From 044b52b5d17bfb373bd615af1d8a6ff3a40581ba Mon Sep 17 00:00:00 2001
From: Umit Cavus Buyuksahin <ucbuyuksahin@gmail.com>
Date: Fri, 17 Dec 2021 14:00:36 +0100
Subject: [PATCH] #48: Added linting and coverage analysis (#49)

* Added linting and coverage analysis

* Added Lua linting to CI

* Corrected Lua unittest paths

* Prepared for the release

* Updated dependecies

* Updated Lua CI pipeline
---
 .github/workflows/build.yaml                  |   7 +-
 ...ase_droid_upload_github_release_assets.yml |   7 +-
 doc/changes/changelog.md                      |   1 +
 doc/changes/changes_0.2.2.md                  |  23 ++++
 exasol_sagemaker_extension/__init__.py        |   2 +-
 .../lua/src/autopilot_training_main.lua       |   4 +-
 .../lua/src/db_metadata_reader.lua            |   3 +-
 .../test/test_autopilot_endpoint_deletion.lua |   2 +-
 .../test_autopilot_endpoint_deployment.lua    |   2 +-
 .../test_autopilot_job_status_polling.lua     |   2 +-
 .../lua/test/test_autopilot_training_main.lua |  10 +-
 .../lua/test/test_aws_s3_handler.lua          |   4 +-
 .../lua/test/test_aws_sagemaker_handler.lua   |   2 +-
 .../lua/test/test_db_metadata_reader.lua      |   2 +-
 .../lua/test/test_db_metadata_writer.lua      |   2 +-
 .../test/test_endpoint_connection_handler.lua |   2 +-
 .../test_install_autopilot_prediction_udf.lua |   2 +-
 .../lua/test/test_validate_input.lua          |   2 +-
 pyproject.toml                                |  12 +-
 scripts/lua_tests.sh                          | 114 ++++++++++++++++++
 scripts/run_lua_unittests.sh                  |  34 ------
 setup.py                                      |   4 +-
 tests/unit_tests/test_sagemaker_extension.py  |   2 +-
 23 files changed, 179 insertions(+), 66 deletions(-)
 create mode 100644 doc/changes/changes_0.2.2.md
 create mode 100755 scripts/lua_tests.sh
 delete mode 100755 scripts/run_lua_unittests.sh

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 914e78f..e4e2ab2 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -33,10 +33,11 @@ jobs:
           sudo luarocks install luaunit
           sudo luarocks install mockagne
           sudo luarocks install exaerror
+          sudo luarocks install luacheck
+          sudo luarocks install luacov
 
-      - name: Run Lua unit tests
-        run: |
-          ./scripts/run_lua_unittests.sh
+      - name: Run Lua unit tests and static code analyzer
+        run: poetry run poe lua-tests
 
       - name: Setup integration test environment
         run: ./scripts/setup_integration_test.sh
diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml
index 45c5cd8..008d75b 100644
--- a/.github/workflows/release_droid_upload_github_release_assets.yml
+++ b/.github/workflows/release_droid_upload_github_release_assets.yml
@@ -38,10 +38,11 @@ jobs:
           sudo luarocks install luaunit
           sudo luarocks install mockagne
           sudo luarocks install exaerror
+          sudo luarocks install luacheck
+          sudo luarocks install luacov
 
-      - name: Run Lua unit tests
-        run: |
-          ./scripts/run_lua_unittests.sh
+      - name: Run Lua unit tests and static code analyzer
+        run: poetry run poe lua-tests
 
       - name: Setup integration test environment
         run: ./scripts/setup_integration_test.sh
diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md
index 9ad97b7..40f172a 100644
--- a/doc/changes/changelog.md
+++ b/doc/changes/changelog.md
@@ -1,5 +1,6 @@
 # Changelog
 
+* [0.2.2](changes_0.2.2.md)
 * [0.2.1](changes_0.2.1.md)
 * [0.2.0](changes_0.2.0.md)
 * [0.1.0](changes_0.1.0.md)
diff --git a/doc/changes/changes_0.2.2.md b/doc/changes/changes_0.2.2.md
new file mode 100644
index 0000000..e57c10f
--- /dev/null
+++ b/doc/changes/changes_0.2.2.md
@@ -0,0 +1,23 @@
+# SageMaker Extension 0.2.2, released 2021-12-17
+
+Code name: Add static code analysis for Lua implementations to CI 
+
+## Summary
+
+This release includes a linter and a coverage analyzer of the Lua implementation 
+to the CI pipeline of the SageMaker-Extension. Thus, the static code analysis of 
+Lua scripts are performed without executing them and, the coding standards and 
+conventions are adhered.
+
+
+### Features
+
+  - #48: Added static code analysis to CI for Lua
+
+
+
+
+
+
+  
+    
diff --git a/exasol_sagemaker_extension/__init__.py b/exasol_sagemaker_extension/__init__.py
index fb13a35..9dd16a3 100644
--- a/exasol_sagemaker_extension/__init__.py
+++ b/exasol_sagemaker_extension/__init__.py
@@ -1 +1 @@
-__version__ = '0.2.1'
\ No newline at end of file
+__version__ = '0.2.2'
\ No newline at end of file
diff --git a/exasol_sagemaker_extension/lua/src/autopilot_training_main.lua b/exasol_sagemaker_extension/lua/src/autopilot_training_main.lua
index 06e7cdc..5c1ce2f 100644
--- a/exasol_sagemaker_extension/lua/src/autopilot_training_main.lua
+++ b/exasol_sagemaker_extension/lua/src/autopilot_training_main.lua
@@ -27,7 +27,7 @@ local required_args = {
 	max_candidates = nil,
 	model_info_schema_name = nil,
 	model_info_table_name = nil,
-	objective = Nil,
+	objective = nil,
 	aws_tags = nil
 }
 
@@ -187,7 +187,7 @@ function get_table_columns(schema_name, table_name)
 		_G.global_env.error(tostring(error_obj))
 	end
 
-   	local col_names, col_types = {}, {}
+	local col_names, col_types = {}, {}
 	for i=1, #res do
 		col_names[#col_names+1] = res[i][1]
 		col_types[#col_types+1] = res[i][2]
diff --git a/exasol_sagemaker_extension/lua/src/db_metadata_reader.lua b/exasol_sagemaker_extension/lua/src/db_metadata_reader.lua
index aaafa85..156dd98 100644
--- a/exasol_sagemaker_extension/lua/src/db_metadata_reader.lua
+++ b/exasol_sagemaker_extension/lua/src/db_metadata_reader.lua
@@ -39,7 +39,8 @@ function M.read_metadata(schema, job_name, columns)
 		end
 	end
 
-	local query_reading = [[SELECT ]] .. table.concat(columns_ids, ",") .. [[ FROM ::schema."SME_METADATA_AUTOPILOT_JOBS" WHERE JOB_NAME = :job_name]]
+	local query_reading = [[SELECT ]] .. table.concat(columns_ids, ",")
+			.. [[ FROM ::schema."SME_METADATA_AUTOPILOT_JOBS" WHERE JOB_NAME = :job_name]]
 	local success, result = _G.global_env.pquery(query_reading, params)
 	if not success then
 		local error_obj = exaerror.create("F-SME-3",
diff --git a/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deletion.lua b/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deletion.lua
index ee393a7..da8f416 100644
--- a/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deletion.lua
+++ b/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deletion.lua
@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-require("src/autopilot_endpoint_deletion")
+require("autopilot_endpoint_deletion")
 
 
 test_autopilot_endpoint_deletion = {
diff --git a/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deployment.lua b/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deployment.lua
index a0b641b..944847a 100644
--- a/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deployment.lua
+++ b/exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deployment.lua
@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-require("src/autopilot_endpoint_deployment")
+require("autopilot_endpoint_deployment")
 
 
 test_autopilot_endpoint_deployment = {
diff --git a/exasol_sagemaker_extension/lua/test/test_autopilot_job_status_polling.lua b/exasol_sagemaker_extension/lua/test/test_autopilot_job_status_polling.lua
index 82f0a18..8010730 100644
--- a/exasol_sagemaker_extension/lua/test/test_autopilot_job_status_polling.lua
+++ b/exasol_sagemaker_extension/lua/test/test_autopilot_job_status_polling.lua
@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-require("src/autopilot_job_status_polling")
+require("autopilot_job_status_polling")
 
 
 test_autopilot_job_status_polling = {
diff --git a/exasol_sagemaker_extension/lua/test/test_autopilot_training_main.lua b/exasol_sagemaker_extension/lua/test/test_autopilot_training_main.lua
index 94cc873..b959370 100644
--- a/exasol_sagemaker_extension/lua/test/test_autopilot_training_main.lua
+++ b/exasol_sagemaker_extension/lua/test/test_autopilot_training_main.lua
@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-require("./src/autopilot_training_main")
+require("autopilot_training_main")
 
 
 
@@ -33,10 +33,10 @@ end
 
 function test_train_autopilot_main.test_parse_used_optional_arguments()
     local args_missing_json_str = [[{
-       	"job_name"						    : "jobname",
-       	"aws_credentials_connection_name"	: "S3_CONNECTION",
-       	"aws_region"						: "eu-central-1",
-       	"iam_sagemaker_role"				: "arn:aws:iam::1234:role",
+	    "job_name"							: "jobname",
+	    "aws_credentials_connection_name"	: "S3_CONNECTION",
+	    "aws_region"						: "eu-central-1",
+	    "iam_sagemaker_role"				: "arn:aws:iam::1234:role",
 	    "s3_bucket_uri"						: "s3://sme-bucket",
 	    "s3_output_path"					: "path",
 	    "input_schema_name"					: "TEST_SCHEMA",
diff --git a/exasol_sagemaker_extension/lua/test/test_aws_s3_handler.lua b/exasol_sagemaker_extension/lua/test/test_aws_s3_handler.lua
index 09a7838..4967de4 100644
--- a/exasol_sagemaker_extension/lua/test/test_aws_s3_handler.lua
+++ b/exasol_sagemaker_extension/lua/test/test_aws_s3_handler.lua
@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-local aws_s3_handler = require("./src/aws_s3_handler")
+local aws_s3_handler = require("aws_s3_handler")
 
 
 test_aws_s3_handler = {
@@ -67,7 +67,7 @@ end
 
 function test_aws_s3_handler.test_prepare_export_query()
     local parallelism_factor = 1
-    for i = 1, 5 do
+    for _ = 1, 5 do
         local n_nodes = math.random(1, 20)
         local n_exporter = n_nodes * parallelism_factor
 
diff --git a/exasol_sagemaker_extension/lua/test/test_aws_sagemaker_handler.lua b/exasol_sagemaker_extension/lua/test/test_aws_sagemaker_handler.lua
index d8da180..2af8bd3 100644
--- a/exasol_sagemaker_extension/lua/test/test_aws_sagemaker_handler.lua
+++ b/exasol_sagemaker_extension/lua/test/test_aws_sagemaker_handler.lua
@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-local aws_sagemaker_handler = require("./src/aws_sagemaker_handler")
+local aws_sagemaker_handler = require("aws_sagemaker_handler")
 
 
 test_aws_sagemaker_handler = {
diff --git a/exasol_sagemaker_extension/lua/test/test_db_metadata_reader.lua b/exasol_sagemaker_extension/lua/test/test_db_metadata_reader.lua
index 6f27fe1..60da800 100644
--- a/exasol_sagemaker_extension/lua/test/test_db_metadata_reader.lua
+++ b/exasol_sagemaker_extension/lua/test/test_db_metadata_reader.lua
@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-local db_metadata_reader = require("./src/db_metadata_reader")
+local db_metadata_reader = require("db_metadata_reader")
 
 
 test_db_metadata_reader = {
diff --git a/exasol_sagemaker_extension/lua/test/test_db_metadata_writer.lua b/exasol_sagemaker_extension/lua/test/test_db_metadata_writer.lua
index 6de425c..ef347c9 100644
--- a/exasol_sagemaker_extension/lua/test/test_db_metadata_writer.lua
+++ b/exasol_sagemaker_extension/lua/test/test_db_metadata_writer.lua
@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-local db_metadata_writer = require("./src/db_metadata_writer")
+local db_metadata_writer = require("db_metadata_writer")
 
 
 test_db_metadata_writer = {
diff --git a/exasol_sagemaker_extension/lua/test/test_endpoint_connection_handler.lua b/exasol_sagemaker_extension/lua/test/test_endpoint_connection_handler.lua
index a2f4fb3..aa7fd93 100644
--- a/exasol_sagemaker_extension/lua/test/test_endpoint_connection_handler.lua
+++ b/exasol_sagemaker_extension/lua/test/test_endpoint_connection_handler.lua
@@ -1,7 +1,7 @@
 local json = require('cjson')
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-local endpoint_connection_handler = require("src/endpoint_connection_handler")
+local endpoint_connection_handler = require("endpoint_connection_handler")
 
 
 local aws_s3_connection = 'aws_s3_connection'
diff --git a/exasol_sagemaker_extension/lua/test/test_install_autopilot_prediction_udf.lua b/exasol_sagemaker_extension/lua/test/test_install_autopilot_prediction_udf.lua
index d9105d9..187ddb7 100644
--- a/exasol_sagemaker_extension/lua/test/test_install_autopilot_prediction_udf.lua
+++ b/exasol_sagemaker_extension/lua/test/test_install_autopilot_prediction_udf.lua
@@ -1,7 +1,7 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
 local install_autopilot_prediction_udf = require(
-		"./src/install_autopilot_prediction_udf")
+		"install_autopilot_prediction_udf")
 
 
 test_install_autopilot_prediction_udf = {}
diff --git a/exasol_sagemaker_extension/lua/test/test_validate_input.lua b/exasol_sagemaker_extension/lua/test/test_validate_input.lua
index 58905eb..c0dedde 100644
--- a/exasol_sagemaker_extension/lua/test/test_validate_input.lua
+++ b/exasol_sagemaker_extension/lua/test/test_validate_input.lua
@@ -1,5 +1,5 @@
 local luaunit = require("luaunit")
-local validate_input = require("./src/validate_input")
+local validate_input = require("validate_input")
 
 
 test_validate_input= {
diff --git a/pyproject.toml b/pyproject.toml
index f8d9ce9..790ba9c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "exasol-sagemaker-extension"
-version = "0.2.1"
+version = "0.2.2"
 description = "Exasol SageMaker Integration"
 
 license = "MIT"
@@ -20,7 +20,7 @@ keywords = ['exasol', 'sagemaker']
 include = ["lua", "resources", "deployment"]
 
 [tool.poetry.dependencies]
-python = ">=3.6.1,<4.0"
+python = ">=3.6.2,<4.0"
 pandas = ">=1.1.3,<2.0.0"
 boto = "^2.49.0"
 sagemaker = "^2.59.1"
@@ -30,12 +30,18 @@ importlib-resources = "^5.2.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2" 
-pytest-cov = "^2.12.1"
+pytest-cov = "^2.10.1"
 Sphinx = "^4.1.2"
 coverage = "^5.5"
 exasol-udf-mock-python = { git = "https://github.com/exasol/udf-mock-python.git", branch = "master" }
 exasol-bucketfs-utils-python = { git = "https://github.com/exasol/bucketfs-utils-python.git", branch = "main" }
+poethepoet = "^0.10.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
+
+
+[tool.poe.tasks]
+lua-tests = {"shell" = "bash ./scripts/lua_tests.sh"}
+
diff --git a/scripts/lua_tests.sh b/scripts/lua_tests.sh
new file mode 100755
index 0000000..38a5744
--- /dev/null
+++ b/scripts/lua_tests.sh
@@ -0,0 +1,114 @@
+#!/bin/bash
+
+# This script finds and runs Lua unit tests, collects coverage and runs static code analysis.
+
+readonly script_dir=$(dirname "$(readlink -f "$0")")
+if [[ -v $1 ]]
+then
+    readonly base_dir="$1"
+else
+    readonly base_dir=$(readlink -f "$script_dir/..")
+fi
+
+readonly exit_ok=0
+readonly exit_software=2
+readonly src_module_path="$base_dir/exasol_sagemaker_extension/lua/src"
+readonly test_module_path="$base_dir/exasol_sagemaker_extension/lua/test"
+readonly target_dir="$base_dir/target"
+readonly reports_dir="$target_dir/luaunit-reports"
+readonly luacov_dir="$target_dir/luacov-reports"
+
+function create_target_directories {
+    mkdir -p "$reports_dir"
+    mkdir -p "$luacov_dir"
+}
+
+##
+# Run the unit tests and collect code coverage.
+#
+# Return error status in case there were failures.
+#
+function run_tests {
+    cd "$test_module_path" || exit
+    readonly tests="$(find . -name '*.lua')"
+    test_suites=0
+    failures=0
+    successes=0
+    for testcase in $tests
+    do
+        ((test_suites++))
+
+        if LUA_PATH="$src_module_path/?.lua;$(luarocks path --lr-path)" \
+            lua -lluacov "$testcase" -o junit -n "$reports_dir/$testcase"
+        then
+            ((successes++))
+        else
+            ((failures++))
+        fi
+        echo
+    done
+    echo -n "Ran $test_suites test suites. $successes successes, "
+    if [[ "$failures" -eq 0 ]]
+    then
+        echo -e "\e[1m\e[32m$failures failures\e[0m."
+        return "$exit_ok"
+    else
+        echo -e "\e[1m\e[31m$failures failures\e[0m."
+        return "$exit_software"
+    fi
+}
+
+##
+# Collect the coverage results into a single file.
+#
+# Return exit status of coverage collector.
+#
+function collect_coverage_results {
+    echo
+    echo "Collecting code coverage results"
+    luacov
+    return "$?"
+}
+
+##
+# Move the coverage results into the target directory.
+#
+# Return exit status of `mv` command.
+#
+function move_coverage_results {
+    echo "Moving coverage results to $luacov_dir"
+    mv "$test_module_path"/luacov.*.out "$luacov_dir"
+    return "$?"
+}
+
+##
+# Print the summary section of the code coverage report to the console
+#
+function print_coverage_summary {
+    echo
+    grep --after 500 'File\s*Hits' "$luacov_dir/luacov.report.out"
+}
+
+##
+# Analyze the Lua code with "luacheck".
+#
+# Return exit status of code coverage.
+#
+function run_static_code_analysis {
+    echo
+    echo "Running static code analysis"
+    echo
+    luacheck "$src_module_path" "$test_module_path" --codes \
+      --ignore 111 --ignore 112  --ignore 113 --ignore 142
+    return "$?"
+}
+
+create_target_directories
+run_tests \
+&& collect_coverage_results \
+&& move_coverage_results \
+&& print_coverage_summary \
+&& run_static_code_analysis \
+|| exit "$exit_software"
+
+exit "$exit_ok"
\ No newline at end of file
diff --git a/scripts/run_lua_unittests.sh b/scripts/run_lua_unittests.sh
deleted file mode 100755
index 754da1e..0000000
--- a/scripts/run_lua_unittests.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-
-# This scripts run all Lua scripts implemented in lua/test directory
-
-
-readonly script_dir=$(dirname "$(readlink -f "$0")")
-readonly base_dir=$(readlink -f "$script_dir/..")
-readonly lua_dir="$base_dir/exasol_sagemaker_extension/lua"
-
-
-if ! cd "$lua_dir"
-then
-  echo  "Lua scripts path is not correct: " $lua_dir
-  exit  2
-fi
-
-n_test=0
-n_success=0
-n_fail=0
-readonly tests="$(find './test/' -name '*.lua')"
-for unittest in $tests
-do
-    ((n_test+=1))
-    if  lua  "$unittest" -v
-    then
-        ((n_success+=1))
-    else
-        ((n_fail+=1))
-    fi
-    echo ""
-done
-
-echo -n "Ran $n_test tests, $n_success successes, $n_fail failures"
-exit 0
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 8c3c0dd..2bbdb2e 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@
 
 setup_kwargs = {
     'name': 'exasol-sagemaker-extension',
-    'version': '0.2.1',
+    'version': '0.2.2',
     'description': 'Exasol SageMaker Integration',
     'long_description': '# SageMaker Extension\n\nThis project provides a Python library that trains data stored in Exasol using AWS SageMaker.\n\n\n## Table of Contents\n\n### Information for Users\n\n* [User Guide](doc/user_guide/user_guide.md)\n* [Tutorial](https://github.com/exasol/data-science-examples/blob/main/tutorials/machine-learning/sagemaker-extension/tutorial.md)\n* [Changelog](doc/changes/changelog.md)\n\n### Information for Contributors\n\n\n* [System Requirement Specification](doc/system_requirements.md)\n* [Design](doc/design.md)\n* [Dependencies](doc/dependencies.md)',
     'author': 'Umit Buyuksahin',
@@ -31,7 +31,7 @@
     'packages': packages,
     'package_data': package_data,
     'install_requires': install_requires,
-    'python_requires': '>=3.6.1,<4.0',
+    'python_requires': '>=3.6.2,<4.0',
 }
 
 
diff --git a/tests/unit_tests/test_sagemaker_extension.py b/tests/unit_tests/test_sagemaker_extension.py
index 23da44a..66125b4 100644
--- a/tests/unit_tests/test_sagemaker_extension.py
+++ b/tests/unit_tests/test_sagemaker_extension.py
@@ -2,4 +2,4 @@
 
 
 def test_version():
-    assert __version__ == '0.2.1'
\ No newline at end of file
+    assert __version__ == '0.2.2'
\ No newline at end of file