#48: Added linting and coverage analysis (#49)

* Added linting and coverage analysis

* Added Lua linting to CI

* Corrected Lua unittest paths

* Prepared for the release

* Updated dependecies

* Updated Lua CI pipeline

.github/workflows/build.yaml

@@ -33,10 +33,11 @@ jobs:
           sudo luarocks install luaunit
           sudo luarocks install mockagne
           sudo luarocks install exaerror
+          sudo luarocks install luacheck
+          sudo luarocks install luacov
 
-      - name: Run Lua unit tests
-        run: |
-          ./scripts/run_lua_unittests.sh
+      - name: Run Lua unit tests and static code analyzer
+        run: poetry run poe lua-tests
 
       - name: Setup integration test environment
         run: ./scripts/setup_integration_test.sh

.github/workflows/release_droid_upload_github_release_assets.yml

@@ -38,10 +38,11 @@ jobs:
           sudo luarocks install luaunit
           sudo luarocks install mockagne
           sudo luarocks install exaerror
+          sudo luarocks install luacheck
+          sudo luarocks install luacov
 
-      - name: Run Lua unit tests
-        run: |
-          ./scripts/run_lua_unittests.sh
+      - name: Run Lua unit tests and static code analyzer
+        run: poetry run poe lua-tests
 
       - name: Setup integration test environment
         run: ./scripts/setup_integration_test.sh

doc/changes/changelog.md

@@ -1,5 +1,6 @@
 # Changelog
 
+* [0.2.2](changes_0.2.2.md)
 * [0.2.1](changes_0.2.1.md)
 * [0.2.0](changes_0.2.0.md)
 * [0.1.0](changes_0.1.0.md)

doc/changes/changes_0.2.2.md

@@ -0,0 +1,23 @@
+# SageMaker Extension 0.2.2, released 2021-12-17
+
+Code name: Add static code analysis for Lua implementations to CI 
+
+## Summary
+
+This release includes a linter and a coverage analyzer of the Lua implementation 
+to the CI pipeline of the SageMaker-Extension. Thus, the static code analysis of 
+Lua scripts are performed without executing them and, the coding standards and 
+conventions are adhered.
+
+
+### Features
+
+  - #48: Added static code analysis to CI for Lua
+
+
+
+
+
+
+
+

exasol_sagemaker_extension/__init__.py

@@ -1 +1 @@
-__version__ = '0.2.1'
+__version__ = '0.2.2'

exasol_sagemaker_extension/lua/src/autopilot_training_main.lua

@@ -27,7 +27,7 @@ local required_args = {
 	max_candidates = nil,
 	model_info_schema_name = nil,
 	model_info_table_name = nil,
-	objective = Nil,
+	objective = nil,
 	aws_tags = nil
 }
 
@@ -187,7 +187,7 @@ function get_table_columns(schema_name, table_name)
 		_G.global_env.error(tostring(error_obj))
 	end
 
-   	local col_names, col_types = {}, {}
+	local col_names, col_types = {}, {}
 	for i=1, #res do
 		col_names[#col_names+1] = res[i][1]
 		col_types[#col_types+1] = res[i][2]

exasol_sagemaker_extension/lua/src/db_metadata_reader.lua

@@ -39,7 +39,8 @@ function M.read_metadata(schema, job_name, columns)
 		end
 	end
 
-	local query_reading = [[SELECT ]] .. table.concat(columns_ids, ",") .. [[ FROM ::schema."SME_METADATA_AUTOPILOT_JOBS" WHERE JOB_NAME = :job_name]]
+	local query_reading = [[SELECT ]] .. table.concat(columns_ids, ",")
+			.. [[ FROM ::schema."SME_METADATA_AUTOPILOT_JOBS" WHERE JOB_NAME = :job_name]]
 	local success, result = _G.global_env.pquery(query_reading, params)
 	if not success then
 		local error_obj = exaerror.create("F-SME-3",

exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deletion.lua

@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-require("src/autopilot_endpoint_deletion")
+require("autopilot_endpoint_deletion")
 
 
 test_autopilot_endpoint_deletion = {

exasol_sagemaker_extension/lua/test/test_autopilot_endpoint_deployment.lua

@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-require("src/autopilot_endpoint_deployment")
+require("autopilot_endpoint_deployment")
 
 
 test_autopilot_endpoint_deployment = {

exasol_sagemaker_extension/lua/test/test_autopilot_job_status_polling.lua

@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-require("src/autopilot_job_status_polling")
+require("autopilot_job_status_polling")
 
 
 test_autopilot_job_status_polling = {

exasol_sagemaker_extension/lua/test/test_autopilot_training_main.lua

@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-require("./src/autopilot_training_main")
+require("autopilot_training_main")
 
 
 
@@ -33,10 +33,10 @@ end
 
 function test_train_autopilot_main.test_parse_used_optional_arguments()
     local args_missing_json_str = [[{
-       	"job_name"						    : "jobname",
-       	"aws_credentials_connection_name"	: "S3_CONNECTION",
-       	"aws_region"						: "eu-central-1",
-       	"iam_sagemaker_role"				: "arn:aws:iam::1234:role",
+	    "job_name"							: "jobname",
+	    "aws_credentials_connection_name"	: "S3_CONNECTION",
+	    "aws_region"						: "eu-central-1",
+	    "iam_sagemaker_role"				: "arn:aws:iam::1234:role",
 	    "s3_bucket_uri"						: "s3://sme-bucket",
 	    "s3_output_path"					: "path",
 	    "input_schema_name"					: "TEST_SCHEMA",

exasol_sagemaker_extension/lua/test/test_aws_s3_handler.lua

@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-local aws_s3_handler = require("./src/aws_s3_handler")
+local aws_s3_handler = require("aws_s3_handler")
 
 
 test_aws_s3_handler = {
@@ -67,7 +67,7 @@ end
 
 function test_aws_s3_handler.test_prepare_export_query()
     local parallelism_factor = 1
-    for i = 1, 5 do
+    for _ = 1, 5 do
         local n_nodes = math.random(1, 20)
         local n_exporter = n_nodes * parallelism_factor
 

exasol_sagemaker_extension/lua/test/test_aws_sagemaker_handler.lua

@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-local aws_sagemaker_handler = require("./src/aws_sagemaker_handler")
+local aws_sagemaker_handler = require("aws_sagemaker_handler")
 
 
 test_aws_sagemaker_handler = {

exasol_sagemaker_extension/lua/test/test_db_metadata_reader.lua

@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-local db_metadata_reader = require("./src/db_metadata_reader")
+local db_metadata_reader = require("db_metadata_reader")
 
 
 test_db_metadata_reader = {

exasol_sagemaker_extension/lua/test/test_db_metadata_writer.lua

@@ -1,6 +1,6 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-local db_metadata_writer = require("./src/db_metadata_writer")
+local db_metadata_writer = require("db_metadata_writer")
 
 
 test_db_metadata_writer = {

exasol_sagemaker_extension/lua/test/test_endpoint_connection_handler.lua

@@ -1,7 +1,7 @@
 local json = require('cjson')
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
-local endpoint_connection_handler = require("src/endpoint_connection_handler")
+local endpoint_connection_handler = require("endpoint_connection_handler")
 
 
 local aws_s3_connection = 'aws_s3_connection'

exasol_sagemaker_extension/lua/test/test_install_autopilot_prediction_udf.lua

@@ -1,7 +1,7 @@
 local luaunit = require("luaunit")
 local mockagne = require("mockagne")
 local install_autopilot_prediction_udf = require(
-		"./src/install_autopilot_prediction_udf")
+		"install_autopilot_prediction_udf")
 
 
 test_install_autopilot_prediction_udf = {}

exasol_sagemaker_extension/lua/test/test_validate_input.lua

@@ -1,5 +1,5 @@
 local luaunit = require("luaunit")
-local validate_input = require("./src/validate_input")
+local validate_input = require("validate_input")
 
 
 test_validate_input= {

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "exasol-sagemaker-extension"
-version = "0.2.1"
+version = "0.2.2"
 description = "Exasol SageMaker Integration"
 
 license = "MIT"
@@ -20,7 +20,7 @@ keywords = ['exasol', 'sagemaker']
 include = ["lua", "resources", "deployment"]
 
 [tool.poetry.dependencies]
-python = ">=3.6.1,<4.0"
+python = ">=3.6.2,<4.0"
 pandas = ">=1.1.3,<2.0.0"
 boto = "^2.49.0"
 sagemaker = "^2.59.1"
@@ -30,12 +30,18 @@ importlib-resources = "^5.2.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2" 
-pytest-cov = "^2.12.1"
+pytest-cov = "^2.10.1"
 Sphinx = "^4.1.2"
 coverage = "^5.5"
 exasol-udf-mock-python = { git = "https://github.com/exasol/udf-mock-python.git", branch = "master" }
 exasol-bucketfs-utils-python = { git = "https://github.com/exasol/bucketfs-utils-python.git", branch = "main" }
+poethepoet = "^0.10.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
+
+
+[tool.poe.tasks]
+lua-tests = {"shell" = "bash ./scripts/lua_tests.sh"}
+

scripts/lua_tests.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+# This script finds and runs Lua unit tests, collects coverage and runs static code analysis.
+
+readonly script_dir=$(dirname "$(readlink -f "$0")")
+if [[ -v $1 ]]
+then
+    readonly base_dir="$1"
+else
+    readonly base_dir=$(readlink -f "$script_dir/..")
+fi
+
+readonly exit_ok=0
+readonly exit_software=2
+readonly src_module_path="$base_dir/exasol_sagemaker_extension/lua/src"
+readonly test_module_path="$base_dir/exasol_sagemaker_extension/lua/test"
+readonly target_dir="$base_dir/target"
+readonly reports_dir="$target_dir/luaunit-reports"
+readonly luacov_dir="$target_dir/luacov-reports"
+
+function create_target_directories {
+    mkdir -p "$reports_dir"
+    mkdir -p "$luacov_dir"
+}
+
+##
+# Run the unit tests and collect code coverage.
+#
+# Return error status in case there were failures.
+#
+function run_tests {
+    cd "$test_module_path" || exit
+    readonly tests="$(find . -name '*.lua')"
+    test_suites=0
+    failures=0
+    successes=0
+    for testcase in $tests
+    do
+        ((test_suites++))
+
+        if LUA_PATH="$src_module_path/?.lua;$(luarocks path --lr-path)" \
+            lua -lluacov "$testcase" -o junit -n "$reports_dir/$testcase"
+        then
+            ((successes++))
+        else
+            ((failures++))
+        fi
+        echo
+    done
+    echo -n "Ran $test_suites test suites. $successes successes, "
+    if [[ "$failures" -eq 0 ]]
+    then
+        echo -e "\e[1m\e[32m$failures failures\e[0m."
+        return "$exit_ok"
+    else
+        echo -e "\e[1m\e[31m$failures failures\e[0m."
+        return "$exit_software"
+    fi
+}
+
+##
+# Collect the coverage results into a single file.
+#
+# Return exit status of coverage collector.
+#
+function collect_coverage_results {
+    echo
+    echo "Collecting code coverage results"
+    luacov
+    return "$?"
+}
+
+##
+# Move the coverage results into the target directory.
+#
+# Return exit status of `mv` command.
+#
+function move_coverage_results {
+    echo "Moving coverage results to $luacov_dir"
+    mv "$test_module_path"/luacov.*.out "$luacov_dir"
+    return "$?"
+}
+
+##
+# Print the summary section of the code coverage report to the console
+#
+function print_coverage_summary {
+    echo
+    grep --after 500 'File\s*Hits' "$luacov_dir/luacov.report.out"
+}
+
+##
+# Analyze the Lua code with "luacheck".
+#
+# Return exit status of code coverage.
+#
+function run_static_code_analysis {
+    echo
+    echo "Running static code analysis"
+    echo
+    luacheck "$src_module_path" "$test_module_path" --codes \
+      --ignore 111 --ignore 112  --ignore 113 --ignore 142
+    return "$?"
+}
+
+create_target_directories
+run_tests \
+&& collect_coverage_results \
+&& move_coverage_results \
+&& print_coverage_summary \
+&& run_static_code_analysis \
+|| exit "$exit_software"
+
+exit "$exit_ok"