diff --git a/.github/workflows/check-code-generation.yml b/.github/workflows/check-code-generation.yml new file mode 100644 index 00000000..527782c1 --- /dev/null +++ b/.github/workflows/check-code-generation.yml @@ -0,0 +1,35 @@ +name: Check Code Generation + +on: + push: + branches-ignore: + - main + +jobs: + check_code_generation: + name: Lua Amalgate and Example in User Guide + strategy: + fail-fast: false + matrix: + python-version: [ "3.10" ] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@0.14.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Development Environment + run: poetry run nox -s install_dev_env + + - name: Poetry install + run: poetry run -- nox -s run_in_dev_env -- poetry install + + - name: Amalgate Lua Scripts + run: poetry run nox -s amalgate_lua_scripts + + - name: Check if re-generated files differ from commit + run: git diff --exit-code diff --git a/.github/workflows/check-packaging.yml b/.github/workflows/check-packaging.yml deleted file mode 100644 index 8514ce77..00000000 --- a/.github/workflows/check-packaging.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Check packaging of the python package - -on: - push: - branches-ignore: - - main - -jobs: - check_packaging: - strategy: - fail-fast: false - matrix: - python-version: [ "3.10" ] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Setup Python & Poetry Environment - uses: exasol/python-toolbox/.github/actions/python-environment@0.14.0 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Development Environment - run: poetry run -- nox -s install_dev_env - - - name: Poetry install - run: poetry run -- nox -s run_in_dev_env -- poetry install - - - name: Run packaging update - # re-generates / amalgate the lua script - # refactor pre-commit as nox task - # and call in pre-commit - run: bash ./githooks/pre-commit - - - name: Show changes on working copy - # check if re-generated lua script is still up-to-date - run: git status --porcelain=v1 -uno - - - name: Show diff on working copy - run: git diff --cached - - - name: Check if packaging changed - run: | - [ -z "$(git status --porcelain=v1 -uno 2>/dev/null)" ] diff --git a/doc/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md index 2bdcba1c..cb1bd98a 100644 --- a/doc/changes/changes_0.1.0.md +++ b/doc/changes/changes_0.1.0.md @@ -52,6 +52,7 @@ Code name: * #176: Updated usage of `exasol-bucketfs` to new API * #185: Removed directory and script for building SLC AAF * #191: Renamed UDF json element "parameters" to "parameter" +* #190: Added dynamic module generation and used it in the example UDF in the user guide * #178: Fixed names of mock objects: * Renamed `testing.mock_query_handler_runner.MockQueryHandlerRunner` to `query_handler.python_query_handler_runner.PythonQueryHandlerRunner` * Renamed method `PythonQueryHandlerRunner.execute_query()` to `execute_queries()` diff --git a/doc/developer_guide/developer_guide.md b/doc/developer_guide/developer_guide.md index 2f66f71c..b0cf4568 100644 --- a/doc/developer_guide/developer_guide.md +++ b/doc/developer_guide/developer_guide.md @@ -14,6 +14,16 @@ poetry run nox -s build_language_container Installing the SLC ins described in the [AAF User Guide](../user_guide/user_guide.md#script-language-container-slc). +## Update Generated Files + +AAF contains the amalgated Lua script [create_query_loop.sql](https://github.com/exasol/advanced-analytics-framework/blob/main/exasol_advanced_analytics_framework/resources/outputs/create_query_loop.sql) originating from the files in the directory [exasol_advanced_analytics_framework/lua/src](https://github.com/exasol/advanced-analytics-framework/blob/main/exasol_advanced_analytics_framework/lua/src/). + +The following command updates the amalgated script: + +```shell +poetry run nox -s amalgate_lua_scripts +``` + ## Running Tests AAF comes with different automated tests implemented in different programming languages and requiring different environments: diff --git a/doc/user_guide/example-udf-script/create.sql b/doc/user_guide/example-udf-script/create.sql new file mode 100644 index 00000000..565afc65 --- /dev/null +++ b/doc/user_guide/example-udf-script/create.sql @@ -0,0 +1,88 @@ +--/ +CREATE OR REPLACE PYTHON3_AAF SET SCRIPT "EXAMPLE_SCHEMA"."EXAMPLE_QUERY_HANDLER_UDF"(...) +EMITS (outputs VARCHAR(2000000)) AS + +from typing import Union +from exasol_advanced_analytics_framework.udf_framework.udf_query_handler import UDFQueryHandler +from exasol_advanced_analytics_framework.udf_framework.dynamic_modules import create_module +from exasol_advanced_analytics_framework.query_handler.context.query_handler_context import QueryHandlerContext +from exasol_advanced_analytics_framework.query_result.query_result import QueryResult +from exasol_advanced_analytics_framework.query_handler.result import Result, Continue, Finish +from exasol_advanced_analytics_framework.query_handler.query.select_query import SelectQuery, SelectQueryWithColumnDefinition +from exasol_advanced_analytics_framework.query_handler.context.proxy.bucketfs_location_proxy import \ + BucketFSLocationProxy +from exasol_data_science_utils_python.schema.column import Column +from exasol_data_science_utils_python.schema.column_name import ColumnName +from exasol_data_science_utils_python.schema.column_type import ColumnType +from datetime import datetime +from exasol.bucketfs import as_string + + +example_module = create_module("example_module") + +class ExampleQueryHandler(UDFQueryHandler): + + def __init__(self, parameter: str, query_handler_context: QueryHandlerContext): + super().__init__(parameter, query_handler_context) + self.parameter = parameter + self.query_handler_context = query_handler_context + self.bfs_proxy = None + self.db_table_proxy = None + + def _bfs_file(self, proxy: BucketFSLocationProxy): + return proxy.bucketfs_location() / "temp_file.txt" + + def start(self) -> Union[Continue, Finish[str]]: + def sample_content(key: str) -> str: + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + return f"{timestamp} {key} {self.parameter}" + + def table_query_string(statement: str, **kwargs): + table_name = self.db_table_proxy._db_object_name.fully_qualified + return statement.format(table_name=table_name, **kwargs) + + def table_query(statement: str, **kwargs): + return SelectQuery(table_query_string(statement, **kwargs)) + + self.bfs_proxy = self.query_handler_context.get_temporary_bucketfs_location() + self._bfs_file(self.bfs_proxy).write(sample_content("bucketfs")) + self.db_table_proxy = self.query_handler_context.get_temporary_table_name() + query_list = [ + table_query('CREATE TABLE {table_name} ("c1" VARCHAR(100), "c2" INTEGER)'), + table_query("INSERT INTO {table_name} VALUES ('{value}', 4)", + value=sample_content("table-insert")), + ] + query_handler_return_query = SelectQueryWithColumnDefinition( + query_string=table_query_string('SELECT "c1", "c2" from {table_name}'), + output_columns=[ + Column(ColumnName("c1"), ColumnType("VARCHAR(100)")), + Column(ColumnName("c2"), ColumnType("INTEGER")), + ]) + return Continue( + query_list=query_list, + input_query=query_handler_return_query) + + def handle_query_result(self, query_result: QueryResult) -> Union[Continue, Finish[str]]: + c1 = query_result.c1 + c2 = query_result.c2 + bfs_content = as_string(self._bfs_file(self.bfs_proxy).read()) + return Finish(result=f"Final result: from query '{c1}', {c2} and bucketfs: '{bfs_content}'") + + +example_module.add_to_module(ExampleQueryHandler) + +class ExampleQueryHandlerFactory: + def create(self, parameter: str, query_handler_context: QueryHandlerContext): + return example_module.ExampleQueryHandler(parameter, query_handler_context) + +example_module.add_to_module(ExampleQueryHandlerFactory) + +from exasol_advanced_analytics_framework.udf_framework.query_handler_runner_udf \ + import QueryHandlerRunnerUDF + +udf = QueryHandlerRunnerUDF(exa) + +def run(ctx): + return udf.run(ctx) + +/ diff --git a/doc/user_guide/example-udf-script/execute.sql b/doc/user_guide/example-udf-script/execute.sql new file mode 100644 index 00000000..bd20b329 --- /dev/null +++ b/doc/user_guide/example-udf-script/execute.sql @@ -0,0 +1,20 @@ +EXECUTE SCRIPT "AAF_DB_SCHEMA"."AAF_RUN_QUERY_HANDLER"('{ + "query_handler": { + "factory_class": { + "module": "example_module", + "name": "ExampleQueryHandlerFactory" + }, + "parameter": "bla-bla", + "udf": { + "schema": "EXAMPLE_SCHEMA", + "name": "EXAMPLE_QUERY_HANDLER_UDF" + } + }, + "temporary_output": { + "bucketfs_location": { + "connection_name": "EXAMPLE_BFS_CON", + "directory": "temp" + }, + "schema_name": "EXAMPLE_TEMP_SCHEMA" + } +}') diff --git a/doc/user_guide/proxies.md b/doc/user_guide/proxies.md new file mode 100644 index 00000000..642a8703 --- /dev/null +++ b/doc/user_guide/proxies.md @@ -0,0 +1,13 @@ +## AAF Proxies + +The Advanced Analytics Framework (AAF) uses _Object Proxies_ to manage temporary objects. + +An _Object Proxy_ +* Encapsulates a temporary object +* Provides a reference enabling using the object, i.e. its name incl. the database schema or the path in the BucketFS +* Ensures the object is removed when leaving the current scope, e.g. the Query Handler. + +All Object Proxies are derived from class `exasol_advanced_analytics_framework.query_handler.context.proxy.object_proxy.ObjectProxy`: +* `BucketFSLocationProxy` encapsulates a location in the BucketFS +* `DBObjectNameProxy` encapsulates a database object, e.g. a table + diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index c308aad9..e51a5e5b 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -73,42 +73,29 @@ pip install exasol-advanced-analytics-framework Exasol executes User Defined Functions (UDFs) in an isolated Container whose root filesystem is derived from a Script Language Container (SLC). -Running the AAF requires a SLC. The following command -* downloads the specified version `` (preferrably the latest) of a prebuilt AAF SLC from the [AAF releases](https://github.com/exasol/advanced-analytics-framework/releases/latest) on GitHub, +Running the AAF requires an SLC. The following command +* downloads the related prebuilt AAF SLC from the [AAF releases](https://github.com/exasol/advanced-analytics-framework/releases) on GitHub, * uploads the file into the BucketFS, * and registers it to the database. -The variable `$LANGUAGE_ALIAS` will be reused in [Additional Scripts](#additional-scripts). - ```shell -LANGUAGE_ALIAS=PYTHON3_AAF -python -m exasol_advanced_analytics_framework.deploy language-container \ - --dsn "$DB_HOST:$DB_PORT" \ - --db-user "$DB_USER" \ - --db-pass "$DB_PASSWORD" \ - --bucketfs-name "$BUCKETFS_NAME" \ - --bucketfs-host "$BUCKETFS_HOST" \ - --bucketfs-port "$BUCKETFS_PORT" \ - --bucketfs-user "$BUCKETFS_USER" \ - --bucketfs-password "$BUCKETFS_PASSWORD" \ - --bucket "$BUCKETFS_NAME" \ - --path-in-bucket "$PATH_IN_BUCKET" \ - --version "$VERSION" \ - --language-alias "$LANGUAGE_ALIAS" +python -m exasol_advanced_analytics_framework.deploy language-container ``` -### Additional Scripts +See the documentation in the Exasol Python Extension Common package for [options common to all Exasol extensions](https://github.com/exasol/python-extension-common/blob/0.8.0/doc/user_guide/user-guide). + +### Defining Additional SQL Scripts Besides the BucketFS connection, the SLC, and the Python package AAF also requires some additional Lua scripts to be created in the Exasol database. -The following command deploys the additional scripts to the specified `DB_SCHEMA` using the `LANGUAGE_ALIAS` of the SLC: +The following command deploys the additional scripts to the specified database schema `$AAF_DB_SCHEMA` using the same language alias `$LANGUAGE_ALIAS` as for uploading the SLC before: ```shell python -m exasol_advanced_analytics_framework.deploy scripts \ - --dsn "$DB_HOST:DB_PORT" \ + --dsn "$DB_HOST:$DB_PORT" \ --db-user "$DB_USER" \ --db-pass "$DB_PASSWORD" \ - --schema "$DB_SCHEMA" \ + --schema "$AAF_DB_SCHEMA" \ --language-alias "$LANGUAGE_ALIAS" ``` @@ -124,7 +111,7 @@ This script takes the necessary parameters to execute the desired algorithm in s The following SQL statement shows how to call an AAF query handler: ```sql -EXECUTE SCRIPT AAF_RUN_QUERY_HANDLER('{ +EXECUTE SCRIPT ""."AAF_RUN_QUERY_HANDLER"('{ "query_handler": { "factory_class": { "module": "", @@ -148,10 +135,11 @@ EXECUTE SCRIPT AAF_RUN_QUERY_HANDLER('{ See [Implementing a Custom Algorithm as Example Query Handler](#implementing-a-custom-algorithm-as-example-query-handler) for a complete example. -### Parameters +### Placeholders -| Parameter | Required? | Description | +| Placeholder | Required? | Description | |------------------------------|-----------|-------------------------------------------------------------------------------| +| `` | yes | Name of the database schema containing the default Query Handler, See [Defining Additional SQL Scripts](#defining-additional-sql-scripts) | | `` | yes | Name of the query handler class | | `` | yes | Module name of the query handler class | | `` | yes | Parameters of the query handler class encoded as string | @@ -188,88 +176,29 @@ Each algorithm should extend the `UDFQueryHandler` abstract class and then imple ### Concrete Example Using an Adhoc Implementation Within the UDF -The example uses the module `builtins` and dynamically adds `ExampleQueryHandler` and `ExampleQueryHandlerFactory` to it. - -```python ---/ -CREATE OR REPLACE PYTHON3_AAF SET SCRIPT "MY_SCHEMA"."MY_QUERY_HANDLER_UDF"(...) -EMITS (outputs VARCHAR(2000000)) AS - -from typing import Union -from exasol_advanced_analytics_framework.udf_framework.udf_query_handler import UDFQueryHandler -from exasol_advanced_analytics_framework.query_handler.context.query_handler_context import QueryHandlerContext -from exasol_advanced_analytics_framework.query_result.query_result import QueryResult -from exasol_advanced_analytics_framework.query_handler.result import Result, Continue, Finish -from exasol_advanced_analytics_framework.query_handler.query.select_query import SelectQuery, SelectQueryWithColumnDefinition -from exasol_data_science_utils_python.schema.column import Column -from exasol_data_science_utils_python.schema.column_name import ColumnName -from exasol_data_science_utils_python.schema.column_type import ColumnType - - -class ExampleQueryHandler(UDFQueryHandler): - def __init__(self, parameter: str, query_handler_context: QueryHandlerContext): - super().__init__(parameter, query_handler_context) - self.parameter = parameter - self.query_handler_context = query_handler_context - - def start(self) -> Union[Continue, Finish[str]]: - query_list = [ - SelectQuery("SELECT 1 FROM DUAL"), - SelectQuery("SELECT 2 FROM DUAL")] - query_handler_return_query = SelectQueryWithColumnDefinition( - query_string="SELECT 5 AS 'return_column' FROM DUAL", - output_columns=[ - Column(ColumnName("return_column"), ColumnType("INTEGER"))]) - - return Continue( - query_list=query_list, - input_query=query_handler_return_query) - - def handle_query_result(self, query_result: QueryResult) -> Union[Continue, Finish[str]]: - return_value = query_result.return_column - result = 2 ** return_value - return Finish(result=result) - -import builtins -builtins.ExampleQueryHandler=ExampleQueryHandler # required for pickle - -class ExampleQueryHandlerFactory: - def create(self, parameter: str, query_handler_context: QueryHandlerContext): - return builtins.ExampleQueryHandler(parameter, query_handler_context) +The example dynamically creates a python module `example_module` and adds classes `ExampleQueryHandler` and `ExampleQueryHandlerFactory` to it. -builtins.ExampleQueryHandlerFactory=ExampleQueryHandlerFactory +In order to execute the example successfully you need to +1. [Create a BucketFS connection](#bucketfs-connection) +2. Activate the AAF's SLC +3. Make sure the database schemas used in the example exist. -from exasol_advanced_analytics_framework.udf_framework.query_handler_runner_udf \ - import QueryHandlerRunnerUDF +The example assumes +* the name for the BucketFS Connection `` to be `EXAMPLE_BFS_CON` +* the name for the AAF database schema `` to be `AAF_DB_SCHEMA`, see [Defining Additional SQL Scripts](#defining-additional-sql-scripts). -udf = QueryHandlerRunnerUDF(exa) +The following SQL statements create the required database schemas unless they already exist: -def run(ctx): - return udf.run(ctx) -/ +```sql +create schema IF NOT EXISTS "EXAMPLE_SCHEMA"; +create schema IF NOT EXISTS "EXAMPLE_TEMP_SCHEMA"; +``` +The following files contain the SQL statements for creating and executing the UDF script +* [example-udf-script/create.sql](example-udf-script/create.sql) +* [example-udf-script/execute.sql](example-udf-script/execute.sql) -EXECUTE SCRIPT MY_SCHEMA.AAF_RUN_QUERY_HANDLER('{ - "query_handler": { - "factory_class": { - "module": "builtins", - "name": "ExampleQueryHandlerFactory" - }, - "parameter": "bla-bla", - "udf": { - "schema": "MY_SCHEMA", - "name": "MY_QUERY_HANDLER_UDF" - } - }, - "temporary_output": { - "bucketfs_location": { - "connection_name": "BFS_CON", - "directory": "temp" - }, - "schema_name": "TEMP_SCHEMA" - } -}'); -``` +### Sequence Diagram The figure below illustrates the execution of this algorithm implemented in class `ExampleQueryHandler`. * When method `start()` is called, it executes two queries and an additional `input_query` to obtain the input for the next iteration. @@ -278,3 +207,7 @@ The figure below illustrates the execution of this algorithm implemented in clas In this example, the algorithm is finished at this iteration and returns 2_return value_ as final result. ![Sample Execution](../images/sample_execution.png "Sample Execution") + +## Additional Information + +* [Object Proxies](proxies.md) for managing temporary locations in the database and BucketFS diff --git a/exasol_advanced_analytics_framework/lua/src/query_loop.lua b/exasol_advanced_analytics_framework/lua/src/query_loop.lua index 7d628844..f97d9540 100644 --- a/exasol_advanced_analytics_framework/lua/src/query_loop.lua +++ b/exasol_advanced_analytics_framework/lua/src/query_loop.lua @@ -55,7 +55,7 @@ function M.prepare_init_query(arguments, meta) local udf_schema = udf['schema'] local udf_name = udf['name'] - local full_qualified_udf_name = string.format("%s.%s", udf_schema, udf_name) + local full_qualified_udf_name = string.format("\"%s\".\"%s\"", udf_schema, udf_name) local udf_args = string.format("(%d,'%s','%s','%s','%s','%s','%s','%s')", iter_num, temporary_bfs_location_conn, diff --git a/exasol_advanced_analytics_framework/lua/test/test_query_handler_runner.lua b/exasol_advanced_analytics_framework/lua/test/test_query_handler_runner.lua index 801113fd..fdadbfc7 100644 --- a/exasol_advanced_analytics_framework/lua/test/test_query_handler_runner.lua +++ b/exasol_advanced_analytics_framework/lua/test/test_query_handler_runner.lua @@ -24,7 +24,7 @@ test_query_handler_runner = { parameter = "param" }, }, - query = "SELECT UDF_SCHEMA.UDF_NAME(" .. + query = "SELECT \"UDF_SCHEMA\".\"UDF_NAME\"(" .. "0,'bfs_conn','directory','db_name_1122334455_1','temp_schema'," .. "'cls_name','package.module','param')", return_query_result = { diff --git a/exasol_advanced_analytics_framework/lua/test/test_query_loop.lua b/exasol_advanced_analytics_framework/lua/test/test_query_loop.lua index b3c39679..07804bcf 100644 --- a/exasol_advanced_analytics_framework/lua/test/test_query_loop.lua +++ b/exasol_advanced_analytics_framework/lua/test/test_query_loop.lua @@ -55,7 +55,7 @@ test_query_loop = { parameter = "param" }, }, - query = "SELECT UDF_SCHEMA.UDF_NAME(" .. + query = "SELECT \"UDF_SCHEMA\".\"UDF_NAME\"(" .. "0,'bfs_conn','directory','db_name_1122334455_1','temp_schema'," .. "'cls_name','package.module','param')" }, @@ -76,7 +76,7 @@ test_query_loop = { parameter = "param" }, }, - query = "SELECT script_schema.AAF_QUERY_HANDLER_UDF(" .. + query = "SELECT \"script_schema\".\"AAF_QUERY_HANDLER_UDF\"(" .. "0,'bfs_conn','directory','db_name_1122334455_1','temp_schema'," .. "'cls_name','package.module','param')" }, diff --git a/exasol_advanced_analytics_framework/resources/outputs/create_query_loop.sql b/exasol_advanced_analytics_framework/resources/outputs/create_query_loop.sql index 811a9df1..0f4940fe 100644 --- a/exasol_advanced_analytics_framework/resources/outputs/create_query_loop.sql +++ b/exasol_advanced_analytics_framework/resources/outputs/create_query_loop.sql @@ -510,7 +510,7 @@ function M.prepare_init_query(arguments, meta) local udf_schema = udf['schema'] local udf_name = udf['name'] - local full_qualified_udf_name = string.format("%s.%s", udf_schema, udf_name) + local full_qualified_udf_name = string.format("\"%s\".\"%s\"", udf_schema, udf_name) local udf_args = string.format("(%d,'%s','%s','%s','%s','%s','%s','%s')", iter_num, temporary_bfs_location_conn, diff --git a/exasol_advanced_analytics_framework/udf_framework/dynamic_modules.py b/exasol_advanced_analytics_framework/udf_framework/dynamic_modules.py new file mode 100644 index 00000000..add6dde9 --- /dev/null +++ b/exasol_advanced_analytics_framework/udf_framework/dynamic_modules.py @@ -0,0 +1,42 @@ +import sys +import importlib +from typing import Any +from types import ModuleType + + +def _create_module(name: str) -> ModuleType: + spec = importlib.machinery.ModuleSpec(name, None) + return importlib.util.module_from_spec(spec) + + +def _register_module_for_import(name: str, mod: ModuleType): + sys.modules[name] = mod + + + +class ModuleExistsException(Exception): + """ + When trying create a module that already exists. + """ + + +def create_module(name: str) -> ModuleType: + """ + Dynamically create a python module using the specified name and + register the module in sys.modules[] for import. + + Additionally add a function add_to_module() to the module enabling other + code to add classes and functions to the module. + """ + if name in sys.modules: + raise ModuleExistsException(f'Module "{name}" already exists') + + mod = _create_module(name) + _register_module_for_import(name, mod) + + def add_to_module(object: Any): + object.__module__ = name + setattr(mod, object.__name__, object) + + add_to_module(add_to_module) + return mod diff --git a/githooks/pre-commit b/githooks/pre-commit index ba113b9e..0a1616a0 100755 --- a/githooks/pre-commit +++ b/githooks/pre-commit @@ -8,10 +8,7 @@ GITHOOKS_PATH="$REPO_DIR/githooks" pushd "$REPO_DIR" bash "$GITHOOKS_PATH/prohibit_commit_to_main.sh" - -SRC_PATH="$REPO_DIR/exasol_advanced_analytics_framework" -export PYTHONPATH=. -"$REPO_DIR"/scripts/run_in_dev_env.sh poetry run python3 "$SRC_PATH/deployment/regenerate_scripts.py" +poetry run nox -s amalgate_lua_scripts git add "$SRC_PATH/resources/outputs/" popd diff --git a/noxfile.py b/noxfile.py index d66922b5..f7ed8c6b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -52,6 +52,12 @@ def install_dev_env(session: Session): session.run(str(install_script)) +@nox.session(python=False) +def amalgate_lua_scripts(session: Session): + script = ROOT_DIR / "exasol_advanced_analytics_framework" / "deployment" / "regenerate_scripts.py" + _run_in_dev_env_poetry_call(session, "python", str(script)) + + @nox.session(python=False) def run_lua_unit_tests(session: Session): lua_tests_script = SCRIPTS_DIRECTORY / "lua_tests.sh" diff --git a/tests/integration_tests/with_db/test_user_guide_example.py b/tests/integration_tests/with_db/test_user_guide_example.py new file mode 100644 index 00000000..e2e0f377 --- /dev/null +++ b/tests/integration_tests/with_db/test_user_guide_example.py @@ -0,0 +1,47 @@ +import importlib.resources +import pytest +import re + +from contextlib import ExitStack +from exasol_advanced_analytics_framework.deployment import constants +from exasol.python_extension_common.deployment.temp_schema import temp_schema + + +@pytest.fixture +def example_db_schemas(pyexasol_connection): + with ExitStack() as stack: + s1 = stack.enter_context(temp_schema(pyexasol_connection)) + s2 = stack.enter_context(temp_schema(pyexasol_connection)) + yield (s1, s2) + + +def test_user_guide_example(database_with_slc, pyexasol_connection, example_db_schemas): + """ + This test verifies the adhoc implementation of a QueryHandler as shown + in the AAF user guide. The adhoc implementation dynamically creates its + own python module. + """ + bucketfs_connection_name, schema_name = database_with_slc + dir = importlib.resources.files(constants.BASE_DIR) \ + / ".." / "doc" / "user_guide" / "example-udf-script" + + statement = ( + (dir / "create.sql") + .read_text() + .replace("EXAMPLE_SCHEMA", example_db_schemas[0]) + ) + pyexasol_connection.execute(statement) + statement = ( + (dir / "execute.sql") + .read_text() + .replace("EXAMPLE_BFS_CON", bucketfs_connection_name) + .replace("AAF_DB_SCHEMA", schema_name) + .replace("EXAMPLE_SCHEMA", example_db_schemas[0]) + .replace("EXAMPLE_TEMP_SCHEMA", example_db_schemas[1]) + ) + result = pyexasol_connection.execute(statement).fetchall() + expected = ( + "Final result: from query '.* table-insert bla-bla', 4" + " and bucketfs: '.* bucketfs bla-bla'" + ) + assert re.match(expected, result[0][0]) diff --git a/tests/unit_tests/udf_framework/test_dynamic_modules.py b/tests/unit_tests/udf_framework/test_dynamic_modules.py new file mode 100644 index 00000000..b3772725 --- /dev/null +++ b/tests/unit_tests/udf_framework/test_dynamic_modules.py @@ -0,0 +1,36 @@ +import pytest +from exasol_advanced_analytics_framework.udf_framework.dynamic_modules import ( + create_module, + ModuleExistsException, +) + + +class ExampleClass: + pass + + +def example_function(): + return "example_function return value" + + +def test_create_module_with_class(): + mod = create_module("xx1") + mod.add_to_module(ExampleClass) + import xx1 + instance = xx1.ExampleClass() + assert isinstance(instance, ExampleClass) and \ + ExampleClass.__module__ == "xx1" + + +def test_add_function(): + mod = create_module("xx2") + import xx2 + xx2.add_to_module(example_function) + assert xx2.example_function() == "example_function return value" \ + and example_function.__module__ == "xx2" + + +def test_add_function_to_existing_module(): + create_module("xx3") + with pytest.raises(ModuleExistsException, match='Module "xx3" already exists') as ex: + create_module("xx3")