From e7c772f9a2f4a3ec930d2b9e09b733fc508a139e Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Mon, 11 Apr 2022 12:12:31 +0200 Subject: [PATCH] Import from ora: provide explicit type mappings via added switch (#20) * #14 #18 provide explicit type mappings via added switch --- .github/workflows/broken_links_checker.yml | 5 +- .github/workflows/ci-build-next-java.yml | 14 +- .github/workflows/dependencies_check.yml | 10 +- .../release_droid_print_quick_checksum.yml | 10 +- ...ase_droid_upload_github_release_assets.yml | 15 +- .gitignore | 2 + .project-keeper.yml | 10 + dependencies.md | 82 ++--- doc/changes/changes_2.2.0.md | 19 +- doc/user_guide/oracle_user_guide.md | 26 +- lombok.config | 2 + pk_generated_parent.pom | 308 ++++++++++++++++++ pom.xml | 234 ++----------- .../dialects/oracle/OracleProperties.java | 1 + .../dialects/oracle/OracleQueryRewriter.java | 34 +- .../dialects/oracle/OracleSqlDialect.java | 14 +- .../oracle/OracleQueryRewriterTest.java | 35 +- .../dialects/oracle/OracleSqlDialectIT.java | 45 ++- .../scalarFunctionsParameterCache.yml | 2 +- 19 files changed, 567 insertions(+), 301 deletions(-) create mode 100644 .project-keeper.yml create mode 100644 lombok.config create mode 100644 pk_generated_parent.pom diff --git a/.github/workflows/broken_links_checker.yml b/.github/workflows/broken_links_checker.yml index c268e21..20d2ce3 100644 --- a/.github/workflows/broken_links_checker.yml +++ b/.github/workflows/broken_links_checker.yml @@ -11,8 +11,11 @@ on: jobs: linkChecker: runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Configure broken links checker run: | mkdir -p ./target diff --git a/.github/workflows/ci-build-next-java.yml b/.github/workflows/ci-build-next-java.yml index 2abe7cb..88e2d88 100644 --- a/.github/workflows/ci-build-next-java.yml +++ b/.github/workflows/ci-build-next-java.yml @@ -9,9 +9,12 @@ on: jobs: java-17-compatibility: runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true steps: - name: Checkout the repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 17 @@ -19,13 +22,7 @@ jobs: with: distribution: 'temurin' java-version: 17 - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + cache: 'maven' - name: Run tests and build with Maven run: | mvn --batch-mode --update-snapshots clean package -DtrimStackTrace=false \ @@ -35,3 +32,4 @@ jobs: if: ${{ always() && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} with: github_token: ${{ secrets.GITHUB_TOKEN }} + fail_if_no_tests: false diff --git a/.github/workflows/dependencies_check.yml b/.github/workflows/dependencies_check.yml index d28c0b4..3fa137f 100644 --- a/.github/workflows/dependencies_check.yml +++ b/.github/workflows/dependencies_check.yml @@ -9,18 +9,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up JDK 11 uses: actions/setup-java@v2 with: distribution: 'temurin' java-version: 11 - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + cache: 'maven' - name: Checking dependencies for vulnerabilities run: mvn org.sonatype.ossindex.maven:ossindex-maven-plugin:audit -f pom.xml \ No newline at end of file diff --git a/.github/workflows/release_droid_print_quick_checksum.yml b/.github/workflows/release_droid_print_quick_checksum.yml index 746fc43..297eaba 100644 --- a/.github/workflows/release_droid_print_quick_checksum.yml +++ b/.github/workflows/release_droid_print_quick_checksum.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 @@ -16,13 +16,7 @@ jobs: with: distribution: 'temurin' java-version: 11 - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + cache: 'maven' - name: Build with Maven skipping tests run: mvn --batch-mode clean verify -DskipTests - name: Print checksum diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml index 97f9914..b73d1bb 100644 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -12,22 +12,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v2 with: + distribution: 'temurin' java-version: 11 - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + cache: 'maven' - name: Build with Maven skipping tests - run: mvn clean verify -DskipTests + run: mvn --batch-mode clean verify -DskipTests - name: Generate sha256sum files run: find target -maxdepth 1 -name *.jar -exec bash -c 'sha256sum {} > {}.sha256' \; - name: Upload assets to the GitHub release draft diff --git a/.gitignore b/.gitignore index 82f9c5d..50748c7 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ pom.xml.versionsBackup *.old *.md.html .vscode/ + +*.flattened-pom.xml \ No newline at end of file diff --git a/.project-keeper.yml b/.project-keeper.yml new file mode 100644 index 0000000..e58f322 --- /dev/null +++ b/.project-keeper.yml @@ -0,0 +1,10 @@ +sources: + - type: maven + path: pom.xml + modules: + - integration_tests + - udf_coverage + - jar_artifact +excludes: + - "E-PK-CORE-18: Outdated content: '.github/workflows/ci-build.yml'" + - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_prepare_original_checksum.yml'" \ No newline at end of file diff --git a/dependencies.md b/dependencies.md index 451f2c3..f8024fd 100644 --- a/dependencies.md +++ b/dependencies.md @@ -26,70 +26,78 @@ | [udf-debugging-java][26] | [MIT][1] | | [Matcher for SQL Result Sets][28] | [MIT][1] | | [virtual-schema-shared-integration-tests][30] | [MIT][1] | +| [JaCoCo :: Agent][32] | [Eclipse Public License 2.0][33] | ## Plugin Dependencies | Dependency | License | | ------------------------------------------------------- | ---------------------------------------------- | -| [JaCoCo :: Maven Plugin][32] | [Eclipse Public License 2.0][33] | -| [Maven Surefire Plugin][34] | [Apache License, Version 2.0][17] | +| [SonarQube Scanner for Maven][34] | [GNU LGPL 3][35] | | [Apache Maven Compiler Plugin][36] | [Apache License, Version 2.0][17] | -| [Apache Maven Assembly Plugin][38] | [Apache License, Version 2.0][17] | -| [Maven Failsafe Plugin][40] | [Apache License, Version 2.0][17] | -| [Apache Maven Enforcer Plugin][42] | [Apache License, Version 2.0][17] | -| [Versions Maven Plugin][44] | [Apache License, Version 2.0][17] | -| [Artifact reference checker and unifier][46] | [MIT][1] | -| [Project keeper maven plugin][48] | [MIT][1] | -| [error-code-crawler-maven-plugin][50] | [MIT][1] | -| [org.sonatype.ossindex.maven:ossindex-maven-plugin][52] | [ASL2][53] | -| [Maven Dependency Plugin][54] | [The Apache Software License, Version 2.0][53] | -| [Reproducible Build Maven Plugin][56] | [Apache 2.0][53] | -| [Apache Maven JAR Plugin][58] | [Apache License, Version 2.0][17] | -| [Maven Clean Plugin][60] | [The Apache Software License, Version 2.0][53] | -| [Maven Resources Plugin][62] | [The Apache Software License, Version 2.0][53] | -| [Maven Install Plugin][64] | [The Apache Software License, Version 2.0][53] | -| [Maven Deploy Plugin][66] | [The Apache Software License, Version 2.0][53] | -| [Maven Site Plugin 3][68] | [The Apache Software License, Version 2.0][53] | +| [Apache Maven Enforcer Plugin][38] | [Apache License, Version 2.0][17] | +| [Maven Flatten Plugin][40] | [Apache Software Licenese][41] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][42] | [ASL2][41] | +| [Reproducible Build Maven Plugin][44] | [Apache 2.0][41] | +| [Maven Surefire Plugin][46] | [Apache License, Version 2.0][17] | +| [Versions Maven Plugin][48] | [Apache License, Version 2.0][17] | +| [Project keeper maven plugin][50] | [The MIT License][51] | +| [Apache Maven Assembly Plugin][52] | [Apache License, Version 2.0][17] | +| [Apache Maven JAR Plugin][54] | [Apache License, Version 2.0][17] | +| [Artifact reference checker and unifier][56] | [MIT][1] | +| [Apache Maven Dependency Plugin][58] | [Apache License, Version 2.0][17] | +| [Maven Failsafe Plugin][60] | [Apache License, Version 2.0][17] | +| [JaCoCo :: Maven Plugin][62] | [Eclipse Public License 2.0][33] | +| [error-code-crawler-maven-plugin][64] | [MIT][1] | +| [Maven Clean Plugin][66] | [The Apache Software License, Version 2.0][41] | +| [Maven Resources Plugin][68] | [The Apache Software License, Version 2.0][41] | +| [Maven Install Plugin][70] | [The Apache Software License, Version 2.0][41] | +| [Maven Deploy Plugin][72] | [The Apache Software License, Version 2.0][41] | +| [Maven Site Plugin 3][74] | [The Apache Software License, Version 2.0][41] | -[48]: https://github.com/exasol/project-keeper-maven-plugin +[32]: https://www.eclemma.org/jacoco/index.html [4]: https://github.com/exasol/error-reporting-java [6]: https://www.oracle.com/database/technologies/maven-central-guide.html [2]: https://github.com/exasol/db-fundamentals-java -[53]: http://www.apache.org/licenses/LICENSE-2.0.txt -[34]: https://maven.apache.org/surefire/maven-surefire-plugin/ -[60]: http://maven.apache.org/plugins/maven-clean-plugin/ +[41]: http://www.apache.org/licenses/LICENSE-2.0.txt +[46]: https://maven.apache.org/surefire/maven-surefire-plugin/ +[66]: http://maven.apache.org/plugins/maven-clean-plugin/ [7]: https://www.oracle.com/downloads/licenses/oracle-free-license.html [1]: https://opensource.org/licenses/MIT [14]: https://github.com/mockito/mockito -[40]: https://maven.apache.org/surefire/maven-failsafe-plugin/ +[60]: https://maven.apache.org/surefire/maven-failsafe-plugin/ [24]: https://github.com/exasol/test-db-builder-java [30]: https://github.com/exasol/virtual-schema-shared-integration-tests -[44]: http://www.mojohaus.org/versions-maven-plugin/ -[54]: http://maven.apache.org/plugins/maven-dependency-plugin/ +[48]: http://www.mojohaus.org/versions-maven-plugin/ +[50]: https://github.com/exasol/project-keeper/ [11]: http://opensource.org/licenses/BSD-3-Clause [36]: https://maven.apache.org/plugins/maven-compiler-plugin/ [21]: http://opensource.org/licenses/MIT [0]: https://github.com/exasol/virtual-schema-common-jdbc [33]: https://www.eclipse.org/legal/epl-2.0/ +[35]: http://www.gnu.org/licenses/lgpl.txt [18]: https://github.com/exasol/exasol-testcontainers -[32]: https://www.jacoco.org/jacoco/trunk/doc/maven.html +[62]: https://www.jacoco.org/jacoco/trunk/doc/maven.html [15]: https://github.com/mockito/mockito/blob/main/LICENSE [28]: https://github.com/exasol/hamcrest-resultset-matcher -[56]: http://zlika.github.io/reproducible-build-maven-plugin +[44]: http://zlika.github.io/reproducible-build-maven-plugin +[51]: https://github.com/exasol/project-keeper/blob/main/LICENSE +[58]: https://maven.apache.org/plugins/maven-dependency-plugin/ [17]: https://www.apache.org/licenses/LICENSE-2.0.txt +[34]: http://sonarsource.github.io/sonar-scanner-maven/ [16]: https://www.jqno.nl/equalsverifier -[42]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ +[38]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ [13]: https://www.eclipse.org/legal/epl-v20.html -[64]: http://maven.apache.org/plugins/maven-install-plugin/ +[70]: http://maven.apache.org/plugins/maven-install-plugin/ [12]: https://junit.org/junit5/ -[52]: https://sonatype.github.io/ossindex-maven/maven-plugin/ +[42]: https://sonatype.github.io/ossindex-maven/maven-plugin/ [20]: https://testcontainers.org +[40]: https://www.mojohaus.org/flatten-maven-plugin/flatten-maven-plugin [26]: https://github.com/exasol/udf-debugging-java [10]: http://hamcrest.org/JavaHamcrest/ -[66]: http://maven.apache.org/plugins/maven-deploy-plugin/ -[68]: http://maven.apache.org/plugins/maven-site-plugin/ -[62]: http://maven.apache.org/plugins/maven-resources-plugin/ -[46]: https://github.com/exasol/artifact-reference-checker-maven-plugin -[50]: https://github.com/exasol/error-code-crawler-maven-plugin -[58]: https://maven.apache.org/plugins/maven-jar-plugin/ -[38]: https://maven.apache.org/plugins/maven-assembly-plugin/ +[72]: http://maven.apache.org/plugins/maven-deploy-plugin/ +[74]: http://maven.apache.org/plugins/maven-site-plugin/ +[68]: http://maven.apache.org/plugins/maven-resources-plugin/ +[56]: https://github.com/exasol/artifact-reference-checker-maven-plugin +[64]: https://github.com/exasol/error-code-crawler-maven-plugin +[54]: https://maven.apache.org/plugins/maven-jar-plugin/ +[52]: https://maven.apache.org/plugins/maven-assembly-plugin/ diff --git a/doc/changes/changes_2.2.0.md b/doc/changes/changes_2.2.0.md index 9bdd798..150bd4d 100644 --- a/doc/changes/changes_2.2.0.md +++ b/doc/changes/changes_2.2.0.md @@ -1,9 +1,24 @@ -# Virtual Schema for Oracle 2.2.0, released 2022-??-?? +# Virtual Schema for Oracle 2.2.0, released 2022-04-11 -Code name: +Code name: Import from ora: explicit data types switch. +## Features + +* #14: Explicitly state types when using IMPORT_FROM_ORA: Added GENERATE_JDBC_DATATYPE_MAPPING_FOR_OCI flag to virtual schema adapter for this functionality. ## Dependency Updates ### Test Dependency Updates * Updated `com.exasol:virtual-schema-shared-integration-tests:2.1.1` to `2.2.0` +* Added `org.jacoco:org.jacoco.agent:0.8.5` + +### Plugin Dependency Updates + +* Updated `com.exasol:error-code-crawler-maven-plugin:0.7.1` to `1.1.0` +* Updated `com.exasol:project-keeper-maven-plugin:1.3.4` to `2.3.0` +* Updated `org.apache.maven.plugins:maven-dependency-plugin:2.8` to `3.2.0` +* Updated `org.apache.maven.plugins:maven-jar-plugin:3.2.2` to `3.2.0` +* Added `org.codehaus.mojo:flatten-maven-plugin:1.2.7` +* Updated `org.codehaus.mojo:versions-maven-plugin:2.9.0` to `2.8.1` +* Added `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` +* Updated `org.sonatype.ossindex.maven:ossindex-maven-plugin:3.2.0` to `3.1.0` diff --git a/doc/user_guide/oracle_user_guide.md b/doc/user_guide/oracle_user_guide.md index 41290eb..a3193a1 100644 --- a/doc/user_guide/oracle_user_guide.md +++ b/doc/user_guide/oracle_user_guide.md @@ -130,6 +130,30 @@ CREATE VIRTUAL SCHEMA IMPORT_FROM_ORA = 'true' ORA_CONNECTION_NAME = 'ORA_CONNECTION'; ``` +### Auto generated datatype mapping list while using IMPORT_FROM_ORA. +Using `IMPORT FROM ORA` might lead to some unexpected datatype mappings. Unlike for a JDBC connection there's no explicit data mapping being generated when using `IMPORT FROM ORA`. +As a current stopgap solution for this issue we now (starting from version 2.2.0) also provide a `GENERATE_JDBC_DATATYPE_MAPPING_FOR_OCI` switch you can specify and enable when creating the virtual schema. +```sql +CREATE VIRTUAL SCHEMA + USING ADAPTER.JDBC_ADAPTER + WITH + CONNECTION_NAME = 'ORACLE_JDBC_CONNECTION' + SCHEMA_NAME = '' + IMPORT_FROM_ORA = 'true' + GENERATE_JDBC_DATATYPE_MAPPING_FOR_OCI = 'true' + ORA_CONNECTION_NAME = 'ORA_CONNECTION'; +``` +This will add explicit datatype mapping to the generated command when using `IMPORT FROM ORA`. + +Example: + +Before + +`IMPORT FROM ORA AT ORACLE_CON STATEMENT ...` + +After setting `GENERATE_JDBC_DATATYPE_MAPPING_FOR_OCI` to `true` + +`IMPORT INTO(c1 DECIMAL(36,1), c2 .... ) FROM ORA AT ORACLE_CON STATEMENT ...` ## Supported Capabilities @@ -137,7 +161,7 @@ The Oracle dialect does not support all capabilities. A complete list can be fou ## Type Mappings and Limitations -| Orcale Data Type | Supported | Converted Exasol Data Type | Comments | +| Oracle Data Type | Supported | Converted Exasol Data Type | Comments | | -------------------------------------------------------------------------------- | --------- | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | | BINARY\_DOUBLE | ✓ | VARCHAR(2000000) | | | BINARY\_FLOAT | ✓ | VARCHAR(2000000) | | diff --git a/lombok.config b/lombok.config new file mode 100644 index 0000000..a23edb4 --- /dev/null +++ b/lombok.config @@ -0,0 +1,2 @@ +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true \ No newline at end of file diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom new file mode 100644 index 0000000..7098512 --- /dev/null +++ b/pk_generated_parent.pom @@ -0,0 +1,308 @@ + + + 4.0.0 + com.exasol + oracle-virtual-schema-generated-parent + 2.2.0 + pom + + UTF-8 + UTF-8 + 11 + + + + MIT License + https://github.com/exasol/oracle-virtual-schema/blob/main/LICENSE + repo + + + + + Exasol + opensource@exasol.com + Exasol AG + https://www.exasol.com/ + + + + scm:git:https://github.com/exasol/oracle-virtual-schema.git + scm:git:https://github.com/exasol/oracle-virtual-schema.git + https://github.com/exasol/oracle-virtual-schema/ + + + + org.jacoco + org.jacoco.agent + 0.8.5 + test + runtime + + + + + + org.sonarsource.scanner.maven + sonar-maven-plugin + 3.9.1.2184 + + + org.apache.maven.plugins + maven-compiler-plugin + 3.9.0 + + ${java.version} + ${java.version} + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.0.0 + + + enforce-maven + + enforce + + + + + 3.6.3 + + + + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.2.7 + + true + oss + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + org.sonatype.ossindex.maven + ossindex-maven-plugin + 3.1.0 + + + audit + package + + audit + + + + + + io.github.zlika + reproducible-build-maven-plugin + 0.15 + + + strip-jar + package + + strip-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} + + + + org.codehaus.mojo + versions-maven-plugin + 2.8.1 + + + display-updates + package + + display-plugin-updates + display-dependency-updates + + + + + file:///${project.basedir}/versionsMavenPluginRules.xml + + + + maven-assembly-plugin + org.apache.maven.plugins + 3.3.0 + + + src/assembly/all-dependencies.xml + + NAME_OF_YOUR_JAR + false + + + + make-assembly + package + + single + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + none + + + + + com.exasol + artifact-reference-checker-maven-plugin + 0.4.0 + + + verify + + verify + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.2.0 + + + copy-jacoco + + copy-dependencies + + compile + + org.jacoco.agent + runtime + ${project.build.directory}/jacoco-agent + true + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.0.0-M5 + + + -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} + + true + + + + + verify + + integration-test + verify + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.7 + + + prepare-agent + + prepare-agent + + + + prepare-agent-integration + + prepare-agent-integration + + + + merge-results + verify + + merge + + + + + ${project.build.directory}/ + + jacoco*.exec + + + + ${project.build.directory}/aggregate.exec + + + + report + verify + + report + + + ${project.build.directory}/aggregate.exec + + + + + + com.exasol + error-code-crawler-maven-plugin + 1.1.0 + + + verify + + verify + + + + + + + diff --git a/pom.xml b/pom.xml index 3489234..9e986f2 100644 --- a/pom.xml +++ b/pom.xml @@ -1,11 +1,11 @@ - + 4.0.0 com.exasol oracle-virtual-schema 2.2.0 Virtual Schema for Oracle + Virtual Schema for Oracle maven.exasol.com @@ -25,8 +25,7 @@ 0.4.1 0.7.1 1.16.3 - target/site/jacoco/jacoco.xml,target/site/jacoco-it/jacoco.xml - + target/site/jacoco/jacoco.xml,target/site/jacoco-it/jacoco.xml @@ -144,152 +143,27 @@ - org.jacoco - jacoco-maven-plugin - 0.8.7 - - - prepare-agent - - prepare-agent - - - - prepare-agent-integration - - prepare-agent-integration - - - - merge-results - verify - - merge - - - - - ${project.build.directory}/ - - jacoco*.exec - - - - ${project.build.directory}/aggregate.exec - - - - report - verify - - report - - - ${project.build.directory}/aggregate.exec - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${surefire.and.failsafe.plugin.version} - - - -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.9.0 - - ${java.version} - ${java.version} - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.3.0 + org.sonatype.ossindex.maven + ossindex-maven-plugin - - src/assembly/all-dependencies.xml - - virtual-schema-dist-${vscjdbc.version}-oracle-${version} - false + + + 7ea56ad4-8a8b-4e51-8ed9-5aad83d8efb1 + - - - make-assembly - package - - single - - - org.apache.maven.plugins maven-failsafe-plugin - ${surefire.and.failsafe.plugin.version} - - -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} 0 - - - - integration-test - verify - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - 3.0.0 - - - enforce-maven - - enforce - - - - - 3.6.3 - - - - - - - - org.codehaus.mojo - versions-maven-plugin - 2.9.0 - - - package - - display-plugin-updates - display-dependency-updates - - - - - file:///${project.basedir}/versionsMavenPluginRules.xml - com.exasol - artifact-reference-checker-maven-plugin - 0.4.0 + project-keeper-maven-plugin + 2.3.0 @@ -297,58 +171,14 @@ - - - /doc/changes/* - - - com.exasol - project-keeper-maven-plugin - 1.3.4 - - - - verify - - - + org.apache.maven.plugins + maven-assembly-plugin - - jar_artifact - integration_tests - - - .github/workflows/release_droid_upload_github_release_assets.yml - + virtual-schema-dist-${vscjdbc.version}-oracle-${project.version} - - com.exasol - error-code-crawler-maven-plugin - ${error.code.crawler.version} - - - - verify - - - - - - org.sonatype.ossindex.maven - ossindex-maven-plugin - 3.2.0 - - - package - - audit - - - - org.apache.maven.plugins maven-dependency-plugin @@ -368,31 +198,13 @@ - - io.github.zlika - reproducible-build-maven-plugin - 0.15 - - - strip-jar - package - - strip-jar - - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.2.2 - - - default-jar - none - - - - \ No newline at end of file + + oracle-virtual-schema-generated-parent + com.exasol + 2.2.0 + pk_generated_parent.pom + + https://github.com/exasol/oracle-virtual-schema/ + diff --git a/src/main/java/com/exasol/adapter/dialects/oracle/OracleProperties.java b/src/main/java/com/exasol/adapter/dialects/oracle/OracleProperties.java index 53a23ce..9399e65 100644 --- a/src/main/java/com/exasol/adapter/dialects/oracle/OracleProperties.java +++ b/src/main/java/com/exasol/adapter/dialects/oracle/OracleProperties.java @@ -6,6 +6,7 @@ public final class OracleProperties { public static final String ORACLE_CAST_NUMBER_TO_DECIMAL_PROPERTY = "ORACLE_CAST_NUMBER_TO_DECIMAL_WITH_PRECISION_AND_SCALE"; public static final String ORACLE_IMPORT_PROPERTY = "IMPORT_FROM_ORA"; + public static final String GENERATE_JDBC_DATATYPE_MAPPING_FOR_OCI_PROPERTY = "GENERATE_JDBC_DATATYPE_MAPPING_FOR_OCI"; public static final String ORACLE_CONNECTION_NAME_PROPERTY = "ORA_CONNECTION_NAME"; private OracleProperties() { diff --git a/src/main/java/com/exasol/adapter/dialects/oracle/OracleQueryRewriter.java b/src/main/java/com/exasol/adapter/dialects/oracle/OracleQueryRewriter.java index e706595..a4daf7c 100644 --- a/src/main/java/com/exasol/adapter/dialects/oracle/OracleQueryRewriter.java +++ b/src/main/java/com/exasol/adapter/dialects/oracle/OracleQueryRewriter.java @@ -1,10 +1,15 @@ package com.exasol.adapter.dialects.oracle; +import java.sql.Connection; import java.sql.SQLException; +import java.util.logging.Logger; +import com.exasol.adapter.AdapterProperties; import com.exasol.adapter.dialects.SqlDialect; import com.exasol.adapter.dialects.rewriting.AbstractQueryRewriter; +import com.exasol.adapter.jdbc.ColumnMetadataReader; import com.exasol.adapter.jdbc.RemoteMetadataReader; +import com.exasol.adapter.jdbc.ResultSetMetadataReader; /** * This class implements an Oracle-specific query rewriter. @@ -16,13 +21,38 @@ public class OracleQueryRewriter extends AbstractQueryRewriter { * @param dialect Oracle SQl dialect * @param remoteMetadataReader reader for metadata from the remote data source */ - public OracleQueryRewriter(final SqlDialect dialect, final RemoteMetadataReader remoteMetadataReader) { + private final Connection connection; + private final AdapterProperties properties; + + private static final Logger LOGGER = Logger.getLogger(OracleQueryRewriter.class.getName()); + + public OracleQueryRewriter(final SqlDialect dialect, final RemoteMetadataReader remoteMetadataReader, Connection connection, AdapterProperties properties) { super(dialect, remoteMetadataReader, new OracleConnectionDefinitionBuilder()); + this.connection = connection; + this.properties = properties; + } + + private boolean isGenerateJdbcDatatypeMappingForOCIEnabled() { + return this.properties.isEnabled(OracleProperties.GENERATE_JDBC_DATATYPE_MAPPING_FOR_OCI_PROPERTY); } @Override protected String generateImportStatement(final String connectionDefinition, final String pushdownQuery) throws SQLException { - return "IMPORT FROM ORA " + connectionDefinition + " STATEMENT '" + pushdownQuery.replace("'", "''") + "'"; + if (isGenerateJdbcDatatypeMappingForOCIEnabled()) { + final String columnDescription = this.createImportColumnsDescription(pushdownQuery); + return "IMPORT INTO (" + columnDescription + ") FROM ORA " + connectionDefinition + " STATEMENT '" + pushdownQuery.replace("'", "''") + "'"; + } else { + return "IMPORT FROM ORA " + connectionDefinition + " STATEMENT '" + pushdownQuery.replace("'", "''") + "'"; + } + } + + private String createImportColumnsDescription(final String query) { + final ColumnMetadataReader columnMetadataReader = this.remoteMetadataReader.getColumnMetadataReader(); + final ResultSetMetadataReader resultSetMetadataReader = new ResultSetMetadataReader( + connection, columnMetadataReader); + final String columnsDescription = resultSetMetadataReader.describeColumns(query); + LOGGER.finer(() -> "columndescription: " + columnsDescription); + return columnsDescription; } } \ No newline at end of file diff --git a/src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlDialect.java b/src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlDialect.java index fb84b26..7715cbd 100644 --- a/src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlDialect.java +++ b/src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlDialect.java @@ -66,7 +66,7 @@ private static Capabilities createCapabilityList() { */ public OracleSqlDialect(final ConnectionFactory connectionFactory, final AdapterProperties properties) { super(connectionFactory, properties, Set.of(SCHEMA_NAME_PROPERTY, ORACLE_IMPORT_PROPERTY, - ORACLE_CONNECTION_NAME_PROPERTY, ORACLE_CAST_NUMBER_TO_DECIMAL_PROPERTY)); + ORACLE_CONNECTION_NAME_PROPERTY, ORACLE_CAST_NUMBER_TO_DECIMAL_PROPERTY, GENERATE_JDBC_DATATYPE_MAPPING_FOR_OCI_PROPERTY)); this.omitParenthesesMap.add(ScalarFunction.SYSDATE); this.omitParenthesesMap.add(ScalarFunction.SYSTIMESTAMP); } @@ -165,15 +165,21 @@ protected RemoteMetadataReader createRemoteMetadataReader() { return new OracleMetadataReader(this.connectionFactory.getConnection(), this.properties); } catch (final SQLException exception) { throw new RemoteMetadataReaderException(ExaError.messageBuilder("E-VS-ORA-1") - .message("Unable to create Oracle remote metadata reader. Caused by: {{cause}}") - .unquotedParameter("cause", exception.getMessage()).toString(), exception); + .message("Unable to create Oracle remote metadata reader. Caused by: {{cause|uq}}") + .parameter("cause", exception.getMessage()).toString(), exception); } } @Override protected QueryRewriter createQueryRewriter() { if (this.isImportFromOraEnabled()) { - return new OracleQueryRewriter(this, this.createRemoteMetadataReader()); + try { + return new OracleQueryRewriter(this, this.createRemoteMetadataReader(), this.connectionFactory.getConnection(), properties); + } catch (SQLException exception) { + throw new RemoteMetadataReaderException(ExaError.messageBuilder("E-VS-ORA-4") + .message("Unable to create Oracle remote metadata reader. Caused by: {{cause|uq}}") + .parameter("cause", exception.getMessage()).toString(), exception); + } } return new ImportIntoTemporaryTableQueryRewriter(this, this.createRemoteMetadataReader(), this.connectionFactory); } diff --git a/src/test/java/com/exasol/adapter/dialects/oracle/OracleQueryRewriterTest.java b/src/test/java/com/exasol/adapter/dialects/oracle/OracleQueryRewriterTest.java index 7bc9d75..19830e9 100644 --- a/src/test/java/com/exasol/adapter/dialects/oracle/OracleQueryRewriterTest.java +++ b/src/test/java/com/exasol/adapter/dialects/oracle/OracleQueryRewriterTest.java @@ -1,17 +1,21 @@ package com.exasol.adapter.dialects.oracle; -import static com.exasol.adapter.dialects.oracle.OracleProperties.ORACLE_CONNECTION_NAME_PROPERTY; -import static com.exasol.adapter.dialects.oracle.OracleProperties.ORACLE_IMPORT_PROPERTY; +import static com.exasol.adapter.dialects.oracle.OracleProperties.*; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentMatchers; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import com.exasol.adapter.AdapterException; @@ -36,8 +40,33 @@ void testRewriteToImportFromOraWithConnectionDetailsInProperties( ORACLE_CONNECTION_NAME_PROPERTY, "ora_connection")); final SqlDialectFactory dialectFactory = new OracleSqlDialectFactory(); final SqlDialect dialect = dialectFactory.createSqlDialect(connectionFactoryMock, properties); - final QueryRewriter queryRewriter = new OracleQueryRewriter(dialect, null); + final QueryRewriter queryRewriter = new OracleQueryRewriter(dialect, null,connectionFactoryMock.getConnection(),properties); assertThat(queryRewriter.rewrite(this.statement, EXA_METADATA, properties), equalTo("IMPORT FROM ORA AT ora_connection STATEMENT 'SELECT TO_CHAR(1) FROM \"DUAL\"'")); } + @Test + void testRewriteToImportFromOraWithConnectionDetailsInPropertiesJDBCDatatypeMapping( + @Mock final ConnectionFactory connectionFactoryMock) throws AdapterException, SQLException { + final AdapterProperties properties = new AdapterProperties(Map.of( // + ORACLE_IMPORT_PROPERTY, "true", // + ORACLE_CONNECTION_NAME_PROPERTY, "ora_connection", + GENERATE_JDBC_DATATYPE_MAPPING_FOR_OCI_PROPERTY, "true")); + final SqlDialectFactory dialectFactory = new OracleSqlDialectFactory(); + final SqlDialect dialect = dialectFactory.createSqlDialect(connectionFactoryMock, properties); + var oracleMetadataReader = new OracleMetadataReader(mockConnection(), properties); + final QueryRewriter queryRewriter = new OracleQueryRewriter(dialect, oracleMetadataReader,mockConnection(),properties); + assertThat(queryRewriter.rewrite(this.statement, EXA_METADATA, properties), + equalTo("IMPORT INTO (c1 DECIMAL(18, 0)) FROM ORA AT ora_connection STATEMENT 'SELECT TO_CHAR(1) FROM \"DUAL\"'")); + } + @Override + protected Connection mockConnection() throws SQLException { + ResultSetMetaData metadataMock = (ResultSetMetaData) Mockito.mock(ResultSetMetaData.class); + Mockito.lenient().when(metadataMock.getColumnCount()).thenReturn(1); + Mockito.lenient().when(metadataMock.getColumnType(1)).thenReturn(4); + PreparedStatement statementMock = (PreparedStatement)Mockito.mock(PreparedStatement.class); + Mockito.lenient().when(statementMock.getMetaData()).thenReturn(metadataMock); + Connection connectionMock = (Connection)Mockito.mock(Connection.class); + Mockito.lenient().when(connectionMock.prepareStatement((String) ArgumentMatchers.any())).thenReturn(statementMock); + return connectionMock; + } } \ No newline at end of file diff --git a/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlDialectIT.java index f328e54..e98e6fa 100644 --- a/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlDialectIT.java @@ -19,6 +19,8 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; +import com.exasol.udfdebugging.UdfTestSetup; +import com.github.dockerjava.api.model.ContainerNetwork; import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.*; import org.junit.jupiter.params.ParameterizedTest; @@ -53,9 +55,10 @@ class OracleSqlDialectIT { private static final String VIRTUAL_SCHEMA_JDBC = "VIRTUAL_SCHEMA_JDBC"; private static final String VIRTUAL_SCHEMA_ORACLE = "VIRTUAL_SCHEMA_ORACLE"; + private static final String VIRTUAL_SCHEMA_ORACLE_JDBC_MAPPING = "VIRTUAL_SCHEMA_ORACLE_JDBC_MAPPING"; private static final String VIRTUAL_SCHEMA_JDBC_NUMBER_TO_DECIMAL = "VIRTUAL_SCHEMA_JDBC_NUMBER_TO_DECIMAL"; private static final String VIRTUAL_SCHEMA_ORACLE_NUMBER_TO_DECIMAL = "VIRTUAL_SCHEMA_ORACLE_NUMBER_TO_DECIMAL"; - + private static final String VIRTUAL_SCHEMA_ORACLE_NUMBER_TO_DECIMAL_JDBC_MAPPING = "VIRTUAL_SCHEMA_ORACLE_NUMBER_TO_DECIMAL_JDBC_MAPPING"; private static final String TABLE_ORACLE_ALL_DATA_TYPES = "TABLE_ORACLE_ALL_DATA_TYPES"; private static final String TABLE_ORACLE_NUMBER_HANDLING = "TABLE_ORACLE_NUMBER_HANDLING"; private static final String TABLE_ORACLE_TIMESTAMPS = "TABLE_ORACLE_TIMESTAMPS"; @@ -82,7 +85,14 @@ private static void uploadInstantClientToBucket() final String instantClientPath = getPropertyFromFile(RESOURCES_FOLDER_DIALECT_NAME, "instant.client.path"); bucket.uploadFile(Path.of(instantClientPath, instantClientName), "drivers/oracle/" + instantClientName); } - + private static String getTestHostIpFromInsideExasol(ExasolContainer exasolContainer) { + final Map networks = exasolContainer.getContainerInfo().getNetworkSettings() + .getNetworks(); + if (networks.size() == 0) { + return null; + } + return networks.values().iterator().next().getGateway(); + } private static void setupExasolContainer() throws BucketAccessException, TimeoutException, FileNotFoundException, SQLException, InterruptedException { uploadOracleJDBCDriverAndVSToBucket( exasolContainer.getDefaultBucket()); uploadInstantClientToBucket(); @@ -90,7 +100,10 @@ private static void setupExasolContainer() throws BucketAccessException, Timeout exasolContainer.getPassword()); statementExasol = exasolConnection.createStatement(); - final ExasolObjectFactory exasolFactory = new ExasolObjectFactory(exasolContainer.createConnection("")); + final UdfTestSetup udfTestSetup = new UdfTestSetup(getTestHostIpFromInsideExasol(exasolContainer), + exasolContainer.getDefaultBucket(), exasolConnection); + final ExasolObjectFactory exasolFactory = new ExasolObjectFactory(exasolContainer.createConnection(""), + ExasolObjectConfiguration.builder().withJvmOptions(udfTestSetup.getJvmOptions()).build()); final ExasolSchema schema = exasolFactory.createSchema(SCHEMA_EXASOL); final AdapterScript adapterScript = createAdapterScript(schema); @@ -137,11 +150,22 @@ private static void createVirtualSchemasOnExasolDbContainer(ExasolObjectFactory .connectionDefinition(jdbcConnectionDefinition).properties(Map.of("SCHEMA_NAME", SCHEMA_ORACLE, "IMPORT_FROM_ORA", "true", "ORA_CONNECTION_NAME", ORACLE_OCI_CONNECTION_NAME)) .build(); + exasolFactory.createVirtualSchemaBuilder(VIRTUAL_SCHEMA_ORACLE_JDBC_MAPPING).adapterScript(adapterScript) + .connectionDefinition(jdbcConnectionDefinition).properties(Map.of("SCHEMA_NAME", SCHEMA_ORACLE, + "IMPORT_FROM_ORA", "true", "ORA_CONNECTION_NAME", ORACLE_OCI_CONNECTION_NAME, "GENERATE_JDBC_DATATYPE_MAPPING_FOR_OCI", "true")) + .build(); + exasolFactory.createVirtualSchemaBuilder(VIRTUAL_SCHEMA_ORACLE_NUMBER_TO_DECIMAL).adapterScript(adapterScript) .connectionDefinition(jdbcConnectionDefinition) .properties(Map.of("SCHEMA_NAME", SCHEMA_ORACLE, "IMPORT_FROM_ORA", "true", "ORA_CONNECTION_NAME", ORACLE_OCI_CONNECTION_NAME, "oracle_cast_number_to_decimal_with_precision_and_scale", "36,1")) .build(); + exasolFactory.createVirtualSchemaBuilder(VIRTUAL_SCHEMA_ORACLE_NUMBER_TO_DECIMAL_JDBC_MAPPING).adapterScript(adapterScript) + .connectionDefinition(jdbcConnectionDefinition) + .properties(Map.of("SCHEMA_NAME", SCHEMA_ORACLE, "IMPORT_FROM_ORA", "true","GENERATE_JDBC_DATATYPE_MAPPING_FOR_OCI", "true", "ORA_CONNECTION_NAME", + ORACLE_OCI_CONNECTION_NAME, "oracle_cast_number_to_decimal_with_precision_and_scale", "36,1")) + .build(); + // } private static void setupOracleDbContainer() throws SQLException { @@ -370,7 +394,16 @@ void testSelectAllColsNumberFromOra() throws SQLException { assertThat(statementExasol.executeQuery("SELECT * FROM " + qualifiedTableNameActual), // matchesResultSet(expected)); } - + @Test + void testSelectAllColsNumberFromOraWithJDBCTypemapping() throws SQLException { + final String qualifiedTableNameActual = VIRTUAL_SCHEMA_ORACLE_NUMBER_TO_DECIMAL_JDBC_MAPPING + "." + + TABLE_ORACLE_NUMBER_HANDLING; + //previously final ResultSet expected = getExpectedResultSet("(A VARCHAR(100), B VARCHAR(100), C VARCHAR(100))", + final ResultSet expected = getExpectedResultSet("(A DECIMAL(36,1), B DECIMAL(36,1), C DECIMAL(36,2))", + "('12.3456789012345678901234567890123460E32', '12.3456789012345678901234567890E26', '12.3456789012345678901234567890123456E32')"); + assertThat(statementExasol.executeQuery("SELECT * FROM " + qualifiedTableNameActual), // + matchesResultSet(expected)); + } @ParameterizedTest @ValueSource(strings = {VIRTUAL_SCHEMA_JDBC_NUMBER_TO_DECIMAL, VIRTUAL_SCHEMA_ORACLE_NUMBER_TO_DECIMAL}) void testNumberDataTypes(final String virtualSchemaName) { @@ -549,9 +582,11 @@ private void assertExpressionExecutionStringResult(final String query, final Str final String actual = result.getString(1); MatcherAssert.assertThat(actual, containsString(expected)); } - +// @CsvSource(value = {"VIRTUAL_SCHEMA_JDBC, 12355.12345", // +// "VIRTUAL_SCHEMA_ORACLE, 01.2355123450E4"}) @ParameterizedTest @CsvSource(value = {"VIRTUAL_SCHEMA_JDBC, 12355.12345", // + "VIRTUAL_SCHEMA_ORACLE_JDBC_MAPPING, 12355.12345", "VIRTUAL_SCHEMA_ORACLE, 01.2355123450E4"}) void testFilterExpression(final String virtualSchemaName, final String expectedColumnValue) { final String qualifiedTableNameActual = virtualSchemaName + "." + TABLE_ORACLE_ALL_DATA_TYPES; diff --git a/src/test/resources/integration/scalarFunctionsParameterCache.yml b/src/test/resources/integration/scalarFunctionsParameterCache.yml index 478356e..4cf3be3 100644 --- a/src/test/resources/integration/scalarFunctionsParameterCache.yml +++ b/src/test/resources/integration/scalarFunctionsParameterCache.yml @@ -1041,7 +1041,7 @@ truncate: ['"DOUBLE_PRECISION_C0"', '"DECIMAL_C1"', '"DECIMAL_C2"', '"NUMBER1_C3 "DECIMAL_C2"', '"NUMBER1_C3", "NUMBER1_C3"'] unicode: ['"DECIMAL_C1"', '"DECIMAL_C2"', '"VARCHAR22_CHAR_C4"'] unicodechr: ['"DOUBLE_PRECISION_C0"', '"DECIMAL_C1"', '"DECIMAL_C2"', '"NUMBER1_C3"'] -upper: [ '"DECIMAL_C1"', '"DECIMAL_C2"', '"NUMBER1_C3"', '"VARCHAR22_CHAR_C4"' ] +upper: ['"DECIMAL_C1"', '"DECIMAL_C2"', '"NUMBER1_C3"', '"VARCHAR22_CHAR_C4"'] user: [''] week: ['"DATE_C5"', '"TIMESTAMP_0_C6"'] year: ['"DATE_C5"', '"TIMESTAMP_0_C6"']