From 5c36c7365b1dbd157556676d7164c8b4f04f487e Mon Sep 17 00:00:00 2001 From: Christoph Kuhnke Date: Thu, 12 Jan 2023 13:19:55 +0100 Subject: [PATCH] Feature/13 missing functionality (#20) * Added ListCommand, support for profile file, DeleteCommand, recursive operations, require read password * Updated documentation and requirements * Removed support to decode base64-encoded passwords Co-authored-by: Christoph Pirkl --- .gitattributes | 13 + ...ase_droid_upload_github_release_assets.yml | 6 +- .project-keeper.yml | 2 - .vscode/settings.json | 17 ++ dependencies.md | 160 +++++------ doc/changes/changelog.md | 1 + doc/changes/changes_1.0.0.md | 58 ++++ doc/design.md | 158 ++++++++++- doc/system_requirements.md | 106 ++++++-- doc/user_guide/user_guide.md | 250 +++++++++++++++--- error_code_config.yml | 2 +- pk_generated_parent.pom | 48 ++-- pom.xml | 96 ++----- release_config.yml | 2 +- .../java/com/exasol/bucketfs/Fallback.java | 27 ++ .../bucketfs/client/BucketFsClient.java | 72 ++++- .../ConsoleReaderWithFallbackToStdIn.java | 36 +++ .../exasol/bucketfs/client/CopyCommand.java | 93 +++---- .../exasol/bucketfs/client/DeleteCommand.java | 52 ++++ .../exasol/bucketfs/client/Downloader.java | 87 ++++++ .../exasol/bucketfs/client/ListCommand.java | 60 +++++ .../bucketfs/client/ListingProvider.java | 37 +++ .../bucketfs/client/PasswordReader.java | 78 ++++++ .../com/exasol/bucketfs/client/Uploader.java | 53 ++++ .../bucketfs/client/UseCaseDetector.java | 96 +++++++ .../com/exasol/bucketfs/profile/Profile.java | 96 +++++++ .../bucketfs/profile/ProfileReader.java | 105 ++++++++ .../exasol/bucketfs/url/BucketFsLocation.java | 92 +++++++ .../com/exasol/bucketfs/url/BucketFsPath.java | 60 +++++ .../com/exasol/bucketfs/url/BucketFsUrl.java | 178 ++++++------- .../java/com/exasol/bucketfs/url/OsCheck.java | 49 ++++ .../exasol/bucketfs/url/PathCompleter.java | 37 +++ .../com/exasol/bucketfs/url/UriConverter.java | 53 ++++ src/test/java/com/exasol/bucketfs/Lines.java | 7 + .../java/com/exasol/bucketfs/client/BFSC.java | 62 +++-- .../client/BucketFsClientExecutableJarIT.java | 51 ++-- .../exasol/bucketfs/client/CopyCommandIT.java | 162 ------------ .../bucketfs/client/DeleteCommandIT.java | 106 ++++++++ .../bucketfs/client/DownloadCommandIT.java | 165 ++++++++++++ .../bucketfs/client/IntegrationTestSetup.java | 143 ++++++++++ .../exasol/bucketfs/client/ListCommandIT.java | 131 +++++++++ .../bucketfs/client/UploadCommandIT.java | 172 ++++++++++++ .../bucketfs/profile/ProfileReaderTest.java | 83 ++++++ .../exasol/bucketfs/profile/ProfileTest.java | 29 ++ .../bucketfs/url/BucketFsLocationTest.java | 81 ++++++ .../exasol/bucketfs/url/BucketFsUrlTest.java | 172 ++++++++---- .../com/exasol/bucketfs/url/OsCheckTest.java | 48 ++++ .../bucketfs/url/PathCompleterTest.java | 90 +++++++ .../exasol/bucketfs/url/UriConverterTest.java | 47 ++++ 49 files changed, 3159 insertions(+), 670 deletions(-) create mode 100644 .gitattributes create mode 100644 .vscode/settings.json create mode 100644 doc/changes/changes_1.0.0.md create mode 100644 src/main/java/com/exasol/bucketfs/Fallback.java create mode 100644 src/main/java/com/exasol/bucketfs/client/ConsoleReaderWithFallbackToStdIn.java create mode 100644 src/main/java/com/exasol/bucketfs/client/DeleteCommand.java create mode 100644 src/main/java/com/exasol/bucketfs/client/Downloader.java create mode 100644 src/main/java/com/exasol/bucketfs/client/ListCommand.java create mode 100644 src/main/java/com/exasol/bucketfs/client/ListingProvider.java create mode 100644 src/main/java/com/exasol/bucketfs/client/PasswordReader.java create mode 100644 src/main/java/com/exasol/bucketfs/client/Uploader.java create mode 100644 src/main/java/com/exasol/bucketfs/client/UseCaseDetector.java create mode 100644 src/main/java/com/exasol/bucketfs/profile/Profile.java create mode 100644 src/main/java/com/exasol/bucketfs/profile/ProfileReader.java create mode 100644 src/main/java/com/exasol/bucketfs/url/BucketFsLocation.java create mode 100644 src/main/java/com/exasol/bucketfs/url/BucketFsPath.java create mode 100644 src/main/java/com/exasol/bucketfs/url/OsCheck.java create mode 100644 src/main/java/com/exasol/bucketfs/url/PathCompleter.java create mode 100644 src/main/java/com/exasol/bucketfs/url/UriConverter.java create mode 100644 src/test/java/com/exasol/bucketfs/Lines.java delete mode 100644 src/test/java/com/exasol/bucketfs/client/CopyCommandIT.java create mode 100644 src/test/java/com/exasol/bucketfs/client/DeleteCommandIT.java create mode 100644 src/test/java/com/exasol/bucketfs/client/DownloadCommandIT.java create mode 100644 src/test/java/com/exasol/bucketfs/client/IntegrationTestSetup.java create mode 100644 src/test/java/com/exasol/bucketfs/client/ListCommandIT.java create mode 100644 src/test/java/com/exasol/bucketfs/client/UploadCommandIT.java create mode 100644 src/test/java/com/exasol/bucketfs/profile/ProfileReaderTest.java create mode 100644 src/test/java/com/exasol/bucketfs/profile/ProfileTest.java create mode 100644 src/test/java/com/exasol/bucketfs/url/BucketFsLocationTest.java create mode 100644 src/test/java/com/exasol/bucketfs/url/OsCheckTest.java create mode 100644 src/test/java/com/exasol/bucketfs/url/PathCompleterTest.java create mode 100644 src/test/java/com/exasol/bucketfs/url/UriConverterTest.java diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c87b87d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,13 @@ +*.sh text eol=lf +*.bat text eol=crlf + +pk_generated_parent.pom linguist-generated=true +dependencies.md linguist-generated=true +doc/changes/changelog.md linguist-generated=true +.github/workflows/broken_links_checker.yml linguist-generated=true +.github/workflows/ci-build-next-java.yml linguist-generated=true +.github/workflows/ci-build.yml linguist-generated=true +.github/workflows/dependencies_check.yml linguist-generated=true +.github/workflows/release_droid_prepare_original_checksum.yml linguist-generated=true +.github/workflows/release_droid_print_quick_checksum.yml linguist-generated=true +.github/workflows/release_droid_upload_github_release_assets.yml linguist-generated=true diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml index 1fd0b60..7350faf 100644 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -24,7 +24,9 @@ jobs: - name: Build with Maven skipping tests run: mvn --batch-mode clean verify -DskipTests - name: Generate sha256sum files - run: find target -maxdepth 1 -name *.jar -exec bash -c 'sha256sum {} > {}.sha256' \; + run: | + cd target + find . -maxdepth 1 -name \*.jar -exec bash -c 'sha256sum {} > {}.sha256' \; - name: Upload assets to the GitHub release draft uses: shogo82148/actions-upload-release-asset@v1 with: @@ -39,4 +41,4 @@ jobs: uses: shogo82148/actions-upload-release-asset@v1 with: upload_url: ${{ github.event.inputs.upload_url }} - asset_path: target/error_code_report.json \ No newline at end of file + asset_path: target/error_code_report.json diff --git a/.project-keeper.yml b/.project-keeper.yml index a5a893c..0ce4cac 100644 --- a/.project-keeper.yml +++ b/.project-keeper.yml @@ -4,5 +4,3 @@ sources: modules: - integration_tests - jar_artifact -linkReplacements: - - "https://www.mojohaus.org/flatten-maven-plugin/flatten-maven-plugin|https://www.mojohaus.org/flatten-maven-plugin/" diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..21c0cc1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,17 @@ +{ + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true, + "source.generate.finalModifiers": true, + "source.fixAll": true + }, + "java.codeGeneration.useBlocks": true, + "java.saveActions.organizeImports": true, + "java.sources.organizeImports.starThreshold": 3, + "java.sources.organizeImports.staticStarThreshold": 3, + "java.test.config": { + "vmArgs": [ + "-Djava.util.logging.config.file=src/test/resources/logging.properties" + ] + }, +} diff --git a/dependencies.md b/dependencies.md index 68560ac..0e80593 100644 --- a/dependencies.md +++ b/dependencies.md @@ -3,95 +3,99 @@ ## Compile Dependencies -| Dependency | License | -| ---------------------------------------- | --------------------------------------------- | -| [BucketFS Java][0] | [MIT][1] | -| [error-reporting-java][2] | [MIT][1] | -| [picocli][4] | [The Apache Software License, version 2.0][5] | -| [EqualsVerifier | release normal jar][6] | [Apache License, Version 2.0][7] | -| [JUnit5 System Extensions][8] | [Eclipse Public License v2.0][9] | +| Dependency | License | +| ------------------------- | --------------------------------------------- | +| [BucketFS Java][0] | [MIT License][1] | +| [error-reporting-java][2] | [MIT License][3] | +| [picocli][4] | [The Apache Software License, version 2.0][5] | +| [java-ini-parser][6] | [Apache License, Version 2.0][5] | ## Test Dependencies | Dependency | License | | ----------------------------------------------- | --------------------------------- | -| [JUnit Jupiter Engine][10] | [Eclipse Public License v2.0][11] | -| [JUnit Jupiter Params][10] | [Eclipse Public License v2.0][11] | -| [Hamcrest][14] | [BSD License 3][15] | -| [mockito-junit-jupiter][16] | [The MIT License][17] | -| [Test containers for Exasol on Docker][18] | [MIT][1] | -| [Testcontainers :: JUnit Jupiter Extension][20] | [MIT][21] | +| [JUnit Jupiter Engine][7] | [Eclipse Public License v2.0][8] | +| [JUnit Jupiter Params][7] | [Eclipse Public License v2.0][8] | +| [Hamcrest][9] | [BSD License 3][10] | +| [mockito-junit-jupiter][11] | [The MIT License][12] | +| [Test containers for Exasol on Docker][13] | [MIT License][14] | +| [Testcontainers :: JUnit Jupiter Extension][15] | [MIT][16] | +| [EqualsVerifier | release normal jar][17] | [Apache License, Version 2.0][18] | +| [JUnit5 System Extensions][19] | [Eclipse Public License v2.0][20] | +| [junit-pioneer][21] | [Eclipse Public License v2.0][8] | ## Plugin Dependencies | Dependency | License | | ------------------------------------------------------- | --------------------------------------------- | | [SonarQube Scanner for Maven][22] | [GNU LGPL 3][23] | -| [Apache Maven Source Plugin][24] | [Apache License, Version 2.0][7] | -| [Apache Maven Javadoc Plugin][26] | [Apache License, Version 2.0][7] | -| [Apache Maven Compiler Plugin][28] | [Apache License, Version 2.0][7] | -| [Apache Maven Enforcer Plugin][30] | [Apache License, Version 2.0][7] | -| [Maven Flatten Plugin][32] | [Apache Software Licenese][5] | -| [org.sonatype.ossindex.maven:ossindex-maven-plugin][34] | [ASL2][5] | -| [Reproducible Build Maven Plugin][36] | [Apache 2.0][5] | -| [Maven Surefire Plugin][38] | [Apache License, Version 2.0][7] | -| [OpenFastTrace Maven Plugin][40] | [GNU General Public License v3.0][41] | -| [Project keeper maven plugin][42] | [The MIT License][43] | -| [Versions Maven Plugin][44] | [Apache License, Version 2.0][7] | -| [Apache Maven Assembly Plugin][46] | [Apache License, Version 2.0][7] | -| [Apache Maven JAR Plugin][48] | [Apache License, Version 2.0][7] | -| [Artifact reference checker and unifier][50] | [MIT][1] | -| [Maven Failsafe Plugin][52] | [Apache License, Version 2.0][7] | -| [JaCoCo :: Maven Plugin][54] | [Eclipse Public License 2.0][55] | -| [error-code-crawler-maven-plugin][56] | [MIT][1] | -| [Maven Clean Plugin][58] | [The Apache Software License, Version 2.0][5] | -| [Maven Resources Plugin][60] | [The Apache Software License, Version 2.0][5] | -| [Maven Install Plugin][62] | [The Apache Software License, Version 2.0][5] | -| [Maven Deploy Plugin][64] | [The Apache Software License, Version 2.0][5] | -| [Maven Site Plugin 3][66] | [The Apache Software License, Version 2.0][5] | +| [OpenFastTrace Maven Plugin][24] | [GNU General Public License v3.0][25] | +| [Project keeper maven plugin][26] | [The MIT License][27] | +| [Apache Maven Compiler Plugin][28] | [Apache License, Version 2.0][18] | +| [Apache Maven Enforcer Plugin][29] | [Apache License, Version 2.0][18] | +| [Maven Flatten Plugin][30] | [Apache Software Licenese][18] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][31] | [ASL2][5] | +| [Maven Surefire Plugin][32] | [Apache License, Version 2.0][18] | +| [Versions Maven Plugin][33] | [Apache License, Version 2.0][18] | +| [Apache Maven Assembly Plugin][34] | [Apache License, Version 2.0][18] | +| [Apache Maven JAR Plugin][35] | [Apache License, Version 2.0][18] | +| [Artifact reference checker and unifier][36] | [MIT License][37] | +| [Maven Failsafe Plugin][38] | [Apache License, Version 2.0][18] | +| [JaCoCo :: Maven Plugin][39] | [Eclipse Public License 2.0][40] | +| [error-code-crawler-maven-plugin][41] | [MIT License][42] | +| [Reproducible Build Maven Plugin][43] | [Apache 2.0][5] | +| [Maven Clean Plugin][44] | [The Apache Software License, Version 2.0][5] | +| [Maven Resources Plugin][45] | [The Apache Software License, Version 2.0][5] | +| [Maven Install Plugin][46] | [The Apache Software License, Version 2.0][5] | +| [Maven Deploy Plugin][47] | [The Apache Software License, Version 2.0][5] | +| [Maven Site Plugin 3][48] | [The Apache Software License, Version 2.0][5] | -[0]: https://github.com/exasol/bucketfs-java -[2]: https://github.com/exasol/error-reporting-java -[9]: http://www.eclipse.org/legal/epl-v20.html +[0]: https://github.com/exasol/bucketfs-java/ +[1]: https://github.com/exasol/bucketfs-java/blob/main/LICENSE +[2]: https://github.com/exasol/error-reporting-java/ +[3]: https://github.com/exasol/error-reporting-java/blob/main/LICENSE +[4]: https://picocli.info [5]: http://www.apache.org/licenses/LICENSE-2.0.txt -[38]: https://maven.apache.org/surefire/maven-surefire-plugin/ -[58]: http://maven.apache.org/plugins/maven-clean-plugin/ -[1]: https://opensource.org/licenses/MIT -[16]: https://github.com/mockito/mockito -[52]: https://maven.apache.org/surefire/maven-failsafe-plugin/ -[32]: https://www.mojohaus.org/flatten-maven-plugin/ -[42]: https://github.com/exasol/project-keeper/ -[44]: http://www.mojohaus.org/versions-maven-plugin/ -[15]: http://opensource.org/licenses/BSD-3-Clause -[28]: https://maven.apache.org/plugins/maven-compiler-plugin/ -[21]: http://opensource.org/licenses/MIT -[40]: https://github.com/itsallcode/openfasttrace-maven-plugin -[55]: https://www.eclipse.org/legal/epl-2.0/ -[23]: http://www.gnu.org/licenses/lgpl.txt -[18]: https://github.com/exasol/exasol-testcontainers -[54]: https://www.jacoco.org/jacoco/trunk/doc/maven.html -[17]: https://github.com/mockito/mockito/blob/main/LICENSE -[36]: http://zlika.github.io/reproducible-build-maven-plugin -[43]: https://github.com/exasol/project-keeper/blob/main/LICENSE -[41]: https://www.gnu.org/licenses/gpl-3.0.html -[7]: https://www.apache.org/licenses/LICENSE-2.0.txt +[6]: https://github.com/vincentrussell/java-ini-parser +[7]: https://junit.org/junit5/ +[8]: https://www.eclipse.org/legal/epl-v20.html +[9]: http://hamcrest.org/JavaHamcrest/ +[10]: http://opensource.org/licenses/BSD-3-Clause +[11]: https://github.com/mockito/mockito +[12]: https://github.com/mockito/mockito/blob/main/LICENSE +[13]: https://github.com/exasol/exasol-testcontainers/ +[14]: https://github.com/exasol/exasol-testcontainers/blob/main/LICENSE +[15]: https://testcontainers.org +[16]: http://opensource.org/licenses/MIT +[17]: https://www.jqno.nl/equalsverifier +[18]: https://www.apache.org/licenses/LICENSE-2.0.txt +[19]: https://github.com/itsallcode/junit5-system-extensions +[20]: http://www.eclipse.org/legal/epl-v20.html +[21]: https://junit-pioneer.org/ [22]: http://sonarsource.github.io/sonar-scanner-maven/ -[6]: https://www.jqno.nl/equalsverifier -[30]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ -[11]: https://www.eclipse.org/legal/epl-v20.html -[62]: http://maven.apache.org/plugins/maven-install-plugin/ -[10]: https://junit.org/junit5/ -[34]: https://sonatype.github.io/ossindex-maven/maven-plugin/ -[20]: https://testcontainers.org -[8]: https://github.com/itsallcode/junit5-system-extensions -[24]: https://maven.apache.org/plugins/maven-source-plugin/ -[14]: http://hamcrest.org/JavaHamcrest/ -[64]: http://maven.apache.org/plugins/maven-deploy-plugin/ -[66]: http://maven.apache.org/plugins/maven-site-plugin/ -[60]: http://maven.apache.org/plugins/maven-resources-plugin/ -[26]: https://maven.apache.org/plugins/maven-javadoc-plugin/ -[50]: https://github.com/exasol/artifact-reference-checker-maven-plugin -[56]: https://github.com/exasol/error-code-crawler-maven-plugin -[48]: https://maven.apache.org/plugins/maven-jar-plugin/ -[4]: http://picocli.info -[46]: https://maven.apache.org/plugins/maven-assembly-plugin/ +[23]: http://www.gnu.org/licenses/lgpl.txt +[24]: https://github.com/itsallcode/openfasttrace-maven-plugin +[25]: https://www.gnu.org/licenses/gpl-3.0.html +[26]: https://github.com/exasol/project-keeper/ +[27]: https://github.com/exasol/project-keeper/blob/main/LICENSE +[28]: https://maven.apache.org/plugins/maven-compiler-plugin/ +[29]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ +[30]: https://www.mojohaus.org/flatten-maven-plugin/ +[31]: https://sonatype.github.io/ossindex-maven/maven-plugin/ +[32]: https://maven.apache.org/surefire/maven-surefire-plugin/ +[33]: https://www.mojohaus.org/versions-maven-plugin/ +[34]: https://maven.apache.org/plugins/maven-assembly-plugin/ +[35]: https://maven.apache.org/plugins/maven-jar-plugin/ +[36]: https://github.com/exasol/artifact-reference-checker-maven-plugin/ +[37]: https://github.com/exasol/artifact-reference-checker-maven-plugin/blob/main/LICENSE +[38]: https://maven.apache.org/surefire/maven-failsafe-plugin/ +[39]: https://www.jacoco.org/jacoco/trunk/doc/maven.html +[40]: https://www.eclipse.org/legal/epl-2.0/ +[41]: https://github.com/exasol/error-code-crawler-maven-plugin/ +[42]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE +[43]: http://zlika.github.io/reproducible-build-maven-plugin +[44]: http://maven.apache.org/plugins/maven-clean-plugin/ +[45]: http://maven.apache.org/plugins/maven-resources-plugin/ +[46]: http://maven.apache.org/plugins/maven-install-plugin/ +[47]: http://maven.apache.org/plugins/maven-deploy-plugin/ +[48]: http://maven.apache.org/plugins/maven-site-plugin/ diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index b081b25..b89740d 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [1.0.0](changes_1.0.0.md) * [0.2.2](changes_0.2.2.md) * [0.2.1](changes_0.2.1.md) * [0.2.0](changes_0.2.0.md) diff --git a/doc/changes/changes_1.0.0.md b/doc/changes/changes_1.0.0.md new file mode 100644 index 0000000..c8ba749 --- /dev/null +++ b/doc/changes/changes_1.0.0.md @@ -0,0 +1,58 @@ +# bucketfs-client 1.0.0, released 2023-01-12 + +Code name: Added Missing Functionality + +## Summary + +Added missing functionality and additional features: + +* Enabled to list buckets, files and directories inside a bucket or with a common path prefix. +* Enabled to delete files and directories in buckets. +* Added support for profiles in a configuration file. +* Added option for recursive operations. +* Added support for read password for private buckets. +* Updated documentation and requirements. +* Removed support to decode base64-encoded passwords. +* Refactored URL conversion and password entry and tests and updated dependencies. + +## Features + +* #13: Added missing functionality + +## Dependency Updates + +### Compile Dependency Updates + +* Updated `com.exasol:bucketfs-java:2.3.0` to `2.6.0` +* Updated `com.exasol:error-reporting-java:0.4.1` to `1.0.0` +* Added `com.github.vincentrussell:java-ini-parser:1.3` +* Updated `info.picocli:picocli:4.6.3` to `4.7.0` +* Removed `nl.jqno.equalsverifier:equalsverifier:3.10` +* Removed `org.itsallcode:junit5-system-extensions:1.2.0` + +### Test Dependency Updates + +* Updated `com.exasol:exasol-testcontainers:6.1.1` to `6.4.0` +* Added `nl.jqno.equalsverifier:equalsverifier:3.12.1` +* Added `org.itsallcode:junit5-system-extensions:1.2.0` +* Added `org.junit-pioneer:junit-pioneer:1.7.1` +* Updated `org.junit.jupiter:junit-jupiter-engine:5.8.2` to `5.9.1` +* Updated `org.junit.jupiter:junit-jupiter-params:5.8.2` to `5.9.1` +* Updated `org.mockito:mockito-junit-jupiter:4.5.1` to `4.9.0` +* Updated `org.testcontainers:junit-jupiter:1.17.1` to `1.17.6` + +### Plugin Dependency Updates + +* Updated `com.exasol:artifact-reference-checker-maven-plugin:0.4.1` to `0.4.2` +* Updated `com.exasol:error-code-crawler-maven-plugin:1.1.1` to `1.2.1` +* Updated `com.exasol:project-keeper-maven-plugin:2.4.2` to `2.9.1` +* Updated `io.github.zlika:reproducible-build-maven-plugin:0.15` to `0.16` +* Updated `org.apache.maven.plugins:maven-assembly-plugin:3.3.0` to `3.4.2` +* Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.0.0` to `3.1.0` +* Updated `org.apache.maven.plugins:maven-failsafe-plugin:3.0.0-M6` to `3.0.0-M7` +* Updated `org.apache.maven.plugins:maven-jar-plugin:3.2.2` to `3.3.0` +* Removed `org.apache.maven.plugins:maven-javadoc-plugin:3.4.0` +* Removed `org.apache.maven.plugins:maven-source-plugin:3.2.1` +* Updated `org.apache.maven.plugins:maven-surefire-plugin:3.0.0-M6` to `3.0.0-M7` +* Updated `org.codehaus.mojo:flatten-maven-plugin:1.2.7` to `1.3.0` +* Updated `org.codehaus.mojo:versions-maven-plugin:2.10.0` to `2.13.0` diff --git a/doc/design.md b/doc/design.md index e826563..35669c5 100644 --- a/doc/design.md +++ b/doc/design.md @@ -10,6 +10,22 @@ This section introduces technical system constraints. +## No Confirmation for Delete Operation +`const~delete-ansynchronous~1` + +Deleting a file in BucketFS is an asynchronous operation. Additionally BucketFS service does not signal whether the delete operation succeeded or failed. + +Needs: dsn + +## Ambigue Entries +`const~ambigue-entries~1` + +A bucket can hold entries with common prefix and a slash `/` as separator. When interpreting this as a hierarchy similar to a file system, we need to consider that BucketFS also allows having a file with the same name as a directory. For example `ambigue/file.txt` and `ambigue` can exist at the same time. + +When copying from the local file system to BucketFS this does not create any difficulty but an ambigue entry cannot be copied from BucketFS to the local file system as this allows an entry only to be either a file or a directory but not both at the same time. + +Needs: dsn + # Solution Strategy The BucketFS Client mimics the popular GNU core utilities in terms of user interface and behavior. It is designed to work independently of the platform so that it can be used on any OS that Java can run on and has a console (Linux, Mac OS X, Windows). @@ -43,7 +59,6 @@ This section describes the runtime behavior of the software. The `BucketFsUrl` represents the uniform resource locator for objects on BucketFs. Covers: - * `req~bucketfs-url~1` Needs: impl, utest @@ -54,32 +69,156 @@ Needs: impl, utest The `CopyCommand` copies a single file from a public bucket to the local filesystem. Covers: - * `req~copy-single-file-from-public-bucket~1` Needs: impl, itest -### CopyCommand Copies File To Bucket With Interactive Password +### CopyCommand Copies File To Bucket With Write Password `dsn~copy-command-copies-file-to-bucket~1` The `CopyCommand` copies a single file from the local filesystem to a bucket. Covers: +* `req~copy-single-file-to-bucket-with-interactive-password~2` + +Needs: impl, itest + +### Copying Ambigue Entries From Bucket to Local File System +`dsn~download-ambigue-file~1` + +If the user requests to copy an ambigue entry from a bucket to the local file system non-recursively then BFSC copies only the regular file but not the directory with the same name. + +BFSC downloads an ambigue regular file if all of the following conditions hold true +* The user requests to copy an ambigue entry from a bucket to the local file system +* The user does not request recursive mode + +Covers +* `const~ambigue-entries~1` + +Needs: impl, itest + +`dsn~download-ambigue-directory-recursively~1` + +BFSC recursively downloads an ambigue directory if all of the following conditions hold true +* The user requests to copy an ambigue entry from a bucket to the local file system. +* The user requests recursive mode. +* The user resolves the ambiguity by appending a slash to the name of the ambigue entry. + +Covers +* `const~ambigue-entries~1` + +Needs: impl, itest + +`dsn~download-ambigue-entry-recursively~1` + +BFSC reports an error if all of the following conditions hold true +* The user requests to copy an ambigue entry from a bucket to the local file system. +* The user requests recursive mode. +* The entry requested by the user does have a trailing a slash. + +Covers +* `const~ambigue-entries~1` + +Needs: impl, itest + +`dsn~copy-ambigue-entrie-on-lower-level~1` + +When copying recursively from a bucket to the local file system and BFSC detects an ambigue entry on a lower level then BFSC reports a warning and copies only the directory but not the regular file with the same name. + +Covers +* `const~ambigue-entries~1` + +Needs: impl, itest + +## Listing Contents + +### Listing Contents of a Bucket +`dsn~list-contents~1` + +The `ListCommand` displays the contents of a directory in a bucket as a sorted list. + +Covers: +* `req~list-contents-of-a-bucket~1` + +Needs: impl, itest + +### Files and Directories +`dsn~list-files-and-directories~1` + +The output of the `ListCommand` contains files as well as directories if there are such. + +Covers: +* `req~list-contents-of-a-bucket~1` + +Needs: impl, itest + +### Indicate the Type of Entrys +`dsn~highlight-type-of-entries~1` + +The `ListCommand` indicates directories with a trailing slash. + +Rationale: +* By appending a slash to directories BFSC enables users to distinguish ambigue entries in BucketFs, i.e. regular files and directories with identical names. + +Covers: +* `req~list-contents-of-a-bucket~1` +* `const~ambigue-entries~1` + +Needs: impl, itest + +## Deleting Files + +### Deleting a File in a Bucket +`dsn~delete-file~1` + +The `DeleteCommand` deletes a file in a bucket. + +Covers: +* `req~delete-files~1` -* `req~copy-single-file-to-bucket-with-interactive-password~1` +Needs: impl, itest + +### No Error When Deleting a Non-Existing File +`dsn~no-error-when-deleting-a-non-existing-file~1` + +BFSC does not signal an error in case of trying to delete a non-existing file. + +Covers: +* `const~delete-ansynchronous~1` + +Needs: impl, itest + +### Deleting Ambigue Entries +`dsn~delete-ambigue-entry-non-recursively~1` + +When the user asks to delete an ambigue entry in non-recursive mode then BFSC will only delete the regular file but not the directory with the same name. + +Covers: +* `const~ambigue-entries~1` + +Needs: impl, itest + +`dsn~delete-ambigue-entry-recursively~1` + +When the user asks to delete an ambigue entry in recursive mode then BFSC will delete +* the regular file +* the directory recursively + +Covers: +* `const~ambigue-entries~1` Needs: impl, itest ## Command Line Interface ### Sub-command Requires Hidden Write Password -`dsn~sub-command-requires-hidden-password~1` +`dsn~sub-command-requires-hidden-password~2` -In case of interactive password entry, sub-command (e.g. `cp`) tells the `picocli` library to prompt for a password with hidden entry. +In case of interactive password entry BFSC prompts for a password with hidden entry. Covers: - -* `req~interactive-password-entry~1` +* `req~no-command-line-option-for-password~1` +* `req~password-protected-bucket-access~1` Needs: impl, itest @@ -113,7 +252,7 @@ The choice of the CLI library has an impact on usability and maintainability of ### Alternatives considered 1. [args4J](http://args4j.kohsuke.org/). No dependencies. [Latest release from 2016](https://search.maven.org/artifact/args4j/args4j). -1. [JCommander](https://jcommander.org). Fulfills all criteria except for sub-commands and positional parameters, which are possible, but must be hand-coded. Leaner than the selected solution though. +1. [JCommander](https://jcommander.org). Fulfills all criteria except for sub-commands and positional parameters, which are possible, but must be hand-coded. Leaner than the selected solution, though. 1. [JOpt Simple](https://jopt-simple.github.io/jopt-simple). [Latest release from 2018](https://search.maven.org/artifact/net.sf.jopt-simple/jopt-simple) is in alpha stage. Latest stable from 2017. 1. [Spring Shell](https://github.com/spring-projects/spring-shell) has way too many dependencies. Latest release was from 2017. @@ -127,7 +266,6 @@ We are using [picocli](https://picocli.info/). It is small enough, under active BFSC uses [picocli](https://picocli.info/) to parse the command line. Covers: - * `req~gnu-style-command-line-arguments~1` Needs: impl diff --git a/doc/system_requirements.md b/doc/system_requirements.md index 2fbeba9..3387630 100644 --- a/doc/system_requirements.md +++ b/doc/system_requirements.md @@ -53,11 +53,25 @@ Needs: req ### Copying Files `feat~copying-files~1` -BFSC supports copying file from and to buckets in BucketFS. +BFSC supports copying files from and to buckets in BucketFS. Rationale: -This allows uploading UFDs, configuration and drivers or checking the contents of files on BucketFS. +This allows uploading UFDs, configuration files and drivers, or checking the contents of files on BucketFS. + +Needs: req + +### Listing Contents +`feat~listing-contents~1` + +BFSC supports listing the contents of a bucket in BucketFS. + +Needs: req + +### Deleting Files +`feat~deleting-files~1` + +BFSC supports deleting files inside a bucket in BucketFS. Needs: req @@ -72,21 +86,23 @@ To make BucketFS access similar to other file systems, BFSC introduces the conce A Bucket URL locates a resource inside a bucket with the following syntax: - bucket-url = protocol-identifier "://" bucketfs-service-name "/" bucket-name path-in-bucket - - protocol-identifier = "bfs" - - bucketfs-service-name = segment - - bucket-name = segment - - path-in-bucket = 1*("/" segment) - - segment = segment-start-character *segment-character - - segment-start-character = (ALPHA / DIGIT / "_") - - segment-character = (ALPHA / DIGIT / "-" / "_" / ".") +```bash +bucket-url = protocol-identifier "://" bucketfs-service-name "/" bucket-name path-in-bucket + +protocol-identifier = "bfs" + +bucketfs-service-name = segment + +bucket-name = segment + +path-in-bucket = 1*("/" segment) + +segment = segment-start-character *segment-character + +segment-start-character = (ALPHA / DIGIT / "_") + +segment-character = (ALPHA / DIGIT / "-" / "_" / ".") +``` Covers: @@ -101,7 +117,9 @@ Needs: dsn Users can copy a single file from BucketFS to local file storage with the following command: - bfs cp +```bash +bfs cp +``` Covers: @@ -110,15 +128,17 @@ Covers: Needs: dsn #### Copy Single File to Bucket With Interactive Password -`req~copy-single-file-to-bucket-with-interactive-password~1` +`req~copy-single-file-to-bucket-with-interactive-password~2` Users can copy a single file from local file storage to a public bucket with the following command: - bfs cp --password +```bash +bfs cp +``` Comment: -See also: [Interactive Password Entry](#interactive-password-entry) +See also: [Password Protected Bucket Access](#interactive-password-entry) Covers: @@ -126,6 +146,28 @@ Covers: Needs: dsn +### Listing Contents +`req~list-contents-of-a-bucket~1` + +Users can list the contents of a bucket in BucketFS. + +Covers: + +* `feat~listing-contents~1` + +Needs: dsn + +### Deleting Files +`req~delete-files~1` + +Users can delete files in a bucket in BucketFS. + +Covers: + +* `feat~deleting-files~1` + +Needs: dsn + ### Command Line Interface BFSC is a command line tool. GNU tools like `cp` and `ls` serve as template for the arguments, mostly because a lot of users are familiar with them. @@ -147,21 +189,27 @@ Covers: Needs: dsn -#### Interactive Password Entry -`req~interactive-password-entry~1` +#### Password Protected Bucket Access +`req~password-protected-bucket-access~1` -When the `--password` switch is set, BFSC brings up a password prompt where users have to enter the password interactively. +For write operations like copying files to the BucketFS or deleting files from the BucketFS BFSC will retrieve the required write password. BFSC supports to read the password either from an [environment variable](#environment-variables-for-default-parameters) or from an interactive prompt hiding the characters typed by the user. -Rationale: +Covers: +* `feat~command-line-interface~1` -Passwords should never be supplied via the command line because they otherwise are logged in the command history. This is not the case with interactive input. +Needs: dsn -Comment: +#### No Password on Command Line +`req~no-command-line-option-for-password~1` -If batch mode is required, the password must be taken from a key store. But this is outside of the scope of this requirement. +BFSC does not enable users to supply the write password on the command line. Covers: - * `feat~command-line-interface~1` +Rationale: + +Passwords should never be supplied via the command line because they otherwise are logged in the command history. This is not the case with interactive input. + + Needs: dsn \ No newline at end of file diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index d10a3fa..8de7d5b 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -2,15 +2,13 @@ The BucketFS Client (BFSC) is a client program for Exasol's BucketFS distributed file system. -It allows you to read and write the contents of buckets. +BFSC allows you to read and write the contents of buckets. -A bucket is a storage space on an Exasol cluster that automatically takes care of distributing files loaded onto it. - -Those files can then be used in [User Defined Functions (UDFs)](https://docs.exasol.com/database_concepts/udf_scripts.htm). +A bucket is a storage space on an Exasol cluster that automatically takes care of distributing files loaded onto it. Those files can then be used in [User Defined Functions (UDFs)](https://docs.exasol.com/database_concepts/udf_scripts.htm), e.g. for [Virtual Schemas](https://docs.exasol.com/db/latest/database_concepts/virtual_schemas.htm) which are a special type of UDFs. See [official documentation](https://docs.exasol.com/db/latest/database_concepts/bucketfs/bucketfs.htm) for more details on BucketFS. ## Use Cases -The most common use for BucketFS is storing files on it that UDFs need: +The most common use case for BucketFS is storing files on it that UDFs need: * Libraries for your UDFs * Drivers for external data sources or sinks @@ -26,84 +24,272 @@ Installation depends on your operating system. * Any operating system: [Adoptium OpenJDK build](https://adoptium.net/) * Linux * Ubuntu, Debian: - ```bash + ```shell sudo apt install openjdk-11-jre-headless ``` * RedHat - ```bash + ```shell sudo yum install java-11-openjdk ``` * SuSE: [OpenJDK build of the Leap project](https://software.opensuse.org/download/package?package=java-11-openjdk&project=openSUSE%3ALeap%3A15.1%3AUpdate) -* Windows: [Microsoft OpenJDK build](https://www.microsoft.com/openjdk) ## Using the Client The client is a Java program. -### Using the JAR File Directly - The purist way of starting that is of course starting the application straight out of the JAR archive. -```bash +```shell java -jar ""