Merge branch 'dev' into 3dgan_integration

.github/workflows/documentation.yml

@@ -0,0 +1,38 @@
+name: Build and Deploy Sphinx Documentation
+
+# on:
+#   push:
+#     branches:
+#       - docs_dev
+#     tags:
+#       - 'v*'
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install dependencies
+      run: |
+        pip install -r docs/requirements.txt
+        pip install sphinx sphinx-rtd-theme
+
+    - name: Build documentation
+      run: |
+        cd docs
+        make html
+
+    - name: Deploy to GitHub Pages
+      uses: peaceiris/actions-gh-pages@v3
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: ./docs/_build/html

.github/workflows/lint.yml

@@ -30,7 +30,7 @@ jobs:
 
       # Runs the Super-Linter action
       - name: Run Super-Linter on new changes
-        uses: github/super-linter/slim@v5
+        uses: super-linter/super-linter/slim@v6.3.0
         env:
           DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -43,10 +43,14 @@ jobs:
           VALIDATE_PYTHON_PYLINT: false
           VALIDATE_HTML: false
           VALIDATE_GITLEAKS: false
+          VALIDATE_BASH_EXEC: false
+          VALIDATE_CHECKOV: false # activate to lint k8s pods
+          VALIDATE_SHELL_SHFMT: false
 
           # Only check new or edited files
           VALIDATE_ALL_CODEBASE: false
           # Fail on errors
           DISABLE_ERRORS: false
           # Skip linting of docs
-          FILTER_REGEX_EXCLUDE: .*docs/index.md|.*docs/docs/.*|.*ISSUE_TEMPLATE/.*|use-cases/.*
+          FILTER_REGEX_EXCLUDE: .*docs/index.md|.*docs/docs/.*|.*ISSUE_TEMPLATE/.*|use-cases/.*|experimental/.*
+          BASH_SEVERITY: error

.github/workflows/workflows-dt.yml → .github/workflows/pytest.yml

@@ -1,10 +1,12 @@
 ---
-name: Test workflows
+name: Unit and integration tests
 
 on:
   pull_request:
     branches: [main, dev]
 
+# TODO: use container and set custom TORCH_ENV and TF_ENV env variables
+
 jobs:
   test-itwinai:
     name: Test itwinai with pytest

.github/workflows/sqaaas.yml

@@ -7,13 +7,28 @@ name: SQAaaS
 on:
   push:
     branches: [main, dev]
-  pull_request:
-    branches: [main, dev]
+  # pull_request:
+  #   branches: [main, dev]
 
 jobs:
   sqaaas_job:
     runs-on: ubuntu-latest
     name: Job that triggers SQAaaS platform
     steps:
-      - name: SQAaaS assessment step
+      - name: Step definition for validating the workflow
+        uses: eosc-synergy/sqaaas-step-action@v1
+        with:
+          name: workflow_validation_step
+          tool: commands
+          commands: |
+            make torch-cpu
+            make tf-2.13
+            micromamba run -p ./.venv-pytorch pip install .[dev]
+            micromamba run -p ./.venv-pytorch pytest -v ./tests/ -m "not slurm and not memory_heavy"
+          container: eoscsynergy/sqaaas-micromamba:1.5.3-1
+      - name: Print out payload
+        run: cat workflow_validation_step.json
+      - name: SQAaaS assessment with unit testing (QC.Uni) step
         uses: eosc-synergy/sqaaas-assessment-action@v2
+        with:
+          qc_uni_steps: workflow_validation_step

.gitignore

@@ -1,33 +1,47 @@
+s*.png
+*.pdf
 *_logs
-exp_data/
+logs_*
 TODO
 /data
 nohup*
-lightning_logs
-mlruns
 tmp*
 .tmp*
 checkpoints/
 mamba*
-MNIST
-mllogs
-*.out
-*.err
-.logs/
 pl-training.yml
 *-predictions/
 *-data/
-*.pth
 *.tar.gz
+*.pth
+*.csv
+*tar.gz
+0
+*.tar
+
+# Use cases files
+MNIST
+3dgan-generated-data/
+mnist-sample-data/
+exp_data/
+
 
 # Kubernetes
 secret*.yaml
 
 # Custom envs
 .venv*
+envAI_*
 
 # Logs
 logs/
+ml_logs/
+mllogs/
+*.out
+*.err
+.logs/
+lightning_logs/
+mlruns/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.readthedocs.yaml

@@ -0,0 +1,32 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+    # You can also specify other tool versions:
+    # nodejs: "19"
+    # rust: "1.64"
+    # golang: "1.19"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#    - pdf
+#    - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+   install:
+   - requirements: docs/requirements.txt

.vscode/settings.json

@@ -1,6 +1,4 @@
 {
-    "python.linting.flake8Enabled": true,
-    "python.linting.enabled": true,
     "editor.formatOnSave": true,
     "editor.defaultFormatter": null,
     "cSpell.ignoreWords": [
@@ -17,6 +15,7 @@
         "fromlist",
         "hyperparameters",
         "hyperparams",
+        "imagenet",
         "ipython",
         "itwinai",
         "Lockfiles",
@@ -56,10 +55,6 @@
     "[python]": {
         "editor.defaultFormatter": "ms-python.autopep8"
     },
-    "python.formatting.provider": "none",
-    "[markdown]": {
-        "editor.formatOnSave": false
-    },
     "python.testing.pytestArgs": [
         "tests"
     ],

Makefile

@@ -1,14 +1,21 @@
 # Install PyTorch env (GPU support)
 torch-gpu: env-files/torch/pytorch-env-gpu.yml
 	micromamba env create -p ./.venv-pytorch --file env-files/torch/pytorch-env-gpu.yml -y
-	micromamba run -p ./.venv-pytorch python -m pip install -e .[dev]
+	micromamba run -p ./.venv-pytorch python -m pip install -e .[distributed,dev]
+
+# Install PyTorch env (GPU support) on Juelich Super Computer (tested on HDFML system)
+torch-gpu-jsc: env-files/torch/createEnvJSC.sh
+	sh env-files/torch/createEnvJSC.sh
+
+# Install Tensorflow env (GPU support) on Juelich Super Computer (tested on HDFML system)
+tf-gpu-jsc: env-files/tensorflow/createEnvJSCTF.sh
+	sh env-files/tensorflow/createEnvJSCTF.sh
 
 # Install PyTorch env (CPU only)
 torch-cpu: env-files/torch/pytorch-env-cpu.yml
 	micromamba env create -p ./.venv-pytorch --file env-files/torch/pytorch-env-cpu.yml -y
 	micromamba run -p ./.venv-pytorch python -m pip install -e .[dev]
 
-
 # Install TensorFlow 2.13. Creates ./.venv-tf folder.
 # Ref: https://www.tensorflow.org/install/pip#step-by-step_instructions
 tf-2.13: env-files/tensorflow/tensorflow-2.13.yml

README.md

@@ -4,12 +4,9 @@
 [![GitHub Super-Linter](https://github.com/interTwin-eu/T6.5-AI-and-ML/actions/workflows/check-links.yml/badge.svg)](https://github.com/marketplace/actions/markdown-link-check)
  [![SQAaaS source code](https://github.com/EOSC-synergy/itwinai.assess.sqaaas/raw/dev/.badge/status_shields.svg)](https://sqaaas.eosc-synergy.eu/#/full-assessment/report/https://raw.githubusercontent.com/eosc-synergy/itwinai.assess.sqaaas/dev/.report/assessment_output.json)
 
-See the latest version of our [docs](https://intertwin-eu.github.io/T6.5-AI-and-ML/)
+See the latest version of our [docs](https://intertwin-eu.github.io/itwinai/)
 for a quick overview of this platform for advanced AI/ML workflows in digital twin applications.
 
-If you want to integrate a new use case, you can follow this
-[step-by-step guide](https://intertwin-eu.github.io/T6.5-AI-and-ML/docs/How-to-use-this-software.html).
-
 ## Installation
 
 Requirements:
@@ -21,7 +18,7 @@ Requirements:
 To manage Conda environments we use micromamba, a light weight version of conda.
 
 It is suggested to refer to the
-[Manual installation guide](https://mamba.readthedocs.io/en/latest/micromamba-installation.html#umamba-install).
+[Manual installation guide](https://mamba.readthedocs.io/en/latest/installation/micromamba-installation.html#manual-installation).
 
 Consider that Micromamba can eat a lot of space when building environments because packages are cached on
 the local filesystem after being downloaded. To clear cache you can use `micromamba clean -a`.
@@ -44,12 +41,12 @@ MAMBA_ROOT_PREFIX='my-mamba-root'
 echo 'PATH="$(dirname $MAMBA_EXE):$PATH"' >> ~/.bashrc
 ```
 
-**Reference**: [Micromamba installation guide](https://mamba.readthedocs.io/en/latest/installation.html#micromamba).
+**Reference**: [Micromamba installation guide](https://mamba.readthedocs.io/en/latest/installation/micromamba-installation.html).
 
 ### Documentation folder
 
 Documentation for this repository is maintained under `./docs` location.
-If you are using code from a previous release, you can build the docs webpage
+If you are using code from a previous release, you can build the docs web page
 locally using [these instructions](docs/README#building-and-previewing-your-site-locally).
 
 ## Environment setup
@@ -99,7 +96,35 @@ pip install -e .[dev]
 
 #### Test with `pytest`
 
-To run tests on itwinai package:
+Do this only if you are a developer wanting to test your code with pytest.
+
+First, you need to create virtual environments both for torch and tensorflow.
+For instance, you can use:
+
+```bash
+make torch-cpu
+make make tf-2.13-cpu
+```
+
+To select the name of the torch and tf environments you can set the following
+environment variables, which allow to run the tests in environments with
+custom names which are different from `.venv-pytorch` and `.venv-tf`.
+
+```bash
+export TORCH_ENV="my_torch_env"
+export TF_ENV="my_tf_env"
+```
+
+Functional tests (marked with `pytest.mark.functional`) will be executed under
+`/tmp/pytest` location to guarantee they are run in a clean environment.
+
+To run functional tests use:
+
+```bash
+pytest -v tests/ -m "functional"
+```
+
+To run all tests on itwinai package:
 
 ```bash
 # Activate env