feat(task_engine): implement a new declarative task engine for dioptra

Adds the new task engine for Dioptra that replaces Prefect and allows users to create experiments
using a declarative YAML file. The structure of the YAML files is validated using a JSON-Schema
schema. In addition, the task engine includes a type validation module that checks if the task
plugin inputs and outputs are consistent throughout the task graph set up in the YAML file.
Two commandline scripts are added for running the task engine (`run-experiment`) and validating
the YAML files (`validate-experiment`).

This update introduces unit tests for this module and adds documentation to the Dioptra User Guide
that explains how the declarative task engine works. The project lockfiles have also been
regenerated.

Closes #101

MANIFEST.in

@@ -1,4 +1,5 @@
 include CHANGELOG.md tox.ini
+recursive-include src/dioptra *.json
 
 graft docs
 graft tests

docker/requirements/linux-aarch64-py3.9-mlflow-tracking-requirements.txt

@@ -2,23 +2,23 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --config=pyproject.toml --output-file=venvs/linux-aarch64-py3.9-mlflow-tracking-requirements.txt --resolver=backtracking docker/pip-tools/mlflow-tracking-requirements.in
+#    pip-compile --config=pyproject.toml --output-file=venvs/linux-aarch64-py3.9-mlflow-tracking-requirements.txt docker/pip-tools/mlflow-tracking-requirements.in
 #
 alembic==1.11.1
     # via mlflow
 blinker==1.6.2
     # via flask
-boto3==1.28.2
+boto3==1.28.16
     # via -r docker/pip-tools/mlflow-tracking-requirements.in
-botocore==1.31.2
+botocore==1.31.16
     # via
     #   boto3
     #   s3transfer
-certifi==2023.5.7
+certifi==2023.7.22
     # via requests
 charset-normalizer==3.2.0
     # via requests
-click==8.1.5
+click==8.1.6
     # via
     #   databricks-cli
     #   flask
@@ -66,7 +66,7 @@ markupsafe==2.1.3
     #   werkzeug
 mlflow==1.30.1
     # via -r docker/pip-tools/mlflow-tracking-requirements.in
-numpy==1.25.1
+numpy==1.25.2
     # via
     #   mlflow
     #   pandas
@@ -87,9 +87,9 @@ protobuf==4.23.4
     # via mlflow
 psycopg2-binary==2.9.6
     # via -r docker/pip-tools/mlflow-tracking-requirements.in
-pyjwt==2.7.0
+pyjwt==2.8.0
     # via databricks-cli
-pyparsing==3.1.0
+pyparsing==3.1.1
     # via packaging
 python-dateutil==2.8.2
     # via
@@ -99,7 +99,7 @@ pytz==2022.7.1
     # via
     #   mlflow
     #   pandas
-pyyaml==6.0
+pyyaml==6.0.1
     # via mlflow
 querystring-parser==1.2.4
     # via mlflow
@@ -141,7 +141,7 @@ websocket-client==1.6.1
     # via docker
 werkzeug==2.3.6
     # via flask
-zipp==3.16.1
+zipp==3.16.2
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:

docker/requirements/linux-aarch64-py3.9-pytorch-cpu-requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --extra=sdk --extra=taskplugins --output-file=venvs/linux-aarch64-py3.9-pytorch-cpu-requirements.txt --resolver=backtracking docker/pip-tools/worker-requirements.in pyproject.toml requirements-dev-pytorch.in
+#    pip-compile --extra=sdk --extra=taskplugins --output-file=venvs/linux-aarch64-py3.9-pytorch-cpu-requirements.txt docker/pip-tools/worker-requirements.in pyproject.toml requirements-dev-pytorch.in
 #
 --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
 --find-links https://dl.fbaipublicfiles.com/detectron2/wheels/cpu/torch1.10/index.html
@@ -11,7 +11,7 @@ absl-py==1.4.0
     # via tensorboard
 adversarial-robustness-toolbox==1.15.0
     # via dioptra (pyproject.toml)
-aiohttp==3.8.4
+aiohttp==3.8.5
     # via fsspec
 aiosignal==1.3.1
     # via aiohttp
@@ -31,25 +31,25 @@ attrs==23.1.0
     #   aiohttp
     #   jsonschema
     #   referencing
-boto3==1.28.2
+boto3==1.28.16
     # via dioptra (pyproject.toml)
-botocore==1.31.2
+botocore==1.31.16
     # via
     #   boto3
     #   s3transfer
 box2d-py==2.3.5
     # via gym
 cachetools==5.3.1
     # via google-auth
-certifi==2023.5.7
+certifi==2023.7.22
     # via requests
 cffi==1.15.1
     # via cryptography
 charset-normalizer==3.2.0
     # via
     #   aiohttp
     #   requests
-click==8.1.5
+click==8.1.6
     # via
     #   dask
     #   databricks-cli
@@ -76,17 +76,17 @@ cryptography==3.4.8
     # via dioptra (pyproject.toml)
 cycler==0.11.0
     # via matplotlib
-dask==2023.7.0
+dask==2023.7.1
     # via
     #   distributed
     #   prefect
 databricks-cli==0.17.7
     # via mlflow
 decorator==4.4.2
     # via moviepy
-distributed==2023.7.0
+distributed==2023.7.1
     # via prefect
-dnspython==2.3.0
+dnspython==2.4.1
     # via email-validator
 docker==6.1.3
     # via
@@ -127,7 +127,7 @@ flask-sqlalchemy==2.5.1
     #   flask-migrate
 flask-wtf==1.1.1
     # via dioptra (pyproject.toml)
-fonttools==4.41.0
+fonttools==4.41.1
     # via matplotlib
 frozenlist==1.4.0
     # via
@@ -149,7 +149,7 @@ google-auth-oauthlib==1.0.0
     # via tensorboard
 greenlet==2.0.2
     # via sqlalchemy
-grpcio==1.56.0
+grpcio==1.56.2
     # via tensorboard
 gunicorn==20.1.0
     # via mlflow
@@ -182,7 +182,7 @@ importlib-resources==6.0.0
     # via
     #   matplotlib
     #   prefect
-injector==0.20.1
+injector==0.21.0
     # via
     #   dioptra (pyproject.toml)
     #   flask-injector
@@ -200,9 +200,11 @@ jmespath==1.0.1
     #   botocore
 joblib==1.3.1
     # via scikit-learn
-jsonschema==4.18.2
-    # via flask-restx
-jsonschema-specifications==2023.6.1
+jsonschema==4.18.4
+    # via
+    #   dioptra (pyproject.toml)
+    #   flask-restx
+jsonschema-specifications==2023.7.1
     # via jsonschema
 kiwisolver==1.4.4
     # via matplotlib
@@ -220,7 +222,7 @@ lz4==4.3.2
     # via gym
 mako==1.2.4
     # via alembic
-markdown==3.4.3
+markdown==3.4.4
     # via tensorboard
 markdown-it-py==3.0.0
     # via rich
@@ -229,7 +231,7 @@ markupsafe==2.1.3
     #   jinja2
     #   mako
     #   wtforms
-marshmallow==3.19.0
+marshmallow==3.20.1
     # via
     #   dioptra (pyproject.toml)
     #   flask-accepts
@@ -263,7 +265,7 @@ mypy-extensions==1.0.0
     # via prefect
 networkx==3.1
     # via scikit-image
-numpy==1.25.1
+numpy==1.25.2
     # via
     #   adversarial-robustness-toolbox
     #   cma
@@ -361,7 +363,7 @@ pygame==2.1.0
     # via gym
 pygments==2.15.1
     # via rich
-pyjwt==2.7.0
+pyjwt==2.8.0
     # via databricks-cli
 pyparsing==3.0.9
     # via
@@ -394,9 +396,10 @@ pytzdata==2020.1
     # via pendulum
 pywavelets==1.4.1
     # via scikit-image
-pyyaml==6.0
+pyyaml==6.0.1
     # via
     #   dask
+    #   dioptra (pyproject.toml)
     #   distributed
     #   mlflow
     #   prefect
@@ -407,7 +410,7 @@ redis==4.6.0
     # via
     #   dioptra (pyproject.toml)
     #   rq
-referencing==0.29.1
+referencing==0.30.0
     # via
     #   jsonschema
     #   jsonschema-specifications
@@ -424,9 +427,9 @@ requests==2.31.0
     #   tensorboard
 requests-oauthlib==1.3.1
     # via google-auth-oauthlib
-rich==13.4.2
+rich==13.5.2
     # via -r docker/pip-tools/worker-requirements.in
-rpds-py==0.8.10
+rpds-py==0.9.2
     # via
     #   jsonschema
     #   referencing
@@ -492,7 +495,7 @@ text-unidecode==1.3
     # via python-slugify
 threadpoolctl==3.2.0
     # via scikit-learn
-tifffile==2023.7.10
+tifffile==2023.7.18
     # via scikit-image
 toml==0.10.2
     # via prefect
@@ -547,7 +550,7 @@ werkzeug==2.1.2
     #   flask-accepts
     #   flask-restx
     #   tensorboard
-wheel==0.40.0
+wheel==0.41.0
     # via tensorboard
 wtforms[email]==3.0.1
     # via
@@ -557,7 +560,7 @@ yarl==1.9.2
     # via aiohttp
 zict==3.0.0
     # via distributed
-zipp==3.16.1
+zipp==3.16.2
     # via
     #   importlib-metadata
     #   importlib-resources