Pydantic 2.4 support (#59)

* Loosen dependencies, add config for pylint and mypy, add docs dependencies section, disable apache-airflow (needs to be manually side loaded as poetry is unable to)

* managed to get apache-airflow to play happy; ensure poetry is updated to latest version!

---------

Co-authored-by: Danny Meijer <[email protected]>

brickflow/cli/__init__.py

@@ -122,8 +122,8 @@ def deploy_only_workflows(
             ] = ",".join(value)
 
     def set_up_bundle_for_workflow_dir(
-        ctx: click.Context, param: str, value: Any  # noqa
-    ) -> None:
+        ctx: click.Context, param: str, value: Any  # pylint: disable=unused-argument
+    ) -> None:  # noqa
         if value is not None:
             return value
 

brickflow/engine/workflow.py

@@ -239,8 +239,7 @@ def parents(self, node: str) -> Iterator:
     def check_no_active_task(self) -> None:
         if self.active_task is not None:
             raise AnotherActiveTaskError(
-                "You are calling another active task in another task. "
-                "Please abstract the code more."
+                "You are calling another active task in another task. Please abstract the code more."
             )
 
     @wraps_keyerror(TaskNotFoundError, "Unable to find task: ")

brickflow_plugins/databricks/workflow_dependency_sensor.py

@@ -1,12 +1,13 @@
-from datetime import datetime, timedelta
 import functools
 import logging
 import os
-from pydantic import SecretStr
+from datetime import datetime, timedelta
+from typing import Union
+
 import requests
-from requests.adapters import HTTPAdapter
 import time
-from typing import Union
+from pydantic import SecretStr
+from requests.adapters import HTTPAdapter
 
 from brickflow.context import ctx
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "brickflows"
-version = "0.0.1-pre.36+4a13620"
+version = "0.11.0a0"
 description = "Deploy scalable workflows to databricks using python"
 authors = ["Ashok Singamaneni, Sriharsha Tikkireddy"]
 readme = "README.md"
@@ -20,7 +20,7 @@ exclude = ["sample_workflows", "tests"]
 
 [tool.black]
 line-length = 88
-target-version = ['py38']
+target-version = ['py38', 'py39', 'py310']
 include = '\.pyi?$'
 extend-exclude = '''
 /(
@@ -34,45 +34,47 @@ extend-exclude = '''
 '''
 
 [tool.poetry.dependencies]
-python = "^3.8"
-python-decouple = "3.8"
-networkx = "3.1"
-pendulum = "2.1.2"
+python = ">=3.8,<3.11"  # pyspark <3.5 does not play happy with python 3.11
 Jinja2 = "3.1.2"
 click = "^8.1.3"
+databricks-sdk = ">=0.1.8 <1.0.0"
+networkx = "3.1"
+pendulum = "2.1.2"
 pluggy = "^1.0.0"
+pydantic = ">=1.10.0 <3.0.0"
+python-decouple = "3.8"
+pyyaml = "^6.0"
 requests = ">=2.28.2 <3.0.0"
-pydantic = "^1.10.9"
-databricks-sdk = ">=0.1.8 <1.0.0"
-pyyaml = "^6.0.0"
 # cerberus-python-client = {version = "~2.5.4",  optional = true } # Users might have to manually install cerberus-python-client if required
 
 [tool.poetry.scripts]
 bf = "brickflow.cli:cli"
 brickflow = "brickflow.cli:cli"
 
 [tool.poetry.group.dev.dependencies]
-types-requests = ">=2.28.11.16 <3.0.0.0" # only for development purposes no need to make installation req
-types-PyYAML = "^6.0.0" # only for development purposes no need to make installation req
-pytest = "7.3.1"
-pytest-mock = "3.10.0"
-coverage = "7.2.5"
-black = "23.3.0"
-prospector = "1.10.0"
+black = "^23.3.0"
+coverage = "^7.2.5"
+datamodel-code-generator = "^0.22.1"
 deepdiff = "^6.3.0"
-mypy = "1.3.0"
+mypy = "^1.3.0"
+pre-commit = "^3.3.1"
+prospector = "*"
+py4j = "^0.10.9.7"
+pytest = ">=7.3.1"
+pytest-mock = "^3.10.0"
+types-PyYAML = "*" # only for development purposes no need to make installation req
+types-requests = ">=2.28.11.16 <3.0.0.0" # only for development purposes no need to make installation req
+apache-airflow = "^2.7.3"
+
+[tool.poetry.group.docs.dependencies]
+mdx-include = ">=1.4.1,<2.0.0"
+mike = "^1.1.2"
 mkdocs = "1.4.3"
+mkdocs-autorefs = "0.4.1"
 mkdocs-click = "0.8.0"
-mkdocs-material = "9.1.12"
-mdx-include = ">=1.4.1,<2.0.0"
 mkdocs-markdownextradata-plugin = "0.2.5"
+mkdocs-material = "9.1.12"
 mkdocstrings = {version = "0.20.0", extras=["python"]}
-mkdocs-autorefs = "0.4.1"
-mike = "^1.1.2"
-py4j = "^0.10.9.7"
-pre-commit = "3.3.1"
-datamodel-code-generator = "^0.19.0"
-apache-airflow = ">=2.6.3 <2.7.0"
 
 [build-system]
 requires = ["poetry-core", "poetry-dynamic-versioning"]
@@ -111,4 +113,18 @@ files = [
     "brickflow/resolver/*.py",
     "brickflow/codegen/*.py",
 ]
-follow_imports = "skip"
+follow_imports = "skip"
+
+[tool.pylint]
+    [tool.pylint."MASTER"]
+    fail-under = 9.0
+
+    [tool.pylint.'FORMAT']
+    max-line-length = 120
+    # Regexp for a line that is allowed to be longer than the limit.
+    ignore-long-lines = "^\\s*(# )?<?https?://\\S+>?$"
+
+    [tool.pylint.'MESSAGES CONTROL']
+    disable = """
+    django-not-configured,
+    """

tests/codegen/test_databricks_bundle.py

@@ -398,7 +398,8 @@ def test_mutators(self):
         ]
         databricks_fake_client.jobs.list.assert_called_once_with(name=job_name)
         databricks_fake_client.current_user.me.assert_called_once()
+        # pylint: disable=unsubscriptable-object
+        jobs: dict = resource.jobs
         assert (
-            resource.jobs is not None
-            and resource.jobs[job_name].name == f"{fake_user_name}_{job_name}"  # noqa
+            jobs is not None and jobs[job_name].name == f"{fake_user_name}_{job_name}"
         )

tests/engine/test_engine.py

@@ -11,6 +11,6 @@ def test_get_current_commit(self, mocker):
         mocker.patch("subprocess.check_output")
         subprocess.check_output.return_value = branch.encode("utf-8")
         assert get_current_commit() == branch
-        subprocess.check_output.assert_called_once_with(  # noqa
+        subprocess.check_output.assert_called_once_with(
             ['git log -n 1 --pretty=format:"%H"'], shell=True
-        )
+        )  # noqa