From f53aa37a34dc37026d430e71b5e0d1b871bc5ac1 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Tue, 2 Jul 2024 07:09:12 -0400 Subject: [PATCH] move py.typed to correct places (#403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * move py.typed to correct places https://peps.python.org/pep-0561/ says 'For namespace packages (see PEP 420), the py.typed file should be in the submodules of the namespace, to avoid conflicts and for clarity.'. Previously, when I added the py.typed file to this project, https://github.com/databricks/databricks-sql-python/pull/382 , I was unaware this was a namespace package (although, curiously, it seems I had done it right initially and then changed to the wrong way). As PEP 561 warns us, this does create conflicts; other libraries in the databricks namespace package (such as, in my case, databricks-vectorsearch) are then treated as though they are typed, which they are not. This commit moves the py.typed file to the correct places, the submodule folders, fixing that problem. Signed-off-by: wyattscarpenter * change target of mypy to src/databricks instead of src. I think this might fix the CI code-quality checks failure, but unfortunately I can't replicate that failure locally and the error message is unhelpful Signed-off-by: wyattscarpenter * Possible workaround for bad error message 'error: --install-types failed (no mypy cache directory)'; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153 Signed-off-by: wyattscarpenter * fix invalid yaml syntax Signed-off-by: wyattscarpenter * Best fix (#3) Fixes the problem by cding and supplying a flag to mypy (that mypy needs this flag is seemingly fixed/changed in later versions of mypy; but that's another pr altogether...). Also fixes a type error that was somehow in the arguments of the program (?!) (I guess this is because you guys are still using implicit optional) --------- Signed-off-by: wyattscarpenter * return the old result_links default (#5) Return the old result_links default, make the type optional, & I'm pretty sure the original problem is that add_file_links can't take a None, so these statements should be in the body of the if-statement that ensures it is not None Signed-off-by: wyattscarpenter * Update src/databricks/sql/utils.py "self.download_manager is unconditionally used later, so must be created. Looks this part of code is totally not covered with tests 🤔" Co-authored-by: Levko Kravets Signed-off-by: wyattscarpenter --------- Signed-off-by: wyattscarpenter Co-authored-by: Levko Kravets --- .github/workflows/code-quality-checks.yml | 5 ++++- src/databricks/{ => sql}/py.typed | 0 src/databricks/sql/utils.py | 7 +++---- src/databricks/sqlalchemy/py.typed | 0 4 files changed, 7 insertions(+), 5 deletions(-) rename src/databricks/{ => sql}/py.typed (100%) create mode 100755 src/databricks/sqlalchemy/py.typed diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 03c3991d..2dd55b77 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -160,4 +160,7 @@ jobs: # mypy the code #---------------------------------------------- - name: Mypy - run: poetry run mypy --install-types --non-interactive src + run: | + cd src # Need to be in the actual databricks/ folder or mypy does the wrong thing. + mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153 + poetry run mypy --config-file ../pyproject.toml --install-types --non-interactive --namespace-packages databricks diff --git a/src/databricks/py.typed b/src/databricks/sql/py.typed similarity index 100% rename from src/databricks/py.typed rename to src/databricks/sql/py.typed diff --git a/src/databricks/sql/utils.py b/src/databricks/sql/utils.py index 9f21a8eb..6063eca1 100644 --- a/src/databricks/sql/utils.py +++ b/src/databricks/sql/utils.py @@ -7,7 +7,7 @@ from collections.abc import Iterable from decimal import Decimal from enum import Enum -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union import re import lz4.frame @@ -134,7 +134,7 @@ def __init__( schema_bytes, max_download_threads: int, start_row_offset: int = 0, - result_links: List[TSparkArrowResultLink] = None, + result_links: Optional[List[TSparkArrowResultLink]] = None, lz4_compressed: bool = True, description: List[List[Any]] = None, ): @@ -168,11 +168,10 @@ def __init__( result_link.startRowOffset, result_link.rowCount ) ) - self.download_manager = ResultFileDownloadManager( self.max_download_threads, self.lz4_compressed ) - self.download_manager.add_file_links(result_links) + self.download_manager.add_file_links(result_links or []) self.table = self._create_next_table() self.table_row_index = 0 diff --git a/src/databricks/sqlalchemy/py.typed b/src/databricks/sqlalchemy/py.typed new file mode 100755 index 00000000..e69de29b