Pydantic vendoring (#137)

* Pydantic vendoring
* Updating dependency versions
* fixing issues related to mypy
* Poetry updates
* revering the changes to the public docs
* Updated readme file
* Changelog updates
* pyproject updates
* adding latest version of the dask package
* fixing issues related to poetry lock file
* poetry.lock updates
* Poetry updates on github actions
* adding init file
* dask version mismatch
* dask version upgrades

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
       - name: Install dependencies
         run: |
           python3 -m pip install --upgrade pip poetry==1.2.2
-          poetry install --extras "settings"
+          poetry install
 
       - name: Linting
         run: |
@@ -33,7 +33,7 @@ jobs:
       - name: Install dependencies
         run: |
           python3 -m pip install --upgrade pip poetry==1.2.2
-          poetry install --extras "settings"
+          poetry install
 
       - name: Run tests
         env:

.github/workflows/publish.yaml

@@ -23,7 +23,7 @@ jobs:
         run: |
           python3 -m pip install --upgrade pip poetry==1.2.2
           poetry config virtualenvs.create false
-          poetry install --extras "settings"
+          poetry install
 
       - name: Linting
         run: |
@@ -41,7 +41,7 @@ jobs:
           poetry run pytest tests/ -v --cov-report xml:coverage.xml --cov
           poetry run coverage xml
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
         with:
           file: ./coverage.xml
 

.pre-commit-config.yaml

@@ -25,6 +25,7 @@ repos:
     rev: 6.1.0
     hooks:
       - id: flake8
+        exclude: ^(vendor/)
         additional_dependencies: ["flake8-pyproject"]
   - repo: https://github.com/PyCQA/pydocstyle
     rev: 6.3.0

CHANGELOG.md

@@ -170,3 +170,17 @@ Changes are grouped as follows
 ### Fixed
 - Dependency updates for source code.
 - Updated Github actions to use the latest versions.
+
+## [0.3.4] - 2024-02-01
+
+### Fixed
+- Dependency updates for source code.
+- Added support for pydantic v1
+
+## [0.3.5] - 2024-02-01
+
+### Fixed
+- Dependency updates for source code.
+- Updated Github actions to use the latest versions of Code Cov.
+- Removed the multi-version support for pydantic.
+- Introduced vendoring for pydantic v1.

README.md

@@ -50,5 +50,10 @@ Important steps to follow when working with CDF Files using the `fsspec` support
 
 Refer [cdffs.readthedocs.io](https://cdffs.readthedocs.io) for more details.
 
+## Vendoring
+
+`cdffs` uses pydandic.v1 package using vendoring. It was mainly introduced to overcome version conflicts
+related to the Cognite Notebooks.
+
 ## Contributing
 Want to contribute? Check out [CONTRIBUTING](CONTRIBUTING.md).

cognite/cdffs/__init__.py

@@ -4,7 +4,7 @@
 
 from .spec import CdfFileSystem
 
-__version__ = "0.3.4"
+__version__ = "0.3.5"
 __all__ = ["CdfFileSystem"]
 
 fsspec.register_implementation(CdfFileSystem.protocol, CdfFileSystem)

cognite/cdffs/credentials.py

@@ -6,15 +6,9 @@
 from cognite.client import ClientConfig, CogniteClient
 from cognite.client.credentials import OAuthClientCredentials, Token
 
-try:
-    from pydantic import BaseSettings, ConfigDict, SecretStr, validator
-
-    is_pydantic_v2 = False
-except ImportError:
-    from pydantic import ConfigDict, SecretStr, field_validator
-    from pydantic_settings import BaseSettings
-
-    is_pydantic_v2 = True
+from vendor.pydantic.class_validators import validator
+from vendor.pydantic.env_settings import BaseSettings
+from vendor.pydantic.types import SecretStr
 
 
 def validate_scopes(cls: Any, value: str) -> Optional[List]:
@@ -27,7 +21,14 @@ def validate_scopes(cls: Any, value: str) -> Optional[List]:
 class FsConfig(BaseSettings):
     """Base config to parse environment variables."""
 
-    model_config: ConfigDict = ConfigDict(env_nested_delimiter="__", env_file=".env", env_file_encoding="utf-8")
+    class Config:
+        """Global config for Base Settings."""
+
+        env_nested_delimiter = "__"
+        env_file = ".env"
+        env_file_encoding = "utf-8"
+
+    # model_config: ConfigDict = ConfigDict(env_nested_delimiter="__", env_file=".env", env_file_encoding="utf-8")
 
 
 class FsCredentials(FsConfig, ABC):
@@ -79,10 +80,7 @@ class FsOAuthCredentials(FsCredentials, FsConfig):
     scopes: Optional[Union[str, List]] = None
 
     # Validator
-    if is_pydantic_v2:
-        _scopes = field_validator("scopes")(validate_scopes)
-    else:
-        _scopes = validator("scopes")(validate_scopes)
+    _scopes = validator("scopes")(validate_scopes)
 
     def get_credentials(self) -> OAuthClientCredentials:
         """Construct credentials based on environment variables.
@@ -122,23 +120,13 @@ def get_credentials(self) -> Token:
 
 def get_connection_config(env_file: str) -> CogniteClient:
     """Construct Cognite Client from environment variables."""
-    credentials = FsOAuthCredentials(_env_file=env_file)
+    credentials = FsOAuthCredentials(_env_file=env_file)  # type:ignore
     connection_config = None
-
-    if is_pydantic_v2:
-        credentials_dump = credentials.model_dump().items()
-    else:
-        credentials_dump = credentials.dict().items()
-
-    if all(value is not None for _, value in credentials_dump):
+    if all(value is not None for _, value in credentials.dict().items()):
         connection_config = credentials.get_client_config()
     else:
-        token = FsToken(_env_file=env_file)
-        if is_pydantic_v2:
-            token_dump = token.model_dump().items()
-        else:
-            token_dump = token.dict().items()
-        if all(value is not None for _, value in token_dump):
+        token = FsToken(_env_file=env_file)  # type:ignore
+        if all(value is not None for _, value in token.dict().items()):
             connection_config = token.get_client_config()
 
     return connection_config

docs/source/index.rst

@@ -9,13 +9,7 @@ the list of all supported/compatible python packages.
 
 Installation
 ^^^^^^^^^^^^
-To install this package(Recommended):
-
-  .. code-block:: bash
-
-   pip install cognite-cdffs[settings]
-
-If you need `cdffs` to be compatible with pydantic-v1, choose to install the expected pydantic-v1 (`^1.10.7`) and use,
+To install this package:
 
   .. code-block:: bash
 

mypy.ini

@@ -9,3 +9,7 @@ ignore_missing_imports = false
 disallow_untyped_defs = true
 follow_imports = normal
 show_error_codes = true
+
+
+[mypy-vendor.pydantic.*]
+ignore_errors = true