Merge pull request #342 from jtcohen6/jerco/migrate-dbt-common-adapte…

…rs-interfaces

Migrate to dbt-common + dbt-adapters

dbt/adapters/duckdb/connections.py

@@ -1,18 +1,21 @@
 import atexit
 import threading
 from contextlib import contextmanager
+from multiprocessing.context import SpawnContext
 from typing import Optional
 from typing import Tuple
 from typing import TYPE_CHECKING
 
 import dbt.exceptions
 from . import environments
+from dbt.adapters.contracts.connection import AdapterRequiredConfig
+from dbt.adapters.contracts.connection import AdapterResponse
+from dbt.adapters.contracts.connection import Connection
+from dbt.adapters.contracts.connection import ConnectionState
+from dbt.adapters.events.logging import AdapterLogger
 from dbt.adapters.sql import SQLConnectionManager
-from dbt.contracts.connection import AdapterRequiredConfig
-from dbt.contracts.connection import AdapterResponse
-from dbt.contracts.connection import Connection
-from dbt.contracts.connection import ConnectionState
-from dbt.logger import GLOBAL_LOGGER as logger
+
+logger = AdapterLogger("DuckDB")
 
 if TYPE_CHECKING:
     import agate
@@ -23,9 +26,9 @@ class DuckDBConnectionManager(SQLConnectionManager):
     _LOCK = threading.RLock()
     _ENV = None
 
-    def __init__(self, profile: AdapterRequiredConfig):
-        super().__init__(profile)
-        self.disable_transactions = profile.credentials.disable_transactions  # type: ignore
+    def __init__(self, config: AdapterRequiredConfig, mp_context: SpawnContext) -> None:
+        super().__init__(config, mp_context)
+        self.disable_transactions = config.credentials.disable_transactions  # type: ignore
 
     @classmethod
     def env(cls) -> environments.Environment:
@@ -52,7 +55,7 @@ def open(cls, connection: Connection) -> Connection:
                 logger.debug("Got an error when attempting to connect to DuckDB: '{}'".format(e))
                 connection.handle = None
                 connection.state = ConnectionState.FAIL
-                raise dbt.exceptions.FailedToConnectError(str(e))
+                raise dbt.adapters.exceptions.FailedToConnectError(str(e))
 
             return connection
 

dbt/adapters/duckdb/credentials.py

@@ -10,9 +10,10 @@
 from typing import Tuple
 from urllib.parse import urlparse
 
-import dbt.exceptions
-from dbt.adapters.base import Credentials
-from dbt.dataclass_schema import dbtClassMixin
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.exceptions import DbtRuntimeError
+
+from dbt.adapters.contracts.connection import Credentials
 
 
 @dataclass
@@ -187,12 +188,12 @@ def __pre_deserialize__(cls, data: Dict[Any, Any]) -> Dict[Any, Any]:
             data["database"] = path_db
         elif path_db and data["database"] != path_db:
             if not data.get("remote"):
-                raise dbt.exceptions.DbtRuntimeError(
+                raise DbtRuntimeError(
                     "Inconsistency detected between 'path' and 'database' fields in profile; "
                     f"the 'database' property must be set to '{path_db}' to match the 'path'"
                 )
         elif not path_db:
-            raise dbt.exceptions.DbtRuntimeError(
+            raise DbtRuntimeError(
                 "Unable to determine target database name from 'path' field in profile"
             )
         return data

dbt/adapters/duckdb/environments/__init__.py

@@ -9,14 +9,14 @@
 from typing import Optional
 
 import duckdb
+from dbt_common.exceptions import DbtRuntimeError
 
 from ..credentials import DuckDBCredentials
 from ..plugins import BasePlugin
 from ..utils import SourceConfig
 from ..utils import TargetConfig
-from dbt.contracts.connection import AdapterResponse
-from dbt.contracts.connection import Connection
-from dbt.exceptions import DbtRuntimeError
+from dbt.adapters.contracts.connection import AdapterResponse
+from dbt.adapters.contracts.connection import Connection
 
 
 def _ensure_event_loop():

dbt/adapters/duckdb/environments/buenavista.py

@@ -5,8 +5,8 @@
 from . import Environment
 from .. import credentials
 from .. import utils
-from dbt.contracts.connection import AdapterResponse
-from dbt.contracts.connection import Connection
+from dbt.adapters.contracts.connection import AdapterResponse
+from dbt.adapters.contracts.connection import Connection
 
 
 class BVEnvironment(Environment):

dbt/adapters/duckdb/environments/local.py

@@ -1,11 +1,12 @@
 import threading
 
+from dbt_common.exceptions import DbtRuntimeError
+
 from . import Environment
 from .. import credentials
 from .. import utils
-from dbt.contracts.connection import AdapterResponse
-from dbt.contracts.connection import Connection
-from dbt.exceptions import DbtRuntimeError
+from dbt.adapters.contracts.connection import AdapterResponse
+from dbt.adapters.contracts.connection import Connection
 
 
 class DuckDBCursorWrapper:

dbt/adapters/duckdb/impl.py

@@ -5,24 +5,25 @@
 from typing import Sequence
 from typing import TYPE_CHECKING
 
+from dbt_common.contracts.constraints import ColumnLevelConstraint
+from dbt_common.contracts.constraints import ConstraintType
+from dbt_common.exceptions import DbtInternalError
+from dbt_common.exceptions import DbtRuntimeError
+
 from dbt.adapters.base import BaseRelation
 from dbt.adapters.base.column import Column as BaseColumn
 from dbt.adapters.base.impl import ConstraintSupport
 from dbt.adapters.base.meta import available
+from dbt.adapters.contracts.connection import AdapterResponse
+from dbt.adapters.contracts.relation import Path
+from dbt.adapters.contracts.relation import RelationType
 from dbt.adapters.duckdb.column import DuckDBColumn
 from dbt.adapters.duckdb.connections import DuckDBConnectionManager
 from dbt.adapters.duckdb.relation import DuckDBRelation
 from dbt.adapters.duckdb.utils import TargetConfig
 from dbt.adapters.duckdb.utils import TargetLocation
 from dbt.adapters.sql import SQLAdapter
-from dbt.context.providers import RuntimeConfigObject
-from dbt.contracts.connection import AdapterResponse
-from dbt.contracts.graph.nodes import ColumnLevelConstraint
-from dbt.contracts.graph.nodes import ConstraintType
-from dbt.contracts.relation import Path
-from dbt.contracts.relation import RelationType
-from dbt.exceptions import DbtInternalError
-from dbt.exceptions import DbtRuntimeError
+
 
 TEMP_SCHEMA_NAME = "temp_schema_name"
 DEFAULT_TEMP_SCHEMA_NAME = "dbt_temp"
@@ -103,7 +104,7 @@ def store_relation(
         column_list: Sequence[BaseColumn],
         path: str,
         format: str,
-        config: RuntimeConfigObject,
+        config: Any,
     ) -> None:
         target_config = TargetConfig(
             relation=relation,

dbt/adapters/duckdb/plugins/__init__.py

@@ -4,12 +4,12 @@
 from typing import Dict
 from typing import Optional
 
+from dbt_common.dataclass_schema import dbtClassMixin
 from duckdb import DuckDBPyConnection
 
 from ..credentials import DuckDBCredentials
 from ..utils import SourceConfig
 from ..utils import TargetConfig
-from dbt.dataclass_schema import dbtClassMixin
 
 
 class PluginConfig(dbtClassMixin):

dbt/adapters/duckdb/relation.py

@@ -7,16 +7,30 @@
 from .connections import DuckDBConnectionManager
 from .utils import SourceConfig
 from dbt.adapters.base.relation import BaseRelation
-from dbt.adapters.base.relation import Self
-from dbt.contracts.graph.nodes import SourceDefinition
+from dbt.adapters.contracts.relation import HasQuoting
+from dbt.adapters.contracts.relation import RelationConfig
 
 
 @dataclass(frozen=True, eq=False, repr=False)
 class DuckDBRelation(BaseRelation):
     external: Optional[str] = None
 
     @classmethod
-    def create_from_source(cls: Type[Self], source: SourceDefinition, **kwargs: Any) -> Self:
+    def create_from(
+        cls: Type["DuckDBRelation"],
+        quoting: HasQuoting,
+        relation_config: RelationConfig,
+        **kwargs: Any,
+    ) -> "DuckDBRelation":
+        if relation_config.resource_type == "source":
+            return cls.create_from_source(quoting, relation_config, **kwargs)
+        else:
+            return super().create_from(quoting, relation_config, **kwargs)
+
+    @classmethod
+    def create_from_source(
+        cls: Type["DuckDBRelation"], quoting: HasQuoting, source: RelationConfig, **kwargs: Any
+    ) -> "DuckDBRelation":
         """
         This method creates a new DuckDBRelation instance from a source definition.
         It first checks if a 'plugin' is defined in the meta argument for the source or its parent configuration.
@@ -59,7 +73,7 @@ def create_from_source(cls: Type[Self], source: SourceDefinition, **kwargs: Any)
                 ext_location = f"'{ext_location}'"
             kwargs["external"] = ext_location
 
-        return super().create_from_source(source, **kwargs)  # type: ignore
+        return super().create_from(quoting, source, **kwargs)  # type: ignore
 
     def render(self) -> str:
         if self.external:

dbt/adapters/duckdb/utils.py

@@ -7,8 +7,9 @@
 
 from dbt.adapters.base.column import Column
 from dbt.adapters.base.relation import BaseRelation
-from dbt.context.providers import RuntimeConfigObject
-from dbt.contracts.graph.nodes import SourceDefinition
+from dbt.adapters.contracts.relation import RelationConfig
+# TODO
+# from dbt.context.providers import RuntimeConfigObject
 
 
 @dataclass
@@ -47,11 +48,11 @@ def as_dict(self) -> Dict[str, Any]:
         return base
 
     @classmethod
-    def create_from_source(cls, source: SourceDefinition) -> "SourceConfig":
-        meta = source.source_meta.copy()
-        meta.update(source.meta)
+    def create_from_source(cls, source: RelationConfig) -> "SourceConfig":
+        meta = source.meta.copy()
         # Use the config properties as well if they are present
-        meta.update(source.config._extra)
+        config_properties = source.config.extra if source.config else {}
+        meta.update(config_properties)
         return SourceConfig(
             name=source.name,
             identifier=source.identifier,
@@ -75,7 +76,7 @@ def as_dict(self) -> Dict[str, Any]:
 class TargetConfig:
     relation: BaseRelation
     column_list: Sequence[Column]
-    config: RuntimeConfigObject
+    config: Any  # TODO
     location: Optional[TargetLocation] = None
 
     def as_dict(self) -> Dict[str, Any]:

dev-requirements.txt

@@ -1,8 +1,8 @@
-# install latest changes in dbt-core
+# install latest changes in dbt-core + dbt-tests-adapter
 # git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
-# git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter
+# git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter
 
-dbt-tests-adapter==1.7.14
+dbt-tests-adapter>=1.8.0b1
 
 boto3
 mypy-boto3-glue

setup.py

@@ -37,8 +37,11 @@ def _dbt_duckdb_version():
     packages=find_namespace_packages(include=["dbt", "dbt.*"]),
     include_package_data=True,
     install_requires=[
-        "dbt-core~=1.7.0",
+        "dbt-common>=1,<2",
+        "dbt-adapters>=1,<2",
         "duckdb>=0.7.0",
+        # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency
+        "dbt-core>=1.8.0b1",
     ],
     extras_require={"glue": ["boto3", "mypy-boto3-glue"], "md": ["duckdb>=0.10.2"]},
     classifiers=[

tests/functional/adapter/test_unit_testing.py

@@ -0,0 +1,36 @@
+import pytest
+
+from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes
+from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity
+from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput
+
+
+@pytest.mark.skip_profile("buenavista")
+class TestUnitTestingTypesDuckDB(BaseUnitTestingTypes):
+    @pytest.fixture
+    def data_types(self):
+        # sql_value, yaml_value
+        return [
+            ["1", "1"],
+            ["2.0", "2.0"],
+            ["'12345'", "12345"],
+            ["'string'", "string"],
+            ["true", "true"],
+            ["DATE '2020-01-02'", "2020-01-02"],
+            ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"],
+            ["'2013-11-03 00:00:00-0'::TIMESTAMPTZ", "2013-11-03 00:00:00-0"],
+            [
+                "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}",
+                "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}",
+            ],
+            ["ARRAY['a','b','c']", "['a','b','c']"],
+            ["ARRAY[1,2,3]", "[1, 2, 3]"],
+        ]
+
+
+class TestUnitTestCaseInsensitivityDuckDB(BaseUnitTestCaseInsensivity):
+    pass
+
+
+class TestUnitTestInvalidInputDuckDB(BaseUnitTestInvalidInput):
+    pass

tests/unit/test_duckdb_adapter.py

@@ -39,8 +39,9 @@ def setUp(self):
 
     @property
     def adapter(self):
+        self.mock_mp_context = mock.MagicMock()
         if self._adapter is None:
-            self._adapter = DuckDBAdapter(self.config)
+            self._adapter = DuckDBAdapter(self.config, self.mock_mp_context)
         return self._adapter
 
     @mock.patch("dbt.adapters.duckdb.environments.duckdb")

tests/unit/test_external_utils.py

@@ -1,5 +1,6 @@
 import unittest
 from argparse import Namespace
+from unittest import mock
 
 from dbt.flags import set_from_args
 from dbt.adapters.duckdb import DuckDBAdapter
@@ -32,8 +33,9 @@ def setUp(self):
 
     @property
     def adapter(self):
+        self.mock_mp_context = mock.MagicMock()
         if self._adapter is None:
-            self._adapter = DuckDBAdapter(self.config)
+            self._adapter = DuckDBAdapter(self.config, self.mock_mp_context)
         return self._adapter
 
     def test_external_write_options(self):