Feature/external catalog config

dbt/adapters/base/__init__.py

@@ -1,5 +1,6 @@
 from dbt.adapters.base.meta import available
 from dbt.adapters.base.column import Column
+from dbt.adapters.base.catalog import ExternalCatalogIntegration
 from dbt.adapters.base.connections import BaseConnectionManager
 from dbt.adapters.base.impl import (
     AdapterConfig,

dbt/adapters/base/catalog.py

@@ -0,0 +1,59 @@
+import abc
+from typing import Self, ValuesView
+
+from dbt_config.catalog_config import ExternalCatalog
+
+from dbt.adapters.base import BaseRelation, BaseConnectionManager
+
+
+class ExternalCatalogIntegration(abc.ABC):
+    name: str
+    external_catalog: ExternalCatalog
+    _connection_manager: BaseConnectionManager
+    _exists: bool
+
+    @classmethod
+    def create(cls, external_catalog: ExternalCatalog, connection_manager: BaseConnectionManager) -> Self:
+        integration = ExternalCatalogIntegration()
+        integration.external_catalog = external_catalog
+        integration.name = external_catalog.name
+        _connection_manager = connection_manager
+        return integration
+
+    @abc.abstractmethod
+    def _exists(self) -> bool:
+        pass
+
+    def exists(self) -> bool:
+        return self._exists
+    @abc.abstractmethod
+    def relation_exists(self, relation: BaseRelation) -> bool:
+        pass
+
+    @abc.abstractmethod
+    def refresh_relation(self, table_name: str) -> None:
+        pass
+
+    @abc.abstractmethod
+    def create_relation(self, table_name: str) -> None:
+        pass
+
+
+class ExternalCatalogIntegrations:
+    def get(self, name: str) -> ExternalCatalogIntegration:
+        return self.integrations[name]
+
+    @property
+    def integrations(self) -> dict[str, ExternalCatalogIntegration]:
+        return self.integrations
+
+    @classmethod
+    def from_json_strings(cls, json_strings: ValuesView[str],
+                          integration_class: ExternalCatalogIntegration,
+                          connection_manager: BaseConnectionManager) -> Self:
+        new_instance = cls()
+        for json_string in json_strings:
+            external_catalog = ExternalCatalog.model_validate_json(json_string)
+            integration = integration_class.create(external_catalog, connection_manager)
+            new_instance.integrations[integration.name] = integration
+        return new_instance

dbt/adapters/base/impl.py

@@ -50,6 +50,7 @@
 )
 
 from dbt.adapters.base.column import Column as BaseColumn
+from dbt.adapters.base.catalog import ExternalCatalogIntegration
 from dbt.adapters.base.connections import (
     AdapterResponse,
     BaseConnectionManager,
@@ -228,6 +229,7 @@ class BaseAdapter(metaclass=AdapterMeta):
         - expand_column_types
         - list_relations_without_caching
         - is_cancelable
+        - execute
         - create_schema
         - drop_schema
         - quote
@@ -241,11 +243,14 @@ class BaseAdapter(metaclass=AdapterMeta):
 
     Macros:
         - get_catalog
+
+    External Catalog support: Attach an implementation of ExternalCatalogIntegration
     """
 
     Relation: Type[BaseRelation] = BaseRelation
     Column: Type[BaseColumn] = BaseColumn
     ConnectionManager: Type[BaseConnectionManager]
+    ExternalCatalogIntegration: Type[ExternalCatalogIntegration]
 
     # A set of clobber config fields accepted by this adapter
     # for use in materializations

dbt/adapters/base/relation.py

@@ -71,6 +71,7 @@ class BaseRelation(FakeAPIObject, Hashable):
     # e.g. adding RelationType.View in dbt-postgres requires that you define:
     # include/postgres/macros/relations/view/replace.sql::postgres__get_replace_view_sql()
     replaceable_relations: SerializableIterable = field(default_factory=frozenset)
+    catalog: Optional[str] = None
 
     def _is_exactish_match(self, field: ComponentName, value: str) -> bool:
         if self.dbt_created and self.quote_policy.get_part(field) is False:

dbt/adapters/capability.py

@@ -21,6 +21,8 @@ class Capability(str, Enum):
     """Indicates support for getting catalog information including table-level and column-level metadata for a single
     relation."""
 
+    CreateExternalCatalog = "CreateExternalCatalog"
+
 
 class Support(str, Enum):
     Unknown = "Unknown"

dbt/adapters/contracts/connection.py

@@ -19,6 +19,8 @@
     ValidatedStringMixin,
     dbtClassMixin,
 )
+from dbt_config.catalog_config import ExternalCatalogConfig
+
 
 # TODO: this is a very bad dependency - shared global state
 from dbt_common.events.contextvars import get_node_info
@@ -226,3 +228,4 @@ class AdapterRequiredConfig(HasCredentials, Protocol):
     cli_vars: Dict[str, Any]
     target_path: str
     log_cache_events: bool
+    catalogs = Optional[ExternalCatalogConfig]

dbt/adapters/contracts/relation.py

@@ -10,6 +10,7 @@
 from dbt_common.dataclass_schema import StrEnum, dbtClassMixin
 from dbt_common.exceptions import CompilationError, DataclassNotDictError
 from dbt_common.utils import deep_merge
+from dbt_config.catalog_config import ExternalCatalog
 from typing_extensions import Protocol
 
 
@@ -58,6 +59,7 @@ class RelationConfig(Protocol):
     tags: List[str]
     quoting_dict: Dict[str, bool]
     config: Optional[MaterializationConfig]
+    catalog_name: Optional[str]
 
 
 class ComponentName(StrEnum):

dbt/adapters/protocol.py

@@ -41,6 +41,9 @@ class ConnectionManagerProtocol(Protocol):
 class ColumnProtocol(Protocol):
     pass
 
+class ExternalCatalogIntegrationProtocol(Protocol):
+    pass
+
 
 Self = TypeVar("Self", bound="RelationProtocol")
 
@@ -62,6 +65,7 @@ def create_from(
 ConnectionManager_T = TypeVar("ConnectionManager_T", bound=ConnectionManagerProtocol)
 Relation_T = TypeVar("Relation_T", bound=RelationProtocol)
 Column_T = TypeVar("Column_T", bound=ColumnProtocol)
+ExtCatInteg_T = TypeVar("ExtCatInteg_T", bound=ExternalCatalogIntegrationProtocol)
 
 
 class MacroContextGeneratorCallable(Protocol):
@@ -82,6 +86,7 @@ class AdapterProtocol(  # type: ignore[misc]
         ConnectionManager_T,
         Relation_T,
         Column_T,
+        ExtCatInteg_T,
     ],
 ):
     # N.B. Technically these are ClassVars, but mypy doesn't support putting type vars in a
@@ -92,6 +97,7 @@ class AdapterProtocol(  # type: ignore[misc]
     Relation: Type[Relation_T]
     ConnectionManager: Type[ConnectionManager_T]
     connections: ConnectionManager_T
+    ExternalCatalogIntegration: Type[ExtCatInteg_T]
 
     def __init__(self, config: AdapterRequiredConfig) -> None: ...
 

pyproject.toml

@@ -25,6 +25,7 @@ classifiers = [
 ]
 dependencies = [
     "dbt-common>=1.10,<2.0",
+    "dbt-config<1.0",
     "pytz>=2015.7",
     # installed via dbt-common but used directly
     "agate>=1.0,<2.0",
@@ -54,6 +55,7 @@ include = ["dbt/adapters", "dbt/include", "dbt/__init__.py"]
 
 [tool.hatch.envs.default]
 dependencies = [
+    "dbt-config @ git+https://github.com/dbt-labs/dbt-common.git@feature/externalCatalogConfig#subdirectory=config",
     "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git",
     'pre-commit==3.7.0;python_version>="3.9"',
     'pre-commit==3.5.0;python_version=="3.8"',