diff --git a/dbt/adapters/base/__init__.py b/dbt/adapters/base/__init__.py index ade1af3d..9e7549f8 100644 --- a/dbt/adapters/base/__init__.py +++ b/dbt/adapters/base/__init__.py @@ -1,5 +1,6 @@ from dbt.adapters.base.meta import available from dbt.adapters.base.column import Column +from dbt.adapters.base.catalog import ExternalCatalogIntegration from dbt.adapters.base.connections import BaseConnectionManager from dbt.adapters.base.impl import ( AdapterConfig, diff --git a/dbt/adapters/base/catalog.py b/dbt/adapters/base/catalog.py new file mode 100644 index 00000000..a1aa66e6 --- /dev/null +++ b/dbt/adapters/base/catalog.py @@ -0,0 +1,59 @@ +import abc +from typing import Self, ValuesView + +from dbt_config.catalog_config import ExternalCatalog + +from dbt.adapters.base import BaseRelation, BaseConnectionManager + + +class ExternalCatalogIntegration(abc.ABC): + name: str + external_catalog: ExternalCatalog + _connection_manager: BaseConnectionManager + _exists: bool + + @classmethod + def create(cls, external_catalog: ExternalCatalog, connection_manager: BaseConnectionManager) -> Self: + integration = ExternalCatalogIntegration() + integration.external_catalog = external_catalog + integration.name = external_catalog.name + _connection_manager = connection_manager + return integration + + @abc.abstractmethod + def _exists(self) -> bool: + pass + + def exists(self) -> bool: + return self._exists + @abc.abstractmethod + def relation_exists(self, relation: BaseRelation) -> bool: + pass + + @abc.abstractmethod + def refresh_relation(self, table_name: str) -> None: + pass + + @abc.abstractmethod + def create_relation(self, table_name: str) -> None: + pass + + +class ExternalCatalogIntegrations: + def get(self, name: str) -> ExternalCatalogIntegration: + return self.integrations[name] + + @property + def integrations(self) -> dict[str, ExternalCatalogIntegration]: + return self.integrations + + @classmethod + def from_json_strings(cls, json_strings: ValuesView[str], + integration_class: ExternalCatalogIntegration, + connection_manager: BaseConnectionManager) -> Self: + new_instance = cls() + for json_string in json_strings: + external_catalog = ExternalCatalog.model_validate_json(json_string) + integration = integration_class.create(external_catalog, connection_manager) + new_instance.integrations[integration.name] = integration + return new_instance diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index f3788fe3..7f8820dc 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -50,6 +50,7 @@ ) from dbt.adapters.base.column import Column as BaseColumn +from dbt.adapters.base.catalog import ExternalCatalogIntegration from dbt.adapters.base.connections import ( AdapterResponse, BaseConnectionManager, @@ -228,6 +229,7 @@ class BaseAdapter(metaclass=AdapterMeta): - expand_column_types - list_relations_without_caching - is_cancelable + - execute - create_schema - drop_schema - quote @@ -241,11 +243,14 @@ class BaseAdapter(metaclass=AdapterMeta): Macros: - get_catalog + + External Catalog support: Attach an implementation of ExternalCatalogIntegration """ Relation: Type[BaseRelation] = BaseRelation Column: Type[BaseColumn] = BaseColumn ConnectionManager: Type[BaseConnectionManager] + ExternalCatalogIntegration: Type[ExternalCatalogIntegration] # A set of clobber config fields accepted by this adapter # for use in materializations diff --git a/dbt/adapters/base/relation.py b/dbt/adapters/base/relation.py index 80dbd34b..db6f86b6 100644 --- a/dbt/adapters/base/relation.py +++ b/dbt/adapters/base/relation.py @@ -71,6 +71,7 @@ class BaseRelation(FakeAPIObject, Hashable): # e.g. adding RelationType.View in dbt-postgres requires that you define: # include/postgres/macros/relations/view/replace.sql::postgres__get_replace_view_sql() replaceable_relations: SerializableIterable = field(default_factory=frozenset) + catalog: Optional[str] = None def _is_exactish_match(self, field: ComponentName, value: str) -> bool: if self.dbt_created and self.quote_policy.get_part(field) is False: diff --git a/dbt/adapters/capability.py b/dbt/adapters/capability.py index 2bd49112..4e410bb2 100644 --- a/dbt/adapters/capability.py +++ b/dbt/adapters/capability.py @@ -21,6 +21,8 @@ class Capability(str, Enum): """Indicates support for getting catalog information including table-level and column-level metadata for a single relation.""" + CreateExternalCatalog = "CreateExternalCatalog" + class Support(str, Enum): Unknown = "Unknown" diff --git a/dbt/adapters/contracts/connection.py b/dbt/adapters/contracts/connection.py index e3baf284..751d6135 100644 --- a/dbt/adapters/contracts/connection.py +++ b/dbt/adapters/contracts/connection.py @@ -19,6 +19,8 @@ ValidatedStringMixin, dbtClassMixin, ) +from dbt_config.catalog_config import ExternalCatalogConfig + # TODO: this is a very bad dependency - shared global state from dbt_common.events.contextvars import get_node_info @@ -226,3 +228,4 @@ class AdapterRequiredConfig(HasCredentials, Protocol): cli_vars: Dict[str, Any] target_path: str log_cache_events: bool + catalogs = Optional[ExternalCatalogConfig] diff --git a/dbt/adapters/contracts/relation.py b/dbt/adapters/contracts/relation.py index 42beb579..636ea9cc 100644 --- a/dbt/adapters/contracts/relation.py +++ b/dbt/adapters/contracts/relation.py @@ -10,6 +10,7 @@ from dbt_common.dataclass_schema import StrEnum, dbtClassMixin from dbt_common.exceptions import CompilationError, DataclassNotDictError from dbt_common.utils import deep_merge +from dbt_config.catalog_config import ExternalCatalog from typing_extensions import Protocol @@ -58,6 +59,7 @@ class RelationConfig(Protocol): tags: List[str] quoting_dict: Dict[str, bool] config: Optional[MaterializationConfig] + catalog_name: Optional[str] class ComponentName(StrEnum): diff --git a/dbt/adapters/protocol.py b/dbt/adapters/protocol.py index 35219866..58ff001b 100644 --- a/dbt/adapters/protocol.py +++ b/dbt/adapters/protocol.py @@ -41,6 +41,9 @@ class ConnectionManagerProtocol(Protocol): class ColumnProtocol(Protocol): pass +class ExternalCatalogIntegrationProtocol(Protocol): + pass + Self = TypeVar("Self", bound="RelationProtocol") @@ -62,6 +65,7 @@ def create_from( ConnectionManager_T = TypeVar("ConnectionManager_T", bound=ConnectionManagerProtocol) Relation_T = TypeVar("Relation_T", bound=RelationProtocol) Column_T = TypeVar("Column_T", bound=ColumnProtocol) +ExtCatInteg_T = TypeVar("ExtCatInteg_T", bound=ExternalCatalogIntegrationProtocol) class MacroContextGeneratorCallable(Protocol): @@ -82,6 +86,7 @@ class AdapterProtocol( # type: ignore[misc] ConnectionManager_T, Relation_T, Column_T, + ExtCatInteg_T, ], ): # N.B. Technically these are ClassVars, but mypy doesn't support putting type vars in a @@ -92,6 +97,7 @@ class AdapterProtocol( # type: ignore[misc] Relation: Type[Relation_T] ConnectionManager: Type[ConnectionManager_T] connections: ConnectionManager_T + ExternalCatalogIntegration: Type[ExtCatInteg_T] def __init__(self, config: AdapterRequiredConfig) -> None: ... diff --git a/pyproject.toml b/pyproject.toml index 76ca3dee..6efffefe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ classifiers = [ ] dependencies = [ "dbt-common>=1.10,<2.0", + "dbt-config<1.0", "pytz>=2015.7", # installed via dbt-common but used directly "agate>=1.0,<2.0", @@ -54,6 +55,7 @@ include = ["dbt/adapters", "dbt/include", "dbt/__init__.py"] [tool.hatch.envs.default] dependencies = [ + "dbt-config @ git+https://github.com/dbt-labs/dbt-common.git@feature/externalCatalogConfig#subdirectory=config", "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"',