diff --git a/.changes/unreleased/Features-20241001-165406.yaml b/.changes/unreleased/Features-20241001-165406.yaml new file mode 100644 index 00000000..609684d4 --- /dev/null +++ b/.changes/unreleased/Features-20241001-165406.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Use a behavior flag to gate microbatch functionality (instead of an environment + variable) +time: 2024-10-01T16:54:06.121016-05:00 +custom: + Author: QMalcolm + Issue: "327" diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 8c657eec..977620c3 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.10.2" +version = "1.10.3" diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_microbatch.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_microbatch.py index 5bbabbe1..d35dee8c 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_microbatch.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_microbatch.py @@ -1,6 +1,4 @@ -import os from pprint import pformat -from unittest import mock import pytest @@ -63,7 +61,16 @@ def assert_row_count(self, project, relation_name: str, expected_row_count: int) assert len(result) == expected_row_count, f"{relation_name}:{pformat(result)}" - @mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"}) + +class BaseTestMicrobatchOn(BaseMicrobatch): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": { + "require_builtin_microbatch_strategy": True, + } + } + def test_run_with_event_time(self, project, insert_two_rows_sql): # initial run -- backfills all data with patch_microbatch_end_time("2020-01-03 13:57:00"): @@ -94,3 +101,17 @@ def test_run_with_event_time(self, project, insert_two_rows_sql): with patch_microbatch_end_time("2020-01-05 14:57:00"): run_dbt(["run", "--select", "microbatch_model"]) self.assert_row_count(project, "microbatch_model", 5) + + +class BaseTestMicrobatchOff(BaseMicrobatch): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": { + "require_builtin_microbatch_strategy": False, + } + } + + def test_run_with_event_time(self, project): + with patch_microbatch_end_time("2020-01-03 13:57:00"): + run_dbt(["run"], expect_pass=False) diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index f3788fe3..28d50d08 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -22,7 +22,6 @@ Union, TYPE_CHECKING, ) -import os import pytz from dbt_common.behavior_flags import Behavior, BehaviorFlag from dbt_common.clients.jinja import CallableMacroGenerator @@ -312,7 +311,13 @@ def _behavior_flags(self) -> List[BehaviorFlag]: """ This method should be overwritten by adapter maintainers to provide platform-specific flags """ - return [] + return [ + { + "name": "require_builtin_microbatch_strategy", + "default": False, + "docs_url": "https://docs.getdbt.com/docs/build/incremental-microbatch", + } + ] ### # Methods that pass through to the connection manager @@ -1570,7 +1575,7 @@ def valid_incremental_strategies(self): def builtin_incremental_strategies(self): builtin_strategies = ["append", "delete+insert", "merge", "insert_overwrite"] - if os.environ.get("DBT_EXPERIMENTAL_MICROBATCH"): + if self.behavior.require_builtin_microbatch_strategy: builtin_strategies.append("microbatch") return builtin_strategies