diff --git a/test.env.example b/test.env.example index 08dfcf6b..9a8c9b92 100644 --- a/test.env.example +++ b/test.env.example @@ -7,8 +7,6 @@ DBT_DATABRICKS_HTTP_PATH= ## Also able to set the http-path values for each profile ## Server http-path value for `databricks_cluster` profile # DBT_DATABRICKS_CLUSTER_HTTP_PATH= -## Server http-path value for `databricks_sql_endpoint` profile -# DBT_DATABRICKS_ENDPOINT_HTTP_PATH= ## Server http-path value for `databricks_uc_cluster` profile # DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH= ## Server http-path value for `databricks_uc_sql_endpoint` profile diff --git a/tests/functional/adapter/test_changing_relation_type.py b/tests/functional/adapter/test_changing_relation_type.py index 94bae797..afe38f5b 100644 --- a/tests/functional/adapter/test_changing_relation_type.py +++ b/tests/functional/adapter/test_changing_relation_type.py @@ -1,15 +1,15 @@ import pytest -from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator +from dbt.tests.adapter.relations.test_changing_relation_type import ( + BaseChangeRelationTypeValidator, +) class TestChangeRelationTypesDatabricks(BaseChangeRelationTypeValidator): pass -@pytest.mark.skip_profile( - "databricks_uc_cluster", "databricks_sql_endpoint", "databricks_uc_sql_endpoint" -) +@pytest.mark.skip_profile("databricks_uc_cluster", "databricks_uc_sql_endpoint") class TestChangeRelationTypesParquetDatabricks(BaseChangeRelationTypeValidator): @pytest.fixture(scope="class") def project_config_update(self): diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 361b1f26..374f4672 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -75,7 +75,7 @@ def data_types(self, schema_int_type, int_type, string_type): ] -@pytest.mark.skip_profile("databricks_sql_endpoint", "databricks_cluster") +@pytest.mark.skip_profile("databricks_cluster") class TestSparkTableConstraintsColumnsEqualDatabricksHTTP( DatabricksHTTPSetup, BaseTableConstraintsColumnsEqual ): @@ -88,7 +88,7 @@ def models(self): } -@pytest.mark.skip_profile("databricks_sql_endpoint", "databricks_cluster") +@pytest.mark.skip_profile("databricks_cluster") class TestSparkViewConstraintsColumnsEqualDatabricksHTTP( DatabricksHTTPSetup, BaseViewConstraintsColumnsEqual ): @@ -101,7 +101,7 @@ def models(self): } -@pytest.mark.skip_profile("databricks_sql_endpoint", "databricks_cluster") +@pytest.mark.skip_profile("databricks_cluster") class TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP( DatabricksHTTPSetup, BaseIncrementalConstraintsColumnsEqual ): @@ -128,7 +128,7 @@ def expected_sql(self): return _expected_sql_spark -@pytest.mark.skip_profile("databricks_sql_endpoint", "databricks_cluster") +@pytest.mark.skip_profile("databricks_cluster") class TestSparkTableConstraintsDdlEnforcement( BaseSparkConstraintsDdlEnforcementSetup, BaseConstraintsRuntimeDdlEnforcement ): @@ -140,7 +140,7 @@ def models(self): } -@pytest.mark.skip_profile("databricks_sql_endpoint", "databricks_cluster") +@pytest.mark.skip_profile("databricks_cluster") class TestSparkIncrementalConstraintsDdlEnforcement( BaseSparkConstraintsDdlEnforcementSetup, BaseIncrementalConstraintsRuntimeDdlEnforcement, @@ -179,7 +179,7 @@ def assert_expected_error_messages(self, error_message, expected_error_messages) assert any(msg in error_message for msg in expected_error_messages) -@pytest.mark.skip_profile("databricks_sql_endpoint", "databricks_cluster") +@pytest.mark.skip_profile("databricks_cluster") class TestSparkTableConstraintsRollback(BaseSparkConstraintsRollbackSetup, BaseConstraintsRollback): @pytest.fixture(scope="class") def models(self): @@ -196,7 +196,7 @@ def expected_color(self): return "red" -@pytest.mark.skip_profile("databricks_sql_endpoint", "databricks_cluster") +@pytest.mark.skip_profile("databricks_cluster") class TestSparkIncrementalConstraintsRollback( BaseSparkConstraintsRollbackSetup, BaseIncrementalConstraintsRollback ): diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py index 35c93943..4e3428b7 100644 --- a/tests/functional/adapter/test_grants.py +++ b/tests/functional/adapter/test_grants.py @@ -7,7 +7,7 @@ @pytest.mark.skip(reason="DECO team must provide DBT_TEST_USER_1/2/3 before we re-enable") -# @pytest.mark.skip_profile("databricks_cluster", "databricks_sql_endpoint") +# @pytest.mark.skip_profile("databricks_cluster") class TestModelGrantsDatabricks(BaseModelGrants): def privilege_grantee_name_overrides(self): # insert --> modify @@ -20,13 +20,13 @@ def privilege_grantee_name_overrides(self): @pytest.mark.skip(reason="DECO team must provide DBT_TEST_USER_1/2/3 before we re-enable") -# @pytest.mark.skip_profile("databricks_cluster", "databricks_sql_endpoint") +# @pytest.mark.skip_profile("databricks_cluster") class TestIncrementalGrantsDatabricks(BaseIncrementalGrants): pass @pytest.mark.skip(reason="DECO team must provide DBT_TEST_USER_1/2/3 before we re-enable") -# @pytest.mark.skip_profile("databricks_cluster", "databricks_sql_endpoint") +# @pytest.mark.skip_profile("databricks_cluster") class TestSeedGrantsDatabricks(BaseSeedGrants): # seeds in dbt-spark are currently "full refreshed," in such a way that # the grants are not carried over @@ -36,13 +36,13 @@ def seeds_support_partial_refresh(self): @pytest.mark.skip(reason="DECO team must provide DBT_TEST_USER_1/2/3 before we re-enable") -# @pytest.mark.skip_profile("databricks_cluster", "databricks_sql_endpoint") +# @pytest.mark.skip_profile("databricks_cluster") class TestSnapshotGrantsDatabricks(BaseSnapshotGrants): pass @pytest.mark.skip(reason="DECO team must provide DBT_TEST_USER_1/2/3 before we re-enable") -# @pytest.mark.skip_profile("databricks_cluster", "databricks_sql_endpoint") +# @pytest.mark.skip_profile("databricks_cluster") class TestInvalidGrantsDatabricks(BaseInvalidGrants): def grantee_does_not_exist_error(self): return "PRINCIPAL_DOES_NOT_EXIST" diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index e2ee9bc3..073cb511 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -15,12 +15,12 @@ ) -@pytest.mark.skip_profile("databricks_sql_endpoint", "databricks_uc_sql_endpoint") +@pytest.mark.skip_profile("databricks_uc_sql_endpoint") class TestPythonModelDatabricks(BasePythonModelTests): pass -@pytest.mark.skip_profile("databricks_sql_endpoint", "databricks_uc_sql_endpoint") +@pytest.mark.skip_profile("databricks_uc_sql_endpoint") class TestPythonIncrementalModelDatabricks(BasePythonIncrementalTests): @pytest.fixture(scope="class") def project_config_update(self): @@ -49,7 +49,7 @@ def model(dbt, spark): """ -@pytest.mark.skip_profile("databricks_sql_endpoint", "databricks_uc_sql_endpoint") +@pytest.mark.skip_profile("databricks_uc_sql_endpoint") class TestChangingSchemaDatabricks: @pytest.fixture(scope="class") def models(self): @@ -72,7 +72,7 @@ def test_changing_schema_with_log_validation(self, project, logs_dir): assert "Execution status: OK in" in log -@pytest.mark.skip_profile("databricks_sql_endpoint", "databricks_uc_sql_endpoint") +@pytest.mark.skip_profile("databricks_uc_sql_endpoint") class TestPySparkDatabricks(BasePySparkTests): @pytest.fixture(scope="class") def models(self): diff --git a/tests/integration/base.py b/tests/integration/base.py index c10e10f8..6283d396 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -96,7 +96,6 @@ def _profile_from_test_name(test_name): adapter_names = ( "databricks_cluster", "databricks_uc_cluster", - "databricks_sql_endpoint", "databricks_uc_sql_endpoint", ) adapters_in_name = sum(x in test_name for x in adapter_names) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c9f790e3..92b2f5e7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,5 +1,4 @@ def pytest_configure(config): config.addinivalue_line("markers", "profile_databricks_cluster") config.addinivalue_line("markers", "profile_databricks_uc_cluster") - config.addinivalue_line("markers", "profile_databricks_sql_endpoint") config.addinivalue_line("markers", "profile_databricks_uc_sql_endpoint") diff --git a/tests/integration/copy_into/test_copy_into.py b/tests/integration/copy_into/test_copy_into.py index d265be09..665300d6 100644 --- a/tests/integration/copy_into/test_copy_into.py +++ b/tests/integration/copy_into/test_copy_into.py @@ -44,7 +44,8 @@ def prepare(self): raise Exception("No location found for the source table") return path - def test_copy_into(self): + @use_profile("databricks_cluster") + def test_copy_into_databricks_cluster(self): path = self.prepare() self.run_dbt( @@ -57,7 +58,8 @@ def test_copy_into(self): ) self.assertTablesEqual("target", "expected_target") - def test_copy_into_with_expression_list(self): + @use_profile("databricks_cluster") + def test_copy_into_with_expression_list_databricks_cluster(self): path = self.prepare() self.run_dbt( @@ -71,11 +73,3 @@ def test_copy_into_with_expression_list(self): self.assertTablesEqual( "target_with_expression_list", "expected_target_with_expression_list" ) - - @use_profile("databricks_cluster") - def test_copy_into_databricks_cluster(self): - self.test_copy_into() - - @use_profile("databricks_cluster") - def test_copy_into_with_expression_list_databricks_cluster(self): - self.test_copy_into_with_expression_list() diff --git a/tests/integration/debug/test_debug.py b/tests/integration/debug/test_debug.py index c66de17a..00c51948 100644 --- a/tests/integration/debug/test_debug.py +++ b/tests/integration/debug/test_debug.py @@ -26,10 +26,6 @@ def run_and_test(self, contains_catalog: bool): def test_debug_databricks_cluster(self): self.run_and_test(contains_catalog=False) - @use_profile("databricks_sql_endpoint") - def test_debug_databricks_sql_endpoint(self): - self.run_and_test(contains_catalog=False) - @use_profile("databricks_uc_cluster") def test_debug_databricks_uc_cluster(self): self.run_and_test(contains_catalog=True) diff --git a/tests/integration/fail_fast/test_fail_fast.py b/tests/integration/fail_fast/test_fail_fast.py index ce2639d9..31759262 100644 --- a/tests/integration/fail_fast/test_fail_fast.py +++ b/tests/integration/fail_fast/test_fail_fast.py @@ -12,7 +12,7 @@ def schema(self): def models(self): return "models" - def test_fail_fast(self): + def _test_fail_fast(self): self.run_dbt(["run"]) # PECO-738 Original except message we tested for was: @@ -30,16 +30,12 @@ def test_fail_fast(self): @use_profile("databricks_cluster") def test_fail_fast_databricks_cluster(self): - self.test_fail_fast() + self._test_fail_fast() @use_profile("databricks_uc_cluster") def test_fail_fast_databricks_uc_cluster(self): - self.test_fail_fast() - - @use_profile("databricks_sql_endpoint") - def test_fail_fast_databricks_sql_endpoint(self): - self.test_fail_fast() + self._test_fail_fast() @use_profile("databricks_uc_sql_endpoint") def test_fail_fast_databricks_uc_sql_endpoint(self): - self.test_fail_fast() + self._test_fail_fast() diff --git a/tests/integration/get_columns_in_relation/test_get_columns_in_relation.py b/tests/integration/get_columns_in_relation/test_get_columns_in_relation.py index 5d737a5e..b0ceb0dd 100644 --- a/tests/integration/get_columns_in_relation/test_get_columns_in_relation.py +++ b/tests/integration/get_columns_in_relation/test_get_columns_in_relation.py @@ -22,10 +22,6 @@ def test_get_columns_in_relation_in_same_run_databricks_cluster(self): def test_get_columns_in_relation_in_same_run_databricks_uc_cluster(self): self.run_and_test() - @use_profile("databricks_sql_endpoint") - def test_get_columns_in_relation_in_same_run_databricks_sql_endpoint(self): - self.run_and_test() - @use_profile("databricks_uc_sql_endpoint") def test_get_columns_in_relation_in_same_run_databricks_uc_sql_endpoint(self): self.run_and_test() diff --git a/tests/integration/incremental_on_schema_change/test_incremental_on_schema_change.py b/tests/integration/incremental_on_schema_change/test_incremental_on_schema_change.py index a3aa9b07..b19e8e2d 100644 --- a/tests/integration/incremental_on_schema_change/test_incremental_on_schema_change.py +++ b/tests/integration/incremental_on_schema_change/test_incremental_on_schema_change.py @@ -15,7 +15,6 @@ def project_config(self): return {"config-version": 2, "test-paths": ["tests"]} def run_twice_and_assert(self, include, compare_source, compare_target): - # dbt run (twice) run_args = ["run"] if include: @@ -89,18 +88,6 @@ def test__databricks_uc_cluster__run_incremental_fail_on_schema_change(self): def test__databricks_uc_cluster__run_incremental_sync_all_columns(self): self.run_incremental_sync_all_columns() - @use_profile("databricks_sql_endpoint") - def test__databricks_sql_endpoint__run_incremental_ignore(self): - self.run_incremental_ignore() - - @use_profile("databricks_sql_endpoint") - def test__databricks_sql_endpoint__run_incremental_fail_on_schema_change(self): - self.run_incremental_fail_on_schema_change() - - @use_profile("databricks_sql_endpoint") - def test__databricks_sql_endpoint__run_incremental_sync_all_columns(self): - self.run_incremental_sync_all_columns() - @use_profile("databricks_uc_sql_endpoint") def test__databricks_uc_sql_endpoint__run_incremental_ignore(self): self.run_incremental_ignore() @@ -157,22 +144,6 @@ def test__databricks_uc_cluster__run_incremental_append_new_columns(self): def test__databricks_uc_cluster__run_incremental_sync_all_columns(self): self.run_incremental_sync_all_columns() - @use_profile("databricks_sql_endpoint") - def test__databricks_sql_endpoint__run_incremental_ignore(self): - self.run_incremental_ignore() - - @use_profile("databricks_sql_endpoint") - def test__databricks_sql_endpoint__run_incremental_fail_on_schema_change(self): - self.run_incremental_fail_on_schema_change() - - @use_profile("databricks_sql_endpoint") - def test__databricks_sql_endpoint__run_incremental_append_new_columns(self): - self.run_incremental_append_new_columns() - - @use_profile("databricks_sql_endpoint") - def test__databricks_sql_endpoint__run_incremental_sync_all_columns(self): - self.run_incremental_sync_all_columns() - @use_profile("databricks_uc_sql_endpoint") def test__databricks_uc_sql_endpoint__run_incremental_ignore(self): self.run_incremental_ignore() diff --git a/tests/integration/incremental_strategies/test_incremental_strategies.py b/tests/integration/incremental_strategies/test_incremental_strategies.py index e882b3f4..53a2cc88 100644 --- a/tests/integration/incremental_strategies/test_incremental_strategies.py +++ b/tests/integration/incremental_strategies/test_incremental_strategies.py @@ -67,10 +67,6 @@ def run_and_test_warehouse(self): self.assertTablesEqual("merge_exclude_columns", "expected_exclude_upsert") self.assertTablesEqual("replace_where", "expected_replace_where") - @use_profile("databricks_sql_endpoint") - def test_delta_strategies_databricks_sql_endpoint(self): - self.run_and_test_warehouse() - @use_profile("databricks_uc_sql_endpoint") def test_delta_strategies_databricks_uc_sql_endpoint(self): self.run_and_test_warehouse() @@ -130,10 +126,6 @@ def test_bad_strategies_databricks_cluster(self): def test_bad_strategies_databricks_uc_cluster(self): self.run_and_test() - @use_profile("databricks_sql_endpoint") - def test_bad_strategies_databricks_sql_endpoint(self): - self.run_and_test() - @use_profile("databricks_uc_sql_endpoint") def test_bad_strategies_databricks_uc_sql_endpoint(self): self.run_and_test() diff --git a/tests/integration/materialized_view/test_materialized_view.py b/tests/integration/materialized_view/test_materialized_view.py index 9f320668..7f513fb7 100644 --- a/tests/integration/materialized_view/test_materialized_view.py +++ b/tests/integration/materialized_view/test_materialized_view.py @@ -12,7 +12,7 @@ def schema(self): def models(self): return "models" - def test_materialized_view_base(self): + def _test_materialized_view_base(self): self.run_dbt(["seed"]) self.run_dbt(["run", "--select", "+mv"]) @@ -30,7 +30,7 @@ def test_materialized_view_base(self): self.run_dbt(["run", "--full-refresh", "--select", "mv"]) self.assertTablesEqual("mv", "expected2") - def test_materialized_view_no_cdf(self): + def _test_materialized_view_no_cdf(self): # The base table is not CDF. result = self.run_dbt(["run", "--select", "+mv_nocdf"], expect_pass=False) assert len(result.results) == 2 @@ -41,7 +41,7 @@ def test_materialized_view_no_cdf(self): and "The input for materialized view must have Change Data Feed enabled." in res.message ) - def test_materialized_view_based_on_view(self): + def _test_materialized_view_based_on_view(self): result = self.run_dbt(["run", "--select", "+mv_on_view"], expect_pass=False) assert len(result.results) == 3 res: RunResult = result.results[2] @@ -54,14 +54,14 @@ def test_materialized_view_based_on_view(self): @pytest.mark.skip(reason="not yet ready for production") @use_profile("databricks_uc_sql_endpoint") def test_materialized_view_base_databricks_uc_sql_endpoint(self): - self.test_materialized_view_base() + self._test_materialized_view_base() @pytest.mark.skip(reason="not yet ready for production") @use_profile("databricks_uc_sql_endpoint") def test_materialized_view_no_cdf_databricks_uc_sql_endpoint(self): - self.test_materialized_view_no_cdf() + self._test_materialized_view_no_cdf() @pytest.mark.skip(reason="not yet ready for production") @use_profile("databricks_uc_sql_endpoint") def test_materialized_view_based_on_view_databricks_uc_sql_endpoint(self): - self.test_materialized_view_based_on_view() + self._test_materialized_view_based_on_view() diff --git a/tests/integration/model_contract/test_model_contract.py b/tests/integration/model_contract/test_model_contract.py index 6dc4a216..a653eec2 100644 --- a/tests/integration/model_contract/test_model_contract.py +++ b/tests/integration/model_contract/test_model_contract.py @@ -44,7 +44,7 @@ def check_staging_table_cleaned(self): class TestModelContractConstraints(TestModelContract): - def test_table_constraints(self): + def _test_table_constraints(self): self.run_dbt(["seed"]) model_name = "table_model" expected_model_name = "expected_model" @@ -69,23 +69,19 @@ def test_table_constraints(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_table_constraints() + self._test_table_constraints() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_table_constraints() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_table_constraints() + self._test_table_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_table_constraints() + self._test_table_constraints() class TestIncrementalModelContractConstraints(TestModelContract): - def test_incremental_constraints(self): + def _test_incremental_constraints(self): self.run_dbt(["seed"]) model_name = "incremental_model" self.run_dbt(["run", "--select", model_name, "--full-refresh"]) @@ -117,19 +113,15 @@ def test_incremental_constraints(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_incremental_constraints() + self._test_incremental_constraints() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_incremental_constraints() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_incremental_constraints() + self._test_incremental_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_incremental_constraints() + self._test_incremental_constraints() class TestInvalidModelContractCheckConstraints(TestModelContract): @@ -156,7 +148,7 @@ def run_dbt(self, args=None, profiles_dir=True): res = dbtRunner().invoke(args, log_cache_events=True, log_path=self._logs_dir) return res - def test_invalid_check_constraints(self): + def _test_invalid_check_constraints(self): model_name = "invalid_check_constraint" res = self.run_dbt(["run", "--select", model_name]) self.assertFalse(res.success, "dbt exit state did not match expected") @@ -168,23 +160,19 @@ def test_invalid_check_constraints(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_invalid_check_constraints() + self._test_invalid_check_constraints() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_invalid_check_constraints() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_invalid_check_constraints() + self._test_invalid_check_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_invalid_check_constraints() + self._test_invalid_check_constraints() class TestInvalidModelContractColumnConstraints(TestModelContract): - def test_invalid_column_constraints(self): + def _test_invalid_column_constraints(self): self.run_dbt(["seed"]) model_name = "invalid_column_constraint" self.run_and_check_failure( @@ -194,23 +182,19 @@ def test_invalid_column_constraints(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_invalid_column_constraints() + self._test_invalid_column_constraints() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_invalid_column_constraints() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_invalid_column_constraints() + self._test_invalid_column_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_invalid_column_constraints() + self._test_invalid_column_constraints() class TestTableWithModelContractConstraintsDisabled(TestModelContract): - def test_delta_constraints_disabled(self): + def _test_delta_constraints_disabled(self): self.run_dbt(["seed"]) model_name = "table_model_disable_constraints" expected_model_name = "expected_model" @@ -232,54 +216,50 @@ def test_delta_constraints_disabled(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_delta_constraints_disabled() + self._test_delta_constraints_disabled() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_delta_constraints_disabled() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_delta_constraints_disabled() + self._test_delta_constraints_disabled() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_delta_constraints_disabled() + self._test_delta_constraints_disabled() class TestModelLevelPrimaryKey(TestModelContract): - def test_table_constraints(self): + def _test_table_constraints(self): self.run_dbt(["seed"]) model_name = "primary_key" self.run_dbt(["run", "--select", model_name]) @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_table_constraints() + self._test_table_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_table_constraints() + self._test_table_constraints() class TestModelLevelForeignKey(TestModelContract): - def test_table_constraints(self): + def _test_table_constraints(self): self.run_dbt(["seed"]) self.run_dbt(["run", "--select", "foreign_key_parent"]) self.run_dbt(["run", "--select", "foreign_key"]) @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_table_constraints() + self._test_table_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_table_constraints() + self._test_table_constraints() @pytest.mark.skip(reason="test needs redesign") class TestModelContractNotDelta(TestModelContract): - def test_table_constraints(self): + def _test_table_constraints(self): self.run_dbt(["seed"]) model_name = "not_delta" result, logoutput = self.run_dbt_and_capture( @@ -289,23 +269,19 @@ def test_table_constraints(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_table_constraints() + self._test_table_constraints() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_table_constraints() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_table_constraints() + self._test_table_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_table_constraints() + self._test_table_constraints() class TestModelContractView(TestModelContract): - def test_table_constraints(self): + def _test_table_constraints(self): # persist_constraints should not be called for a view materialization. If it is and # error is raised. # Successfully creating the view indicates that it wasn't called. @@ -315,16 +291,12 @@ def test_table_constraints(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_table_constraints() + self._test_table_constraints() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_table_constraints() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_table_constraints() + self._test_table_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_table_constraints() + self._test_table_constraints() diff --git a/tests/integration/persist_constraints/test_persist_constraints.py b/tests/integration/persist_constraints/test_persist_constraints.py index 69399f5f..144a1053 100644 --- a/tests/integration/persist_constraints/test_persist_constraints.py +++ b/tests/integration/persist_constraints/test_persist_constraints.py @@ -45,7 +45,7 @@ def check_staging_table_cleaned(self): class TestTableConstraints(TestConstraints): - def test_table_constraints(self): + def _test_table_constraints(self): self.run_dbt(["seed"]) model_name = "table_model" expected_model_name = "expected_model" @@ -69,23 +69,19 @@ def test_table_constraints(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_table_constraints() + self._test_table_constraints() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_table_constraints() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_table_constraints() + self._test_table_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_table_constraints() + self._test_table_constraints() class TestIncrementalConstraints(TestConstraints): - def test_incremental_constraints(self): + def _test_incremental_constraints(self): self.run_dbt(["seed"]) model_name = "incremental_model" self.run_dbt(["run", "--select", model_name, "--full-refresh"]) @@ -117,19 +113,15 @@ def test_incremental_constraints(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_incremental_constraints() + self._test_incremental_constraints() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_incremental_constraints() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_incremental_constraints() + self._test_incremental_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_incremental_constraints() + self._test_incremental_constraints() class TestSnapshotConstraints(TestConstraints): @@ -137,7 +129,7 @@ def check_snapshot_results(self, num_rows: int): results = self.run_sql("select * from {database_schema}.my_snapshot", fetch="all") self.assertEqual(len(results), num_rows) - def test_snapshot(self): + def _test_snapshot(self): self.run_dbt(["seed"]) self.run_dbt(["snapshot"]) self.check_snapshot_results(num_rows=2) @@ -159,46 +151,38 @@ def test_snapshot(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_snapshot() + self._test_snapshot() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_snapshot() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_snapshot() + self._test_snapshot() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_snapshot() + self._test_snapshot() class TestInvalidCheckConstraints(TestConstraints): - def test_invalid_check_constraints(self): + def _test_invalid_check_constraints(self): model_name = "invalid_check_constraint" self.run_dbt(["seed"]) self.run_and_check_failure(model_name, err_msg="Invalid check constraint condition") @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_invalid_check_constraints() + self._test_invalid_check_constraints() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_invalid_check_constraints() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_invalid_check_constraints() + self._test_invalid_check_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_invalid_check_constraints() + self._test_invalid_check_constraints() class TestInvalidColumnConstraints(TestConstraints): - def test_invalid_column_constraints(self): + def _test_invalid_column_constraints(self): model_name = "invalid_column_constraint" self.run_dbt(["seed"]) self.run_and_check_failure( @@ -208,23 +192,19 @@ def test_invalid_column_constraints(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_invalid_column_constraints() + self._test_invalid_column_constraints() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_invalid_column_constraints() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_invalid_column_constraints() + self._test_invalid_column_constraints() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_invalid_column_constraints() + self._test_invalid_column_constraints() class TestTableWithConstraintsDisabled(TestConstraints): - def test_delta_constraints_disabled(self): + def _test_delta_constraints_disabled(self): self.run_dbt(["seed"]) model_name = "table_model_disable_constraints" expected_model_name = "expected_model" @@ -246,16 +226,12 @@ def test_delta_constraints_disabled(self): @use_profile("databricks_cluster") def test_databricks_cluster(self): - self.test_delta_constraints_disabled() + self._test_delta_constraints_disabled() @use_profile("databricks_uc_cluster") def test_databricks_uc_cluster(self): - self.test_delta_constraints_disabled() - - @use_profile("databricks_sql_endpoint") - def test_databricks_sql_endpoint(self): - self.test_delta_constraints_disabled() + self._test_delta_constraints_disabled() @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): - self.test_delta_constraints_disabled() + self._test_delta_constraints_disabled() diff --git a/tests/integration/persist_docs/test_persist_docs.py b/tests/integration/persist_docs/test_persist_docs.py index 23b00e53..8326950c 100644 --- a/tests/integration/persist_docs/test_persist_docs.py +++ b/tests/integration/persist_docs/test_persist_docs.py @@ -34,7 +34,7 @@ def project_config(self): }, } - def test_delta_comments(self): + def _test_delta_comments(self): self.run_dbt(["seed"]) self.run_dbt(["run"]) @@ -59,16 +59,12 @@ def test_delta_comments(self): @use_profile("databricks_cluster") def test_delta_comments_databricks_cluster(self): - self.test_delta_comments() + self._test_delta_comments() @use_profile("databricks_uc_cluster") def test_delta_comments_databricks_uc_cluster(self): - self.test_delta_comments() - - @use_profile("databricks_sql_endpoint") - def test_delta_comments_databricks_sql_endpoint(self): - self.test_delta_comments() + self._test_delta_comments() @use_profile("databricks_uc_sql_endpoint") def test_delta_comments_databricks_uc_sql_endpoint(self): - self.test_delta_comments() + self._test_delta_comments() diff --git a/tests/integration/python/test_python_autostart.py b/tests/integration/python/test_python_autostart.py index 20c48d6c..c41cce3f 100644 --- a/tests/integration/python/test_python_autostart.py +++ b/tests/integration/python/test_python_autostart.py @@ -28,10 +28,6 @@ def project_config(self): def python_exc(self): self.run_dbt(["run"]) - @use_profile("databricks_sql_endpoint") - def test_python_databricks_sql_endpoint(self): - self.python_exc() - @use_profile("databricks_uc_sql_endpoint") def test_python_databricks_uc_sql_endpoint(self): self.use_default_project( diff --git a/tests/integration/run_results_json/test_run_results_json.py b/tests/integration/run_results_json/test_run_results_json.py index e77d4377..38f94864 100644 --- a/tests/integration/run_results_json/test_run_results_json.py +++ b/tests/integration/run_results_json/test_run_results_json.py @@ -40,10 +40,6 @@ def run_and_check_for_query_id(self): "Query ID column was not written to run_results.json", ) - @use_profile("databricks_sql_endpoint") - def test_run_results_json_databricks_sql_endpoint(self): - self.run_and_check_for_query_id() - @use_profile("databricks_uc_sql_endpoint") def test_run_results_json_databricks_uc_sql_endpoint(self): self.run_and_check_for_query_id() diff --git a/tests/integration/seed_column_types/test_seed_column_types.py b/tests/integration/seed_column_types/test_seed_column_types.py index 3a9ad9fd..65f55f26 100644 --- a/tests/integration/seed_column_types/test_seed_column_types.py +++ b/tests/integration/seed_column_types/test_seed_column_types.py @@ -26,10 +26,6 @@ def test_seed_column_types_databricks_cluster(self): def test_seed_column_types_databricks_uc_cluster(self): self.run_dbt(["seed"]) - @use_profile("databricks_sql_endpoint") - def test_seed_column_types_databricks_sql_endpoint(self): - self.run_dbt(["seed"]) - @use_profile("databricks_uc_sql_endpoint") def test_seed_column_types_databricks_uc_sql_endpoint(self): self.run_dbt(["seed"]) diff --git a/tests/integration/set_tblproperties/test_set_tblproperties.py b/tests/integration/set_tblproperties/test_set_tblproperties.py index 1c65f2e4..91937464 100644 --- a/tests/integration/set_tblproperties/test_set_tblproperties.py +++ b/tests/integration/set_tblproperties/test_set_tblproperties.py @@ -27,7 +27,7 @@ def check_snapshot_results(self, num_rows: int): results = self.run_sql("select * from {database_schema}.my_snapshot", fetch="all") self.assertEqual(len(results), num_rows) - def test_set_tblproperties(self): + def _test_set_tblproperties(self): self.run_dbt(["seed"]) self.run_dbt(["run"]) self.run_dbt(["run"]) @@ -48,16 +48,12 @@ def test_set_tblproperties(self): @use_profile("databricks_cluster") def test_set_tblproperties_databricks_cluster(self): - self.test_set_tblproperties() + self._test_set_tblproperties() @use_profile("databricks_uc_cluster") def test_set_tblproperties_databricks_uc_cluster(self): - self.test_set_tblproperties() - - @use_profile("databricks_sql_endpoint") - def test_set_tblproperties_databricks_sql_endpoint(self): - self.test_set_tblproperties() + self._test_set_tblproperties() @use_profile("databricks_uc_sql_endpoint") def test_set_tblproperties_databricks_uc_sql_endpoint(self): - self.test_set_tblproperties() + self._test_set_tblproperties() diff --git a/tests/integration/store_failures/test_store_failures.py b/tests/integration/store_failures/test_store_failures.py index ee09646f..16f2748a 100644 --- a/tests/integration/store_failures/test_store_failures.py +++ b/tests/integration/store_failures/test_store_failures.py @@ -20,7 +20,7 @@ def project_config(self): }, } - def test_store_failures(self): + def _test_store_failures(self): self.run_dbt(["run"]) self.run_dbt(["test", "--store-failures"]) @@ -28,19 +28,15 @@ def test_store_failures(self): class TestStoreFailuresDelta(TestStoreFailures): @use_profile("databricks_cluster") def test_store_failures_databricks_cluster(self): - self.test_store_failures() + self._test_store_failures() @use_profile("databricks_uc_cluster") def test_store_failures_databricks_uc_cluster(self): - self.test_store_failures() - - @use_profile("databricks_sql_endpoint") - def test_store_failures_databricks_sql_endpoint(self): - self.test_store_failures() + self._test_store_failures() @use_profile("databricks_uc_sql_endpoint") def test_store_failures_databricks_uc_sql_endpoint(self): - self.test_store_failures() + self._test_store_failures() class TestStoreFailuresParquet(TestStoreFailures): @@ -57,4 +53,4 @@ def project_config(self): @use_profile("databricks_cluster") def test_store_failures_databricks_cluster(self): - self.test_store_failures() + self._test_store_failures() diff --git a/tests/integration/streaming_table/test_streaming_table.py b/tests/integration/streaming_table/test_streaming_table.py index e89cbdf7..35ef93b0 100644 --- a/tests/integration/streaming_table/test_streaming_table.py +++ b/tests/integration/streaming_table/test_streaming_table.py @@ -12,7 +12,7 @@ def schema(self): def models(self): return "models" - def test_streaming_table_base(self): + def _test_streaming_table_base(self): self.run_dbt(["seed"]) self.run_dbt(["run", "--select", "+st"]) @@ -30,7 +30,7 @@ def test_streaming_table_base(self): self.run_dbt(["run", "--full-refresh", "--select", "st"]) self.assertTablesEqual("st", "expected2") - def test_streaming_table_no_cdf(self): + def _test_streaming_table_no_cdf(self): # The base table is not CDF. result = self.run_dbt(["run", "--select", "+st_nocdf"], expect_pass=True) self.run_dbt(["run", "--select", "base"]) @@ -43,7 +43,7 @@ def test_streaming_table_no_cdf(self): and "The input for materialized view must have Change Data Feed enabled." in res.message ) - def test_streaming_table_based_on_view(self): + def _test_streaming_table_based_on_view(self): result = self.run_dbt(["run", "--select", "+st_on_view"], expect_pass=False) assert len(result.results) == 3 res: RunResult = result.results[2] @@ -56,14 +56,14 @@ def test_streaming_table_based_on_view(self): @pytest.mark.skip(reason="not yet ready for production") @use_profile("databricks_uc_sql_endpoint") def test_streaming_table_base_databricks_uc_sql_endpoint(self): - self.test_streaming_table_base() + self._test_streaming_table_base() @pytest.mark.skip(reason="not yet ready for production") @use_profile("databricks_uc_sql_endpoint") def test_streaming_table_no_cdf_databricks_uc_sql_endpoint(self): - self.test_streaming_table_no_cdf() + self._test_streaming_table_no_cdf() @pytest.mark.skip(reason="not yet ready for production") @use_profile("databricks_uc_sql_endpoint") def test_streaming_table_based_on_view_databricks_uc_sql_endpoint(self): - self.test_streaming_table_based_on_view() + self._test_streaming_table_based_on_view() diff --git a/tests/integration/zorder/test_zorder.py b/tests/integration/zorder/test_zorder.py index 982d203f..aa468b22 100644 --- a/tests/integration/zorder/test_zorder.py +++ b/tests/integration/zorder/test_zorder.py @@ -14,22 +14,18 @@ def models(self): def project_config(self): return {"config-version": 2} - def test_zorder(self): + def _test_zorder(self): self.run_dbt(["run"]) self.run_dbt(["run"]) # make sure it also run in incremental @use_profile("databricks_cluster") def test_zorder_databricks_cluster(self): - self.test_zorder() + self._test_zorder() @use_profile("databricks_uc_cluster") def test_zorder_databricks_uc_cluster(self): - self.test_zorder() - - @use_profile("databricks_sql_endpoint") - def test_zorder_databricks_sql_endpoint(self): - self.test_zorder() + self._test_zorder() @use_profile("databricks_uc_sql_endpoint") def test_zorder_databricks_uc_sql_endpoint(self): - self.test_zorder() + self._test_zorder() diff --git a/tests/profiles.py b/tests/profiles.py index 3cfdac9c..73c660ff 100644 --- a/tests/profiles.py +++ b/tests/profiles.py @@ -5,8 +5,6 @@ def get_databricks_cluster_target(profile_type: str): if profile_type == "databricks_cluster": return databricks_cluster_target() - elif profile_type == "databricks_sql_endpoint": - return databricks_sql_endpoint_target() elif profile_type == "databricks_uc_cluster": return databricks_uc_cluster_target() elif profile_type == "databricks_uc_sql_endpoint": @@ -47,14 +45,6 @@ def databricks_cluster_target(): ) -def databricks_sql_endpoint_target(): - return _build_databricks_cluster_target( - http_path=os.getenv( - "DBT_DATABRICKS_ENDPOINT_HTTP_PATH", os.getenv("DBT_DATABRICKS_HTTP_PATH") - ) - ) - - def databricks_uc_cluster_target(): return _build_databricks_cluster_target( http_path=os.getenv( @@ -67,7 +57,8 @@ def databricks_uc_cluster_target(): def databricks_uc_sql_endpoint_target(): return _build_databricks_cluster_target( http_path=os.getenv( - "DBT_DATABRICKS_UC_ENDPOINT_HTTP_PATH", os.getenv("DBT_DATABRICKS_HTTP_PATH") + "DBT_DATABRICKS_UC_ENDPOINT_HTTP_PATH", + os.getenv("DBT_DATABRICKS_HTTP_PATH"), ), catalog=os.getenv("DBT_DATABRICKS_UC_INITIAL_CATALOG", "main"), ) diff --git a/tox.ini b/tox.ini index a13588b8..5adee10a 100644 --- a/tox.ini +++ b/tox.ini @@ -60,19 +60,6 @@ deps = -r{toxinidir}/requirements.txt allowlist_externals = /bin/bash -[testenv:integration-databricks-sql-endpoint] -basepython = python3 -commands = - /bin/bash -c '{envpython} -m pytest -v -m profile_databricks_sql_endpoint -n4 tests/integration/* {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' - /bin/bash -c '{envpython} -m pytest -v --profile databricks_sql_endpoint -n4 tests/functional/adapter/* {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -passenv = - DBT_* - PYTEST_ADDOPTS -deps = - -r{toxinidir}/dev-requirements.txt - -r{toxinidir}/requirements.txt -allowlist_externals = /bin/bash - [testenv:integration-databricks-uc-sql-endpoint] basepython = python3 commands =