From 4daa20d1c6fc81f0943237018a8450244dae388d Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 09:33:36 -0500 Subject: [PATCH 01/19] Update changelog --- CHANGELOG.md | 126 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 94 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c223dd6..aa37afb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,32 @@ -## New features -- `generate_model_yaml` with `upstream_descriptions=True` now reads from upstream sources in addition to models. +# dbt-codegen v0.12.0-b1 + +## What's Changed + +### Features + +- `generate_model_yaml` with `upstream_descriptions=True` now reads from upstream sources in addition to models.(#112)[https://github.com/dbt-labs/dbt-codegen/issues/112] +- `generate_source` now has options for case sensitivity in all fields ([#112](https://github.com/dbt-labs/dbt-codegen/issues/112)) + +### Fixes -## Fixes - Column `description` fields are now correctly escaped in `generate_model_yaml` ([#142](https://github.com/dbt-labs/dbt-codegen/issues/142)) -- Fix `generate_source` behavior of applying a lowercase function to all object names ([#112](https://github.com/dbt-labs/dbt-codegen/issues/112)) + +### Docs + +- Fixed `generate_source` documentation +- Rewrote the contributor README at `integration_tests/README.md` + +## New Contributors + +- @wircho made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/159 +- @yatsky made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/164 +- @pnadolny made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/168 +- @esegal made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/154 +- @gwenwindflower made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/163 + +**Full Changelog**: https://github.com/dbt-labs/dbt-codegen/compare/0.11.0...0.12.0-b1 + +## New features # dbt-codegen v0.11.0 @@ -13,6 +36,7 @@ and are lowercase to align with the dbt style guide. Scale & precision are **not** included. Previous logic for `generate_source` defaulted to `false` and the resulting data types were uppercase and included scale & precision ([#122](https://github.com/dbt-labs/dbt-codegen/pull/122)). [Dispatch](https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch) can be used to utilize the column data type formatting of previous versions. Namely, by adding this macro to your project: + ```sql {% macro default__data_type_format_source(column) %} {{ return(column.data_type | upper) }} @@ -20,43 +44,48 @@ and are lowercase to align with the dbt style guide. Scale & precision are **not ``` And then adding this within `dbt_project.yml`: + ```yaml dispatch: - macro_namespace: codegen - search_order: ['my_project', 'codegen'] + search_order: ["my_project", "codegen"] ``` ## What's Changed -* GitHub Action to add/remove triage labels as-needed by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/133 -* GitHub Action to close issues as stale as-needed by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/134 -* Update README.md by @cohms in https://github.com/dbt-labs/dbt-codegen/pull/129 -* Remove hard-coded values for database and schema by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/139 -* Instructions for the release process by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/137 -* Add `include_data_types` argument to `generate_model_yaml` macro by @linbug in https://github.com/dbt-labs/dbt-codegen/pull/122 + +- GitHub Action to add/remove triage labels as-needed by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/133 +- GitHub Action to close issues as stale as-needed by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/134 +- Update README.md by @cohms in https://github.com/dbt-labs/dbt-codegen/pull/129 +- Remove hard-coded values for database and schema by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/139 +- Instructions for the release process by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/137 +- Add `include_data_types` argument to `generate_model_yaml` macro by @linbug in https://github.com/dbt-labs/dbt-codegen/pull/122 ## New Contributors -* @cohms made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/129 -* @linbug made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/122 + +- @cohms made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/129 +- @linbug made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/122 **Full Changelog**: https://github.com/dbt-labs/dbt-codegen/compare/0.10.0...v0.10.0 # dbt-codegen v0.10.0 ## What's Changed -* added comments to verbose regex in generate_model_import_ctes by @graciegoheen in https://github.com/dbt-labs/dbt-codegen/pull/93 -* Feature/hackathon model generator by @fivetran-joemarkiewicz in https://github.com/dbt-labs/dbt-codegen/pull/83 -* Suggestion to include packages.yml example in README.md by @Maayan-s in https://github.com/dbt-labs/dbt-codegen/pull/77 -* Add include_data_types flag to generate_source macro by @GSokol in https://github.com/dbt-labs/dbt-codegen/pull/76 -* Expected result of nested struct in BigQuery by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/105 -* issue106/get_models helper macro by @erkanncelen in https://github.com/dbt-labs/dbt-codegen/pull/115 -* Feat/generate sources add database and schema by @jeremyholtzman in https://github.com/dbt-labs/dbt-codegen/pull/124 + +- added comments to verbose regex in generate_model_import_ctes by @graciegoheen in https://github.com/dbt-labs/dbt-codegen/pull/93 +- Feature/hackathon model generator by @fivetran-joemarkiewicz in https://github.com/dbt-labs/dbt-codegen/pull/83 +- Suggestion to include packages.yml example in README.md by @Maayan-s in https://github.com/dbt-labs/dbt-codegen/pull/77 +- Add include_data_types flag to generate_source macro by @GSokol in https://github.com/dbt-labs/dbt-codegen/pull/76 +- Expected result of nested struct in BigQuery by @dbeatty10 in https://github.com/dbt-labs/dbt-codegen/pull/105 +- issue106/get_models helper macro by @erkanncelen in https://github.com/dbt-labs/dbt-codegen/pull/115 +- Feat/generate sources add database and schema by @jeremyholtzman in https://github.com/dbt-labs/dbt-codegen/pull/124 ## New Contributors -* @fivetran-joemarkiewicz made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/83 -* @Maayan-s made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/77 -* @GSokol made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/76 -* @erkanncelen made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/115 -* @jeremyholtzman made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/124 + +- @fivetran-joemarkiewicz made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/83 +- @Maayan-s made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/77 +- @GSokol made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/76 +- @erkanncelen made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/115 +- @jeremyholtzman made their first contribution in https://github.com/dbt-labs/dbt-codegen/pull/124 **Full Changelog**: https://github.com/dbt-labs/dbt-codegen/compare/0.9.0...0.10.0 @@ -67,30 +96,40 @@ dispatch: # dbt-codegen v0.8.0 # Unreleased + ## Breaking changes + ## New features + ## Quality of life + - Now uses `print` instead of `log` to output the generated text into the console. This enables you to invoke dbt with the `--quiet` flag and directly pipe the codegen output into a new file, ending up with valid yaml ## Under the hood + ## Contributors: + - [@JorgenG](https://github.com/JorgenG) (#86) # dbt-codegen v0.7.0 ## 🚨 Breaking change + - Add support for including description placeholders for the source and table, which changes the behavior of `generate_source` when `include_descriptions` is set to `True`. Previous logic only created description placeholders for the columns ([#64](https://github.com/dbt-labs/dbt-codegen/issues/64), [#66](https://github.com/dbt-labs/dbt-codegen/pull/66)) ## New features + - Add optional `table_names` arg to `generate_source` ([#50](https://github.com/dbt-labs/dbt-codegen/issues/50), [#51](https://github.com/dbt-labs/dbt-codegen/pull/51)) - Add support for importing descriptions from columns with the same names in upstream models. It is available by setting the parameter `upstream_descriptions` to `True` in `generate_model_yaml` ([#61](https://github.com/dbt-labs/dbt-codegen/pull/61)) - Added `case_sensitive_cols` argument to `generate_base_model` macro ([#63](https://github.com/dbt-labs/dbt-codegen/pull/63)) - Add optional `name` arg to `generate_source` ([#64](https://github.com/dbt-labs/dbt-codegen/issues/64), [#66](https://github.com/dbt-labs/dbt-codegen/pull/66)) ## Fixes + - `generate_model_yaml` now correctly handles nested `STRUCT` fields in BigQuery ([#27](https://github.com/dbt-labs/dbt-codegen/issues/27), [#54](https://github.com/dbt-labs/dbt-codegen/pull/54)) ## Contributors: + - [@rahulj51](https://github.com/rahulj51) (#51) - [@bodschut](https://github.com/bodschut) (#54) - [@b-per](https://github.com/b-per) (#61) @@ -102,6 +141,7 @@ dispatch: This release creates breaking changes to the `generate_source.sql` macro. ## Features + - add optional `table_pattern` argument to `generate_source.sql` macro. Default value is '%' to pull all tables in the raw data schema to preserve existing behavior if the `table_pattern` argument is not specified by the user. # dbt-codegen v0.5.0 @@ -109,6 +149,7 @@ This release creates breaking changes to the `generate_source.sql` macro. This release supports any version (minor and patch) of v1, which means far less need for compatibility releases in the future. ## Under the hood + - Change `require-dbt-version` to `[">=1.0.0", "<2.0.0"]` - Bump dbt-utils dependency - Replace `source-paths` and `data-paths` with `model-paths` and `seed-paths` respectively @@ -116,22 +157,27 @@ This release supports any version (minor and patch) of v1, which means far less - Replace `dbt_modules` with `dbt_packages` in `clean-targets` # dbt-codegen v0.4.1 + 🚨 This is a compatibility release in preparation for `dbt-core` v1.0.0 (🎉). Projects using this version with `dbt-core` v1.0.x can expect to see a deprecation warning. This will be resolved in the next minor release. # dbt-codegen v0.4.0 ## Breaking changes + - Requires `dbt>=0.20.0` and `dbt-utils>=0.7.0` - Depends on `dbt-labs/dbt_utils` (instead of `fishtown-analytics/dbt_utils`) ## Features + - Add optional `leading_commas` arg to `generate_base_model` (#41 @jaypeedevlin) - Add optional `include_descriptions` arg to `generate_source` (#40 @djbelknapdbs) ## Fixes + - In the `generate_source` macro, use `dbt_utils.get_relations_by_pattern` instead of `get_relations_by_prefix`, since the latter will be deprecated in the future (#42) ## Under the hood + - Use new adapter.dispatch syntax (#44) # dbt-codegen v0.3.2 @@ -139,57 +185,73 @@ This release supports any version (minor and patch) of v1, which means far less This is a quality of life release ## Other -* Fix rendering issues on hub.getdbt.com -* Fix integration tests due to python version compatibility + +- Fix rendering issues on hub.getdbt.com +- Fix integration tests due to python version compatibility # dbt-codegen v0.3.1 + This is a bugfix release ## Fixes + - Use latest version of dbt-utils (0.6.2) to ensure generate_source_yaml works for non-target schemata (#34) # dbt-codegen v0.3.0 -## 🚨 Breaking change + +## 🚨 Breaking change + This release requires dbt v0.18.0, and dbt-utils v0.6.1. If you're not ready to upgrade, consider using a previous release of this package. ## Quality of life + - Use dbt v0.18.0 (#31) -- Fix README rendering on hub (#32 @calvingiles) +- Fix README rendering on hub (#32 @calvingiles) # dbt-codegen v0.2.0 + ## 🚨 Breaking change + The lower bound of `dbt-utils` is now `0.4.0`. This won't affect most users, since you're likely already using version of dbt-utils higher than this to achieve 0.17.0 compatibility. ## Quality of life: + - Change dbt-utils dependencies to `[>=0.4.0, <0.6.0]` (#29) - Fix tests (#29) # dbt-codegen v0.1.0 + ## 🚨 Breaking change! This package now requires dbt v0.17.x! ## Features: -* Add `generate_model_yaml` (#18 @jtalmi) +- Add `generate_model_yaml` (#18 @jtalmi) ## Under the hood: -* Update to v0.17.0, including `dbt_project.yml` version 2 syntax (#23) -* Add GitHub templates and installation instructions (#23) + +- Update to v0.17.0, including `dbt_project.yml` version 2 syntax (#23) +- Add GitHub templates and installation instructions (#23) ## Acknowledgements + @marzaccaro made a PR for `generate_model_yaml`, and, although I had reviewed it, I let the PR go stale and somehow completely forgot about it when merging PR #18 — this is completely my bad! So equal credit to @marzaccaro and @jtalmi for their work :clap: # dbt-codegen v0.0.4 + This is a bugfix release to improve compatibility with Snowflake # dbt-codegen v0.0.3 + Bump utils version range # dbt-codegen v0.0.2 + Small quality of life improvements # dbt-codegen v0.0.1 + Initial release From bf7338f8e2bd00d6494ea4ded378f4d6d3cf9f0d Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 10:44:46 -0500 Subject: [PATCH 02/19] Fix typo in CHANGELOG --- CHANGELOG.md | 2 +- integration_tests/dbt_project.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa37afb..8f8658e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Features - `generate_model_yaml` with `upstream_descriptions=True` now reads from upstream sources in addition to models.(#112)[https://github.com/dbt-labs/dbt-codegen/issues/112] -- `generate_source` now has options for case sensitivity in all fields ([#112](https://github.com/dbt-labs/dbt-codegen/issues/112)) +- `generate_source` now has options for case sensitivity in all fields ([#112](https://github.com/dbt-labs/dbt-codegen/issues/112) ### Fixes diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 166f04e..c0a6873 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -27,4 +27,3 @@ vars: models: +bind: false - From acb92ea9f94517e5c2d2b0055e4e72244f894757 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 10:58:59 -0500 Subject: [PATCH 03/19] Attempt to fix integration test case sensitivity for redshift --- .../test_generate_source_case_sensitive.sql | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/integration_tests/tests/test_generate_source_case_sensitive.sql b/integration_tests/tests/test_generate_source_case_sensitive.sql index 48f3d69..b7afbfb 100644 --- a/integration_tests/tests/test_generate_source_case_sensitive.sql +++ b/integration_tests/tests/test_generate_source_case_sensitive.sql @@ -19,9 +19,21 @@ version: 2 sources: - - name: codegen_integration_tests_postgres_raw_data_case_sensitive +{% if target.database == 'redshift' -%} + - name: {{ raw_schema | trim | lower }} + database: {{ target.database | trim | lower }} + schema: {{ raw_schema | trim }} + tables: + - name: data__Case_Sensitive + columns: + - name: Col_A + data_type: integer + - name: Col_B + data_type: text +{%- else -%} +- name: {{ raw_schema | trim | lower }} database: circle_test - schema: codegen_integration_tests_postgres_Raw_Data_Case_Sensitive + schema: {{ raw_schema | trim }} tables: - name: data__Case_Sensitive columns: @@ -29,6 +41,7 @@ sources: data_type: integer - name: Col_B data_type: text +{%- endif -%} {% endset %} From b5992374854c0c07961b5c9530d0df579ce1350f Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 11:05:51 -0500 Subject: [PATCH 04/19] Always lower on Redshift --- .../tests/test_generate_source_case_sensitive.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/tests/test_generate_source_case_sensitive.sql b/integration_tests/tests/test_generate_source_case_sensitive.sql index b7afbfb..527add0 100644 --- a/integration_tests/tests/test_generate_source_case_sensitive.sql +++ b/integration_tests/tests/test_generate_source_case_sensitive.sql @@ -24,11 +24,11 @@ sources: database: {{ target.database | trim | lower }} schema: {{ raw_schema | trim }} tables: - - name: data__Case_Sensitive + - name: data__case_sensitive columns: - - name: Col_A + - name: col_a data_type: integer - - name: Col_B + - name: col_b data_type: text {%- else -%} - name: {{ raw_schema | trim | lower }} From 66100a53eaf12fa134be82aa7da725424a117672 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 11:11:18 -0500 Subject: [PATCH 05/19] Fix indentation --- integration_tests/tests/test_generate_source_case_sensitive.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/tests/test_generate_source_case_sensitive.sql b/integration_tests/tests/test_generate_source_case_sensitive.sql index 527add0..9be744f 100644 --- a/integration_tests/tests/test_generate_source_case_sensitive.sql +++ b/integration_tests/tests/test_generate_source_case_sensitive.sql @@ -31,7 +31,7 @@ sources: - name: col_b data_type: text {%- else -%} -- name: {{ raw_schema | trim | lower }} + - name: {{ raw_schema | trim | lower }} database: circle_test schema: {{ raw_schema | trim }} tables: From c314e809bf8a75eb17f47c6003aa164bf37cae6a Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 11:13:54 -0500 Subject: [PATCH 06/19] Fix whitespace --- integration_tests/tests/test_generate_source_case_sensitive.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/tests/test_generate_source_case_sensitive.sql b/integration_tests/tests/test_generate_source_case_sensitive.sql index 9be744f..03f35d4 100644 --- a/integration_tests/tests/test_generate_source_case_sensitive.sql +++ b/integration_tests/tests/test_generate_source_case_sensitive.sql @@ -30,7 +30,7 @@ sources: data_type: integer - name: col_b data_type: text -{%- else -%} +{%- else %} - name: {{ raw_schema | trim | lower }} database: circle_test schema: {{ raw_schema | trim }} From 61010eb330e15f910f9ea72ca33c253cef378ad4 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 11:16:39 -0500 Subject: [PATCH 07/19] Fix whitespace pt 2 --- .../tests/test_generate_source_case_sensitive.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration_tests/tests/test_generate_source_case_sensitive.sql b/integration_tests/tests/test_generate_source_case_sensitive.sql index 03f35d4..78ee8ce 100644 --- a/integration_tests/tests/test_generate_source_case_sensitive.sql +++ b/integration_tests/tests/test_generate_source_case_sensitive.sql @@ -18,8 +18,8 @@ {% set expected_source_yaml %} version: 2 -sources: {% if target.database == 'redshift' -%} +sources: - name: {{ raw_schema | trim | lower }} database: {{ target.database | trim | lower }} schema: {{ raw_schema | trim }} @@ -30,7 +30,8 @@ sources: data_type: integer - name: col_b data_type: text -{%- else %} +{%- else -%} +sources: - name: {{ raw_schema | trim | lower }} database: circle_test schema: {{ raw_schema | trim }} From 2587996d15f233766c22ab314a66476c1de2e7b9 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 11:24:29 -0500 Subject: [PATCH 08/19] Update case sensitive seeds into folder --- integration_tests/dbt_project.yml | 2 +- .../seeds/{ => case_sensitive}/data__Case_Sensitive.csv | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename integration_tests/seeds/{ => case_sensitive}/data__Case_Sensitive.csv (100%) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index c0a6873..92d951d 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -19,7 +19,7 @@ seeds: +schema: raw_data +quote_columns: false codegen_integration_tests: - data__Case_Sensitive: + case_sensitive: +schema: Raw_Data_Case_Sensitive quote_columns: true vars: diff --git a/integration_tests/seeds/data__Case_Sensitive.csv b/integration_tests/seeds/case_sensitive/data__Case_Sensitive.csv similarity index 100% rename from integration_tests/seeds/data__Case_Sensitive.csv rename to integration_tests/seeds/case_sensitive/data__Case_Sensitive.csv From 21c82d94d79099c05685e7379cc4d5ebc3c051fd Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 11:25:00 -0500 Subject: [PATCH 09/19] Use + for quote_columns config --- integration_tests/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 92d951d..8f9fac9 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -21,7 +21,7 @@ seeds: codegen_integration_tests: case_sensitive: +schema: Raw_Data_Case_Sensitive - quote_columns: true + +quote_columns: true vars: my_table_reference: table_c From 4236c1380ad640d6cd9fbd1dcf04d6ce58edc374 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 11:46:49 -0500 Subject: [PATCH 10/19] Lower schema on redshift --- integration_tests/dbt_project.yml | 2 +- integration_tests/tests/test_generate_source_case_sensitive.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 8f9fac9..d4d0679 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -20,7 +20,7 @@ seeds: +quote_columns: false codegen_integration_tests: case_sensitive: - +schema: Raw_Data_Case_Sensitive + +schema: "Raw_Data_Case_Sensitive" +quote_columns: true vars: my_table_reference: table_c diff --git a/integration_tests/tests/test_generate_source_case_sensitive.sql b/integration_tests/tests/test_generate_source_case_sensitive.sql index 78ee8ce..f3b2f67 100644 --- a/integration_tests/tests/test_generate_source_case_sensitive.sql +++ b/integration_tests/tests/test_generate_source_case_sensitive.sql @@ -22,7 +22,7 @@ version: 2 sources: - name: {{ raw_schema | trim | lower }} database: {{ target.database | trim | lower }} - schema: {{ raw_schema | trim }} + schema: {{ raw_schema | trim | lower }} tables: - name: data__case_sensitive columns: From a10038a6f047cef16309a486303b2538738e25a4 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 11:56:58 -0500 Subject: [PATCH 11/19] Use target.type --- integration_tests/tests/test_generate_source_case_sensitive.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/tests/test_generate_source_case_sensitive.sql b/integration_tests/tests/test_generate_source_case_sensitive.sql index f3b2f67..d6f1660 100644 --- a/integration_tests/tests/test_generate_source_case_sensitive.sql +++ b/integration_tests/tests/test_generate_source_case_sensitive.sql @@ -18,7 +18,7 @@ {% set expected_source_yaml %} version: 2 -{% if target.database == 'redshift' -%} +{% if target.type == 'redshift' -%} sources: - name: {{ raw_schema | trim | lower }} database: {{ target.database | trim | lower }} From 40cae7620f3ee919a09836dd8be5421f24e7e75d Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 12:08:08 -0500 Subject: [PATCH 12/19] Do some nonsense to make this work for Redshift --- README.md | 9 ++++----- integration_tests/dbt_project.yml | 5 +++++ .../data__case_sensitive_redshift.csv | 3 +++ .../tests/test_generate_source_case_sensitive.sql | 8 +++++--- 4 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 integration_tests/seeds/case_sensitive_redshift/data__case_sensitive_redshift.csv diff --git a/README.md b/README.md index 29066e4..ae1d8e0 100644 --- a/README.md +++ b/README.md @@ -70,14 +70,13 @@ which you can then paste into a schema file. - `include_schema` (optional, default=False): Whether you want to add the schema to your source definition - `case_sensitive_databases` (optional, default=False): Whether you want database names to be -in lowercase, or to match the case in the source table + in lowercase, or to match the case in the source table — not compatible with Redshift - `case_sensitive_schemas` (optional, default=False): Whether you want schema names to be -in lowercase, or to match the case in the source table + in lowercase, or to match the case in the source table — not compatible with Redshift - `case_sensitive_tables` (optional, default=False): Whether you want table names to be -in lowercase, or to match the case in the source table + in lowercase, or to match the case in the source table — not compatible with Redshift - `case_sensitive_cols` (optional, default=False): Whether you want column names to be -in lowercase, or to match the case in the source table - + in lowercase, or to match the case in the source table ### Outputting to a file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index d4d0679..a05f00d 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -22,6 +22,11 @@ seeds: case_sensitive: +schema: "Raw_Data_Case_Sensitive" +quote_columns: true + +enabled: {{ target.type != 'redshift' }} + case_sensitive_redshift: + +schema: "raw_data_case_sensitive" + +quote_columns: true + +enabled: {{ target.type == 'redshift' }} vars: my_table_reference: table_c diff --git a/integration_tests/seeds/case_sensitive_redshift/data__case_sensitive_redshift.csv b/integration_tests/seeds/case_sensitive_redshift/data__case_sensitive_redshift.csv new file mode 100644 index 0000000..d0f2e97 --- /dev/null +++ b/integration_tests/seeds/case_sensitive_redshift/data__case_sensitive_redshift.csv @@ -0,0 +1,3 @@ +Col_A,Col_B +1,a +2,b diff --git a/integration_tests/tests/test_generate_source_case_sensitive.sql b/integration_tests/tests/test_generate_source_case_sensitive.sql index d6f1660..4952868 100644 --- a/integration_tests/tests/test_generate_source_case_sensitive.sql +++ b/integration_tests/tests/test_generate_source_case_sensitive.sql @@ -1,7 +1,9 @@ +{% if target.type == 'redshift' %} + {% set raw_schema = generate_schema_name('raw_data_case_sensitive') %} +{% else %} + {% set raw_schema = generate_schema_name('Raw_Data_Case_Sensitive') %} +{% endif %} -{% set raw_schema = generate_schema_name('Raw_Data_Case_Sensitive') %} - --- test default args {% set actual_source_yaml = codegen.generate_source( schema_name=raw_schema, database_name=target.database, From e6d824b9cec6ad8643814f6751623c0064b4ca78 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 12:15:10 -0500 Subject: [PATCH 13/19] Move seeds config to properties.yml --- integration_tests/dbt_project.yml | 10 +--------- .../{case_sensitive => }/data__Case_Sensitive.csv | 0 .../data__case_sensitive_redshift.csv | 0 integration_tests/seeds/properties.yml | 14 ++++++++++++++ 4 files changed, 15 insertions(+), 9 deletions(-) rename integration_tests/seeds/{case_sensitive => }/data__Case_Sensitive.csv (100%) rename integration_tests/seeds/{case_sensitive_redshift => }/data__case_sensitive_redshift.csv (100%) create mode 100644 integration_tests/seeds/properties.yml diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index a05f00d..6f194d0 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -18,15 +18,7 @@ clean-targets: seeds: +schema: raw_data +quote_columns: false - codegen_integration_tests: - case_sensitive: - +schema: "Raw_Data_Case_Sensitive" - +quote_columns: true - +enabled: {{ target.type != 'redshift' }} - case_sensitive_redshift: - +schema: "raw_data_case_sensitive" - +quote_columns: true - +enabled: {{ target.type == 'redshift' }} + vars: my_table_reference: table_c diff --git a/integration_tests/seeds/case_sensitive/data__Case_Sensitive.csv b/integration_tests/seeds/data__Case_Sensitive.csv similarity index 100% rename from integration_tests/seeds/case_sensitive/data__Case_Sensitive.csv rename to integration_tests/seeds/data__Case_Sensitive.csv diff --git a/integration_tests/seeds/case_sensitive_redshift/data__case_sensitive_redshift.csv b/integration_tests/seeds/data__case_sensitive_redshift.csv similarity index 100% rename from integration_tests/seeds/case_sensitive_redshift/data__case_sensitive_redshift.csv rename to integration_tests/seeds/data__case_sensitive_redshift.csv diff --git a/integration_tests/seeds/properties.yml b/integration_tests/seeds/properties.yml new file mode 100644 index 0000000..cd34ce8 --- /dev/null +++ b/integration_tests/seeds/properties.yml @@ -0,0 +1,14 @@ +version: 2 + +seeds: + - name: data__Case_Sensitive + config: + quote_columns: true + enabled: {{ target.type != 'redshift' }} + schema: "Raw_Data_Case_Sensitive" + + - name: data__case_sensitive_redshift + config: + quote_columns: true + enabled: {{ target.type == 'redshift' }} + schema: raw_data_case_sensitive From dfda71fd02c886fc890186030ab3d1d6fa1deca8 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 12:40:08 -0500 Subject: [PATCH 14/19] Bypass redshift completely --- .../seeds/data__case_sensitive_redshift.csv | 3 --- integration_tests/seeds/properties.yml | 9 +------- .../test_generate_source_case_sensitive.sql | 21 +++---------------- 3 files changed, 4 insertions(+), 29 deletions(-) delete mode 100644 integration_tests/seeds/data__case_sensitive_redshift.csv diff --git a/integration_tests/seeds/data__case_sensitive_redshift.csv b/integration_tests/seeds/data__case_sensitive_redshift.csv deleted file mode 100644 index d0f2e97..0000000 --- a/integration_tests/seeds/data__case_sensitive_redshift.csv +++ /dev/null @@ -1,3 +0,0 @@ -Col_A,Col_B -1,a -2,b diff --git a/integration_tests/seeds/properties.yml b/integration_tests/seeds/properties.yml index cd34ce8..cb4689f 100644 --- a/integration_tests/seeds/properties.yml +++ b/integration_tests/seeds/properties.yml @@ -1,14 +1,7 @@ version: 2 seeds: - - name: data__Case_Sensitive + - name: "data__Case_Sensitive" config: quote_columns: true - enabled: {{ target.type != 'redshift' }} schema: "Raw_Data_Case_Sensitive" - - - name: data__case_sensitive_redshift - config: - quote_columns: true - enabled: {{ target.type == 'redshift' }} - schema: raw_data_case_sensitive diff --git a/integration_tests/tests/test_generate_source_case_sensitive.sql b/integration_tests/tests/test_generate_source_case_sensitive.sql index 4952868..68977e7 100644 --- a/integration_tests/tests/test_generate_source_case_sensitive.sql +++ b/integration_tests/tests/test_generate_source_case_sensitive.sql @@ -1,8 +1,7 @@ {% if target.type == 'redshift' %} - {% set raw_schema = generate_schema_name('raw_data_case_sensitive') %} + select 'ok' limit 0 {% else %} - {% set raw_schema = generate_schema_name('Raw_Data_Case_Sensitive') %} -{% endif %} +{% set raw_schema = generate_schema_name('Raw_Data_Case_Sensitive') %} {% set actual_source_yaml = codegen.generate_source( schema_name=raw_schema, @@ -20,19 +19,6 @@ {% set expected_source_yaml %} version: 2 -{% if target.type == 'redshift' -%} -sources: - - name: {{ raw_schema | trim | lower }} - database: {{ target.database | trim | lower }} - schema: {{ raw_schema | trim | lower }} - tables: - - name: data__case_sensitive - columns: - - name: col_a - data_type: integer - - name: col_b - data_type: text -{%- else -%} sources: - name: {{ raw_schema | trim | lower }} database: circle_test @@ -44,8 +30,7 @@ sources: data_type: integer - name: Col_B data_type: text -{%- endif -%} {% endset %} - {{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} +{%- endif -%} From a249c74bdebb3629a3e097cbbf82a8347d9a2b56 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 17 Apr 2024 12:57:02 -0500 Subject: [PATCH 15/19] Temporarily bypass Redshift in CI completely --- .circleci/config.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2585c60..96148c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,18 +43,18 @@ jobs: dbt --warn-error run --target postgres dbt --warn-error test --target postgres - - run: - name: "Run Tests - Redshift" - command: | - . dbt_venv/bin/activate - echo `pwd` - cd integration_tests - dbt --warn-error deps --target redshift - dbt --warn-error run-operation create_source_table --target redshift - dbt --warn-error seed --target redshift --full-refresh - dbt --warn-error run --target redshift - dbt --warn-error test --target redshift - + # - run: + # name: "Run Tests - Redshift" + # command: | + # . dbt_venv/bin/activate + # echo `pwd` + # cd integration_tests + # dbt --warn-error deps --target redshift + # dbt --warn-error run-operation create_source_table --target redshift + # dbt --warn-error seed --target redshift --full-refresh + # dbt --warn-error run --target redshift + # dbt --warn-error test --target redshift + # - run: name: "Run Tests - Snowflake" command: | From 41cb17e0435f0a4d2722b7b4a4e4cf067346a125 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Thu, 18 Apr 2024 13:05:01 -0500 Subject: [PATCH 16/19] Turn Redshift CI back on --- .circleci/config.yml | 24 +++++++++---------- .../test_generate_source_case_sensitive.sql | 19 +++++++++++---- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 96148c9..2585c60 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,18 +43,18 @@ jobs: dbt --warn-error run --target postgres dbt --warn-error test --target postgres - # - run: - # name: "Run Tests - Redshift" - # command: | - # . dbt_venv/bin/activate - # echo `pwd` - # cd integration_tests - # dbt --warn-error deps --target redshift - # dbt --warn-error run-operation create_source_table --target redshift - # dbt --warn-error seed --target redshift --full-refresh - # dbt --warn-error run --target redshift - # dbt --warn-error test --target redshift - # + - run: + name: "Run Tests - Redshift" + command: | + . dbt_venv/bin/activate + echo `pwd` + cd integration_tests + dbt --warn-error deps --target redshift + dbt --warn-error run-operation create_source_table --target redshift + dbt --warn-error seed --target redshift --full-refresh + dbt --warn-error run --target redshift + dbt --warn-error test --target redshift + - run: name: "Run Tests - Snowflake" command: | diff --git a/integration_tests/tests/test_generate_source_case_sensitive.sql b/integration_tests/tests/test_generate_source_case_sensitive.sql index 68977e7..f1e8e1b 100644 --- a/integration_tests/tests/test_generate_source_case_sensitive.sql +++ b/integration_tests/tests/test_generate_source_case_sensitive.sql @@ -1,6 +1,3 @@ -{% if target.type == 'redshift' %} - select 'ok' limit 0 -{% else %} {% set raw_schema = generate_schema_name('Raw_Data_Case_Sensitive') %} {% set actual_source_yaml = codegen.generate_source( @@ -19,8 +16,21 @@ {% set expected_source_yaml %} version: 2 +{% if target.type == 'redshift' -%} sources: - name: {{ raw_schema | trim | lower }} + database: {{ target.database | trim | lower }} + schema: {{ raw_schema | trim | lower }} + tables: + - name: data__case_sensitive + columns: + - name: col_a + data_type: integer + - name: col_b + data_type: text +{%- else -%} +sources: +- name: {{ raw_schema | trim | lower }} database: circle_test schema: {{ raw_schema | trim }} tables: @@ -30,7 +40,8 @@ sources: data_type: integer - name: Col_B data_type: text +{%- endif -%} {% endset %} + {{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} -{%- endif -%} From 7a0eb602d5b3c775aff8cddb1528c8890803ea05 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Tue, 4 Jun 2024 11:32:55 -0500 Subject: [PATCH 17/19] Delete case sensitivity test --- .../test_generate_source_case_sensitive.sql | 47 ------------------- macros/generate_source.sql | 2 +- 2 files changed, 1 insertion(+), 48 deletions(-) delete mode 100644 integration_tests/tests/test_generate_source_case_sensitive.sql diff --git a/integration_tests/tests/test_generate_source_case_sensitive.sql b/integration_tests/tests/test_generate_source_case_sensitive.sql deleted file mode 100644 index f1e8e1b..0000000 --- a/integration_tests/tests/test_generate_source_case_sensitive.sql +++ /dev/null @@ -1,47 +0,0 @@ -{% set raw_schema = generate_schema_name('Raw_Data_Case_Sensitive') %} - -{% set actual_source_yaml = codegen.generate_source( - schema_name=raw_schema, - database_name=target.database, - generate_columns=True, - name=raw_schema, - include_database=True, - include_schema=True, - case_sensitive_databases=True, - case_sensitive_schemas=True, - case_sensitive_tables=True, - case_sensitive_cols=True -) %} - -{% set expected_source_yaml %} -version: 2 - -{% if target.type == 'redshift' -%} -sources: - - name: {{ raw_schema | trim | lower }} - database: {{ target.database | trim | lower }} - schema: {{ raw_schema | trim | lower }} - tables: - - name: data__case_sensitive - columns: - - name: col_a - data_type: integer - - name: col_b - data_type: text -{%- else -%} -sources: -- name: {{ raw_schema | trim | lower }} - database: circle_test - schema: {{ raw_schema | trim }} - tables: - - name: data__Case_Sensitive - columns: - - name: Col_A - data_type: integer - - name: Col_B - data_type: text -{%- endif -%} -{% endset %} - - -{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} diff --git a/macros/generate_source.sql b/macros/generate_source.sql index f769911..b3790f2 100644 --- a/macros/generate_source.sql +++ b/macros/generate_source.sql @@ -1,5 +1,5 @@ {% macro get_tables_in_schema(schema_name, database_name=target.database, table_pattern='%', exclude='') %} - + {% set tables=dbt_utils.get_relations_by_pattern( schema_pattern=schema_name, database=database_name, From 772412a3505630a43f547a8fbdb095a37df51da3 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Tue, 4 Jun 2024 11:39:01 -0500 Subject: [PATCH 18/19] Delete case sensitive seed --- integration_tests/seeds/data__Case_Sensitive.csv | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 integration_tests/seeds/data__Case_Sensitive.csv diff --git a/integration_tests/seeds/data__Case_Sensitive.csv b/integration_tests/seeds/data__Case_Sensitive.csv deleted file mode 100644 index d0f2e97..0000000 --- a/integration_tests/seeds/data__Case_Sensitive.csv +++ /dev/null @@ -1,3 +0,0 @@ -Col_A,Col_B -1,a -2,b From f9003dc3331bfd9c334715b7473dfa82b967ab76 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Tue, 4 Jun 2024 11:45:02 -0500 Subject: [PATCH 19/19] Delete properties.yml for case sensitive seeds --- integration_tests/seeds/properties.yml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 integration_tests/seeds/properties.yml diff --git a/integration_tests/seeds/properties.yml b/integration_tests/seeds/properties.yml deleted file mode 100644 index cb4689f..0000000 --- a/integration_tests/seeds/properties.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: 2 - -seeds: - - name: "data__Case_Sensitive" - config: - quote_columns: true - schema: "Raw_Data_Case_Sensitive"