From 131ee9c7c8b26a66fc7bf682ee2826a78737dcd3 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Sat, 5 Mar 2022 10:31:35 -0700 Subject: [PATCH 01/13] optional table_pattern on generate_source.sql --- macros/generate_source.sql | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/macros/generate_source.sql b/macros/generate_source.sql index 9bd6bca..0da8848 100644 --- a/macros/generate_source.sql +++ b/macros/generate_source.sql @@ -1,10 +1,19 @@ -{% macro get_tables_in_schema(schema_name, database_name=target.database) %} - - {% set tables=dbt_utils.get_relations_by_pattern( +{% macro get_tables_in_schema(schema_name, database_name=target.database, table_pattern='') %} + + {% if table_pattern != '' %} + {% set tables=dbt_utils.get_relations_by_pattern( + database=database_name, + schema_pattern=schema_name, + table_pattern=table_pattern + ) %} + {% else %} + {% set tables=dbt_utils.get_relations_by_pattern( database=database_name, schema_pattern=schema_name, table_pattern='%' - ) %} + ) %} + + {% endif %} {% set table_list= tables | map(attribute='identifier') %} @@ -14,7 +23,7 @@ --- -{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False) %} +{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, table_pattern='') %} {% set sources_yaml=[] %} @@ -29,7 +38,7 @@ {% do sources_yaml.append(' tables:') %} -{% set tables=codegen.get_tables_in_schema(schema_name, database_name) %} +{% set tables=codegen.get_tables_in_schema(schema_name, database_name, table_pattern) %} {% for table in tables %} {% do sources_yaml.append(' - name: ' ~ table | lower ) %} From d52b393a7d30883435242db0d2e849d20254ea65 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 7 Mar 2022 17:23:31 -0700 Subject: [PATCH 02/13] expose table_pattern config --- macros/generate_source.sql | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/macros/generate_source.sql b/macros/generate_source.sql index 0da8848..feb6f34 100644 --- a/macros/generate_source.sql +++ b/macros/generate_source.sql @@ -1,19 +1,10 @@ -{% macro get_tables_in_schema(schema_name, database_name=target.database, table_pattern='') %} +{% macro get_tables_in_schema(schema_name, table_pattern, database_name=target.database) %} - {% if table_pattern != '' %} - {% set tables=dbt_utils.get_relations_by_pattern( - database=database_name, - schema_pattern=schema_name, - table_pattern=table_pattern - ) %} - {% else %} - {% set tables=dbt_utils.get_relations_by_pattern( - database=database_name, - schema_pattern=schema_name, - table_pattern='%' - ) %} - - {% endif %} + {% set tables=dbt_utils.get_relations_by_pattern( + database=database_name, + schema_pattern=schema_name, + table_pattern=table_pattern + ) %} {% set table_list= tables | map(attribute='identifier') %} @@ -23,7 +14,7 @@ --- -{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, table_pattern='') %} +{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, table_pattern='%') %} {% set sources_yaml=[] %} From 8d15d6597c246dad64bff00bde511a2cdbdb478f Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 7 Mar 2022 17:27:09 -0700 Subject: [PATCH 03/13] fix argument order error --- macros/generate_source.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/generate_source.sql b/macros/generate_source.sql index feb6f34..a703983 100644 --- a/macros/generate_source.sql +++ b/macros/generate_source.sql @@ -29,7 +29,7 @@ {% do sources_yaml.append(' tables:') %} -{% set tables=codegen.get_tables_in_schema(schema_name, database_name, table_pattern) %} +{% set tables=codegen.get_tables_in_schema(schema_name, table_pattern, database_name) %} {% for table in tables %} {% do sources_yaml.append(' - name: ' ~ table | lower ) %} From 152d19f47ab47aa45ef5de308e8991c4d260646b Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Wed, 9 Mar 2022 09:31:55 -0700 Subject: [PATCH 04/13] update changelog --- CHANGELOG.md | 8 +++++++- macros/generate_source.sql | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31f148f..eb53603 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ -# dbt-codegen 0.5.0 +# dbt-codegen v0.5.1 +#TODO: confirm minor vs. patch version + +## Features +- add optional `table_pattern` argument to `generate_source.sql` macro. Default value is '%' to pull all tables in the raw data schema to preserve existing behavior if the `table_pattern` argument is not specified by the user. + +# dbt-codegen v0.5.0 This release supports any version (minor and patch) of v1, which means far less need for compatibility releases in the future. diff --git a/macros/generate_source.sql b/macros/generate_source.sql index a703983..532a593 100644 --- a/macros/generate_source.sql +++ b/macros/generate_source.sql @@ -14,7 +14,7 @@ --- -{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, table_pattern='%') %} +{% macro generate_source(schema_name, table_pattern='%', database_name=target.database, generate_columns=False, include_descriptions=False) %} {% set sources_yaml=[] %} From 4e3454de68bf03731c9ea35c3519b6659c6e54d3 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Wed, 9 Mar 2022 09:33:59 -0700 Subject: [PATCH 05/13] update readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7be3bc8..9db8b48 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ which you can then paste into a schema file. ### Arguments * `schema_name` (required): The schema name that contains your source data +* `table_pattern` (optional, default='%'): A table prefix / postfix that you +want to subselect from all available tables within a given schema. * `database_name` (optional, default=target.database): The database that your source data is in. * `generate_columns` (optional, default=False): Whether you want to add the From 0f14d1e95257ae29774fc8ce94a6db6625b0562c Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Wed, 9 Mar 2022 10:25:59 -0700 Subject: [PATCH 06/13] adding integration test --- .../tests/test_generate_source_all_args.sql | 1 + .../test_generate_source_table_pattern.sql | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 integration_tests/tests/test_generate_source_table_pattern.sql diff --git a/integration_tests/tests/test_generate_source_all_args.sql b/integration_tests/tests/test_generate_source_all_args.sql index f0c7967..60b6b52 100644 --- a/integration_tests/tests/test_generate_source_all_args.sql +++ b/integration_tests/tests/test_generate_source_all_args.sql @@ -3,6 +3,7 @@ -- test all args {% set actual_source_yaml = codegen.generate_source( schema_name=raw_schema, + table_pattern='%', database_name=target.database, generate_columns=True, include_descriptions=True diff --git a/integration_tests/tests/test_generate_source_table_pattern.sql b/integration_tests/tests/test_generate_source_table_pattern.sql new file mode 100644 index 0000000..4a47945 --- /dev/null +++ b/integration_tests/tests/test_generate_source_table_pattern.sql @@ -0,0 +1,17 @@ + +{% set raw_schema = generate_schema_name('raw_data') %} + +-- test default args +{% set actual_source_yaml = codegen.generate_source(raw_schema, table_pattern='data__%') %} + +{% set expected_source_yaml %} +version: 2 + +sources: + - name: {{ raw_schema | trim }} + tables: + - name: data__a_relation +{% endset %} + + +{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} From bb1e5673090603fbbff734534a4bce1680f18996 Mon Sep 17 00:00:00 2001 From: matt-winkler <75497565+matt-winkler@users.noreply.github.com> Date: Mon, 14 Mar 2022 09:18:13 -0600 Subject: [PATCH 07/13] Update macros/generate_source.sql accepting Joel's suggestion. Co-authored-by: Joel Labes --- macros/generate_source.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/generate_source.sql b/macros/generate_source.sql index 532a593..6e553a5 100644 --- a/macros/generate_source.sql +++ b/macros/generate_source.sql @@ -1,4 +1,4 @@ -{% macro get_tables_in_schema(schema_name, table_pattern, database_name=target.database) %} +{% macro get_tables_in_schema(schema_name, table_pattern='%', database_name=target.database) %} {% set tables=dbt_utils.get_relations_by_pattern( database=database_name, From 16e158e89744f3ce7ae64d9869975f87b6089f96 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 14 Mar 2022 09:25:35 -0600 Subject: [PATCH 08/13] update changelog --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb53603..53f6c97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ -# dbt-codegen v0.5.1 -#TODO: confirm minor vs. patch version +# dbt-codegen v0.6.0 + +This release creates breaking changes to the `generate_source.sql` macro. ## Features - add optional `table_pattern` argument to `generate_source.sql` macro. Default value is '%' to pull all tables in the raw data schema to preserve existing behavior if the `table_pattern` argument is not specified by the user. From f12a55aeeebffd9df7347a93ac6777182d2a658d Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 14 Mar 2022 09:34:45 -0600 Subject: [PATCH 09/13] add new seed file --- integration_tests/seeds/data__b_relation.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 integration_tests/seeds/data__b_relation.csv diff --git a/integration_tests/seeds/data__b_relation.csv b/integration_tests/seeds/data__b_relation.csv new file mode 100644 index 0000000..9b9932e --- /dev/null +++ b/integration_tests/seeds/data__b_relation.csv @@ -0,0 +1,3 @@ +col_a,col_b +3,c +4,d \ No newline at end of file From 42bb9b649be016d3c47e6a5034115cb0c4d0bc6f Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 14 Mar 2022 09:41:38 -0600 Subject: [PATCH 10/13] update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9db8b48..3c2c077 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ which you can then paste into a schema file. * `schema_name` (required): The schema name that contains your source data * `table_pattern` (optional, default='%'): A table prefix / postfix that you want to subselect from all available tables within a given schema. +* `exclude` (optional, default=''): A string you want to exclude from the selection criteria * `database_name` (optional, default=target.database): The database that your source data is in. * `generate_columns` (optional, default=False): Whether you want to add the From 0566165d1004bfdc1e00e7373265091b7b11bc32 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 14 Mar 2022 09:59:15 -0600 Subject: [PATCH 11/13] update tests --- .../tests/test_generate_source.sql | 1 + .../tests/test_generate_source_all_args.sql | 7 +++++++ .../tests/test_generate_source_exclude.sql | 17 +++++++++++++++++ .../test_generate_source_table_pattern.sql | 4 ++-- macros/generate_source.sql | 12 +++++++----- 5 files changed, 34 insertions(+), 7 deletions(-) create mode 100644 integration_tests/tests/test_generate_source_exclude.sql diff --git a/integration_tests/tests/test_generate_source.sql b/integration_tests/tests/test_generate_source.sql index a41aed9..875d1f7 100644 --- a/integration_tests/tests/test_generate_source.sql +++ b/integration_tests/tests/test_generate_source.sql @@ -11,6 +11,7 @@ sources: - name: {{ raw_schema | trim }} tables: - name: data__a_relation + - name: data__b_relation {% endset %} diff --git a/integration_tests/tests/test_generate_source_all_args.sql b/integration_tests/tests/test_generate_source_all_args.sql index 60b6b52..d5401ce 100644 --- a/integration_tests/tests/test_generate_source_all_args.sql +++ b/integration_tests/tests/test_generate_source_all_args.sql @@ -4,6 +4,7 @@ {% set actual_source_yaml = codegen.generate_source( schema_name=raw_schema, table_pattern='%', + exclude='', database_name=target.database, generate_columns=True, include_descriptions=True @@ -23,6 +24,12 @@ sources: - name: col_b description: "" + - name: data__b_relation + - name: col_a + description: "" + - name: col_b + description: "" + {% endset %} {{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} diff --git a/integration_tests/tests/test_generate_source_exclude.sql b/integration_tests/tests/test_generate_source_exclude.sql new file mode 100644 index 0000000..7677628 --- /dev/null +++ b/integration_tests/tests/test_generate_source_exclude.sql @@ -0,0 +1,17 @@ + +{% set raw_schema = generate_schema_name('raw_data') %} + +-- test default args +{% set actual_source_yaml = codegen.generate_source(raw_schema, table_pattern='data__%', exclude='data__a_%') %} + +{% set expected_source_yaml %} +version: 2 + +sources: + - name: {{ raw_schema | trim }} + tables: + - name: data__b_relation +{% endset %} + + +{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} diff --git a/integration_tests/tests/test_generate_source_table_pattern.sql b/integration_tests/tests/test_generate_source_table_pattern.sql index 4a47945..823024f 100644 --- a/integration_tests/tests/test_generate_source_table_pattern.sql +++ b/integration_tests/tests/test_generate_source_table_pattern.sql @@ -2,7 +2,7 @@ {% set raw_schema = generate_schema_name('raw_data') %} -- test default args -{% set actual_source_yaml = codegen.generate_source(raw_schema, table_pattern='data__%') %} +{% set actual_source_yaml = codegen.generate_source(raw_schema, table_pattern='data__b_%') %} {% set expected_source_yaml %} version: 2 @@ -10,7 +10,7 @@ version: 2 sources: - name: {{ raw_schema | trim }} tables: - - name: data__a_relation + - name: data__b_relation {% endset %} diff --git a/macros/generate_source.sql b/macros/generate_source.sql index 532a593..2b142b4 100644 --- a/macros/generate_source.sql +++ b/macros/generate_source.sql @@ -1,9 +1,10 @@ -{% macro get_tables_in_schema(schema_name, table_pattern, database_name=target.database) %} +{% macro get_tables_in_schema(schema_name, database_name=target.database, table_pattern='%', exclude='') %} {% set tables=dbt_utils.get_relations_by_pattern( - database=database_name, schema_pattern=schema_name, - table_pattern=table_pattern + database=database_name, + table_pattern=table_pattern, + exclude=exclude ) %} {% set table_list= tables | map(attribute='identifier') %} @@ -14,10 +15,11 @@ --- -{% macro generate_source(schema_name, table_pattern='%', database_name=target.database, generate_columns=False, include_descriptions=False) %} +{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, table_pattern='%', exclude='') %} {% set sources_yaml=[] %} +{% do sources_yaml.append('') %} {% do sources_yaml.append('version: 2') %} {% do sources_yaml.append('') %} {% do sources_yaml.append('sources:') %} @@ -29,7 +31,7 @@ {% do sources_yaml.append(' tables:') %} -{% set tables=codegen.get_tables_in_schema(schema_name, table_pattern, database_name) %} +{% set tables=codegen.get_tables_in_schema(schema_name, database_name, table_pattern, exclude) %} {% for table in tables %} {% do sources_yaml.append(' - name: ' ~ table | lower ) %} From a6773ae5d79874bc6499a84de1bd5c2bbb2e79e4 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 14 Mar 2022 10:27:33 -0600 Subject: [PATCH 12/13] fix integration test --- integration_tests/tests/test_generate_source_all_args.sql | 2 ++ macros/generate_source.sql | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/tests/test_generate_source_all_args.sql b/integration_tests/tests/test_generate_source_all_args.sql index d5401ce..545ea29 100644 --- a/integration_tests/tests/test_generate_source_all_args.sql +++ b/integration_tests/tests/test_generate_source_all_args.sql @@ -12,6 +12,7 @@ {% set expected_source_yaml %} + version: 2 sources: @@ -25,6 +26,7 @@ sources: description: "" - name: data__b_relation + columns: - name: col_a description: "" - name: col_b diff --git a/macros/generate_source.sql b/macros/generate_source.sql index 2b142b4..75d4302 100644 --- a/macros/generate_source.sql +++ b/macros/generate_source.sql @@ -19,7 +19,6 @@ {% set sources_yaml=[] %} -{% do sources_yaml.append('') %} {% do sources_yaml.append('version: 2') %} {% do sources_yaml.append('') %} {% do sources_yaml.append('sources:') %} From f53e4558ce004b965026c34e69bf64f2a3330a3d Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 14 Mar 2022 10:34:43 -0600 Subject: [PATCH 13/13] update generate_source args in readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3c2c077..d53f75d 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,15 @@ which you can then paste into a schema file. ### Arguments * `schema_name` (required): The schema name that contains your source data -* `table_pattern` (optional, default='%'): A table prefix / postfix that you -want to subselect from all available tables within a given schema. -* `exclude` (optional, default=''): A string you want to exclude from the selection criteria * `database_name` (optional, default=target.database): The database that your source data is in. * `generate_columns` (optional, default=False): Whether you want to add the column names to your source definition. * `include_descriptions` (optional, default=False): Whether you want to add description placeholders to your source definition. +* `table_pattern` (optional, default='%'): A table prefix / postfix that you +want to subselect from all available tables within a given schema. +* `exclude` (optional, default=''): A string you want to exclude from the selection criteria ### Usage: 1. Copy the macro into a statement tab in the dbt Cloud IDE, or into an analysis file, and compile your code