From a659ecedb1a37ccfb21175a2ff586b3c3fa3c7ab Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Wed, 25 Jan 2023 17:04:46 +0100 Subject: [PATCH 01/13] adding get_models helper macro --- macros/helpers/helpers.sql | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/macros/helpers/helpers.sql b/macros/helpers/helpers.sql index 262c7d9..fd28c87 100644 --- a/macros/helpers/helpers.sql +++ b/macros/helpers/helpers.sql @@ -26,4 +26,35 @@ {% endfor %} {{ return(glob_dict) }} {% endif %} +{% endmacro %} + +{# build a list of models looping through all models in the project #} +{# filter by directory or prefix arguments, if provided #} +{% macro get_models(directory=None, prefix=None) %} + {% set model_names=[] %} + {% set models = graph.nodes.values() | selectattr('resource_type', "equalto", 'model') %} + {% if directory and prefix %} + {% for model in models %} + {% set model_path = "/".join(model.path.split("/")[:-1]) %} + {% if model_path == directory and model.name.startswith(prefix) %} + {% do model_names.append(model.name) %} + {% endif %} + {% endfor %} + {% elif directory %} + {% for model in models %} + {% set model_path = "/".join(model.path.split("/")[:-1]) %} + {% if model_path == directory %} + {% do model_names.append(model.name) %} + {% endif %} + {% endfor %} + {% elif prefix %} + {% for model in models if model.name.startswith(prefix) %} + {% do model_names.append(model.name) %} + {% endfor %} + {% else %} + {% for model in models %} + {% do model_names.append(model.name) %} + {% endfor %} + {% endif %} + {{ return(model_names) }} {% endmacro %} \ No newline at end of file From e02b03d87b10c162db23510c6adfe163fc548939 Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Fri, 27 Jan 2023 16:45:17 +0100 Subject: [PATCH 02/13] test for prefix --- ...l_yaml_multiple_models_using_get_model.sql | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 integration_tests/tests/test_generate_model_yaml_multiple_models_using_get_model.sql diff --git a/integration_tests/tests/test_generate_model_yaml_multiple_models_using_get_model.sql b/integration_tests/tests/test_generate_model_yaml_multiple_models_using_get_model.sql new file mode 100644 index 0000000..ace8459 --- /dev/null +++ b/integration_tests/tests/test_generate_model_yaml_multiple_models_using_get_model.sql @@ -0,0 +1,21 @@ +{% set actual_model_yaml_prefix = codegen.generate_model_yaml( + model_names = codegen.get_models(prefix='child_') + ) +%} + +{% set expected_model_yaml_prefix %} +version: 2 + +models: + - name: child_model + description: "" + columns: + - name: col_a + description: "" + + - name: col_b + description: "" + +{% endset %} + +{{ assert_equal (actual_model_yaml_prefix | trim, expected_model_yaml_prefix | trim) }} From ed839e52674b1f1bae98209332bd60e07af4d0c2 Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Fri, 27 Jan 2023 16:54:46 +0100 Subject: [PATCH 03/13] readme and changelog --- CHANGELOG.md | 1 + README.md | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e297377..0e37590 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Addition of the [create_base_models](macros/create_base_models.sql) This macro generates a series of terminal commands (appended w) bash script which creates a new file in your dbt project based off the results of the [generate_base_model](macros/generate_base_model.sql) macro. Therefore, instead of outputting in the terminal, it will create the file for you. - Add `include_data_types` flag to `generate_source` macro ([#76](https://github.com/dbt-labs/dbt-codegen/pull/76)) +- Add `get_models` macro in helper macros. This macro retrieves a list of models with specified prefix at the specified directory. It is designed to make creating yamls for multiple models easier. ## Fixes - Fix handling of nested `STRUCT` fields in BigQuery ([#98](https://github.com/dbt-labs/dbt-codegen/issues/98), [#105](https://github.com/dbt-labs/dbt-codegen/pull/105)) diff --git a/README.md b/README.md index cf70350..e57ec7e 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,14 @@ schema.yml file. ) }} ``` +You can use the helper function codegen.get_models and specify a directory and/or prefix to get a list of all matching models, to be passed into model_names list. + +``` +{{ codegen.generate_model_yaml( + model_names= codegen.get_models(directory='marts','prefix='fct_') +) }} +``` + Alternatively, call the macro as an [operation](https://docs.getdbt.com/docs/using-operations): ``` From 9c3ca18479730ce6b3e6f8968473e447b6d362de Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Mon, 30 Jan 2023 12:07:03 +0100 Subject: [PATCH 04/13] integration tests --- ...l_yaml_multiple_models_using_get_model.sql | 21 ------------- ...st_generate_model_yaml_using_get_model.sql | 30 +++++++++++++++++++ 2 files changed, 30 insertions(+), 21 deletions(-) delete mode 100644 integration_tests/tests/test_generate_model_yaml_multiple_models_using_get_model.sql create mode 100644 integration_tests/tests/test_generate_model_yaml_using_get_model.sql diff --git a/integration_tests/tests/test_generate_model_yaml_multiple_models_using_get_model.sql b/integration_tests/tests/test_generate_model_yaml_multiple_models_using_get_model.sql deleted file mode 100644 index ace8459..0000000 --- a/integration_tests/tests/test_generate_model_yaml_multiple_models_using_get_model.sql +++ /dev/null @@ -1,21 +0,0 @@ -{% set actual_model_yaml_prefix = codegen.generate_model_yaml( - model_names = codegen.get_models(prefix='child_') - ) -%} - -{% set expected_model_yaml_prefix %} -version: 2 - -models: - - name: child_model - description: "" - columns: - - name: col_a - description: "" - - - name: col_b - description: "" - -{% endset %} - -{{ assert_equal (actual_model_yaml_prefix | trim, expected_model_yaml_prefix | trim) }} diff --git a/integration_tests/tests/test_generate_model_yaml_using_get_model.sql b/integration_tests/tests/test_generate_model_yaml_using_get_model.sql new file mode 100644 index 0000000..69988b5 --- /dev/null +++ b/integration_tests/tests/test_generate_model_yaml_using_get_model.sql @@ -0,0 +1,30 @@ +{% set actual_model_yaml = codegen.generate_model_yaml( + model_names= codegen.get_modelsd(prefix='data__') + ) +%} + +{% set expected_model_yaml %} +version: 2 + +models: + - name: data__a_relation + description: "" + columns: + - name: col_a + description: "" + + - name: col_b + description: "" + + - name: data__b_relation + description: "" + columns: + - name: col_a + description: "" + + - name: col_b + description: "" + +{% endset %} + +{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }} From e42281c892004b973c14947282379c99f93bc574 Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Mon, 30 Jan 2023 12:12:27 +0100 Subject: [PATCH 05/13] typo --- .../tests/test_generate_model_yaml_using_get_model.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/tests/test_generate_model_yaml_using_get_model.sql b/integration_tests/tests/test_generate_model_yaml_using_get_model.sql index 69988b5..32c45d5 100644 --- a/integration_tests/tests/test_generate_model_yaml_using_get_model.sql +++ b/integration_tests/tests/test_generate_model_yaml_using_get_model.sql @@ -1,5 +1,5 @@ {% set actual_model_yaml = codegen.generate_model_yaml( - model_names= codegen.get_modelsd(prefix='data__') + model_names= codegen.get_models(prefix='data__') ) %} From c6e7a98bcb96192deae145200375e737ac107601 Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Mon, 30 Jan 2023 13:33:19 +0100 Subject: [PATCH 06/13] fixing graph object issues in testing --- ...st_generate_model_yaml_using_get_model.sql | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/integration_tests/tests/test_generate_model_yaml_using_get_model.sql b/integration_tests/tests/test_generate_model_yaml_using_get_model.sql index 32c45d5..27c1955 100644 --- a/integration_tests/tests/test_generate_model_yaml_using_get_model.sql +++ b/integration_tests/tests/test_generate_model_yaml_using_get_model.sql @@ -1,22 +1,44 @@ +-- depends_on: {{ ref('model_data_a'),ref('model_struct'), }} +-- depends_on: {{ ref('model_struct') }} +-- depends_on: {{ ref('model_without_import_ctes') }} +-- depends_on: {{ ref('model_without_any_ctes') }} + +{% if execute %} {% set actual_model_yaml = codegen.generate_model_yaml( - model_names= codegen.get_models(prefix='data__') + model_names = codegen.get_models(prefix='model_') ) %} +{% endif %} {% set expected_model_yaml %} version: 2 models: - - name: data__a_relation + - name: model_without_import_ctes description: "" columns: - - name: col_a + - name: model_without_any_ctes + description: "" + columns: + - name: model_struct + description: "" + columns: + - name: analytics description: "" - - name: col_b + - name: analytics.source + description: "" + + - name: analytics.medium + description: "" + + - name: analytics.source_medium + description: "" + + - name: col_x description: "" - - name: data__b_relation + - name: model_data_a description: "" columns: - name: col_a From b0f85f9a2ec1d395e867b3c9e1317fc0787977a2 Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Mon, 30 Jan 2023 13:37:19 +0100 Subject: [PATCH 07/13] correcting expected test output --- ...st_generate_model_yaml_using_get_model.sql | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/integration_tests/tests/test_generate_model_yaml_using_get_model.sql b/integration_tests/tests/test_generate_model_yaml_using_get_model.sql index 27c1955..d192cb4 100644 --- a/integration_tests/tests/test_generate_model_yaml_using_get_model.sql +++ b/integration_tests/tests/test_generate_model_yaml_using_get_model.sql @@ -14,37 +14,49 @@ version: 2 models: - - name: model_without_import_ctes - description: "" - columns: - - name: model_without_any_ctes + - name: model_data_a description: "" columns: + - name: col_a + description: "" + + - name: col_b + description: "" + - name: model_struct description: "" columns: - name: analytics description: "" - - name: analytics.source + - name: source description: "" - - name: analytics.medium + - name: medium description: "" - - name: analytics.source_medium + - name: source_medium description: "" - name: col_x description: "" - - name: model_data_a + - name: model_without_any_ctes description: "" columns: + - name: id + description: "" + - name: col_a description: "" - - name: col_b + - name: col2 + description: "" + + - name: model_without_import_ctes + description: "" + columns: + - name: id description: "" {% endset %} From 0ab9d6ecac0f14476dad94c5589b31a784de4ff1 Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Mon, 30 Jan 2023 13:48:36 +0100 Subject: [PATCH 08/13] changing test logic to model list comparison --- ...st_generate_model_yaml_using_get_model.sql | 64 ------------------- .../tests/test_helper_get_models.sql | 14 ++++ 2 files changed, 14 insertions(+), 64 deletions(-) delete mode 100644 integration_tests/tests/test_generate_model_yaml_using_get_model.sql create mode 100644 integration_tests/tests/test_helper_get_models.sql diff --git a/integration_tests/tests/test_generate_model_yaml_using_get_model.sql b/integration_tests/tests/test_generate_model_yaml_using_get_model.sql deleted file mode 100644 index d192cb4..0000000 --- a/integration_tests/tests/test_generate_model_yaml_using_get_model.sql +++ /dev/null @@ -1,64 +0,0 @@ --- depends_on: {{ ref('model_data_a'),ref('model_struct'), }} --- depends_on: {{ ref('model_struct') }} --- depends_on: {{ ref('model_without_import_ctes') }} --- depends_on: {{ ref('model_without_any_ctes') }} - -{% if execute %} -{% set actual_model_yaml = codegen.generate_model_yaml( - model_names = codegen.get_models(prefix='model_') - ) -%} -{% endif %} - -{% set expected_model_yaml %} -version: 2 - -models: - - name: model_data_a - description: "" - columns: - - name: col_a - description: "" - - - name: col_b - description: "" - - - name: model_struct - description: "" - columns: - - name: analytics - description: "" - - - name: source - description: "" - - - name: medium - description: "" - - - name: source_medium - description: "" - - - name: col_x - description: "" - - - name: model_without_any_ctes - description: "" - columns: - - name: id - description: "" - - - name: col_a - description: "" - - - name: col2 - description: "" - - - name: model_without_import_ctes - description: "" - columns: - - name: id - description: "" - -{% endset %} - -{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }} diff --git a/integration_tests/tests/test_helper_get_models.sql b/integration_tests/tests/test_helper_get_models.sql new file mode 100644 index 0000000..d63b338 --- /dev/null +++ b/integration_tests/tests/test_helper_get_models.sql @@ -0,0 +1,14 @@ +-- depends_on: {{ ref('model_data_a') }} +-- depends_on: {{ ref('model_struct') }} +-- depends_on: {{ ref('model_without_import_ctes') }} +-- depends_on: {{ ref('model_without_any_ctes') }} + +{% if execute %} +{% set actual_list = codegen.get_models(prefix='model_') %} +{% endif %} + +{% set expected_list %} +['model_without_import_ctes', 'model_without_any_ctes', 'model_struct', 'model_data_a'] +{% endset %} + +{{ assert_equal (actual_list | trim, expected_list | trim) }} From 55dfb963f75c6cb7e7c3779315702a2740a538aa Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Mon, 30 Jan 2023 13:57:42 +0100 Subject: [PATCH 09/13] sorting actual list result --- integration_tests/tests/test_helper_get_models.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/tests/test_helper_get_models.sql b/integration_tests/tests/test_helper_get_models.sql index d63b338..32ad0f9 100644 --- a/integration_tests/tests/test_helper_get_models.sql +++ b/integration_tests/tests/test_helper_get_models.sql @@ -4,11 +4,11 @@ -- depends_on: {{ ref('model_without_any_ctes') }} {% if execute %} -{% set actual_list = codegen.get_models(prefix='model_') %} +{% set actual_list = codegen.get_models(prefix='model_')|sort %} {% endif %} {% set expected_list %} -['model_without_import_ctes', 'model_without_any_ctes', 'model_struct', 'model_data_a'] +['model_data_a', 'model_struct', 'model_without_any_ctes', 'model_without_import_ctes'] {% endset %} {{ assert_equal (actual_list | trim, expected_list | trim) }} From 312552c35e40ddc10031c5fc2a31179b6ec3e89a Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Tue, 31 Jan 2023 10:21:57 +0100 Subject: [PATCH 10/13] Update README.md Co-authored-by: Joel Labes --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e57ec7e..a91570c 100644 --- a/README.md +++ b/README.md @@ -211,8 +211,9 @@ schema.yml file. You can use the helper function codegen.get_models and specify a directory and/or prefix to get a list of all matching models, to be passed into model_names list. ``` +{% set models_to_generate = codegen.get_models(directory='marts','prefix='fct_') %} {{ codegen.generate_model_yaml( - model_names= codegen.get_models(directory='marts','prefix='fct_') + model_names = models_to_generate ) }} ``` From 052ce674545478bf504662935aa6733e9b07527e Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Tue, 31 Jan 2023 10:22:16 +0100 Subject: [PATCH 11/13] Update integration_tests/tests/test_helper_get_models.sql Co-authored-by: Joel Labes --- integration_tests/tests/test_helper_get_models.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/tests/test_helper_get_models.sql b/integration_tests/tests/test_helper_get_models.sql index 32ad0f9..33c87f8 100644 --- a/integration_tests/tests/test_helper_get_models.sql +++ b/integration_tests/tests/test_helper_get_models.sql @@ -11,4 +11,4 @@ ['model_data_a', 'model_struct', 'model_without_any_ctes', 'model_without_import_ctes'] {% endset %} -{{ assert_equal (actual_list | trim, expected_list | trim) }} +{{ assert_equal (actual_list, expected_list) }} From e244ac8f0fb695bfbf31a57e5c9b1b9fb4c1a6f2 Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Tue, 31 Jan 2023 10:30:59 +0100 Subject: [PATCH 12/13] fixing test mismatch --- integration_tests/tests/test_helper_get_models.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration_tests/tests/test_helper_get_models.sql b/integration_tests/tests/test_helper_get_models.sql index 33c87f8..1d6e380 100644 --- a/integration_tests/tests/test_helper_get_models.sql +++ b/integration_tests/tests/test_helper_get_models.sql @@ -7,8 +7,6 @@ {% set actual_list = codegen.get_models(prefix='model_')|sort %} {% endif %} -{% set expected_list %} -['model_data_a', 'model_struct', 'model_without_any_ctes', 'model_without_import_ctes'] -{% endset %} +{% set expected_list = ['model_data_a', 'model_struct', 'model_without_any_ctes', 'model_without_import_ctes'] %} {{ assert_equal (actual_list, expected_list) }} From b45a1d2b87c4bd42f653103c227577fb2821e9df Mon Sep 17 00:00:00 2001 From: Erkan Celen Date: Tue, 31 Jan 2023 10:43:04 +0100 Subject: [PATCH 13/13] typo in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a91570c..472ae8a 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,7 @@ schema.yml file. You can use the helper function codegen.get_models and specify a directory and/or prefix to get a list of all matching models, to be passed into model_names list. ``` -{% set models_to_generate = codegen.get_models(directory='marts','prefix='fct_') %} +{% set models_to_generate = codegen.get_models(directory='marts', prefix='fct_') %} {{ codegen.generate_model_yaml( model_names = models_to_generate ) }}