diff --git a/README.md b/README.md index e431fc0f..ff6d1d4b 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ dbt-coves generate sources -h ```console --flatten-json-fields -# Flag: flatten model JSON fields +# Action to perform when JSON fields exist: 'yes', 'no', 'ask' (per file) ``` ```console @@ -478,6 +478,7 @@ generate: update_strategy: ask # Action to perform when a property file already exists. Options: update, recreate, fail, ask (per file) templates_folder: ".dbt_coves/templates" # Folder where source generation jinja templates are located. Override default templates creating source_props.yml, source_model_props.yml, and source_model.sql under this folder metadata: "metadata.csv" # Path to csv file containing metadata + flatten_json_fields: ask properties: destination: "{{model_folder_path}}/{{model_file_name}}.yml" # Where models yml files will be generated diff --git a/dbt_coves/config/config.py b/dbt_coves/config/config.py index d2db8f22..ad92f65e 100644 --- a/dbt_coves/config/config.py +++ b/dbt_coves/config/config.py @@ -34,7 +34,7 @@ class GenerateSourcesModel(BaseModel): templates_folder: Optional[str] = ".dbt_coves/templates" metadata: Optional[str] = "" no_prompt: Optional[bool] = False - flatten_json_fields: Optional[bool] = False + flatten_json_fields: Optional[str] = "ask" overwrite_staging_models: Optional[bool] = False skip_model_props: Optional[bool] = False diff --git a/dbt_coves/tasks/generate/sources.py b/dbt_coves/tasks/generate/sources.py index 6915d799..6dac84ab 100644 --- a/dbt_coves/tasks/generate/sources.py +++ b/dbt_coves/tasks/generate/sources.py @@ -85,7 +85,10 @@ def register_parser(cls, sub_parsers, base_subparser): help="Path to csv file containing metadata, i.e. 'metadata.csv'", ) subparser.add_argument( - "--flatten-json-fields", help="Flatten JSON fields", action="store_true", default=False + "--flatten-json-fields", + type=str, + choices=["yes", "no", "ask"], + help="Action to perform when JSON fields exist:" "'yes', 'no', 'ask' (per file)", ) subparser.add_argument( "--overwrite-staging-models", @@ -135,7 +138,7 @@ def generate(self, rels): models_destination = self.get_config_value("models_destination") options = { "override_all": "Yes" if self.overwrite_sqls else None, - "flatten_all": "Yes" if self.flatten_json else None, + "flatten_all": "Yes" if self.flatten_json == "yes" else None, "model_prop_update_all": False, "model_prop_recreate_all": False, "source_prop_update_all": False, @@ -189,7 +192,7 @@ def generate_model(self, relation, destination, options): columns = self.adapter.get_columns_in_relation(relation) nested_field_type = self.NESTED_FIELD_TYPES.get(self.adapter.__class__.__name__) nested = [col.name for col in columns if col.dtype == nested_field_type] - if not options["flatten_all"]: + if not options["flatten_all"] and self.flatten_json == "ask": if nested: field_nlg = "field" flatten_nlg = "flatten it" diff --git a/dbt_coves/utils/flags.py b/dbt_coves/utils/flags.py index 5d341d5c..bbfaa20c 100644 --- a/dbt_coves/utils/flags.py +++ b/dbt_coves/utils/flags.py @@ -46,7 +46,7 @@ def __init__(self, cli_parser: ArgumentParser) -> None: "templates_folder": None, "metadata": None, "no_prompt": False, - "flatten_json_fields": False, + "flatten_json_fields": None, "overwrite_staging_models": False, "skip_model_props": False, }, @@ -189,7 +189,9 @@ def parse_args(self, cli_args: List[str] = list()) -> None: if self.args.no_prompt: self.generate["sources"]["no_prompt"] = True if self.args.flatten_json_fields: - self.generate["sources"]["flatten_json_fields"] = True + self.generate["sources"][ + "flatten_json_fields" + ] = self.args.flatten_json_fields.lower() if self.args.overwrite_staging_models: self.generate["sources"]["overwrite_staging_models"] = True if self.args.skip_model_props: diff --git a/dbt_coves/utils/jinja.py b/dbt_coves/utils/jinja.py index 58633763..bb207b0f 100644 --- a/dbt_coves/utils/jinja.py +++ b/dbt_coves/utils/jinja.py @@ -4,7 +4,7 @@ def add_env_vars(context): - context["env_vars"] = os.environ.copy() + context["env_var"] = os.environ.copy() return context