Update dags new operator (#199)

* update dag operator * set temp env vars until dbt-coves is fixed --------- Co-authored-by: Noel Gomez <[email protected]>
datacoves · Feb 7, 2024 · 6ab5f70 · 6ab5f70
1 parent 23bce86
commit 6ab5f70
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 4 deletions.
diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml
@@ -53,6 +53,13 @@ jobs:
       # cannot be applied when using the Datacoves permifrost security model.
       DATACOVES__DROP_DB_ON_FAIL:  ${{ vars.DATACOVES__DROP_DB_ON_FAIL }}
 
+
+      # this is only here temporarilly until we updtae dbt-coves
+      DATACOVES__AIRFLOW_DAGS_YML_PATH: ''
+      DATACOVES__AIRFLOW_DAGS_PATH: ''
+      DATACOVES__AIRBYTE_HOST_NAME: ''
+      DATACOVES__AIRBYTE_PORT: ''
+
     steps:
       - name: Checkout branch
         uses: actions/[email protected]

diff --git a/orchestrate/dags/daily_loan_run.py b/orchestrate/dags/daily_loan_run.py
@@ -61,6 +61,7 @@ def extract_and_load_fivetran():
     tg_extract_and_load_fivetran = extract_and_load_fivetran()
     extract_and_load_dlt = DatacovesBashOperator(
         task_id="extract_and_load_dlt",
+        activate_venv=True,
         tooltip="dlt Extract and Load",
         bash_command="python load/dlt/csv_to_snowflake/load_csv_data.py",
     )

diff --git a/orchestrate/dags/datacoves_sample_dag.py b/orchestrate/dags/datacoves_sample_dag.py
@@ -33,14 +33,17 @@ def datacoves_sample_dag():
     dbt_task = DatacovesDbtOperator(
         task_id = "run_dbt_task",
         bash_command = "dbt debug",
+        doc_md = """\
+            #### Task Documentation
+            This task leveraged the DatacovesDbtOperator
+        """
     )
 
     # This is calling an external Python file after activating the venv
     # use this instead of the Python Operator
     python_task = DatacovesBashOperator(
         task_id = "run_python_script",
-        # Virtual Environment is automatically activated
-        # activate_venv=True,
+        activate_venv=True,
         bash_command = "python orchestrate/python_scripts/sample_script.py"
     )
 

diff --git a/orchestrate/dags_yml_definitions/daily_loan_run.yml b/orchestrate/dags_yml_definitions/daily_loan_run.yml
@@ -26,8 +26,7 @@ nodes:
   extract_and_load_dlt:
     type: task
     operator: operators.datacoves.bash.DatacovesBashOperator
-    # activate_venv: true
-    # Virtual Environment is automatically activated
+    activate_venv: true
 
     tooltip: "dlt Extract and Load"
     bash_command: "python load/dlt/csv_to_snowflake/load_csv_data.py"