-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
d40bd98
commit 9cbf90e
Showing
3 changed files
with
15 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,14 +1,15 @@ | ||
"""## Datacoves Bash Operator DAG | ||
This DAG is a sample using the Datacoves Airflow Operators""" | ||
|
||
from pendulum import datetime | ||
from airflow import DAG | ||
from airflow.decorators import dag, task | ||
from operators.datacoves.dbt import DatacovesDbtOperator | ||
from operators.datacoves.bash import DatacovesBashOperator | ||
from operators.datacoves.dbt import DatacovesDbtOperator | ||
from pendulum import datetime | ||
|
||
# Only here for reference, this is automatically activated by Datacoves Operator | ||
DATACOVES_VIRTIAL_ENV = '/opt/datacoves/virtualenvs/main/bin/activate' | ||
DATACOVES_VIRTIAL_ENV = "/opt/datacoves/virtualenvs/main/bin/activate" | ||
|
||
|
||
@dag( | ||
default_args={ | ||
|
@@ -17,39 +18,37 @@ | |
"email": "[email protected]", | ||
"email_on_failure": True, | ||
}, | ||
catchup=False, | ||
tags = ["version_6"], | ||
description = "Datacoves Sample dag", | ||
tags=["version_6"], | ||
description="Datacoves Sample dag", | ||
# This is a regular CRON schedule. Helpful resources | ||
# https://cron-ai.vercel.app/ | ||
# https://crontab.guru/ | ||
schedule_interval = "0 0 1 */12 *" | ||
schedule_interval="0 0 1 */12 *", | ||
) | ||
def datacoves_sample_dag(): | ||
|
||
# Calling dbt commands | ||
dbt_task = DatacovesDbtOperator( | ||
task_id = "run_dbt_task", | ||
bash_command = "dbt debug", | ||
doc_md = """\ | ||
task_id="run_dbt_task", | ||
bash_command="dbt debug", | ||
doc_md="""\ | ||
#### Task Documentation | ||
This task leveraged the DatacovesDbtOperator | ||
""" | ||
""", | ||
) | ||
|
||
# This is calling an external Python file after activating the venv | ||
# use this instead of the Python Operator | ||
python_task = DatacovesBashOperator( | ||
task_id = "run_python_script", | ||
activate_venv=True, | ||
bash_command = "python orchestrate/python_scripts/sample_script.py" | ||
task_id="run_python_script", | ||
bash_command="python orchestrate/python_scripts/sample_script.py", | ||
) | ||
|
||
# Define task dependencies | ||
python_task.set_upstream([dbt_task]) | ||
|
||
|
||
# Invoke Dag | ||
dag = datacoves_sample_dag() | ||
dag.doc_md = __doc__ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters