-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[dagster-airlift] Multi code locations working
- Loading branch information
Showing
21 changed files
with
273 additions
and
129 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
41 changes: 41 additions & 0 deletions
41
...al/dagster-airlift/examples/kitchen-sink/kitchen_sink/airflow_dags/multi_location_dags.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
from datetime import datetime | ||
from pathlib import Path | ||
|
||
from airflow import DAG | ||
from airflow.operators.python import PythonOperator | ||
from dagster_airlift.in_airflow import proxying_to_dagster | ||
from dagster_airlift.in_airflow.proxied_state import load_proxied_state_from_yaml | ||
|
||
|
||
def print_hello() -> None: | ||
print("Hello") # noqa: T201 | ||
|
||
|
||
default_args = { | ||
"owner": "airflow", | ||
"depends_on_past": False, | ||
"start_date": datetime(2023, 1, 1), | ||
"retries": 0, | ||
} | ||
|
||
with DAG( | ||
"dag_first_code_location", | ||
default_args=default_args, | ||
schedule_interval=None, | ||
is_paused_upon_creation=False, | ||
) as first_dag: | ||
PythonOperator(task_id="task", python_callable=print_hello) | ||
|
||
with DAG( | ||
"dag_second_code_location", | ||
default_args=default_args, | ||
schedule_interval=None, | ||
is_paused_upon_creation=False, | ||
) as second_dag: | ||
PythonOperator(task_id="task", python_callable=print_hello) | ||
|
||
|
||
proxying_to_dagster( | ||
proxied_state=load_proxied_state_from_yaml(Path(__file__).parent / "proxied_state"), | ||
global_vars=globals(), | ||
) |
3 changes: 3 additions & 0 deletions
3
...xamples/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_first_code_location.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
tasks: | ||
- id: task | ||
proxied: False |
3 changes: 3 additions & 0 deletions
3
...amples/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_second_code_location.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
tasks: | ||
- id: task | ||
proxied: False |
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
17 changes: 17 additions & 0 deletions
17
...airlift/examples/kitchen-sink/kitchen_sink/dagster_multi_code_locations/first_dag_defs.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
from dagster import AssetSpec, Definitions | ||
from dagster_airlift.core import assets_with_task_mappings, build_defs_from_airflow_instance | ||
|
||
from kitchen_sink.airflow_instance import local_airflow_instance | ||
|
||
defs = build_defs_from_airflow_instance( | ||
airflow_instance=local_airflow_instance(), | ||
defs=Definitions( | ||
assets=assets_with_task_mappings( | ||
dag_id="dag_first_code_location", | ||
task_mappings={ | ||
"task": [AssetSpec(key="dag_first_code_location__asset")], | ||
}, | ||
), | ||
), | ||
dag_selector_fn=lambda dag_info: dag_info.dag_id == "dag_first_code_location", | ||
) |
17 changes: 17 additions & 0 deletions
17
...irlift/examples/kitchen-sink/kitchen_sink/dagster_multi_code_locations/second_dag_defs.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
from dagster import AssetSpec, Definitions | ||
from dagster_airlift.core import assets_with_task_mappings, build_defs_from_airflow_instance | ||
|
||
from kitchen_sink.airflow_instance import local_airflow_instance | ||
|
||
defs = build_defs_from_airflow_instance( | ||
airflow_instance=local_airflow_instance(), | ||
defs=Definitions( | ||
assets=assets_with_task_mappings( | ||
dag_id="dag_second_code_location", | ||
task_mappings={ | ||
"task": [AssetSpec(key="dag_second_code_location__asset")], | ||
}, | ||
), | ||
), | ||
dag_selector_fn=lambda dag_info: dag_info.dag_id == "dag_second_code_location", | ||
) |
7 changes: 7 additions & 0 deletions
7
...er-airlift/examples/kitchen-sink/kitchen_sink/dagster_multi_code_locations/workspace.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
load_from: | ||
- python_module: | ||
module_name: kitchen_sink.dagster_multi_code_locations.first_dag_defs | ||
location_name: first_dag_location | ||
- python_module: | ||
module_name: kitchen_sink.dagster_multi_code_locations.second_dag_defs | ||
location_name: second_dag_location |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.