Created
April 3, 2022 00:06
-
-
Save EricSchles/660537c669f9251e4d5afc421f44bbe4 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from datetime import datetime, timedelta | |
| from textwrap import dedent | |
| from functor import my_function | |
| # The DAG object; we'll need this to instantiate a DAG | |
| from airflow import DAG | |
| # Operators; we need this to operate! | |
| from airflow.operators.bash import BashOperator | |
| from airflow.operators.python import PythonOperator | |
| with DAG( | |
| 'eric_lior_tutorial', | |
| # These args will get passed on to each operator | |
| # You can override them on a per-task basis during operator initialization | |
| default_args={ | |
| 'depends_on_past': False, | |
| 'email': ['airflow@example.com'], | |
| 'email_on_failure': False, | |
| 'email_on_retry': False, | |
| 'retries': 1, | |
| 'retry_delay': timedelta(minutes=5), | |
| # 'queue': 'bash_queue', | |
| # 'pool': 'backfill', | |
| # 'priority_weight': 10, | |
| # 'end_date': datetime(2016, 1, 1), | |
| # 'wait_for_downstream': False, | |
| # 'sla': timedelta(hours=2), | |
| # 'execution_timeout': timedelta(seconds=300), | |
| # 'on_failure_callback': some_function, | |
| # 'on_success_callback': some_other_function, | |
| # 'on_retry_callback': another_function, | |
| # 'sla_miss_callback': yet_another_function, | |
| # 'trigger_rule': 'all_success' | |
| }, | |
| description='A simple tutorial DAG', | |
| schedule_interval=timedelta(days=1), | |
| start_date=datetime(2021, 1, 1), | |
| catchup=False, | |
| tags=['example'], | |
| ) as dag: | |
| # t1, t2 and t3 are examples of tasks created by instantiating operators | |
| t1 = BashOperator( | |
| task_id='print_date', | |
| bash_command='date', | |
| ) | |
| t2 = BashOperator( | |
| task_id='sleep', | |
| depends_on_past=False, | |
| bash_command='sleep 5', | |
| retries=3, | |
| ) | |
| t3 = PythonOperator( | |
| task_id='print', | |
| python_callable= my_function, | |
| op_kwargs = {"x" : "Apache Airflow"}, | |
| dag=dag, | |
| ) | |
| t1.doc_md = dedent( | |
| """\ | |
| #### Task Documentation | |
| You can document your task using the attributes `doc_md` (markdown), | |
| `doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets | |
| rendered in the UI's Task Instance Details page. | |
|  | |
| """ | |
| ) | |
| dag.doc_md = __doc__ # providing that you have a docstring at the beginning of the DAG | |
| dag.doc_md = """ | |
| This is a documentation placed anywhere | |
| """ # otherwise, type it like this | |
| templated_command = dedent( | |
| """ | |
| {% for i in range(5) %} | |
| echo "{{ ds }}" | |
| echo "{{ macros.ds_add(ds, 7)}}" | |
| {% endfor %} | |
| """ | |
| ) | |
| t4 = BashOperator( | |
| task_id='templated', | |
| depends_on_past=False, | |
| bash_command=templated_command, | |
| ) | |
| t1 >> [t2, t3] >> t4 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment