DAG: ingest_dag_bhp_spence Functions that add fields in files related to Collahuasi

schedule: 20 15,23 * * *

ingest_dag_bhp_spence

Toggle wrap 
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
# Airflow dependencies
from asyncio import Task
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from airflow.utils.dates import days_ago
from airflow.models import Variable

from jinja2 import Template
from functools import reduce
from datetime import datetime, timedelta
from standardized_process.dags.functions.data_ingest import *

# Default arguments
default_args = {
    'owner': 'pedro',
    'start_date': datetime(2023, 11, 6, 0, 0, 0)
}

bhp_spence_ingest_dag = DAG(
    'ingest_dag_bhp_spence',
    default_args=default_args,
    description='Functions that add fields in files related to Collahuasi',
    schedule_interval='20 15,23 * * *',# Hora local 17:30
    catchup=False,
    is_paused_upon_creation=True, 
    tags=['Standardized Process'],

)

year = datetime.now().year
month = datetime.now().month
day = datetime.now().day
 
ingest_task = PythonOperator(
    task_id = 'ingest_bhp_spence_data',
    python_callable = ingest_data_bhp,
    op_kwargs = {"container":"raw",
                "final_container":"raw",
                "final_blob":f"BHP Spence/Drilling/Surface Manager/{int(year)}/{int(month)}/{int(day)}/data_drill.json",
                },
                
    dag = bhp_spence_ingest_dag,
)

ingest_task