arnaud9

import boto3

s3 = boto3.resource('s3')

def upload_file_to_S3(filename, key, bucket_name):
    s3.Bucket(bucket_name).upload_file(filename, key)

arnaud9

mkdir airflow_project
cd airflow_project
virutalenv venv
source venv/bin/activate
mkdir airflow_home

arnaud9

from airflow import DAG
from airflow.operators import DummyOperator, PythonOperator

default_args = {
    'owner': 'arnaud',
    'start_date': datetime(2019, 1, 1),
    'retry_delay': timedelta(minutes=5)
}
# Using the context manager alllows you not to duplicate the dag parameter in each operator
with DAG('S3_dag_test', default_args=default_args, schedule_interval='@once') as dag:

arnaud9

upload_to_S3_task = PythonOperator(
    task_id='upload_to_S3',
    python_callable=upload_file_to_S3,
    op_kwargs={
        'filename': 'path/to/my_file.csv',
        'key': 'my_S3_file.csv',
        'bucket_name': 'my-S3-bucket',
    },
    dag=my_dag)

arnaud9

import airflow.hooks.S3_hook

def upload_file_to_S3_with_hook(filename, key, bucket_name):
    hook = airflow.hooks.S3_hook.S3Hook('my_S3_conn')
    hook.load_file(filename, key, bucket_name)

arnaud9

Editor commands

Commande	Nom JetBrain
Priorité	1
Ouvrir un fichier	File...
Naviguer de mot en mot	Move caret to next/previous word
Aller au bout d'une ligne	Move caret to line end/start
Multicurseur sélection manuelle des positions	Add or remove caret