Skip to content

Instantly share code, notes, and snippets.

@lakshay-arora
Created November 23, 2020 12:25
Show Gist options
  • Save lakshay-arora/62e958a8340e1102c9774583336cde2f to your computer and use it in GitHub Desktop.
Save lakshay-arora/62e958a8340e1102c9774583336cde2f to your computer and use it in GitHub Desktop.
from collections import Counter
# define the python function
def my_function():
# get the variable value
file_path = Variable.get("data_path")
# open the file
file_ = open(file_path)
# read the file and calculate the word count
data = Counter(file_.read().split())
return data
# define the DAG
dag = DAG(
'word_count_example',
default_args=default_args,
description='How to use the Python Operator?',
schedule_interval=timedelta(days=1),
)
# define the first task
t1 = PythonOperator(
task_id='print_word_count',
python_callable= my_function,
dag=dag,
)
t1
### importing the required libraries
from datetime import timedelta
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from airflow.utils.dates import days_ago
from airflow.models import Variable
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment