nickefy

pip install delta-spark==2.4.0
pip install pyspark

import pyspark
from delta import *

builder = pyspark.sql.SparkSession.builder.appName("MyApp") \
    .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \
    .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")

nickefy

from airflow import models
from airflow import DAG
from datetime import datetime, timedelta
from operators import DataSourceToCsv
from operators import CsvToBigquery
from operators import ExternalSensor

transformation_query_sample = """Select 
                        a.user_id,
                        b.country,

nickefy

from airflow import models
from airflow import DAG
from datetime import datetime, timedelta
from operators import DataSourceToCsv
from operators import CsvToBigquery

extract_query_source = """Select 
                        a.user_id,
                        b.country,
                        a.revenue

nickefy

from airflow import models
from airflow import DAG
from datetime import datetime, timedelta
from operators import DataSourceToCsv
from operators import CsvToBigquery

extract_query_source = """select * from transactions"""

default_dag_args = {
    'start_date': datetime(2019, 5, 1, 7),

nickefy

import pandas as pd

# Standard plotly imports
import plotly as py
import plotly.tools as tls
from plotly.offline import iplot, init_notebook_mode
import plotly.graph_objs as go
import cufflinks

# Using plotly + cufflinks in offline mode

nickefy

import pandas as pd

# Standard plotly imports
import plotly as py
import plotly.tools as tls
from plotly.offline import iplot, init_notebook_mode
import plotly.graph_objs as go
import cufflinks

# Using plotly + cufflinks in offline mode

nickefy

import os,sys,glob
from pathlib import Path
import json
from datetime import datetime, timedelta
import time

from airflow import models
from airflow.utils import trigger_rule
from airflow import DAG, AirflowException
from airflow.operators.dummy_operator import DummyOperator

nickefy

from airflow import DAG
from operators.create_dag_operator import CreateDagOperator


default_dag_args = {
    'owner': 'airflow',
    }  

dag = CreateDagOperator.create_dag('Hello_world',default_dag_args)

nickefy

{
   "destination": [
      {
         "task_name": "Print Hello World 1",
         "destination_action" : "print",
         "text_to_print": "Hello World!"
      },
      {
         "task_name": "Print Hello World 2",
         "destination_action" : "print",

nickefy

{
  "job_name": "Hello_World!",
  "retries": "0",
  "start_date": "2020-02-27",
  "email": ["[email protected]"],
  "schedule_interval": "None"
}