r39132

import os
from airflow import DAG
from airflow.operators import *
from datetime import datetime, timedelta
from datetime import date

START_DATE = datetime(2016, 2, 4, 0, 0, 0)

default_args = {
  'owner':'sanand',

r39132

from airflow.operators import ShortCircuitOperator, DummyOperator
from airflow.models import DAG
import airflow.utils
from datetime import datetime, timedelta

seven_days_ago = datetime.combine(datetime.today() - timedelta(7),
                                  datetime.min.time())
args = {
    'owner': 'airflow',
    'start_date': seven_days_ago,

r39132

import logging
import datetime
  
from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators import BashOperator, ExternalTaskSensor
from telemetry_pipeline_utils import *
# constants
  
START = datetime.combine(datetime.today() - timedelta(days=2), datetime.min.time()) + timedelta(hours=10)

r39132

"""
Automation
This script will reload data (e.g. re-ingest) data into a database
"""

# ## Imports
import getopt
import logging
import os
import psycopg2

r39132

from airflow.models import Variable
from boto.s3.key import Key
from collections import OrderedDict
from datetime import date, datetime, time, timedelta
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from slackclient import SlackClient
from subprocess import Popen, PIPE, STDOUT

import base64

r39132

from airflow import DAG, utils
from airflow.operators import *
from datetime import date, datetime, time, timedelta
from ep_telemetry_pipeline_utils import *

now = datetime.now()
now_to_the_hour = now.replace(hour=now.time().hour, minute=0, second=0, microsecond=0)
START_DATE = now_to_the_hour + timedelta(hours=-3)
DAG_NAME = 'ep_telemetry_v2'
ORG_IDS = get_active_org_ids_string()

r39132

import logging
import datetime

from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators import PythonOperator, BashOperator, ExternalTaskSensor
from ep_telemetry_pipeline_utils import *
import rt_scorer_asg

# constants

r39132

import avro.schema
import io, random
from avro.io import DatumWriter, DatumReader
import avro.io

# Path to user.avsc avro schema
schema_path="user.avsc"
schema = avro.schema.parse(open(schema_path).read())

r39132

import sys, os
from datetime import datetime

from airflow import DAG
from airflow.operators import PythonOperator

import generate_spoofs.generate_spoofed_message

# build DAG
default_args = {

r39132

deploy@workflow-00:~/airflow/dags$ python
Python 2.7.6 (default, Jun 22 2015, 17:58:13) 
[GCC 4.8.2] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import generate_spoofs.generate_spoofed_message
>>> generate_spoofs.generate_spoofed_message.main()
--- Injecting Record ---
{'spf_result': '', 'from': '[email protected]', 'timestamp': '2015-10-24 02:24:25', 'mfrom': '', 'dmarc_result': '', 'msg_id': '<[email protected]>', 'to': '[email protected]', 'ip': '207.46.100.71', 'dkim_s': '', 'dkim_result': '', 'dkim_d': '', 'subject': '[Expensify] Please approve and forward expense report "%{month} Expenses"'}
{"body": "ApT46/cKAkhmYTZlNzI2MS1kYzc0LTRkY2ItYmQ3My0wOWY5OThmZmZjZGKU+Ov3CgAaMjA3LjQ2LjEwMC43MQIaMjA3LjQ2LjEwMC43MQAcZXhwZW5zaWlmeS5jb20CAAACAAIAACR2YXBwYXJhb0BhZ2FyaS5jb20AAAAAAF5pcC0xMC00NC0wLTIxMi51cy13ZXN0LTIuY29tcHV0ZS5pbnRlcm5hbFx1MDAwMAAAAAAAAAB+PGZhY2FkZTJjODc5NTRmOWM5MWM3NTVkMTdmMTEzNGNiQEJZMlBSMTJNQjAwNTQuZXhwZW5zaWlme