r39132

import os
import yaml

import yaml

from airflow import configuration as conf
from airflow import DAG
from airflow.operators import BashOperator, PostgresOperator
from datetime import datetime
from pprint import pprint

r39132

from airflow import configuration as conf
from airflow import DAG
from airflow.operators import BashOperator
from datetime import datetime

# build DAG
default_args = {
  'owner': 'jrideout',
  'pool': 'ep_generate_spoofs',
  'depends_on_past': False,

r39132

deploy@workflow-00:~/airflow/dags$ python
Python 2.7.6 (default, Jun 22 2015, 17:58:13) 
[GCC 4.8.2] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import generate_spoofs.generate_spoofed_message
>>> generate_spoofs.generate_spoofed_message.main()
--- Injecting Record ---
{'spf_result': '', 'from': '[email protected]', 'timestamp': '2015-10-24 02:24:25', 'mfrom': '', 'dmarc_result': '', 'msg_id': '<[email protected]>', 'to': '[email protected]', 'ip': '207.46.100.71', 'dkim_s': '', 'dkim_result': '', 'dkim_d': '', 'subject': '[Expensify] Please approve and forward expense report "%{month} Expenses"'}
{"body": "ApT46/cKAkhmYTZlNzI2MS1kYzc0LTRkY2ItYmQ3My0wOWY5OThmZmZjZGKU+Ov3CgAaMjA3LjQ2LjEwMC43MQIaMjA3LjQ2LjEwMC43MQAcZXhwZW5zaWlmeS5jb20CAAACAAIAACR2YXBwYXJhb0BhZ2FyaS5jb20AAAAAAF5pcC0xMC00NC0wLTIxMi51cy13ZXN0LTIuY29tcHV0ZS5pbnRlcm5hbFx1MDAwMAAAAAAAAAB+PGZhY2FkZTJjODc5NTRmOWM5MWM3NTVkMTdmMTEzNGNiQEJZMlBSMTJNQjAwNTQuZXhwZW5zaWlme

r39132

import sys, os
from datetime import datetime

from airflow import DAG
from airflow.operators import PythonOperator

import generate_spoofs.generate_spoofed_message

# build DAG
default_args = {

r39132

import avro.schema
import io, random
from avro.io import DatumWriter, DatumReader
import avro.io

# Path to user.avsc avro schema
schema_path="user.avsc"
schema = avro.schema.parse(open(schema_path).read())

r39132

import logging
import datetime

from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators import PythonOperator, BashOperator, ExternalTaskSensor
from ep_telemetry_pipeline_utils import *
import rt_scorer_asg

# constants

r39132

from airflow import DAG, utils
from airflow.operators import *
from datetime import date, datetime, time, timedelta
from ep_telemetry_pipeline_utils import *

now = datetime.now()
now_to_the_hour = now.replace(hour=now.time().hour, minute=0, second=0, microsecond=0)
START_DATE = now_to_the_hour + timedelta(hours=-3)
DAG_NAME = 'ep_telemetry_v2'
ORG_IDS = get_active_org_ids_string()

r39132

from airflow.models import Variable
from boto.s3.key import Key
from collections import OrderedDict
from datetime import date, datetime, time, timedelta
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from slackclient import SlackClient
from subprocess import Popen, PIPE, STDOUT

import base64

r39132

"""
Automation
This script will reload data (e.g. re-ingest) data into a database
"""

# ## Imports
import getopt
import logging
import os
import psycopg2

r39132

import logging
import datetime
  
from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators import BashOperator, ExternalTaskSensor
from telemetry_pipeline_utils import *
# constants
  
START = datetime.combine(datetime.today() - timedelta(days=2), datetime.min.time()) + timedelta(hours=10)