WillianFuks

#!/usr/bin/env bash

function_exists () {
    declare -f -F $1 > /dev/null
    return $?
}

throw () {
    echo "$*" >&2
    echo

WillianFuks

WillianFuks

import operator
import math
import time

from base import JobsBase
from pyspark.sql import SparkSession
from pyspark.sql import types as stypes

class NaiveJob(JobsBase):
    def run(self, sc, args):

WillianFuks

import utils
from flask import Flask, request 
from config import config
from connector.gcp import GCPService
from scheduler import SchedulerJob


app = Flask(__name__)
gcp_service = GCPService() 
scheduler = SchedulerJob()

WillianFuks

import utils
import base_utils
from config import config
from flask import Flask, request, jsonify
from factory import JobsFactory
from google.appengine.ext import ndb
import time


app = Flask(__name__)

WillianFuks

import utils
from flask import Flask, request 
from config import config
from connector.gcp import GCPService
from scheduler import SchedulerJob


app = Flask(__name__)
gcp_service = GCPService() 
scheduler = SchedulerJob()

WillianFuks

service: dataproc-twitter
runtime: python27
api_version: 1
threadsafe: true

handlers:
- url: /.*/
  script: main.app
  login: admin

WillianFuks

runtime: python27
api_version: 1
threadsafe: true
service: worker

handlers:
- url: /.*
  script: worker.app
  login: admin

WillianFuks

queue:
- name: default
  rate: 1/m

WillianFuks

cron:
- description: daily export job from BigQuery to Cloud Storage
  url: /run_job/export_customers_from_bq/?url=/export_customers&target=worker
  target: dataproc-twitter
  schedule: every day 01:05
- description: Runs DIMSUM job in Dataproc 
  url: /run_job/run_dimsum/?url=/dataproc_dimsum&target=worker&extended_args=--days_init=30,--days_end=1,--threshold=0.1&force=no
  target: dataproc-twitter
  schedule: every day 01:15