robert8138

HiveToMySqlTransfer(
    task_id='data_to_mysql_transfer',
    sql="SELECT * FROM robert.dim_bitcoin_revenue WHERE ds = '{{ ds }}'",
    mysql_conn_id='robert_mysql',
    mysql_table='dim_bitcoin_revenue',
    hiveserver2_conn_id='hiveserver2_silver',
    dag=dag)

robert8138

template = """
SELECT
  SUM(m_bitcoin_trade_revenue) AS m_revenue
FROM
  robert.fct_bitcoin_revenue
WHERE
  ds = '{{ ds }}'
"""

HiveOperator(

robert8138

NamedHivePartitionSensor(
        task_id='waiting_for_my_awesome_hive_table',
        partition_names=['robert.fct_bitcoin_revenue'],
        dag=dag
    )

robert8138

"""
A DAG docstring might be a good way to explain at a high level
what problem space the DAG is looking at.
Links to design documents, upstream dependencies etc
are highly recommended.
"""
from datetime import datetime, timedelta
from airflow.models import DAG  # Import the DAG class
from airflow.operators.sensors import NamedHivePartitionSensor
from airflow.operators.hive_operator import HiveOperator

robert8138

"""
A DAG docstring might be a good way to explain at a high level
what problem space the DAG is looking at.
Links to design documents, upstream dependencies etc
are highly recommended.
"""
from datetime import datetime, timedelta
from airflow.models import DAG  # Import the DAG class
from airflow.operators.bash_operator import BashOperator
from airflow.operators.python_operator import PythonOperator

robert8138

"""
A DAG definition file in Airflow, written in Python.
"""
from datetime import datetime, timedelta
from airflow.models import DAG  # Import the DAG class
from airflow.operators.bash_operator import BashOperator
from airflow.operators.sensors import TimeDeltaSensor

default_args = {
    'owner': 'you',

robert8138

def fit(X_train, y_train):
    import multiprocessing
    from ml_helpers.sklearn_extensions import DenseMatrixConverter
    from ml_helpers.data import split_records
    from xgboost import XGBRegressor

    global model
    
    model = {}
    n_subset = N_EXAMPLES

robert8138

transforms = []

transforms.append(
    ('select_binary', ColumnSelector(features=binary))
)

transforms.append(
    ('numeric', ExtendedPipeline([
        ('select', ColumnSelector(features=numeric)),
        ('impute', Imputer(missing_values='NaN', strategy='mean', axis=0)),

robert8138

source: {
  type: hive
  query:"""
    SELECT
        id_listing as listing
      , dim_city as city
      , dim_country as country
      , dim_is_active as is_active
      , CONCAT(ds, ' 23:59:59.999') as ts
    FROM

robert8138

Object.keys(window.calendarMap).forEach(function(key){
	document.getElementById(key)
		.addEventListener(‘click’, function(){ 
			makeCalendar(key); 
		});
});

// ... a bunch of D3 code to set up my calendar visualization ... //

function makeCalendar(eventType) {