etheleon

etheleon

etheleon

gcloud projects create <your>-dl --enable-cloud-apis
gcloud config set project  <your>-dl
gcloud services enable compute.googleapis.com
gcloud iam service-accounts create gcp-terraform-dl --display-name gcp-terraform-dl
gcloud projects add-iam-policy-binding  <your>-dl \
            --member='serviceAccount:gcp-terraform-dl@ <your>-dl.iam.gserviceaccount.com' --role='roles/owner'

gcloud iam service-accounts keys create 'credentials.json' --iam-account='gcp-terraform-dl@<your>-dl.iam.gserviceaccount.com'

etheleon

etheleon

etheleon

etheleon

etheleon

data_stream <- sc %>% stream_read_json(s3_folder)      %>%
    filter(event=="Added to Cart")                 %>%
    filter(properties.serviceType == 'Food')       %>% 
    sdf_select(event, 
        cart_id     = properties.CartId,
        recommender = properties.experimentDetails.id,
        variant     = properties.experimentDetails.variant) %>% 
    filter(!is.na(recommender))                    %>% 
    group_by(recommender)                          %>%
    summarize(n = n())                             %>%

etheleon

add_to_cart_hourly <-
sc %>% 
    spark_read_json( 
        name = "atc_tbl",
        path = "s3a://bucket/segment-logs/sourceId/day/*", 
        overwrite = TRUE
    ) %>% 
    filter(
        event == "Added to Cart", 
        properties.serviceType == 'Food'

etheleon

conf <- spark_config()
conf$sparklyr.defaultPackages <- c("com.databricks:spark-csv_2.10:1.5.0",
                                   "com.amazonaws:aws-java-sdk-pom:1.10.34",
                                   "org.apache.hadoop:hadoop-aws:2.7.3")

conf$`sparklyr.shell.driver-memory.local` = "150G"
# Connect to Spark:
sc <- spark_connect(master = "local", 
                    version = "2.4.0",
                    config = conf)