joshuahendinata

import string

from google.ads.googleads.client import GoogleAdsClient
from google.protobuf.json_format import MessageToDict
from pyspark.sql import functions as F
from pyspark.sql.types import ArrayType, StringType, StructField, StructType

segment_dates = ['2022-01-01', '2022-01-02']  # dummy values for brevity
gads_account_ids = ['123456789', '987654321']  # dummy values for brevity
first_letters = list(string.digits) + list(string.ascii_lowercase)

joshuahendinata

spec:
  template:
    spec:
      containers:
      - name: airflow-worker
        resources:
          requests:
            memory: 2Gi
          limits:
            memory: 2Gi

joshuahendinata

apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
  name: airflow-worker
  namespace: # use ${AIRFLOW_WORKER_NS} variable defined above
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: airflow-worker

joshuahendinata

#!/usr/bin/env bash
set -e

PROJECT=[provide your gcp project id]
COMPOSER_NAME=[provide your composer environment name]
COMPOSER_LOCATION=[provide the selected composer’s location e.g. us-central]
CLUSTER_ZONE=[provide the selected composer’s zone e.g. us-central1-a]

# Enable autoscaling in node level
GKE_CLUSTER=$(gcloud composer environments describe \