Geremie

# Get the dags folder
dags_folder=$(gcloud composer environments describe <your_environment_name> \
    --project <your_gcp_project> --location <your_location_name> --format="get(config.dagGcsPrefix)")
# Copy the pip.conf file to the config/pip folder
config_folder=$(echo $dags_folder | cut -d / -f-3)/config/pip/
gsutil cp pip.conf $config_folder

Geremie

# Get the python repository url
gcloud artifacts print-settings python --project <your_gcp_project> \
  --repository <your_repository_name> --location <your_repository_location>

Geremie

gcloud iam service-accounts create <your_service_account_id>
# Add the composer worker role to the service account
gcloud projects add-iam-policy-binding <project_id> \
  --member=serviceAccount:<your_service_account_id>@<project_id>.iam.gserviceaccount.com \
  --role=roles/composer.worker
# Deploy the cloud composer environment
gcloud composer environments create <your_environment_name> --location=<your_location_name> \
  --zone=<your_zone_name> --service-account=<your_service_account_id>@<project_id>.iam.gserviceaccount.com \
  --python-version=3 --image-version=composer-1.17.8-airflow-2.1.4

Geremie

# Get the function url
function_url=$(gcloud functions describe <your_function_name> --region <your_function_location> --format="get(httpsTrigger.url)")
# Call the function to compute the haversine distance
curl -X POST -d '{"x1": 1, "y1": 2, "x2": 3, "y2": 4}' $function_url

Geremie

import json
from mypythonlib import myfunctions

def main(request):
    decoded_request = json.loads(request.get_data().decode('utf-8'))
    x1 = decoded_request.get('x1')
    y1 = decoded_request.get('y1')
    x2 = decoded_request.get('x2')
    y2 = decoded_request.get('y2')
    haversine_distance = myfunctions.haversine(x1, y1, x2, y2)

Geremie

# Clone the cloud function repository
git clone https://gitlab.com/marcdjoh/cloud-functions-pull-from-artifact-registry.git
# Activate the cloud functions API
gcloud services enable cloudfunctions.googleapis.com
# Activate the cloud build API
gcloud services enable cloudbuild.googleapis.com
# Create a service account for the fucntion
gcloud iam service-accounts create <your_service_account_id>
# Deploy the cloud function
gcloud functions deploy <your_function_name> --runtime=python37 --entry-point=main \

Geremie

# Clone the sample python package
git clone https://gitlab.com/marcdjoh/sample-python-package.git
# Create a python virtual environment (command for python3)
python3 -m venv py3-env
# Activate the python virtual environment
source py3-env/bin/activate
# Install twine, keyrings.google-artifactregistry-auth and wheel
pip install twine keyrings.google-artifactregistry-auth wheel
# Build the sample python package
python setup.py bdist_wheel

Geremie

# activate the artifact registry API
gcloud services enable artifactregistry.googleapis.com
# create a python repository
gcloud artifacts repositories create <your_repository_name> --repository-format=python --location=<your_repository_location> \
--description="<your_repository_description>"

Geremie

def files_processor():

    wait_for_objects_task = MultipleGcsBucketSensor(
        task_id='wait_for_objects',
        buckets=['sandbox-mdjohossou-first-bucket', 'sandbox-mdjohossou-second-bucket']
    )

    process_objects_task = PythonOperator(
        task_id='process_objects',
        python_callable=process_objects,

Geremie

def execute(self, context):
        super().execute(context)
        return self._matches