Transfer Files from GCS to GCS

transfer_function.py

#add google-cloud-storage == 2.1.0 to requirements.txt

"""
curl -m 70 -X POST https://<cf-url> \
-H "Authorization:bearer $(gcloud auth print-identity-token)" \
-H "Content-Type:application/json" \
-d '{
    "source_bucket_name":"<replace-with-bucket-name>",
    "destination_bucket_name":"<replace-with-dest-bucket-name>",
    "folder":"<folder-to-transfer>/" 
}'

"""

from google.cloud import storage
from zipfile import ZipFile
import os


def transfer(request):
    """Responds to any HTTP request.
    Args:
        request (flask.Request): HTTP request object.
    Returns:
        The response text or any set of values that can be turned into a
        Response object using
        `make_response <http://flask.pocoo.org/docs/1.0/api/#flask.Flask.make_response>`.
    """
    request_json = request.get_json()
    # GCS Buckets 
    client = storage.Client()
    # Params
    # source
    source_bucket_name = request_json["source_bucket_name"]
    source_bucket = client.get_bucket(source_bucket_name)
    folder = request_json["folder"] #prefix
    #destination
    destination_bucket_name = request_json["destination_bucket_name"]
    destination_bucket = client.get_bucket(destination_bucket_name)
    # Transfer Job
    for blob in client.list_blobs(source_bucket_name, prefix=folder):
        if blob.name == folder:
            continue
        print(blob.name)
        file_name = str(blob.name)
        file_path = "{}".format(file_name)
        blob.download_to_filename("/tmp/{}".format(file_name.split("/")[-1]))
        new_blob = destination_bucket.blob("{}".format(file_path))
        new_blob.upload_from_filename(filename="/tmp/{}".format(file_name.split("/")[-1]))
    print("Transfer done")
    return "200"