Python script to fill out missing directories in GCS bucket so the directory tree can be read by gcsfuse.

fillout_gcsfuse_bucket.py

#! /usr/bin/env python
# 2020-01-16
# Brewster Malevich <bmalevich@rhg.com>

"""Fill out missing directories in GCS bucket (TARGET_BUCKET), for gcsfuse.

Looks at blob names in Google Cloud Storage bucket to pull out all implied
directories. Then add a blob directory placeholder - an empty blob for each
directory that doesn't already have a blob. All directories should then
appear when mounted by gcsfuse.
"""


from pathlib import Path
from tqdm import tqdm
from google.cloud import storage


TARGET_BUCKET = "bucket_name_here"


def _fetch_dirs(bucket):
    """Return set of all directory paths within a GCS bucket blob names

    Parameters
    ----------
    bucket : google.cloud.storage.bucket.Bucket

    Returns
    -------
    all_dirs : set
    """
    all_dirs = set([])

    for blob in list(bucket.list_blobs()):
        p = Path(blob.name).parent

        # Skip root dir.
        if p == Path("."):
            continue

        # Parse parent dirs in blob name.
        for i in range(1, len(p.parts) + 1):
            parentdir = str(Path(*p.parts[:i])) + "/"
            all_dirs.add(parentdir)

    return all_dirs


def main(bucket_name, client=None):
    if client is None:
        client = storage.Client()

    bucket = client.get_bucket(bucket_name)
    assert bucket.exists()

    # Create empty blob for any non-exist directories
    for target_dir in tqdm(_fetch_dirs(bucket)):
        dir_blob = bucket.blob(target_dir)
        if dir_blob.exists():
            continue
        # Create empty blob as dir placeholder.
        dir_blob.upload_from_string(
            "",
            content_type="application/x-www-form-urlencoded;charset=UTF-8"
        )


if __name__ == "__main__":
    main(TARGET_BUCKET)

I've occasionally had to swap in client = storage.Client(project="compute-impactlab") for client = storage.Client() in main(). I think it's because I didn't have a default gcloud project configured properly on my machine.

We can fix this if it becomes a persistent problem.