Recursively traverse the directory tree of a Google Drive folder as variation of os.walk()

walk_gdrive.py

"""os.walk() variation with Google Drive API."""

import os

from apiclient.discovery import build  # pip install google-api-python-client

FOLDER = 'application/vnd.google-apps.folder'


def get_credentials(scopes, *,
                    secrets='~/client_secrets.json',
                    storage='~/storage.json'):
    from oauth2client import file, client, tools

    store = file.Storage(os.path.expanduser(storage))
    creds = store.get()

    if creds is None or creds.invalid:
        flow = client.flow_from_clientsecrets(os.path.expanduser(secrets), scopes)
        flags = tools.argparser.parse_args([])
        creds = tools.run_flow(flow, store, flags)
    return creds


creds = get_credentials(scopes=['https://www.googleapis.com/auth/drive.metadata.readonly'])

service = build('drive', version='v3', credentials=creds)


def iterfiles(name=None, *, is_folder=None, parent=None,
              order_by='folder,name,createdTime'):
    q = []
    if name is not None:
        q.append("name = '{}'".format(name.replace("'", "\\'")))
    if is_folder is not None:
        q.append("mimeType {} '{}'".format('=' if is_folder else '!=', FOLDER))
    if parent is not None:
        q.append("'{}' in parents".format(parent.replace("'", "\\'")))

    params = {'pageToken': None, 'orderBy': order_by}
    if q:
        params['q'] = ' and '.join(q)

    while True:
        response = service.files().list(**params).execute()
        for f in response['files']:
            yield f
        try:
            params['pageToken'] = response['nextPageToken']
        except KeyError:
            return


def walk(top='root', *, by_name: bool = False):
    if by_name:
        top, = iterfiles(name=top, is_folder=True)
    else:
        top = service.files().get(fileId=top).execute()
        if top['mimeType'] != FOLDER:
            raise ValueError(f'not a folder: {top!r}')

    stack = [((top['name'],), top)]
    while stack:
        path, top = stack.pop()

        dirs, files = is_file = [], []
        for f in iterfiles(parent=top['id']):
            is_file[f['mimeType'] != FOLDER].append(f)

        yield path, top, dirs, files

        if dirs:
            stack.extend((path + (d['name'],), d) for d in reversed(dirs))


for kwargs in [{'top': 'spam', 'by_name': True}, {}]:
    for path, root, dirs, files in walk(**kwargs):
        print('/'.join(path), f'{len(dirs):d} {len(files):d}', sep='\t')

One thing to note, if you have more then 1 folder with the same name this wont work. To accommodate I made a quick update.

def walk(top):
    if top:
        top = service.files().get(fileId=top).execute()
    else:
        top = service.files().get(fileId = "root").execute()
    stack = [((top['name'],), top)]
    while stack:
        path, top = stack.pop()
        dirs, files = is_file = [], []
        for f in iterfiles(parent=top['id']):
            is_file[f['mimeType'] != FOLDER].append(f)
        yield path, top, dirs, files
        if dirs:
            stack.extend((path + (d['name'],), d) for d in dirs)

To get the folder Id you can open google drive, then copy the value from the url.

Thanks a lot @dbfanmanga and @john-delivuk for pointing this out (sorry for the late response).

Updated the signature of walk() to accept a folder id instead of a name per default (use by_name for the previous behaviour).
Using 'root' as default for walking complete drive contents.

nice one. thanks!

as i needed to download files from share drive i added some args

top = service.files().get(fileId=top, supportsAllDrives=True).execute()

params = {
"pageToken": None,
"orderBy": order_by,
"supportsAllDrives": True,
"includeItemsFromAllDrives": True,
}

Thank You

Hello there!

I am a new user to GDrive Python tools. Could you please give us an example of how ~/client_secrets.json should be?

An use case would be nice for us dummies to be able to set up this code, :)

I tried downloading a json on google's service account screen but it gave me the following error:

"InvalidClientSecretsError: Invalid file format. See https://developers.google.com/api-client-library/python/guide/aaa_client_secrets Expected a JSON object with a single property for a "web" or "installed" application"

Hey, :) maybe this helps: https://medium.com/@ashokyogi5/a-beginners-guide-to-google-oauth-and-google-apis-450f36389184 (there is an example for a clients_secrets.json).

https://github.com/googleapis/google-auth-library-python/blob/main/docs/index.rst

oauth2client was recently deprecated in favor of this library. For more details on the deprecation, see :doc:oauth2client-deprecation.