generating signed url for requester pays bucket, bill to project that owns the bucket

g_sign.py

# NOTE: need to install oauth2client
# Signed URL generates correctly but I keep getting:
# <Error>
# <Code>SignatureDoesNotMatch</Code>
# <Message>
# The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.
# </Message>
# ...
# </Error>

import base64
import time
import json

from oauth2client.service_account import ServiceAccountCredentials


PATH_TO_RESOURCE = "bucket-with-r-pays-on/some-file.txt"
GOOGLE_PROJECT_ID = "project-with-bucket"
SERVICE_ACCOUNT_CREDS = "/path/to/SA/creds/with/sufficient/permissions/on/project.json"


def get_signed_url(
    path_to_resource,
    http_verb,
    expires,
    google_project_id,
    extension_headers=None,
    content_type="",
    md5_value="",
    service_account_creds=None,
):
    """
    Requirements/process:
        https://cloud.google.com/storage/docs/access-control/create-signed-urls-program

    Args:
        path_to_resource (str): Description
        http_verb (str): Description
        expires (int): Description
        extension_headers (None, optional): Description
        content_type (str, optional): Description
        md5_value (str, optional): Description
        service_account_creds (dict, optional): JSON keyfile dict for Google
            Service Account (can be obtained by calling `get_access_key`)

    Returns:
        str: Completed signed URL
    """
    path_to_resource = path_to_resource.strip("/")
    string_to_sign = _get_string_to_sign(
        path_to_resource,
        http_verb,
        expires,
        google_project_id,
        extension_headers,
        content_type,
        md5_value,
    )

    if service_account_creds:
        creds = ServiceAccountCredentials.from_json_keyfile_dict(service_account_creds)
    else:
        pass
        # creds = get_default_service_account_credentials()

    client_id = creds.service_account_email
    signature = creds.sign_blob(string_to_sign)[1]

    # needs to be url safe so percent-encode + and /
    encoded_signature = (
        base64.b64encode(signature).replace("+", "%2B").replace("/", "%2F")
    )

    final_url = (
        "https://storage.googleapis.com/"
        + path_to_resource
        + "?GoogleAccessId="
        + client_id
        + "&Expires="
        + str(expires)
        + "&userProject="
        + str(google_project_id)
        + "&Signature="
        + encoded_signature
    )

    return final_url


def _get_string_to_sign(
    path_to_resource,
    http_verb,
    expires,
    google_project_id,
    extension_headers=None,
    content_type="",
    md5_value="",
):
    path_to_resource = path_to_resource.strip("/")
    extension_headers = extension_headers or []
    string_to_sign = (
        str(http_verb)
        + "\n"
        + str(md5_value)
        + "\n"
        + str(content_type)
        + "\n"
        + str(expires)
        + "\n"
        + str("userProject={}".format(google_project_id))
        # also tried -> + str("userProject")
        # also tried -> + str("{}".format(google_project_id))
        + "\n"
    )

    # TODO actually need to sort these before adding
    # https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers
    for ext_header in extension_headers:
        string_to_sign += (
            str(ext_header).lower().replace("\n", " ").replace(": ", ":") + "\n"
        )

    string_to_sign += "/" + str(path_to_resource)

    return string_to_sign


if __name__ == "__main__":
    http_verb = "GET"
    expires = int(time.time()) + int(1800)
    with open(SERVICE_ACCOUNT_CREDS) as creds:
        json_creds = json.load(creds)

    print(
        get_signed_url(
            PATH_TO_RESOURCE,
            http_verb,
            expires,
            GOOGLE_PROJECT_ID,
            service_account_creds=json_creds,
        )
    )