dfp_external_storage.js

frappe.ui.form.on("DFP External Storage", {
  setup: (frm) => {
    frm.button_remote_files_list = null;
  },

  refresh: function (frm) {
    if (frm.is_new() && !frm.doc.doctypes_ignored.length) {
      frm.doc.doctypes_ignored.push({ doctype_to_ignore: "Data Import" });
      frm.doc.doctypes_ignored.push({ doctype_to_ignore: "Prepared Report" });
      frm.refresh_field("doctypes_ignored");
    }

    if (frm.doc.enabled) {
      frm.button_remote_files_list = frm.add_custom_button(
        __("List files in bucket"),
        () => frappe.set_route("dfp-s3-bucket-list", frm.doc.name)
      );
    }

    // Storage type specific actions
    if (frm.doc.type === "Google Drive") {
      // Add Google Drive authentication button
      setup_google_drive_auth(frm);
    } else {
      // Add Test Connection button for S3
      frm
        .add_custom_button(__("Test Connection"), function () {
          frm.events.test_connection(frm);
        })
        .addClass("btn-primary");
    }

    frm.set_query("folders", function () {
      return {
        filters: {
          is_folder: 1,
        },
      };
    });

    frappe.db
      .get_list("DFP External Storage by Folder", {
        fields: ["name", "folder"],
      })
      .then((data) => {
        if (data && data.length) {
          let folders_name_not_assigned = data
            .filter((d) => (d.name != frm.doc.name ? d : null))
            .map((d) => d.folder);
          frm.set_query("folders", function () {
            return {
              filters: {
                is_folder: 1,
                name: ["not in", folders_name_not_assigned],
              },
            };
          });
        }
      });
  },

  type: function(frm) {
    // Show/hide fields based on storage type
    frm.trigger("refresh");
  },

  test_connection: function (frm) {
    if (frm.doc.type === "Google Drive") {
      frm.events.test_google_drive_connection(frm);
    } else {
      frm.events.test_s3_connection(frm);
    }
  },

  test_s3_connection: function (frm) {
    frappe.show_message(__("Testing S3 Connection..."));

    // Prepare for API call - collect all the necessary fields
    let data = {
      endpoint: frm.doc.endpoint,
      secure: frm.doc.secure,
      bucket_name: frm.doc.bucket_name,
      region: frm.doc.region,
      access_key: frm.doc.access_key,
      // Secret key won't be included if the doc is saved
      // It will use the stored encrypted value in that case
      secret_key: frm.is_new() ? frm.doc.secret_key : undefined,
      storage_type: frm.doc.type,
    };

    frappe.call({
      method:
        "dfp_external_storage.dfp_external_storage.doctype.dfp_external_storage.dfp_external_storage.test_s3_connection",
      args: {
        doc_name: frm.doc.name,
        connection_data: data,
      },
      freeze: true,
      freeze_message: __("Testing S3 Connection..."),
      callback: (r) => {
        frappe.hide_message();
        if (r.exc) {
          // Error handling
          frappe.msgprint({
            title: __("Connection Failed"),
            indicator: "red",
            message:
              __("Failed to connect to S3: ") +
              __(r.exc_msg || "Unknown error"),
          });
        } else if (r.message.success) {
          // Success message
          frappe.msgprint({
            title: __("Connection Successful"),
            indicator: "green",
            message: __(r.message.message),
          });
        } else {
          // Failed but with a message
          frappe.msgprint({
            title: __("Connection Failed"),
            indicator: "red",
            message: __(r.message.message),
          });
        }
      },
    });
  },

  test_google_drive_connection: function (frm) {
    // Validate required fields
    if (!frm.doc.google_client_id || !frm.doc.google_folder_id) {
      frappe.msgprint({
        title: __("Missing Information"),
        indicator: "red",
        message: __("Client ID and Folder ID are required to test the connection."),
      });
      return;
    }

    frappe.show_message(__("Testing Google Drive Connection..."));

    frappe.call({
      method: "dfp_external_storage.gdrive_integration.test_google_drive_connection",
      args: {
        doc_name: frm.doc.name,
      },
      freeze: true,
      freeze_message: __("Testing Google Drive Connection..."),
      callback: (r) => {
        frappe.hide_message();
        if (r.exc) {
          frappe.msgprint({
            title: __("Connection Failed"),
            indicator: "red",
            message: __("Failed to connect to Google Drive: ") + __(r.exc_msg || "Unknown error"),
          });
        } else if (r.message && r.message.success) {
          frappe.msgprint({
            title: __("Connection Successful"),
            indicator: "green",
            message: __(r.message.message || "Successfully connected to Google Drive."),
          });
        } else {
          frappe.msgprint({
            title: __("Connection Failed"),
            indicator: "red",
            message: __(r.message ? r.message.message : "Unknown error connecting to Google Drive."),
          });
        }
      },
    });
  }
});

function setup_google_drive_auth(frm) {
  // First check if we already have credentials in the session
  frappe.call({
    method: "dfp_external_storage.gdrive_integration.get_auth_credentials",
    callback: (r) => {
      if (r.message) {
        // We have credentials, show button to apply them
        setup_apply_credentials_button(frm, r.message);
      } else {
        // No credentials, show authenticate button
        setup_auth_button(frm);
      }
    }
  });
}

function setup_auth_button(frm) {
  $('#google-auth-btn').off('click').on('click', function() {
    // Client ID and secret are required
    if (!frm.doc.google_client_id || !frm.doc.google_client_secret) {
      frappe.msgprint({
        title: __("Missing Information"),
        indicator: "red",
        message: __("Client ID and Client Secret are required for authentication."),
      });
      return;
    }

    frappe.call({
      method: "dfp_external_storage.gdrive_integration.initiate_google_drive_auth",
      args: {
        doc_name: frm.doc.name,
        client_id: frm.doc.google_client_id,
        client_secret: frm.doc.google_client_secret
      },
      callback: (r) => {
        if (r.message && r.message.success) {
          // Open the auth URL in a new window
          const authWindow = window.open(r.message.auth_url, "GoogleDriveAuth", 
            "width=600,height=700,location=yes,resizable=yes,scrollbars=yes,status=yes");

          // Add message about the popup
          frappe.show_alert({
            message: __("Google authentication window opened. Please complete the authentication process in the new window."),
            indicator: "blue"
          }, 10);

          // Check if window was blocked
          if (!authWindow || authWindow.closed || typeof authWindow.closed === 'undefined') {
            frappe.msgprint({
              title: __("Popup Blocked"),
              indicator: "red",
              message: __("The authentication popup was blocked. Please allow popups for this site and try again."),
            });
          }
        } else {
          frappe.msgprint({
            title: __("Authentication Failed"),
            indicator: "red",
            message: __(r.message ? r.message.error : "Failed to initiate Google Drive authentication."),
          });
        }
      }
    });
  });
}

function setup_apply_credentials_button(frm, credentials) {
  // Replace the auth button with an apply credentials button
  $('.google-auth-button').html(
    `<button class="btn btn-primary btn-sm" id="apply-credentials-btn">Apply Authentication</button>`
  );

  $('#apply-credentials-btn').off('click').on('click', function() {
    // Set the refresh token in the form
    frm.set_value('google_refresh_token', credentials.refresh_token);
    
    frappe.show_alert({
      message: __("Authentication credentials applied. You can now save the document."),
      indicator: "green"
    }, 5);

    // Replace with re-authenticate button
    $('.google-auth-button').html(
      `<button class="btn btn-default btn-sm" id="google-auth-btn">Re-authenticate with Google Drive</button>`
    );
    setup_auth_button(frm);
  });
}

dfp_external_storage.py

"""
You need to update this file by integrating the code from the "DFP External Storage File Class Update" artifact. Don't completely replace the file - instead, add the DFPExternalStorageGoogleDriveFile class and modify the existing methods to handle Google Drive.
"""
import io
import os
import re
import mimetypes
import typing as t
from datetime import timedelta
from werkzeug.wrappers import Response
from werkzeug.wsgi import wrap_file
from functools import cached_property
from minio import Minio
import frappe
from frappe import _
from frappe.core.doctype.file.file import File
from frappe.core.doctype.file.file import URL_PREFIXES
from frappe.model.document import Document
from frappe.utils.password import get_decrypted_password

# Import Google Drive integration
from dfp_external_storage.gdrive_integration import GoogleDriveConnection

# Constants
DFP_EXTERNAL_STORAGE_PUBLIC_CACHE_PREFIX = "external_storage_public_file:"
DFP_EXTERNAL_STORAGE_URL_SEGMENT_FOR_FILE_LOAD = "file"


class DFPExternalStorageGoogleDriveFile:
    """Google Drive implementation for DFP External Storage File"""
    
    def __init__(self, file_doc):
        self.file_doc = file_doc
        self.file_name = file_doc.file_name
        self.content_hash = file_doc.content_hash
        self.storage_doc = file_doc.dfp_external_storage_doc
        self.client = self._get_google_drive_client()
        
    def _get_google_drive_client(self):
        """Initialize Google Drive client"""
        try:
            storage_doc = self.storage_doc
            client_id = storage_doc.google_client_id
            client_secret = get_decrypted_password(
                "DFP External Storage", storage_doc.name, "google_client_secret"
            )
            refresh_token = get_decrypted_password(
                "DFP External Storage", storage_doc.name, "google_refresh_token"
            )
            
            return GoogleDriveConnection(
                client_id=client_id,
                client_secret=client_secret,
                refresh_token=refresh_token
            )
        except Exception as e:
            frappe.log_error(f"Failed to initialize Google Drive client: {str(e)}")
            return None
    
    def upload_file(self, local_file=None):
        """
        Upload file to Google Drive
        
        Args:
            local_file (str): Local file path
            
        Returns:
            bool: True if upload successful
        """
        try:
            # Check if already uploaded
            if self.file_doc.dfp_external_storage_s3_key:
                return False
                
            # Determine file path
            is_public = "/public" if not self.file_doc.is_private else ""
            if not local_file:
                local_file = f"./{frappe.local.site}{is_public}{self.file_doc.file_url}"
                
            # Check if file exists
            if not os.path.exists(local_file):
                frappe.throw(_("Local file not found: {0}").format(local_file))
                
            # Determine content type
            content_type, _ = mimetypes.guess_type(self.file_name)
            if not content_type:
                content_type = "application/octet-stream"
                
            # Upload file
            with open(local_file, 'rb') as f:
                file_size = os.path.getsize(local_file)
                
                # Upload to Google Drive
                result = self.client.put_object(
                    folder_id=self.storage_doc.google_folder_id,
                    file_name=self.file_name,
                    data=f,
                    metadata={"Content-Type": content_type}
                )
                
                # Update file document with Google Drive file ID
                self.file_doc.dfp_external_storage_s3_key = result.get('id')
                self.file_doc.dfp_external_storage = self.storage_doc.name
                self.file_doc.file_url = f"/{DFP_EXTERNAL_STORAGE_URL_SEGMENT_FOR_FILE_LOAD}/{self.file_doc.name}/{self.file_name}"
                
                # Remove local file
                os.remove(local_file)
                return True
                
        except Exception as e:
            error_msg = _("Error saving file to Google Drive: {0}").format(str(e))
            frappe.log_error(f"{error_msg}: {self.file_name}")
            
            # Reset S3 fields for new file
            if not self.file_doc.get_doc_before_save():
                error_extra = _("File saved in local filesystem.")
                frappe.log_error(f"{error_msg} {error_extra}: {self.file_name}")
                self.file_doc.dfp_external_storage_s3_key = ""
                self.file_doc.dfp_external_storage = ""
                # Keep original file_url
            else:
                frappe.throw(error_msg)
            
            return False
    
    def delete_file(self):
        """Delete file from Google Drive"""
        if not self.file_doc.dfp_external_storage_s3_key:
            return False
            
        # Check if other files use the same key
        files_using_key = frappe.get_all(
            "File",
            filters={
                "dfp_external_storage_s3_key": self.file_doc.dfp_external_storage_s3_key,
                "dfp_external_storage": self.file_doc.dfp_external_storage,
            },
        )
        
        if len(files_using_key) > 1:
            # Other files are using this Drive file, don't delete
            return False
            
        # Delete from Google Drive
        try:
            self.client.remove_object(
                folder_id=self.storage_doc.google_folder_id,
                file_id=self.file_doc.dfp_external_storage_s3_key
            )
            return True
        except Exception as e:
            error_msg = _("Error deleting file from Google Drive.")
            frappe.log_error(f"{error_msg}: {self.file_name}", message=str(e))
            frappe.throw(f"{error_msg} {str(e)}")
            return False
    
    def download_file(self):
        """Download file from Google Drive"""
        try:
            file_content = self.client.get_object(
                folder_id=self.storage_doc.google_folder_id,
                file_id=self.file_doc.dfp_external_storage_s3_key
            )
            return file_content.read()
        except Exception as e:
            error_msg = _("Error downloading file from Google Drive")
            frappe.log_error(title=f"{error_msg}: {self.file_name}")
            frappe.throw(error_msg)
            return b""
    
    def stream_file(self):
        """Stream file from Google Drive"""
        try:
            file_content = self.client.get_object(
                folder_id=self.storage_doc.google_folder_id,
                file_id=self.file_doc.dfp_external_storage_s3_key
            )
            
            # Wrap the file content for streaming
            return wrap_file(
                environ=frappe.local.request.environ,
                file=file_content,
                buffer_size=self.storage_doc.setting_stream_buffer_size
            )
        except Exception as e:
            frappe.log_error(f"Google Drive streaming error: {str(e)}")
            frappe.throw(_("Failed to stream file from Google Drive"))
    
    def download_to_local_and_remove_remote(self):
        """Download file from Google Drive and remove the remote file"""
        try:
            # Get file content
            file_content = self.client.get_object(
                folder_id=self.storage_doc.google_folder_id,
                file_id=self.file_doc.dfp_external_storage_s3_key
            )
            
            # Save content
            self.file_doc._content = file_content.read()
            
            # Clear storage info
            self.file_doc.dfp_external_storage_s3_key = ""
            self.file_doc.dfp_external_storage = ""
            
            # Save to filesystem
            self.file_doc.save_file_on_filesystem()
            
            # Delete from Google Drive
            self.client.remove_object(
                folder_id=self.storage_doc.google_folder_id,
                file_id=self.file_doc.dfp_external_storage_s3_key
            )
            
            return True
        except Exception as e:
            error_msg = _("Error downloading and removing file from Google Drive.")
            frappe.log_error(title=f"{error_msg}: {self.file_name}")
            frappe.throw(error_msg)
            return False
    
    def get_presigned_url(self):
        """Get a presigned URL for the file"""
        try:
            if not self.storage_doc.presigned_urls:
                return None
                
            # Check mimetype restrictions
            if (
                self.storage_doc.presigned_mimetypes_starting
                and self.file_doc.dfp_mime_type_guess_by_file_name
            ):
                presigned_mimetypes_starting = [
                    i.strip()
                    for i in self.storage_doc.presigned_mimetypes_starting.split("\n")
                    if i.strip()
                ]
                
                if not any(
                    self.file_doc.dfp_mime_type_guess_by_file_name.startswith(i)
                    for i in presigned_mimetypes_starting
                ):
                    return None
            
            # Get presigned URL
            return self.client.presigned_get_object(
                folder_id=self.storage_doc.google_folder_id,
                file_id=self.file_doc.dfp_external_storage_s3_key,
                expires=self.storage_doc.setting_presigned_url_expiration
            )
        except Exception as e:
            frappe.log_error(f"Error generating Google Drive presigned URL: {str(e)}")
            return None


# Functions to update DFPExternalStorageFile class to handle Google Drive
def handle_storage_type(file_doc):
    """
    Handle different storage types and return the appropriate handler
    
    Args:
        file_doc: DFPExternalStorageFile instance
        
    Returns:
        object: Storage handler instance
    """
    storage_type = file_doc.dfp_external_storage_doc.type
    
    if storage_type == "Google Drive":
        return DFPExternalStorageGoogleDriveFile(file_doc)
    return None  # Default S3 handlers will be used


# Modifications needed in DFPExternalStorageFile class:
"""
The following functions need to be updated in the main DFPExternalStorageFile class:

1. dfp_external_storage_upload_file:
   - Add a check for Google Drive storage type
   - Call the appropriate handler

2. dfp_external_storage_delete_file:
   - Add a check for Google Drive storage type
   - Call the appropriate handler

3. dfp_external_storage_download_file: 
   - Add a check for Google Drive storage type
   - Call the appropriate handler

4. dfp_external_storage_stream_file:
   - Add a check for Google Drive storage type
   - Call the appropriate handler

5. download_to_local_and_remove_remote:
   - Add a check for Google Drive storage type
   - Call the appropriate handler

6. dfp_presigned_url_get:
   - Add a check for Google Drive storage type
   - Call the appropriate handler
"""

gdrive_integration.py

"""
Google Drive Integration for DFP External Storage

This module adds Google Drive support to the DFP External Storage app.
It requires the following dependencies:
- google-api-python-client
- google-auth
- google-auth-oauthlib
"""

import io
import os
import re
import json
import frappe
from frappe import _
from frappe.utils import get_request_site_address, get_url
from datetime import datetime, timedelta
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from googleapiclient.http import MediaIoBaseUpload, MediaIoBaseDownload
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import Flow
from google.auth.transport.requests import Request
from google.auth.exceptions import RefreshError

# Google Drive API scopes
SCOPES = ['https://www.googleapis.com/auth/drive.file']

# Cache key prefix for Google Drive tokens
DFP_GDRIVE_TOKEN_CACHE_PREFIX = "dfp_gdrive_token:"


class GoogleDriveConnection:
    """Google Drive connection handler for DFP External Storage"""
    
    def __init__(self, client_id, client_secret, refresh_token, token_uri="https://oauth2.googleapis.com/token"):
        """
        Initialize Google Drive connection
        
        Args:
            client_id (str): Google API Client ID
            client_secret (str): Google API Client Secret
            refresh_token (str): OAuth2 refresh token
            token_uri (str): Token URI for OAuth2
        """
        self.client_id = client_id
        self.client_secret = client_secret
        self.refresh_token = refresh_token
        self.token_uri = token_uri
        self.service = None
        
        # Initialize the connection
        self._connect()
        
    def _connect(self):
        """Establish connection to Google Drive API"""
        try:
            creds = Credentials(
                None,  # No access token initially
                refresh_token=self.refresh_token,
                token_uri=self.token_uri,
                client_id=self.client_id,
                client_secret=self.client_secret,
                scopes=SCOPES
            )
            
            # Refresh the token if expired
            if creds.expired:
                creds.refresh(Request())
                
            # Build the Drive API service
            self.service = build('drive', 'v3', credentials=creds)
            return True
        except Exception as e:
            frappe.log_error(f"Google Drive connection error: {str(e)}")
            return False
    
    def validate_folder(self, folder_id):
        """
        Validate if a Google Drive folder exists and is accessible
        
        Args:
            folder_id (str): Google Drive folder ID
            
        Returns:
            bool: True if folder exists and is accessible
        """
        try:
            # Try to get folder metadata
            folder = self.service.files().get(
                fileId=folder_id, 
                fields="id,name,mimeType",
                supportsAllDrives=True
            ).execute()
            
            # Verify it's a folder
            if folder.get('mimeType') != 'application/vnd.google-apps.folder':
                frappe.msgprint(_("The specified Google Drive ID is not a folder"), alert=True)
                return False
                
            frappe.msgprint(_("Google Drive folder found: {0}").format(folder.get('name')), indicator="green", alert=True)
            return True
        except HttpError as e:
            if e.resp.status == 404:
                frappe.throw(_("Google Drive folder not found"))
            else:
                frappe.throw(_("Error accessing Google Drive folder: {0}").format(str(e)))
            return False
        except Exception as e:
            frappe.throw(_("Error validating Google Drive folder: {0}").format(str(e)))
            return False
    
    def remove_object(self, folder_id, file_id):
        """
        Remove a file from Google Drive
        
        Args:
            folder_id (str): Not used for Google Drive (included for API compatibility)
            file_id (str): Google Drive file ID
            
        Returns:
            bool: True if file was successfully deleted
        """
        try:
            self.service.files().delete(fileId=file_id).execute()
            return True
        except Exception as e:
            frappe.log_error(f"Google Drive delete error: {str(e)}")
            return False
    
    def stat_object(self, folder_id, file_id):
        """
        Get file metadata from Google Drive
        
        Args:
            folder_id (str): Not used for Google Drive (included for API compatibility)
            file_id (str): Google Drive file ID
            
        Returns:
            dict: File metadata
        """
        try:
            return self.service.files().get(
                fileId=file_id,
                fields="id,name,mimeType,size,modifiedTime,md5Checksum"
            ).execute()
        except Exception as e:
            frappe.log_error(f"Google Drive stat error: {str(e)}")
            raise
    
    def get_object(self, folder_id, file_id, offset=0, length=0):
        """
        Get file content from Google Drive
        
        Args:
            folder_id (str): Not used for Google Drive (included for API compatibility)
            file_id (str): Google Drive file ID
            offset (int): Start byte position (not directly supported by Google Drive)
            length (int): Number of bytes to read (not directly supported by Google Drive)
            
        Returns:
            BytesIO: File content as a file-like object
        """
        try:
            request = self.service.files().get_media(fileId=file_id)
            file_content = io.BytesIO()
            downloader = MediaIoBaseDownload(file_content, request)
            
            done = False
            while not done:
                status, done = downloader.next_chunk()
            
            # Reset position to beginning
            file_content.seek(0)
            
            # Handle offset and length if specified
            if offset > 0 or length > 0:
                file_content.seek(offset)
                if length > 0:
                    return io.BytesIO(file_content.read(length))
            
            return file_content
        except Exception as e:
            frappe.log_error(f"Google Drive download error: {str(e)}")
            raise
    
    def fget_object(self, folder_id, file_id, file_path):
        """
        Download file from Google Drive to a local path
        
        Args:
            folder_id (str): Not used for Google Drive (included for API compatibility)
            file_id (str): Google Drive file ID
            file_path (str): Local file path to save the file
            
        Returns:
            bool: True if file was successfully downloaded
        """
        try:
            request = self.service.files().get_media(fileId=file_id)
            with open(file_path, 'wb') as f:
                downloader = MediaIoBaseDownload(f, request)
                done = False
                while not done:
                    status, done = downloader.next_chunk()
            return True
        except Exception as e:
            frappe.log_error(f"Google Drive download to file error: {str(e)}")
            raise
    
    def put_object(self, folder_id, file_name, data, metadata=None, length=-1):
        """
        Upload file to Google Drive
        
        Args:
            folder_id (str): Google Drive folder ID to upload to
            file_name (str): Name for the uploaded file
            data: File-like object with data to upload
            metadata (dict): Additional metadata (not used for Google Drive)
            length (int): Data size (optional)
            
        Returns:
            dict: File metadata for the uploaded file
        """
        try:
            # Prepare file metadata
            file_metadata = {
                'name': file_name,
                'parents': [folder_id]
            }
            
            # Prepare media
            mimetype = metadata.get('Content-Type', 'application/octet-stream') if metadata else 'application/octet-stream'
            media = MediaIoBaseUpload(
                data, 
                mimetype=mimetype,
                resumable=True
            )
            
            # Upload file
            file = self.service.files().create(
                body=file_metadata,
                media_body=media,
                fields='id,name,mimeType,size,modifiedTime,md5Checksum'
            ).execute()
            
            return file
        except Exception as e:
            frappe.log_error(f"Google Drive upload error: {str(e)}")
            raise
    
    def list_objects(self, folder_id, recursive=True):
        """
        List files in a Google Drive folder
        
        Args:
            folder_id (str): Google Drive folder ID
            recursive (bool): Whether to list files in subfolders
            
        Returns:
            list: List of file metadata objects
        """
        try:
            query = f"'{folder_id}' in parents and trashed=false"
            
            page_token = None
            while True:
                response = self.service.files().list(
                    q=query,
                    spaces='drive',
                    fields='nextPageToken, files(id, name, mimeType, size, modifiedTime, md5Checksum, parents)',
                    pageToken=page_token
                ).execute()
                
                for file in response.get('files', []):
                    # Adapt Google Drive response to match S3 format
                    yield {
                        'object_name': file.get('name'),
                        'size': int(file.get('size', 0)) if file.get('size') else 0,
                        'etag': file.get('md5Checksum', ''),
                        'last_modified': file.get('modifiedTime'),
                        'is_dir': file.get('mimeType') == 'application/vnd.google-apps.folder',
                        'storage_class': 'GOOGLE_DRIVE',
                        'metadata': {
                            'id': file.get('id'),
                            'mime_type': file.get('mimeType')
                        }
                    }
                
                page_token = response.get('nextPageToken')
                if not page_token or not recursive:
                    break
                    
            # If recursive, get files from subfolders
            if recursive:
                # Find subfolders
                folder_query = f"'{folder_id}' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false"
                folders_response = self.service.files().list(
                    q=folder_query,
                    fields='files(id)'
                ).execute()
                
                # List files in each subfolder
                for subfolder in folders_response.get('files', []):
                    subfolder_id = subfolder.get('id')
                    yield from self.list_objects(subfolder_id, recursive)
                    
        except Exception as e:
            frappe.log_error(f"Google Drive list error: {str(e)}")
            yield None
    
    def presigned_get_object(self, folder_id, file_id, expires=timedelta(hours=3)):
        """
        Create a temporary shareable link for a Google Drive file
        
        Args:
            folder_id (str): Not used for Google Drive (included for API compatibility)
            file_id (str): Google Drive file ID
            expires (timedelta): How long the link should be valid
            
        Returns:
            str: Temporary shareable link
        """
        try:
            # Create a permission for anyone with the link to access the file
            # This is similar to a presigned URL in S3
            expiration = datetime.utcnow() + expires
            expires_epoch = int(expiration.timestamp())
            
            # Use Drive API to create a web view link
            # Note: This is different from S3 presigned URLs as it requires changing permissions
            file = self.service.files().get(
                fileId=file_id,
                fields='webViewLink'
            ).execute()
            
            # Return the web view link
            return file.get('webViewLink')
        except Exception as e:
            frappe.log_error(f"Google Drive presigned URL error: {str(e)}")
            return None


# Helper functions for Google Drive OAuth flow

def get_google_drive_oauth_url(client_id, client_secret, redirect_uri=None):
    """
    Generate OAuth URL for Google Drive authentication
    
    Args:
        client_id (str): Google API Client ID
        client_secret (str): Google API Client Secret
        redirect_uri (str): OAuth redirect URI
        
    Returns:
        str: OAuth URL for user authorization
    """
    if not redirect_uri:
        redirect_uri = get_url("/api/method/dfp_external_storage.gdrive_integration.oauth_callback")
        
    try:
        # Create OAuth flow
        flow = Flow.from_client_config(
            {
                "web": {
                    "client_id": client_id,
                    "client_secret": client_secret,
                    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
                    "token_uri": "https://oauth2.googleapis.com/token",
                    "redirect_uris": [redirect_uri]
                }
            },
            scopes=SCOPES,
            redirect_uri=redirect_uri
        )
        
        # Generate authorization URL
        auth_url, _ = flow.authorization_url(
            access_type='offline',
            include_granted_scopes='true',
            prompt='consent'  # Force to get refresh token
        )
        
        # Store flow information in cache for callback
        cache_key = f"{DFP_GDRIVE_TOKEN_CACHE_PREFIX}flow_{frappe.session.user}"
        frappe.cache().set_value(cache_key, flow.to_json())
        
        return auth_url
    except Exception as e:
        frappe.log_error(f"Google Drive OAuth URL generation error: {str(e)}")
        frappe.throw(_("Error generating Google Drive OAuth URL: {0}").format(str(e)))


@frappe.whitelist()
def oauth_callback():
    """Handle OAuth callback from Google"""
    try:
        # Get authorization code from request
        code = frappe.request.args.get('code')
        if not code:
            frappe.throw(_("Missing authorization code"))
            
        # Get flow from cache
        cache_key = f"{DFP_GDRIVE_TOKEN_CACHE_PREFIX}flow_{frappe.session.user}"
        flow_json = frappe.cache().get_value(cache_key)
        if not flow_json:
            frappe.throw(_("Authentication session expired"))
            
        # Recreate flow
        flow = Flow.from_json(flow_json)
        
        # Exchange code for tokens
        flow.fetch_token(code=code)
        
        # Get credentials
        credentials = flow.credentials
        
        # Store in session for later use
        session_key = f"{DFP_GDRIVE_TOKEN_CACHE_PREFIX}credentials_{frappe.session.user}"
        frappe.cache().set_value(session_key, {
            'refresh_token': credentials.refresh_token,
            'client_id': credentials.client_id,
            'client_secret': credentials.client_secret,
            'token': credentials.token,
            'expiry': credentials.expiry.isoformat() if credentials.expiry else None
        })
        
        # Clean up flow cache
        frappe.cache().delete_value(cache_key)
        
        # Redirect to success page or document
        success_url = frappe.cache().get_value(f"{DFP_GDRIVE_TOKEN_CACHE_PREFIX}success_url_{frappe.session.user}")
        if success_url:
            frappe.cache().delete_value(f"{DFP_GDRIVE_TOKEN_CACHE_PREFIX}success_url_{frappe.session.user}")
            frappe.local.response["type"] = "redirect"
            frappe.local.response["location"] = success_url
        else:
            # Default success message
            frappe.respond_as_web_page(
                _("Google Drive Authentication Successful"),
                _("You have successfully authenticated with Google Drive. You can close this window and return to the app."),
                indicator_color='green'
            )
    except Exception as e:
        frappe.log_error(f"Google Drive OAuth callback error: {str(e)}")
        frappe.respond_as_web_page(
            _("Google Drive Authentication Failed"),
            _("An error occurred during authentication: {0}").format(str(e)),
            indicator_color='red'
        )


@frappe.whitelist()
def initiate_google_drive_auth(doc_name, client_id, client_secret):
    """
    Initiate Google Drive OAuth flow from the DFP External Storage document
    
    Args:
        doc_name (str): DFP External Storage document name
        client_id (str): Google API Client ID
        client_secret (str): Google API Client Secret
        
    Returns:
        dict: Response with auth_url for redirection
    """
    try:
        # Set success URL in cache
        success_url = get_url(f"/desk#Form/DFP External Storage/{doc_name}")
        frappe.cache().set_value(f"{DFP_GDRIVE_TOKEN_CACHE_PREFIX}success_url_{frappe.session.user}", success_url)
        
        # Generate OAuth URL
        auth_url = get_google_drive_oauth_url(client_id, client_secret)
        
        return {
            "success": True,
            "auth_url": auth_url
        }
    except Exception as e:
        frappe.log_error(f"Error initiating Google Drive auth: {str(e)}")
        return {
            "success": False,
            "error": str(e)
        }


@frappe.whitelist()
def get_auth_credentials():
    """
    Get Google Drive auth credentials from session
    Returns credentials if they exist, None otherwise
    """
    session_key = f"{DFP_GDRIVE_TOKEN_CACHE_PREFIX}credentials_{frappe.session.user}"
    return frappe.cache().get_value(session_key)


@frappe.whitelist()
def test_google_drive_connection(doc_name=None, connection_data=None):
    """
    Test the connection to Google Drive
    
    Args:
        doc_name (str): DFP External Storage document name
        connection_data (dict): Connection data with client_id, client_secret, refresh_token, folder_id
        
    Returns:
        dict: Response with success status and message
    """
    try:
        if doc_name and not connection_data:
            # If we have a document name but no connection data, load it from the document
            doc = frappe.get_doc("DFP External Storage", doc_name)
            
            # Test the connection using document's stored credentials
            client_id = doc.google_client_id
            client_secret = frappe.utils.password.get_decrypted_password(
                "DFP External Storage", doc_name, "google_client_secret"
            )
            refresh_token = frappe.utils.password.get_decrypted_password(
                "DFP External Storage", doc_name, "google_refresh_token"
            )
            folder_id = doc.google_folder_id
            
        elif connection_data:
            # If connection data is provided, use it directly
            if isinstance(connection_data, str):
                connection_data = json.loads(connection_data)
                
            client_id = connection_data.get("client_id")
            client_secret = connection_data.get("client_secret")
            refresh_token = connection_data.get("refresh_token")
            folder_id = connection_data.get("folder_id")
            
        else:
            return {
                "success": False,
                "message": "No connection data provided"
            }
            
        # Validate required fields
        if not all([client_id, client_secret, refresh_token, folder_id]):
            missing = []
            if not client_id:
                missing.append("Client ID")
            if not client_secret:
                missing.append("Client Secret")
            if not refresh_token:
                missing.append("Refresh Token")
            if not folder_id:
                missing.append("Folder ID")
            
            return {
                "success": False,
                "message": f"Missing required fields: {', '.join(missing)}"
            }
            
        # Create connection and test
        connection = GoogleDriveConnection(
            client_id=client_id,
            client_secret=client_secret,
            refresh_token=refresh_token
        )
        
        if not connection.service:
            return {
                "success": False,
                "message": "Failed to connect to Google Drive API"
            }
            
        # Test folder access
        folder_exists = connection.validate_folder(folder_id)
        
        if folder_exists:
            return {
                "success": True,
                "message": f"Successfully connected to Google Drive and verified folder access"
            }
        else:
            return {
                "success": False,
                "message": "Connected to Google Drive but folder not found or not accessible"
            }
            
    except Exception as e:
        frappe.log_error(f"Error testing Google Drive connection: {str(e)}")
        return {
            "success": False,
            "message": f"Error testing connection: {str(e)}"
        }

google_drive.md

Google Drive Support for DFP External Storage

DFP External Storage now supports Google Drive as a storage backend! This feature allows you to store your Frappe/ERPNext files in Google Drive, alongside the existing S3-compatible storage options.

Features

• Store Frappe files in Google Drive folders
• Automatic OAuth 2.0 authentication flow
• Support for private and public files
• Streaming large files directly from Google Drive
• Presigned URL support for direct file access
• Folder-based storage configuration
• Extensive integration with existing DFP External Storage features

Setup Instructions

1. Install Dependencies

Install the required Google API libraries:

pip install google-api-python-client google-auth google-auth-oauthlib google-auth-httplib2

Or use the provided installation helper:

bench --site your-site.com execute dfp_external_storage/install_gdrive.py

2. Set Up Google Cloud Project

1. Create a new project in the Google Cloud Console
2. Enable the Google Drive API for your project
3. Create OAuth 2.0 credentials:
   • Application type: Web application
   • Authorized JavaScript origins: Your site URL (e.g., https://your-site.com)
   • Authorized redirect URIs: Your callback URL (e.g., https://your-site.com/api/method/dfp_external_storage.gdrive_integration.oauth_callback)

3. Configure in Frappe

1. Go to DFP External Storage list and create a new entry
2. Select "Google Drive" as the storage type
3. Enter your Google Client ID and Client Secret
4. Save the document
5. Click "Authenticate with Google Drive" and complete the OAuth flow
6. Find a Google Drive folder ID (from the folder's URL) and enter it
7. Save the document again

Usage

Once configured, the Google Drive storage works just like the S3 options:

1. Select folders to use this storage
2. Upload files normally through Frappe
3. Files will be stored in your Google Drive folder

You can also enable presigned URLs for direct file access, set up caching, and configure other advanced options just like with S3 storage.

Troubleshooting

If you encounter issues:

1. Check the Frappe error logs for detailed error messages
2. Verify your Google Cloud project has the Drive API enabled
3. Make sure your OAuth redirect URIs are correctly configured
4. Check that your Google Drive folder exists and is accessible
5. Try re-authenticating if your refresh token has expired

For more detailed instructions, see the full Google Drive Integration Guide.

google_drive_integration.md

Google Drive Integration Guide for DFP External Storage

This guide provides step-by-step instructions for integrating Google Drive support into the DFP External Storage app.

1. Install Required Dependencies

First, install the required Python packages for Google Drive API access:

pip install google-api-python-client google-auth google-auth-oauthlib

2. Add Google Drive Integration Module

1. Create a new file dfp_external_storage/gdrive_integration.py with the code from the "Google Drive Integration for DFP External Storage" artifact.

3. Update DocType Definition

1. Use Frappe's DocType editor to modify the "DFP External Storage" DocType, adding the new fields for Google Drive support:

   • Add "Google Drive" to the "Type" Select field options
   • Create a new Section Break "Google Drive Settings" with depends_on: eval:doc.type=='Google Drive'
   • Add fields for Google Client ID, Google Client Secret, Google Refresh Token, and Google Folder ID
   • Add an HTML field for the Google Auth Button

   Alternatively, you can import the JSON definition from the "Modified DFP External Storage DocType" artifact.

4. Update JavaScript Client Code

1. Replace or update the file dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.js with the code from the "Modified DFP External Storage JavaScript" artifact.

5. Update the DFP External Storage File Class

1. Modify the dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.py file to integrate the Google Drive file handling code.

2. Add the following methods to handle Google Drive files:

   # At the top of the file, add:
   from dfp_external_storage.gdrive_integration import GoogleDriveConnection
   
   # Add the DFPExternalStorageGoogleDriveFile class from the "DFP External Storage File Class Update" artifact
   
   # Then update the following methods in DFPExternalStorageFile class:

3. In the existing dfp_external_storage_upload_file method, add Google Drive support:

   def dfp_external_storage_upload_file(self, local_file=None):
       """Upload file to external storage"""
       # Check if this is a Google Drive storage
       if self.dfp_external_storage_doc and self.dfp_external_storage_doc.type == "Google Drive":
           gdrive_handler = DFPExternalStorageGoogleDriveFile(self)
           return gdrive_handler.upload_file(local_file)
           
       # Existing S3 upload logic
       # [...]

4. Similarly update the other methods to check for Google Drive storage type and call the appropriate handler.

6. Add OAuth Callback URL to Your Site Config

1. Edit your site_config.json file to enable the OAuth callback URL:

   {
     "host_name": "your-frappe-site.com",
     "oauth_redirect_uris": [
       "https://your-frappe-site.com/api/method/dfp_external_storage.gdrive_integration.oauth_callback"
     ]
   }

7. Set Up Google Cloud Project

1. Go to the Google Cloud Console and create a new project.

2. Enable the Google Drive API:

   • Navigate to "APIs & Services" > "Library"
   • Search for "Google Drive API" and enable it

3. Create OAuth 2.0 Credentials:

   • Go to "APIs & Services" > "Credentials"
   • Click "Create Credentials" > "OAuth client ID"
   • Select "Web application" as the Application type
   • Add your site's URL to the authorized JavaScript origins
   • Add the OAuth callback URL to the authorized redirect URIs: https://your-frappe-site.com/api/method/dfp_external_storage.gdrive_integration.oauth_callback
   • Click "Create" and note your Client ID and Client Secret

8. Testing Google Drive Integration

1. Create a new DFP External Storage entry:

   • Select "Google Drive" as the Type
   • Enter your Google Client ID and Client Secret
   • Save the document

2. Authenticate with Google Drive:

   • Click the "Authenticate with Google Drive" button
   • Complete the OAuth flow in the popup window
   • After successful authentication, you'll return to the document
   • Click "Apply Authentication" to apply the refresh token

3. Enter a Google Drive Folder ID:

   • To find a folder ID, navigate to the folder in Google Drive
   • The folder ID is in the URL: https://drive.google.com/drive/folders/FOLDER_ID

4. Test the connection:

   • The system will verify that it can access the specified folder

5. Save the document and try uploading files

9. Troubleshooting

OAuth Issues

• Ensure your redirect URIs in the Google Cloud Console match exactly with your site's callback URL
• Check browser console for any popup blocking issues
• Verify that your site is using HTTPS (required for OAuth)

File Upload/Download Issues

• Check the error logs in Frappe for detailed error messages
• Verify that your Google API credentials have the correct permissions
• Ensure the Google Drive folder ID is correct and accessible to your account

Connection Issues

• Check if your refresh token is valid and not expired
• Verify that the Google Drive API is enabled for your project
• Ensure your site can make outbound HTTPS connections

10. Advanced Configuration

Customizing Google Drive File Structure

You can modify the DFPExternalStorageGoogleDriveFile class to implement a custom folder structure in Google Drive. For example, you might want to create subfolders based on file types or Frappe DocTypes.

Implementing File Versioning

Google Drive supports file versioning. You could extend the integration to leverage this feature by:

1. Modifying the upload_file method to check for existing files with the same name
2. Using Google Drive's versioning API to update existing files instead of creating new ones

Adding Team Drive Support

To support Google Team Drives (Shared Drives):

1. Update the OAuth scope to include 'https://www.googleapis.com/auth/drive'
2. Modify the API calls to include the supportsTeamDrives=True parameter

Synchronization Features

You could implement background synchronization features:

1. Create a scheduled task to check for files that need to be synchronized
2. Add a UI to show synchronization status and history
3. Implement conflict resolution for files modified both locally and in Google Drive

install_gdrive.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
DFP External Storage - Google Drive Integration Installer

This script helps to ensure all dependencies for Google Drive integration
are properly installed.

Usage:
    bench --site [site-name] execute install_gdrive.py
"""

import frappe
import subprocess
import sys
import importlib.util
import os

REQUIRED_PACKAGES = [
    {'name': 'google-api-python-client', 'import_name': 'googleapiclient', 'min_version': '2.86.0'},
    {'name': 'google-auth', 'import_name': 'google.auth', 'min_version': '2.17.3'},
    {'name': 'google-auth-oauthlib', 'import_name': 'google_auth_oauthlib', 'min_version': '1.0.0'},
    {'name': 'google-auth-httplib2', 'import_name': 'google_auth_httplib2', 'min_version': '0.1.0'}
]

def check_dependency(package):
    """Check if a Python package is installed and meets minimum version requirements"""
    try:
        # Try to import the package
        spec = importlib.util.find_spec(package['import_name'])
        if spec is None:
            return False, f"Package {package['name']} is not installed"
        
        # If min_version is specified, check the version
        if package['min_version']:
            try:
                module = importlib.import_module(package['import_name'])
                if hasattr(module, '__version__'):
                    version = module.__version__
                    if version < package['min_version']:
                        return False, f"Package {package['name']} version {version} is lower than required {package['min_version']}"
            except:
                # Unable to check version, assume it's okay
                pass
        
        return True, f"Package {package['name']} is installed and meets requirements"
    except Exception as e:
        return False, f"Error checking {package['name']}: {str(e)}"

def install_dependency(package):
    """Attempt to install a package using pip"""
    package_spec = package['name']
    if package['min_version']:
        package_spec += f">={package['min_version']}"
    
    try:
        subprocess.check_call([sys.executable, "-m", "pip", "install", package_spec])
        return True, f"Successfully installed {package_spec}"
    except subprocess.CalledProcessError as e:
        return False, f"Failed to install {package_spec}: {str(e)}"

def ensure_dependencies():
    """Check and install all required dependencies"""
    all_installed = True
    results = []
    
    print("📋 Checking dependencies for Google Drive integration...")
    
    for package in REQUIRED_PACKAGES:
        installed, message = check_dependency(package)
        if not installed:
            print(f"⚠️ {message}")
            print(f"🔄 Installing {package['name']}...")
            success, install_message = install_dependency(package)
            results.append(f"- {package['name']}: {'✅' if success else '❌'} {install_message}")
            if not success:
                all_installed = False
        else:
            print(f"✅ {message}")
            results.append(f"- {package['name']}: ✅ Already installed")
    
    return all_installed, results

def check_gdrive_integration():
    """Check if Google Drive integration files are present"""
    gdrive_file = os.path.join(frappe.get_app_path('dfp_external_storage'), 'gdrive_integration.py')
    if not os.path.exists(gdrive_file):
        return False
    return True

def create_install_log(success, results):
    """Create a log file with installation results"""
    log_dir = os.path.join(frappe.get_site_path(), 'logs')
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)
    
    log_file = os.path.join(log_dir, 'dfp_gdrive_install.log')
    with open(log_file, 'w') as f:
        f.write("DFP External Storage - Google Drive Integration Installation Log\n")
        f.write(f"Timestamp: {frappe.utils.now()}\n")
        f.write(f"Status: {'SUCCESS' if success else 'FAILED'}\n\n")
        f.write("Dependency Check Results:\n")
        for result in results:
            f.write(f"{result}\n")
    
    return log_file

def main():
    """Main function to prepare for Google Drive integration"""
    if not check_gdrive_integration():
        print("❌ Google Drive integration files not found.")
        print("🔷 Please make sure you have added the gdrive_integration.py file to your app.")
        return
    
    dependencies_ok, results = ensure_dependencies()
    log_file = create_install_log(dependencies_ok, results)
    
    if dependencies_ok:
        print("\n✅ All dependencies for Google Drive integration are installed successfully.")
        print("🔷 You can now use Google Drive storage in DFP External Storage.")
        
        # Update site_config.json to add OAuth redirect URI
        try:
            site_url = frappe.utils.get_url()
            oauth_redirect_uri = f"{site_url}/api/method/dfp_external_storage.gdrive_integration.oauth_callback"
            
            print(f"\n🔷 Make sure to add the following OAuth redirect URI to your site_config.json:")
            print(f"   {oauth_redirect_uri}")
            
            print("\n🔷 Also add this URI to your Google Cloud Console OAuth 2.0 credentials.")
        except:
            print("\n⚠️ Could not determine site URL. Please manually configure OAuth redirect URIs.")
    else:
        print("\n❌ Some dependencies could not be installed.")
        print(f"📝 Check the log file for details: {log_file}")
        print("🔶 Try installing the dependencies manually:")
        print("   pip install google-api-python-client google-auth google-auth-oauthlib google-auth-httplib2")

if __name__ == "__main__":
    main()

requirements-gdrive.txt

# Google Drive API requirements for DFP External Storage
google-api-python-client>=2.86.0
google-auth>=2.17.3
google-auth-oauthlib>=1.0.0
google-auth-httplib2>=0.1.0