Expo SDK 47 ImagePicker media handler modified to support Swift async-await syntax

MediaHandler.swift

// Copyright 2022-present 650 Industries. All rights reserved.

import ExpoModulesCore
import MobileCoreServices
import Photos
import PhotosUI

internal struct MediaHandler {
  internal weak var fileSystem: EXFileSystemInterface?
  internal let options: ImagePickerOptions
  
  internal func handleMedia(_ mediaInfo: MediaInfo, completion: @escaping (ImagePickerResult) -> Void) {
    let mediaType: String? = mediaInfo[UIImagePickerController.InfoKey.mediaType] as? String
    let imageType = kUTTypeImage as String
    let videoType = kUTTypeMovie as String
    
    Task {
      do {
        switch mediaType {
        case imageType: return completion(.success(try await handleImage(mediaInfo: mediaInfo)))
        case videoType: return completion(.success(try await handleVideo(mediaInfo: mediaInfo)))
        default: return completion(.failure(InvalidMediaTypeException(mediaType)))
        }
      } catch let exception as Exception {
        return completion(.failure(exception))
      } catch {
        return completion(.failure(UnexpectedException(error)))
      }
    }
  }
  
  @available(iOS 14, *)
  internal func handleMultipleMedia(_ selection: [PHPickerResult], completion: @escaping (ImagePickerResult) -> Void) {
    Task {
      async let getResults = withThrowingTaskGroup(of: (Int, AssetInfo).self) { group in
        var results = [AssetInfo?](repeating: nil, count: selection.count)
        
        // concurrently process each result
        for (index, selectedItem) in selection.enumerated() {
          group.addTask {
            let itemProvider = selectedItem.itemProvider
            if itemProvider.hasItemConformingToTypeIdentifier(UTType.image.identifier) {
              return try await handleImage(from: selectedItem, atIndex: index)
            } else if itemProvider.hasItemConformingToTypeIdentifier(UTType.movie.identifier) {
              return try await handleVideo(from: selectedItem, atIndex: index)
            } else {
              throw InvalidMediaTypeException(itemProvider.registeredTypeIdentifiers.first)
            }
          }
        }
        
        // aggregate them, preserving their selection index
        for try await (index, item) in group {
          results[index] = item
        }
        return results
      }
      
      do {
        let results = try await getResults;
        return completion(.success(
          ImagePickerResponse(assets: results.compactMap({ $0 }), canceled: false)
        ))
      } catch let exception as Exception {
        return completion(.failure(exception))
      } catch {
        return completion(.failure(UnexpectedException(error)))
      }
    }
  }
  
  // MARK: - Image
  
  private func handleImage(mediaInfo: MediaInfo) async throws -> ImagePickerResponse {
    guard let image = ImageUtils.readImageFrom(mediaInfo: mediaInfo, shouldReadCroppedImage: options.allowsEditing) else {
      throw FailedToReadImageException()
    }
    
    let (imageData, fileExtension) = try ImageUtils.readDataAndFileExtension(image: image,
                                                                             mediaInfo: mediaInfo,
                                                                             options: options)
    
    let targetUrl = try generateUrl(withFileExtension: fileExtension)
    
    // no modification requested
    let imageModified = options.allowsEditing || options.quality != nil
    let fileWasCopied = !imageModified && ImageUtils.tryCopyingOriginalImageFrom(mediaInfo: mediaInfo, to: targetUrl)
    if !fileWasCopied {
      try ImageUtils.write(imageData: imageData, to: targetUrl)
    }
    
    // as calling this already requires media library permission, we can access it here
    // if user gave limited permissions, in the worst case this will be null
    let asset = mediaInfo[.phAsset] as? PHAsset
    let fileName = asset?.value(forKey: "filename") as? String
    let fileSize = getFileSize(from: targetUrl)
    
    let base64 = try ImageUtils.optionallyReadBase64From(imageData: imageData,
                                                         orImageFileUrl: targetUrl,
                                                         tryReadingFile: fileWasCopied,
                                                         shouldReadBase64: self.options.base64)
    
    let exif = await ImageUtils.optionallyReadExifFrom(mediaInfo: mediaInfo, shouldReadExif: self.options.exif)
    let imageInfo = AssetInfo(assetId: asset?.localIdentifier,
                              uri: targetUrl.absoluteString,
                              width: image.size.width,
                              height: image.size.height,
                              fileName: fileName,
                              fileSize: fileSize,
                              base64: base64,
                              exif: exif)
    return ImagePickerResponse(assets: [imageInfo], canceled: false)
  }
  
  @available(iOS 14, *)
  private func handleImage(from selectedImage: PHPickerResult,
                           atIndex index: Int = -1) async throws -> (Int, AssetInfo) {
    let itemProvider = selectedImage.itemProvider
    let rawData = try await itemProvider.loadDataRepresentation(forTypeIdentifier: UTType.image.identifier)
    
    guard let rawData = rawData,
          let image = UIImage(data: rawData) else {
      throw FailedToReadImageException()
    }
    
    let (imageData, fileExtension) = try ImageUtils.readDataAndFileExtension(image: image,
                                                                             rawData: rawData,
                                                                             itemProvider: itemProvider,
                                                                             options: self.options)
    
    let targetUrl = try generateUrl(withFileExtension: fileExtension)
    try ImageUtils.write(imageData: imageData, to: targetUrl)
    let fileSize = getFileSize(from: targetUrl)
    let fileName = itemProvider.suggestedName.map { $0 + fileExtension }
    
    // We need to get EXIF from original image data, as it is being lost in UIImage
    let exif = ImageUtils.optionallyReadExifFrom(data: rawData, shouldReadExif: self.options.exif)
    
    let base64 = try ImageUtils.optionallyReadBase64From(imageData: imageData,
                                                         orImageFileUrl: targetUrl,
                                                         tryReadingFile: false,
                                                         shouldReadBase64: self.options.base64)
    
    let assetInfo =  AssetInfo(assetId: selectedImage.assetIdentifier,
                     uri: targetUrl.absoluteString,
                     width: image.size.width,
                     height: image.size.height,
                     fileName: fileName,
                     fileSize: fileSize,
                     base64: base64,
                     exif: exif)
    return (index, assetInfo)
  }
  
  
  // MARK: - Video

  func handleVideo(mediaInfo: MediaInfo) async throws -> ImagePickerResponse {
    guard let pickedVideoUrl = VideoUtils.readVideoUrlFrom(mediaInfo: mediaInfo) else {
      throw FailedToReadVideoException()
    }
    
    let targetUrl = try generateUrl(withFileExtension: ".mov")
    
    try VideoUtils.tryCopyingVideo(at: pickedVideoUrl, to: targetUrl)
    
    guard let dimensions = VideoUtils.readSizeFrom(url: targetUrl) else {
      throw FailedToReadVideoSizeException()
    }
    
    // If video was edited (the duration is affected) then read the duration from the original edited video.
    // Otherwise read the duration from the target video file.
    // TODO: (@bbarthec): inspect whether it makes sense to read duration from two different assets
    let videoUrlToReadDurationFrom = options.allowsEditing ? pickedVideoUrl : targetUrl
    let duration = VideoUtils.readDurationFrom(url: videoUrlToReadDurationFrom)
    
    let asset = mediaInfo[.phAsset] as? PHAsset
    let fileName = asset?.value(forKey: "filename") as? String
    let fileSize = getFileSize(from: targetUrl)
    let videoInfo = AssetInfo(assetId: asset?.localIdentifier,
                              type: "video",
                              uri: targetUrl.absoluteString,
                              width: dimensions.width,
                              height: dimensions.height,
                              fileName: fileName,
                              fileSize: fileSize,
                              duration: duration)
    
    return ImagePickerResponse(assets: [videoInfo], canceled: false)
  }
  
  @available(iOS 14, *)
  private func handleVideo(from selectedVideo: PHPickerResult,
                           atIndex index: Int = -1) async throws -> (Int, AssetInfo) {
    let itemProvider = selectedVideo.itemProvider
    
    guard let videoUrl = try await itemProvider.loadFileRepresentation(forTypeIdentifier: UTType.movie.identifier)
    else {
      throw FailedToReadVideoException()
    }
    
    
    // In case of passthrough, we want original file extension, mp4 otherwise
    // TODO: (barthap) Support other file extensions?
    let transcodeFileType = AVFileType.mp4
    let transcodeFileExtension = ".mp4"
    let originalExtension = ".\(videoUrl.pathExtension)"
    
    // We need to copy the result into a place that we control, because the picker
    // can remove the original file during conversion.
    // Also, the transcoding may need a separate url - one of these will be used as a final result
    let assetUrl = try generateUrl(withFileExtension: originalExtension)
    let transcodedUrl = try generateUrl(withFileExtension: transcodeFileExtension)
    try VideoUtils.tryCopyingVideo(at: videoUrl, to: assetUrl)
    
    let targetUrl = try await VideoUtils.transcodeVideoAsync(sourceAssetUrl: assetUrl,
                                                             destinationUrl: transcodedUrl,
                                                             outputFileType: transcodeFileType,
                                                             exportPreset: options.videoExportPreset)
    
    let fileName = itemProvider.suggestedName.map { $0 + transcodeFileExtension }
    let videoResult = try buildVideoResult(for: targetUrl, withName: fileName, assetId: selectedVideo.assetIdentifier)
    return (index, videoResult)
    
  }
  
  // MARK: - utils
  
  private func generateUrl(withFileExtension: String) throws -> URL {
    guard let fileSystem = self.fileSystem else {
      throw FileSystemModuleNotFoundException()
    }
    let directory =  fileSystem.cachesDirectory.appending(
      fileSystem.cachesDirectory.hasSuffix("/") ? "" : "/" + "ImagePicker"
    )
    let path = fileSystem.generatePath(inDirectory: directory, withExtension: withFileExtension)
    let url = URL(fileURLWithPath: path)
    return url
  }
  
  private func buildVideoResult(for videoUrl: URL, withName fileName: String?, assetId: String?) throws -> AssetInfo {
    guard let size = VideoUtils.readSizeFrom(url: videoUrl) else {
      throw FailedToReadVideoSizeException()
    }
    let duration = VideoUtils.readDurationFrom(url: videoUrl)
    let fileSize = getFileSize(from: videoUrl)
    
    return AssetInfo(assetId: assetId,
                     type: "video",
                     uri: videoUrl.absoluteString,
                     width: size.width,
                     height: size.height,
                     fileName: fileName,
                     fileSize: fileSize,
                     duration: duration)
  }
  
  private func getFileSize(from fileUrl: URL) -> Int? {
    do {
      let resources = try fileUrl.resourceValues(forKeys: [.fileSizeKey])
      return resources.fileSize
    } catch {
      log.error("Failed to get file size for \(fileUrl.absoluteString)")
      return nil
    }
  }
  
  
  private struct ImageUtils {
    static func readImageFrom(mediaInfo: MediaInfo, shouldReadCroppedImage: Bool) -> UIImage? {
      guard let originalImage = mediaInfo[.originalImage] as? UIImage,
            let image = originalImage.fixOrientation()
      else {
        return nil
      }
      
      if !shouldReadCroppedImage {
        return image
      }
      
      guard let cropRect = mediaInfo[.cropRect] as? CGRect,
            let croppedImage = ImageUtils.crop(image: image, to: cropRect)
      else {
        return nil
      }
      
      return croppedImage
    }
    
    static func crop(image: UIImage, to: CGRect) -> UIImage? {
      guard let cgImage = image.cgImage?.cropping(to: to) else {
        return nil
      }
      return UIImage(cgImage: cgImage,
                     scale: image.scale,
                     orientation: image.imageOrientation)
    }
    
    static func readDataAndFileExtension(
      image: UIImage,
      mediaInfo: MediaInfo,
      options: ImagePickerOptions
    ) throws -> (imageData: Data?, fileExtension: String) {
      let compressionQuality = options.quality ?? DEFAULT_QUALITY
      
      // nil when an image is picked from camera
      let referenceUrl = mediaInfo[.referenceURL] as? URL
      
      switch referenceUrl?.absoluteString {
      case .some(let s) where s.contains("ext=PNG"):
        let data = image.pngData()
        return (data, ".png")
        
      case .some(let s) where s.contains("ext=BMP"):
        if options.allowsEditing || options.quality != nil {
          // switch to png if editing
          let data = image.pngData()
          return (data, ".png")
        }
        return (nil, ".bmp")
        
      case .some(let s) where s.contains("ext=GIF"):
        var rawData: Data?
        if let imgUrl = mediaInfo[.imageURL] as? URL {
          rawData = try? Data(contentsOf: imgUrl)
        }
        let inputData = rawData ?? image.jpegData(compressionQuality: compressionQuality)
        let metadata = mediaInfo[.mediaMetadata] as? [String: Any]
        let cropRect = options.allowsEditing ? mediaInfo[.cropRect] as? CGRect : nil
        let gifData = try processGifData(inputData: inputData,
                                         compressionQuality: options.quality,
                                         initialMetadata: metadata,
                                         cropRect: cropRect)
        return (gifData, ".gif")
      default:
        let data = image.jpegData(compressionQuality: compressionQuality)
        return (data, ".jpg")
      }
    }
    
    @available(iOS 14, *)
    static func readDataAndFileExtension(
      image: UIImage,
      rawData: Data,
      itemProvider: NSItemProvider,
      options: ImagePickerOptions
    ) throws -> (imageData: Data?, fileExtension: String) {
      let compressionQuality = options.quality ?? DEFAULT_QUALITY
      let preferredFormat = itemProvider.registeredTypeIdentifiers.first
      
      switch preferredFormat {
      case UTType.png.identifier:
        let data = image.pngData()
        return (data, ".png")
      case UTType.gif.identifier:
        let gifData = try processGifData(inputData: rawData,
                                         compressionQuality: options.quality,
                                         initialMetadata: nil)
        return (gifData, ".gif")
      default:
        let data = image.jpegData(compressionQuality: compressionQuality)
        return (data, ".jpg")
      }
    }
    
    static func write(imageData: Data?, to: URL) throws {
      do {
        try imageData?.write(to: to, options: [.atomic])
      } catch {
        throw FailedToWriteImageException()
          .causedBy(error)
      }
    }
    
    /**
     @returns `true` upon copying success and `false` otherwise
     */
    static func tryCopyingOriginalImageFrom(mediaInfo: MediaInfo, to: URL) -> Bool {
      guard let from = mediaInfo[.imageURL] as? URL else {
        return false
      }
      
      do {
        try FileManager.default.copyItem(atPath: from.path, toPath: to.path)
        return true
      } catch {
        return false
      }
    }
    
    /**
     Reads base64 representation of the image data. If the data is `nil` fallbacks to reading the data from the url.
     */
    static func optionallyReadBase64From(
      imageData: Data?,
      orImageFileUrl url: URL,
      tryReadingFile: Bool,
      shouldReadBase64: Bool
    ) throws -> String? {
      if !shouldReadBase64 {
        return nil
      }
      
      if tryReadingFile {
        do {
          let data = try Data(contentsOf: url)
          return data.base64EncodedString()
        } catch {
          throw FailedToReadImageDataException()
            .causedBy(error)
        }
      }
      
      guard let data = imageData else {
        throw FailedToReadImageDataForBase64Exception()
      }
      
      return data.base64EncodedString()
    }
    
    static func optionallyReadExifFrom(
      mediaInfo: MediaInfo,
      shouldReadExif: Bool
    ) async -> ExifInfo? {
      if !shouldReadExif {
        return nil
      }
      
      let metadata = mediaInfo[.mediaMetadata] as? [String: Any]
      
      if metadata != nil {
        let exif = ImageUtils.readExifFrom(imageMetadata: metadata!)
        return exif
      }
      
      guard let imageUrl = mediaInfo[.referenceURL] as? URL else {
        log.error("Could not fetch metadata for image")
        return nil
      }
      
      let assets = PHAsset.fetchAssets(withALAssetURLs: [imageUrl], options: nil)
      
      guard let asset = assets.firstObject else {
        log.error("Could not fetch metadata for image '\(imageUrl.absoluteString)'.")
        return nil
      }
      
      let options = PHContentEditingInputRequestOptions()
      options.isNetworkAccessAllowed = true
      let (input, _) = await asset.requestContentEditingInput(with: options)
      
      guard let imageUrl = input?.fullSizeImageURL,
            let properties = CIImage(contentsOf: imageUrl)?.properties
      else {
        log.error("Could not fetch metadata for '\(imageUrl.absoluteString)'.")
        return nil
      }
      return ImageUtils.readExifFrom(imageMetadata: properties)
    }
    
    static func optionallyReadExifFrom(data: Data, shouldReadExif: Bool) -> ExifInfo? {
      if shouldReadExif,
         let cgImageSource = CGImageSourceCreateWithData(data as CFData, nil),
         let properties = CGImageSourceCopyPropertiesAtIndex(cgImageSource, 0, nil) {
        return ImageUtils.readExifFrom(imageMetadata: properties as! [String: Any])
      }
      return nil
    }
    
    static func readExifFrom(imageMetadata: [String: Any]) -> ExifInfo {
      var exif: ExifInfo = imageMetadata[kCGImagePropertyExifDictionary as String] as? ExifInfo ?? [:]
      
      // Copy ["{GPS}"]["<tag>"] to ["GPS<tag>"]
      let gps = imageMetadata[kCGImagePropertyGPSDictionary as String] as? [String: Any]
      if gps != nil {
        gps!.forEach { key, value in
          exif["GPS\(key)"] = value
        }
      }
      
      // Inject orientation into exif
      let orientationKey = kCGImagePropertyOrientation as String
      let orientationValue = imageMetadata[orientationKey]
      if orientationValue != nil {
        exif[orientationKey] = orientationValue
      }
      
      return exif
    }
    
    static func processGifData(inputData: Data?,
                               compressionQuality quality: Double?,
                               initialMetadata: [String: Any]?,
                               cropRect: CGRect? = nil) throws -> Data? {
      // for uncropped, maximum quality image we can just pass through the raw data
      if cropRect == nil,
         quality == nil || quality == MAXIMUM_QUALITY {
        return inputData
      }
      
      guard let sourceData = inputData,
            let imageSource = CGImageSourceCreateWithData(sourceData as CFData, nil)
      else {
        throw FailedToReadImageException()
      }
      
      let gifProperties = CGImageSourceCopyProperties(imageSource, nil) as? [String: Any]
      let frameCount = CGImageSourceGetCount(imageSource)
      
      let destinationData = NSMutableData()
      guard let imageDestination = CGImageDestinationCreateWithData(destinationData, kUTTypeGIF, frameCount, nil)
      else {
        throw FailedToCreateGifException()
      }
      
      let gifMetadata = initialMetadata ?? gifProperties
      CGImageDestinationSetProperties(imageDestination, gifMetadata as CFDictionary?);
      
      for frameIndex in 0 ..< frameCount {
        guard var cgImage = CGImageSourceCreateImageAtIndex(imageSource, frameIndex, nil),
              var frameProperties = CGImageSourceCopyPropertiesAtIndex(imageSource, frameIndex, nil) as? [String: Any]
        else {
          throw FailedToCreateGifException()
        }
        if cropRect != nil {
          cgImage = cgImage.cropping(to: cropRect!)!
        }
        if quality != nil {
          frameProperties[kCGImageDestinationLossyCompressionQuality as String] = quality
        }
        CGImageDestinationAddImage(imageDestination, cgImage, frameProperties as CFDictionary)
      }
      
      if !CGImageDestinationFinalize(imageDestination) {
        throw FailedToExportGifException()
      }
      
      return destinationData as Data
    }
  }
  
  private struct VideoUtils {
    static func tryCopyingVideo(at: URL, to: URL) throws {
      do {
        // we copy the file as `moveItem(at:,to:)` throws an error in iOS 13 due to missing permissions
        try FileManager.default.copyItem(at: at, to: to)
      } catch {
        throw FailedToPickVideoException()
          .causedBy(error)
      }
    }
    
    /**
     @returns duration in milliseconds
     */
    static func readDurationFrom(url: URL) -> Double {
      let asset = AVURLAsset(url: url)
      return Double(asset.duration.value) / Double(asset.duration.timescale) * 1_000
    }
    
    static func readSizeFrom(url: URL) -> CGSize? {
      let asset = AVURLAsset(url: url)
      let size: CGSize? = asset.tracks(withMediaType: .video).first?.naturalSize
      return size
    }
    
    static func readVideoUrlFrom(mediaInfo: MediaInfo) -> URL? {
      return mediaInfo[.mediaURL] as? URL
      ?? mediaInfo[.referenceURL] as? URL
    }
    
    /**
     Asynchronously transcodes asset provided as `sourceAssetUrl` according to `exportPreset`.
     Result URL is returned to the `completion` closure.
     Transcoded video is saved at `destinationUrl`, unless `exportPreset` is set to `passthrough`.
     In this case, `sourceAssetUrl` is returned.
     */
    static func transcodeVideoAsync(sourceAssetUrl: URL,
                                    destinationUrl: URL,
                                    outputFileType: AVFileType,
                                    exportPreset: VideoExportPreset) async throws -> URL {
      if case .passthrough = exportPreset {
        return sourceAssetUrl
      }
      
      let asset = AVURLAsset(url: sourceAssetUrl)
      let preset = exportPreset.toAVAssetExportPreset()
      let canBeTranscoded = await AVAssetExportSession.compatibility(ofExportPreset: preset,
                                                                     with: asset,
                                                                     outputFileType: outputFileType)
      guard canBeTranscoded else {
        throw UnsupportedVideoExportPresetException(preset.description)
      }
      guard let exportSession = AVAssetExportSession(asset: asset,
                                                     presetName: preset) else {
        throw FailedToTranscodeVideoException()
      }
      exportSession.outputFileType = outputFileType
      exportSession.outputURL = destinationUrl
      try await exportSession.awaitExport()
      return destinationUrl
    }
  }
}

extension NSItemProvider {
  // TODO: (Barthap) It should be possible to use `async NSItemProvider.loadItem()` instead
  // but this has some strange type-casting policy. Need to investigate this more:
  // https://developer.apple.com/documentation/foundation/nsitemprovider/1403900-loaditem
  func loadDataRepresentation(forTypeIdentifier typeIdentifier: String) async throws -> Data? {
    try await withCheckedThrowingContinuation { continuation in
      self.loadDataRepresentation(forTypeIdentifier: typeIdentifier) { data, error in
        guard error == nil else {
          return continuation.resume(throwing: FailedToReadImageException().causedBy(error!))
        }
        continuation.resume(returning: data)
      }
    }
  }
  
  func loadFileRepresentation(forTypeIdentifier typeIdentifier: String) async throws -> URL? {
    try await withCheckedThrowingContinuation { continuation in
      self.loadFileRepresentation(forTypeIdentifier: typeIdentifier) { url, error in
        guard error == nil else {
          return continuation.resume(throwing: FailedToReadVideoException().causedBy(error!))
        }
        continuation.resume(returning: url)
      }
    }
  }
}

extension AVAssetExportSession {
  func awaitExport() async throws {
    try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
      self.exportAsynchronously {
        switch self.status {
        case .failed:
          continuation.resume(throwing: FailedToTranscodeVideoException().causedBy(self.error))
        default:
          continuation.resume()
        }
      }
    }
  }
}

extension PHAsset {
  func requestContentEditingInput(with options: PHContentEditingInputRequestOptions?) async -> (PHContentEditingInput?, [AnyHashable : Any]) {
    await withCheckedContinuation { continuation in
      self.requestContentEditingInput(with: options) {input, info in
        continuation.resume(returning: (input, info))
      }
    }
  }
}