iOS 27: Media Intelligence Framework - Face Detection example

MediaIntelligenceView.swift

import SwiftUI
import UIKit
import MediaIntelligence


struct MediaIntelligenceView: View {
    var body: some View {
        VStack {
            if #available(iOS 27.0, *) {
                PeopleDemoView()
            } else {
                ContentUnavailableView(
                    "iOS 27 Required",
                    systemImage: "person.2.slash",
                    description: Text("This demo uses the beta Media Intelligence framework.")
                )
            }
        }
    }
}

#Preview {
    MediaIntelligenceView()
}

@available(iOS 27.0, *)
@MainActor
@Observable
final class PeopleDemoModel {

    enum LoadingState: Equatable {
        case idle
        case preparing
        case detecting(current: Int, total: Int)
        case grouping
        case finished
        case failed(String)

        var title: String {
            switch self {
            case .idle:
                "Ready"
            case .preparing:
                "Preparing images"
            case let .detecting(current, total):
                "Detecting faces \(current) of \(total)"
            case .grouping:
                "Grouping people"
            case .finished:
                "Analysis complete"
            case let .failed(message):
                message
            }
        }

        var isWorking: Bool {
            switch self {
            case .preparing, .detecting, .grouping:
                true
            default:
                false
            }
        }

        var progress: Double? {
            switch self {
            case let .detecting(current, total) where total > 0:
                Double(current) / Double(total)
            case .finished:
                1
            default:
                nil
            }
        }
    }

    struct Photo: Identifiable, Hashable {
        let id: MediaIntelligenceImageAsset.ID
        let name: String
        let url: URL
        let image: UIImage
    }

    struct DetectedFace: Identifiable, Hashable {
        let id: String
        let assetID: MediaIntelligenceImageAsset.ID
        let entityID: String?
        let bounds: CGRect
    }

    struct PersonGroup: Identifiable, Hashable {
        let id: String
        let faces: [DetectedFace]

        var assetIDs: [MediaIntelligenceImageAsset.ID] {
            Array(Set(faces.map(\.assetID))).sorted { String(describing: $0) < String(describing: $1) }
        }
    }

    private struct AnalysisResult {
        let facesByAssetID: [MediaIntelligenceImageAsset.ID: [DetectedFace]]
        let groups: [PersonGroup]
    }

    private(set) var state: LoadingState = .idle
    private(set) var photos: [Photo] = []
    private(set) var facesByAssetID: [MediaIntelligenceImageAsset.ID: [DetectedFace]] = [:]
    private(set) var groups: [PersonGroup] = []

    private let resourceNames = (0...4).map { "photo\($0)" }

    
    func analyze() async {
        guard !state.isWorking else { return }

        do {
            state = .preparing
            let preparedPhotos = try preparePhotos()
            photos = preparedPhotos

            let workingDirectory = URL.applicationSupportDirectory
                .appending(path: "FaceGroupData", directoryHint: .isDirectory)

            try FileManager.default.createDirectory(
                at: workingDirectory,
                withIntermediateDirectories: true
            )

            let assets = preparedPhotos.map { photo in
                MediaIntelligenceImageAsset(
                    id: photo.id,
                    kind: .url(photo.url)
                )
            }

            facesByAssetID = [:]
            groups = []

            state = .detecting(current: 0, total: assets.count)
            let result = try await analyzeAssets(
                assets,
                workingDirectory: workingDirectory
            )

            facesByAssetID = result.facesByAssetID
            groups = result.groups
            state = .finished
        } catch is CancellationError {
            state = .idle
        } catch {
            state = .failed(error.localizedDescription)
        }
    }

    func reset() async {
        do {
            let workingDirectory = URL.applicationSupportDirectory
                .appending(path: "FaceGroupData", directoryHint: .isDirectory)

            let isDirectory = try? workingDirectory.resourceValues(forKeys: [.isDirectoryKey]).isDirectory

            if isDirectory == nil {
                try FileManager.default.createDirectory(
                    at: workingDirectory,
                    withIntermediateDirectories: true
                )
            } else if isDirectory != true {
                throw DemoError.invalidWorkingDirectory
            }

            let analyzer = try FaceGroupAnalyzer(
                workingDirectory: workingDirectory
            )
            try await analyzer.deleteAllAssets()

            facesByAssetID = [:]
            groups = []
            state = .idle
        } catch {
            state = .failed(error.localizedDescription)
        }
    }

    func photo(for assetID: MediaIntelligenceImageAsset.ID) -> Photo? {
        photos.first { $0.id == assetID }
    }

    func photos(in group: PersonGroup) -> [Photo] {
        group.assetIDs.compactMap(photo(for:))
    }

    @concurrent
    private func analyzeAssets(
        _ assets: [MediaIntelligenceImageAsset],
        workingDirectory: URL
    ) async throws -> AnalysisResult {
        let analyzer = try FaceGroupAnalyzer(
            workingDirectory: workingDirectory
        )

        let stream = try await analyzer.insertOrUpdateAssets(assets)
        var facesByAssetID: [MediaIntelligenceImageAsset.ID: [DetectedFace]] = [:]

        for try await (assetID, faces) in stream {
            facesByAssetID[assetID] = faces.map { face in
                DetectedFace(
                    id: String(describing: face.id),
                    assetID: face.assetID,
                    entityID: face.entityID.map(String.init(describing:)),
                    bounds: face.bounds
                )
            }
        }

        if await analyzer.state == .stale {
            try await analyzer.update()
        }

        var loadedGroups: [PersonGroup] = []
        var refreshedFaces = facesByAssetID

        for try await (entityID, faces) in analyzer.allFacesByEntityID {
            let entityKey = String(describing: entityID)

            let mappedFaces = faces.map { face in
                DetectedFace(
                    id: String(describing: face.id),
                    assetID: face.assetID,
                    entityID: entityKey,
                    bounds: face.bounds
                )
            }

            loadedGroups.append(
                PersonGroup(
                    id: entityKey,
                    faces: mappedFaces
                )
            )

            for face in mappedFaces {
                guard var stored = refreshedFaces[face.assetID],
                      let index = stored.firstIndex(where: { $0.id == face.id })
                else {
                    continue
                }

                stored[index] = face
                refreshedFaces[face.assetID] = stored
            }
        }

        let groups = loadedGroups.sorted {
            if $0.faces.count == $1.faces.count {
                return $0.id < $1.id
            }
            return $0.faces.count > $1.faces.count
        }

        return AnalysisResult(
            facesByAssetID: refreshedFaces,
            groups: groups
        )
    }

    private func preparePhotos() throws -> [Photo] {
        let destinationDirectory = URL.applicationSupportDirectory
            .appending(path: "DemoPhotos", directoryHint: .isDirectory)

        try FileManager.default.createDirectory(
            at: destinationDirectory,
            withIntermediateDirectories: true
        )

        return try resourceNames.map { name in
            guard let sourceURL = Bundle.main.url(
                forResource: name,
                withExtension: "png"
            ) else {
                throw DemoError.missingResource("\(name).png")
            }

            let destinationURL = destinationDirectory
                .appending(path: "\(name).png")

            if !FileManager.default.fileExists(atPath: destinationURL.path) {
                try FileManager.default.copyItem(
                    at: sourceURL,
                    to: destinationURL
                )
            }

            guard let image = UIImage(contentsOfFile: destinationURL.path) else {
                throw DemoError.invalidImage(name)
            }

            return Photo(
                id: MediaIntelligenceImageAsset.ID(name),
                name: name,
                url: destinationURL,
                image: image
            )
        }
    }
}

@available(iOS 27.0, *)
private enum DemoError: LocalizedError {
    case missingResource(String)
    case invalidImage(String)
    case invalidWorkingDirectory

    var errorDescription: String? {
        switch self {
        case let .missingResource(filename):
            "Add \(filename) to the app target."
        case let .invalidImage(name):
            "The app could not decode \(name)."
        case .invalidWorkingDirectory:
            "FaceGroupData exists but is not a directory."
        }
    }
}

@available(iOS 27.0, *)
struct PeopleDemoView: View {

    @State private var model = PeopleDemoModel()

    var body: some View {
        NavigationStack {
            ScrollView {
                LazyVStack(alignment: .leading, spacing: 24) {
                    statusCard
                    detectedPhotosSection
                    groupedPeopleSection
                }
                .padding()
            }
            .navigationTitle("Media Intelligence")
            .toolbar {
                ToolbarItemGroup(placement: .topBarTrailing) {
                    Button("Reset", systemImage: "trash") {
                        Task { await model.reset() }
                    }
                    .disabled(model.state.isWorking)

                    Button("Analyze", systemImage: "sparkles") {
                        Task { await model.analyze() }
                    }
                    .disabled(model.state.isWorking)
                }
            }
//            .task {
//                if model.state == .idle {
//                    await model.analyze()
//                }
//            }
        }
    }

    private var statusCard: some View {
        HStack(spacing: 12) {
            if model.state.isWorking {
                ProgressView()
            } else {
                Image(systemName: statusSymbol)
                    .foregroundStyle(.secondary)
            }

            VStack(alignment: .leading, spacing: 8) {
                Text(model.state.title)
                    .font(.headline)

                Text("\(model.photos.count) photos · \(faceCount) faces · \(model.groups.count) groups")
                    .font(.subheadline)
                    .foregroundStyle(.secondary)

                if model.state.isWorking {
                    if let progress = model.state.progress {
                        ProgressView(value: progress)
                    } else {
                        ProgressView()
                    }
                }
            }

            Spacer()
        }
        .padding()
        .background(.thinMaterial, in: .rect(cornerRadius: 16))
    }

    @ViewBuilder
    private var detectedPhotosSection: some View {
        VStack(alignment: .leading, spacing: 12) {
            Text("Detected Faces")
                .font(.title2.bold())

            if faceCount == 0 && !model.state.isWorking {
                ContentUnavailableView(
                    "No Detected Faces",
                    systemImage: "face.smiling.inverse",
                    description: Text("Run analysis on photos that contain visible faces.")
                )
            } else {
                ForEach(model.photos) { photo in
                    VStack(alignment: .leading, spacing: 8) {
                        FaceOverlayImage(
                            photo: photo,
                            faces: model.facesByAssetID[photo.id, default: []]
                        )
                        .frame(height: 240)
                        .clipShape(.rect(cornerRadius: 16))

                        Text("\(photo.name) · \(model.facesByAssetID[photo.id, default: []].count) faces")
                            .font(.caption)
                            .foregroundStyle(.secondary)
                    }
                }
            }
        }
    }

    @ViewBuilder
    private var groupedPeopleSection: some View {
        VStack(alignment: .leading, spacing: 12) {
            Text("People Groups")
                .font(.title2.bold())

            if model.groups.isEmpty && !model.state.isWorking {
                ContentUnavailableView(
                    "No Groups",
                    systemImage: "person.2.slash",
                    description: Text("Add photos containing repeated appearances of the same people.")
                )
            } else {
                ForEach(Array(model.groups.enumerated()), id: \.element.id) { index, group in
                    NavigationLink {
                        PersonGroupView(
                            title: "Person \(index + 1)",
                            group: group,
                            photos: model.photos(in: group)
                        )
                    } label: {
                        PersonGroupRow(
                            title: "Person \(index + 1)",
                            group: group,
                            photos: model.photos(in: group)
                        )
                    }
                    .buttonStyle(.plain)
                }
            }
        }
    }

    private var faceCount: Int {
        model.facesByAssetID.values.reduce(0) { $0 + $1.count }
    }

    private var statusSymbol: String {
        switch model.state {
        case .finished:
            "checkmark.circle.fill"
        case .failed:
            "exclamationmark.triangle.fill"
        default:
            "person.2"
        }
    }
}

@available(iOS 27.0, *)
private struct FaceOverlayImage: View {

    let photo: PeopleDemoModel.Photo
    let faces: [PeopleDemoModel.DetectedFace]

    var body: some View {
        GeometryReader { proxy in
            let imageSize = photo.image.size
            let fitted = aspectFitRect(
                contentSize: imageSize,
                containerSize: proxy.size
            )

            ZStack(alignment: .topLeading) {
                Image(uiImage: photo.image)
                    .resizable()
                    .scaledToFit()
                    .frame(width: proxy.size.width, height: proxy.size.height)

                ForEach(faces) { face in
                    let rect = displayedFaceRect(
                        normalizedBounds: face.bounds,
                        imageRect: fitted
                    )

                    RoundedRectangle(cornerRadius: 6)
                        .stroke(.green, lineWidth: 3)
                        .frame(width: rect.width, height: rect.height)
                        .offset(x: rect.minX, y: rect.minY)
                        .accessibilityLabel("Detected face")
                }
            }
        }
        .background(.quaternary)
    }

    private func aspectFitRect(
        contentSize: CGSize,
        containerSize: CGSize
    ) -> CGRect {
        guard contentSize.width > 0,
              contentSize.height > 0,
              containerSize.width > 0,
              containerSize.height > 0
        else {
            return .zero
        }

        let scale = min(
            containerSize.width / contentSize.width,
            containerSize.height / contentSize.height
        )

        let size = CGSize(
            width: contentSize.width * scale,
            height: contentSize.height * scale
        )

        return CGRect(
            x: (containerSize.width - size.width) / 2,
            y: (containerSize.height - size.height) / 2,
            width: size.width,
            height: size.height
        )
    }

    private func displayedFaceRect(
        normalizedBounds: CGRect,
        imageRect: CGRect
    ) -> CGRect {
        CGRect(
            x: imageRect.minX + normalizedBounds.minX * imageRect.width,
            y: imageRect.minY + (1 - normalizedBounds.maxY) * imageRect.height,
            width: normalizedBounds.width * imageRect.width,
            height: normalizedBounds.height * imageRect.height
        )
    }
}

@available(iOS 27.0, *)
private struct PersonGroupRow: View {

    let title: String
    let group: PeopleDemoModel.PersonGroup
    let photos: [PeopleDemoModel.Photo]

    var body: some View {
        HStack(spacing: 12) {
            thumbnail

            VStack(alignment: .leading, spacing: 4) {
                Text(title)
                    .font(.headline)

                Text("\(group.faces.count) appearances in \(photos.count) photos")
                    .font(.subheadline)
                    .foregroundStyle(.secondary)
            }

            Spacer()
            Image(systemName: "chevron.right")
                .font(.caption.bold())
                .foregroundStyle(.tertiary)
        }
        .padding()
        .background(.thinMaterial, in: .rect(cornerRadius: 16))
    }

    @ViewBuilder
    private var thumbnail: some View {
        if let photo = photos.first {
            GroupedFaceImageView(
                photo: photo,
                faces: group.faces.filter { $0.assetID == photo.id }
            )
            .frame(width: 64, height: 64)
            .clipShape(.rect(cornerRadius: 12))
        } else {
            Image(systemName: "person.crop.square")
                .font(.largeTitle)
                .frame(width: 64, height: 64)
                .background(.quaternary, in: .rect(cornerRadius: 12))
        }
    }
}

@available(iOS 27.0, *)
private struct PersonGroupView: View {

    let title: String
    let group: PeopleDemoModel.PersonGroup
    let photos: [PeopleDemoModel.Photo]

    private let columns = [
        GridItem(.adaptive(minimum: 140), spacing: 12)
    ]

    var body: some View {
        ScrollView {
            LazyVGrid(columns: columns, spacing: 12) {
                ForEach(photos) { photo in
                    GroupedFacePhotoView(
                        photo: photo,
                        faces: group.faces.filter { $0.assetID == photo.id }
                    )
                }
            }
            .padding()
        }
        .navigationTitle(title)
        .navigationBarTitleDisplayMode(.inline)
    }
}

@available(iOS 27.0, *)
private struct GroupedFacePhotoView: View {

    let photo: PeopleDemoModel.Photo
    let faces: [PeopleDemoModel.DetectedFace]

    var body: some View {
        GroupedFaceImageView(photo: photo, faces: faces)
            .overlay(alignment: .bottomLeading) {
                Text(photo.name)
                    .font(.caption.bold())
                    .padding(8)
                    .foregroundStyle(.white)
                    .background(.black.opacity(0.55), in: .capsule)
                    .padding(8)
            }
            .frame(height: 160)
            .frame(maxWidth: .infinity)
            .clipShape(.rect(cornerRadius: 14))
    }
}

@available(iOS 27.0, *)
private struct GroupedFaceImageView: View {

    let photo: PeopleDemoModel.Photo
    let faces: [PeopleDemoModel.DetectedFace]

    var body: some View {
        GeometryReader { proxy in
            let imageRect = aspectFillRect(
                contentSize: photo.image.size,
                containerSize: proxy.size
            )

            ZStack(alignment: .topLeading) {
                Image(uiImage: photo.image)
                    .resizable()
                    .scaledToFill()
                    .frame(width: proxy.size.width, height: proxy.size.height)
                    .clipped()

                ForEach(faces) { face in
                    let rect = displayedFaceRect(
                        normalizedBounds: face.bounds,
                        imageRect: imageRect
                    )

                    RoundedRectangle(cornerRadius: 6)
                        .stroke(.green, lineWidth: 3)
                        .frame(width: rect.width, height: rect.height)
                        .offset(x: rect.minX, y: rect.minY)
                        .accessibilityLabel(face.entityID.map { "Grouped face entity \($0)" } ?? "Grouped face")
                }
            }
        }
    }

    private func aspectFillRect(
        contentSize: CGSize,
        containerSize: CGSize
    ) -> CGRect {
        guard contentSize.width > 0,
              contentSize.height > 0,
              containerSize.width > 0,
              containerSize.height > 0
        else {
            return .zero
        }

        let scale = max(
            containerSize.width / contentSize.width,
            containerSize.height / contentSize.height
        )

        let size = CGSize(
            width: contentSize.width * scale,
            height: contentSize.height * scale
        )

        return CGRect(
            x: (containerSize.width - size.width) / 2,
            y: (containerSize.height - size.height) / 2,
            width: size.width,
            height: size.height
        )
    }

    private func displayedFaceRect(
        normalizedBounds: CGRect,
        imageRect: CGRect
    ) -> CGRect {
        CGRect(
            x: imageRect.minX + normalizedBounds.minX * imageRect.width,
            y: imageRect.minY + (1 - normalizedBounds.maxY) * imageRect.height,
            width: normalizedBounds.width * imageRect.width,
            height: normalizedBounds.height * imageRect.height
        )
    }
}