SwiftDataReplaceTable.swift

func replaceCachedContacts(_ contacts: [UserOrContact]) throws {
        let modelContext = ModelContext(modelContainer)

        let internalIdSet = Set(contacts.map { $0.internalId })

        var internalIdsToNotInsert = Set<String>()
        var existingInternalIdsToDelete = Set<String>()

        let existingContacts = try modelContext.fetch(
            FetchDescriptor<UserOrContact>(
                predicate: #Predicate { internalIdSet.contains($0.internalId) }
            )
        )

        let internalIdsToExistingContacts = existingContacts
            .reduce(into: [String: UserOrContact]()) {
                $0[$1.internalId] = $1
            }

        for contact in contacts {
            if contact.equalProperties(to: internalIdsToExistingContacts[contact.internalId]) {
                internalIdsToNotInsert.insert(contact.internalId)
            } else {
                existingInternalIdsToDelete.insert(contact.internalId)
            }
        }

        try modelContext.transaction {
            // Delete models which have changed or – if deleteOthers is true – do not exist in
            // the models parameter
            try modelContext.delete(model: UserOrContact.self, where: #Predicate {
                existingInternalIdsToDelete.contains($0.internalId) || !internalIdSet.contains($0.internalId)
            })

            // Insert new and changed models
            for contact in contacts {
                guard !internalIdsToNotInsert.contains(contact.internalId) else {
                    continue
                }

                modelContext.insert(contact)
            }
        }
    }

Hey Robert! First off, really appreciate you taking the time to provide some feedback. It's all very helpful and I've incorporated all of it into my project.

It's an excellent point to avoid instantiating PersistentModel instances for objects that will (very likely) be discarded; using a struct instead should improve the performance here.