Observe presented UIViewController is dismissed with UIModalPresentationStyle is custom

UIViewController+Reactive.swift

extension Reactive where Base: UIViewController {
    var presenting: Observable<UIViewController> {
        return sentMessage(#selector(Base.present(_:animated:completion:)))
        .compactMap({ args -> UIViewController? in
            return args.first as? UIViewController
        })
        .do(onError: { error in
            assertionFailure("\(error)")
        })
    }

    var dismissing: Observable<Void> {
        return sentMessage(#selector(Base.dismiss(animated:completion:)))
        .map { _ in Void() }
        .do(onError: { error in
            assertionFailure("\(error)")
        })
    }

    var presentedViewControllerDismissed: Observable<UIViewController?> {
        return makePresentingVC().flatMap({ presentedVC -> Observable<UIViewController?> in
            let topVC: UIViewController? = (presentedVC as? UINavigationController)?.topViewController
            let vc: UIViewController = topVC ?? presentedVC
            return vc.rx.dismissing
                .map({ [weak vc] _ -> UIViewController? in
                    return vc
                })
        })
    }
    
    private func makePresentingVC() -> Observable<UIViewController> {
        if let presentedVC: UIViewController = base.presentedViewController {
            return Observable.merge(
                Observable<UIViewController>.just(presentedVC),
                presenting
            )
        }
        return presenting
    }
}

How to use

let yourVC = UIViewController()
let disposeBag = DisposeBage()

yourVC.rx.presentedViewControllerDismissed
            .subscribe(onNext: { [weak self] presentedVC in
                print("presentedVC is dismissed", presentedVC)
            })
            .disposed(by: disposeBag)