A sketch of safely(?) passing a mutable reference into Python

pyo3_mut_atomic.rs

// Fifth attempt at safe wrapper, this time using atomics to avoid race
// when dropped without holding GIL.
// User code is same as pyo3_mut_rc

mod safe_wrapper {
    use pyo3::Python;
    use std::sync::atomic::{AtomicPtr, Ordering};

    pub struct SafeWrapper<T>(*mut AtomicPtr<T>);

    unsafe impl<T> Send for SafeWrapper<T> {}

    impl<T> Drop for SafeWrapper<T> {
        fn drop(&mut self) {
            unsafe {
                let box_ptr = self.0;
                let ptr = &*(box_ptr as *const AtomicPtr<String>);
                let old = ptr.load(Ordering::Acquire);
                ptr.store(std::ptr::null_mut(), Ordering::Release);
                if old.is_null() {
                    std::mem::drop(Box::from_raw(box_ptr));
                }
            }
        }
    }

    impl<T> SafeWrapper<T> {
        pub fn scoped<'p, U>(
            _py: Python<'p>,
            obj: &mut T,
            f: impl FnOnce(SafeWrapper<T>) -> U,
        ) -> U {
            let box_ptr = Box::into_raw(Box::new(AtomicPtr::new(obj)));
            let wrapper = SafeWrapper(box_ptr);
            let result = f(wrapper);
            std::mem::drop(SafeWrapper(box_ptr));
            result
        }

        pub fn try_get_mut<'p>(&mut self, _py: Python<'p>) -> Option<&mut T> {
            unsafe {
                let ptr = (*self.0).load(Ordering::Relaxed);
                if ptr.is_null() {
                    None
                } else {
                    Some(&mut *ptr)
                }
            }
        }
    }
}

pyo3_mut_ptr.rs

use safe_wrapper::SafeWrapper;

/// A object that holds a mutable reference, that can be passed to Python.
#[pyclass]
struct StringRef {
    inner: SafeWrapper<String>,
}

#[pymethods]
impl StringRef {
    fn foo(&mut self, py: Python<'_>) {
        if let Some(s) = self.inner.try_get_mut(py) {
            s.push_str(", world");
        } else {
            println!("trying to use reference after borrow lifetime");
        }
    }
}

impl StringRef {
    fn new(inner: safe_wrapper::SafeWrapper<String>) -> Self {
        StringRef { inner }
    }
}

#[pyfunction]
fn test_string_ref(py: Python<'_>) -> PyResult<()> {
    let mut s = "hello".to_string();
    let mut leaked_cell = None;
    SafeWrapper::scoped(py, &mut s, StringRef::new, |s_cell| {
        leaked_cell = Some(s_cell);
        py_run!(py, s_cell, "s_cell.foo()");
    });
    if let Some(s_cell) = leaked_cell {
        py_run!(py, s_cell, "s_cell.foo()");
    }
    println!("{}", s);
    Ok(())
}

// Another approach

mod safe_wrapper {
    use pyo3::type_object::PyBorrowFlagLayout;
    use pyo3::{PyCell, PyClass, PyClassInitializer, Python};
    use std::ptr::NonNull;

    pub struct SafeWrapper<T>(NonNull<*mut T>);

    unsafe impl<T> Send for SafeWrapper<T> {}

    impl<T> SafeWrapper<T> {
        pub fn scoped<'a, 'p, U: 'p + PyClass, V, W: Into<PyClassInitializer<U>>>(
            py: Python<'p>,
            obj: &'a mut T,
            f: impl FnOnce(SafeWrapper<T>) -> W,
            g: impl FnOnce(&'p PyCell<U>) -> V,
        ) -> V
        where
            U::BaseLayout: PyBorrowFlagLayout<U::BaseType>,
        {
            unsafe {
                let mut obj_ptr: *mut T = obj;
                let obj_ptr_ptr: *mut *mut T = &mut obj_ptr;
                let wrapper = SafeWrapper(NonNull::new_unchecked(obj_ptr_ptr));
                // TODO: propagate result instead of unwrap
                let cell = PyCell::new(py, f(wrapper)).unwrap();
                let result = g(cell);
                (*obj_ptr_ptr) = std::ptr::null_mut();
                result
            }
        }

        pub fn try_get_mut<'p>(&mut self, _py: Python<'p>) -> Option<&mut T> {
            unsafe {
                let ptr = *self.0.as_ptr();
                if ptr.is_null() {
                    None
                } else {
                    Some(&mut *ptr)
                }
            }
        }
    }
}

pyo3_mut_rc.rs

// An attempt at passing safe mutable references from Rust to Python,
// this time using Rc.

use safe_wrapper::SafeWrapper;

/// A object that holds a mutable reference, that can be passed to Python.
#[pyclass]
struct StringRef {
    inner: SafeWrapper<String>,
}

#[pymethods]
impl StringRef {
    fn foo(&mut self, py: Python<'_>) {
        if let Some(s) = self.inner.try_get_mut(py) {
            s.push_str(", world");
        } else {
            println!("trying to use reference after borrow lifetime");
        }
    }
}

impl StringRef {
    fn new(inner: safe_wrapper::SafeWrapper<String>) -> Self {
        StringRef { inner }
    }
}

#[pyfunction]
fn test_string_ref(py: Python<'_>) -> PyResult<()> {
    let mut s = "hello".to_string();
    let mut leaked_cell = None;
    SafeWrapper::scoped(py, &mut s, |wrapper| {
        let s_cell = PyCell::new(py, StringRef::new(wrapper)).unwrap();
        leaked_cell = Some(s_cell);
        py_run!(py, s_cell, "s_cell.foo()");
    });
    if let Some(s_cell) = leaked_cell {
        py_run!(py, s_cell, "s_cell.foo()");
    }
    println!("{}", s);
    Ok(())
}

mod safe_wrapper {
    use pyo3::Python;
    use std::rc::{Rc, Weak};

    pub struct SafeWrapper<T>(Weak<*mut T>);

    unsafe impl<T> Send for SafeWrapper<T> {}

    impl<T> SafeWrapper<T> {
        pub fn scoped<'p, U>(
            _py: Python<'p>,
            obj: &mut T,
            f: impl FnOnce(SafeWrapper<T>) -> U,
        ) -> U {
            let obj_ptr: *mut T = obj;
            let rc = Rc::new(obj_ptr);
            let wrapper = SafeWrapper(Rc::downgrade(&rc));
            f(wrapper)
        }

        pub fn try_get_mut<'p>(&mut self, _py: Python<'p>) -> Option<&mut T> {
            unsafe { self.0.upgrade().map(|ptr| &mut **ptr) }
        }
    }
}

pyo3_mut_ref.rs

/// A object that holds a mutable reference, that can be passed to Python.
#[pyclass]
struct StringRef {
    ref_cell: RefCell<Option<&'static mut String>>,
}

/// A guard object that ensures safe use of a mutable borrow.
///
/// It has two functions. First, it will clear the reference on drop,
/// preventing use of the borrowed reference after its lifetime. Second,
/// it has a marker that prevents other access to the borrowed reference.
struct RefGuard<'a, 'p> {
    s_cell: &'p PyCell<StringRef>,
    marker: std::marker::PhantomData<&'a mut String>,
}

impl<'a, 'p> Drop for RefGuard<'a, 'p> {
    fn drop(&mut self) {
        *self.s_cell.borrow().ref_cell.borrow_mut() = None;
    }
}

#[pymethods]
impl StringRef {
    fn foo(&self) {
        if let Some(mut c) = self.try_borrow_mut() {
            if let Some(s) = &mut *c {
                s.push_str(", world");    
            } else {
                println!("refcell already borrowed");
            }
        } else {
            println!("cell has been freed");
        }
    }
}

impl StringRef {
    /// Create a new reference object, along with guard.
    fn new<'a, 'p>(
        py: Python<'p>,
        s: &'a mut String,
    ) -> (&'p PyCell<StringRef>, RefGuard<'a, 'p>) {
        let string_ref = StringRef {
            // Transmute is only to adjust lifetimes.
            ref_cell: unsafe { std::mem::transmute(RefCell::new(Some(s))) },
        };
        let s_cell = PyCell::new(py, string_ref).unwrap();
        let guard = RefGuard { s_cell, marker: Default::default() };
        (s_cell, guard)
    }

    fn try_borrow_mut(&self) -> Option<RefMut<Option<&mut String>>> {
            // Transmute is only to adjust lifetimes.
            unsafe { std::mem::transmute(self.ref_cell.try_borrow_mut().ok()) }
    }
}

#[pyfunction]
fn test_string_ref(py: Python<'_>) -> PyResult<()> {
    let mut s = "hello".to_string();
    let leak_cell;
    {
        let (s_cell, _guard) = StringRef::new(py, &mut s);
        leak_cell = s_cell;
        py_run!(py, s_cell, "s_cell.foo()");
    }
    py_run!(py, leak_cell, "leak_cell.foo()");
    println!("{}", s);
    Ok(())
}

pyo3_mut_scoped.rs

/// A object that holds a mutable reference, that can be passed to Python.
#[pyclass]
struct StringRef {
    ref_cell: RefCell<Option<&'static mut String>>,
}

#[pymethods]
impl StringRef {
    fn foo(&self) {
        if let Some(mut r) = self.try_borrow_mut() {
            if let Some(s) = &mut *r {
                s.push_str(", world");
            } else {
                println!("trying to use reference after borrow lifetime");
            }
        } else {
            println!("refcell already borrowed");
        }
    }
}

impl StringRef {
    fn try_borrow_mut(&self) -> Option<RefMut<Option<&mut String>>> {
        // Transmute is only to adjust lifetimes.
        unsafe { std::mem::transmute(self.ref_cell.try_borrow_mut().ok()) }
    }

    /// Run a function in a scope with the mutable reference.
    fn scoped<'a, 'p, T>(
        py: Python<'p>,
        s: &'a mut String,
        f: impl FnOnce(&'p PyCell<StringRef>) -> T,
    ) -> T {
        let string_ref = StringRef {
            // Transmute is only to adjust lifetimes.
            ref_cell: unsafe { std::mem::transmute(RefCell::new(Some(s))) },
        };
        let s_cell = PyCell::new(py, string_ref).unwrap();
        let result = f(s_cell);
        *s_cell.borrow().ref_cell.borrow_mut() = None;
        result
    }
}

#[pyfunction]
fn test_string_ref(py: Python<'_>) -> PyResult<()> {
    let mut s = "hello".to_string();
    let mut leaked_cell = None;
    StringRef::scoped(py, &mut s, |s_cell| {
        leaked_cell = Some(s_cell);
        py_run!(py, s_cell, "s_cell.foo()");
    });
    if let Some(s_cell) = leaked_cell {
        py_run!(py, s_cell, "s_cell.foo()");
    }
    println!("{}", s);
    Ok(())
}