VTabThreadLocal.rs

#[repr(C)]
pub struct InitData {
    pub parser: ...
}

pub struct ThreadLocalState {
    pub spillover: Option<Vec<RawMrtRecord>>
}

// ...

    fn func(
        func: &TableFunctionInfo<Self>,
        output: &mut DataChunkHandle,
    ) -> Result<(), Box<dyn std::error::Error>> {
        let init_data = func.get_init_data();

        thread_local! {
            pub static THREAD_STATE: RefCell<ThreadLocalState> = const { RefCell::new(ThreadLocalState { spillover: None }) };
        }

        // Drain any leftover records from the previous call.
        let mut rows = THREAD_STATE.with_borrow_mut(|state| {
            state.spillover.take().unwrap_or_default()
        });

        loop {
            if rows.is_empty() {
                let mut parser_guard = init_data.parser.lock().unwrap();
                rows = parser_guard.as_mut().take(512 * max_chunk_size).collect();
                if rows.is_empty() {
                    break; // parser exhausted
                }
            }

            let mut iter = rows.into_iter();
            for row in &mut iter {
                // ...
                found_rows += 1;
                if found_rows == max_chunk_size {
                    // Collect the unvisited tail; moves, no clone.
                    let remaining: Vec<_> = iter.collect();
                    if !remaining.is_empty() {
                        THREAD_STATE.with_borrow_mut(|state| {
                            state.spillover = Some(remaining);
                        });
                    }
                    break;
                }
            }

            if found_rows == max_chunk_size {
                break;
            }
            // iter fully consumed; reinit so the fetch at the top of the loop fires.
            rows = Vec::new();
        }

        output.set_len(found_rows);

        Ok(())
    }