mimalloc.diff

diff --git a/Include/mimalloc/mimalloc-atomic.h b/Include/mimalloc/mimalloc-atomic.h
index 7b7cb383d6..aa777cce05 100644
--- a/Include/mimalloc/mimalloc-atomic.h
+++ b/Include/mimalloc/mimalloc-atomic.h
@@ -51,6 +51,40 @@ static inline void* mi_atomic_exchange_ptr(volatile void** p, void* exchange) {
   return (void*)mi_atomic_exchange((volatile uintptr_t*)p, (uintptr_t)exchange);
 }
 
+// Atomically read a value
+static inline uintptr_t mi_atomic_read(const volatile uintptr_t* p);
+
+// Atomically read a uint16
+static inline uint16_t mi_atomic_read16(const volatile uint16_t* p);
+
+// Atomically read a value with acquire semantics
+static inline uintptr_t mi_atomic_acquire(const volatile uintptr_t* p);
+
+// Atomically write a value
+static inline void mi_atomic_write(volatile uintptr_t* p, uintptr_t x);
+
+// Atomically write a pointer
+static inline void mi_atomic_write_ptr(volatile void** p, void* x) {
+  mi_atomic_write((volatile uintptr_t*)p, (uintptr_t)x);
+}
+
+// Atomically read a pointer
+static inline void* mi_atomic_read_ptr(const volatile void** p) {
+  return (void*)mi_atomic_read( (const volatile uintptr_t*)p );
+}
+static inline void* mi_atomic_acquire_ptr(const volatile void** p) {
+  return (void*)mi_atomic_acquire( (const volatile uintptr_t*)p );
+}
+// Atomically read a value
+static inline int64_t mi_atomic_read64(volatile int64_t* p);
+
+// Atomically write a value
+static inline void mi_atomic_write64(volatile int64_t* p, int64_t x);
+
+// Atomically write a value
+static inline void mi_atomic_write16(volatile uint16_t* p, uint16_t x);
+
+
 
 #ifdef _MSC_VER
 #define WIN32_LEAN_AND_MEAN
@@ -87,6 +121,18 @@ static inline bool mi_atomic_compare_exchange(volatile uintptr_t* p, uintptr_t e
 static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exchange) {
   return (uintptr_t)RC64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
 }
+static inline uintptr_t mi_atomic_read(const volatile uintptr_t* p) {
+  return *p;
+}
+static inline int64_t mi_atomic_read64(const volatile int64_t* p) {
+  return *p;
+}
+static inline void mi_atomic_write(volatile uintptr_t* p, uintptr_t x) {
+  *p = x;
+}
+static inline void mi_atomic_write64(volatile int64_t* p, int64_t x) {
+  *p = x;
+}
 static inline void mi_atomic_yield(void) {
   YieldProcessor();
 }
@@ -149,7 +195,42 @@ static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exch
   MI_USING_STD
   return atomic_exchange_explicit((volatile atomic_uintptr_t*)p, exchange, memory_order_acquire);
 }
-
+static inline uintptr_t mi_atomic_read(const volatile uintptr_t* p) {
+  MI_USING_STD
+  return atomic_load_explicit((const volatile atomic_uintptr_t*)p, memory_order_relaxed);
+}
+static inline bool mi_atomic_read_bool(const volatile bool* p) {
+  MI_USING_STD
+  return atomic_load_explicit((const volatile _Atomic(bool)*)p, memory_order_relaxed);
+}
+static inline uint16_t mi_atomic_read16(const volatile uint16_t* p) {
+  MI_USING_STD
+  return atomic_load_explicit((const volatile _Atomic(uint16_t)*)p, memory_order_relaxed);
+}
+static inline uintptr_t mi_atomic_acquire(const volatile uintptr_t* p) {
+  MI_USING_STD
+  return atomic_load_explicit((const volatile atomic_uintptr_t*)p, memory_order_acquire);
+}
+static inline int64_t mi_atomic_read64(volatile int64_t* p) {
+  MI_USING_STD
+  return atomic_load_explicit((volatile _Atomic(int64_t)*)p, memory_order_relaxed);
+}
+static inline void mi_atomic_write(volatile uintptr_t* p, uintptr_t x) {
+  MI_USING_STD
+  return atomic_store_explicit((volatile atomic_uintptr_t*)p, x, memory_order_relaxed);
+}
+static inline void mi_atomic_write_bool(volatile bool* p, bool x) {
+  MI_USING_STD
+  return atomic_store_explicit((volatile _Atomic(bool)*)p, x, memory_order_relaxed);
+}
+static inline void mi_atomic_write64(volatile int64_t* p, int64_t x) {
+  MI_USING_STD
+  return atomic_store_explicit((volatile _Atomic(int64_t)*)p, x, memory_order_relaxed);
+}
+static inline void mi_atomic_write16(volatile uint16_t* p, uint16_t x) {
+  MI_USING_STD
+  return atomic_store_explicit((volatile _Atomic(uint16_t)*)p, x, memory_order_relaxed);
+}
 #if defined(__cplusplus)
   #include <thread>
   static inline void mi_atomic_yield(void) {
diff --git a/Include/mimalloc/mimalloc-internal.h b/Include/mimalloc/mimalloc-internal.h
index a1d1d835d5..5c98b050ed 100644
--- a/Include/mimalloc/mimalloc-internal.h
+++ b/Include/mimalloc/mimalloc-internal.h
@@ -9,6 +9,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #define MIMALLOC_INTERNAL_H
 
 #include "mimalloc-types.h"
+#include "mimalloc-atomic.h"
 
 #if defined(MI_MALLOC_OVERRIDE) && defined(__APPLE__)
 #define MI_TLS_RECURSE_GUARD
@@ -17,7 +18,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #if (MI_DEBUG>0)
 #define mi_trace_message(...)  _mi_trace_message(__VA_ARGS__)
 #else
-#define mi_trace_message(...)  
+#define mi_trace_message(...)
 #endif
 
 
@@ -35,6 +36,8 @@ bool       _mi_is_main_thread(void);
 uintptr_t  _mi_ptr_cookie(const void* p);
 uintptr_t  _mi_random_shuffle(uintptr_t x);
 uintptr_t  _mi_random_init(uintptr_t seed /* can be zero */);
+void       _mi_heap_delete_backing(mi_heap_t* heap);
+
 
 // "os.c"
 bool       _mi_os_reset(void* p, size_t size, mi_stats_t* stats);
@@ -56,6 +59,7 @@ void       _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld);
 bool       _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld);
 void       _mi_segment_thread_collect(mi_segments_tld_t* tld);
 uint8_t*   _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t block_size, size_t* page_size); // page start for any page
+mi_segment_t* _mi_segment_abandoned(void);
 
 // "page.c"
 void*      _mi_malloc_generic(mi_heap_t* heap, size_t size)  mi_attr_noexcept mi_attr_malloc;
@@ -126,8 +130,8 @@ bool        _mi_page_is_valid(mi_page_t* page);
   Inlined definitions
 ----------------------------------------------------------- */
 #define UNUSED(x)     (void)(x)
-#if (MI_DEBUG>0) 
-#define UNUSED_RELEASE(x)  
+#if (MI_DEBUG>0)
+#define UNUSED_RELEASE(x)
 #else
 #define UNUSED_RELEASE(x)  UNUSED(x)
 #endif
@@ -260,7 +264,7 @@ static inline mi_thread_free_t mi_tf_set_block(mi_thread_free_t tf, mi_block_t*
 // are all blocks in a page freed?
 static inline bool mi_page_all_free(const mi_page_t* page) {
   mi_assert_internal(page != NULL);
-  return (page->used - page->thread_freed == 0);
+  return (page->used - mi_atomic_read(&page->thread_freed) == 0);
 }
 
 // are there immediately available blocks
diff --git a/Include/mimalloc/mimalloc-types.h b/Include/mimalloc/mimalloc-types.h
index 2f16020f94..b36076de57 100644
--- a/Include/mimalloc/mimalloc-types.h
+++ b/Include/mimalloc/mimalloc-types.h
@@ -161,6 +161,7 @@ typedef struct mi_page_s {
   uint8_t               segment_idx;       // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]`
   bool                  segment_in_use:1;  // `true` if the segment allocated this page
   bool                  is_reset:1;        // `true` if the page memory was reset
+  unsigned int          tag:6;
 
   // layout like this to optimize access in `mi_malloc` and `mi_free`
   mi_page_flags_t       flags;
@@ -245,6 +246,8 @@ typedef struct mi_page_queue_s {
 
 #define MI_BIN_FULL  (MI_BIN_HUGE+1)
 
+struct _gc_runtime_state;
+
 // A heap owns a set of pages.
 struct mi_heap_s {
   mi_tld_t*             tld;
@@ -256,6 +259,9 @@ struct mi_heap_s {
   uintptr_t             random;                                      // random number used for secure allocation
   size_t                page_count;                                  // total number of pages in the `pages` queues.
   bool                  no_reclaim;                                  // `true` if this heap should not reclaim abandoned pages
+  unsigned char         tag;
+  bool                  visited;                                     // used by gcmodule.c
+  struct _gc_runtime_state* gcstate;
 };
 
 
@@ -390,6 +396,8 @@ typedef struct mi_os_tld_s {
 struct mi_tld_s {
   unsigned long long  heartbeat;     // monotonic heartbeat count
   mi_heap_t*          heap_backing;  // backing heap of this thread (cannot be deleted)
+  mi_heap_t*          heap_obj;
+  mi_heap_t*          heap_gc;
   mi_segments_tld_t   segments;      // segment tld
   mi_os_tld_t         os;            // os tld
   mi_stats_t          stats;         // statistics
diff --git a/Include/mimalloc/mimalloc.h b/Include/mimalloc/mimalloc.h
index ec09f1195d..13059d24e4 100644
--- a/Include/mimalloc/mimalloc.h
+++ b/Include/mimalloc/mimalloc.h
@@ -52,8 +52,8 @@ terms of the MIT license. A copy of the license can be found in the file
   #define mi_attr_alloc_size2(s1,s2)
   #else
   #define mi_attr_alloc_size(s)       __attribute__((alloc_size(s)))
-  #define mi_attr_alloc_size2(s1,s2)  __attribute__((alloc_size(s1,s2)))  
-  #define mi_cdecl                    // leads to warnings... __attribute__((cdecl))  
+  #define mi_attr_alloc_size2(s1,s2)  __attribute__((alloc_size(s1,s2)))
+  #define mi_cdecl                    // leads to warnings... __attribute__((cdecl))
   #endif
 #else
   #define mi_decl_thread              __thread
@@ -62,7 +62,7 @@ terms of the MIT license. A copy of the license can be found in the file
   #define mi_attr_malloc
   #define mi_attr_alloc_size(s)
   #define mi_attr_alloc_size2(s1,s2)
-  #define mi_cdecl                   
+  #define mi_cdecl
 #endif
 
 // ------------------------------------------------------
@@ -148,6 +148,8 @@ mi_decl_export void       mi_heap_destroy(mi_heap_t* heap);
 mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
 mi_decl_export mi_heap_t* mi_heap_get_default(void);
 mi_decl_export mi_heap_t* mi_heap_get_backing(void);
+mi_decl_export mi_heap_t* mi_heap_get_obj(void);
+mi_decl_export mi_heap_t* mi_heap_get_gc(void);
 mi_decl_export void       mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept;
 
 mi_decl_export mi_decl_allocator void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
@@ -174,6 +176,13 @@ mi_decl_export mi_decl_allocator void* mi_heap_realloc_aligned(mi_heap_t* heap,
 mi_decl_export mi_decl_allocator void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3);
 
 
+typedef enum mi_heap_tag_e {
+  mi_heap_tag_default,
+  mi_heap_tag_obj,
+  mi_heap_tag_gc,
+  _mi_heap_tag_last
+} mi_heap_tag_t;
+
 // ------------------------------------------------------
 // Analysis
 // ------------------------------------------------------
diff --git a/Objects/mimalloc/alloc.c b/Objects/mimalloc/alloc.c
index d5050b03b4..d57c9655d4 100644
--- a/Objects/mimalloc/alloc.c
+++ b/Objects/mimalloc/alloc.c
@@ -8,6 +8,9 @@ terms of the MIT license. A copy of the license can be found in the file
 #include "mimalloc-internal.h"
 #include "mimalloc-atomic.h"
 
+#include "Python.h"
+#include "objimpl.h"
+
 #include <string.h>  // memset
 
 #define MI_IN_ALLOC_C
@@ -29,7 +32,8 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz
   mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);
   // pop from the free list
   page->free = mi_block_next(page,block);
-  page->used++;
+  mi_atomic_write(&page->used, mi_atomic_read(&page->used) + 1);
+  // page->used++; // RACE with alloc.c 116
   mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);
 #if (MI_DEBUG)
   memset(block, MI_DEBUG_UNINIT, size);
@@ -111,9 +115,10 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc
   bool use_delayed;
 
   do {
-    tfree = page->thread_free;
+    tfree = mi_atomic_read(&page->thread_free);
     use_delayed = (mi_tf_delayed(tfree) == MI_USE_DELAYED_FREE ||
-                   (mi_tf_delayed(tfree) == MI_NO_DELAYED_FREE && page->used == page->thread_freed+1)
+                   (mi_tf_delayed(tfree) == MI_NO_DELAYED_FREE && 
+                    mi_atomic_read(&page->used) == mi_atomic_read(&page->thread_freed)+1)
                   );
     if (mi_unlikely(use_delayed)) {
       // unlikely: this only happens on the first concurrent free in a page that is in the full list
@@ -165,7 +170,8 @@ static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block
     // owning thread can free a block directly
     mi_block_set_next(page, block, page->local_free);
     page->local_free = block;
-    page->used--;
+    mi_atomic_write(&page->used, mi_atomic_read(&page->used) - 1);
+    // page->used--;
     if (mi_unlikely(mi_page_all_free(page))) {
       _mi_page_retire(page);
     }
@@ -189,7 +195,8 @@ mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* p
 
 
 static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool local, void* p) {
-  mi_block_t* block = (page->flags.has_aligned ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p);
+  bool has_aligned = mi_atomic_read_bool(&page->flags.has_aligned);
+  mi_block_t* block = (has_aligned ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p);
   _mi_free_block(page, local, block);
 }
 
@@ -208,7 +215,7 @@ void mi_free(void* p) mi_attr_noexcept
 
   const mi_segment_t* const segment = _mi_ptr_segment(p);
   if (segment == NULL) return;  // checks for (p==NULL)
-  bool local = (_mi_thread_id() == segment->thread_id);  // preload, note: putting the thread_id in the page->flags does not improve performance
+  bool local = (_mi_thread_id() == mi_atomic_read(&segment->thread_id));  // preload, note: putting the thread_id in the page->flags does not improve performance
 
 #if (MI_DEBUG>0)
   if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) {
@@ -229,13 +236,15 @@ void mi_free(void* p) mi_attr_noexcept
 #endif
 
   // adjust if it might be an un-aligned block
-  if (mi_likely(page->flags.value==0)) {  // note: merging both tests (local | value) does not matter for performance
+  bool has_aligned = mi_atomic_read_bool(&page->flags.has_aligned);
+  if (mi_likely(!has_aligned)) {  // note: merging both tests (local | value) does not matter for performance
     mi_block_t* block = (mi_block_t*)p;
     if (mi_likely(local)) {
       // owning thread can free a block directly
       mi_block_set_next(page, block, page->local_free);  // note: moving this write earlier does not matter for performance
       page->local_free = block;
-      page->used--;
+      mi_atomic_write(&page->used, mi_atomic_read(&page->used) - 1);
+      // page->used--;
       if (mi_unlikely(mi_page_all_free(page))) { _mi_page_retire(page); }
     }
     else {
@@ -271,7 +280,7 @@ size_t mi_usable_size(const void* p) mi_attr_noexcept {
   const mi_segment_t* segment = _mi_ptr_segment(p);
   const mi_page_t* page = _mi_segment_page_of(segment,p);
   size_t size = page->block_size;
-  if (mi_unlikely(page->flags.has_aligned)) {
+  if (mi_unlikely(mi_atomic_read_bool(&page->flags.has_aligned))) {
     ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)_mi_page_ptr_unalign(segment,page,p);
     mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
     return (size - adjust);
@@ -405,6 +414,7 @@ void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {
   return mi_heap_reallocf(mi_get_default_heap(),p,newsize);
 }
 
+
 // ------------------------------------------------------
 // strdup, strndup, and realpath
 // ------------------------------------------------------
@@ -522,7 +532,7 @@ static bool mi_try_new_handler(bool nothrow) {
 #ifndef ENOMEM
 #define ENOMEM 12
 #endif
-typedef void (*std_new_handler_t)();
+typedef void (*std_new_handler_t)(void);
 
 #if (defined(__GNUC__) || defined(__clang__))
 std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv() {
diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c
index dc21bd0a6f..b3738f5af1 100644
--- a/Objects/mimalloc/heap.c
+++ b/Objects/mimalloc/heap.c
@@ -130,19 +130,19 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
   // if abandoning, mark all pages to no longer add to delayed_free
   if (collect == ABANDON) {
     //for (mi_page_t* page = heap->pages[MI_BIN_FULL].first; page != NULL; page = page->next) {
-    //  _mi_page_use_delayed_free(page, false);  // set thread_free.delayed to MI_NO_DELAYED_FREE      
-    //}    
+    //  _mi_page_use_delayed_free(page, false);  // set thread_free.delayed to MI_NO_DELAYED_FREE
+    //}
     mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL);
   }
 
-  // free thread delayed blocks. 
+  // free thread delayed blocks.
   // (if abandoning, after this there are no more local references into the pages.)
   _mi_heap_delayed_free(heap);
 
   // collect all pages owned by this thread
   mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL);
   mi_assert_internal( collect != ABANDON || heap->thread_delayed_free == NULL );
-  
+
   // collect segment caches
   if (collect >= FORCE) {
     _mi_segment_thread_collect(&heap->tld->segments);
@@ -180,6 +180,17 @@ mi_heap_t* mi_heap_get_backing(void) {
   return bheap;
 }
 
+mi_heap_t* mi_heap_get_obj(void) {
+  mi_heap_t* def = mi_heap_get_default();
+  return def->tld->heap_obj;
+}
+
+mi_heap_t* mi_heap_get_gc(void) {
+  mi_heap_t* def = mi_heap_get_default();
+  return def->tld->heap_gc;
+}
+
+
 uintptr_t _mi_heap_random(mi_heap_t* heap) {
   uintptr_t r = heap->random;
   heap->random = _mi_random_shuffle(r);
@@ -216,7 +227,7 @@ static void mi_heap_reset_pages(mi_heap_t* heap) {
 static void mi_heap_free(mi_heap_t* heap) {
   mi_assert_internal(mi_heap_is_initialized(heap));
   if (mi_heap_is_backing(heap)) return; // dont free the backing heap
-  
+
   // reset default
   if (mi_heap_is_default(heap)) {
     _mi_heap_default = heap->tld->heap_backing;
@@ -237,7 +248,7 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
   UNUSED(pq);
 
   // ensure no more thread_delayed_free will be added
-  _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE);  
+  _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE);
 
   // stats
   if (page->block_size > MI_LARGE_SIZE_MAX) {
@@ -294,19 +305,19 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
   if (from==NULL || from->page_count == 0) return;
 
   // unfull all full pages
-  mi_page_t* page = heap->pages[MI_BIN_FULL].first; 
+  mi_page_t* page = from->pages[MI_BIN_FULL].first;
   while (page != NULL) {
     mi_page_t* next = page->next;
     _mi_page_unfull(page);
     page = next;
   }
-  mi_assert_internal(heap->pages[MI_BIN_FULL].first == NULL);
+  mi_assert_internal(from->pages[MI_BIN_FULL].first == NULL);
 
   // free outstanding thread delayed free blocks
   _mi_heap_delayed_free(from);
 
   // transfer all pages by appending the queues; this will set
-  // a new heap field which is ok as all pages are unfull'd and thus 
+  // a new heap field which is ok as all pages are unfull'd and thus
   // other threads won't access this field anymore (see `mi_free_block_mt`)
   for (size_t i = 0; i < MI_BIN_FULL; i++) {
     mi_page_queue_t* pq = &heap->pages[i];
@@ -317,7 +328,7 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
   }
   mi_assert_internal(from->thread_delayed_free == NULL);
   mi_assert_internal(from->page_count == 0);
-  
+
   // and reset the `from` heap
   mi_heap_reset_pages(from);
 }
@@ -501,7 +512,7 @@ typedef struct mi_visit_blocks_args_s {
 
 static bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t* xarea, void* arg) {
   mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg;
-  if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, arg)) return false;
+  if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg)) return false;
   if (args->visit_blocks) {
     return mi_heap_area_visit_blocks(xarea, args->visitor, args->arg);
   }
diff --git a/Objects/mimalloc/init.c b/Objects/mimalloc/init.c
index 5b2d3c8e15..f137a0f208 100644
--- a/Objects/mimalloc/init.c
+++ b/Objects/mimalloc/init.c
@@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file
 
 // Empty page used to initialize the small free pages array
 const mi_page_t _mi_page_empty = {
-  0, false, false, {0},
+  0, false, false, 0, {0},
   0, 0,
   NULL, 0, 0,   // free, used, cookie
   NULL, 0, 0,
@@ -80,7 +80,9 @@ const mi_heap_t _mi_heap_empty = {
   0,
   0,
   0,
-  false
+  false,
+  0,
+  0
 };
 
 mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;
@@ -88,9 +90,15 @@ mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;
 
 #define tld_main_stats  ((mi_stats_t*)((uint8_t*)&tld_main + offsetof(mi_tld_t,stats)))
 
+// Initialized in mi_heap_init
+static mi_heap_t _mi_heap_main_obj;
+static mi_heap_t _mi_heap_main_gc;
+
 static mi_tld_t tld_main = {
   0,
   &_mi_heap_main,
+  &_mi_heap_main_obj,
+  &_mi_heap_main_gc,
   { { NULL, NULL }, {NULL ,NULL}, 0, 0, 0, 0, {NULL,NULL}, tld_main_stats }, // segments
   { 0, NULL, NULL, 0, tld_main_stats },              // os
   { MI_STATS_NULL }                                  // stats
@@ -182,9 +190,20 @@ uintptr_t _mi_ptr_cookie(const void* p) {
 
 typedef struct mi_thread_data_s {
   mi_heap_t  heap;  // must come first due to cast in `_mi_heap_done`
+  mi_heap_t  heap_obj;
+  mi_heap_t  heap_gc;
   mi_tld_t   tld;
 } mi_thread_data_t;
 
+static void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, int tag) {
+    memcpy(heap, &_mi_heap_empty, sizeof(*heap));
+    heap->thread_id = _mi_thread_id();
+    heap->random = _mi_random_init(heap->thread_id);
+    heap->cookie = ((uintptr_t)heap ^ _mi_heap_random(heap)) | 1;
+    heap->tld = tld;
+    heap->tag = tag;
+}
+
 // Initialize the thread local default heap, called from `mi_thread_init`
 static bool _mi_heap_init(void) {
   if (mi_heap_is_initialized(_mi_heap_default)) return true;
@@ -201,38 +220,44 @@ static bool _mi_heap_init(void) {
       return false;
     }
     mi_tld_t*  tld = &td->tld;
-    mi_heap_t* heap = &td->heap;
-    memcpy(heap, &_mi_heap_empty, sizeof(*heap));
-    heap->thread_id = _mi_thread_id();
-    heap->random = _mi_random_init(heap->thread_id);
-    heap->cookie = ((uintptr_t)heap ^ _mi_heap_random(heap)) | 1;
-    heap->tld = tld;
+    _mi_heap_init_ex(&td->heap, &td->tld, mi_heap_tag_default);
+    _mi_heap_init_ex(&td->heap_obj, &td->tld, mi_heap_tag_obj);
+    _mi_heap_init_ex(&td->heap_gc, &td->tld, mi_heap_tag_gc);
     memset(tld, 0, sizeof(*tld));
-    tld->heap_backing = heap;
+    tld->heap_backing = &td->heap;
+    tld->heap_obj = &td->heap_obj;
+    tld->heap_gc = &td->heap_gc;
     tld->segments.stats = &tld->stats;
     tld->os.stats = &tld->stats;
-    _mi_heap_default = heap;
+    _mi_heap_default = &td->heap;
   }
   return false;
 }
 
 // Free the thread local default heap (called from `mi_thread_done`)
-static bool _mi_heap_done(void) {
-  mi_heap_t* heap = _mi_heap_default;
-  if (!mi_heap_is_initialized(heap)) return true;
+// static bool _mi_heap_done(void) {
+//   mi_heap_t* heap = _mi_heap_default;
+//   if (!mi_heap_is_initialized(heap)) return true;
 
-  // reset default heap
-  _mi_heap_default = (_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty);
+//   // reset default heap
+//   _mi_heap_default = (_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty);
 
-  // todo: delete all non-backing heaps?
+//   // todo: delete all non-backing heaps?
+//   _mi_heap_delete_backing(heap);
+//   return false;
+// }
 
+void _mi_heap_delete_backing(mi_heap_t* heap) {
   // switch to backing heap and free it
   heap = heap->tld->heap_backing;
-  if (!mi_heap_is_initialized(heap)) return false;
+  if (!mi_heap_is_initialized(heap)) return;
 
-  // collect if not the main thread 
+  // collect if not the main thread
   if (heap != &_mi_heap_main) {
     _mi_heap_collect_abandon(heap);
+    // TODO(sgross): maybe set no reclaim temporarily to prevent reclamation of segments from heaps???
+    _mi_heap_collect_abandon(heap->tld->heap_obj);
+    _mi_heap_collect_abandon(heap->tld->heap_gc);
   }
 
   // merge stats
@@ -248,7 +273,6 @@ static bool _mi_heap_done(void) {
     mi_assert_internal(heap->tld->heap_backing == &_mi_heap_main);
   }
 #endif
-  return false;
 }
 
 
@@ -348,8 +372,10 @@ void mi_thread_done(void) mi_attr_noexcept {
     _mi_stat_decrease(&heap->tld->stats.threads, 1);
   }
 
-  // abandon the thread local heap
-  if (_mi_heap_done()) return; // returns true if already ran
+  if (!mi_heap_is_initialized(heap)) return;
+
+  // reset default heap
+  _mi_heap_default = (_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty);
 
   #if (MI_DEBUG>0)
   if (!_mi_is_main_thread()) {
@@ -383,6 +409,8 @@ void mi_process_init(void) mi_attr_noexcept {
   #if (MI_DEBUG)
   _mi_verbose_message("debug level : %d\n", MI_DEBUG);
   #endif
+  _mi_heap_init_ex(&_mi_heap_main_obj, &tld_main, mi_heap_tag_obj);
+  _mi_heap_init_ex(&_mi_heap_main_gc, &tld_main, mi_heap_tag_gc);
   atexit(&mi_process_done);
   mi_process_setup_auto_thread_done();
   mi_stats_reset();
diff --git a/Objects/mimalloc/options.c b/Objects/mimalloc/options.c
index d75764aba6..8420f396de 100644
--- a/Objects/mimalloc/options.c
+++ b/Objects/mimalloc/options.c
@@ -154,15 +154,18 @@ void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, co
 // Initialize options by checking the environment
 // --------------------------------------------------------
 
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable:4996)
+#endif
+
 static void mi_strlcpy(char* dest, const char* src, size_t dest_size) {
   dest[0] = 0;
-  #pragma warning(suppress:4996)
   strncpy(dest, src, dest_size - 1);
   dest[dest_size - 1] = 0;
 }
 
 static void mi_strlcat(char* dest, const char* src, size_t dest_size) {
-  #pragma warning(suppress:4996)
   strncat(dest, src, dest_size - 1);
   dest[dest_size - 1] = 0;
 }
@@ -173,14 +176,12 @@ static void mi_option_init(mi_option_desc_t* desc) {
   char buf[32];
   mi_strlcpy(buf, "mimalloc_", sizeof(buf));
   mi_strlcat(buf, desc->name, sizeof(buf));
-  #pragma warning(suppress:4996)
   char* s = getenv(buf);
   if (s == NULL) {
     size_t buf_size = strlen(buf);
     for (size_t i = 0; i < buf_size; i++) {
       buf[i] = toupper(buf[i]);
     }
-    #pragma warning(suppress:4996)
     s = getenv(buf);
   }
   if (s != NULL) {
@@ -210,3 +211,7 @@ static void mi_option_init(mi_option_desc_t* desc) {
     }
   }
 }
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
diff --git a/Objects/mimalloc/os.c b/Objects/mimalloc/os.c
index 68a2314389..f9f165d702 100644
--- a/Objects/mimalloc/os.c
+++ b/Objects/mimalloc/os.c
@@ -152,6 +152,10 @@ void _mi_os_init() {
 /* -----------------------------------------------------------
   Raw allocation on Windows (VirtualAlloc) and Unix's (mmap).  
 ----------------------------------------------------------- */
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable:4996)
+#endif
 
 static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
 {
@@ -165,7 +169,6 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
   _mi_stat_decrease(&stats->committed, size); // TODO: what if never committed?
   _mi_stat_decrease(&stats->reserved, size);
   if (err) {
-#pragma warning(suppress:4996)
     _mi_warning_message("munmap failed: %s, addr 0x%8li, size %lu\n", strerror(errno), (size_t)addr, size);
     return false;
   }
@@ -174,6 +177,10 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
   }
 }
 
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
 #ifdef _WIN32
 static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) {
 #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
diff --git a/Objects/mimalloc/page-queue.c b/Objects/mimalloc/page-queue.c
index ebe858b34c..cbbba86ee8 100644
--- a/Objects/mimalloc/page-queue.c
+++ b/Objects/mimalloc/page-queue.c
@@ -260,6 +260,7 @@ static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) {
   page->next = NULL;
   page->prev = NULL;
   page->heap = NULL;
+
   page->flags.in_full = false;
 }
 
diff --git a/Objects/mimalloc/page.c b/Objects/mimalloc/page.c
index fc7c4f0149..d0bfe11e75 100644
--- a/Objects/mimalloc/page.c
+++ b/Objects/mimalloc/page.c
@@ -15,6 +15,9 @@ terms of the MIT license. A copy of the license can be found in the file
 #include "mimalloc-internal.h"
 #include "mimalloc-atomic.h"
 
+#include "Python.h"
+#include "pycore_pystate.h"
+
 #include <string.h>  // memset, memcpy
 
 /* -----------------------------------------------------------
@@ -114,14 +117,14 @@ void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay  ) {
   mi_thread_free_t tfreex;
 
   do {
-    tfreex = tfree = page->thread_free;
+    tfreex = tfree = mi_atomic_read(&page->thread_free);
     if (mi_unlikely(mi_tf_delayed(tfree) < MI_DELAYED_FREEING)) {
       tfreex = mi_tf_set_delayed(tfree,delay);
     }
     else if (mi_unlikely(mi_tf_delayed(tfree) == MI_DELAYED_FREEING)) {
       mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done.
       continue;          // and try again
-    }    
+    }
   }
   while((mi_tf_delayed(tfreex) !=  mi_tf_delayed(tfree)) && // avoid atomic operation if already equal
         !mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex, tfree));
@@ -142,7 +145,7 @@ static void mi_page_thread_free_collect(mi_page_t* page)
   mi_thread_free_t tfree;
   mi_thread_free_t tfreex;
   do {
-    tfreex = tfree = page->thread_free;
+    tfreex = tfree = mi_atomic_acquire(&page->thread_free);
     head = mi_tf_block(tfree);
     tfreex = mi_tf_set_block(tfree,NULL);
   } while (!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex, tfree));
@@ -189,7 +192,7 @@ void _mi_page_free_collect(mi_page_t* page) {
     page->local_free = NULL;
   }
   // and the thread free list
-  if (mi_tf_block(page->thread_free) != NULL) {  // quick test to avoid an atomic operation
+  if (mi_tf_block(mi_atomic_read(&page->thread_free)) != NULL) {  // quick test to avoid an atomic operation
     mi_page_thread_free_collect(page);
   }
 }
@@ -204,6 +207,7 @@ void _mi_page_free_collect(mi_page_t* page) {
 void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) {
   mi_assert_expensive(mi_page_is_valid_init(page));
   mi_assert_internal(page->heap == NULL);
+  mi_assert_internal(page->tag == heap->tag);
   _mi_page_free_collect(page);
   mi_page_queue_t* pq = mi_page_queue(heap, page->block_size);
   mi_page_queue_push(heap, pq, page);
@@ -252,7 +256,7 @@ void _mi_heap_delayed_free(mi_heap_t* heap) {
   // take over the list
   mi_block_t* block;
   do {
-    block = (mi_block_t*)heap->thread_delayed_free;
+    block = (mi_block_t*)mi_atomic_acquire(&heap->thread_delayed_free);
   } while (block != NULL && !mi_atomic_compare_exchange_ptr((volatile void**)&heap->thread_delayed_free, NULL, block));
 
   // and free them all
@@ -260,7 +264,7 @@ void _mi_heap_delayed_free(mi_heap_t* heap) {
     mi_block_t* next = mi_block_nextx(heap->cookie,block);
     // use internal free instead of regular one to keep stats etc correct
     if (!_mi_free_delayed_block(block)) {
-      // we might already start delayed freeing while another thread has not yet 
+      // we might already start delayed freeing while another thread has not yet
       // reset the delayed_freeing flag; in that case delay it further by reinserting.
       mi_block_t* dfree;
       do {
@@ -277,6 +281,7 @@ void _mi_heap_delayed_free(mi_heap_t* heap) {
   Unfull, abandon, free and retire
 ----------------------------------------------------------- */
 
+
 // Move a page from the full list back to a regular list
 void _mi_page_unfull(mi_page_t* page) {
   mi_assert_internal(page != NULL);
@@ -287,6 +292,9 @@ void _mi_page_unfull(mi_page_t* page) {
   if (!page->flags.in_full) return;
 
   mi_heap_t* heap = page->heap;
+  if (page->tag == mi_heap_tag_gc && heap->gcstate) {
+    mi_atomic_add(&heap->gcstate->gc_live, -page->capacity);
+  }
   mi_page_queue_t* pqfull = &heap->pages[MI_BIN_FULL];
   page->flags.in_full = false; // to get the right queue
   mi_page_queue_t* pq = mi_heap_page_queue_of(heap, page);
@@ -302,7 +310,12 @@ static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {
   _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE);
   if (page->flags.in_full) return;
 
-  mi_page_queue_enqueue_from(&page->heap->pages[MI_BIN_FULL], pq, page);
+  mi_heap_t* heap = page->heap;
+  if (page->tag == mi_heap_tag_gc && heap->gcstate) {
+    mi_atomic_add(&heap->gcstate->gc_live, page->capacity);
+  }
+
+  mi_page_queue_enqueue_from(&heap->pages[MI_BIN_FULL], pq, page);
   mi_page_thread_free_collect(page);  // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set
 }
 
@@ -541,6 +554,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
   mi_assert_internal(page_size / block_size < (1L<<16));
   page->reserved = (uint16_t)(page_size / block_size);
   page->cookie = _mi_heap_random(heap) | 1;
+  page->tag = heap->tag;
 
   mi_assert_internal(page->capacity == 0);
   mi_assert_internal(page->free == NULL);
@@ -690,7 +704,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) {
     mi_assert_internal(mi_page_immediate_available(page));
     mi_assert_internal(page->block_size == block_size);
     mi_heap_stat_increase( heap, huge, block_size);
-  }  
+  }
   return page;
 }
 
diff --git a/Objects/mimalloc/segment.c b/Objects/mimalloc/segment.c
index 3317554474..4080c13521 100644
--- a/Objects/mimalloc/segment.c
+++ b/Objects/mimalloc/segment.c
@@ -125,9 +125,9 @@ static mi_segment_queue_t* mi_segment_free_queue(mi_segment_t* segment, mi_segme
 static void mi_segment_remove_from_free_queue(mi_segment_t* segment, mi_segments_tld_t* tld) {
   mi_segment_queue_t* queue = mi_segment_free_queue(segment,tld); // may be NULL
   bool in_queue = (queue!=NULL && (segment->next != NULL || segment->prev != NULL || queue->first == segment));
-  if (in_queue) {    
+  if (in_queue) {
     mi_segment_queue_remove(queue,segment);
-  }  
+  }
 }
 
 static void mi_segment_insert_in_free_queue(mi_segment_t* segment, mi_segments_tld_t* tld) {
@@ -254,10 +254,10 @@ static mi_segment_t* _mi_segment_cache_findx(mi_segments_tld_t* tld, size_t requ
       mi_segment_os_free(segment, segment->segment_size, tld);
       // and look further...
     }
-    else if (segment->segment_size >= required) {      
+    else if (segment->segment_size >= required) {
       // always remove it from the cache
       mi_segment_cache_remove(segment, tld);
-      
+
       // exact size match?
       if (required==0 || segment->segment_size == required) {
         return segment;
@@ -435,7 +435,7 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t
   //fprintf(stderr,"mimalloc: free segment at %p\n", (void*)segment);
   mi_assert(segment != NULL);
   mi_segment_remove_from_free_queue(segment,tld);
-  
+
   mi_assert_expensive(!mi_segment_queue_contains(&tld->small_free, segment));
   mi_assert_expensive(!mi_segment_queue_contains(&tld->medium_free, segment));
   mi_assert(segment->next == NULL);
@@ -560,6 +560,10 @@ void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld)
 static volatile mi_segment_t* abandoned = NULL;
 static volatile uintptr_t     abandoned_count = 0;
 
+extern inline mi_segment_t* _mi_segment_abandoned(void) {
+  return abandoned;
+}
+
 static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) {
   mi_assert_internal(segment->used == segment->abandoned);
   mi_assert_internal(segment->used > 0);
@@ -569,9 +573,9 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) {
   mi_segment_remove_from_free_queue(segment,tld);
   mi_assert_internal(segment->next == NULL && segment->prev == NULL);
   // all pages in the segment are abandoned; add it to the abandoned list
-  segment->thread_id = 0;
+  mi_atomic_write(&segment->thread_id, 0);
   do {
-    segment->abandoned_next = (mi_segment_t*)abandoned;
+    segment->abandoned_next = (mi_segment_t*)mi_atomic_read_ptr((const volatile void**)&abandoned);
   } while (!mi_atomic_compare_exchange_ptr((volatile void**)&abandoned, segment, segment->abandoned_next));
   mi_atomic_increment(&abandoned_count);
   mi_stat_increase( tld->stats->segments_abandoned,1);
@@ -594,26 +598,30 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen
   uintptr_t reclaimed = 0;
   uintptr_t atmost;
   if (try_all) {
-    atmost = abandoned_count+16;   // close enough
+    atmost = mi_atomic_read(&abandoned_count)+16;   // close enough
   }
   else {
-    atmost = abandoned_count/8;    // at most 1/8th of all outstanding (estimated)
+    atmost = mi_atomic_read(&abandoned_count)/8;    // at most 1/8th of all outstanding (estimated)
     if (atmost < 8) atmost = 8;    // but at least 8
   }
 
   // for `atmost` `reclaimed` abandoned segments...
   while(atmost > reclaimed) {
     // try to claim the head of the abandoned segments
+    // ABA problem????
     mi_segment_t* segment;
     do {
-      segment = (mi_segment_t*)abandoned;
-    } while(segment != NULL && !mi_atomic_compare_exchange_ptr((volatile void**)&abandoned, segment->abandoned_next, segment));
-    if (segment==NULL) break; // stop early if no more segments available
+      segment = (mi_segment_t*)mi_atomic_acquire_ptr((const volatile void**)&abandoned);
+      if (segment == NULL) {
+        // stop early if no more segments available
+        return reclaimed > 0;
+      }
+    } while(!mi_atomic_compare_exchange_ptr((volatile void**)&abandoned, segment->abandoned_next, segment));
 
     // got it.
     mi_atomic_decrement(&abandoned_count);
-    segment->thread_id = _mi_thread_id();
-    segment->abandoned_next = NULL;
+    mi_atomic_write(&segment->thread_id, _mi_thread_id());
+    mi_atomic_write_ptr((volatile void**)&segment->abandoned_next, NULL);
     mi_segments_track_size((long)segment->segment_size,tld);
     mi_assert_internal(segment->next == NULL && segment->prev == NULL);
     mi_assert_expensive(mi_segment_is_valid(segment));
@@ -636,7 +644,17 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen
         }
         else {
           // otherwise reclaim it
-          _mi_page_reclaim(heap,page);
+          if (page->tag == heap->tag) {
+            _mi_page_reclaim(heap, page);
+          } else if (page->tag == mi_heap_tag_default) {
+            _mi_page_reclaim(heap->tld->heap_backing, page);
+          } else if (page->tag == mi_heap_tag_obj) {
+            _mi_page_reclaim(heap->tld->heap_obj, page);
+          } else if (page->tag == mi_heap_tag_gc) {
+            _mi_page_reclaim(heap->tld->heap_gc, page);
+          } else {
+            _mi_error_message("unknown page tag: %d\n", page->tag);
+          }
         }
       }
     }
diff --git a/Objects/mimalloc/stats.c b/Objects/mimalloc/stats.c
index 462088e025..0c679b5367 100644
--- a/Objects/mimalloc/stats.c
+++ b/Objects/mimalloc/stats.c
@@ -36,7 +36,10 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
   {
     // add atomically (for abandoned pages)
     int64_t current = mi_atomic_add(&stat->current,amount);
-    if (current > stat->peak) stat->peak = stat->current;  // racing.. it's ok
+    int64_t peak = mi_atomic_read64(&stat->peak);
+    if (current > peak) {
+      mi_atomic_write64(&stat->peak, current);  // racing.. it's ok
+    }
     if (amount > 0) {
       mi_atomic_add(&stat->allocated,amount);
     }