Releasing the GIL in Nim

gil_test.nim

import strutils, strformat, os, random, times
import dynlib
import nimpy, nimpy/py_lib, nimpy/py_types
{.pragma: pyfunc, cdecl, gcsafe.}
import locks
import malebolgia, math
import random, os

import chronos

{.pragma: pyfunc, cdecl, gcsafe.}



initPyLibIfNeeded()
let m = py_lib.pyLib.module

type
  PyInterpreterStateObj {.incompleteStruct.} = object
  PyInterpreterState = ptr PyInterpreterStateObj
  PyThreadStateObj {.incompleteStruct.} = object
    prev: ptr PyThreadState3
    next: ptr PyThreadState3
    interp: PyInterpreterState
    frame: pointer
  PyThreadState* = ptr PyThreadStateObj
  PyGILState_STATE* = bool
let
  PyGILState_Ensure* = cast[proc(): PyGILState_STATE {.pyfunc.}](m.symAddr("PyGILState_Ensure"))
  PyGILState_Release* = cast[proc(s: PyGILState_STATE) {.pyfunc.}](m.symAddr("PyGILState_Release"))
  PyGILState_Check* = cast[proc(): bool {.pyfunc.}](m.symAddr("PyGILState_Check"))
  PyGILState_GetThisThreadState* = cast[proc(): PyThreadState {.pyfunc.}](
      m.symAddr("PyGILState_GetThisThreadState"))
  PyThreadState_Get* = cast[proc(): PyThreadState {.pyfunc.}](m.symAddr("PyThreadState_Get"))
  PyThreadState_New* = cast[proc(interp: PyInterpreterState): PyThreadState{.pyfunc.}](
      m.symAddr("PyThreadState_New"))
  PyThreadState_Swap* = cast[proc(s: PyThreadState): PyThreadState{.pyfunc.}](
      m.symAddr("PyThreadState_Swap"))
  PyEval_SaveThread* = cast[proc(): PyThreadState {.pyfunc.}](m.symAddr("PyEval_SaveThread"))
  PyEval_RestoreThread* = cast[proc(s: PyThreadState) {.pyfunc.}](m.symAddr("PyEval_RestoreThread"))
  PyEval_AcquireThread* = cast[proc(s: PyThreadState) {.pyfunc.}](m.symAddr("PyEval_AcquireThread"))
  PyEval_ReleaseThread* = cast[proc(s: PyThreadState) {.pyfunc.}](m.symAddr("PyEval_ReleaseThread"))
  PyEval_InitThreads* = cast[proc() {.pyfunc.}](m.symAddr("PyEval_InitThreads"))
  PyEval_AcquireLock* = cast[proc() {.pyfunc.}](m.symAddr("PyEval_AcquireLock"))
  PyEval_ReleaseLock* = cast[proc() {.pyfunc.}](m.symAddr("PyEval_ReleaseLock"))
  Py_IsInitialized* = cast[proc(): bool {.pyfunc.}](m.symAddr("Py_IsInitialized"))
  Py_Initialize* = cast[proc() {.pyfunc.}](m.symAddr("Py_Initialize"))
  PyOS_AfterFork_Child* = cast[proc() {.pyfunc.}](m.symAddr("PyOS_AfterFork_Child"))
  PyThreadState_GetInterpreter* = cast[proc(state: PyThreadState): PyInterpreterState {.pyfunc.}](
          m.symAddr("PyThreadState_GetInterpreter"))
  PyInterpreterState_Get* = cast[proc(): PyInterpreterState {.pyfunc.}](
      m.symAddr("PyInterpreterState_Get"))

let pyErrClear* = py_lib.pyLib.PyErr_Clear
proc pyErrOccurred*(): bool = not py_lib.pyLib.PyErr_Occurred().isNil
proc pyCurrentThread*(): auto = cast[PyThreadState](PyThreadState_Get())
let gilLocked* = PyGILState_Check

when defined(pyAsync):
  type
    PyGilObj = object
      lock: Lock
      currentLockHolder: int
      state: PyGILState_STATE
    PyGil = ptr PyGilObj

  var pyGil*: PyGil
  var pyGilLock*: Lock
  var pyMainThread: PyThreadState
  proc initPyGil*() =
    assert PyGILState_Check()
    pyGil = create(PyGilObj)
    pyGil.currentLockHolder = getThreadID()
    pyGil.lock.initLock()
    pyGilLock = pyGil.lock
    pyMainThread = PyEval_SaveThread()

  proc acquire*(gil: PyGil): void =
    gil.lock.acquire()
    let id = getThreadId()
    gil.currentLockHolder = id
    gil.state = Py_GILState_Ensure()

  proc tryAcquire*(gil: PyGil): bool =
    if gil.lock.tryAcquire():
      let id = getThreadId()
      gil.currentLockHolder = id
      gil.state = Py_GILState_Ensure()
      return true

  proc release*(gil: PyGil) {.inline.} =
    doassert gil.currentLockHolder == getThreadId(), "Can't release gil lock from a different thread."
    doassert gilLocked()
    Py_GILState_Release(gil.state)
    gil.lock.release

  proc globalAcquire*(gil: PyGil) {.inline.} =
    gil.acquire()

  proc locked*(gil: PyGil): bool {.inline.} =
    if gil.lock.tryAcquire:
      gil.lock.release
      return false
    else:
      return true

  initPyGil()

#  proc `=destroy`*(p: var PyObject) =
#    let acquired = pygil.tryAcquire()
#    if not p.rawPyObj.isnil:
#      decRef p.rawPyObj
#      p.rawPyObj = nil
#    if acquired:

#      pygil.release()

else:
  let pyMainThread = PyEval_SaveThread()

proc f() {.gcsafe.} =
  echo getThreadId()
  pyGil.acquire()
  let nx = pyImport("networkx")
  echo nx.path_graph(100)
  pyGil.release()
  var r = 0.0

  for idx in (0..1000000):
    r += cos(idx.float)

var ms = createMaster()
ms.awaitAll:
  for idx in (0..1000):
    ms.spawn f()