Trick for using multiprocessing with nested functions and lambda expressions

nested.py

import concurrent.futures
import multiprocessing
import sys
import uuid

def globalize(func):
  def result(*args, **kwargs):
    return func(*args, **kwargs)
  result.__name__ = result.__qualname__ = uuid.uuid4().hex
  setattr(sys.modules[result.__module__], result.__name__, result)
  return result

def main():
  @globalize
  def func1(x):
    return x

  func2 = globalize(lambda x: x)

  with multiprocessing.Pool() as pool:
    print(pool.map(func1, range(10)))
    print(pool.map(func2, range(10)))

  with concurrent.futures.ThreadPoolExecutor() as executor:
    print(list(executor.map(func1, range(10))))
    print(list(executor.map(func2, range(10))))

if __name__ == '__main__':
  main()

@pk1234dva As far as I understand it, multiprocessing pickles the function name so that the worker processes know where to start. The trick here simply automates the creation of wrappers that can be pickled. Nothing else is pickled, neither in the global scope nor in the local scope, so the module must still be available in some form for the program to work. Whether the worker processes inherit the module in memory as a result of a fork or import the module anew is mostly an implementation detail.

@EdwinChan Thank you.