Convert NumPy `ndarray` to C# `System.Array` and back again

clr_array_convert.md

Originally for Python 3.7 and PythonNet 2.4.0 I wrote a snippet of code to transform NumPy ndarray into System.Array from CLR and back again using pure python and the ctypes package memmove function:

https://github.com/pythonnet/pythonnet/issues/514
https://github.com/pythonnet/pythonnet/issues/652

However, after the release of PythonNet 2.5.0 there were some changes to the PythonNet interface that created some small breaks in my code snippet:

https://github.com/pythonnet/pythonnet/issues/1187

Since a lot of people seem to have found the snippet to be useful I thought it would be useful to maintain a gist.

Code herein was tested against Python 3.9.4, NumPy 1.20.2, and PythonNet 2.5.1:

import ctypes

import numpy as np

import clr 
import System
from System import Array, Int32
from System.Runtime.InteropServices import GCHandle, GCHandleType

_MAP_NP_NET = {
    np.dtype(np.float32): System.Single,
    np.dtype(np.float64): System.Double,
    np.dtype(np.int8)   : System.SByte,
    np.dtype(np.int16)  : System.Int16,
    np.dtype(np.int32)  : System.Int32,
    np.dtype(np.int64)  : System.Int64,
    np.dtype(np.uint8)  : System.Byte,
    np.dtype(np.uint16) : System.UInt16,
    np.dtype(np.uint32) : System.UInt32,
    np.dtype(np.uint64) : System.UInt64,
    np.dtype(np.bool)   : System.Boolean,
}
_MAP_NET_NP = {
    'Single' : np.dtype(np.float32),
    'Double' : np.dtype(np.float64),
    'SByte'  : np.dtype(np.int8),
    'Int16'  : np.dtype(np.int16), 
    'Int32'  : np.dtype(np.int32),
    'Int64'  : np.dtype(np.int64),
    'Byte'   : np.dtype(np.uint8),
    'UInt16' : np.dtype(np.uint16),
    'UInt32' : np.dtype(np.uint32),
    'UInt64' : np.dtype(np.uint64),
    'Boolean': np.dtype(np.bool),
}

def asNumpyArray(netArray: System.Array):
    """
    Converts a .NET array to a NumPy array. See `_MAP_NET_NP` for 
    the mapping of CLR types to Numpy ``dtype``.

    Parameters
    ----------
    netArray: System.Array
        The array to be converted

    Returns
    -------
    numpy.ndarray 
    """
    dims = np.empty(netArray.Rank, dtype=int)
    for I in range(netArray.Rank):
        dims[I] = netArray.GetLength(I)
    netType = netArray.GetType().GetElementType().Name

    try:
        npArray = np.empty(dims, order='C', dtype=_MAP_NET_NP[netType])
    except KeyError:
        raise NotImplementedError(f'asNumpyArray does support System type {netType}')

    try: # Memmove 
        sourceHandle = GCHandle.Alloc(netArray, GCHandleType.Pinned)
        sourcePtr = sourceHandle.AddrOfPinnedObject().ToInt64()
        destPtr = npArray.__array_interface__['data'][0]
        ctypes.memmove(destPtr, sourcePtr, npArray.nbytes)
    finally:
        if sourceHandle.IsAllocated: 
            sourceHandle.Free()
    return npArray


def asNetArray(npArray):
    """
    Converts a NumPy array to a .NET array. See `_MAP_NP_NET` for 
    the mapping of CLR types to Numpy ``dtype``.

    Parameters
    ----------
    npArray: numpy.ndarray
        The array to be converted

    Returns
    -------
    System.Array

    Warning
    -------
    ``complex64`` and ``complex128`` arrays are converted to ``float32``
    and ``float64`` arrays respectively with shape ``[m,n,...] -> [m,n,...,2]``

    """
    dims = npArray.shape
    dtype = npArray.dtype

    # For complex arrays, we must make a view of the array as its corresponding 
    # float type as if it's (real, imag)
    if dtype == np.complex64:
        dtype = np.dtype(np.float32)
        dims += (2,)
        npArray = npArray.view(dtype).reshape(dims)
    elif dtype == np.complex128:
        dtype = np.dtype(np.float64)
        dims += (2,)
        npArray = npArray.view(dtype).reshape(dims)

    if not npArray.flags.c_contiguous or not npArray.flags.aligned:
        npArray = np.ascontiguousarray(npArray)
    assert npArray.flags.c_contiguous

    try:
        netArray = Array.CreateInstance(_MAP_NP_NET[dtype], *dims)
    except KeyError:
        raise NotImplementedError(f'asNetArray does not yet support dtype {dtype}')

    try: # Memmove 
        destHandle = GCHandle.Alloc(netArray, GCHandleType.Pinned)
        sourcePtr = npArray.__array_interface__['data'][0]
        destPtr = destHandle.AddrOfPinnedObject().ToInt64()
        ctypes.memmove(destPtr, sourcePtr, npArray.nbytes)
    finally:
        if destHandle.IsAllocated: 
            destHandle.Free()
    return netArray

For testing, here is a quick and dirty py.test module:

import psutil
import pytest
import numpy as np
import numpy.testing as npt

from clr_array_convert import asNumpyArray, asNetArray, _MAP_NP_NET

def test_dtypes():
    for dtype in _MAP_NP_NET.keys():
        nd0 = np.full([16, 16], 3, dtype=dtype)
        array0 = asNetArray(nd0)
        nd1 = asNumpyArray(array0)

        assert(nd0.dtype == nd1.dtype)
        npt.assert_array_almost_equal(nd0, nd1)

def test_dimensions():
    for ndim in range(1, 6):
        shape = tuple(range(2, 2 + ndim))
        nd0 = np.full(shape, 2.2, dtype=np.float32)
        array0 = asNetArray(nd0)
        nd1 = asNumpyArray(array0)

        assert(np.all(nd0.shape == nd1.shape))
        npt.assert_array_almost_equal(nd0, nd1)

thanks!

in python i want create array Array.CreateInstance(System, *dims = (24459153, 15)) it return error System.OutOfMemoryException: Array dimensions exceeded supported range. how to resolve it thanks