Write to managed arrays from an `IJob` ! A collections of useful utilities when working with `Unity.Jobs`.

.write_to_managed_array_from_Burst-compiled_IJob.md

👉write to managed array from a Burst-compiled IJob

Highlights:

• cast T[] to NativeArray<T>
• schedule jobs that read/write to managed arrays
• pointer safety assertion that prevent crashes and data corruption

more info in the first comment under the source code

ArrayExtensionMethods.cs

// src* https://gist.github.com/andrew-raphael-lukasik/812e152f95e38cc13c44e5040c5739bc
using UnityEngine;
using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
using Unity.Profiling;

public static class ArrayExtensionMethods
{
    /// <summary>
    /// Pins this GC array and turns it's pointer into a NativeArray.
    /// Do not Dispose but call UnsafeUtility.ReleaseGCObject(gcHandle) when done with it.
    /// </summary>
    public static unsafe NativeArray<T> AsNativeArray<T>(this T[] array, out ulong gcHandle) where T : unmanaged
    {
        void* ptr = UnsafeUtility.PinGCArrayAndGetDataAddress(array, out gcHandle);
        var nativeArray = NativeArrayUnsafeUtility.ConvertExistingDataToNativeArray<T>(ptr, array.Length, Allocator.None);
        #if ENABLE_UNITY_COLLECTIONS_CHECKS
        NativeArrayUnsafeUtility.SetAtomicSafetyHandle(ref nativeArray, AtomicSafetyHandle.Create());
        #endif
        return nativeArray;
    }
    /// <inheritdoc />
    public static unsafe NativeArray<T> AsNativeArray<T>(this T[,] array, out ulong gcHandle) where T : unmanaged
    {
        void* ptr = UnsafeUtility.PinGCArrayAndGetDataAddress(array, out gcHandle);
        var nativeArray = NativeArrayUnsafeUtility.ConvertExistingDataToNativeArray<T>(ptr, array.Length, Allocator.None);
        #if ENABLE_UNITY_COLLECTIONS_CHECKS
        NativeArrayUnsafeUtility.SetAtomicSafetyHandle(ref nativeArray, AtomicSafetyHandle.Create());
        #endif
        return nativeArray;
    }
    /// <inheritdoc />
    public static unsafe NativeArray<T> AsNativeArray<T>(this T[,,] array, out ulong gcHandle) where T : unmanaged
    {
        void* ptr = UnsafeUtility.PinGCArrayAndGetDataAddress(array, out gcHandle);
        var nativeArray = NativeArrayUnsafeUtility.ConvertExistingDataToNativeArray<T>(ptr, array.Length, Allocator.None);
        #if ENABLE_UNITY_COLLECTIONS_CHECKS
        NativeArrayUnsafeUtility.SetAtomicSafetyHandle(ref nativeArray, AtomicSafetyHandle.Create());
        #endif
        return nativeArray;
    }
    /// <inheritdoc />
    public static unsafe NativeArray<(T1,T2)> AsNativeArray<T1,T2>(this (T1,T2)[] array, out ulong gcHandle)
        where T1 : unmanaged
        where T2 : unmanaged
    {
        void* ptr = UnsafeUtility.PinGCArrayAndGetDataAddress(array, out gcHandle);
        var nativeArray = NativeArrayUnsafeUtility.ConvertExistingDataToNativeArray<(T1,T2)>(ptr, array.Length, Allocator.None);
        #if ENABLE_UNITY_COLLECTIONS_CHECKS
        NativeArrayUnsafeUtility.SetAtomicSafetyHandle(ref nativeArray, AtomicSafetyHandle.Create());
        #endif
        return nativeArray;
    }
    /// <inheritdoc />
    public static unsafe NativeArray<(T1,T2,T3)> AsNativeArray<T1,T2,T3>(this (T1,T2,T3)[] array, out ulong gcHandle)
        where T1 : unmanaged
        where T2 : unmanaged
        where T3 : unmanaged
    {
        void* ptr = UnsafeUtility.PinGCArrayAndGetDataAddress(array, out gcHandle);
        var nativeArray = NativeArrayUnsafeUtility.ConvertExistingDataToNativeArray<(T1,T2,T3)>(ptr, array.Length, Allocator.None);
        #if ENABLE_UNITY_COLLECTIONS_CHECKS
        NativeArrayUnsafeUtility.SetAtomicSafetyHandle(ref nativeArray, AtomicSafetyHandle.Create());
        #endif
        return nativeArray;
    }

    public static string ToReadableString<T>(this T[] arr)
    {
        if (arr.Length == 0) return "()";
        var sb = new System.Text.StringBuilder();
        sb.Append($"({arr[0]}");
        for (int i = 1; i < arr.Length; i++)
            sb.Append($",{arr[i]}");
        sb.Append(')');
        return sb.ToString();
    }
    public static string ToReadableString<T>(this T[,] arr)
    {
        int
            lengthY = arr.GetLength(0),
            lengthX = arr.GetLength(1),
            length = arr.Length;
        var sb = new System.Text.StringBuilder($"[{lengthY},{lengthX}]( ");
        for (int y = 0; y < lengthY; y++)
        {
            if (y != 0)
                sb.Append(" , ");
            if (lengthX != 0)
                sb.Append($"({arr[y, 0]}");
            for (int x = 1; x < lengthX; x++)
                sb.Append($",{arr[y, x]}");
            sb.Append(')');
        }
        sb.Append($" )");
        return sb.ToString();
    }
    public static string ToReadableString<T>(this T[,,] arr)
    {
        int
            lengthZ = arr.GetLength(0),
            lengthY = arr.GetLength(1),
            lengthX = arr.GetLength(2),
            length = arr.Length;
        var sb = new System.Text.StringBuilder($"[{lengthZ},{lengthY},{lengthX}]( ");
        for (int z = 0; z < lengthZ; z++)
        {
            if (z != 0) sb.Append(" , ");
            sb.Append("( ");
            for (int y = 0; y < lengthY; y++)
            {
                if (y != 0)
                    sb.Append(" , ");
                if (lengthX != 0)
                sb.Append($"({arr[z, y, 0]}");
                for (int x = 1; x < lengthX; x++)
                    sb.Append($",{arr[z, y, x]}");
                sb.Append(')');
            }
            sb.Append(" )");
        }
        sb.Append($" )");
        return sb.ToString();
    }

}

JobUtility.cs

// src* https://gist.github.com/andrew-raphael-lukasik/812e152f95e38cc13c44e5040c5739bc
using Unity.Jobs;

public static class JobUtility
{
    /// <summary>
    /// Schedules a job that calls UnsafeUtility.ReleaseGCObject(gcHandle).
    /// </summary>
    public static JobHandle ReleaseGCObject(ulong gcHandle, JobHandle dependency = default)
        => new ReleaseGCObjectJob(gcHandle).Schedule(dependency);

    /// <summary>
    /// This equation is yet to be (im)proved experimentally, the current version is merely an initial guesswork
    /// </summary>
    public static int OptimalLoopBatchCount(int length)
        => Unity.Mathematics.math.max(length / (UnityEngine.SystemInfo.processorCount * 3), 1);
}

NativeArrayExtensionMethods.cs

// src* https://gist.github.com/andrew-raphael-lukasik/812e152f95e38cc13c44e5040c5739bc
using UnityEngine;
using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
using Unity.Jobs;
using Unity.Profiling;

public static class NativeArrayExtensionMethods
{

    static readonly ProfilerMarker
        ___CopyTo = new ProfilerMarker("CopyTo"),
        ___CopyFrom = new ProfilerMarker("CopyFrom");

    public static unsafe void CopyTo<T>(this NativeArray<T> src, T[,] dst) where T : unmanaged
    {
        ___CopyTo.Begin();
        if (src.Length == dst.Length)
        {
            int size = src.Length * UnsafeUtility.SizeOf<T>();
            void* srcPtr = NativeArrayUnsafeUtility.GetUnsafePtr(src);
            void* dstPtr = UnsafeUtility.PinGCArrayAndGetDataAddress(dst, out ulong dstHandle);
            UnsafeUtility.MemCpy(destination: dstPtr, source: srcPtr, size: size);
            UnsafeUtility.ReleaseGCObject(dstHandle);
        }
        else Debug.LogError($"<b>{nameof(src)}.Length</b> ({src}[b]) and <b>{nameof(dst)}.Length</b> ({dst.Length}[b]) must be equal. MemCpy aborted.");
        ___CopyTo.End();
    }
    public static unsafe void CopyTo<T>(this NativeArray<T> src, T[,,] dst) where T : unmanaged
    {
        ___CopyTo.Begin();
        if (src.Length == dst.Length)
        {
            int size = src.Length * UnsafeUtility.SizeOf<T>();
            void* srcPtr = NativeArrayUnsafeUtility.GetUnsafePtr(src);
            void* dstPtr = UnsafeUtility.PinGCArrayAndGetDataAddress(dst, out ulong dstHandle);
            UnsafeUtility.MemCpy(destination: dstPtr, source: srcPtr, size: size);
            UnsafeUtility.ReleaseGCObject(dstHandle);
        }
        else Debug.LogError($"<b>{nameof(src)}.Length</b> ({src}[b]) and <b>{nameof(dst)}.Length</b> ({dst.Length}[b]) must be equal. MemCpy aborted.");
        ___CopyTo.End();
    }

    public static unsafe void CopyFrom<T>(this NativeArray<T> dst, T[,] src) where T : unmanaged
    {
        ___CopyFrom.Begin();
        if (src.Length == dst.Length)
        {
            int size = src.Length * UnsafeUtility.SizeOf<T>();
            void* srcPtr = UnsafeUtility.PinGCArrayAndGetDataAddress(src, out ulong dstHandle);
            void* dstPtr = NativeArrayUnsafeUtility.GetUnsafePtr(dst);
            UnsafeUtility.MemCpy(destination: dstPtr, source: srcPtr, size: size);
            UnsafeUtility.ReleaseGCObject(dstHandle);
        }
        else Debug.LogError($"<b>{nameof(src)}.Length</b> ({src}[b]) and <b>{nameof(dst)}.Length</b> ({dst.Length}[b]) must be equal. MemCpy aborted.");
        ___CopyFrom.End();
    }
    public static unsafe void CopyFrom<T>(this NativeArray<T> dst, T[,,] src) where T : unmanaged
    {
        ___CopyFrom.Begin();
        if (src.Length == dst.Length)
        {
            int size = src.Length * UnsafeUtility.SizeOf<T>();
            void* srcPtr = UnsafeUtility.PinGCArrayAndGetDataAddress(src, out ulong dstHandle);
            void* dstPtr = NativeArrayUnsafeUtility.GetUnsafePtr(dst);
            UnsafeUtility.MemCpy(destination: dstPtr, source: srcPtr, size: size);
            UnsafeUtility.ReleaseGCObject(dstHandle);
        }
        else Debug.LogError($"<b>{nameof(src)}.Length</b> ({src}[b]) and <b>{nameof(dst)}.Length</b> ({dst.Length}[b]) must be equal. MemCpy aborted.");
        ___CopyFrom.End();
    }


    /// <summary> Schedules a <see cref="CopyToJob{T}"/>. </summary>
    [Unity.Burst.BurstDiscard]// Burst warning without it, not sure why
    public static JobHandle CopyTo<T>(this NativeArray<T> src, NativeArray<T> dst, JobHandle dependency) where T : unmanaged
        => new CopyToJob<T>(src,dst).Schedule(dependency);
    
    /// <summary> Schedules a <see cref="CopyToJob{T}"/>. </summary>
    public static JobHandle CopyFrom<T>(this NativeArray<T> dst, NativeArray<T> src, JobHandle dependency) where T : unmanaged
        => new CopyToJob<T>(src,dst).Schedule(dependency);

    /// <summary> Fills entire array using given value. </summary>
    public static unsafe void Fill<T>(this NativeArray<T> Array, T value) where T : unmanaged
    {
        void* src = UnsafeUtility.Malloc(UnsafeUtility.SizeOf<T>(), UnsafeUtility.AlignOf<T>(), Allocator.Temp);
        void* dst = NativeArrayUnsafeUtility.GetUnsafePtr<T>(Array);
        int size = UnsafeUtility.SizeOf<T>();
        int count = Array.Length;
        UnsafeUtility.MemCpyReplicate(destination: dst, source: src, size: size, count: count);
    }


    /// <summary>
    /// Raises an exception when given pointer refers to address outside this array.
    /// This makes sure this pointer won't crash the game.
    /// <b>NOTE</b>: Editor and DEBUG builds only.
    /// </summary>
    [System.Diagnostics.Conditional("DEBUG")]
    public static unsafe void AssertPtrScope<ARRAY_ITEM, POINTER>(this NativeSlice<ARRAY_ITEM> array, POINTER* ptr)
        where ARRAY_ITEM : unmanaged
        where POINTER : unmanaged
    {
        long addr = (long)ptr;
        long firstItemAddr = (long)NativeSliceUnsafeUtility.GetUnsafeReadOnlyPtr(array);
        long lastItemAddr = firstItemAddr + array.Length * (long)UnsafeUtility.SizeOf<ARRAY_ITEM>() - UnsafeUtility.SizeOf<POINTER>();
        if (!(addr >= firstItemAddr && addr < lastItemAddr))
        {
            string message = $"Pointer is out of scope, so considered unsafe (possible crash prevented). Ptr:{addr}, array first item addr:{firstItemAddr}, array last item addr:{lastItemAddr}";
            Debug.LogWarning(message);
            throw new System.ArgumentOutOfRangeException(message);
        }
    }

    /// <inheritdoc />
    [System.Diagnostics.Conditional("DEBUG")]
    public static unsafe void AssertPtrScope<ARRAY_ITEM, POINTER>(this NativeArray<ARRAY_ITEM> array, POINTER* ptr)
        where ARRAY_ITEM : unmanaged
        where POINTER : unmanaged
    {
        long addr = (long)ptr;
        long firstItemAddr = (long)NativeArrayUnsafeUtility.GetUnsafeReadOnlyPtr(array);
        long lastItemAddr = firstItemAddr + array.Length * (long)UnsafeUtility.SizeOf<ARRAY_ITEM>() - UnsafeUtility.SizeOf<POINTER>();
        if (!(addr >= firstItemAddr && addr <= lastItemAddr))
        {
            string message = $"Pointer is out of scope, so considered unsafe (possible crash prevented). Ptr:{addr}, array first item addr:{firstItemAddr}, array last item addr:{lastItemAddr}";
            Debug.LogWarning(message);
            throw new System.ArgumentOutOfRangeException(message);
        }
    }

    public static string ToReadableString<T>(this NativeArray<T> arr) where T : unmanaged
    {
        if (arr.Length == 0) return "()";
        var sb = new System.Text.StringBuilder();
        sb.Append($"({arr[0]}");
        for (int i = 1; i < arr.Length; i++)
            sb.Append($",{arr[i]}");
        sb.Append(')');
        return sb.ToString();
    }


}

UniversalJobs.cs

// src* https://gist.github.com/andrew-raphael-lukasik/812e152f95e38cc13c44e5040c5739bc
using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
using Unity.Jobs;

[Unity.Burst.BurstCompile]
public struct DeallocateArrayJob<T> : IJob where T : unmanaged
{
    [ReadOnly] [DeallocateOnJobCompletion] NativeArray<T> Array;
    public DeallocateArrayJob(NativeArray<T> array) => this.Array = array;
    void IJob.Execute() { }
}

[Unity.Burst.BurstCompile]
public struct FillJob<T> : IJob where T : unmanaged
{
    T Value;
    [WriteOnly] NativeArray<T> Array;
    public FillJob(NativeArray<T> array, T value)
    {
        this.Value = value;
        this.Array = array;
    }
    unsafe void IJob.Execute()
    {
        void* src = UnsafeUtility.Malloc(UnsafeUtility.SizeOf<T>(), UnsafeUtility.AlignOf<T>(), Allocator.Temp);
        void* dst = NativeArrayUnsafeUtility.GetUnsafePtr<T>(Array);
        int size = UnsafeUtility.SizeOf<T>();
        int count = this.Array.Length;
        UnsafeUtility.MemCpyReplicate(destination: dst, source: src, size: size, count: count);
    }
}

[Unity.Burst.BurstCompile]
public struct CopyToJob<T> : IJob where T : unmanaged
{
    [ReadOnly] NativeArray<T> Src;
    [WriteOnly] NativeArray<T> Dst;
    int SrcIndex, DstIndex, Length;
    public CopyToJob(NativeArray<T> src, NativeArray<T> dst)
    {
        this.Src = src;
        this.SrcIndex = -1;
        this.Dst = dst;
        this.DstIndex = -1;
        this.Length = -1;
    }
    public CopyToJob(NativeArray<T> src, int srcIndex, NativeArray<T> dst, int dstIndex, int length)
    {
        this.Src = src;
        this.SrcIndex = srcIndex;
        this.Dst = dst;
        this.DstIndex = dstIndex;
        this.Length = length;
    }
    void IJob.Execute()
    {
        if (Length == -1) NativeArray<T>.Copy(Src, Dst);
        else NativeArray<T>.Copy(Src, SrcIndex, Dst, DstIndex, Length);
    }
}

[Unity.Burst.BurstCompile]
public struct ReleaseGCObjectJob : IJob
{
    ulong Handle;
    public ReleaseGCObjectJob(ulong handle) => this.Handle = handle;
    void IJob.Execute() => UnsafeUtility.ReleaseGCObject(Handle);
}

Check this out:

• cast managed arrays to native ones

int[] myArray = new int[] { 0 , 1 , 2 };
NativeArray<int> myArrayNativeView = myArray.AsNativeArray(out ulong gcHandle);// this is NOT A COPY

myArrayNativeView[0] = 255;
myArrayNativeView[1] = 255;
myArrayNativeView[2] = 255;
JobUtility.ReleaseGCObject(gcHandle);
// myArray is filled with 255 now

• schedule jobs that read or write to managed arrays

int[] myArray = new int[] { 0 , 1 , 2 , 3 , 4 };
JobHandle jobHandle = new MyJob(myArray.AsNativeArray(out ulong gcHandle)).Schedule();
JobHandle released = JobUtility.ReleaseGCObject(gcHandle, jobHandle);

• pointer safety

NativeArray<VertexBuffer> Buffer;
unsafe void IJobParallelFor.Execute(int index)
{
    VertexBuffer* vertexPtr = (VertexBuffer*)NativeArrayUnsafeUtility.GetUnsafePtr(Buffer) + index;

    // ptr arithmetic
    float3* pos    = (float3*)vertexPtr;
    short3* normal = (short3*)((byte*)pos + 3*4);
    byte4* tangent = (byte4*)((byte*)normal + 3*2);
    short2* uv     = (short2*)((byte*)tangent + 4*1);
    
    // assertions that prevent buffer overflow i.e.: editor from crashing and surprise memory corruptions
    Buffer.AssertPtrScope(pos);
    Buffer.AssertPtrScope(normal);
    Buffer.AssertPtrScope(tangent);
    Buffer.AssertPtrScope(uv);

    // mem access
    *pos     = ___;
    *normal  = ___;
    *tangent = ___;
    *uv      = ___;
}

Hi @andrew-raphael-lukasik , I've had your gist laying around for a while and finally had a need to use the Fill method. However, it seems to be bugged.

    public static unsafe void Fill<T>(this NativeArray<T> Array, T value) where T : unmanaged
    {
        void* src = UnsafeUtility.Malloc(UnsafeUtility.SizeOf<T>(), UnsafeUtility.AlignOf<T>(), Allocator.Temp);
        void* dst = NativeArrayUnsafeUtility.GetUnsafePtr<T>(Array);
        int size = UnsafeUtility.SizeOf<T>();
        int count = Array.Length;
        UnsafeUtility.MemCpyReplicate(destination: dst, source: src, size: size, count: count);
    }

it doesn't actually use the value that is passed in, and instead seems to fill Array with copies of whatever garbage src was allocated with. I assume you meant to then copy value to src - though I'm wondering why you're allocating new data as opposed to getting a pointer to value directly. For example, this does appear to give correct results:

    public static unsafe void Fill<T>(this NativeArray<T> Array, T value) where T : unmanaged
    {
        void* dst = NativeArrayUnsafeUtility.GetUnsafePtr<T>(Array);
        int size = UnsafeUtility.SizeOf<T>();
        int count = Array.Length;
        void* src = UnsafeUtility.AddressOf(ref value); // Get the address of the value directly
        UnsafeUtility.MemCpyReplicate(destination: dst, source: src, size: size, count: count);
    }