Viewport to World Point Unity.cs

private static Vector3 ViewportToWorldPointUnity (Vector3 point, float fieldOfView, float aspect, Vector3 from, Vector3 to, Vector3 up)
{
    // The zNear / zFar magic numbers are what Unity's viewport always uses regardless of camera settings
    var matP = Matrix4x4.Perspective (fieldOfView, aspect, 0.6f /* zNear */, 1000f /* zFar */);

    // For some reason we have to reverse these coordinates
    from = -from;
    to = -to;

    var matT = Matrix4x4.Translate (from);
    var matR = Matrix4x4.Rotate (Quaternion.LookRotation (to-from, up));

    // Unity uses a viewport in the range [0f, 1f] and flipped on X
    var stupidUnityViewport = Matrix4x4.TRS (new Vector3 (0.5f, 0.5f, 0f), Quaternion.identity, new Vector3 (-0.5f, 0.5f, 1f));
    var invMatVP = (stupidUnityViewport * matP * matR.transpose * matT).inverse;

    // Another set of magic numbers!
    const float zNear = 1.2f;
    const float zFar = 1000f;

    // Convert the point's Z coordinate into depth-buffer space. Why? Who knows.
    float x = point.x;
    float y = point.y;
    float z = 0.5f * ((zFar + zNear - 2*zNear*zFar/point.z) / (zFar - zNear) + 1f);
    return invMatVP.MultiplyPoint (new Vector3 (x, y, z));
}