Interactive Camera

InteractiveCamera.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using CodeX;
using BaseX;

namespace FrooxEngine
{
    [Category("Media/Capture")]
    public class InteractiveCamera : Component, ITriggerActionReceiver, IGrabbableReparentBlock
    {
        public enum Mode
        {
            Camera2D,
            CameraStereo,
            Camera360,
            CameraObject
        };

        public enum EncodeFormat
        {
            PNG,
            JPG,
            WebP
        }

        public readonly Sync<Mode> CameraMode;

        public readonly Sync<int> PreviewWidth;
        public readonly Sync<int> PreviewHeight;

        public readonly Sync<int> RenderWidth;

        public readonly Sync<float> StereoSeparation;

        public readonly Sync<float> TimerInterval;
        public readonly Sync<bool> TimerEnabled;

        public readonly FieldDrive<string> TimerCountIndicator;
        public readonly FieldDrive<color> TimerColorIndicator;

        readonly SyncTime _timerStartPoint;
        readonly SyncRef<User> _timerUser;

        public readonly RelayRef<Camera> MainCamera;
        public readonly SyncRef<Camera> SecondaryCamera;

        public readonly SyncRef<RenderTextureProvider> PreviewTexture;

        public readonly SyncRef<IStereoMaterial> DisplayMaterial;

        public readonly Sync<EncodeFormat> Format;
        public readonly Sync<float> Quality;

        public readonly SyncRef<Slot> PhotoSpawnPoint;
        public readonly Sync<float> PhotoSpawnSize;

        public readonly SyncRef<Slot> PanoramaIndicator;
        public readonly FieldDrive<float3> PanoramaIndicatorSize;

        public readonly SyncRef<Slot> ObjectTargetSource;
        public readonly FieldDrive<bool> ObjectTargetSourceActive;

        public readonly Sync<bool> ObjectAutoPose;

        public readonly AssetRef<AudioClip> CaptureSound;

        readonly FieldDrive<float3> _leftCamOffset;
        readonly FieldDrive<float3> _rightCamOffset;
        readonly FieldDrive<floatQ> _leftCamOrientation;
        readonly FieldDrive<floatQ> _rightCamOrientation;

        public bool DontReparent => true;

        protected override void OnAwake()
        {
            base.OnAwake();

            PreviewWidth.Value = 400;
            PreviewHeight.Value = 300;

            RenderWidth.Value = 2048;

            Format.Value = EncodeFormat.PNG;
            Quality.Value = 80;

            PhotoSpawnSize.Value = 0.1f;
        }

        protected override void OnAttach()
        {
            var cameras = Slot.AddSlot("Cameras");
            var mainCamSlot = cameras.AddSlot("MainCamera");
            var secondaryCameraSlot = cameras.AddSlot("SecondaryCamera");

            MainCamera.Target = mainCamSlot.AttachComponent<Camera>();
            SecondaryCamera.Target = secondaryCameraSlot.AttachComponent<Camera>();

            _leftCamOffset.Target = mainCamSlot.Position_Field;
            _rightCamOffset.Target = secondaryCameraSlot.Position_Field;

            PreviewTexture.Target = Slot.AttachComponent<RenderTextureProvider>();

            MainCamera.Target.RenderTexture.Target = PreviewTexture.Target;
        }

        Slot RaycastRenderObject()
        {
            var source = ObjectTargetSource.Target ?? Slot;

            var hits = Pool.BorrowList<RaycastHit>();

            Physics.RaycastAll(source.GlobalPosition, source.Forward, hits);

            Slot renderObject = null;

            if (hits.Count > 0)
                renderObject = hits[0].Collider.Slot.GetObjectRoot();

            Pool.Return(hits);

            return renderObject;
        }

        protected override void OnCommonUpdate()
        {
            if(TimerEnabled.Value)
            {
                if(_timerUser.Target != null)
                {
                    var remainingTime = MathX.Max(0f, TimerInterval.Value - _timerStartPoint.CurrentTime);

                    var flashInterval = remainingTime < 2f ? 0.25f : 0.5f;

                    TimerCountIndicator.Target.Value = remainingTime.ToString("0.0", System.Globalization.CultureInfo.InvariantCulture);
                    TimerColorIndicator.Target.Value = (((int)(remainingTime / flashInterval)) % 2 == 0) ? color.Red : color.White;

                    if(_timerUser.Target.IsLocal && remainingTime <= 0)
                    {
                        Capture();
                        _timerUser.Target = null;
                    }
                }
                else
                {
                    TimerCountIndicator.Target.Value = TimerInterval.Value.ToString("0.0", System.Globalization.CultureInfo.InvariantCulture);
                    TimerColorIndicator.Target.Value = color.White;
                }
            }

            if (PanoramaIndicatorSize.IsLinkValid)
                PanoramaIndicatorSize.Target.Value = MathX.Lerp(PanoramaIndicatorSize.Target.Value,
                    CameraMode.Value == Mode.Camera360 ? float3.One : float3.Zero, Time.Delta * 4);

            if(CameraMode.Value == Mode.CameraObject)
            {
                if (MainCamera.Target.SelectiveRender.Count < 1)
                    MainCamera.Target.SelectiveRender.Add();

                while (MainCamera.Target.SelectiveRender.Count > 1)
                    MainCamera.Target.SelectiveRender.RemoveAt(0);

                var renderObject = RaycastRenderObject();

                MainCamera.Target.SelectiveRender[0] = renderObject;

                if(renderObject != null && ObjectAutoPose.Value)
                {
                    renderObject.GetRenderPoint(out float3 renderPoint, out floatQ renderRotation, GetSingleRenderResolution(),
                        MainCamera.Target.Slot.Parent.GlobalPosition, MainCamera.Target.FieldOfView);

                    MainCamera.Target.Slot.GlobalPosition = renderPoint;
                    MainCamera.Target.Slot.GlobalRotation = renderRotation;
                }
                else
                {
                    MainCamera.Target.Slot.LocalPosition = float3.Zero;
                    MainCamera.Target.Slot.LocalRotation = floatQ.Identity;
                }
            }
        }

        protected override void OnChanges()
        {
            PreviewTexture.Target.Size.Value = GetFinalPreviewResolution();

            if (ObjectTargetSourceActive.IsLinkValid)
                ObjectTargetSourceActive.Target.Value = CameraMode.Value == Mode.CameraObject;

            // make sure selective render is off
            if(CameraMode.Value != Mode.CameraObject)
            {
                while (MainCamera.Target.SelectiveRender.Count > 0)
                    MainCamera.Target.SelectiveRender.RemoveAt(0);
            }

            MainCamera.Target.Slot.LocalPosition = float3.Zero;
            MainCamera.Target.Slot.LocalRotation = floatQ.Identity;

            SecondaryCamera.Target.Slot.LocalPosition = float3.Zero;
            SecondaryCamera.Target.Slot.LocalRotation = floatQ.Identity;

            switch (CameraMode.Value)
            {
                case Mode.Camera2D:
                    MainCamera.Target.Clear.Value = CameraClearMode.Skybox;
                    SetStereo(false);
                    break;

                case Mode.Camera360:
                    MainCamera.Target.Clear.Value = CameraClearMode.Skybox;
                    SetStereo(false);
                    break;

                case Mode.CameraObject:
                    MainCamera.Target.Clear.Value = CameraClearMode.Color;
                    MainCamera.Target.ClearColor.Value = color.Clear;
                    SetStereo(false);
                    break;

                case Mode.CameraStereo:
                    MainCamera.Target.Clear.Value = CameraClearMode.Skybox;
                    SetStereo(true);
                    break;
            }
        }

        void SetStereo(bool stereo)
        {
            if(stereo)
            {
                // copy camera properties
                var _m = MainCamera.Target;
                var _s = SecondaryCamera.Target;

                _s.FieldOfView.Value = _m.FieldOfView;
                _s.NearClipping.Value = _m.NearClipping;
                _s.FarClipping.Value = _m.FarClipping;
                _s.Clear.Value = _m.Clear;
                _s.ClearColor.Value = _m.ClearColor;
                _s.Projection.Value = _m.Projection;
                _s.OrthographicSize.Value = _m.OrthographicSize;
                _s.RenderTexture.Target = _m.RenderTexture.Target;
                _s.Depth.Value = _m.Depth;
                _s.DoubleBuffered.Value = _m.DoubleBuffered;

                // configure cameras
                SecondaryCamera.Target.Enabled = true;
                MainCamera.Target.Viewport.Value = new Rect(0f, 0f, 0.5f, 1f);
                SecondaryCamera.Target.Viewport.Value = new Rect(0.5f, 0f, 0.5f, 1f);

                // configure the projection material
                if (DisplayMaterial.Target != null)
                    SetupStereoMaterial(DisplayMaterial.Target);

                SetSeparation(MainCamera.Target.Slot.GlobalScaleToLocal(StereoSeparation));
            }
            else
            {
                // configure cameras
                SecondaryCamera.Target.Enabled = false;
                MainCamera.Target.Viewport.Value = new Rect(0f, 0f, 1f, 1f);

                // configure the projection material
                if (DisplayMaterial.Target != null)
                    SetupMonoMaterial(DisplayMaterial.Target);

                SetSeparation(0f);
            }
        }

        int2 GetFinalPreviewResolution()
        {
            if (CameraMode.Value == Mode.CameraStereo)
                return new int2(PreviewWidth * 2, PreviewHeight);

            return new int2(PreviewWidth, PreviewHeight);
        }

        int2 GetSingleRenderResolution()
        {
            var height = MathX.RoundToInt((PreviewHeight / (float)PreviewWidth) * RenderWidth);
            return new int2(RenderWidth, height);
        }

        int2 GetFinalRenderResolution()
        {
            var single = GetSingleRenderResolution();

            if (CameraMode.Value == Mode.CameraStereo)
                return new int2(single.x * 2, single.y);

            return single;
        }

        void SetSeparation(float separation)
        {
            var half = separation * 0.5f;
            _leftCamOffset.Target.Value = new float3(-half, 0f, 0f);
            _rightCamOffset.Target.Value = new float3(half, 0f, 0f);
        }

        string GetFormatExt()
        {
            switch (Format.Value)
            {
                case EncodeFormat.WebP:
                    return "webp";
                case EncodeFormat.PNG:
                    return "png";
                case EncodeFormat.JPG:
                    return "jpg";

                default:
                    throw new Exception("invalid Format: " + Format.Value);
            }
        }

        int GetQualitySetting() => MathX.Clamp(MathX.RoundToInt(Quality), 0, 101);

        public void Trigger()
        {
            if(TimerEnabled.Value)
            {
                _timerUser.Target = World.LocalUser;
                _timerStartPoint.SetNow();
            }
            else
                Capture();
        }

        public void Capture()
        {
            bool captured = false;

            switch (CameraMode.Value)
            {
                case Mode.Camera2D:
                    MainCamera.Target.RenderToAsset(GetFinalRenderResolution(), GetFormatExt(), GetQualitySetting()).OnResultDone +=
                        texture => RunSynchronously(() => SpawnPhoto(texture, Mode.Camera2D));

                    captured = true;
                    break;

                case Mode.Camera360:
                    var settings = MainCamera.Target.GetRenderSettings(GetFinalRenderResolution());
                    settings.fov = 360f;
                    settings.position = (PanoramaIndicator.Target ?? Slot).GlobalPosition;
                    settings.rotation = floatQ.Identity;
                    settings.excludeObjects = new List<Slot>() { Slot };

                    World.Render.RenderToAsset(settings, GetFormatExt(), GetQualitySetting()).OnResultDone +=
                        texture => RunSynchronously(() => SpawnPhoto(texture, Mode.Camera360));

                    captured = true;
                    break;

                case Mode.CameraObject:
                    var renderObject = RaycastRenderObject();

                    if(renderObject != null)
                    {
                        var resolution = GetFinalRenderResolution();
                        var renderSettings = MainCamera.Target.GetRenderSettings(resolution);

                        if(ObjectAutoPose.Value)
                        {
                            renderObject.GetRenderPoint(out renderSettings.position, out renderSettings.rotation, resolution,
                                MainCamera.Target.Slot.GlobalPosition, renderSettings.fov);
                        }

                        renderObject.RenderToAsset(renderSettings, GetFormatExt(), GetQualitySetting()).OnResultDone += 
                                texture => RunSynchronously(() => SpawnPhoto(texture, Mode.CameraObject));

                        captured = true;
                    }
                    break;

                case Mode.CameraStereo:
                    StartCoroutine(RenderStereo(GetFormatExt(), GetQualitySetting()));
                    captured = true;
                    break;
            }

            if(captured)
            {
                Slot.TryVibrateLong();

                if (CaptureSound.IsAssetAvailable)
                    Slot.PlayOneShot(CaptureSound);
            }
        }

        IEnumerator<Context> RenderStereo(string formatExt, int quality)
        {
            var resolution = GetSingleRenderResolution();

            var leftTask = MainCamera.Target.RenderToBitmap(resolution);
            var rightTask = SecondaryCamera.Target.RenderToBitmap(resolution);

            yield return Context.ToBackground();

            yield return Context.WaitFor(leftTask);
            yield return Context.WaitFor(rightTask);

            var leftTex = leftTask.Result;
            var rightTex = rightTask.Result;

            var final = new Bitmap2D(resolution.x * 2, resolution.y, leftTex.Format, false);
            final.CopyFrom(leftTex, 0, 0, 0, 0, resolution.x, resolution.y);
            final.CopyFrom(rightTex, 0, 0, resolution.x, 0, resolution.x, resolution.y);

            var uri = Engine.LocalDB.SaveAsset(final, formatExt, quality);

            yield return Context.ToWorld();

            SpawnPhoto(uri, Mode.CameraStereo);
        }

        void SpawnPhoto(Uri texture, Mode mode)
        {
            var s = World.AddSlot("Image");

            var tex = s.AttachTexture(texture);

            var exporter = s.AttachComponent<TextureExportable>();
            exporter.Texture.Target = tex;

            var grabbable = s.AttachComponent<Grabbable>();
            grabbable.Scalable = true;

            if(mode == Mode.Camera360)
            {
                var sphere = s.AttachMesh<IcoSphereMesh, Projection360Material>();

                sphere.material.Texture.Target = tex;

                sphere.mesh.Subdivisions.Value = 2;

                var collider = s.AttachComponent<MeshCollider>();
                collider.Mesh.Target = sphere.mesh;

                s.GlobalPosition = Slot.GlobalPosition;
            }
            else
            {
                var quad = s.AttachMesh<QuadMesh, UnlitMaterial>();

                quad.material.Texture.Target = tex;
                quad.material.Sidedness.Value = Sidedness.Double;

                var driver = s.AttachComponent<TextureSizeDriver>();
                driver.Texture.Target = tex;
                driver.DriveMode.Value = TextureSizeDriver.Mode.Normalized;
                driver.Target.Target = quad.mesh.Size;

                var spawnPoint = PhotoSpawnPoint.Target ?? Slot;

                s.GlobalPosition = spawnPoint.GlobalPosition;
                s.GlobalRotation = spawnPoint.GlobalRotation;
                s.LocalScale = PhotoSpawnSize.Value * float3.One;

                s.Parent = Slot;

                var collider = s.AttachComponent<MeshCollider>();
                collider.Mesh.Target = quad.mesh;

                if (mode == Mode.CameraStereo)
                {
                    SetupStereoMaterial(quad.material);
                    driver.Premultiply.Value = new float2(0.5f, 1f);
                }

                if (mode == Mode.CameraObject)
                    quad.material.BlendMode.Value = BlendMode.Alpha;
            }

            ScreenshotType screenshotType;

            switch(mode)
            {
                case Mode.Camera360:
                    screenshotType = ScreenshotType.Mono360;
                    break;

                case Mode.CameraStereo:
                    screenshotType = ScreenshotType.Stereo;
                    break;

                default:
                    screenshotType = ScreenshotType.Mono;
                    break;
            }

            var file = Engine.LocalDB.TryFetchAssetRecord(texture).path;
            Engine.PlatformInterface.NotifyOfScreenshot(World, file, screenshotType);
        }

        void SetupMonoMaterial(IStereoMaterial material)
        {
            material.StereoTextureTransform = false;
            material.LeftEyeTextureOffset = float2.Zero;
            material.LeftEyeTextureScale = float2.One;
        }

        void SetupStereoMaterial(IStereoMaterial material)
        {
            material.StereoTextureTransform = true;

            material.LeftEyeTextureOffset = float2.Zero;
            material.LeftEyeTextureScale = new float2(0.5f, 1f);

            material.RightEyeTextureOffset = new float2(0.5f, 0f);
            material.RightEyeTextureScale = new float2(0.5f, 1f);
        }        
    }
}