PopCloud.cs

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.IO;

/// <summary>
/// General utilities
/// </summary>
public static class PopCloud
{
	//	gr: make this a more intelligent extension->loader mapping
	public static List<string>		FileExtensions = new List<string>() {".xyz",".off"};





	static void ExtractLineStarts(string Filename,ref byte[] Data,ref List<int> LineStartIndexes)
	{
		Data = System.IO.File.ReadAllBytes (Filename);

		//	find all line feeds
		LineStartIndexes = new List<int>();
		var NewLine = true;
		for (int i = 0;	i < Data.Length;	i++) 
		{
			var c = Data [i];

			//	walk over delins
			if (c == '\n' || c == '\r') {
				NewLine = true;
				continue;
			}

			//	first valid character of a new line
			if (NewLine) {
				LineStartIndexes.Add (i);
				NewLine = false;
			}
		}
	}

	struct XyzRgba
	{
		public float	x;
		public float	y;
		public float	z;

		public float	r;
		public float	g;
		public float	b;
		public float	a;

		//	gr: need to align to 4, so use int, not bool
		public int		Valid;	

		public static int	GetSize()
		{
			return (sizeof(float) * 7) + (sizeof(int) * 1);
		}
	};

	public static void LoadXyzGpu(string Filename,System.Action<Vector3,Color> OnPointDecoded,System.Action<List<string>> OnUnhandledHeader,System.Action<string,float> UpdateProgress)
	{
		UpdateProgress.Invoke ("Parsing file lines...", 0.0f);

		byte[] Data = null;
		List<int> LineStarts = null;
		ExtractLineStarts (Filename, ref Data, ref LineStarts);
		XyzRgba[] PosColours = new XyzRgba[LineStarts.Count];

		//	gr: DX REQUIRES buffer to be multiples of 4.
		//		metal packs these bytes into int's, silently. So lets just pretend they're packed
		int PackSize = 4;
		var DataBuffer = new ComputeBuffer (Data.Length/PackSize, sizeof(byte)*PackSize, ComputeBufferType.Default );

		DataBuffer.SetData (Data);
		var LineStartsBuffer = new ComputeBuffer (LineStarts.Count, sizeof(int));
		LineStartsBuffer.SetData (LineStarts.ToArray());
		var PositionColourBuffer = new ComputeBuffer (PosColours.Length, XyzRgba.GetSize() );
		PositionColourBuffer.SetData (PosColours);

		var KernelName = "ParsePositionColour";
		var ShaderAssetName = "Assets/PopCloud/PopXyzToTexture.compute";

		var Shader = UnityEditor.AssetDatabase.LoadAssetAtPath (ShaderAssetName, typeof(ComputeShader)) as ComputeShader;
		if (!Shader)
			throw new System.Exception ("Couldn't find " + ShaderAssetName);

		var KernelId = Shader.FindKernel (KernelName);
		Shader.SetBuffer (KernelId, "Data", DataBuffer);
		Shader.SetBuffer (KernelId, "LineStarts", LineStartsBuffer);
		Shader.SetBuffer (KernelId, "PositionColours", PositionColourBuffer);

		UpdateProgress.Invoke ("Executing compute parser...", 0.0f);

		Shader.Dispatch(KernelId, LineStarts.Count, 1, 1);

		PositionColourBuffer.GetData (PosColours);

		{
			var v3 = new Vector3 ();
			var c = new Color ();
			int Count = 0;
			foreach (var Point in PosColours) {

				//	GUI update eats cpu time
				if ((Count % 1000) == 0) {
					var Progress = Count / (float)PosColours.Length;
					UpdateProgress.Invoke ("Loading points...", Progress);
				}

				if ( Count == 0 )
					Debug.Log ("#" + Count + " valid=" + Point.Valid + " (" + Point.x + "," + Point.y + "," + Point.z + "," + Point.r + "," + Point.g + "," + Point.b + ")");

				Count++;
				if (Point.Valid == 0)
					continue;
				
				v3.x = Point.x;
				v3.y = Point.y;
				v3.z = Point.z;
				c.r = Point.r;
				c.g = Point.g;
				c.b = Point.b;
				c.a = Point.a;
				OnPointDecoded.Invoke (v3, c);
			}
		}			

		DataBuffer.Release ();
		LineStartsBuffer.Release ();
		PositionColourBuffer.Release ();
	}

	public static void LoadXyz(string Filename,System.Action<Vector3,Color> OnPointDecoded,System.Action<List<string>> OnUnhandledHeader,System.Action<string,float> UpdateProgress)
	{
		var Stream = File.OpenText (Filename);

		//	unaccounted header strings we keep
		var Headers = new List<string>();

		//	once we parse something good, we dont keep "headers"
		bool PassedHeader = false;

		var Pos3 = new Vector3 ();
		var Colour = new Color ();
		int LinesParsed = 0;
		int LinesSkipped = 0;


		System.Action<string> ParseLine = (XyzLine) =>
		{
			//	parse line
			try
			{
				var Floats = FastParse.Floats(XyzLine);
				if ( Floats.Count != 6 && Floats.Count != 7 )
					throw new System.Exception("Expecting 6/7 values");

				PassedHeader = true;

				Pos3.x = Floats[0];
				Pos3.y = Floats[1];
				Pos3.z = Floats[2];

				Colour.r = Floats[3] / 255.0f;
				Colour.g = Floats[4] / 255.0f;
				Colour.b = Floats[5] / 255.0f;
				Colour.a = ( Floats.Count > 6 ) ? Floats[6] / 255.0f : 1;

				OnPointDecoded.Invoke( Pos3, Colour );
				LinesParsed++;
			}
			catch
			{
				//	no floats in line
				if (!PassedHeader)
					Headers.Add (XyzLine);
				LinesSkipped++;
			}
		};


		var JobRunner = new JobPool_Action();

		System.Action<string> ParseLineThreaded = (XyzLine) => {
			System.Action Job = () => {
				ParseLine (XyzLine);
			};
			JobRunner.PushJob (Job);
		};
				

		{
			string Line;
			int LineCount = 0;
			while ((Line = Stream.ReadLine ()) != null) {
				var Progress = Stream.BaseStream.Position / (float)Stream.BaseStream.Length;
				//Progress = JobRunner.jobProgress;
				LineCount++;

				if (LineCount > 800000)
					break;
				
				ParseLineThreaded (Line);

				if ((LinesParsed + LinesSkipped) % 1000 == 0) {
					var ProgressName = "Parsing line " + LinesParsed + " (" + LinesSkipped + " skipped)";
					UpdateProgress.Invoke (ProgressName, Progress);
				}
			}
		}

		do {
			var Pending = JobRunner.jobsPending;
			var Completed = JobRunner.jobsCompleted;
			var Step = "Waiting for jobs " + Completed + "/" + (Pending + Completed);
			UpdateProgress.Invoke (Step, JobRunner.jobProgress);
			JobRunner.WaitForJobs (500);
		} while (JobRunner.jobsPending > 0);


		OnUnhandledHeader.Invoke (Headers);
	}
}