Created
March 24, 2023 15:26
-
-
Save MarkPflug/7470c364eb94560b2ac347d2ee94274e to your computer and use it in GitHub Desktop.
A low-allocation CSV struct writer
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System.Collections; | |
using System.Collections.ObjectModel; | |
using System.Data.Common; | |
using System.Diagnostics; | |
var rand = new Random(); | |
var data = | |
Enumerable | |
.Range(0, 10000000) | |
.Select( | |
i => | |
new MyData | |
{ | |
Id = Guid.NewGuid(), | |
TimeStamp = DateTime.Now.AddDays(rand.NextDouble() * -10.0), | |
LocX = rand.NextDouble() * 1000.0, | |
LocY = rand.NextDouble() * 1000.0, | |
} | |
).ToArray(); | |
var start = GC.GetTotalAllocatedBytes(true); | |
var sw = Stopwatch.StartNew(); | |
using var csvWriter = Sylvan.Data.Csv.CsvDataWriter.Create("data.csv"); | |
csvWriter.Write(new MyDataReader(data)); | |
sw.Stop(); | |
var end = GC.GetTotalAllocatedBytes(true); | |
Console.WriteLine($"Time={sw.Elapsed} Allocated={end - start} OutputSize={new FileInfo("Data.csv").Length}"); | |
struct MyData | |
{ | |
public Guid Id; | |
public DateTime TimeStamp; | |
public double LocX; | |
public double LocY; | |
} | |
sealed class MyDataReader : DbDataReader, IDbColumnSchemaGenerator | |
{ | |
readonly MyData[] data; | |
readonly ReadOnlyCollection<DbColumn> schema; | |
int idx = -1; | |
public MyDataReader(MyData[] data) | |
{ | |
this.data = data; | |
var cols = new DbColumn[] | |
{ | |
new Col("Id", typeof(Guid)), | |
new Col("Timestamp", typeof(DateTime)), | |
new Col("LocX", typeof(double)), | |
new Col("LocY", typeof(double)), | |
}; | |
this.schema = new ReadOnlyCollection<DbColumn>(cols); | |
} | |
class Col : DbColumn | |
{ | |
public Col(string name, Type type) | |
{ | |
this.ColumnName = name; | |
this.DataType = type; | |
this.AllowDBNull = false; | |
} | |
} | |
public override object this[int ordinal] => GetValue(ordinal); | |
public override object this[string name] => GetValue(GetOrdinal(name)); | |
public override int Depth => 0; | |
public override int FieldCount => 4; | |
public override bool HasRows => true; | |
public override bool IsClosed => false; | |
public override int RecordsAffected => 0; | |
public override bool GetBoolean(int ordinal) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override byte GetByte(int ordinal) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override long GetBytes(int ordinal, long dataOffset, byte[]? buffer, int bufferOffset, int length) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override char GetChar(int ordinal) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override long GetChars(int ordinal, long dataOffset, char[]? buffer, int bufferOffset, int length) | |
{ | |
throw new NotImplementedException(); | |
} | |
public ReadOnlyCollection<DbColumn> GetColumnSchema() | |
{ | |
return schema; | |
} | |
public override string GetDataTypeName(int ordinal) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override DateTime GetDateTime(int ordinal) | |
{ | |
if (ordinal == 1) | |
return data[idx].TimeStamp; | |
throw new ArgumentOutOfRangeException(); | |
} | |
public override decimal GetDecimal(int ordinal) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override double GetDouble(int ordinal) | |
{ | |
switch (ordinal) | |
{ | |
case 2: | |
return data[idx].LocX; | |
case 3: | |
return data[idx].LocY; | |
} | |
throw new ArgumentOutOfRangeException(); | |
} | |
public override IEnumerator GetEnumerator() | |
{ | |
throw new NotImplementedException(); | |
} | |
public override Type GetFieldType(int ordinal) | |
{ | |
return ordinal switch | |
{ | |
0 => typeof(Guid), | |
1 => typeof(DateTime), | |
2 => typeof(double), | |
3 => typeof(double), | |
_ => throw new ArgumentOutOfRangeException() | |
}; | |
} | |
public override float GetFloat(int ordinal) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override Guid GetGuid(int ordinal) | |
{ | |
if (ordinal == 0) | |
{ | |
return data[idx].Id; | |
} | |
throw new ArgumentOutOfRangeException(); | |
} | |
public override short GetInt16(int ordinal) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override int GetInt32(int ordinal) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override long GetInt64(int ordinal) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override string GetName(int ordinal) | |
{ | |
return schema[ordinal].ColumnName; | |
} | |
public override int GetOrdinal(string name) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override string GetString(int ordinal) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override object GetValue(int ordinal) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override int GetValues(object[] values) | |
{ | |
throw new NotImplementedException(); | |
} | |
public override bool IsDBNull(int ordinal) | |
{ | |
// You might have to worry about this if any of your members are nullable refs. | |
return false; | |
} | |
public override bool NextResult() | |
{ | |
return false; | |
} | |
public override bool Read() | |
{ | |
idx++; | |
return idx < data.Length; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment