Created
December 24, 2023 02:37
-
-
Save leandromoh/ed4a2d30ea43da39c3df56c6fa0d7d74 to your computer and use it in GitHub Desktop.
serialize objects to json then get md5 hashes without too much string allocations
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System.Buffers; | |
using System.Reflection; | |
using System.Text.Json; | |
using System.Text.Json.Serialization.Metadata; | |
public interface IHashGenerator | |
{ | |
void GenerateHashes<T>(IEnumerable<T> source, Action<T, string> action); | |
void GenerateHashes<T, TKey>(IEnumerable<T> source, Func<T, TKey> selector, Action<T, string> action); | |
} | |
public class HashGenerator : IHashGenerator | |
{ | |
private int _size; | |
public HashGenerator() : this((int)Math.Pow(2, 13)) { } | |
public HashGenerator(int bufferSize) | |
{ | |
if (bufferSize < 1) | |
throw new ArgumentOutOfRangeException(nameof(bufferSize), $"{nameof(bufferSize)} must be greater than zero."); | |
_size = bufferSize; | |
} | |
public void GenerateHashes<T>(IEnumerable<T> source, Action<T, string> action) | |
{ | |
GenerateHashes(source, x => x, action); | |
} | |
public void GenerateHashes<T, TKey>(IEnumerable<T> source, Func<T, TKey> selector, Action<T, string> action) | |
{ | |
var pool = (byte[])null; | |
var mem = (MemoryStream)null; | |
var writer = (Utf8JsonWriter)null; | |
Init(); | |
var options = new JsonSerializerOptions | |
{ | |
TypeInfoResolver = new DefaultJsonTypeInfoResolver | |
{ | |
Modifiers = { JsonExtensions.AlphabetizeProperties() }, | |
}, | |
}; | |
try | |
{ | |
foreach (var item in source) | |
{ | |
var attempts = 0; | |
reloop: | |
//if (attempts++ > 15) | |
// throw new Exception($"Double buffer size {15}x was not enough. Abort with size of {_size}."); | |
try | |
{ | |
var key = selector(item); | |
System.Text.Json.JsonSerializer.Serialize(writer, key, inputType: key.GetType(), options); | |
var md5 = pool.ToMd5(0, (int)mem.Position); | |
action(item, md5); | |
writer.Reset(); | |
mem.Position = 0; | |
} | |
catch (NotSupportedException ex) | |
when (ex.Message == "Memory stream is not expandable.") | |
{ | |
Reset(); | |
goto reloop; | |
} | |
catch (TargetInvocationException ex) | |
when (ex.InnerException is System.Text.Json.JsonException js && js.Message == "The object or value could not be serialized.Path: $.") | |
{ | |
Reset(); | |
goto reloop; | |
} | |
} | |
} | |
finally | |
{ | |
Dispose(); | |
} | |
void Init() | |
{ | |
pool = ArrayPool<byte>.Shared.Rent(_size); | |
mem = new MemoryStream(pool, writable: true); | |
writer = new Utf8JsonWriter(mem); | |
} | |
void Reset() | |
{ | |
Dispose(); | |
_size *= 2; | |
Init(); | |
} | |
void Dispose() | |
{ | |
ArrayPool<byte>.Shared.Return(pool); | |
mem.Dispose(); | |
writer.Dispose(); | |
} | |
} | |
} | |
public static class JsonExtensions | |
{ | |
public static Action<JsonTypeInfo> AlphabetizeProperties(Type type) | |
{ | |
return typeInfo => | |
{ | |
if (typeInfo.Kind != JsonTypeInfoKind.Object || !type.IsAssignableFrom(typeInfo.Type)) | |
return; | |
AlphabetizeProperties()(typeInfo); | |
}; | |
} | |
// https://stackoverflow.com/a/72593993 | |
public static Action<JsonTypeInfo> AlphabetizeProperties() | |
{ | |
return static typeInfo => | |
{ | |
if (typeInfo.Kind != JsonTypeInfoKind.Object) | |
return; | |
var properties = typeInfo.Properties.OrderBy(p => p.Name, StringComparer.Ordinal).ToList(); | |
typeInfo.Properties.Clear(); | |
for (int i = 0; i < properties.Count; i++) | |
{ | |
properties[i].Order = i; | |
typeInfo.Properties.Add(properties[i]); | |
} | |
}; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment