|
|
|
## Protobuf Serialization & Deserialization |
|
|
|
# This is the process to serialize and deserialize data into/from protobuf |
|
# Protobuf is supposed to be much smaller than JSON or XML and that's its main advantage |
|
|
|
# Date: 09-Jun-2026 |
|
|
|
# you will need to following thing: |
|
# - nuget tool (ex. scoop install nuget) |
|
|
|
# the following was done in PowerShell 7.6.2 on Windows 11 |
|
# which means we are running on a .NET v10.0 runtime (that is PS v7.6) |
|
|
|
# This example uses the "service.proto" sample file as the protocol buffer message definition |
|
|
|
|
|
## ** Initial Setup ** |
|
|
|
# go to a sample folder (create one if need be) |
|
cd (md C:\temp\ProtoService) |
|
|
|
# download the protoc tool (that's the proto compiler) |
|
# get it from: https://github.com/protocolbuffers/protobuf/releases |
|
. C:\temp\Get-ProtoC.ps1 |
|
Get-ProtoC -Path C:\temp\ProtoService |
|
|
|
# generate the .cs file from that .proto file |
|
$DST_DIR = 'C:\temp\ProtoService' |
|
.\protoc --proto_path=. --csharp_out="$DST_DIR" .\service.proto |
|
# Note: this will generate the C# file c:\temp\proto\MyCustomProto\Service.cs |
|
|
|
# download the Google.Protobuf library |
|
cd (md ProtobufLibs) |
|
nuget install Google.Protobuf |
|
cd .. |
|
|
|
# load the google library |
|
$gDll = (dir 'ProtobufLibs\Google*\lib\net5.0\*.dll').FullName |
|
Add-Type -Path $gDll |
|
if (-not 'Google.Protobuf.IMessage' -as [type]) {Write-Warning 'Cannot find the Protobuf classes!!'} |
|
|
|
# now load our library |
|
$ref = [Google.Protobuf.MessageParser].Assembly.FullName |
|
Add-Type -Path .\Service.cs -ReferencedAssemblies $ref -IgnoreWarnings -WarningAction Ignore |
|
|
|
|
|
## ** Serialization ** |
|
|
|
# we'll do an example with the following windows services: WinRM,Spooler,RPCSS,DcomLaunch |
|
|
|
# let's use our library |
|
$ProtoSvcList = [ProtoServiceProcess.ServiceList]::new() |
|
Get-Service WinRM,Spooler,RPCSS,DcomLaunch | foreach { |
|
$obj = [ProtoServiceProcess.Service]@{ |
|
BinaryPathName = $_.BinaryPathName |
|
DelayedAutoStart = $_.DelayedAutoStart |
|
Description = $_.Description |
|
DisplayName = $_.DisplayName |
|
ServiceName = $_.ServiceName |
|
ServiceType = $_.ServiceType.ToString() -as [ProtoServiceProcess.ServiceTypeEnum] |
|
StartType = $_.StartType.ToString() -as [ProtoServiceProcess.StartTypeEnum] |
|
Status = $_.Status.ToString() -as [ProtoServiceProcess.StatusEnum] |
|
UserName = $_.UserName |
|
} |
|
if ($_.RequiredServices) {$obj.RequiredServices.AddRange([string[]]($_.RequiredServices.Name))} |
|
if ($_.ServicesDependedOn) {$obj.ServicesDependedOn.AddRange([string[]]($_.ServicesDependedOn.Name))} |
|
# Note: you cant set a value to the repeated properties (RequiredServices,ServicesDependedOn) |
|
# you can only mutate them in place, by using their methods, for ex. .Add() or .AddRange(), etc.. |
|
|
|
$ProtoSvcList.Services.Add($obj) |
|
} |
|
|
|
# convert the data into a byte array |
|
$MemStream = [System.IO.MemoryStream]::new() |
|
$ProtoStream = [Google.Protobuf.CodedOutputStream]::new($MemStream) |
|
$ProtoSvcList.WriteTo($ProtoStream) |
|
$ProtoStream.Flush() |
|
$SerializedBytes = $MemStream.ToArray() |
|
|
|
# save the byte array into a file |
|
$Path = Join-Path $PWD service.dat |
|
$File = [System.IO.File]::Create($Path) |
|
$File.Write($SerializedBytes,0,$SerializedBytes.Length) |
|
$File.Close() |
|
|
|
# clean up (order matters here) |
|
$ProtoStream.Dispose() |
|
$MemStream.Dispose() |
|
$File.Dispose() |
|
|
|
# you can now send that data anywhere you want |
|
|
|
|
|
<# by now you have the following files in the c:\temp\proto folder |
|
- Service.cs |
|
- service.dat |
|
- service.proto |
|
- protoc.exe |
|
And the \ProtobufLibs sub directory |
|
#> |
|
|
|
## ** Deserialization ** |
|
|
|
# To properly check the deserialization, we can open up a new powershell console |
|
# to simulate a different environment |
|
|
|
# load the google library |
|
cd C:\temp\ProtoService |
|
$gDll = (dir 'ProtobufLibs\Google*\lib\net5.0\*.dll').FullName |
|
Add-Type -Path $gDll |
|
|
|
# load up our class |
|
$ref = [Google.Protobuf.MessageParser].Assembly.FullName |
|
Add-Type -Path .\Service.cs -ReferencedAssemblies $ref -IgnoreWarnings -WarningAction Ignore |
|
|
|
# Get the parser for the service message |
|
$Parser = [ProtoServiceProcess.ServiceList]::Parser |
|
|
|
# get the byte array from our data |
|
$Path = 'C:\temp\ProtoService\service.dat' |
|
$ByteData = [System.IO.File]::ReadAllBytes($Path) # <-- this needs a full path, not relative path |
|
|
|
# Deserialize the byte array back into a service object |
|
$DeserializedServices = $Parser.ParseFrom($ByteData) |
|
|
|
|
|
|
|
|
|
### === Size Comparison |
|
|
|
$list = Get-Service WinRM,Spooler,RPCSS,DcomLaunch | foreach { |
|
@{ |
|
BinaryPathName = $_.BinaryPathName |
|
DelayedAutoStart = $_.DelayedAutoStart |
|
Description = $_.Description |
|
DisplayName = $_.DisplayName |
|
ServiceName = $_.ServiceName |
|
ServiceType = $_.ServiceType |
|
StartType = $_.StartType |
|
Status = $_.Status |
|
UserName = $_.UserName |
|
RequiredServices = $_.RequiredServices.Name |
|
ServicesDependedOn = $_.ServicesDependedOn.Name |
|
} |
|
} |
|
($list | ConvertTo-Json -Compress).Length |
|
# returns: 3049 bytes |
|
|
|
|
|
# The size difference is not that big, because if we compress the json string (from $list) with gzip (native .net library) |
|
# then the 2 sizes (the Protobuf and the Zipped Json) won't be that different |
|
|
|
|
|
# compression details: |
|
|
|
$json = $list | ConvertTo-Json -Compress |
|
|
|
# compress the json string |
|
$ms = [System.IO.MemoryStream]::new() |
|
$mode = [System.IO.Compression.CompressionMode]::Compress |
|
$gz = [System.IO.Compression.GZipStream]::new($ms, $mode) |
|
$sw = [System.IO.StreamWriter]::new($gz) |
|
$sw.Write($json) |
|
$sw.Close() |
|
$bytes = $ms.ToArray() |
|
$OutStr = [System.Convert]::ToBase64String($bytes) |
|
$ms.Close() # MemoryStream |
|
$gz.Close() # GZipStream |
|
|
|
|
|
$OutStr.Length # <-- returns 1472 bytes, which is smaller than the Protocol Buffer size |
|
$OutStr | Out-File Base64CompressedJson.txt -Encoding UTF8 |
|
|
|
|
|
# decompression details (to get back the json string) |
|
$InputString = Get-Content Base64CompressedJson.txt -Raw -Encoding UTF8 |
|
|
|
$data = [System.Convert]::FromBase64String($InputString) |
|
$ms = [System.IO.MemoryStream]::new() |
|
$ms.Write($data, 0, $data.Length) |
|
$ms.Seek(0,0) | Out-Null |
|
$mode = [System.IO.Compression.CompressionMode]::Decompress |
|
$gz = [System.IO.Compression.GZipStream]::new($ms, $mode) |
|
$sr = [System.IO.StreamReader]::new($gz) |
|
$OutStr = $sr.ReadToEnd() |
|
$sr.Close() # StreamReader |
|
$ms.Close() # MemoryStream |
|
$gz.Close() # GZipStream |
|
|
|
$OutStr | ConvertFrom-Json # <-- you get back the custom service objects |
|
|
|
## ** Verdict ** |
|
|
|
<# |
|
- You cannot use Protobuf with adhoc objects, because you have to write up the .proto file in advance in order to |
|
get a.cs and then load up the classes from the .cs file |
|
|
|
Which means you need to know before hand the objects you are going to emit, and their corresponding properties |
|
which makes the solution quite difficult to use on-the-fly. It's probably only good for pre-defined workflows |
|
like a CICD pipeline where you know what kind of data you send/receive. |
|
|
|
- We cant use the already thousands of .NET Framework objects (ex. Diagnostics.Process, ServiceController, etc) |
|
because we have to supply/write a .proto file for each object we are going to serialize/deserialize |
|
|
|
Again, this makes the solution quite cumbersome. |
|
|
|
- The size benefits do not seem that great, at least when tested on a small scale (4 windows services) |
|
Actually the zipped json string was smaller than the serialized protobuf data (which supposedly is compressed as well) |
|
|
|
- We need an automated way (a script or function perhaps) that can take a .Net object and generate the .proto file for it |
|
with all the relevant protobuf messages and enums. |
|
Because the process to write the .proto file for each class and their properties, and also the enums required, is quite time consuming. |
|
(essentially that would be a transpiler in a way, just like the protoc tool which takes a proto file and converts it to a C sharp file) |
|
|
|
Date: 09-Jun-2026 |
|
#> |