Skip to content

Instantly share code, notes, and snippets.

@jeremybeavon
Last active April 16, 2025 09:57
Show Gist options
  • Select an option

  • Save jeremybeavon/deedb7565b5083c02b75c4524a00a006 to your computer and use it in GitHub Desktop.

Select an option

Save jeremybeavon/deedb7565b5083c02b75c4524a00a006 to your computer and use it in GitHub Desktop.
Powershell parallel task runner using msbuild
function Invoke-ParallelPowershellFiles
{
[CmdletBinding()]
param(
[hashtable]$PowershellFiles,
[int]$NumberOfFilesToRunInParallel,
[string]$LogDirectory = ((Get-Location).Path)
)
if (!$NumberOfFilesToRunInParallel)
{
$NumberOfFilesToRunInParallel = $PowershellFiles.Count
}
$tempDirectory = [System.IO.Path]::GetTempPath()
$directoryPath = Join-Path $tempDirectory ([System.Guid]::NewGuid().ToString("N"))
New-Item -Type Directory $directoryPath | Out-Null
$PowershellFiles.GetEnumerator() | ForEach-Object {
Set-Content -Path (Join-Path $directoryPath $_.Name) -Value "powershell -ExecutionPolicy ByPass -File $($_.Value)"
}
& C:\Windows\Microsoft.NET\Framework\v4.0.30319\msbuild.exe /nologo /noconsolelogger /verbosity:minimal /nodeReuse:false `
/maxcpucount:$NumberOfFilesToRunInParallel /property:TaskDirectory="$directoryPath" /property:LogDirectory="$LogDirectory" `
/distributedlogger:PowershellCentralLogger,PowershellMsBuildLoggers.dll*PowershellForwardingLogger,PowershellMsBuildLoggers.dll `
"$PSScriptRoot\ParallelTaskRunner.targets"
$MsBuildExitCode = $LASTEXITCODE
Remove-Item -Recurse -Force $directoryPath
if ($MsBuildExitCode -ne 0)
{
throw "Parallel powershell scripts failed."
}
}
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="RunParallelTasks" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="4.0">
<Target Name="RunParallelTasks">
<Error Text="Property missing: TaskDirectory" Condition="'$(TaskDirectory)' == ''"/>
<Error Text="Property missing: LogDirectory" Condition="'$(LogDirectory)' == ''"/>
<Error Text="$(TaskDirectory) does not exist" Condition="!Exists('$(TaskDirectory)')" />
<ItemGroup>
<TasksToRun Include="$(TaskDirectory)\*.*" />
<ProjectsToRun Include="$(MSBuildThisFileFullPath)">
<Properties>TaskToRunFile=%(TasksToRun.Identity);LogDirectory=$(LogDirectory)</Properties>
</ProjectsToRun>
</ItemGroup>
<MSBuild Projects="@(ProjectsToRun)" Targets="RunTaskWithLogging" BuildInParallel="true" />
</Target>
<Target Name="RunTaskWithLogging">
<Error Text="Property missing: TaskToRunFile" Condition="'$(TaskToRunFile)' == ''"/>
<Error Text="$(TaskToRunFile) does not exist" Condition="!Exists('$(TaskToRunFile)')" />
<PropertyGroup>
<MSBuildCommand>C:\Windows\Microsoft.NET\Framework\v4.0.30319\msbuild.exe</MSBuildCommand>
<MSBuildCommand>$(MSBuildCommand) /nologo</MSBuildCommand>
<MSBuildCommand>$(MSBuildCommand) /verbosity:minimal</MSBuildCommand>
<MSBuildCommand>$(MSBuildCommand) /target:RunTask</MSBuildCommand>
<MSBuildCommand>$(MSBuildCommand) /property:TaskToRunFile=&quot;$(TaskToRunFile)&quot;</MSBuildCommand>
<MSBuildCommand>$(MSBuildCommand) /fileLogger</MSBuildCommand>
<MSBuildCommand>$(MSBuildCommand) /fileloggerparameters:LogFile=&quot;$(LogDirectory)\$([System.IO.Path]::GetFileName($(TaskToRunFile))).log&quot;;Verbosity=minimal</MSBuildCommand>
<MSBuildCommand>$(MSBuildCommand) &quot;$(MSBuildThisFileFullPath)&quot;</MSBuildCommand>
</PropertyGroup>
<Exec Command="$(MSBuildCommand)" EchoOff="true" />
<OnError ExecuteTargets="OnTaskFailed" />
</Target>
<Target Name="OnTaskFailed">
<PropertyGroup>
<LogFile>$(LogDirectory)\$([System.IO.Path]::GetFileName($(TaskToRunFile)))</LogFile>
</PropertyGroup>
<Move SourceFiles="$(LogFile).log" DestinationFiles="$(LogFile).failed.log" Condition="Exists('$(LogFile).log')" />
</Target>
<Target Name="RunTask">
<Error Text="Property missing: TaskToRunFile" Condition="'$(TaskToRunFile)' == ''"/>
<Error Text="$(TaskToRunFile) does not exist" Condition="!Exists('$(TaskToRunFile)')" />
<ReadLinesFromFile File="$(TaskToRunFile)">
<Output TaskParameter="Lines" ItemName="TaskToRun" />
</ReadLinesFromFile>
<Exec Command="@(TaskToRun)" />
</Target>
</Project>
using Microsoft.Build.Framework;
using System;
public class PowershellCentralLogger : INodeLogger
{
public string Parameters { get; set; }
public LoggerVerbosity Verbosity { get; set; }
public void Initialize(IEventSource eventSource)
{
eventSource.MessageRaised += MessageRaised;
}
private void MessageRaised(object sender, BuildMessageEventArgs e)
{
if (e.Importance != MessageImportance.Low)
Console.WriteLine(e.Message);
}
public void Initialize(IEventSource eventSource, int nodeCount)
{
Initialize(eventSource);
}
public void Shutdown()
{
}
}
using Microsoft.Build.Framework;
using Microsoft.Build.Logging;
using System;
public sealed class PowershellFileLogger : FileLogger
{
public PowershellFileLogger()
{
WriteHandler defaultWriteHandler = WriteHandler;
WriteHandler = message => defaultWriteHandler(DateTime.Now.ToString("HH:mm:ss") + " " + message);
SkipProjectStartedText = true;
Verbosity = LoggerVerbosity.Minimal;
}
}
using Microsoft.Build.Framework;
using System.IO;
public sealed class PowershellForwardingLogger : IForwardingLogger
{
private string nodeLabel;
public IEventRedirector BuildEventRedirector { get; set; }
public int NodeId { get; set; }
public string Parameters { get; set; }
public LoggerVerbosity Verbosity { get; set; }
public void Initialize(IEventSource eventSource)
{
eventSource.ProjectStarted += ProjectStarted;
eventSource.MessageRaised += MessageRaised;
}
public void Initialize(IEventSource eventSource, int nodeCount)
{
Initialize(eventSource);
}
public void Shutdown()
{
}
private void ProjectStarted(object sender, ProjectStartedEventArgs e)
{
if (e.GlobalProperties.ContainsKey("TaskToRunFile"))
{
nodeLabel = Path.GetFileName(e.GlobalProperties["TaskToRunFile"]);
}
}
private void MessageRaised(object sender, BuildMessageEventArgs e)
{
if (nodeLabel != null)
{
e = new BuildMessageEventArgs(
e.Subcategory,
e.Code,
e.File,
e.LineNumber,
e.ColumnNumber,
e.EndLineNumber,
e.EndColumnNumber,
string.Format("[{0}]: {1}", nodeLabel, e.Message),
e.HelpKeyword,
e.SenderName,
e.Importance,
e.Timestamp)
{
BuildEventContext = e.BuildEventContext
};
}
BuildEventRedirector.ForwardEvent(e);
}
}
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{135E44A6-895D-43F7-9A75-846ED18E040E}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>PowershellMsBuildLoggers</RootNamespace>
<AssemblyName>PowershellMsBuildLoggers</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.Build.Framework" />
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="Microsoft.CSharp" />
</ItemGroup>
<ItemGroup>
<Compile Include="PowershellCentralLogger.cs" />
<Compile Include="PowershellFileLogger.cs" />
<Compile Include="PowershellForwardingLogger.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment