Last active
April 16, 2025 09:57
-
-
Save jeremybeavon/deedb7565b5083c02b75c4524a00a006 to your computer and use it in GitHub Desktop.
Powershell parallel task runner using msbuild
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| function Invoke-ParallelPowershellFiles | |
| { | |
| [CmdletBinding()] | |
| param( | |
| [hashtable]$PowershellFiles, | |
| [int]$NumberOfFilesToRunInParallel, | |
| [string]$LogDirectory = ((Get-Location).Path) | |
| ) | |
| if (!$NumberOfFilesToRunInParallel) | |
| { | |
| $NumberOfFilesToRunInParallel = $PowershellFiles.Count | |
| } | |
| $tempDirectory = [System.IO.Path]::GetTempPath() | |
| $directoryPath = Join-Path $tempDirectory ([System.Guid]::NewGuid().ToString("N")) | |
| New-Item -Type Directory $directoryPath | Out-Null | |
| $PowershellFiles.GetEnumerator() | ForEach-Object { | |
| Set-Content -Path (Join-Path $directoryPath $_.Name) -Value "powershell -ExecutionPolicy ByPass -File $($_.Value)" | |
| } | |
| & C:\Windows\Microsoft.NET\Framework\v4.0.30319\msbuild.exe /nologo /noconsolelogger /verbosity:minimal /nodeReuse:false ` | |
| /maxcpucount:$NumberOfFilesToRunInParallel /property:TaskDirectory="$directoryPath" /property:LogDirectory="$LogDirectory" ` | |
| /distributedlogger:PowershellCentralLogger,PowershellMsBuildLoggers.dll*PowershellForwardingLogger,PowershellMsBuildLoggers.dll ` | |
| "$PSScriptRoot\ParallelTaskRunner.targets" | |
| $MsBuildExitCode = $LASTEXITCODE | |
| Remove-Item -Recurse -Force $directoryPath | |
| if ($MsBuildExitCode -ne 0) | |
| { | |
| throw "Parallel powershell scripts failed." | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <?xml version="1.0" encoding="utf-8"?> | |
| <Project DefaultTargets="RunParallelTasks" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="4.0"> | |
| <Target Name="RunParallelTasks"> | |
| <Error Text="Property missing: TaskDirectory" Condition="'$(TaskDirectory)' == ''"/> | |
| <Error Text="Property missing: LogDirectory" Condition="'$(LogDirectory)' == ''"/> | |
| <Error Text="$(TaskDirectory) does not exist" Condition="!Exists('$(TaskDirectory)')" /> | |
| <ItemGroup> | |
| <TasksToRun Include="$(TaskDirectory)\*.*" /> | |
| <ProjectsToRun Include="$(MSBuildThisFileFullPath)"> | |
| <Properties>TaskToRunFile=%(TasksToRun.Identity);LogDirectory=$(LogDirectory)</Properties> | |
| </ProjectsToRun> | |
| </ItemGroup> | |
| <MSBuild Projects="@(ProjectsToRun)" Targets="RunTaskWithLogging" BuildInParallel="true" /> | |
| </Target> | |
| <Target Name="RunTaskWithLogging"> | |
| <Error Text="Property missing: TaskToRunFile" Condition="'$(TaskToRunFile)' == ''"/> | |
| <Error Text="$(TaskToRunFile) does not exist" Condition="!Exists('$(TaskToRunFile)')" /> | |
| <PropertyGroup> | |
| <MSBuildCommand>C:\Windows\Microsoft.NET\Framework\v4.0.30319\msbuild.exe</MSBuildCommand> | |
| <MSBuildCommand>$(MSBuildCommand) /nologo</MSBuildCommand> | |
| <MSBuildCommand>$(MSBuildCommand) /verbosity:minimal</MSBuildCommand> | |
| <MSBuildCommand>$(MSBuildCommand) /target:RunTask</MSBuildCommand> | |
| <MSBuildCommand>$(MSBuildCommand) /property:TaskToRunFile="$(TaskToRunFile)"</MSBuildCommand> | |
| <MSBuildCommand>$(MSBuildCommand) /fileLogger</MSBuildCommand> | |
| <MSBuildCommand>$(MSBuildCommand) /fileloggerparameters:LogFile="$(LogDirectory)\$([System.IO.Path]::GetFileName($(TaskToRunFile))).log";Verbosity=minimal</MSBuildCommand> | |
| <MSBuildCommand>$(MSBuildCommand) "$(MSBuildThisFileFullPath)"</MSBuildCommand> | |
| </PropertyGroup> | |
| <Exec Command="$(MSBuildCommand)" EchoOff="true" /> | |
| <OnError ExecuteTargets="OnTaskFailed" /> | |
| </Target> | |
| <Target Name="OnTaskFailed"> | |
| <PropertyGroup> | |
| <LogFile>$(LogDirectory)\$([System.IO.Path]::GetFileName($(TaskToRunFile)))</LogFile> | |
| </PropertyGroup> | |
| <Move SourceFiles="$(LogFile).log" DestinationFiles="$(LogFile).failed.log" Condition="Exists('$(LogFile).log')" /> | |
| </Target> | |
| <Target Name="RunTask"> | |
| <Error Text="Property missing: TaskToRunFile" Condition="'$(TaskToRunFile)' == ''"/> | |
| <Error Text="$(TaskToRunFile) does not exist" Condition="!Exists('$(TaskToRunFile)')" /> | |
| <ReadLinesFromFile File="$(TaskToRunFile)"> | |
| <Output TaskParameter="Lines" ItemName="TaskToRun" /> | |
| </ReadLinesFromFile> | |
| <Exec Command="@(TaskToRun)" /> | |
| </Target> | |
| </Project> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| using Microsoft.Build.Framework; | |
| using System; | |
| public class PowershellCentralLogger : INodeLogger | |
| { | |
| public string Parameters { get; set; } | |
| public LoggerVerbosity Verbosity { get; set; } | |
| public void Initialize(IEventSource eventSource) | |
| { | |
| eventSource.MessageRaised += MessageRaised; | |
| } | |
| private void MessageRaised(object sender, BuildMessageEventArgs e) | |
| { | |
| if (e.Importance != MessageImportance.Low) | |
| Console.WriteLine(e.Message); | |
| } | |
| public void Initialize(IEventSource eventSource, int nodeCount) | |
| { | |
| Initialize(eventSource); | |
| } | |
| public void Shutdown() | |
| { | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| using Microsoft.Build.Framework; | |
| using Microsoft.Build.Logging; | |
| using System; | |
| public sealed class PowershellFileLogger : FileLogger | |
| { | |
| public PowershellFileLogger() | |
| { | |
| WriteHandler defaultWriteHandler = WriteHandler; | |
| WriteHandler = message => defaultWriteHandler(DateTime.Now.ToString("HH:mm:ss") + " " + message); | |
| SkipProjectStartedText = true; | |
| Verbosity = LoggerVerbosity.Minimal; | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| using Microsoft.Build.Framework; | |
| using System.IO; | |
| public sealed class PowershellForwardingLogger : IForwardingLogger | |
| { | |
| private string nodeLabel; | |
| public IEventRedirector BuildEventRedirector { get; set; } | |
| public int NodeId { get; set; } | |
| public string Parameters { get; set; } | |
| public LoggerVerbosity Verbosity { get; set; } | |
| public void Initialize(IEventSource eventSource) | |
| { | |
| eventSource.ProjectStarted += ProjectStarted; | |
| eventSource.MessageRaised += MessageRaised; | |
| } | |
| public void Initialize(IEventSource eventSource, int nodeCount) | |
| { | |
| Initialize(eventSource); | |
| } | |
| public void Shutdown() | |
| { | |
| } | |
| private void ProjectStarted(object sender, ProjectStartedEventArgs e) | |
| { | |
| if (e.GlobalProperties.ContainsKey("TaskToRunFile")) | |
| { | |
| nodeLabel = Path.GetFileName(e.GlobalProperties["TaskToRunFile"]); | |
| } | |
| } | |
| private void MessageRaised(object sender, BuildMessageEventArgs e) | |
| { | |
| if (nodeLabel != null) | |
| { | |
| e = new BuildMessageEventArgs( | |
| e.Subcategory, | |
| e.Code, | |
| e.File, | |
| e.LineNumber, | |
| e.ColumnNumber, | |
| e.EndLineNumber, | |
| e.EndColumnNumber, | |
| string.Format("[{0}]: {1}", nodeLabel, e.Message), | |
| e.HelpKeyword, | |
| e.SenderName, | |
| e.Importance, | |
| e.Timestamp) | |
| { | |
| BuildEventContext = e.BuildEventContext | |
| }; | |
| } | |
| BuildEventRedirector.ForwardEvent(e); | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <?xml version="1.0" encoding="utf-8"?> | |
| <Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> | |
| <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> | |
| <PropertyGroup> | |
| <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> | |
| <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> | |
| <ProjectGuid>{135E44A6-895D-43F7-9A75-846ED18E040E}</ProjectGuid> | |
| <OutputType>Library</OutputType> | |
| <AppDesignerFolder>Properties</AppDesignerFolder> | |
| <RootNamespace>PowershellMsBuildLoggers</RootNamespace> | |
| <AssemblyName>PowershellMsBuildLoggers</AssemblyName> | |
| <TargetFrameworkVersion>v4.5</TargetFrameworkVersion> | |
| <FileAlignment>512</FileAlignment> | |
| <TargetFrameworkProfile /> | |
| </PropertyGroup> | |
| <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "> | |
| <DebugSymbols>true</DebugSymbols> | |
| <DebugType>full</DebugType> | |
| <Optimize>false</Optimize> | |
| <OutputPath>bin\Debug\</OutputPath> | |
| <DefineConstants>DEBUG;TRACE</DefineConstants> | |
| <ErrorReport>prompt</ErrorReport> | |
| <WarningLevel>4</WarningLevel> | |
| <Prefer32Bit>false</Prefer32Bit> | |
| </PropertyGroup> | |
| <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' "> | |
| <DebugType>pdbonly</DebugType> | |
| <Optimize>true</Optimize> | |
| <OutputPath>bin\Release\</OutputPath> | |
| <DefineConstants>TRACE</DefineConstants> | |
| <ErrorReport>prompt</ErrorReport> | |
| <WarningLevel>4</WarningLevel> | |
| <Prefer32Bit>false</Prefer32Bit> | |
| </PropertyGroup> | |
| <ItemGroup> | |
| <Reference Include="Microsoft.Build.Framework" /> | |
| <Reference Include="System" /> | |
| <Reference Include="System.Core" /> | |
| <Reference Include="Microsoft.CSharp" /> | |
| </ItemGroup> | |
| <ItemGroup> | |
| <Compile Include="PowershellCentralLogger.cs" /> | |
| <Compile Include="PowershellFileLogger.cs" /> | |
| <Compile Include="PowershellForwardingLogger.cs" /> | |
| <Compile Include="Properties\AssemblyInfo.cs" /> | |
| </ItemGroup> | |
| <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> | |
| <!-- To modify your build process, add your task inside one of the targets below and uncomment it. | |
| Other similar extension points exist, see Microsoft.Common.targets. | |
| <Target Name="BeforeBuild"> | |
| </Target> | |
| <Target Name="AfterBuild"> | |
| </Target> | |
| --> | |
| </Project> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment