Skip to content

Instantly share code, notes, and snippets.

@sean-m
Last active March 7, 2017 01:21
Show Gist options
  • Save sean-m/d293987aa45d3e64bdfae57565999c9f to your computer and use it in GitHub Desktop.
Save sean-m/d293987aa45d3e64bdfae57565999c9f to your computer and use it in GitHub Desktop.
Will take a source and destination directory then spawn a 7za.exe job for each directory. These are enqueued and executed in PowerShell jobs with a throttle to ensure only a limited number of compression jobs happen at any given time.
$queue = New-Object System.Collections.Queue
$wait_state = 2
$job_count = 0
$completed_request = @{}
$job_dispached = @{}
$throttle = 3
$source_dir = "D:\"
$dest_dir = "E:\"
function Make-UniquePath {
param([string]$path)
$ptemp = $path
$inc = 1;
$ext = [IO.Path]::GetExtension($path)
while (Test-Path $ptemp) {
$tmp_ext = [String]::Concat("-",$inc.ToString("D2"),$ext)
if ([String]::IsNullOrEmpty($ext)) {
$ptemp = [String]::Concat($path,$tmp_ext)
} else {
$ptemp = $path.Replace($ext,$tmp_ext)
}
$inc++
}
return $ptemp
}
$xdir_file = "xdir_", $([Guid]::NewGuid()), ".txt" -join ""
$xdir_file = Join-Path $env:temp $xdir_file
gci $source_dir -Directory -Hidden | % { "`"$($_.FullName)`"" } | out-file $xdir_file -Encoding ascii
gci $source_dir -Directory | % { "`"$($_.FullName)`"" } | out-file $xdir_file -Encoding ascii -Append
$task_list = @()
$task_list += {
$dest_file = Make-UniquePath "$dest_dir\_root.7z"
New-Object PSObject -Property @{
"Name"="_root";
"Command"="& 7za a `"$dest_file`" `"$source_dir`" `"@$xdir_file`"";
"Path"=$source_dir;
}
}.Invoke()
$task_list += gci $source_dir -Directory `
| % {
$dest_file = Make-UniquePath "$dest_dir\$($_.Name).7z"
New-Object PSObject -Property @{
"Name"="$($_.Name).7z";
"Command"="& 7za a `"$dest_file`" `"$source_dir\$($_.Name)`"";
"Path"="$($_.FullName)";
} }
$task_list | % { $queue.Enqueue($_) }
while (-not (($queue.Count -eq 0) -and ($job_count -eq 0))) {
## Collect complete jobs
if ($job_count -gt 0) {
Get-Job | ? { $_.Name.StartsWith("JOB-") } | ? { $_.State -eq "Completed" } | % {
Write-Host "Job completed: $($_.Name)" -ForegroundColor Green
$_ | Receive-Job
$_ | Remove-Job
--$job_count
$completed_request.Add($_.Name, "Done")
}
}
## Spawn jobs
while (($queue.Count -gt 0) -and ($job_count -lt $throttle)) {
$task = $queue.Dequeue()
$key = $task.Name
$cmd = $task.Command
if ($job_dispached.ContainsKey($key)) {
## Job already dispatched, avoid race where job is enqueued again before first job completed
Write-Warning "Already dispatched: JOB-$target"
continue
}
Write-Host "Job starting : JOB-$($key)" -ForegroundColor Cyan
Start-Job -Name "JOB-$key" -ScriptBlock {param($cmd) $sb = [ScriptBlock]::Create($cmd); $sb.Invoke() } -ArgumentList @($cmd) | Out-Null
$job_dispached.Add($key,"")
++$job_count
}
Write-Debug "No more requests this round, waiting $wait_state seconds before checking again"
Start-Sleep -Seconds $wait_state
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment