Last active
March 7, 2017 01:21
-
-
Save sean-m/d293987aa45d3e64bdfae57565999c9f to your computer and use it in GitHub Desktop.
Will take a source and destination directory then spawn a 7za.exe job for each directory. These are enqueued and executed in PowerShell jobs with a throttle to ensure only a limited number of compression jobs happen at any given time.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
$queue = New-Object System.Collections.Queue | |
$wait_state = 2 | |
$job_count = 0 | |
$completed_request = @{} | |
$job_dispached = @{} | |
$throttle = 3 | |
$source_dir = "D:\" | |
$dest_dir = "E:\" | |
function Make-UniquePath { | |
param([string]$path) | |
$ptemp = $path | |
$inc = 1; | |
$ext = [IO.Path]::GetExtension($path) | |
while (Test-Path $ptemp) { | |
$tmp_ext = [String]::Concat("-",$inc.ToString("D2"),$ext) | |
if ([String]::IsNullOrEmpty($ext)) { | |
$ptemp = [String]::Concat($path,$tmp_ext) | |
} else { | |
$ptemp = $path.Replace($ext,$tmp_ext) | |
} | |
$inc++ | |
} | |
return $ptemp | |
} | |
$xdir_file = "xdir_", $([Guid]::NewGuid()), ".txt" -join "" | |
$xdir_file = Join-Path $env:temp $xdir_file | |
gci $source_dir -Directory -Hidden | % { "`"$($_.FullName)`"" } | out-file $xdir_file -Encoding ascii | |
gci $source_dir -Directory | % { "`"$($_.FullName)`"" } | out-file $xdir_file -Encoding ascii -Append | |
$task_list = @() | |
$task_list += { | |
$dest_file = Make-UniquePath "$dest_dir\_root.7z" | |
New-Object PSObject -Property @{ | |
"Name"="_root"; | |
"Command"="& 7za a `"$dest_file`" `"$source_dir`" `"@$xdir_file`""; | |
"Path"=$source_dir; | |
} | |
}.Invoke() | |
$task_list += gci $source_dir -Directory ` | |
| % { | |
$dest_file = Make-UniquePath "$dest_dir\$($_.Name).7z" | |
New-Object PSObject -Property @{ | |
"Name"="$($_.Name).7z"; | |
"Command"="& 7za a `"$dest_file`" `"$source_dir\$($_.Name)`""; | |
"Path"="$($_.FullName)"; | |
} } | |
$task_list | % { $queue.Enqueue($_) } | |
while (-not (($queue.Count -eq 0) -and ($job_count -eq 0))) { | |
## Collect complete jobs | |
if ($job_count -gt 0) { | |
Get-Job | ? { $_.Name.StartsWith("JOB-") } | ? { $_.State -eq "Completed" } | % { | |
Write-Host "Job completed: $($_.Name)" -ForegroundColor Green | |
$_ | Receive-Job | |
$_ | Remove-Job | |
--$job_count | |
$completed_request.Add($_.Name, "Done") | |
} | |
} | |
## Spawn jobs | |
while (($queue.Count -gt 0) -and ($job_count -lt $throttle)) { | |
$task = $queue.Dequeue() | |
$key = $task.Name | |
$cmd = $task.Command | |
if ($job_dispached.ContainsKey($key)) { | |
## Job already dispatched, avoid race where job is enqueued again before first job completed | |
Write-Warning "Already dispatched: JOB-$target" | |
continue | |
} | |
Write-Host "Job starting : JOB-$($key)" -ForegroundColor Cyan | |
Start-Job -Name "JOB-$key" -ScriptBlock {param($cmd) $sb = [ScriptBlock]::Create($cmd); $sb.Invoke() } -ArgumentList @($cmd) | Out-Null | |
$job_dispached.Add($key,"") | |
++$job_count | |
} | |
Write-Debug "No more requests this round, waiting $wait_state seconds before checking again" | |
Start-Sleep -Seconds $wait_state | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment