Last active
August 17, 2023 05:35
-
-
Save zudsniper/31369714cb85afdc147734aa8b80097f to your computer and use it in GitHub Desktop.
⓽ 10ZIPPER.ps1 -> the final iteration of the (7+n)ZIPPER.ps1 saga, this will be used for data duplicate detection and archival onto an external drive for system recovery.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # 10ZIPPER.ps1 | |
| # v2.0.3 | |
| # ----------- | |
| # | |
| # @zudsniper | |
| # @openai | |
| param( | |
| [Parameter(Mandatory=$true)] | |
| [Alias("s")] | |
| [string[]]$source, | |
| [Parameter(Mandatory=$true)] | |
| [Alias("d", "dest")] | |
| [string]$destination, | |
| [switch]$ignoreOS, | |
| [Alias("dbg")] | |
| [switch]$debugMode | |
| ) | |
| # Spinner setup | |
| $spinner = '|','/','-','\' | |
| $spinIndex = 0 | |
| # Check if script is running with administrative privileges | |
| $currentPrincipal = New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent()) | |
| $isAdministrator = $currentPrincipal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) | |
| if (-not $isAdministrator) { | |
| Write-Host "This script must be run as an Administrator. Please re-run the script as an Administrator." -ForegroundColor Red | |
| exit | |
| } | |
| if ($debug) { | |
| Write-Host "Script is running with administrative privileges." -ForegroundColor Green | |
| } | |
| # Check if 7zip is installed | |
| $7zipPath = "C:\Program Files\7-Zip\7z.exe" | |
| if (!(Test-Path $7zipPath)) { | |
| Write-Host "7zip is not installed. Installing 7zip..." -ForegroundColor Yellow | |
| # Download and install 7zip | |
| $url64 = "https://www.7-zip.org/a/7z1900-x64.msi" | |
| $url32 = "https://www.7-zip.org/a/7z1900.msi" | |
| $output = "$env:TEMP\7zip.msi" | |
| try { | |
| Invoke-WebRequest -Uri $url64 -OutFile $output | |
| Start-Process -FilePath $output -ArgumentList "/qn" -Wait | |
| } catch { | |
| Write-Host "Failed to install 64-bit 7zip. Trying to install 32-bit 7zip..." -ForegroundColor Yellow | |
| Invoke-WebRequest -Uri $url32 -OutFile $output | |
| Start-Process -FilePath $output -ArgumentList "/qn" -Wait | |
| } | |
| if (!(Test-Path $7zipPath)) { | |
| Write-Host "Failed to install 7zip. Please install it manually and re-run the script." -ForegroundColor Red | |
| exit | |
| } | |
| if ($debug) { | |
| Write-Host "7zip installed successfully." -ForegroundColor Green | |
| } | |
| } | |
| if ($debug) { | |
| Write-Host "7zip is installed." -ForegroundColor Green | |
| } | |
| $timer = [Diagnostics.Stopwatch]::StartNew() | |
| if ($debug) { | |
| Write-Host "Starting script with source directories: $source and destination directory: $destination" -ForegroundColor Cyan | |
| } | |
| # Check if source and destination are external drives | |
| $sourceDriveType = Get-WmiObject -Query "SELECT DriveType FROM Win32_LogicalDisk WHERE DeviceID = '$($source[0].Substring(0,2))'" | |
| $destinationDriveType = Get-WmiObject -Query "SELECT DriveType FROM Win32_LogicalDisk WHERE DeviceID = '$($destination.Substring(0,2))'" | |
| $compress = $sourceDriveType.DriveType -eq 2 -and $destinationDriveType.DriveType -eq 2 | |
| if ($compress) { | |
| if ($debug) { | |
| Write-Host "Source and destination are external drives. Files will be compressed before transfer." -ForegroundColor Cyan | |
| } | |
| } | |
| # Create a log file for failed transfers | |
| $failuresLog = Join-Path $destination "failures.log" | |
| if (Test-Path $failuresLog) { | |
| $i = 1 | |
| while (Test-Path (Join-Path $destination ("failures_" + $i + ".log"))) { | |
| $i++ | |
| } | |
| $failuresLog = Join-Path $destination ("failures_" + $i + ".log") | |
| } | |
| Add-Content -Path $failuresLog -Value ("Script started at: " + (Get-Date).ToString() + "`nOS: " + [System.Environment]::OSVersion.VersionString + "`n") | |
| foreach ($srcDir in $source) { | |
| if ($debug) { | |
| Write-Host "`rProcessing source directory: $($spinner[$spinIndex % $spinner.Length]) $srcDir" -NoNewline | |
| } | |
| $files = Get-ChildItem -Path $srcDir -Recurse -ErrorAction SilentlyContinue | Where-Object { | |
| if ($ignoreOS) { | |
| $_.FullName -notmatch 'Windows|Program Files|Program Files (x86)' | |
| } else { | |
| $true | |
| } | |
| } | |
| if ($debug) { | |
| Write-Host "`rProcessing source directory: $($spinner[$spinIndex % $spinner.Length]) $srcDir - Done!" -ForegroundColor Green | |
| } | |
| $totalFiles = $files.Count | |
| $processedFiles = 0 | |
| foreach ($file in $files) { | |
| $dest = Join-Path $destination $file.FullName.Substring($srcDir.Length) | |
| if (!(Test-Path $dest)) { | |
| try { | |
| if ($debug) { | |
| Write-Host "`rCopying file: $($spinner[$spinIndex % $spinner.Length]) $file.FullName to destination: $dest" -NoNewline | |
| } | |
| New-Item -ItemType Directory -Force -Path (Split-Path -Parent $dest) | Out-Null | |
| if ($compress) { | |
| # Compress file | |
| & $7zipPath a -t7z "$file.7z" $file.FullName | |
| robocopy "$file.7z" $dest /MT:16 /R:0 /W:0 /NP /ETA > $null | |
| Remove-Item "$file.7z" | |
| } else { | |
| robocopy $file.FullName $dest /MT:16 /R:0 /W:0 /NP /ETA > $null | |
| } | |
| $processedFiles++ | |
| $elapsedTime = $timer.Elapsed.TotalSeconds | |
| $averageTimePerFile = $elapsedTime / $processedFiles | |
| $remainingFiles = $totalFiles - $processedFiles | |
| $eta = (Get-Date).AddSeconds($averageTimePerFile * $remainingFiles) | |
| Write-Progress -Activity "Copying files" -Status ("{0:N0} of {1:N0} files processed, ETA: {2}" -f $processedFiles, $totalFiles, $eta) -PercentComplete (($processedFiles / $totalFiles) * 100) | |
| if ($debug) { | |
| Write-Host "`rCopying file: $($spinner[$spinIndex % $spinner.Length]) $file.FullName to destination: $dest - Done!" -ForegroundColor Green | |
| } | |
| $spinIndex++ | |
| } catch { | |
| Write-Host "Error copying file: $file.FullName to destination: $dest" -ForegroundColor Red | |
| Write-Host "Error details: $_.Exception.Message" -ForegroundColor Red | |
| Add-Content -Path $failuresLog -Value ("Failed to copy file: " + $file.FullName + " to destination: " + $dest + ". Error details: " + $_.Exception.Message) | |
| } | |
| } else { | |
| if ($debug) { | |
| Write-Host "File already exists in destination: $dest" -ForegroundColor Yellow | |
| } | |
| } | |
| } | |
| } | |
| $timer.Stop() | |
| Write-Host "Script completed in $($timer.Elapsed.ToString("hh\:mm\:ss"))" -ForegroundColor Green |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # 10ZIPPER.ps1 | |
| # v1.8.0 | |
| # ----------- | |
| # | |
| # @zudsniper | |
| # @openai | |
| param( | |
| [Parameter(Mandatory=$true)] | |
| [string[]]$source, | |
| [Parameter(Mandatory=$true)] | |
| [string]$destination, | |
| [switch]$ignoreOS, | |
| [switch]$debug | |
| ) | |
| # Spinner setup | |
| $spinner = '|','/','-','\' | |
| $spinIndex = 0 | |
| # Check if script is running with administrative privileges | |
| $currentPrincipal = New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent()) | |
| $isAdministrator = $currentPrincipal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) | |
| if (-not $isAdministrator) { | |
| Write-Host "This script must be run as an Administrator. Please re-run the script as an Administrator." -ForegroundColor Red | |
| exit | |
| } | |
| if ($debug) { | |
| Write-Host "Script is running with administrative privileges." -ForegroundColor Green | |
| } | |
| # Check if 7zip is installed | |
| $7zipPath = "C:\Program Files\7-Zip\7z.exe" | |
| if (!(Test-Path $7zipPath)) { | |
| Write-Host "7zip is not installed. Installing 7zip..." -ForegroundColor Yellow | |
| # Download and install 7zip | |
| $url64 = "https://www.7-zip.org/a/7z1900-x64.msi" | |
| $url32 = "https://www.7-zip.org/a/7z1900.msi" | |
| $output = "$env:TEMP\7zip.msi" | |
| try { | |
| Invoke-WebRequest -Uri $url64 -OutFile $output | |
| Start-Process -FilePath $output -ArgumentList "/qn" -Wait | |
| } catch { | |
| Write-Host "Failed to install 64-bit 7zip. Trying to install 32-bit 7zip..." -ForegroundColor Yellow | |
| Invoke-WebRequest -Uri $url32 -OutFile $output | |
| Start-Process -FilePath $output -ArgumentList "/qn" -Wait | |
| } | |
| if (!(Test-Path $7zipPath)) { | |
| Write-Host "Failed to install 7zip. Please install it manually and re-run the script." -ForegroundColor Red | |
| exit | |
| } | |
| if ($debug) { | |
| Write-Host "7zip installed successfully." -ForegroundColor Green | |
| } | |
| } | |
| if ($debug) { | |
| Write-Host "7zip is installed." -ForegroundColor Green | |
| } | |
| $timer = [Diagnostics.Stopwatch]::StartNew() | |
| if ($debug) { | |
| Write-Host "Starting script with source directories: $source and destination directory: $destination" -ForegroundColor Cyan | |
| } | |
| # Check if source and destination are external drives | |
| $sourceDriveType = Get-WmiObject -Query "SELECT DriveType FROM Win32_LogicalDisk WHERE DeviceID = '$($source[0].Substring(0,2))'" | |
| $destinationDriveType = Get-WmiObject -Query "SELECT DriveType FROM Win32_LogicalDisk WHERE DeviceID = '$($destination.Substring(0,2))'" | |
| $compress = $sourceDriveType.DriveType -eq 2 -and $destinationDriveType.DriveType -eq 2 | |
| if ($compress) { | |
| if ($debug) { | |
| Write-Host "Source and destination are external drives. Files will be compressed before transfer." -ForegroundColor Cyan | |
| } | |
| } | |
| foreach ($srcDir in $source) { | |
| if ($debug) { | |
| Write-Host "`rProcessing source directory: $($spinner[$spinIndex % $spinner.Length]) $srcDir" -NoNewline | |
| } | |
| $files = Get-ChildItem -Path $srcDir -Recurse -ErrorAction SilentlyContinue | Where-Object { | |
| if ($ignoreOS) { | |
| $_.FullName -notmatch 'Windows|Program Files|Program Files (x86)' | |
| } else { | |
| $true | |
| } | |
| } | |
| if ($debug) { | |
| Write-Host "`rProcessing source directory: $($spinner[$spinIndex % $spinner.Length]) $srcDir - Done!" -ForegroundColor Green | |
| } | |
| $totalFiles = $files.Count | |
| $processedFiles = 0 | |
| foreach ($file in $files) { | |
| $dest = Join-Path $destination $file.FullName.Substring($srcDir.Length) | |
| if (!(Test-Path $dest)) { | |
| try { | |
| if ($debug) { | |
| Write-Host "`rCopying file: $($spinner[$spinIndex % $spinner.Length]) $file.FullName to destination: $dest" -NoNewline | |
| } | |
| New-Item -ItemType Directory -Force -Path (Split-Path -Parent $dest) | Out-Null | |
| if ($compress) { | |
| # Compress file | |
| & $7zipPath a -t7z "$file.7z" $file.FullName | |
| robocopy "$file.7z" $dest /MT:16 /R:0 /W:0 > $null | |
| Remove-Item "$file.7z" | |
| } else { | |
| robocopy $file.FullName $dest /MT:16 /R:0 /W:0 > $null | |
| } | |
| $processedFiles++ | |
| $elapsedTime = $timer.Elapsed.TotalSeconds | |
| $averageTimePerFile = $elapsedTime / $processedFiles | |
| $remainingFiles = $totalFiles - $processedFiles | |
| $eta = (Get-Date).AddSeconds($averageTimePerFile * $remainingFiles) | |
| Write-Progress -Activity "Copying files" -Status ("{0:N0} of {1:N0} files processed, ETA: {2}" -f $processedFiles, $totalFiles, $eta) -PercentComplete (($processedFiles / $totalFiles) * 100) | |
| if ($debug) { | |
| Write-Host "`rCopying file: $($spinner[$spinIndex % $spinner.Length]) $file.FullName to destination: $dest - Done!" -ForegroundColor Green | |
| } | |
| $spinIndex++ | |
| } catch { | |
| Write-Host "Error copying file: $file.FullName to destination: $dest" -ForegroundColor Red | |
| Write-Host "Error details: $_.Exception.Message" -ForegroundColor Red | |
| } | |
| } else { | |
| if ($debug) { | |
| Write-Host "File already exists at destination: $dest" -ForegroundColor Yellow | |
| } | |
| } | |
| } | |
| } | |
| $timer.Stop() | |
| Write-Host "Total elapsed time: $($timer.Elapsed.TotalSeconds) seconds" -ForegroundColor Cyan |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # 9ZIPPER.ps1 | |
| # v1.2.0 | |
| # ----------- | |
| # | |
| # @zudsniper | |
| # Define the source directories | |
| $srcDirs = @("G:\\", "C:\\", "G:\\OneDrive") | |
| $temp = "E:\\Temp\\MyTempFolder" # Change this to the path of your external SSD | |
| $logFile = "{0}.log" -f [int][double]::Parse((Get-Date (Get-Date).ToUniversalTime() -UFormat %s)) | |
| $summaryFile = "{0}.summary.8zip.md" -f [int][double]::Parse((Get-Date (Get-Date).ToUniversalTime() -UFormat %s)) | |
| # Create the temporary directory if it doesn't exist | |
| if (!(Test-Path $temp)) { New-Item -ItemType Directory -Force -Path $temp } | |
| # Define counters and hashtables | |
| $errorCount = 0 | |
| $duplicateCount = 0 | |
| $filetypeSizes = @{} | |
| $totalSize = 0 | |
| $processedSize = 0 | |
| $lastCopiedFiles = New-Object System.Collections.Queue | |
| # Start time | |
| $startTime = Get-Date | |
| Write-Host "Starting file copy process..." -ForegroundColor Green | |
| # Copy files from the source directories to the temporary directory | |
| foreach ($srcDir in $srcDirs) { | |
| $files = Get-ChildItem -Path $srcDir -Recurse | |
| $totalFiles = $files.Count | |
| $processedFiles = 0 | |
| foreach ($file in $files) { | |
| $dest = Join-Path $temp $file.FullName.Substring($srcDir.Length) | |
| if (!(Test-Path $dest)) { | |
| try { | |
| New-Item -ItemType Directory -Force -Path (Split-Path -Parent $dest) | Out-Null | |
| robocopy $file.FullName $dest /MT:16 /R:0 /W:0 | |
| $totalSize += $file.Length | |
| $processedSize += $file.Length | |
| if ($lastCopiedFiles.Count -eq 15) { | |
| $lastCopiedFiles.Dequeue() | |
| } | |
| $lastCopiedFiles.Enqueue($file.FullName) | |
| } catch { | |
| $errorCount++ | |
| Add-Content -Path $logFile -Value ("Error copying {0} to {1}: {2}" -f $file.FullName, $dest, $_.Exception.Message) | |
| } | |
| } else { | |
| $duplicateCount++ | |
| } | |
| $processedFiles++ | |
| $elapsedTime = (Get-Date) - $startTime | |
| $averageTimePerFile = $elapsedTime.TotalSeconds / $processedFiles | |
| $remainingFiles = $totalFiles - $processedFiles | |
| $eta = (Get-Date).AddSeconds($averageTimePerFile * $remainingFiles) | |
| Write-Progress -Activity "Copying files" -Status ("{0:N0} of {1:N0} files processed, ETA: {2}" -f $processedFiles, $totalFiles, $eta) -PercentComplete (($processedFiles / $totalFiles) * 100) | |
| Write-Host ("Last 15 files copied: {0}" -f ($lastCopiedFiles -join ", ")) | |
| } | |
| } | |
| Write-Host "File copy process finished. Generating summary..." -ForegroundColor Green | |
| # Generate summary | |
| $endTime = Get-Date | |
| $duration = $endTime - $startTime | |
| Add-Content -Path $summaryFile -Value ("Start time: {0}" -f $startTime) | |
| Add-Content -Path $summaryFile -Value ("End time: {0}" -f $endTime) | |
| Add-Content -Path $summaryFile -Value ("Duration: {0}" -f $duration) | |
| Add-Content -Path $summaryFile -Value ("Number of errors: {0:N0}" -f $errorCount) | |
| Add-Content -Path $summaryFile -Value ("Number of duplicates: {0:N0}" -f $duplicateCount) | |
| Add-Content -Path $summaryFile -Value ("Total size transferred: {0:N0}" -f $totalSize) | |
| Write-Host "Summary generated. Process completed." -ForegroundColor Green |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment