rclone-powershell-export-data.ps1

# Windows User Data Backup Script using Rclone - Nextcloud/WebDAV Version
# Simplified version for Nextcloud with reasonable server load
# FIXED FOR POWERSHELL ISE 5 COMPATIBILITY
# 
# DIRECTORY STRUCTURE:
# - SERVERS_DATA/COMPUTERNAME/C  (entire C: drive contents)
# - SERVERS_DATA/COMPUTERNAME/D  (entire D: drive contents)
# - SERVERS_DATA/COMPUTERNAME/E  (etc.)
# 
# SYNC MODES:
# - "sync" = Mirror source to destination (INCLUDES DELETING removed files)
# - "copy" = Only add/update files (no deletions)
# 
# PREREQUISITES:
# 1. Configure your Nextcloud remote first: rclone config
# 2. Set remote name as "nextcloud" (or change $REMOTE_NAME below)
# 3. WebDAV URL example: https://your-nextcloud.com/remote.php/dav/files/USERNAME/

# Configuration Variables (PowerShell ISE 5 compatible)
$BackupConfig = @{
    RCLONE_PATH = "C:\rclone.exe"
    REMOTE_NAME = "nextcloud"  # Your Nextcloud remote name from rclone config
    REMOTE_DIR = "SERVERS_DATA"
    TRANSFERS = 2  # Reduced to avoid overloading Nextcloud
    CHECKERS = 4  # Reduced to avoid overloading Nextcloud
    RETRIES = 3  # Retry failed transfers
    RETRY_SLEEP = "10s"  # Wait between retries
    DRYRUN = $false
    SYNC_MODE = "sync"  # "sync" = mirror (includes delete), "copy" = only add/update
    SHOW_PROGRESS = $true  # Show rclone progress output
    DELETE_EXCLUDED = $false  # Set to $true to delete previously synced files that are now excluded
}

# Exclude Patterns (PowerShell ISE 5 compatible)
$ExcludePatterns = @{
    SYSTEM_EXCLUDES = @(
        # System patterns - using **pattern** to match folders anywhere (rclone --ignore-case handles case variations)
        "**Windows**",
        "**Windows.old**", 
        "**Program Files**",
        "**Program Files (x86)**",
        "**$Recycle.Bin**",
        "**System Volume Information**",
        "**Recovery**",
        "**PerfLogs**",
        "pagefile.sys",
        "hiberfil.sys",
        "**ntuser.dat**",
        "swapfile.sys"
    )
    
    VENDOR_EXCLUDES = @(
        # Vendor patterns - using **pattern** to match folders anywhere (rclone --ignore-case handles case variations)
        "**ricoh**",
        "**xelent**", 
        "**infracom**",
        "**datto**",
        "**centrastage**",
        "**spla**",
        "**licensewatch**",
        "**vmware**",
        "**dattoav**",
        "**kaseya**",
        "**ricoh_drv**",
        "**kaseyaone**",
        "**microsoft**"
    )
    
    GENERIC_EXCLUDES = @(
        # Generic patterns - using **pattern** to match folders anywhere (rclone --ignore-case handles case variations)
        "**logs**",
        "**log**", 
        "**temp**",
        "**cache**",
        "**caches**",
        "**/.git/**",
        "**/node_modules/**",
        "**onedrive**"
    )
}

# Function to build exclude patterns
function Get-AllExcludePatterns {
    $allExcludes = @()
    $allExcludes += $ExcludePatterns.SYSTEM_EXCLUDES
    $allExcludes += $ExcludePatterns.VENDOR_EXCLUDES
    $allExcludes += $ExcludePatterns.GENERIC_EXCLUDES
    
    # Get administrators (works on both regular servers and domain controllers)
    try {
        $admins = Get-LocalAdministrators
        Write-Host "Using Get-LocalAdministrators" -ForegroundColor Green
    } catch {
        # Fallback for domain controllers - use net localgroup
        Write-Host "Get-LocalAdministrators failed, using net localgroup fallback" -ForegroundColor Yellow
        $adminOutput = net localgroup administrators 2>$null
        $admins = $adminOutput | Where-Object { $_ -and $_ -notmatch "^(The command completed|Alias name|Comment|Members|[-]+|\s*$)" } | 
            ForEach-Object { $_.Trim() } | 
            Where-Object { $_ -and $_ -notlike "*\*" }  # Exclude domain accounts with backslash
    }
    
    # Exclude all administrators
    foreach ($admin in $admins) {
        $allExcludes += "/Users/$admin/**"
        $allExcludes += "Users/$admin/**"
        Write-Host "Excluding admin profile: $admin" -ForegroundColor Yellow
    }
    
    return $allExcludes
}

# Simplified rclone sync function for Nextcloud with retry support
function Invoke-RcloneBackup {
    param(
        [string]$Source,
        [string]$Destination,
        [string[]]$ExcludePatterns,
        [string]$Description
    )
    
    Write-Host "Syncing $Description ($Source) to $Destination" -ForegroundColor Green
    if ($BackupConfig.SYNC_MODE -eq "sync") {
        Write-Host "NOTE: Using SYNC mode - files deleted locally will be deleted on Nextcloud" -ForegroundColor Yellow
    } else {
        Write-Host "NOTE: Using COPY mode - only adding/updating files, no deletions" -ForegroundColor Yellow
    }
    
    # Build arguments array - simplified for WebDAV
    $rcloneArgs = @(
        $BackupConfig.SYNC_MODE,  # "sync" or "copy" based on config
        $Source,
        $Destination,
        "--ignore-case"
    )
    
    # Add each exclude
    foreach ($exclude in $ExcludePatterns) {
        $rcloneArgs += "--exclude"
        $rcloneArgs += $exclude
    }
    
    # Simplified parameters for WebDAV
    $rcloneArgs += @(
        "--transfers", $BackupConfig.TRANSFERS,
        "--checkers", $BackupConfig.CHECKERS,
        "--retries", $BackupConfig.RETRIES,
        "--retries-sleep", $BackupConfig.RETRY_SLEEP,
        "--skip-links",
        "--stats", "30s",  # Show stats every 30 seconds
        "--log-level", "INFO",  # Show info messages
        "--fast-list",  # Faster listing for WebDAV
        "--low-level-retries", "3"  # Low level retries for connection issues
    )
    
    # Note: Excluded file deletion is now handled separately with rclone delete
    
    if ($BackupConfig.SHOW_PROGRESS) {
        $rcloneArgs += "--progress"
    }
    
    if ($BackupConfig.DRYRUN) {
        $rcloneArgs += "--dry-run"
        Write-Host "DRY RUN ENABLED" -ForegroundColor Yellow
    }
    
    Write-Host "Excludes applied: $($ExcludePatterns.Count) patterns" -ForegroundColor Red
    Write-Host "Settings: Transfers=$($BackupConfig.TRANSFERS), Checkers=$($BackupConfig.CHECKERS), Retries=$($BackupConfig.RETRIES)" -ForegroundColor Gray
    
    # Execute rclone
    try {
        Write-Host "`nStarting sync operation..." -ForegroundColor Yellow
        Write-Host "From: $Source" -ForegroundColor Gray
        Write-Host "To: $Destination" -ForegroundColor Gray
        Write-Host "Stats will update every 30 seconds..." -ForegroundColor Cyan
        Write-Host "========================================" -ForegroundColor DarkGray
        
        # Execute rclone
        $rclonePath = $BackupConfig.RCLONE_PATH
        
        # Execute via cmd for better ISE output display
        Write-Host "Executing rclone..." -ForegroundColor Gray
        $startTime = Get-Date
        
        # Execute rclone using Start-Process for better reliability (based on rclone forum solution)
        Write-Host "Command: $rclonePath $($rcloneArgs -join ' ')" -ForegroundColor Gray
        
        # Use direct execution to see rclone output in console
        & $rclonePath @rcloneArgs
        $exitCode = $LASTEXITCODE
        
        $endTime = Get-Date
        $duration = $endTime - $startTime
        
        # Check exit code (already set above)
        if ($exitCode -ne 0) {
            Write-Warning "`nRclone exited with code $exitCode for ${Description}"
        } else {
            Write-Host "`nSuccessfully synced ${Description}" -ForegroundColor Green
            Write-Host "Duration: $($duration.ToString('hh\:mm\:ss'))" -ForegroundColor Gray
        }
        
    } catch {
        Write-Error "Failed to sync ${Description}: $_"
        Write-Host "Error details: $($_.Exception.Message)" -ForegroundColor Red
    }
}

# Main execution
Write-Host "==================================" -ForegroundColor Cyan
Write-Host "=== NEXTCLOUD SYNC SCRIPT ===" -ForegroundColor Cyan
Write-Host "==================================" -ForegroundColor Cyan

# Test rclone availability
if (-not (Test-Path $BackupConfig.RCLONE_PATH)) {
    Write-Error "Rclone not found at $($BackupConfig.RCLONE_PATH)"
    Write-Host "Please install rclone or update the path in the script"
    exit 1
} else {
    Write-Host "Rclone found at: $($BackupConfig.RCLONE_PATH)" -ForegroundColor Green
}

# Test Nextcloud connection
Write-Host "`nTesting Nextcloud connection..." -ForegroundColor Yellow
$testResult = & ($BackupConfig.RCLONE_PATH) lsd "$($BackupConfig.REMOTE_NAME):" 2>&1
if ($LASTEXITCODE -eq 0) {
    Write-Host "Nextcloud connection successful!" -ForegroundColor Green
} else {
    Write-Warning "Could not connect to Nextcloud. Please check your rclone config."
    Write-Host "Run 'rclone config' to set up your Nextcloud remote"
    Write-Host "Error: $testResult"
}

Write-Host "`nStarting sync process to Nextcloud..." -ForegroundColor Green
Write-Host "Time: $(Get-Date)" -ForegroundColor Gray
Write-Host ""
if ($BackupConfig.SYNC_MODE -eq "sync") {
    Write-Host "WARNING: Using SYNC mode - files deleted locally will be deleted on Nextcloud!" -ForegroundColor Red
} else {
    Write-Host "Using COPY mode - only adding/updating files" -ForegroundColor Yellow
}

if ($BackupConfig.DELETE_EXCLUDED) {
    Write-Host "WARNING: DELETE_EXCLUDED enabled - previously synced admin/excluded files will be DELETED from Nextcloud!" -ForegroundColor Red
    Write-Host "This includes admin profiles, temp profiles, Microsoft folders, etc. that were synced before exclusions were added." -ForegroundColor Red
}
Write-Host "Computer: $env:COMPUTERNAME" -ForegroundColor Cyan
Write-Host "Remote: $($BackupConfig.REMOTE_NAME):$($BackupConfig.REMOTE_DIR)/$env:COMPUTERNAME/" -ForegroundColor Cyan
Write-Host ""

# Get exclude patterns (this will also display the admin exclusions)
$excludePatterns = Get-AllExcludePatterns

# Backup all drives
Write-Host "`n=== SYNCING ALL DRIVES ===" -ForegroundColor Magenta

# Get all local drives
Write-Host "Detecting drives..." -ForegroundColor Yellow
$allDrives = Get-WmiObject -Class Win32_LogicalDisk | Where-Object { 
    $_.DriveType -eq 3  # Local Disk
}

Write-Host "Detected drives: $(($allDrives | ForEach-Object { $_.DeviceID }) -join ', ')" -ForegroundColor Cyan
Write-Host ""

foreach ($driveInfo in $allDrives) {
    $drive = $driveInfo.DeviceID  # e.g., "C:", "D:", "E:"
    $driveLabel = $driveInfo.VolumeName
    $freeSpace = [math]::Round($driveInfo.FreeSpace / 1GB, 2)
    $totalSize = [math]::Round($driveInfo.Size / 1GB, 2)
    
    Write-Host "`n==================================" -ForegroundColor Cyan
    Write-Host "Processing drive ${drive} (${driveLabel})" -ForegroundColor Yellow
    Write-Host "Size: ${totalSize}GB (${freeSpace}GB free)" -ForegroundColor Yellow
    Write-Host "==================================" -ForegroundColor Cyan
    
    # Check if drive is accessible
    if (-not (Test-Path "${drive}\")) {
        Write-Warning "Drive ${drive} is not accessible, skipping"
        continue
    }
    
    # Build proper path: SERVERS_DATA/COMPUTERNAME/C, SERVERS_DATA/COMPUTERNAME/D, etc.
    $source = "${drive}\"
    $driveLetter = $drive.Replace(':', '')
    $dest = "$($BackupConfig.REMOTE_NAME):$($BackupConfig.REMOTE_DIR)/$env:COMPUTERNAME/${driveLetter}"
    
    Write-Host "Syncing ${drive} to ${dest}" -ForegroundColor Green
    
    # First pass: Delete excluded files using rclone delete with --include patterns
    if ($BackupConfig.DELETE_EXCLUDED) {
        Write-Host "First pass: Cleaning up excluded files from Nextcloud using rclone delete..." -ForegroundColor Yellow
        
        # Build rclone delete command with --include patterns (using excluded paths as includes)
        $deleteArgs = @(
            "delete",
            $dest,
            "--ignore-case",  # This makes it case-insensitive to catch all variations
            "--progress",     # Show progress during deletion
            "--rmdirs"        # Remove empty directories after deletion
        )
        
        # Add each exclude pattern as an include for the delete operation
        foreach ($exclude in $excludePatterns) {
            $deleteArgs += "--include"
            $deleteArgs += $exclude
        }
        
        # Add dry-run if configured
        if ($BackupConfig.DRYRUN) {
            $deleteArgs += "--dry-run"
            Write-Host "DRY RUN: Would delete excluded files" -ForegroundColor Yellow
        }
        
        Write-Host "Deleting excluded files with case-insensitive matching..." -ForegroundColor Yellow
        try {
            & ($BackupConfig.RCLONE_PATH) @deleteArgs
            if ($LASTEXITCODE -eq 0) {
                Write-Host "Successfully cleaned up excluded files" -ForegroundColor Green
            } else {
                Write-Warning "rclone delete completed with exit code $LASTEXITCODE"
            }
        } catch {
            Write-Warning "Error during excluded file cleanup: $_"
        }
    }
    
    # Main sync: Regular sync without any deletion flags
    Write-Host "Main sync: Syncing current files..." -ForegroundColor Green
    Invoke-RcloneBackup -Source $source -Destination $dest -ExcludePatterns $excludePatterns -Description "Drive ${drive}"
    
    Write-Host "`nCompleted ${drive}" -ForegroundColor Green
    Write-Host "----------------------------------------" -ForegroundColor DarkGray
}

Write-Host "`n==================================" -ForegroundColor Green
Write-Host "=== BACKUP COMPLETE ===" -ForegroundColor Green
Write-Host "==================================" -ForegroundColor Green
Write-Host "All drives have been synced to Nextcloud successfully!" -ForegroundColor Green
Write-Host "Structure: $($BackupConfig.REMOTE_NAME):$($BackupConfig.REMOTE_DIR)/$env:COMPUTERNAME/[C, D, E, etc.]" -ForegroundColor Cyan
Write-Host "Time completed: $(Get-Date)" -ForegroundColor Gray