Created
April 11, 2025 14:10
-
-
Save joerodgers/f1d5ebac54382e7975c7a28348d5a7af to your computer and use it in GitHub Desktop.
mockup framework on how you could use SQLite database to track progress and store results for long running (days) scripts. This would allow for a script to be restarted from where it failed rather than having to start all the sites over.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Import-Module -Name PSSQLite | |
function Initialize-Database | |
{ | |
[CmdletBinding()] | |
param | |
( | |
[parameter(mandatory=$true)] | |
[string] | |
$Path, | |
[parameter(mandatory=$true)] | |
[System.Collections.Specialized.OrderedDictionary] | |
$Schema, | |
[parameter(mandatory=$false)] | |
[switch] | |
$Force | |
) | |
begin | |
{ | |
} | |
process | |
{ | |
if( (Test-Path -Path $Path -PathType Leaf) -and -not $Force ) | |
{ | |
return | |
} | |
else | |
{ | |
foreach( $s in $Schema.GetEnumerator() ) | |
{ | |
Write-Verbose "Executing Statement: $($s.value)" | |
Invoke-SqliteQuery -Database $Path -Query $s.value | |
} | |
} | |
} | |
end | |
{ | |
} | |
} | |
# connect to tenant | |
# requires SharePoint > Application > Sites.FullControl.All | |
Connect-PnPOnline -Url "https://$env:CDX_TENANT-admin.sharepoint.com" ` | |
-ClientId $env:CDX_CLIENTID ` | |
-Thumbprint $env:CDX_THUMBPRINT ` | |
-Tenant $env:CDX_TENANTID ` | |
-ErrorAction Stop | |
# initialize database | |
$path = "C:\_temp\sites.sqllite" | |
$schema = [ordered] @{ | |
"INVENTORY" = "CREATE TABLE urls ( url VARCHAR(1000) PRIMARY KEY )" | |
"RESULTS" = "CREATE TABLE results ( url VARCHAR(1000), filename VARCHAR(1000), contenttype VARCHAR(50), actorCount INT, actionCount INT )" | |
} | |
Initialize-Database -Path "C:\_temp\sites.sqllite" -Schema $schema -ErrorAction Stop | |
# load database table | |
if( ( Invoke-SqliteQuery -DataSource $path -Query "SELECT COUNT(*) FROM urls" -As SingleValue) -le 0 ) | |
{ | |
$drives = Get-PnPTenantSite -IncludeOneDriveSites -Filter "Url -like '.sharepoint.com/personal/'" -ErrorAction Stop | |
$datatable = $drives | Select-Object Url | Out-DataTable -ErrorAction Stop | |
Invoke-SQLiteBulkCopy -DataTable $datatable -DataSource $path -Table "urls" -Confirm:$false -ErrorAction Stop | |
} | |
# process workload | |
$urls = Invoke-SqliteQuery -DataSource $path -Query "SELECT url FROM urls" -As PSObject -ErrorAction Stop | Select-Object -ExpandProperty Url | |
foreach( $url in $urls ) | |
{ | |
try | |
{ | |
# do some work on site | |
# update database w/ results of work | |
$query = "INSERT INTO results ( url, fileName, contentType, actorCount, actionCount ) VALUES ( @url, @fileName, @contentType, @actorCount, @actionCount )" | |
Invoke-SqliteQuery -DataSource $path -Query $query -SqlParameters @{ url = $url; filename = "foo.mp4"; contentType = "Recording"; actorCount = 1; actionCount = 1 } -ErrorAction Stop | |
# remove site from inventory so it's not processed again | |
Invoke-SqliteQuery -DataSource $path -Query "DELETE FROM urls WHERE url = @url" -SqlParameters @{ url = $url } | |
} | |
catch | |
{ | |
# tbd - mark site as failed in database for reporting purposes? | |
Write-Error "Site failed: $url. Error: $_" | |
} | |
} | |
# export results from sql to csv | |
$results = Invoke-SqliteQuery -DataSource $path -Query "SELECT * FROM results" -As PSObject | |
$results | Export-Csv -Path "C:\_temp\results.csv" -NoTypeInformation | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment