Skip to content

Instantly share code, notes, and snippets.

@TechByTom

TechByTom/Web-InterfaceCategorizer.ps1

Last active February 27, 2025 19:57
Show Gist options
  • Save TechByTom/04d3ac248b0a197048e27f44700c94f7 to your computer and use it in GitHub Desktop.
Save TechByTom/04d3ac248b0a197048e27f44700c94f7 to your computer and use it in GitHub Desktop.
Download ZIP
Internal Admin Interface Discovery + Categorization + Reporting.
Raw
Web-InterfaceCategorizer.ps1
param(
[Parameter(Mandatory=$true)]
[string]$ScanDirectory,
[Parameter(Mandatory=$true)]
[string]$FingerprintsFile,
[Parameter(Mandatory=$false)]
[string]$OutputDirectory = ".\categorized_interfaces"
)
# Create Windows shortcut (.lnk) file
function New-Shortcut {
param(
[Parameter(Mandatory=$true)]
[string]$SourcePath,
[Parameter(Mandatory=$true)]
[string]$ShortcutPath
)
# Convert paths to absolute paths
$SourcePath = [System.IO.Path]::GetFullPath($SourcePath)
$ShortcutPath = [System.IO.Path]::GetFullPath($ShortcutPath)
$WScriptShell = New-Object -ComObject WScript.Shell
$Shortcut = $WScriptShell.CreateShortcut($ShortcutPath)
$Shortcut.TargetPath = $SourcePath
$Shortcut.WorkingDirectory = [System.IO.Path]::GetDirectoryName($SourcePath)
$Shortcut.Save()
}
# Get original file path from .lnk file
function Get-ShortcutTarget {
param(
[string]$ShortcutPath
)
$WScriptShell = New-Object -ComObject WScript.Shell
$Shortcut = $WScriptShell.CreateShortcut($ShortcutPath)
return $Shortcut.TargetPath
}
# Function to test if a string matches any pattern in an array
function Test-Patterns {
param(
[string]$Text,
[array]$Patterns
)
foreach ($pattern in $Patterns) {
if ($Text -match [regex]::Escape($pattern)) {
return $true
}
}
return $false
}
# Function to check if headers match the specified patterns
function Test-Headers {
param(
[array]$Headers,
$HeaderPatterns
)
# Convert PSObject to Hashtable if needed
$headerPatternsHash = $HeaderPatterns | ConvertTo-Hashtable
foreach ($headerPattern in $headerPatternsHash.GetEnumerator()) {
$headerFound = $false
foreach ($header in $Headers) {
if ($header.Key -eq $headerPattern.Key) {
foreach ($value in $header.Value) {
if (Test-Patterns -Text $value -Patterns $headerPattern.Value) {
$headerFound = $true
break
}
}
}
}
if (-not $headerFound) {
return $false
}
}
return $true
}
# Function to convert PSObject to Hashtable
function ConvertTo-Hashtable {
param (
[Parameter(ValueFromPipeline)]
$InputObject
)
process {
if ($null -eq $InputObject) { return $null }
if ($InputObject -is [System.Collections.Hashtable]) { return $InputObject }
$hash = @{}
$InputObject.PSObject.Properties | ForEach-Object {
$hash[$_.Name] = $_.Value
}
return $hash
}
}
# Function to categorize a single interface based on fingerprints
function Get-InterfaceCategory {
param(
[PSCustomObject]$InterfaceData,
[PSCustomObject]$Fingerprints
)
foreach ($category in $Fingerprints.PSObject.Properties) {
$fingerprint = $category.Value
$allPropertiesMatched = $true
# Check URL patterns if specified
if ($fingerprint.urlPatterns) {
$urlFound = $false
foreach ($port in $InterfaceData.OpenPorts) {
if ($port.WebRequest) {
$requests = @($port.WebRequest)
if ($requests[0] -isnot [array]) { $requests = @($requests) }
foreach ($request in $requests) {
if (Test-Patterns -Text $request.Url -Patterns $fingerprint.urlPatterns) {
$urlFound = $true
break
}
}
}
}
if (-not $urlFound) {
$allPropertiesMatched = $false
continue
}
}
# Check content patterns if specified
if ($allPropertiesMatched -and $fingerprint.contentPatterns) {
$contentFound = $false
foreach ($port in $InterfaceData.OpenPorts) {
if ($port.WebRequest) {
$requests = @($port.WebRequest)
if ($requests[0] -isnot [array]) { $requests = @($requests) }
foreach ($request in $requests) {
if ($request.Content -and (Test-Patterns -Text $request.Content -Patterns $fingerprint.contentPatterns)) {
$contentFound = $true
break
}
}
}
}
if (-not $contentFound) {
$allPropertiesMatched = $false
continue
}
}
# Check header patterns if specified
if ($allPropertiesMatched -and $fingerprint.headerPatterns) {
$headersFound = $false
foreach ($port in $InterfaceData.OpenPorts) {
if ($port.WebRequest) {
$requests = @($port.WebRequest)
if ($requests[0] -isnot [array]) { $requests = @($requests) }
foreach ($request in $requests) {
if ($request.Headers -and (Test-Headers -Headers $request.Headers -HeaderPatterns $fingerprint.headerPatterns)) {
$headersFound = $true
break
}
}
}
}
if (-not $headersFound) {
$allPropertiesMatched = $false
continue
}
}
# Only return the category if ALL specified properties match
if ($allPropertiesMatched) {
return $category.Name
}
}
return "Unknown"
}
# Function to extract title from HTML content
function Get-HtmlTitle {
param(
[string]$Content
)
if (-not $Content) { return $null }
if ($Content -match '<title[^>]*>(.*?)</title>') {
return $matches[1].Trim()
}
return $null
}
# Updated report generation function
function New-CategoryReport {
param(
[string]$OutputDirectory,
[string]$ScanDirectory
)
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
$reportPath = Join-Path $OutputDirectory "categorization_report_${timestamp}.html"
$categories = Get-ChildItem -Path $OutputDirectory -Directory
$html = @"
<!DOCTYPE html>
<html>
<head>
<title>Interface Categorization Report</title>
<style>
body { font-family: Arial, sans-serif; margin: 20px; }
h1 { color: #333; }
h2 { color: #666; margin-top: 30px; }
h3 { color: #666; margin-top: 20px; margin-left: 20px; }
.category { margin-bottom: 30px; }
table { border-collapse: collapse; width: 100%; margin-top: 10px; }
th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
th { background-color: #f5f5f5; }
tr:nth-child(even) { background-color: #f9f9f9; }
tr:hover { background-color: #f5f5f5; }
.stats { margin: 20px 0; padding: 10px; background-color: #f8f9fa; border-radius: 4px; }
.category-count { color: #666; font-size: 0.9em; margin-left: 10px; }
.collapse-btn {
background: none;
border: none;
color: #666;
cursor: pointer;
font-size: 1.2em;
padding: 0 10px;
vertical-align: middle;
}
.collapse-btn:hover { color: #333; }
.section-header {
display: flex;
align-items: center;
}
.collapsed { display: none; }
</style>
<script>
function toggleSection(btn, targetId) {
const section = document.getElementById(targetId);
const isCollapsed = section.classList.contains('collapsed');
section.classList.toggle('collapsed');
btn.textContent = isCollapsed ? '▼' : '▶';
}
</script>
</head>
<body>
<h1>Interface Categorization Report</h1>
<div class="stats">
<p>Total Categories: $($categories.Count)</p>
<p>Generated: $(Get-Date)</p>
</div>
"@
# First process all non-Unknown categories
foreach ($category in ($categories | Where-Object { $_.Name -ne "Unknown" })) {
$files = Get-ChildItem -Path $category.FullName -Filter "*.lnk"
$sectionId = "category-$($category.Name.ToLower())"
$html += @"
<div class="category">
<div class="section-header">
<button class="collapse-btn" onclick="toggleSection(this, '$sectionId')">▼</button>
<h2>$($category.Name) <span class="category-count">($($files.Count) interfaces)</span></h2>
</div>
<div id="$sectionId">
<table>
<tr>
<th>IP Address</th>
<th>Hostname</th>
<th>Ports</th>
<th>Web Links</th>
<th>JSON File</th>
</tr>
"@
foreach ($file in $files) {
$originalPath = Get-ShortcutTarget -ShortcutPath $file.FullName
$interfaceData = Get-Content $originalPath | ConvertFrom-Json
$ports = ($interfaceData.OpenPorts | ForEach-Object { $_.PortNumber }) -join ", "
$links = @()
foreach ($port in $interfaceData.OpenPorts) {
if ($port.WebRequest) {
$requests = @($port.WebRequest)
if ($requests[0] -isnot [array]) { $requests = @($requests) }
foreach ($request in $requests) {
if ($request.Url) {
$links += $request.Url
}
}
}
}
$linksHtml = ($links | Select-Object -Unique | ForEach-Object {
"<a href='$_' target='_blank'>$_</a>"
}) -join "<br>"
$html += @"
<tr>
<td>$($interfaceData.IP)</td>
<td>$($interfaceData.Hostname)</td>
<td>$ports</td>
<td>$linksHtml</td>
<td><a href="file://$originalPath">$([System.IO.Path]::GetFileName($originalPath))</a></td>
</tr>
"@
}
$html += @"
</table>
</div>
</div>
"@
}
# Process Unknown category with title grouping
$unknownCategory = $categories | Where-Object { $_.Name -eq "Unknown" }
if ($unknownCategory) {
$files = Get-ChildItem -Path $unknownCategory.FullName -Filter "*.lnk"
if ($files) {
$html += @"
<div class="category">
<div class="section-header">
<button class="collapse-btn" onclick="toggleSection(this, 'category-unknown')">▼</button>
<h2>Unknown <span class="category-count">($($files.Count) interfaces)</span></h2>
</div>
<div id="category-unknown">
"@
# Group unknown files by title
$titleGroups = @{}
foreach ($file in $files) {
$originalPath = Get-ShortcutTarget -ShortcutPath $file.FullName
$interfaceData = Get-Content $originalPath | ConvertFrom-Json
$title = "No Title"
# Try to find a title in any of the web responses
foreach ($port in $interfaceData.OpenPorts) {
if ($port.WebRequest) {
$requests = @($port.WebRequest)
if ($requests[0] -isnot [array]) { $requests = @($requests) }
foreach ($request in $requests) {
if ($request.Content) {
$extractedTitle = Get-HtmlTitle -Content $request.Content
if ($extractedTitle) {
$title = $extractedTitle
break
}
}
}
}
if ($title -ne "No Title") { break }
}
if (-not $titleGroups.ContainsKey($title)) {
$titleGroups[$title] = @()
}
$titleGroups[$title] += $file
}
# Sort title groups by count (most to least)
foreach ($titleGroup in $titleGroups.GetEnumerator() | Sort-Object { $_.Value.Count } -Descending) {
$titleId = "unknown-title-$([System.Web.HttpUtility]::UrlEncode($titleGroup.Key))"
$html += @"
<div class="section-header">
<button class="collapse-btn" onclick="toggleSection(this, '$titleId')">▼</button>
<h3>Title: $($titleGroup.Key) <span class="category-count">($($titleGroup.Value.Count) interfaces)</span></h3>
</div>
<div id="$titleId">
<table>
<tr>
<th>IP Address</th>
<th>Hostname</th>
<th>Ports</th>
<th>Web Links</th>
<th>JSON File</th>
</tr>
"@
foreach ($file in $titleGroup.Value) {
$originalPath = Get-ShortcutTarget -ShortcutPath $file.FullName
$interfaceData = Get-Content $originalPath | ConvertFrom-Json
$ports = ($interfaceData.OpenPorts | ForEach-Object { $_.PortNumber }) -join ", "
$links = @()
foreach ($port in $interfaceData.OpenPorts) {
if ($port.WebRequest) {
$requests = @($port.WebRequest)
if ($requests[0] -isnot [array]) { $requests = @($requests) }
foreach ($request in $requests) {
if ($request.Url) {
$links += $request.Url
}
}
}
}
$linksHtml = ($links | Select-Object -Unique | ForEach-Object {
"<a href='$_' target='_blank'>$_</a>"
}) -join "<br>"
$html += @"
<tr>
<td>$($interfaceData.IP)</td>
<td>$($interfaceData.Hostname)</td>
<td>$ports</td>
<td>$linksHtml</td>
<td><a href="file://$originalPath">$([System.IO.Path]::GetFileName($originalPath))</a></td>
</tr>
"@
}
$html += @"
</table>
</div>
"@
}
$html += @"
</div>
</div>
"@
}
}
$html += @"
</body>
</html>
"@
$html | Out-File -FilePath $reportPath -Encoding UTF8
return $reportPath
}
# Main script execution
try {
# Convert relative paths to absolute paths
$ScanDirectory = Convert-Path $ScanDirectory
$FingerprintsFile = Convert-Path $FingerprintsFile
# Create timestamped output directory using absolute path
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
$OutputDirectory = Join-Path (Convert-Path .) $OutputDirectory
$OutputDirectory = Join-Path $OutputDirectory $timestamp
if (-not (Test-Path $OutputDirectory)) {
New-Item -ItemType Directory -Path $OutputDirectory | Out-Null
}
# Load fingerprints file
$fingerprints = Get-Content $FingerprintsFile | ConvertFrom-Json
# Track categorization results
$results = @{}
# Process each JSON file in the scan directory
Get-ChildItem -Path $ScanDirectory -Filter "*.json" | ForEach-Object {
Write-Host "Processing $($_.Name)..."
try {
# Read and parse JSON file
$interfaceData = Get-Content $_.FullName | ConvertFrom-Json
# Get category
$category = Get-InterfaceCategory -InterfaceData $interfaceData -Fingerprints $fingerprints
# Create category directory if it doesn't exist
$categoryDir = Join-Path $OutputDirectory $category
if (-not (Test-Path $categoryDir)) {
New-Item -ItemType Directory -Path $categoryDir | Out-Null
}
# Create .lnk file instead of symbolic link
$linkPath = Join-Path $categoryDir "$($_.BaseName).lnk"
if (-not (Test-Path $linkPath)) {
New-Shortcut -SourcePath $_.FullName -ShortcutPath $linkPath
}
# Track result
if (-not $results.ContainsKey($category)) {
$results[$category] = 0
}
$results[$category]++
Write-Host "Categorized as: $category"
}
catch {
Write-Warning "Error processing $($_.Name): $_"
}
}
# Generate HTML report
$reportPath = New-CategoryReport -OutputDirectory $OutputDirectory -ScanDirectory $ScanDirectory
# Display summary
Write-Host "`nCategorization Summary:"
$results.GetEnumerator() | Sort-Object Name | ForEach-Object {
Write-Host "$($_.Key): $($_.Value) interfaces"
}
Write-Host "`nReport generated at: $reportPath"
}
catch {
Write-Error "Script execution failed: $_"
}
Raw
Web-InterfaceFingerprintsExample.jsom
{
"iDRAC": {
"urlPatterns": [
"/restgui/start.html",
"/login.html",
"/data/login"
],
"contentPatterns": [
"ng-app='loginapp'",
"idrac-start-screen",
"images/clarityicons"
],
"headerPatterns": {
"Server": ["Apache"],
"Content-Security-Policy": ["default-src 'self'; connect-src *"]
}
},
"CiscoIOS": {
"urlPatterns": [
"/level/15/exec/-/",
"/login.html",
"/cgi-bin/login"
],
"contentPatterns": [
"Cisco Systems",
"routername",
"class=\"cisco\""
],
"headerPatterns": {
"Server": ["cisco-IOS"]
}
},
"HPiLO": {
"urlPatterns": [
"/json/login_session",
"/html/login.html",
"/redfish/v1"
],
"contentPatterns": [
"iLO",
"hp-navbar",
"integrity_footer"
],
"headerPatterns": {
"Server": ["HP-iLO-Server", "HPE-iLO-Server"],
"X-Frame-Options": ["SAMEORIGIN"]
}
},
"Synology": {
"urlPatterns": [
"/webman/",
"/webapi/",
"/DSM.php"
],
"contentPatterns": [
"SYNO.SDS",
"SynologyDiskStation",
"webman"
],
"headerPatterns": {
"Server": ["Synology"],
"X-Powered-By": ["PHP"]
}
},
"QNAP": {
"urlPatterns": [
"/cgi-bin/authLogin.cgi",
"/cgi-bin/login.html"
],
"contentPatterns": [
"QTS Gateway",
"QNAP",
"qnap-loading-page"
],
"headerPatterns": {
"Server": ["http server 1.0"]
}
},
"UniFiController": {
"urlPatterns": [
"/manage",
"/login",
"/api/login"
],
"contentPatterns": [
"UniFi Network",
"ubnt-UniFi",
"ng-controller=\"LoginController\""
],
"headerPatterns": {
"Server": ["UniFi"],
"X-Frame-Options": ["SAMEORIGIN"]
}
},
"ESXi": {
"urlPatterns": [
"/ui/",
"/folder",
"/sdk"
],
"contentPatterns": [
"VMware ESXi",
"vsphere-client",
"loginButton"
],
"headerPatterns": {
"Server": ["VMware"],
"X-Frame-Options": ["DENY"]
}
},
"Proxmox": {
"urlPatterns": [
"/pve-docs",
"/api2/json/access/ticket",
"/pve2/mobile.html#v"
],
"contentPatterns": [
"Proxmox",
"pve-lang-name",
"proxmoxlib.js"
],
"headerPatterns": {
"Server": ["pve-api-daemon", "proxmox"]
}
},
"PfSense": {
"urlPatterns": [
"/index.php",
"/css/pfSense.css"
],
"contentPatterns": [
"pfSense",
"login-page",
"fadeOut"
],
"headerPatterns": {
"X-Powered-By": ["PHP"],
"X-Frame-Options": ["DENY"]
}
},
"TrueNAS": {
"urlPatterns": [
"/ui/sessions/signin",
"/api/v2.0"
],
"contentPatterns": [
"TrueNAS",
"ix-auto",
"freenas-login"
],
"headerPatterns": {
"Server": ["nginx"],
"X-Frame-Options": ["SAMEORIGIN"]
}
}
}
Raw
Web-InterfaceIdentifier-0.3.2.ps1
param(
[Parameter(Mandatory=$true)]
[string]$XmlPath,
[Parameter(Mandatory=$false)]
[int]$MaxHosts = 15,
[Parameter(Mandatory=$false)]
[int]$RequestTimeout = 10,
[Parameter(Mandatory=$false)]
[int]$MaxResponseSize = 20MB
)
# Function to enable all locally supported TLS/SSL versions
function Set-MaximumTlsSupport {
$protocols = [enum]::GetValues([System.Net.SecurityProtocolType]) |
Where-Object { $_ -ne 'SystemDefault' }
$supportedProtocols = $protocols | ForEach-Object {
try {
[System.Net.ServicePointManager]::SecurityProtocol = $_
$_
} catch {
Write-Host "Protocol $_ not supported"
$null
}
} | Where-Object { $_ -ne $null }
$finalProtocol = [System.Net.SecurityProtocolType]($supportedProtocols -join ',')
[System.Net.ServicePointManager]::SecurityProtocol = $finalProtocol
Write-Host "Enabled protocols: $finalProtocol"
}
# Function definitions must come before usage
function Test-DNSResolution {
param(
[string]$Hostname,
[string]$IP
)
try {
$dnsResults = [System.Net.Dns]::GetHostEntry($Hostname)
$ipAddresses = $dnsResults.AddressList | ForEach-Object { $_.IPAddressToString }
# Check if the IP we have matches any of the DNS results
$ipMatch = $ipAddresses -contains $IP
return @{
Resolves = $true
IPAddresses = $ipAddresses
IPMatch = $ipMatch
Error = $null
}
}
catch {
return @{
Resolves = $false
IPAddresses = @()
IPMatch = $false
Error = $_.Exception.Message
}
}
}
function Get-TargetHostname {
param(
[string]$Hostname,
[string]$IP
)
if (-not $Hostname) {
Write-Host " No hostname provided, using IP: $IP"
return $IP
}
$dnsCheck = Test-DNSResolution -Hostname $Hostname -IP $IP
if (-not $dnsCheck.Resolves) {
Write-Host " Hostname '$Hostname' does not resolve in DNS, using IP: $IP"
return $IP
}
if (-not $dnsCheck.IPMatch) {
Write-Host " Warning: Hostname '$Hostname' resolves to different IPs: $($dnsCheck.IPAddresses -join ', ')"
Write-Host " Target IP '$IP' not in DNS results, using IP instead of hostname"
return $IP
}
Write-Host " Hostname '$Hostname' resolves correctly to IP: $IP"
return $Hostname
}
function Get-Favicon {
param(
[string]$BaseUrl,
[bool]$IsSSL,
[int]$Timeout = 10
)
try {
# First try standard /favicon.ico
$faviconUrl = "{0}/favicon.ico" -f $BaseUrl
Write-Host " → Attempting: $faviconUrl"
if ($IsSSL) {
$response = Invoke-WebRequest -Uri $faviconUrl -MaximumRedirection 0 -SkipCertificateCheck -TimeoutSec $Timeout -ErrorAction SilentlyContinue
} else {
$response = Invoke-WebRequest -Uri $faviconUrl -MaximumRedirection 0 -TimeoutSec $Timeout -ErrorAction SilentlyContinue
}
if ($response.StatusCode -eq 200 -and $response.RawContentLength -gt 0) {
return [Convert]::ToBase64String($response.Content)
}
} catch {
# Don't output anything - the error is expected for sites without favicon.ico
}
return $null
}
function Fetch-Url {
param(
[string]$TargetHost,
[int]$Port,
[bool]$IsSSL,
[int]$Timeout = 10
)
$MaxRedirects = 10
$RedirectCount = 0
$results = @()
$protocol = if ($IsSSL) { "https" } else { "http" }
$currentUrl = "{0}://{1}:{2}" -f $protocol, $TargetHost, $Port
while ($true) {
try {
$requestResult = [ordered]@{
Url = $currentUrl
RedirectDepth = $RedirectCount
StatusCode = $null
Error = $null
Headers = @{}
Content = $null
Favicon = $null
RedirectedRequest = $null
}
Write-Host " → Attempting: $currentUrl"
# Create WebRequest to check content length first
$webRequest = [System.Net.WebRequest]::Create($currentUrl)
$webRequest.Method = "HEAD"
$webRequest.Timeout = $Timeout * 1000
if ($IsSSL) {
# For HTTPS requests
[System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true }
}
try {
$headResponse = $webRequest.GetResponse()
$contentLength = $headResponse.ContentLength
if ($contentLength -gt $MaxResponseSize) {
Write-Host " → Warning: Response too large ($([math]::Round($contentLength/1MB, 2)) MB). Skipping." -ForegroundColor Yellow
$requestResult.Error = "Response size ($([math]::Round($contentLength/1MB, 2)) MB) exceeds maximum allowed size ($([math]::Round($MaxResponseSize/1MB)) MB)"
$results += $requestResult
break
}
} catch {
# If HEAD request fails, proceed with normal GET but limit the read size
}
if ($IsSSL) {
$response = Invoke-WebRequest -Uri $currentUrl -MaximumRedirection 0 -SkipCertificateCheck -TimeoutSec $Timeout -MaximumRetryCount 0 -ErrorAction SilentlyContinue
} else {
$response = Invoke-WebRequest -Uri $currentUrl -MaximumRedirection 0 -TimeoutSec $Timeout -MaximumRetryCount 0 -ErrorAction SilentlyContinue
}
# Check actual content length after download
$actualLength = if ($response.RawContentLength -gt 0) {
$response.RawContentLength
} else {
[System.Text.Encoding]::UTF8.GetByteCount($response.Content)
}
if ($actualLength -gt $MaxResponseSize) {
Write-Host " → Warning: Response too large ($([math]::Round($actualLength/1MB, 2)) MB). Truncating." -ForegroundColor Yellow
$requestResult.Content = $response.Content.Substring(0, $MaxResponseSize)
$requestResult.Error = "Response truncated: exceeded maximum size of $([math]::Round($MaxResponseSize/1MB)) MB"
} else {
$requestResult.Content = $response.Content
Write-Host " → Received $([math]::Round($actualLength/1KB, 2)) KB"
}
# Try to get favicon
$favicon = Get-Favicon -BaseUrl $currentUrl -IsSSL $IsSSL -Timeout $Timeout
if ($favicon) {
$requestResult.Favicon = $favicon
}
$results += $requestResult
break
} catch {
if ($_.Exception.Response) {
$statusCode = [int]$_.Exception.Response.StatusCode
$requestResult.StatusCode = $statusCode
$requestResult.Headers = $_.Exception.Response.Headers
$requestResult.Error = $_.Exception.Message
# Try to get response content even for error status codes
try {
$stream = $_.Exception.Response.GetResponseStream()
$reader = New-Object System.IO.StreamReader($stream)
$responseContent = $reader.ReadToEnd()
$contentLength = [System.Text.Encoding]::UTF8.GetByteCount($responseContent)
$requestResult.Content = $responseContent
Write-Host " → Received $contentLength bytes"
} catch {
# If we can't read the response content, continue with error handling
}
$results += $requestResult
if ($statusCode -ge 300 -and $statusCode -lt 400) {
if ($RedirectCount -ge $MaxRedirects) {
Write-Host " → Warning: Maximum redirects reached" -ForegroundColor Yellow
break
}
$location = $_.Exception.Response.Headers.GetValues("Location")[0]
# Check if redirect goes from HTTP to HTTPS
$isHttpsRedirect = -not $IsSSL -and ($location.StartsWith("https://") -or $location.StartsWith("//"))
if ($isHttpsRedirect) {
Write-Host " → Not following redirect to HTTPS: $location" -ForegroundColor Yellow
break
}
if (-not $location.StartsWith("http")) {
$baseUri = [System.Uri]::new($currentUrl)
$location = "{0}://{1}:{2}{3}" -f $baseUri.Scheme, $baseUri.Host, $baseUri.Port, $location
}
$RedirectCount++
Write-Host " → Following redirect to: $location"
$currentUrl = $location
continue
}
Write-Host " → Warning: HTTP $statusCode - $($_.Exception.Message)" -ForegroundColor Yellow
} else {
$requestResult.Error = $_.Exception.Message
$results += $requestResult
Write-Host " → Warning: $($_.Exception.Message)" -ForegroundColor Yellow
}
break
}
}
return $results
}
# Enable all supported TLS/SSL versions
Set-MaximumTlsSupport
# Convert relative path to absolute
$XmlPath = if ([System.IO.Path]::IsPathRooted($XmlPath)) {
$XmlPath
} else {
Join-Path -Path (Get-Location) -ChildPath $XmlPath
}
if (-not (Test-Path $XmlPath)) {
Write-Error "XML file not found at path: $XmlPath"
exit 1
}
$timestamp = Get-Date -Format "yyyy-MM-dd_HH-mm-ss"
$outputDir = Join-Path -Path (Get-Location) -ChildPath "nmap_results_$timestamp"
New-Item -ItemType Directory -Path $outputDir -Force | Out-Null
try {
$settings = New-Object System.Xml.XmlReaderSettings
$settings.DtdProcessing = [System.Xml.DtdProcessing]::Parse
$reader = [System.Xml.XmlReader]::Create($XmlPath, $settings)
$processedHosts = 0
while ($reader.Read() -and ($processedHosts -lt $MaxHosts)) {
if ($reader.NodeType -eq [System.Xml.XmlNodeType]::Element -and $reader.Name -eq "host") {
$hostXml = [xml]$reader.ReadOuterXml()
$targetHost = $hostXml.host
$hostIP = $targetHost.address | Where-Object { $_.addrtype -eq "ipv4" } | Select-Object -ExpandProperty addr
$hostname = $targetHost.hostnames.hostname.name
Write-Host "`nScanning host: $hostIP $(if ($hostname) { "($hostname)" })"
# Determine the appropriate hostname/IP to use based on DNS resolution
$targetHostname = Get-TargetHostname -Hostname $hostname -IP $hostIP
# Get all open ports from the Nmap XML
$openPorts = @($targetHost.ports.port | Where-Object { $_.state.state -eq "open" })
if ($openPorts.Count -eq 0) {
Write-Host " No open ports found in Nmap results"
$ports = @()
} else {
$ports = $openPorts | ForEach-Object {
$isSSL = $false
if ($_.service.tunnel -eq "ssl" -or $_.service.name -match "ssl|tls|https" -or $_.service.script.output -match "ssl|tls") {
$isSSL = $true
}
Write-Host "Port $($_.portid) ($($_.service.name)$(if ($isSSL) { "/ssl" })):"
$webRequests = Fetch-Url -TargetHost $targetHostname -Port $_.portid -IsSSL $isSSL -Timeout $RequestTimeout
[PSCustomObject]@{
PortNumber = $_.portid
Protocol = $_.protocol
Service = $_.service.name
IsSSL = $isSSL
WebRequest = $webRequests
}
}
}
$dnsCheck = Test-DNSResolution -Hostname $hostname -IP $hostIP
$hostResult = [PSCustomObject]@{
IP = $hostIP
Hostname = $hostname
DNSResolution = $dnsCheck
OpenPorts = $ports
ScanTime = $targetHost.starttime
}
$outputFile = Join-Path -Path $outputDir -ChildPath "$hostIP.json"
$hostResult | ConvertTo-Json -Depth 10 | Out-File $outputFile
$processedHosts++
Write-Host "Completed host: $hostIP ($processedHosts of $MaxHosts)`n"
}
}
} catch {
Write-Error "Error processing NMAP XML: $_"
exit 1
} finally {
if ($reader) {
$reader.Close()
}
}
Write-Host "Processing complete. Results saved in: $outputDir"
Raw
Web-InterfaceIdentifier-0.4.2-CSVified.ps1
param(
[Parameter(Mandatory=$true)]
[string]$XmlPath,
[Parameter(Mandatory=$false)]
[int]$MaxHosts = 0, # Will be set to total hosts if 0
[Parameter(Mandatory=$false)]
[int]$RequestTimeout = 10,
[Parameter(Mandatory=$false)]
[int]$MaxResponseSize = 20MB
)
# Function to count total hosts in the XML file
function Get-TotalHosts {
param(
[string]$XmlPath
)
try {
$settings = New-Object System.Xml.XmlReaderSettings
$settings.DtdProcessing = [System.Xml.DtdProcessing]::Parse
$reader = [System.Xml.XmlReader]::Create($XmlPath, $settings)
$count = 0
while ($reader.Read()) {
if ($reader.NodeType -eq [System.Xml.XmlNodeType]::Element -and $reader.Name -eq "host") {
$count++
}
}
return $count
}
finally {
if ($reader) {
$reader.Close()
}
}
}
# Function to format time span
function Format-TimeSpan {
param (
[TimeSpan]$TimeSpan
)
if ($TimeSpan.TotalHours -ge 1) {
return "{0:0}h {1:0}m {2:0}s" -f $TimeSpan.Hours, $TimeSpan.Minutes, $TimeSpan.Seconds
}
elseif ($TimeSpan.TotalMinutes -ge 1) {
return "{0:0}m {1:0}s" -f $TimeSpan.Minutes, $TimeSpan.Seconds
}
else {
return "{0:0}s" -f $TimeSpan.TotalSeconds
}
}
function Get-AvailableTlsProtocols {
# Define possible protocols in order from newest to oldest
$possibleProtocols = @(
@{ Name = "Tls13"; Enum = "Tls13" },
@{ Name = "Tls12"; Enum = "Tls12" },
@{ Name = "Tls11"; Enum = "Tls11" },
@{ Name = "Tls"; Enum = "Tls" },
@{ Name = "Ssl3"; Enum = "Ssl3" },
@{ Name = "Ssl2"; Enum = "Ssl2" }
)
$availableProtocols = @()
# Dynamically check which protocols are defined in the current .NET version
foreach ($protocol in $possibleProtocols) {
try {
# Try to access the enum value
$enumValue = [System.Security.Authentication.SslProtocols]::($protocol.Enum)
$availableProtocols += @{ Name = $protocol.Name; Value = $enumValue }
Write-Host " → Protocol $($protocol.Name) is available"
}
catch {
Write-Host " → Protocol $($protocol.Name) is not supported on this system" -ForegroundColor Yellow
}
}
if ($availableProtocols.Count -eq 0) {
Write-Host " → Warning: No SSL/TLS protocols are available on this system" -ForegroundColor Red
throw "No SSL/TLS protocols available"
}
Write-Host " → Available protocols (from newest to oldest): $($availableProtocols.Name -join ', ')"
return $availableProtocols
}
# Detect available TLS/SSL protocols once at script startup
Write-Host "Detecting available TLS/SSL protocols..."
$GlobalAvailableTlsProtocols = Get-AvailableTlsProtocols
Write-Host "TLS/SSL protocol detection complete. Ready to scan hosts."
# Function to enable all locally supported TLS/SSL versions
function Set-MaximumTlsSupport {
$protocols = [enum]::GetValues([System.Net.SecurityProtocolType]) |
Where-Object { $_ -ne 'SystemDefault' }
$supportedProtocols = $protocols | ForEach-Object {
try {
[System.Net.ServicePointManager]::SecurityProtocol = $_
$_
} catch {
Write-Host "Protocol $_ not supported"
$null
}
} | Where-Object { $_ -ne $null }
$finalProtocol = [System.Net.SecurityProtocolType]($supportedProtocols -join ',')
[System.Net.ServicePointManager]::SecurityProtocol = $finalProtocol
Write-Host "Enabled protocols: $finalProtocol"
}
function Get-CertificateDetails {
param(
[string]$Hostname,
[int]$Port,
[int]$Timeout = 10,
[array]$AvailableProtocols = $GlobalAvailableTlsProtocols # Use the global variable by default
)
Write-Host (" → Retrieving SSL certificate from ${Hostname}:${Port}")
# Helper function to display inner exceptions
function Get-FullExceptionDetails {
param([Exception]$Exception)
$details = $Exception.Message
$currentEx = $Exception
# Traverse inner exceptions
while ($currentEx.InnerException) {
$currentEx = $currentEx.InnerException
$details += "`n → Inner Exception: $($currentEx.Message)"
# Check for specific exception types
if ($currentEx -is [System.Security.Authentication.AuthenticationException]) {
$details += " (Authentication error - likely certificate or protocol mismatch)"
}
elseif ($currentEx -is [System.IO.IOException]) {
$details += " (IO error - likely connection reset or timeout)"
}
}
return $details
}
# Helper function to check if an exception is timeout-related
function Test-IsTimeoutException {
param([string]$ExceptionMessage)
return ($ExceptionMessage -match "timed? out|timeout|aborted|could not establish trust|operation has timed out|connection failure|connection attempt failed")
}
# Method 1: Try all SSL/TLS protocols from newest to oldest
try {
Write-Host " → Method 1: Using TLS/SSL protocols from newest to oldest"
# Skip protocol detection - use the provided list
if ($AvailableProtocols.Count -eq 0) {
Write-Host " → Warning: No SSL/TLS protocols are available" -ForegroundColor Red
throw "No SSL/TLS protocols available"
}
# Try each protocol with different SNI options (empty or hostname)
$sniOptions = @("", $Hostname)
$timeoutEncountered = $false
foreach ($protocol in $AvailableProtocols) {
# If we already encountered a timeout, skip remaining protocols
if ($timeoutEncountered) {
Write-Host " → Skipping protocol $($protocol.Name) due to previous timeout" -ForegroundColor Yellow
continue
}
foreach ($sniValue in $sniOptions) {
$tcpClient = $null
$sslStream = $null
try {
Write-Host " → Trying handshake with $($protocol.Name), SNI='$sniValue'"
# Create a completely new TCP client for each attempt
$tcpClient = New-Object System.Net.Sockets.TcpClient
$tcpClient.ReceiveTimeout = $Timeout * 1000
$tcpClient.SendTimeout = $Timeout * 1000
# Set a connection timeout
$connectResult = $tcpClient.BeginConnect($Hostname, $Port, $null, $null)
$connectSuccess = $connectResult.AsyncWaitHandle.WaitOne($Timeout * 1000)
if (-not $connectSuccess) {
$timeoutEncountered = $true
throw "TCP connection timed out after $Timeout seconds"
}
# Complete the connection
$tcpClient.EndConnect($connectResult)
# Create SSL stream with callback that ignores ALL certificate errors
$sslStream = New-Object System.Net.Security.SslStream(
$tcpClient.GetStream(),
$false, # don't leave inner stream open
{ param($sender, $cert, $chain, $errors)
# Log the specific errors for troubleshooting
if ($errors -ne [System.Net.Security.SslPolicyErrors]::None) {
# This is just for debugging, we'll still return true
Write-Host " → Certificate errors: $errors" -ForegroundColor Yellow
}
return $true # Always accept the certificate regardless of errors
}
)
# Try SSL/TLS handshake
try {
# Use a reasonable timeout for the handshake (2x the regular timeout)
# We'll set read/write timeouts first
$sslStream.ReadTimeout = $Timeout * 2000
$sslStream.WriteTimeout = $Timeout * 2000
# Then try the handshake
$sslStream.AuthenticateAsClient($sniValue, $null, $protocol.Value, $false)
}
catch [Exception] {
# Get detailed exception info
$exDetails = Get-FullExceptionDetails -Exception $_.Exception
# Check if this is a timeout-related exception
if (Test-IsTimeoutException -ExceptionMessage $exDetails) {
$timeoutEncountered = $true
Write-Host " → Timeout encountered during handshake - will skip remaining protocol attempts" -ForegroundColor Yellow
}
throw $exDetails
}
Write-Host " → SSL/TLS handshake successful with $($protocol.Name)!" -ForegroundColor Green
# Get the certificate
$remoteCertificate = $sslStream.RemoteCertificate
if ($remoteCertificate -eq $null) {
Write-Host " → No certificate found despite successful handshake" -ForegroundColor Yellow
continue
}
# Convert to X509Certificate2 for more details
$x509cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2($remoteCertificate)
# Display basic certificate info
Write-Host " → Certificate details retrieved:" -ForegroundColor Green
Write-Host " Subject: $($x509cert.Subject)"
Write-Host " Issuer: $($x509cert.Issuer)"
Write-Host " Valid from: $($x509cert.NotBefore) to $($x509cert.NotAfter)"
# Extract Subject and Issuer components
$subjectParts = $x509cert.Subject -split ', '
$issuerParts = $x509cert.Issuer -split ', '
$subjectCN = ($subjectParts | Where-Object { $_ -match '^CN=' } | ForEach-Object { $_ -replace '^CN=' })[0]
$issuerCN = ($issuerParts | Where-Object { $_ -match '^CN=' } | ForEach-Object { $_ -replace '^CN=' })[0]
# Check if self-signed
$isSelfSigned = ($x509cert.Subject -eq $x509cert.Issuer)
if ($isSelfSigned) {
Write-Host " Self-signed: Yes" -ForegroundColor Yellow
} else {
Write-Host " Self-signed: No"
}
# Calculate days until expiration
$daysUntilExpiration = [math]::Round(($x509cert.NotAfter - (Get-Date)).TotalDays, 1)
# Return certificate details
return @{
Status = "Success"
Method = "DirectTLS"
Protocol = $protocol.Name
SNI = $sniValue
Subject = @{
CN = $subjectCN
O = ($subjectParts | Where-Object { $_ -match '^O=' } | ForEach-Object { $_ -replace '^O=' })
OU = ($subjectParts | Where-Object { $_ -match '^OU=' } | ForEach-Object { $_ -replace '^OU=' })
C = ($subjectParts | Where-Object { $_ -match '^C=' } | ForEach-Object { $_ -replace '^C=' })
S = ($subjectParts | Where-Object { $_ -match '^S=' -or $_ -match '^ST=' } | ForEach-Object { $_ -replace '^S=|^ST=' })
}
Issuer = @{
CN = $issuerCN
O = ($issuerParts | Where-Object { $_ -match '^O=' } | ForEach-Object { $_ -replace '^O=' })
OU = ($issuerParts | Where-Object { $_ -match '^OU=' } | ForEach-Object { $_ -replace '^OU=' })
C = ($issuerParts | Where-Object { $_ -match '^C=' } | ForEach-Object { $_ -replace '^C=' })
S = ($issuerParts | Where-Object { $_ -match '^S=' -or $_ -match '^ST=' } | ForEach-Object { $_ -replace '^S=|^ST=' })
}
Validity = @{
NotBefore = $x509cert.NotBefore.ToString('yyyy-MM-dd HH:mm:ss')
NotAfter = $x509cert.NotAfter.ToString('yyyy-MM-dd HH:mm:ss')
DaysUntilExpiration = $daysUntilExpiration
}
SerialNumber = $x509cert.SerialNumber
Thumbprint = @{
SHA256 = $x509cert.GetCertHashString('SHA256')
SHA1 = $x509cert.GetCertHashString('SHA1')
}
SignatureAlgorithm = $x509cert.SignatureAlgorithm.FriendlyName
Version = $x509cert.Version
SelfSigned = $isSelfSigned
TLSSupport = @{
SupportedProtocols = @($protocol.Name)
}
}
}
catch {
$exMessage = $_
Write-Host " → Attempt failed: $exMessage" -ForegroundColor Yellow
# Check if this is a timeout-related exception
if (Test-IsTimeoutException -ExceptionMessage $exMessage) {
$timeoutEncountered = $true
Write-Host " → Timeout detected - will skip remaining protocol attempts" -ForegroundColor Yellow
break # Exit this SNI loop
}
# Clean up resources properly
if ($sslStream) {
try { $sslStream.Dispose() } catch {}
}
if ($tcpClient) {
try { $tcpClient.Dispose() } catch {}
}
}
}
# If we encountered a timeout in the inner loop, break out of the outer loop too
if ($timeoutEncountered) {
break
}
}
if ($timeoutEncountered) {
Write-Host " → Skipping remaining protocol attempts due to timeout" -ForegroundColor Yellow
} else {
Write-Host " → All SSL/TLS protocol and SNI combinations failed" -ForegroundColor Yellow
}
}
catch {
$exDetails = Get-FullExceptionDetails -Exception $_.Exception
Write-Host " → Method 1 failed: $exDetails" -ForegroundColor Yellow
}
# Method 2: WebRequest with custom TLS setting
# Only try Method 2 if we didn't encounter a timeout in Method 1
$skipMethod2 = $false
if (Test-Path variable:timeoutEncountered) {
$skipMethod2 = $timeoutEncountered
}
if ($skipMethod2) {
Write-Host " → Skipping Method 2 due to timeout in Method 1" -ForegroundColor Yellow
} else {
try {
Write-Host " → Method 2: Using WebRequest with custom TLS settings"
# Save original settings
$originalProtocol = [System.Net.ServicePointManager]::SecurityProtocol
$originalCallback = [System.Net.ServicePointManager]::ServerCertificateValidationCallback
try {
# Try to use all possible TLS/SSL protocols
# We'll use reflection to get all possible values for SecurityProtocolType
$allValues = 0
# Get all protocols via reflection from SecurityProtocolType enum
$protocolType = [System.Net.SecurityProtocolType]
$protocolFields = $protocolType.GetFields() | Where-Object { $_.IsStatic -and $_.IsLiteral }
foreach ($field in $protocolFields) {
try {
$value = $field.GetValue($null)
Write-Host " → Adding protocol $($field.Name) ($value)"
$allValues = $allValues -bor $value
} catch {
Write-Host " → Couldn't add protocol $($field.Name): $($_.Exception.Message)" -ForegroundColor Yellow
}
}
# If we couldn't get any protocols, fall back to hardcoded combination
if ($allValues -eq 0) {
Write-Host " → Using fallback protocol combination" -ForegroundColor Yellow
try {
$allValues = [System.Net.SecurityProtocolType]::Tls12 -bor
[System.Net.SecurityProtocolType]::Tls11 -bor
[System.Net.SecurityProtocolType]::Tls -bor
[System.Net.SecurityProtocolType]::Ssl3
}
catch {
# If even that fails, try just TLS 1.2
$allValues = [System.Net.SecurityProtocolType]::Tls12
}
}
# Apply our protocol selection
[System.Net.ServicePointManager]::SecurityProtocol = $allValues
# Accept all certificates regardless of errors
[System.Net.ServicePointManager]::ServerCertificateValidationCallback = {
param($sender, $cert, $chain, $errors)
# Log the specific certificate errors for diagnostics
if ($errors -ne [System.Net.Security.SslPolicyErrors]::None) {
Write-Host " → Certificate validation errors: $errors" -ForegroundColor Yellow
}
return $true
}
# Create HTTP request
$url = "https://${Hostname}:${Port}"
$webRequest = [System.Net.WebRequest]::Create($url)
$webRequest.Method = "HEAD"
$webRequest.Timeout = $Timeout * 1000
$webRequest.AllowAutoRedirect = $false
# Attempt to get response (may fail but we might still get the certificate)
try {
$response = $webRequest.GetResponse()
$response.Close()
}
catch [System.Net.WebException] {
# Extract detailed information from the WebException
$ex = $_.Exception
$statusCode = -1
$statusDesc = "Unknown"
if ($ex.Response -ne $null) {
$statusCode = [int]$ex.Response.StatusCode
$statusDesc = $ex.Response.StatusDescription
}
Write-Host " → WebException: Status Code=$statusCode, Description=$statusDesc" -ForegroundColor Yellow
# Check if we have inner exception details
if ($ex.InnerException) {
$innerExMsg = Get-FullExceptionDetails -Exception $ex.InnerException
Write-Host " → Inner exception details: $innerExMsg" -ForegroundColor Yellow
# Check if this is a timeout related exception
if (Test-IsTimeoutException -ExceptionMessage $innerExMsg) {
throw "Timeout encountered in WebRequest method"
}
}
# Check if this is a timeout-related exception directly
if (Test-IsTimeoutException -ExceptionMessage $ex.Message) {
throw "Timeout encountered in WebRequest method"
}
Write-Host " → Still checking for certificate..." -ForegroundColor Yellow
}
# Try to get certificate from the ServicePoint
$cert = $webRequest.ServicePoint.Certificate
if ($cert -eq $null) {
throw "No certificate found in ServicePoint"
}
# Convert to X509Certificate2
$x509cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2($cert)
# Display basic certificate info
Write-Host " → Certificate details retrieved via WebRequest:" -ForegroundColor Green
Write-Host " Subject: $($x509cert.Subject)"
Write-Host " Issuer: $($x509cert.Issuer)"
Write-Host " Valid from: $($x509cert.NotBefore) to $($x509cert.NotAfter)"
# Extract Subject and Issuer components
$subjectParts = $x509cert.Subject -split ', '
$issuerParts = $x509cert.Issuer -split ', '
$subjectCN = ($subjectParts | Where-Object { $_ -match '^CN=' } | ForEach-Object { $_ -replace '^CN=' })[0]
$issuerCN = ($issuerParts | Where-Object { $_ -match '^CN=' } | ForEach-Object { $_ -replace '^CN=' })[0]
# Calculate days until expiration
$daysUntilExpiration = [math]::Round(($x509cert.NotAfter - (Get-Date)).TotalDays, 1)
# Return certificate details
return @{
Status = "Success"
Method = "WebRequest"
Protocol = "Unknown (WebRequest)"
Subject = @{
CN = $subjectCN
O = ($subjectParts | Where-Object { $_ -match '^O=' } | ForEach-Object { $_ -replace '^O=' })
OU = ($subjectParts | Where-Object { $_ -match '^OU=' } | ForEach-Object { $_ -replace '^OU=' })
C = ($subjectParts | Where-Object { $_ -match '^C=' } | ForEach-Object { $_ -replace '^C=' })
S = ($subjectParts | Where-Object { $_ -match '^S=' -or $_ -match '^ST=' } | ForEach-Object { $_ -replace '^S=|^ST=' })
}
Issuer = @{
CN = $issuerCN
O = ($issuerParts | Where-Object { $_ -match '^O=' } | ForEach-Object { $_ -replace '^O=' })
OU = ($issuerParts | Where-Object { $_ -match '^OU=' } | ForEach-Object { $_ -replace '^OU=' })
C = ($issuerParts | Where-Object { $_ -match '^C=' } | ForEach-Object { $_ -replace '^C=' })
S = ($issuerParts | Where-Object { $_ -match '^S=' -or $_ -match '^ST=' } | ForEach-Object { $_ -replace '^S=|^ST=' })
}
Validity = @{
NotBefore = $x509cert.NotBefore.ToString('yyyy-MM-dd HH:mm:ss')
NotAfter = $x509cert.NotAfter.ToString('yyyy-MM-dd HH:mm:ss')
DaysUntilExpiration = $daysUntilExpiration
}
SerialNumber = $x509cert.SerialNumber
Thumbprint = @{
SHA256 = $x509cert.GetCertHashString('SHA256')
SHA1 = $x509cert.GetCertHashString('SHA1')
}
SignatureAlgorithm = $x509cert.SignatureAlgorithm.FriendlyName
Version = $x509cert.Version
SelfSigned = ($x509cert.Subject -eq $x509cert.Issuer)
TLSSupport = @{
SupportedProtocols = @("Unknown - WebRequest method")
}
}
}
finally {
# Restore original settings
[System.Net.ServicePointManager]::SecurityProtocol = $originalProtocol
[System.Net.ServicePointManager]::ServerCertificateValidationCallback = $originalCallback
}
}
catch {
$exDetails = Get-FullExceptionDetails -Exception $_.Exception
# Check if this was a timeout
if (Test-IsTimeoutException -ExceptionMessage $exDetails) {
Write-Host " → Method 2 timed out: $exDetails" -ForegroundColor Yellow
} else {
Write-Host " → Method 2 failed: $exDetails" -ForegroundColor Yellow
}
}
}
# If all methods fail, return a consistent failure object
Write-Host " → Certificate retrieval failed - all methods exhausted" -ForegroundColor Yellow
# Return a failure object
return @{
Status = "Failed"
Method = "AllMethodsFailed"
Error = "Could not retrieve certificate from ${Hostname}:${Port} after trying all methods"
Subject = @{ CN = "Unknown"; O = @(); OU = @() }
Issuer = @{ CN = "Unknown"; O = @(); OU = @() }
Validity = @{
NotBefore = "Unknown"
NotAfter = "Unknown"
DaysUntilExpiration = $null
}
SerialNumber = "Unknown"
SelfSigned = $null
TLSSupport = @{
SupportedProtocols = @()
}
}
}
function Test-DNSResolution {
param(
[string]$Hostname,
[string]$IP
)
try {
$dnsResults = [System.Net.Dns]::GetHostEntry($Hostname)
$ipAddresses = $dnsResults.AddressList | ForEach-Object { $_.IPAddressToString }
# Check if the IP we have matches any of the DNS results
$ipMatch = $ipAddresses -contains $IP
return @{
Resolves = $true
IPAddresses = $ipAddresses
IPMatch = $ipMatch
Error = $null
}
}
catch {
return @{
Resolves = $false
IPAddresses = @()
IPMatch = $false
Error = $_.Exception.Message
}
}
}
# Enhanced URL sanitization function
function Sanitize-Url {
param(
[string]$Url
)
if ([string]::IsNullOrWhiteSpace($Url)) {
return $Url
}
# Remove any backslashes from the URL, but preserve double forward slashes
$Url = $Url.Replace('\', '/')
# Fix any double forward slashes that aren't part of the protocol
if ($Url -match '^(?:https?:\/\/)') {
$protocol = $matches[0]
$rest = $Url.Substring($protocol.Length)
$rest = $rest -replace '//+', '/'
$Url = $protocol + $rest
}
# Ensure no trailing slashes in hostname portion
if ($Url -match '^(https?:\/\/[^\/]+)(.*)$') {
$hostname = $matches[1].TrimEnd('/')
$path = $matches[2]
$Url = $hostname + $path
}
return $Url.Trim()
}
function Get-TargetHostname {
param(
[string]$Hostname,
[string]$IP
)
if (-not $Hostname) {
Write-Host " No hostname provided, using IP: $IP"
return $IP
}
$dnsCheck = Test-DNSResolution -Hostname $Hostname -IP $IP
if (-not $dnsCheck.Resolves) {
Write-Host " Hostname '$Hostname' does not resolve in DNS, using IP: $IP"
return $IP
}
if (-not $dnsCheck.IPMatch) {
Write-Host " Warning: Hostname '$Hostname' resolves to different IPs: $($dnsCheck.IPAddresses -join ', ')"
Write-Host " Target IP '$IP' not in DNS results, using IP instead of hostname"
return $IP
}
Write-Host " Hostname '$Hostname' resolves correctly to IP: $IP"
return $Hostname
}
function Get-Favicon {
param(
[string]$BaseUrl,
[bool]$IsSSL,
[int]$Timeout = 10
)
try {
# First try standard /favicon.ico
$faviconUrl = "{0}/favicon.ico" -f $BaseUrl
Write-Host " → Attempting: $faviconUrl"
if ($IsSSL) {
$response = Invoke-WebRequest -Uri $faviconUrl -MaximumRedirection 0 -SkipCertificateCheck -TimeoutSec $Timeout -ErrorAction SilentlyContinue
} else {
$response = Invoke-WebRequest -Uri $faviconUrl -MaximumRedirection 0 -TimeoutSec $Timeout -ErrorAction SilentlyContinue
}
if ($response.StatusCode -eq 200 -and $response.RawContentLength -gt 0) {
return [Convert]::ToBase64String($response.Content)
}
} catch {
# Don't output anything - the error is expected for sites without favicon.ico
}
return $null
}
# Enable all supported TLS/SSL versions
Set-MaximumTlsSupport
function Sanitize-Hostname {
param(
[string]$Hostname
)
return $Hostname.Replace('\', '').Trim()
}
function Fetch-Url {
param(
[string]$TargetHost,
[int]$Port,
[bool]$IsSSL,
[int]$Timeout = 10
)
$MaxRedirects = 10
$RedirectCount = 0
$results = @()
$protocol = if ($IsSSL) { "https" } else { "http" }
$currentUrl = "${protocol}://${TargetHost}:${Port}"
$originalHost = $TargetHost
$originalPort = $Port
# Helper function to check if an exception is timeout-related
function Test-IsTimeoutException {
param([string]$ExceptionMessage)
return ($ExceptionMessage -match "timed? out|timeout|aborted|could not establish trust|operation has timed out|connection failure|connection attempt failed")
}
# Get certificate details early for SSL connections
$certDetails = $null
if ($IsSSL) {
Write-Host " → Getting certificate details for ${TargetHost}:${Port}"
$certDetails = Get-CertificateDetails -Hostname $TargetHost -Port $Port -Timeout $Timeout -AvailableProtocols $GlobalAvailableTlsProtocols
}
# Create the initial result object with certificate info
$requestResult = [ordered]@{
Url = $currentUrl
RedirectDepth = $RedirectCount
StatusCode = $null
Message = $null # Changed from "Error" to "Message"
Headers = @{}
Content = $null
Favicon = $null
Title = $null
RedirectedRequest = $null
Certificate = $certDetails
}
# Check if we should skip HTTPS connection due to certificate failure or timeout
$skipHttpsRequest = $false
if ($IsSSL) {
# Check if certificate retrieval failed or timed out
if (-not $certDetails -or $certDetails.Status -eq "Failed") {
# Check if the error indicates a timeout
$timeoutError = $false
if ($certDetails -and $certDetails.Error -and (Test-IsTimeoutException -ExceptionMessage $certDetails.Error)) {
$timeoutError = $true
Write-Host " → Certificate retrieval timed out - skipping HTTPS connection attempt" -ForegroundColor Yellow
$requestResult.Message = "HTTPS connection skipped due to certificate retrieval timeout"
} else {
Write-Host " → Certificate retrieval failed - skipping HTTPS connection attempt" -ForegroundColor Yellow
$requestResult.Message = "HTTPS connection skipped due to certificate retrieval failure"
}
$results += $requestResult
$skipHttpsRequest = $true
}
}
# Only attempt HTTP/HTTPS connections if not skipped
if (-not $skipHttpsRequest) {
while ($true) {
try {
# Sanitize the current URL before making the request
$currentUrl = Sanitize-Url -Url $currentUrl
# Update URL in the result object
$requestResult.Url = $currentUrl
Write-Host " → Attempting: $currentUrl"
# Create WebRequest to check content length first
$webRequest = [System.Net.WebRequest]::Create($currentUrl)
$webRequest.Method = "HEAD"
$webRequest.Timeout = $Timeout * 1000
# Only set certificate validation callback for HTTPS requests
if ($IsSSL) {
[System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true }
}
try {
$headResponse = $webRequest.GetResponse()
$contentLength = $headResponse.ContentLength
if ($contentLength -gt $MaxResponseSize) {
Write-Host " → Warning: Response too large ($([math]::Round($contentLength/1MB, 2)) MB). Skipping." -ForegroundColor Yellow
$requestResult.Message = "Response size ($([math]::Round($contentLength/1MB, 2)) MB) exceeds maximum allowed size ($([math]::Round($MaxResponseSize/1MB)) MB)"
$results += $requestResult
break
}
} catch {
# Check if this is a timeout exception
if (Test-IsTimeoutException -ExceptionMessage $_.Exception.Message) {
Write-Host " → HEAD request timed out" -ForegroundColor Yellow
$requestResult.Message = "HEAD request timed out: $($_.Exception.Message)"
$results += $requestResult
break
}
# If HEAD request fails with non-timeout error, proceed with normal GET but limit the read size
Write-Host " → HEAD request failed: $($_.Exception.Message). Trying GET instead." -ForegroundColor Yellow
}
# Use different Invoke-WebRequest parameters based on PowerShell version
$psVersion = $PSVersionTable.PSVersion.Major
$webRequestParams = @{
Uri = $currentUrl
MaximumRedirection = 0
TimeoutSec = $Timeout
MaximumRetryCount = 0
ErrorAction = 'SilentlyContinue'
}
# Different handling for HTTP and HTTPS
if ($IsSSL) {
# HTTPS handling
if ($psVersion -ge 6) {
# For PowerShell 6+ we can use SkipCertificateCheck
$webRequestParams.Add('SkipCertificateCheck', $true)
$response = Invoke-WebRequest @webRequestParams
} else {
# For older PowerShell versions
try {
# Temporarily set certificate callback
$originalCallback = [System.Net.ServicePointManager]::ServerCertificateValidationCallback
[System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true }
$response = Invoke-WebRequest @webRequestParams
} finally {
# Restore original callback
[System.Net.ServicePointManager]::ServerCertificateValidationCallback = $originalCallback
}
}
} else {
# HTTP handling - no SSL/TLS specific code
$response = Invoke-WebRequest @webRequestParams
}
# Check actual content length after download
$actualLength = if ($response.RawContentLength -gt 0) {
$response.RawContentLength
} else {
[System.Text.Encoding]::UTF8.GetByteCount($response.Content)
}
if ($actualLength -gt $MaxResponseSize) {
Write-Host " → Warning: Response too large ($([math]::Round($actualLength/1MB, 2)) MB). Truncating." -ForegroundColor Yellow
$requestResult.Content = $response.Content.Substring(0, $MaxResponseSize)
$requestResult.Message = "Response truncated: exceeded maximum size of $([math]::Round($MaxResponseSize/1MB)) MB"
} else {
$requestResult.Content = $response.Content
Write-Host " → Received $([math]::Round($actualLength/1KB, 2)) KB"
}
# Extract title from HTML content if available
if ($requestResult.Content) {
if ($requestResult.Content -match '<title[^>]*>(.*?)</title>') {
$requestResult.Title = $matches[1].Trim()
Write-Host " → Page title: $($requestResult.Title)"
} else {
$requestResult.Title = $null
Write-Host " → No title found in page"
}
}
# Try to get favicon
$favicon = Get-Favicon -BaseUrl $currentUrl -IsSSL $IsSSL -Timeout $Timeout
if ($favicon) {
$requestResult.Favicon = $favicon
}
$results += $requestResult
break
} catch {
if ($_.Exception.Response) {
$statusCode = [int]$_.Exception.Response.StatusCode
$requestResult.StatusCode = $statusCode
$requestResult.Headers = $_.Exception.Response.Headers
# Instead of treating redirects as errors, treat them as normal status codes
if ($statusCode -ge 300 -and $statusCode -lt 400) {
$requestResult.Message = "Redirect: $statusCode $($_.Exception.Response.StatusDescription)"
} else {
$requestResult.Message = "$statusCode $($_.Exception.Response.StatusDescription)"
}
# Try to get response content even for error status codes
try {
$stream = $_.Exception.Response.GetResponseStream()
$reader = New-Object System.IO.StreamReader($stream)
$responseContent = $reader.ReadToEnd()
$contentLength = [System.Text.Encoding]::UTF8.GetByteCount($responseContent)
$requestResult.Content = $responseContent
Write-Host " → Received $contentLength bytes"
# Extract title from HTML content if available
if ($responseContent -match '<title[^>]*>(.*?)</title>') {
$requestResult.Title = $matches[1].Trim()
Write-Host " → Page title: $($requestResult.Title)"
}
} catch {
# If we can't read the response content, continue with error handling
}
$results += $requestResult
if ($statusCode -ge 300 -and $statusCode -lt 400) {
if ($RedirectCount -ge $MaxRedirects) {
Write-Host " → Warning: Maximum redirects reached" -ForegroundColor Yellow
break
}
$location = $_.Exception.Response.Headers.GetValues("Location")[0]
$location = Sanitize-Url -Url $location
# Check if redirect goes from HTTP to HTTPS
$isHttpsRedirect = -not $IsSSL -and ($location.StartsWith("https://") -or $location.StartsWith("//"))
if ($isHttpsRedirect) {
Write-Host " → Not following redirect to HTTPS: $location" -ForegroundColor Yellow
break
}
try {
if ($location.StartsWith("http://") -or $location.StartsWith("https://")) {
$uri = [System.Uri]::new($location)
if ($uri.Host -ne $originalHost -or $uri.Port -ne $originalPort) {
Write-Host " → Not following redirect to different host/port: $location" -ForegroundColor Yellow
break
}
$currentUrl = $location
}
elseif ($location.StartsWith("//")) {
$currentUrl = "${protocol}:${location}"
}
elseif ($location.StartsWith("/")) {
$currentUrl = "${protocol}://${originalHost}:${originalPort}${location}"
}
else {
$baseUri = [System.Uri]$currentUrl
$currentUrl = [System.Uri]::new($baseUri, $location).ToString()
}
$RedirectCount++
Write-Host " → Following redirect to: $currentUrl"
# Create a new request result for this redirect
$requestResult = [ordered]@{
Url = $currentUrl
RedirectDepth = $RedirectCount
StatusCode = $null
Message = $null # Changed from "Error" to "Message"
Headers = @{}
Content = $null
Favicon = $null
Title = $null
RedirectedRequest = $null
Certificate = $certDetails
}
continue
}
catch {
Write-Host " → Warning: Invalid redirect URL: $location" -ForegroundColor Yellow
break
}
}
Write-Host " → Status: HTTP $statusCode - $($_.Exception.Response.StatusDescription)" -ForegroundColor Yellow
} else {
# Check for timeout exception
if (Test-IsTimeoutException -ExceptionMessage $_.Exception.Message) {
Write-Host " → Request timed out: $($_.Exception.Message)" -ForegroundColor Yellow
} else {
Write-Host " → Warning: $($_.Exception.Message)" -ForegroundColor Yellow
}
$requestResult.Message = $_.Exception.Message
$results += $requestResult
}
break
}
}
}
return $results
}
# Function to export JSON data to CSV in both wide and long formats
function Append-HostToCSV {
param(
[string]$OutputDir,
[string]$WideCSVPath,
[string]$LongCSVPath,
[PSCustomObject]$HostData
)
# Helper function to extract title from HTML content
function Get-HtmlTitle {
param([string]$HtmlContent)
if ([string]::IsNullOrWhiteSpace($HtmlContent)) {
return $null
}
try {
# Try to match the title tag
if ($HtmlContent -match '<title[^>]*>(.*?)</title>') {
return $matches[1].Trim()
}
} catch {
# If there's any error in regex processing, return null
return $null
}
return $null
}
# Create wide format row(s)
$wideRows = @()
foreach ($port in $HostData.OpenPorts) {
$rowData = [ordered]@{
IP = $HostData.IP
Hostname = $HostData.Hostname
DNSResolution = $HostData.DNSResolution.Resolves
DNSMatchesIP = $HostData.DNSResolution.IPMatch
Port = $port.PortNumber
Protocol = $port.Protocol
Service = $port.Service
IsSSL = $port.IsSSL
TotalRedirects = ($port.WebRequest | Measure-Object).Count
}
# Add redirect information
for ($i = 0; $i -lt $port.WebRequest.Count; $i++) {
$request = $port.WebRequest[$i]
$prefix = "Redirect$($i+1)_"
# Extract title from HTML content
$title = Get-HtmlTitle -HtmlContent $request.Content
# Add request details
$rowData["${prefix}URL"] = $request.Url
$rowData["${prefix}Message"] = $request.Message
$rowData["${prefix}Error"] = $request.Error
$rowData["${prefix}HasContent"] = if ($request.Content) { $true } else { $false }
$rowData["${prefix}ContentLength"] = if ($request.Content) { $request.Content.Length } else { 0 }
$rowData["${prefix}HasFavicon"] = if ($request.Favicon) { $true } else { $false }
$rowData["${prefix}Content"] = $request.Content
$rowData["${prefix}Favicon"] = $request.Favicon
$rowData["${prefix}Title"] = $title
# Add headers if available
if ($request.Headers -and $request.Headers.Count -gt 0) {
$headerString = ($request.Headers.GetEnumerator() | ForEach-Object { "$($_.Key): $($_.Value)" }) -join "; "
$rowData["${prefix}Headers"] = $headerString
} else {
$rowData["${prefix}Headers"] = ""
}
# Certificate information (if SSL)
if ($port.IsSSL -and $request.Certificate) {
$cert = $request.Certificate
# Add certificate status and method
$rowData["${prefix}CertStatus"] = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
$rowData["${prefix}CertMethod"] = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
$rowData["${prefix}CertError"] = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }
# Add certificate details if available
if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
$rowData["${prefix}CertSubjectCN"] = $cert.Subject.CN
$rowData["${prefix}CertIssuerCN"] = $cert.Issuer.CN
$rowData["${prefix}CertValidFrom"] = $cert.Validity.NotBefore
$rowData["${prefix}CertValidTo"] = $cert.Validity.NotAfter
$rowData["${prefix}CertDaysUntilExpiration"] = $cert.Validity.DaysUntilExpiration
$rowData["${prefix}CertSignatureAlgorithm"] = $cert.SignatureAlgorithm
$rowData["${prefix}CertSelfSigned"] = $cert.SelfSigned
$rowData["${prefix}CertSupportedTLSVersions"] = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
($cert.TLSSupport.SupportedProtocols -join ", ")
} else {
"Unknown"
}
# Add thumbprints
if ($cert.Thumbprint) {
$rowData["${prefix}CertThumbprintSHA1"] = $cert.Thumbprint.SHA1
$rowData["${prefix}CertThumbprintSHA256"] = $cert.Thumbprint.SHA256
}
# Include the serial number
$rowData["${prefix}CertSerialNumber"] = $cert.SerialNumber
}
}
}
$wideRows += [PSCustomObject]$rowData
}
# Create long format row(s)
$longRows = @()
foreach ($port in $HostData.OpenPorts) {
# For each request in the redirect chain
for ($i = 0; $i -lt $port.WebRequest.Count; $i++) {
$request = $port.WebRequest[$i]
# Extract title from HTML content
$title = Get-HtmlTitle -HtmlContent $request.Content
# Create a new ordered dictionary for each row
$rowData = [ordered]@{
IP = $HostData.IP
Hostname = $HostData.Hostname
DNSResolution = $HostData.DNSResolution.Resolves
DNSMatchesIP = $HostData.DNSResolution.IPMatch
Port = $port.PortNumber
Protocol = $port.Protocol
Service = $port.Service
IsSSL = $port.IsSSL
RedirectIndex = $i
URL = $request.Url
StatusCode = $request.StatusCode
Message = $request.Message
HasContent = if ($request.Content) { $true } else { $false }
ContentLength = if ($request.Content) { $request.Content.Length } else { 0 }
HasFavicon = if ($request.Favicon) { $true } else { $false }
Content = $request.Content
Favicon = $request.Favicon
Title = $title
}
# Add headers if available
if ($request.Headers -and $request.Headers.Count -gt 0) {
$headerString = ($request.Headers.GetEnumerator() | ForEach-Object { "$($_.Key): $($_.Value)" }) -join "; "
$rowData["Headers"] = $headerString
} else {
$rowData["Headers"] = ""
}
# Certificate information (if SSL)
if ($port.IsSSL -and $request.Certificate) {
$cert = $request.Certificate
# Add certificate status and method
$rowData["CertStatus"] = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
$rowData["CertMethod"] = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
$rowData["CertError"] = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }
# Add certificate details if available
if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
$rowData["CertSubjectCN"] = $cert.Subject.CN
$rowData["CertIssuerCN"] = $cert.Issuer.CN
$rowData["CertValidFrom"] = $cert.Validity.NotBefore
$rowData["CertValidTo"] = $cert.Validity.NotAfter
$rowData["CertDaysUntilExpiration"] = $cert.Validity.DaysUntilExpiration
$rowData["CertSignatureAlgorithm"] = $cert.SignatureAlgorithm
$rowData["CertSelfSigned"] = $cert.SelfSigned
$rowData["CertSupportedTLSVersions"] = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
($cert.TLSSupport.SupportedProtocols -join ", ")
} else {
"Unknown"
}
# Add thumbprints
if ($cert.Thumbprint) {
$rowData["CertThumbprintSHA1"] = $cert.Thumbprint.SHA1
$rowData["CertThumbprintSHA256"] = $cert.Thumbprint.SHA256
}
# Include the serial number
$rowData["CertSerialNumber"] = $cert.SerialNumber
}
}
$longRows += [PSCustomObject]$rowData
}
}
# Append to CSV files
try {
# If file doesn't exist, create it with headers
if (-not (Test-Path $WideCSVPath)) {
$wideRows | Export-Csv -Path $WideCSVPath -NoTypeInformation -Encoding UTF8
} else {
$wideRows | Export-Csv -Path $WideCSVPath -NoTypeInformation -Encoding UTF8 -Append -Force
}
if (-not (Test-Path $LongCSVPath)) {
$longRows | Export-Csv -Path $LongCSVPath -NoTypeInformation -Encoding UTF8
} else {
$longRows | Export-Csv -Path $LongCSVPath -NoTypeInformation -Encoding UTF8 -Append -Force
}
Write-Host "CSV data appended for host: $($HostData.IP)"
}
catch {
Write-Host "Error appending CSV data for host $($HostData.IP): $_" -ForegroundColor Red
}
}
# Add this function to the script
function Append-HostToCSV {
param(
[string]$OutputDir,
[string]$WideCSVPath,
[string]$LongCSVPath,
[PSCustomObject]$HostData
)
# Helper function to extract title from HTML content
function Get-HtmlTitle {
param([string]$HtmlContent)
if ([string]::IsNullOrWhiteSpace($HtmlContent)) {
return $null
}
try {
# Try to match the title tag
if ($HtmlContent -match '<title[^>]*>(.*?)</title>') {
return $matches[1].Trim()
}
} catch {
# If there's any error in regex processing, return null
return $null
}
return $null
}
# Create wide format row(s)
$wideRows = @()
foreach ($port in $HostData.OpenPorts) {
$rowData = [ordered]@{
IP = $HostData.IP
Hostname = $HostData.Hostname
DNSResolution = $HostData.DNSResolution.Resolves
DNSMatchesIP = $HostData.DNSResolution.IPMatch
Port = $port.PortNumber
Protocol = $port.Protocol
Service = $port.Service
IsSSL = $port.IsSSL
TotalRedirects = ($port.WebRequest | Measure-Object).Count
}
# Add redirect information
for ($i = 0; $i -lt $port.WebRequest.Count; $i++) {
$request = $port.WebRequest[$i]
$prefix = "Redirect$($i+1)_"
# Extract title from HTML content
$title = Get-HtmlTitle -HtmlContent $request.Content
# Add request details
$rowData["${prefix}URL"] = $request.Url
$rowData["${prefix}StatusCode"] = $request.StatusCode
$rowData["${prefix}Error"] = $request.Error
$rowData["${prefix}HasContent"] = if ($request.Content) { $true } else { $false }
$rowData["${prefix}ContentLength"] = if ($request.Content) { $request.Content.Length } else { 0 }
$rowData["${prefix}HasFavicon"] = if ($request.Favicon) { $true } else { $false }
$rowData["${prefix}Content"] = $request.Content
$rowData["${prefix}Favicon"] = $request.Favicon
$rowData["${prefix}Title"] = $title
# Add headers if available
if ($request.Headers -and $request.Headers.Count -gt 0) {
$headerString = ($request.Headers.GetEnumerator() | ForEach-Object { "$($_.Key): $($_.Value)" }) -join "; "
$rowData["${prefix}Headers"] = $headerString
} else {
$rowData["${prefix}Headers"] = ""
}
# Certificate information (if SSL)
if ($port.IsSSL -and $request.Certificate) {
$cert = $request.Certificate
# Add certificate status and method
$rowData["${prefix}CertStatus"] = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
$rowData["${prefix}CertMethod"] = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
$rowData["${prefix}CertError"] = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }
# Add certificate details if available
if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
$rowData["${prefix}CertSubjectCN"] = $cert.Subject.CN
$rowData["${prefix}CertIssuerCN"] = $cert.Issuer.CN
$rowData["${prefix}CertValidFrom"] = $cert.Validity.NotBefore
$rowData["${prefix}CertValidTo"] = $cert.Validity.NotAfter
$rowData["${prefix}CertDaysUntilExpiration"] = $cert.Validity.DaysUntilExpiration
$rowData["${prefix}CertSignatureAlgorithm"] = $cert.SignatureAlgorithm
$rowData["${prefix}CertSelfSigned"] = $cert.SelfSigned
$rowData["${prefix}CertSupportedTLSVersions"] = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
($cert.TLSSupport.SupportedProtocols -join ", ")
} else {
"Unknown"
}
# Add thumbprints
if ($cert.Thumbprint) {
$rowData["${prefix}CertThumbprintSHA1"] = $cert.Thumbprint.SHA1
$rowData["${prefix}CertThumbprintSHA256"] = $cert.Thumbprint.SHA256
}
# Include the serial number
$rowData["${prefix}CertSerialNumber"] = $cert.SerialNumber
}
}
}
$wideRows += [PSCustomObject]$rowData
}
# Create long format row(s)
$longRows = @()
foreach ($port in $HostData.OpenPorts) {
# For each request in the redirect chain
for ($i = 0; $i -lt $port.WebRequest.Count; $i++) {
$request = $port.WebRequest[$i]
# Extract title from HTML content
$title = Get-HtmlTitle -HtmlContent $request.Content
# Create a new ordered dictionary for each row
$rowData = [ordered]@{
IP = $HostData.IP
Hostname = $HostData.Hostname
DNSResolution = $HostData.DNSResolution.Resolves
DNSMatchesIP = $HostData.DNSResolution.IPMatch
Port = $port.PortNumber
Protocol = $port.Protocol
Service = $port.Service
IsSSL = $port.IsSSL
RedirectIndex = $i
URL = $request.Url
StatusCode = $request.StatusCode
Error = $request.Error
HasContent = if ($request.Content) { $true } else { $false }
ContentLength = if ($request.Content) { $request.Content.Length } else { 0 }
HasFavicon = if ($request.Favicon) { $true } else { $false }
Content = $request.Content
Favicon = $request.Favicon
Title = $title
}
# Add headers if available
if ($request.Headers -and $request.Headers.Count -gt 0) {
$headerString = ($request.Headers.GetEnumerator() | ForEach-Object { "$($_.Key): $($_.Value)" }) -join "; "
$rowData["Headers"] = $headerString
} else {
$rowData["Headers"] = ""
}
# Certificate information (if SSL)
if ($port.IsSSL -and $request.Certificate) {
$cert = $request.Certificate
# Add certificate status and method
$rowData["CertStatus"] = if ($cert.PSObject.Properties.Name -contains "Status") { $cert.Status } else { "Unknown" }
$rowData["CertMethod"] = if ($cert.PSObject.Properties.Name -contains "Method") { $cert.Method } else { "Unknown" }
$rowData["CertError"] = if ($cert.PSObject.Properties.Name -contains "Error") { $cert.Error } else { $null }
# Add certificate details if available
if ($cert.PSObject.Properties.Name -contains "Subject" -and $cert.Subject -ne $null) {
$rowData["CertSubjectCN"] = $cert.Subject.CN
$rowData["CertIssuerCN"] = $cert.Issuer.CN
$rowData["CertValidFrom"] = $cert.Validity.NotBefore
$rowData["CertValidTo"] = $cert.Validity.NotAfter
$rowData["CertDaysUntilExpiration"] = $cert.Validity.DaysUntilExpiration
$rowData["CertSignatureAlgorithm"] = $cert.SignatureAlgorithm
$rowData["CertSelfSigned"] = $cert.SelfSigned
$rowData["CertSupportedTLSVersions"] = if ($cert.TLSSupport -and $cert.TLSSupport.SupportedProtocols) {
($cert.TLSSupport.SupportedProtocols -join ", ")
} else {
"Unknown"
}
# Add thumbprints
if ($cert.Thumbprint) {
$rowData["CertThumbprintSHA1"] = $cert.Thumbprint.SHA1
$rowData["CertThumbprintSHA256"] = $cert.Thumbprint.SHA256
}
# Include the serial number
$rowData["CertSerialNumber"] = $cert.SerialNumber
}
}
$longRows += [PSCustomObject]$rowData
}
}
# Append to CSV files
try {
# If file doesn't exist, create it with headers
if (-not (Test-Path $WideCSVPath)) {
$wideRows | Export-Csv -Path $WideCSVPath -NoTypeInformation -Encoding UTF8
} else {
$wideRows | Export-Csv -Path $WideCSVPath -NoTypeInformation -Encoding UTF8 -Append -Force
}
if (-not (Test-Path $LongCSVPath)) {
$longRows | Export-Csv -Path $LongCSVPath -NoTypeInformation -Encoding UTF8
} else {
$longRows | Export-Csv -Path $LongCSVPath -NoTypeInformation -Encoding UTF8 -Append -Force
}
Write-Host "CSV data appended for host: $($HostData.IP)"
}
catch {
Write-Host "Error appending CSV data for host $($HostData.IP): $_" -ForegroundColor Red
}
}
# Convert relative path to absolute
$XmlPath = if ([System.IO.Path]::IsPathRooted($XmlPath)) {
$XmlPath
} else {
Join-Path -Path (Get-Location) -ChildPath $XmlPath
}
if (-not (Test-Path $XmlPath)) {
Write-Error "XML file not found at path: $XmlPath"
exit 1
}
# Count total hosts and set MaxHosts if not specified
$totalHosts = Get-TotalHosts -XmlPath $XmlPath
if ($MaxHosts -eq 0) {
$MaxHosts = $totalHosts
Write-Host "Total hosts in file: $totalHosts"
} else {
Write-Host "Processing $MaxHosts of $totalHosts hosts"
}
$timestamp = Get-Date -Format "yyyy-MM-dd_HH-mm-ss"
$outputDir = Join-Path -Path (Get-Location) -ChildPath "webinterface_capture_$timestamp"
New-Item -ItemType Directory -Path $outputDir -Force | Out-Null
$startTime = Get-Date
$processedHosts = 0
try {
$settings = New-Object System.Xml.XmlReaderSettings
$settings.DtdProcessing = [System.Xml.DtdProcessing]::Parse
$reader = [System.Xml.XmlReader]::Create($XmlPath, $settings)
while ($reader.Read() -and ($processedHosts -lt $MaxHosts)) {
if ($reader.NodeType -eq [System.Xml.XmlNodeType]::Element -and $reader.Name -eq "host") {
$hostXml = [xml]$reader.ReadOuterXml()
$targetHost = $hostXml.host
$hostIP = $targetHost.address | Where-Object { $_.addrtype -eq "ipv4" } | Select-Object -ExpandProperty addr
$hostname = $targetHost.hostnames.hostname.name
Write-Host "`nScanning host: $hostIP $(if ($hostname) { "($hostname)" })"
$hostStartTime = Get-Date
# Determine the appropriate hostname/IP to use based on DNS resolution
$targetHostname = Get-TargetHostname -Hostname $hostname -IP $hostIP
# Get all open ports from the Nmap XML
$openPorts = @($targetHost.ports.port | Where-Object { $_.state.state -eq "open" })
if ($openPorts.Count -eq 0) {
Write-Host " No open ports found in Nmap results"
$ports = @()
} else {
$ports = $openPorts | ForEach-Object {
$isSSL = $false
if ($_.service.tunnel -eq "ssl" -or $_.service.name -match "ssl|tls|https" -or $_.service.script.output -match "ssl|tls") {
$isSSL = $true
}
Write-Host "Port $($_.portid) ($($_.service.name)$(if ($isSSL) { "/ssl" })):"
$webRequests = Fetch-Url -TargetHost $targetHostname -Port $_.portid -IsSSL $isSSL -Timeout $RequestTimeout
[PSCustomObject]@{
PortNumber = $_.portid
Protocol = $_.protocol
Service = $_.service.name
IsSSL = $isSSL
WebRequest = $webRequests
}
}
}
$dnsCheck = Test-DNSResolution -Hostname $hostname -IP $hostIP
$hostResult = [PSCustomObject]@{
IP = $hostIP
Hostname = $hostname
DNSResolution = $dnsCheck
OpenPorts = $ports
ScanTime = $targetHost.starttime
}
$outputFile = Join-Path -Path $outputDir -ChildPath "$hostIP.json"
$hostResult | ConvertTo-Json -Depth 10 | Out-File $outputFile
# Generate the CSV File content for this host
$csvWideFile = Join-Path -Path $outputDir -ChildPath "web_interfaces_wide.csv"
$csvLongFile = Join-Path -Path $outputDir -ChildPath "web_interfaces_long.csv"
Append-HostToCSV -OutputDir $outputDir -WideCSVPath $csvWideFile -LongCSVPath $csvLongFile -HostData $hostResult
$processedHosts++
$percentComplete = [math]::Round(($processedHosts / $MaxHosts) * 100, 1)
# Calculate time estimates
$elapsedTime = (Get-Date) - $startTime
$averageTimePerHost = $elapsedTime.TotalSeconds / $processedHosts
$remainingHosts = $MaxHosts - $processedHosts
$estimatedRemainingSeconds = $averageTimePerHost * $remainingHosts
$estimatedRemaining = [TimeSpan]::FromSeconds($estimatedRemainingSeconds)
# Format the remaining time
$remainingStr = Format-TimeSpan -TimeSpan $estimatedRemaining
Write-Host ("Completed host: {0} ({1} of {2} - {3}% complete, est. {4} remaining)`n" -f
$hostIP, $processedHosts, $MaxHosts, $percentComplete, $remainingStr)
}
}
} catch {
Write-Error "Error processing NMAP XML: $_"
exit 1
} finally {
if ($reader) {
$reader.Close()
}
}
$totalTime = Format-TimeSpan -TimeSpan ((Get-Date) - $startTime)
Write-Host "Processing complete in $totalTime. Results saved in: $outputDir"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment