Last active
July 14, 2025 14:18
-
-
Save dkarter/a375fbd9395d65a7adcb9ec1c4d5df45 to your computer and use it in GitHub Desktop.
scan all your repos for secrets
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| set -e | |
| # Colors for output | |
| RED='\033[0;31m' | |
| GREEN='\033[0;32m' | |
| YELLOW='\033[1;33m' | |
| BLUE='\033[0;34m' | |
| NC='\033[0m' # No Color | |
| # Maximum number of parallel jobs | |
| MAX_JOBS=5 | |
| # Default settings | |
| INCLUDE_FORKS=false | |
| # Parse command line arguments | |
| while [[ $# -gt 0 ]]; do | |
| case $1 in | |
| --with-forks) | |
| INCLUDE_FORKS=true | |
| shift | |
| ;; | |
| --max-jobs) | |
| if [[ -n $2 && $2 =~ ^[0-9]+$ ]]; then | |
| MAX_JOBS=$2 | |
| shift 2 | |
| else | |
| echo "Error: --max-jobs requires a positive integer argument" | |
| exit 1 | |
| fi | |
| ;; | |
| -h | --help) | |
| echo "Repository Secret Scanner (Parallel)" | |
| echo "Usage: $0 [OPTIONS]" | |
| echo "" | |
| echo "Options:" | |
| echo " --with-forks Include forked repositories in scan (default: false)" | |
| echo " --max-jobs N Maximum number of parallel jobs (default: 5)" | |
| echo " -h, --help Show this help message" | |
| echo "" | |
| echo "Examples:" | |
| echo " $0 # Scan only non-fork repositories with 5 parallel jobs" | |
| echo " $0 --with-forks # Scan all repositories including forks" | |
| echo " $0 --max-jobs 10 # Use 10 parallel jobs" | |
| echo " $0 --with-forks --max-jobs 3 # Include forks with 3 parallel jobs" | |
| exit 0 | |
| ;; | |
| *) | |
| echo "Unknown option: $1" | |
| echo "Use --help for usage information" | |
| exit 1 | |
| ;; | |
| esac | |
| done | |
| echo -e "${BLUE}Repository Secret Scanner (Parallel)${NC}" | |
| echo "=========================================" | |
| echo -e "${GREEN}Include forks: $INCLUDE_FORKS${NC}" | |
| echo -e "${GREEN}Max parallel jobs: $MAX_JOBS${NC}" | |
| # Check if user is authenticated with gh | |
| if ! gh auth status &>/dev/null; then | |
| echo -e "${RED}Please authenticate with GitHub CLI first: gh auth login${NC}" | |
| exit 1 | |
| fi | |
| # Check if gitleaks is available (via mise or system) | |
| if ! command -v gitleaks &>/dev/null && ! mise exec -- gitleaks --version &>/dev/null; then | |
| echo -e "${YELLOW}gitleaks is not installed. Installing...${NC}" | |
| if command -v brew &>/dev/null; then | |
| brew install gitleaks | |
| else | |
| echo -e "${RED}Please install gitleaks manually: https://github.com/gitleaks/gitleaks${NC}" | |
| exit 1 | |
| fi | |
| fi | |
| # Create temporary directory | |
| TEMP_DIR=$(mktemp -d) | |
| REPORT_DIR="$PWD/gitleaks-reports" | |
| FINAL_REPORT="$PWD/gitleaks-summary-report.json" | |
| FINAL_REPORT_HTML="$PWD/gitleaks-summary-report.html" | |
| echo -e "${GREEN}Using temporary directory: $TEMP_DIR${NC}" | |
| echo -e "${GREEN}Reports will be saved to: $REPORT_DIR${NC}" | |
| # Create reports directory | |
| mkdir -p "$REPORT_DIR" | |
| # Initialize summary report | |
| echo '{"repositories": []}' >"$FINAL_REPORT" | |
| # Function to scan a single repository | |
| scan_repo() { | |
| local repo_name="$1" | |
| local repo_url="$2" | |
| local temp_dir="$3" | |
| local report_dir="$4" | |
| echo -e "${YELLOW}Processing repository: $repo_name${NC}" | |
| # Clone repository | |
| repo_dir="$temp_dir/$repo_name" | |
| if git clone "$repo_url" "$repo_dir" 2>/dev/null; then | |
| echo -e "${GREEN}✓ Cloned $repo_name${NC}" | |
| # Run gitleaks scan | |
| report_file="$report_dir/${repo_name}-gitleaks-report.json" | |
| cd "$repo_dir" | |
| # Try gitleaks with mise exec first, fallback to direct command | |
| if command -v gitleaks &>/dev/null; then | |
| gitleaks_cmd="gitleaks" | |
| else | |
| gitleaks_cmd="mise exec -- gitleaks" | |
| fi | |
| if $gitleaks_cmd git --redact --report-path="$report_file" 2>/dev/null; then | |
| echo -e "${GREEN}✓ Gitleaks scan completed for $repo_name${NC}" | |
| else | |
| echo -e "${YELLOW}⚠ Gitleaks scan completed with findings for $repo_name${NC}" | |
| fi | |
| cd - >/dev/null | |
| # Create summary for this repo | |
| if [ -f "$report_file" ] && [ -s "$report_file" ]; then | |
| # Has findings | |
| jq -n \ | |
| --arg name "$repo_name" \ | |
| --arg url "$repo_url" \ | |
| --argjson findings "$(cat "$report_file")" \ | |
| '{ | |
| repository: $name, | |
| url: $url, | |
| scan_date: (now | strftime("%Y-%m-%d %H:%M:%S")), | |
| findings_count: ($findings | length), | |
| findings: $findings | |
| }' >"$report_dir/${repo_name}-summary.json" | |
| else | |
| # No findings | |
| jq -n \ | |
| --arg name "$repo_name" \ | |
| --arg url "$repo_url" \ | |
| '{ | |
| repository: $name, | |
| url: $url, | |
| scan_date: (now | strftime("%Y-%m-%d %H:%M:%S")), | |
| findings_count: 0, | |
| findings: [] | |
| }' >"$report_dir/${repo_name}-summary.json" | |
| fi | |
| else | |
| echo -e "${RED}✗ Failed to clone $repo_name${NC}" | |
| # Create error summary | |
| jq -n \ | |
| --arg name "$repo_name" \ | |
| --arg url "$repo_url" \ | |
| '{ | |
| repository: $name, | |
| url: $url, | |
| scan_date: (now | strftime("%Y-%m-%d %H:%M:%S")), | |
| findings_count: -1, | |
| error: "Failed to clone repository", | |
| findings: [] | |
| }' >"$report_dir/${repo_name}-summary.json" | |
| fi | |
| } | |
| # Export function so it can be used by parallel processes | |
| export -f scan_repo | |
| # Get all repositories | |
| echo -e "${BLUE}Fetching repositories...${NC}" | |
| if [ "$INCLUDE_FORKS" = true ]; then | |
| REPOS=$(gh repo list --limit 1000 --json name,sshUrl,isFork) | |
| else | |
| REPOS=$(gh repo list --limit 1000 --json name,sshUrl,isFork | jq '[.[] | select(.isFork == false)]') | |
| fi | |
| if [ -z "$REPOS" ] || [ "$REPOS" = "[]" ]; then | |
| echo -e "${RED}No repositories found or unable to fetch repositories${NC}" | |
| exit 1 | |
| fi | |
| # Create individual temp directories for each job to avoid conflicts | |
| repo_count=$(echo "$REPOS" | jq length) | |
| echo -e "${GREEN}Found $repo_count repositories. Starting parallel scan...${NC}" | |
| # Process repositories in parallel | |
| echo "$REPOS" | jq -r '.[] | "\(.name) \(.sshUrl)"' | while read -r repo_name repo_url; do | |
| # Wait if we have too many background jobs | |
| while [ $(jobs -r | wc -l) -ge $MAX_JOBS ]; do | |
| sleep 1 | |
| done | |
| # Create individual temp directory for this repo | |
| repo_temp_dir="$TEMP_DIR/repo_$repo_name" | |
| mkdir -p "$repo_temp_dir" | |
| # Start background job | |
| scan_repo "$repo_name" "$repo_url" "$repo_temp_dir" "$REPORT_DIR" & | |
| done | |
| # Wait for all background jobs to complete | |
| wait | |
| echo -e "\n${BLUE}Compiling final report...${NC}" | |
| # Combine all individual summaries into final report | |
| for summary_file in "$REPORT_DIR"/*-summary.json; do | |
| if [ -f "$summary_file" ]; then | |
| repo_summary=$(cat "$summary_file") | |
| jq --argjson new_repo "$repo_summary" '.repositories += [$new_repo]' "$FINAL_REPORT" >"${FINAL_REPORT}.tmp" && mv "${FINAL_REPORT}.tmp" "$FINAL_REPORT" | |
| rm "$summary_file" # Clean up individual summary files | |
| fi | |
| done | |
| # Generate HTML summary report | |
| echo -e "${BLUE}Generating HTML summary report...${NC}" | |
| cat >"$FINAL_REPORT_HTML" <<'EOF' | |
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <title>Gitleaks Security Scan Summary</title> | |
| <style> | |
| body { font-family: Arial, sans-serif; margin: 20px; } | |
| .header { background: #f4f4f4; padding: 20px; border-radius: 5px; } | |
| .repo { margin: 20px 0; padding: 15px; border: 1px solid #ddd; border-radius: 5px; } | |
| .clean { background: #d4edda; border-color: #c3e6cb; } | |
| .findings { background: #f8d7da; border-color: #f5c6cb; } | |
| .error { background: #fff3cd; border-color: #ffeaa7; } | |
| .finding { margin: 10px 0; padding: 15px; background: #fff; border-left: 4px solid #dc3545; border-radius: 3px; } | |
| .finding code { background: #f8f9fa; padding: 2px 4px; border-radius: 3px; font-family: monospace; } | |
| .finding a { color: #007bff; text-decoration: none; } | |
| .finding a:hover { text-decoration: underline; } | |
| .meta { color: #666; font-size: 0.9em; margin-top: 8px; font-style: italic; } | |
| .summary { background: #e3f2fd; padding: 15px; border-radius: 5px; margin: 20px 0; } | |
| .search-container { margin: 20px 0; } | |
| .search-box { width: 100%; max-width: 500px; padding: 10px; border: 1px solid #ddd; border-radius: 5px; font-size: 16px; } | |
| .search-results { margin-top: 10px; color: #666; } | |
| .filter-container { margin: 20px 0; padding: 15px; background: #f8f9fa; border-radius: 5px; } | |
| .filter-input { width: 100%; max-width: 500px; padding: 8px; border: 1px solid #ddd; border-radius: 3px; margin-bottom: 10px; } | |
| .filter-info { font-size: 0.9em; color: #666; } | |
| .ignored-finding { opacity: 0.5; border-left-color: #6c757d !important; } | |
| .ignored-finding .finding { background: #f8f9fa; } | |
| .repo-header { cursor: pointer; display: flex; justify-content: space-between; align-items: center; } | |
| .repo-header:hover { background: rgba(0,0,0,0.05); border-radius: 3px; padding: 5px; margin: -5px; } | |
| .collapse-indicator { font-size: 1.2em; transition: transform 0.2s; } | |
| .collapsed .collapse-indicator { transform: rotate(-90deg); } | |
| .repo-content { overflow: hidden; transition: max-height 0.3s ease-out; } | |
| .collapsed .repo-content { max-height: 0; } | |
| .controls { margin: 20px 0; } | |
| .btn { padding: 8px 16px; margin: 0 5px; border: 1px solid #ddd; background: #fff; border-radius: 3px; cursor: pointer; } | |
| .btn:hover { background: #f8f9fa; } | |
| .hidden { display: none; } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="header"> | |
| <h1>🔒 Gitleaks Security Scan Summary</h1> | |
| <p>Generated on: <span id="scan-date"></span></p> | |
| </div> | |
| <div class="summary"> | |
| <h2>📊 Summary Statistics</h2> | |
| <div id="summary-stats"></div> | |
| </div> | |
| <div class="search-container"> | |
| <h2>🔍 Search Repositories</h2> | |
| <input type="text" id="search-box" class="search-box" placeholder="Search by repository name, file, rule ID, or finding details..."> | |
| <div id="search-results" class="search-results"></div> | |
| </div> | |
| <div class="filter-container"> | |
| <h2>🚫 Ignore Findings Filter</h2> | |
| <input type="text" id="ignore-filter" class="filter-input" placeholder="Enter patterns to ignore (comma-separated, case insensitive). Example: secret_key_base, api_key"> | |
| <div class="filter-info"> | |
| <strong>Examples:</strong> secret_key_base, SECRET_KEY_BASE, development_key<br> | |
| Findings containing these patterns will be visually dimmed. Clear the field to show all findings. | |
| </div> | |
| </div> | |
| <div class="controls"> | |
| <h2>📂 Repository Controls</h2> | |
| <button id="expand-all" class="btn">📂 Expand All</button> | |
| <button id="collapse-all" class="btn">📁 Collapse All</button> | |
| </div> | |
| <div id="repositories"></div> | |
| <script> | |
| EOF | |
| echo -n "const scanData = " >>"$FINAL_REPORT_HTML" | |
| cat "$FINAL_REPORT" >>"$FINAL_REPORT_HTML" | |
| cat >>"$FINAL_REPORT_HTML" <<'EOF' | |
| ; | |
| document.getElementById('scan-date').textContent = new Date().toLocaleString(); | |
| // Calculate summary statistics | |
| const totalRepos = scanData.repositories.length; | |
| const reposWithFindings = scanData.repositories.filter(r => r.findings_count > 0).length; | |
| const reposWithErrors = scanData.repositories.filter(r => r.findings_count === -1).length; | |
| const totalFindings = scanData.repositories.reduce((sum, r) => sum + Math.max(0, r.findings_count), 0); | |
| document.getElementById('summary-stats').innerHTML = ` | |
| <p><strong>Total Repositories Scanned:</strong> ${totalRepos}</p> | |
| <p><strong>Repositories with Findings:</strong> ${reposWithFindings}</p> | |
| <p><strong>Repositories with Errors:</strong> ${reposWithErrors}</p> | |
| <p><strong>Total Findings:</strong> ${totalFindings}</p> | |
| `; | |
| // Sort repositories: findings first, then errors, then clean | |
| const sortedRepos = scanData.repositories.sort((a, b) => { | |
| // Findings first (highest priority) | |
| if (a.findings_count > 0 && b.findings_count <= 0) return -1; | |
| if (b.findings_count > 0 && a.findings_count <= 0) return 1; | |
| // Errors second | |
| if (a.findings_count === -1 && b.findings_count !== -1) return -1; | |
| if (b.findings_count === -1 && a.findings_count !== -1) return 1; | |
| // Within same category, sort by name | |
| return a.repository.localeCompare(b.repository); | |
| }); | |
| // Generate repository sections | |
| const reposContainer = document.getElementById('repositories'); | |
| sortedRepos.forEach(repo => { | |
| const repoDiv = document.createElement('div'); | |
| let repoClass = 'clean'; | |
| let statusIcon = '✅'; | |
| if (repo.findings_count === -1) { | |
| repoClass = 'error'; | |
| statusIcon = '❌'; | |
| } else if (repo.findings_count > 0) { | |
| repoClass = 'findings'; | |
| statusIcon = '🚨'; | |
| } | |
| repoDiv.className = `repo ${repoClass}`; | |
| let findingsHtml = ''; | |
| if (repo.findings_count > 0) { | |
| findingsHtml = repo.findings.map(finding => ` | |
| <div class="finding"> | |
| <strong>File:</strong> ${finding.File || 'Unknown'}<br> | |
| <strong>Rule ID:</strong> ${finding.RuleID || 'Unknown'}<br> | |
| <strong>Line:</strong> ${finding.StartLine || 'Unknown'}<br> | |
| <strong>Match:</strong> <code>${finding.Match || '[REDACTED]'}</code><br> | |
| <strong>Commit Date:</strong> ${finding.Date || 'Unknown'}<br> | |
| ${finding.Link ? `<strong>GitHub Link:</strong> <a href="${finding.Link}" target="_blank">View on GitHub</a><br>` : ''} | |
| <div class="meta">Secret: ${finding.Secret || '[REDACTED]'}</div> | |
| </div> | |
| `).join(''); | |
| } else if (repo.error) { | |
| findingsHtml = `<p><strong>Error:</strong> ${repo.error}</p>`; | |
| } | |
| repoDiv.innerHTML = ` | |
| <div class="repo-header" onclick="toggleRepo(this)"> | |
| <h3>${statusIcon} ${repo.repository}</h3> | |
| <span class="collapse-indicator">🔽</span> | |
| </div> | |
| <div class="repo-content"> | |
| <p><strong>URL:</strong> <a href="${repo.url}">${repo.url}</a></p> | |
| <p><strong>Scan Date:</strong> ${repo.scan_date}</p> | |
| <p><strong>Findings:</strong> ${repo.findings_count === -1 ? 'Error' : repo.findings_count}</p> | |
| ${findingsHtml} | |
| </div> | |
| `; | |
| repoDiv.setAttribute('data-repo-name', repo.repository.toLowerCase()); | |
| repoDiv.setAttribute('data-search-text', JSON.stringify(repo).toLowerCase()); | |
| repoDiv.setAttribute('data-original-status', statusIcon); | |
| repoDiv.setAttribute('data-repo-title', repo.repository); | |
| reposContainer.appendChild(repoDiv); | |
| }); | |
| // Search functionality | |
| const searchBox = document.getElementById('search-box'); | |
| const searchResults = document.getElementById('search-results'); | |
| const allRepos = document.querySelectorAll('.repo'); | |
| function performSearch() { | |
| const query = searchBox.value.toLowerCase().trim(); | |
| let visibleCount = 0; | |
| allRepos.forEach(repoDiv => { | |
| if (query === '') { | |
| repoDiv.classList.remove('hidden'); | |
| visibleCount++; | |
| } else { | |
| const searchText = repoDiv.getAttribute('data-search-text'); | |
| if (searchText.includes(query)) { | |
| repoDiv.classList.remove('hidden'); | |
| visibleCount++; | |
| } else { | |
| repoDiv.classList.add('hidden'); | |
| } | |
| } | |
| }); | |
| if (query === '') { | |
| searchResults.textContent = ''; | |
| } else { | |
| searchResults.textContent = `Found ${visibleCount} repositories matching "${query}"`; | |
| } | |
| } | |
| searchBox.addEventListener('input', performSearch); | |
| searchBox.addEventListener('keyup', performSearch); | |
| // Ignore filter functionality | |
| const ignoreFilter = document.getElementById('ignore-filter'); | |
| function applyIgnoreFilter() { | |
| const patterns = ignoreFilter.value.split(',').map(p => p.trim().toLowerCase()).filter(p => p); | |
| allRepos.forEach(repoDiv => { | |
| const findings = repoDiv.querySelectorAll('.finding'); | |
| let hasUnignoredFindings = false; | |
| let visibleFindingsCount = 0; | |
| findings.forEach(finding => { | |
| // Get the match text from the "Match:" field | |
| let matchText = ''; | |
| // Find the "Match:" field specifically | |
| const strongElements = finding.querySelectorAll('strong'); | |
| for (let strong of strongElements) { | |
| if (strong.textContent === 'Match:') { | |
| const codeElement = strong.parentNode.querySelector('code'); | |
| if (codeElement) { | |
| matchText = codeElement.textContent.toLowerCase(); | |
| } | |
| break; | |
| } | |
| } | |
| const shouldIgnore = patterns.length > 0 && patterns.some(pattern => matchText.includes(pattern)); | |
| if (shouldIgnore) { | |
| finding.style.display = 'none'; | |
| } else { | |
| finding.style.display = 'block'; | |
| hasUnignoredFindings = true; | |
| visibleFindingsCount++; | |
| } | |
| }); | |
| // Update repository status and styling | |
| const repoHeader = repoDiv.querySelector('.repo-header h3'); | |
| const findingsCountElement = repoDiv.querySelector('.repo-content p:nth-of-type(3)'); | |
| const repoTitle = repoDiv.getAttribute('data-repo-title'); | |
| if (findings.length > 0) { | |
| if (hasUnignoredFindings) { | |
| // Has visible findings - keep red | |
| repoDiv.className = 'repo findings'; | |
| if (repoHeader) { | |
| repoHeader.textContent = `🚨 ${repoTitle}`; | |
| } | |
| } else { | |
| // All findings filtered out - make green | |
| repoDiv.className = 'repo clean'; | |
| if (repoHeader) { | |
| repoHeader.textContent = `✅ ${repoTitle}`; | |
| } | |
| } | |
| // Update visible findings count | |
| if (findingsCountElement && patterns.length > 0) { | |
| const originalCount = findings.length; | |
| findingsCountElement.innerHTML = `<strong>Findings:</strong> ${visibleFindingsCount} of ${originalCount} (${originalCount - visibleFindingsCount} filtered)`; | |
| } else if (findingsCountElement) { | |
| findingsCountElement.innerHTML = `<strong>Findings:</strong> ${findings.length}`; | |
| } | |
| } | |
| }); | |
| } | |
| ignoreFilter.addEventListener('input', applyIgnoreFilter); | |
| // Collapsible functionality | |
| function toggleRepo(header) { | |
| const repoDiv = header.closest('.repo'); | |
| repoDiv.classList.toggle('collapsed'); | |
| const content = repoDiv.querySelector('.repo-content'); | |
| if (repoDiv.classList.contains('collapsed')) { | |
| content.style.maxHeight = '0px'; | |
| } else { | |
| content.style.maxHeight = content.scrollHeight + 'px'; | |
| } | |
| } | |
| // Expand/Collapse all functionality | |
| document.getElementById('expand-all').addEventListener('click', function() { | |
| allRepos.forEach(repoDiv => { | |
| repoDiv.classList.remove('collapsed'); | |
| const content = repoDiv.querySelector('.repo-content'); | |
| content.style.maxHeight = content.scrollHeight + 'px'; | |
| }); | |
| }); | |
| document.getElementById('collapse-all').addEventListener('click', function() { | |
| allRepos.forEach(repoDiv => { | |
| repoDiv.classList.add('collapsed'); | |
| const content = repoDiv.querySelector('.repo-content'); | |
| content.style.maxHeight = '0px'; | |
| }); | |
| }); | |
| // Initialize repo content heights for smooth transitions | |
| allRepos.forEach(repoDiv => { | |
| const content = repoDiv.querySelector('.repo-content'); | |
| content.style.maxHeight = content.scrollHeight + 'px'; | |
| }); | |
| </script> | |
| </body> | |
| </html> | |
| EOF | |
| # Clean up temporary directory | |
| rm -rf "$TEMP_DIR" | |
| echo -e "\n${GREEN}✅ Parallel scan completed!${NC}" | |
| echo -e "${GREEN}📄 JSON Report: $FINAL_REPORT${NC}" | |
| echo -e "${GREEN}🌐 HTML Report: $FINAL_REPORT_HTML${NC}" | |
| echo -e "${GREEN}📁 Individual Reports: $REPORT_DIR${NC}" | |
| # Show summary | |
| total_repos=$(jq '.repositories | length' "$FINAL_REPORT") | |
| repos_with_findings=$(jq '[.repositories[] | select(.findings_count > 0)] | length' "$FINAL_REPORT") | |
| repos_with_errors=$(jq '[.repositories[] | select(.findings_count == -1)] | length' "$FINAL_REPORT") | |
| total_findings=$(jq '[.repositories[] | select(.findings_count > 0) | .findings_count] | add // 0' "$FINAL_REPORT") | |
| echo -e "\n${BLUE}📊 Summary:${NC}" | |
| echo -e "Total repositories scanned: $total_repos" | |
| echo -e "Repositories with findings: $repos_with_findings" | |
| echo -e "Repositories with errors: $repos_with_errors" | |
| echo -e "Total findings: $total_findings" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
To run it use:
curl -s https://gist.githubusercontent.com/dkarter/a375fbd9395d65a7adcb9ec1c4d5df45/raw/7e32d335d7eb9b0f85741a2005ca99ba5069f15d/scan_repos_for_secrets_parallel.sh | bash