Reconnaissance automation script that combines multiple tools for thorough target scanning.

init_recon.sh

#!/bin/bash

# Function to display usage/help information
show_help() {
    cat << EOF
Usage: $(basename "$0") [-h] [-i INPUT]

Reconnaissance automation script that combines multiple tools for thorough target scanning.

Options:
    -h          Show this help message
    -i INPUT    Input file containing targets or a single target (domain or IP)
    
The script will:
1. Perform port scanning with Naabu
2. Identify HTTP services with httpx
3. Crawl with Katana
4. Use Waymore to discover additional URLs (limited to the last 2 years)
5. Filter endpoints to match the scope
6. Validate endpoints with FFUF-Workflow

Example usage:
    $(basename "$0") -i targets.txt
    $(basename "$0") -i example.com
EOF
    exit 0
}

# Function to log messages
log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1"
}

# Function to handle errors
handle_error() {
    log "Error: $1" >&2
    exit 1
}

# Function to install Go if not present
install_go() {
    if ! command -v go &> /dev/null; then
        log "Installing Go..."
        sudo apt update
        sudo apt install -y golang || handle_error "Failed to install Go"
        echo 'export GOPATH=$HOME/go' >> ~/.bashrc
        echo 'export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin' >> ~/.bashrc
        source ~/.bashrc
    fi
}

# Function to check and install system dependencies
check_system_deps() {
    log "Checking system dependencies..."
    # Required for naabu
    if ! dpkg -l | grep -q libpcap-dev; then
        log "Installing libpcap-dev..."
        sudo apt update
        sudo apt install -y libpcap-dev || handle_error "Failed to install libpcap-dev"
    fi
}

# Function to install Python version of Waymore
install_waymore() {
    if ! command -v waymore &> /dev/null; then
        log "Installing Waymore (Python version)..."
        pip3 install --break-system-packages git+https://github.com/xnl-h4ck3r/waymore.git -v || handle_error "Failed to install Waymore"
    fi
}

# Function to install Gofuzz
install_gofuzz() {
    if ! command -v gofuzz.py &> /dev/null; then
        log "Installing Gofuzz..."
        # Clone the repository
        local gofuzz_dir="/tmp/gofuzz"
        if [ -d "$gofuzz_dir" ]; then
            rm -rf "$gofuzz_dir"
        fi
        git clone https://github.com/nullenc0de/gofuzz.git "$gofuzz_dir" || handle_error "Failed to clone Gofuzz repository"
        
        # Install required Python dependencies
        log "Installing Gofuzz dependencies..."
        pip3 install --break-system-packages aiohttp || handle_error "Failed to install aiohttp"

        # Move the script to /usr/local/bin
        sudo mv "$gofuzz_dir/gofuzz.py" /usr/local/bin/gofuzz.py || handle_error "Failed to move gofuzz.py to /usr/local/bin"

        # Make the script executable
        sudo chmod +x /usr/local/bin/gofuzz.py || handle_error "Failed to make gofuzz.py executable"

        # Verify installation
        if ! command -v gofuzz.py &> /dev/null; then
            handle_error "Failed to install Gofuzz"
        fi
    fi
}

# Function to install Unfurl
install_unfurl() {
    if ! command -v unfurl &> /dev/null; then
        log "Installing Unfurl..."
        go install github.com/tomnomnom/unfurl@latest || handle_error "Failed to install Unfurl"
        
        # Ensure $GOPATH/bin is in PATH
        if [[ ":$PATH:" != *":$HOME/go/bin:"* ]]; then
            echo 'export PATH=$PATH:$HOME/go/bin' >> ~/.bashrc
            source ~/.bashrc
        fi

        # Verify installation
        if ! command -v unfurl &> /dev/null; then
            handle_error "Failed to install Unfurl"
        fi
    fi
}

# Function to install urless
install_urless() {
    if ! command -v urless &> /dev/null; then
        log "Installing urless..."
        pip3 install --break-system-packages git+https://github.com/xnl-h4ck3r/urless.git -v || handle_error "Failed to install urless"
    fi
}

# Function to check tool dependencies
check_deps() {
    local tools=("naabu" "httpx" "katana" "ffuf" "ffuf-workflow" "waymore" "unfurl" "urless")
    local missing_tools=()
    
    log "Checking tool dependencies..."
    
    # First check Go
    if ! command -v go &> /dev/null; then
        log "Go is not installed. Installing..."
        install_go
    fi

    # Check each tool
    for tool in "${tools[@]}"; do
        if ! command -v "$tool" &> /dev/null; then
            missing_tools+=("$tool")
        fi
    done
    
    # Install missing tools if any
    if [ ${#missing_tools[@]} -ne 0 ]; then
        log "Missing tools: ${missing_tools[*]}"
        check_system_deps
        go install github.com/projectdiscovery/pdtm/cmd/pdtm@latest || handle_error "Failed to install pdtm"
        pdtm -install naabu,httpx,katana || handle_error "Failed to install ProjectDiscovery tools"
        install_unfurl
        install_urless
    fi
    
    # Verify installations
    for tool in "${tools[@]}"; do
        if ! command -v "$tool" &> /dev/null; then
            handle_error "Failed to install $tool"
        fi
    done
    
    # Install Waymore (Python version)
    install_waymore

    # Install Gofuzz
    install_gofuzz
    
    log "All dependencies are installed!"
}

# Parse command line arguments
while getopts "hi:" opt; do
    case $opt in
        h)
            show_help
            ;;
        i)
            input_file="$OPTARG"
            ;;
        \?)
            handle_error "Invalid option: -$OPTARG"
            ;;
        :)
            handle_error "Option -$OPTARG requires an argument."
            ;;
    esac
done

# Check if input is provided
if [ -z "$input_file" ]; then
    handle_error "Input file or target is required."
fi

# Check dependencies before starting
check_deps

# Create output directory with timestamp
timestamp=$(date +%Y%m%d_%H%M%S)
output_dir="/tmp/recon_${timestamp}"
mkdir -p "$output_dir" || handle_error "Could not create output directory"

# Ensure directory exists and is writable
if [ ! -d "$output_dir" ] || [ ! -w "$output_dir" ]; then
    handle_error "Output directory is not accessible or writable"
fi

# Save input for reference
if [ -f "$input_file" ]; then
    cp "$input_file" "${output_dir}/scope.txt" || handle_error "Could not save scope file"
else
    echo "$input_file" > "${output_dir}/scope.txt" || handle_error "Could not save scope file"
fi

# Step 1: Naabu scan
log "Running Naabu scan..."
if [ -f "$input_file" ]; then
    naabu -list "$input_file" -silent -o "${output_dir}/naabu.txt" || handle_error "Naabu scan failed"
else
    naabu -host "$input_file" -silent -o "${output_dir}/naabu.txt" || handle_error "Naabu scan failed"
fi

# Step 2: HTTPX scan
log "Running HTTPX scan..."
if [ ! -f "${output_dir}/naabu.txt" ]; then
    handle_error "Naabu output file not found"
fi

cat "${output_dir}/naabu.txt" | sort -u | httpx -silent -nc -o "${output_dir}/httpx.txt" || handle_error "HTTPX scan failed"

# Step 3: Katana crawl
log "Running Katana crawl..."
sort -u "${output_dir}/httpx.txt" | while read -r target; do
    log "Crawling $target with Katana..."
    katana -u "$target" \
        -d 5 \
        -jc \
        -kf all \
        -aff \
        -timeout 15 \
        -retry 3 \
        -silent \
        -iqp \
        -o "${output_dir}/katana.txt"
done

# Step 4: Extract unique root domains and filter out IP addresses
log "Extracting unique root domains and filtering out IP addresses..."
if [ -f "$input_file" ]; then
    # Extract root domains from the input file and filter out IP addresses
    root_domains=$(cat "$input_file" | unfurl apexes | grep -vE '[0-9]{1,3}(\.[0-9]{1,3}){3}' | sort -u)
else
    # Extract root domain from a single target and filter out IP addresses
    root_domains=$(echo "$input_file" | unfurl apexes | grep -vE '[0-9]{1,3}(\.[0-9]{1,3}){3}')
fi

# Log excluded IP addresses for debugging
excluded_ips=$(cat "$input_file" | unfurl apexes | grep -E '[0-9]{1,3}(\.[0-9]{1,3}){3}')
if [ -n "$excluded_ips" ]; then
    log "Excluded IP addresses: $excluded_ips"
fi

# Step 5: Run Waymore on unique root domains
log "Running Waymore for comprehensive URL discovery (last 2 years)..."
for root_domain in $root_domains; do
    log "Running Waymore on root domain: $root_domain"
    current_year=$(date +%Y)
    from_year=$((current_year - 2))
    waymore -i "$root_domain" -mode U -oU "${output_dir}/waymore_${root_domain}.txt" -xcc -from "$from_year" || handle_error "Waymore failed for $root_domain"
    # Append Waymore results to Katana file
    cat "${output_dir}/waymore_${root_domain}.txt" >> "${output_dir}/katana.txt"
done

# Step 6: Filter katana.txt to match the scope
log "Filtering katana.txt to match the scope..."
grep -F -f "${output_dir}/scope.txt" "${output_dir}/katana.txt" | urless > "${output_dir}/katana_filtered.txt"

# Step 7: Validate with FFUF-Workflow
log "Running FFUF-Workflow validation to verify live endpoints..."
if [ -s "${output_dir}/katana_filtered.txt" ]; then
    ffuf-workflow -wordlist "${output_dir}/katana_filtered.txt" -output "${output_dir}/validated_endpoints.txt" || handle_error "FFUF-Workflow validation failed"
else
    log "No in-scope endpoints found to validate"
fi

# Cleanup and report
rm -f "${output_dir}"/waymore_*.txt "${output_dir}/katana.txt" "${output_dir}/katana_filtered.txt"
log "Reconnaissance complete! Results saved in: ${output_dir}/"
log "    - scope.txt: Original targets"
log "    - naabu.txt: Port scan results ($(wc -l < "${output_dir}/naabu.txt" 2>/dev/null || echo 0) ports found)"
log "    - httpx.txt: HTTP service discovery ($(wc -l < "${output_dir}/httpx.txt" 2>/dev/null || echo 0) services found)"
log "    - validated_endpoints.txt: Live endpoints ($(wc -l < "${output_dir}/validated_endpoints.txt" 2>/dev/null || echo 0) verified)"