wget spider crawler

wget-spider.sh

#!/bin/bash

# Replace with the URL of the sitemap index
SITEMAP_INDEX_URL="https://example.com/sitemap_index.xml"

# Download the sitemap index
wget -q -O sitemap_index.xml "$SITEMAP_INDEX_URL"

# Extract the sitemap URLs
SITEMAP_URLS=$(grep -oP '<loc>\K[^<]+' sitemap_index.xml)

# Loop through each sitemap URL and spider it using wget
for url in $SITEMAP_URLS; do
  echo "Spidering sitemap: $url" >> spider.log
  wget --recursive --level=1 --no-directories --no-check-certificate --spider -o - "$url" 2>&1 | grep -B 1 "^HTTP" | tee -a spider.log
done

# Remove the downloaded sitemap index
rm sitemap_index.xml

echo "Spidering completed." >> spider.log