CajuCLC · June 5, 2018 17:42
diff --git a/magento_crawler.sh b/magento_crawler.sh
 #!/bin/bash
 URL='domain.com'

 wget --quiet http://$URL/sitemap.xml --no-cache --output-document - | egrep -o "http://$URL[^<]+" | while read line; do
    time curl -A 'Cache Warmer' -s -L $line > /dev/null 2>&1
    echo $line
 done
	#!/bin/bash
	URL='domain.com'

	wget --quiet http://$URL/sitemap.xml --no-cache --output-document - \| egrep -o "http://$URL[^<]+" \| while read line; do
	time curl -A 'Cache Warmer' -s -L $line > /dev/null 2>&1
	echo $line
	done