abargnesi · June 21, 2016 01:08
diff --git a/crawl.sh b/crawl.sh
 function crawl_for_urls() {
    if [ $# != 1 ]; then
        echo "usage: crawl_for_urls URL" >2
        exit 1
    fi  

    wget --spider --force-html -r "$1" 2>&1 | \ 
        grep '^--' | \ 
        awk '{ print \$3 }' | \ 
        grep -v '\.\(css\|js\|png\|gif\|jpg\)$'
 }
diff --git a/test.sh b/test.sh
 crawl_for_urls "http://www.infoq.com"
	function crawl_for_urls() {
	if [ $# != 1 ]; then
	echo "usage: crawl_for_urls URL" >2
	exit 1
	fi

	wget --spider --force-html -r "$1" 2>&1 \| \
	grep '^--' \| \
	awk '{ print \$3 }' \| \
	grep -v '\.\(css\\|js\\|png\\|gif\\|jpg\)$'
	}