given a passed URL, extracts all URIs and returns a list, sorted and unique.

geturilist.sh

#!/bin/sh

# *********************************************************************
# script: geturilist
# summary: given a passed URL, extracts all URIs and returns a list, 
#	   sorted and unique.
# dependencies: wget; grep
# by: e.a.soto, [email protected]
# 
# history
# =======
# 
# 2015-09-16: created.
# 
# *********************************************************************

if [ $# -eq 0 ]
  then
    echo "URL must be supplied!"
    exit 1
fi

# wget the passed URI (-q = quiet; -O = output to file '-' which is stdout instead of file)
# pass to grep (-E = extended regex; -o = output only matches; -i = ignore case)

wget -qO- $1 | grep -Eoi '(http|https|ftp|ftps)://[^"]*' | sort -u