cloudsben · January 9, 2013 02:52
diff --git a/gistfile1.sh b/gistfile1.sh
 # -p parameter tells wget to include all files, including images.
 # -e robots=off you don't want wget to obey by the robots.txt file
 # -U mozilla as your browsers identity.
 # --random-wait to let wget chose a random number of seconds to wait, avoid get into black list.
 # Other Useful wget Parameters:
 # --limit-rate=20k limits the rate at which it downloads files.
 # -b continues wget after logging out.
 # -o $HOME/wget_log.txt logs the output

 wget --random-wait -r -p -e robots=off -U mozilla http://www.example.com
	# -p parameter tells wget to include all files, including images.
	# -e robots=off you don't want wget to obey by the robots.txt file
	# -U mozilla as your browsers identity.
	# --random-wait to let wget chose a random number of seconds to wait, avoid get into black list.
	# Other Useful wget Parameters:
	# --limit-rate=20k limits the rate at which it downloads files.
	# -b continues wget after logging out.
	# -o $HOME/wget_log.txt logs the output

	wget --random-wait -r -p -e robots=off -U mozilla http://www.example.com