Created
January 16, 2020 21:39
-
-
Save qskwood/4919c9b8bd979ab51693f8c915509f66 to your computer and use it in GitHub Desktop.
This script uses wget and warcprox to generate a WARC that can be read by a player like OpenWayback.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
if [[ $# -ne 1 ]]; then | |
echo "Must only have one argument, the URL" >&2 | |
exit 1 | |
fi | |
if [[ ! -d "/var/spool/openwayback/files2/${2}" ]]; then | |
mkdir "/var/spool/openwayback/files2/${2}" | |
fi | |
cd "/var/spool/openwayback/files2/${2}" || exit 1 | |
http_proxy=127.0.0.1:8000 | |
https_proxy=127.0.0.1:8000 | |
wget --delete-after \ | |
--execute robots=off \ | |
--no-check-certificate \ | |
--no-directories \ | |
--page-requisites \ | |
--span-hosts \ | |
"${1}" #2> /dev/null |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment