CS 891 Web Archiving Seminar

.setup

#!/usr/bin/env bash

bashColors=".bash_colors"
dlBashColors="https://raw.githubusercontent.com/mercuriev/bash_colors/master/bash_colors.sh"
wgetCol="collections/wget"
heritrixCol="collections/heritrix"
wgetUA="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"
wgetArgs="--mirror
--warc-cdx
--no-warc-compression
--adjust-extension
--page-requisites
--html-extension
--convert-links
--execute robots=off
--no-directories
--span-hosts
"

function curlExists () {
    type -t "curl" | grep -q 'function'
}

function pvExists () {
  type -t "pv" | grep -q 'function'
}

if [[ -f "$bashColors" ]]; then
  source ${bashColors}
else
  if [[ curlExists ]]; then
    curl ${dlBashColors} > ${bashColors}
  else
    wget --user-agent="$wgetUA" -O "$dlBashColors" "$bashColors"
  fi
  source ${bashColors}
  if [[ ! curlExists ]]; then
    echo "$(clr_bold clr_red You Need To Download Curl)"
  fi
fi

function setupWayback () {
  if [[ ! -d "$wgetCol" ]]; then
    wb-manager init wget
  fi
  if [[ ! -d "$heritrixCol" ]]; then
    wb-manager init heritrix
  fi
}

function checkDirs () {
  for maybeDir in "$@"
  do
    if [[ ! -d $maybeDir ]]; then
      mkdir ${maybeDir}
    fi
  done
}

function wgetArchive () {
  if [[ ! -f "$1.warc" ]]; then
    if [[ $2 = "http://example.com" ]]; then
      wget ${wgetArgs} --user-agent="$wgetUA" --domains=example.com,www.example.com,cdn.example.com \
      --warc-file="$1" "$2"
    else
      wget ${wgetArgs} --user-agent="$wgetUA"  --warc-file="$1" "$2"
    fi
  fi
}

function copyWarcCdxTo () {
  cp *.cdx *.warc "$1"
}



function wgetIntro() {
  echo "$(clr_bold clr_cyan Hello) I will be using $(clr_bold clr_cyan wget) to archive two web pages" | pv -qL 25
  echo "The arguments you give to wget are:" | pv -qL 25
  for it in $wgetArgs
  do
    argColor=$(clr_escape "$it" $CLR_BOLD $CLR_CYAN)
    case "$it" in
      "--mirror" )
        echo -e "${argColor}: Recursive infinite crawl depth with timesamping" | pv -qL 25
      ;;
      "--warc-cdx")
        boo=$(clr_escape "--warc-file" $CLR_BOLD $CLR_CYAN)
        echo -e "${boo}: Create the WARC file with the supplied name" | pv -qL 25
        echo -e "${argColor}: Create a cdx file along side the WARC file" | pv -qL 25
      ;;
      "--no-warc-compression")
        echo -e "${argColor}: Do not gzip compress the WARC file" | pv -qL 25
      ;;
      "--adjust-extension")
        echo -e "${argColor}: Ensure files with MIME type ‘application/xhtml+xml’ or ‘text/html’ files are saved as .html" | pv -qL 25
      ;;
      "--page-requisites")
        echo -e "${argColor}: Download all the files 'necessary' to view the given HTML page" | pv -qL 25
      ;;
      "--convert-links")
        echo -e "${argColor}: Make the links contained in the downloaded html files relative to the directory they were saved in" | pv -qL 25
      ;;
      "robots=off")
        boo=$(clr_escape "--execute" $CLR_BOLD $CLR_CYAN)
        echo -e "${boo} ${argColor}: Disregard the robots.txt file" | pv -qL 25
      ;;
      "--no-directories")
        echo -e "${argColor}: Do not create a directory structure that matches the path of the URL" | pv -qL 25
      ;;
      "--span-hosts")
        echo -e ${argColor}: "Crawl all the hosts" | pv -qL 25
      ;;
    esac
  done
  argColor=$(clr_escape "--user-agent" $CLR_BOLD $CLR_CYAN)
  echo -e ${argColor}: "Set the user-agent string for wget" | pv -qL 25
}

README.md

slides prez

archiveWget.sh

#!/usr/bin/env bash
source .setup

cwd=$(pwd)
exampleDir="$cwd/exampleDir"
reactRouterDir="$cwd/reactRouterDir"

wgetIntro

checkDirs ${exampleDir} ${reactRouterDir}

cd ${exampleDir}
echo "Now lets archive http://example.com" | pv -qL 25
wgetArchive exampleWarc http://example.com
if [[ ! -f "$cwd/exampleWarc.warc" ]]; then
  echo "Yay! We archied http://example.com using wget" | pv -qL 25
  copyWarcCdxTo ${cwd}
fi
cd ${reactRouterDir}
echo "Now lets attempt to archive somethign more complex. The documentation for React Router :) " | pv -qL 25
wgetArchive reactRouterWarc https://reacttraining.com/react-router/web/guides/quick-start
if [[ ! -f "$cwd/reactRouterWarc.warc" ]]; then
  echo "Yay! We archied https://reacttraining.com/react-router/web/guides/quick-start using wget" | pv -qL 25
  copyWarcCdxTo ${cwd}
fi

exampleWarc.warc

WARC/1.0
WARC-Type: warcinfo
Content-Type: application/warc-fields
WARC-Date: 2017-10-03T05:23:03Z
WARC-Record-ID: <urn:uuid:d4fb7611-a076-444e-82b0-4a300dfd560e>
WARC-Filename: exampleWarc.warc
WARC-Block-Digest: sha1:I4E6XJGFWIZD4G6A6KHNIPSYMXKZUGLU
Content-Length: 586

software: Wget/1.17.1 (linux-gnu)
format: WARC File Format 1.0
conformsTo: http://bibnum.bnf.fr/WARC/WARC_ISO_28500_version1_latestdraft.pdf
robots: off
wget-arguments: "--mirror" "--warc-cdx" "--no-warc-compression" "--adjust-extension" "--page-requisites" "--html-extension" "--convert-links" "--execute" "robots=off" "--no-directories" "--span-hosts" "--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" "--domains=example.com,www.example.com,cdn.example.com" "--warc-file=exampleWarc" "http://example.com" 



WARC/1.0
WARC-Type: request
WARC-Target-URI: http://example.com/
Content-Type: application/http;msgtype=request
WARC-Date: 2017-10-03T05:23:03Z
WARC-Record-ID: <urn:uuid:79aee7ef-9f60-42b4-a411-5923518bbd69>
WARC-IP-Address: 2606:2800:220:1:248:1893:25c8:1946
WARC-Warcinfo-ID: <urn:uuid:d4fb7611-a076-444e-82b0-4a300dfd560e>
WARC-Block-Digest: sha1:NPGO3XYBPAABOPGEUYINR3NTMAIUMJRH
Content-Length: 220

GET / HTTP/1.1
User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36
Accept: */*
Accept-Encoding: identity
Host: example.com
Connection: Keep-Alive



WARC/1.0
WARC-Type: response
WARC-Record-ID: <urn:uuid:f5ac92a9-e193-480e-baaf-d82c47d87372>
WARC-Warcinfo-ID: <urn:uuid:d4fb7611-a076-444e-82b0-4a300dfd560e>
WARC-Concurrent-To: <urn:uuid:79aee7ef-9f60-42b4-a411-5923518bbd69>
WARC-Target-URI: http://example.com/
WARC-Date: 2017-10-03T05:23:03Z
WARC-IP-Address: 2606:2800:220:1:248:1893:25c8:1946
WARC-Block-Digest: sha1:2C7TU56EJORH6BF6TCWTMCD6PVEMIWT6
WARC-Payload-Digest: sha1:B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A
Content-Type: application/http;msgtype=response
Content-Length: 1597

HTTP/1.1 200 OK
Accept-Ranges: bytes
Cache-Control: max-age=604800
Content-Type: text/html
Date: Tue, 03 Oct 2017 05:23:03 GMT
Etag: "359670651+gzip"
Expires: Tue, 10 Oct 2017 05:23:03 GMT
Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT
Server: ECS (atl/FC90)
Vary: Accept-Encoding
X-Cache: HIT
Content-Length: 1270

<!doctype html>
<html>
<head>
    <title>Example Domain</title>

    <meta charset="utf-8" />
    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <style type="text/css">
    body {
        background-color: #f0f0f2;
        margin: 0;
        padding: 0;
        font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
        
    }
    div {
        width: 600px;
        margin: 5em auto;
        padding: 50px;
        background-color: #fff;
        border-radius: 1em;
    }
    a:link, a:visited {
        color: #38488f;
        text-decoration: none;
    }
    @media (max-width: 700px) {
        body {
            background-color: #fff;
        }
        div {
            width: auto;
            margin: 0 auto;
            border-radius: 0;
            padding: 1em;
        }
    }
    </style>    
</head>

<body>
<div>
    <h1>Example Domain</h1>
    <p>This domain is established to be used for illustrative examples in documents. You may use this
    domain in examples without prior coordination or asking for permission.</p>
    <p><a href="http://www.iana.org/domains/example">More information...</a></p>
</div>
</body>
</html>


WARC/1.0
WARC-Type: metadata
WARC-Record-ID: <urn:uuid:a17053cc-c276-41bb-b154-220e33893e8d>
WARC-Warcinfo-ID: <urn:uuid:d4fb7611-a076-444e-82b0-4a300dfd560e>
WARC-Target-URI: metadata://gnu.org/software/wget/warc/MANIFEST.txt
WARC-Date: 2017-10-03T05:23:03Z
WARC-Block-Digest: sha1:CRSRUWV4ISHW72AGDHPBH7RGBDSU723S
Content-Type: text/plain
Content-Length: 48

<urn:uuid:d4fb7611-a076-444e-82b0-4a300dfd560e>


WARC/1.0
WARC-Type: resource
WARC-Record-ID: <urn:uuid:90a380a0-c704-4477-81fc-3ad330fb000c>
WARC-Warcinfo-ID: <urn:uuid:d4fb7611-a076-444e-82b0-4a300dfd560e>
WARC-Concurrent-To: <urn:uuid:a17053cc-c276-41bb-b154-220e33893e8d>
WARC-Target-URI: metadata://gnu.org/software/wget/warc/wget_arguments.txt
WARC-Date: 2017-10-03T05:23:03Z
WARC-Block-Digest: sha1:HTQNGRCNFKIMEELR6GOAI4IYXT6CTLHY
Content-Type: text/plain
Content-Length: 410

"--mirror" "--warc-cdx" "--no-warc-compression" "--adjust-extension" "--page-requisites" "--html-extension" "--convert-links" "--execute" "robots=off" "--no-directories" "--span-hosts" "--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" "--domains=example.com,www.example.com,cdn.example.com" "--warc-file=exampleWarc" "http://example.com" 


WARC/1.0
WARC-Type: resource
WARC-Record-ID: <urn:uuid:3db8b68a-7fea-40a5-841a-5ea98c4e6cb1>
WARC-Warcinfo-ID: <urn:uuid:d4fb7611-a076-444e-82b0-4a300dfd560e>
WARC-Concurrent-To: <urn:uuid:a17053cc-c276-41bb-b154-220e33893e8d>
WARC-Target-URI: metadata://gnu.org/software/wget/warc/wget.log
WARC-Date: 2017-10-03T05:23:03Z
WARC-Block-Digest: sha1:35JP22NWKHB5UOEIAFL6N7YVULQDQ37C
Content-Type: text/plain
Content-Length: 719

Opening WARC file ‘exampleWarc.warc’.

--2017-10-03 01:23:03--  http://example.com/
Resolving example.com (example.com)... 2606:2800:220:1:248:1893:25c8:1946, 93.184.216.34
Connecting to example.com (example.com)|2606:2800:220:1:248:1893:25c8:1946|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 1270 (1.2K) [text/html]
Saving to: ‘index.html’

     0K .                                                     100%  212M=0s

2017-10-03 01:23:03 (212 MB/s) - ‘index.html’ saved [1270/1270]

FINISHED --2017-10-03 01:23:03--
Total wall clock time: 0.05s
Downloaded: 1 files, 1.2K in 0s (212 MB/s)
Converting links in index.html... nothing to do.
Converted links in 1 files in 0 seconds.

reactRouterWarc.warc