------------------------------

crawl.pseudo

import List, Queue, Hash from lang.data
import fetch, normalize_url from http.utils
import write_file from lang.io

function crawl(start_url) {
  crawled = new List
  queue = new Queue
  visited = new Hash

  start_url = normalize_url(start_url)
  queue.push(start_url)

  while (not queue.empty?) {
    url = queue.pop()
    page = fetch(url)
    visited[url] = true

    for asset in page.assets {
      data = fetch(asset)
      write_file(data.filename, data.bytes)
    }
    
    for link in page.links {
      link = normalize_url(link)
      queue.push(link) if not visited[link]
    }
    
    crawled.append({ url, page.assets })
  }
  
  return crawled
}