gistfile1.py

from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.item import Item, Field

class LinkItem(Item):
    url = Field()
    referer = Field()
    status = Field()

class Spider(CrawlSpider):
    name = "ua_spider"
    allowed_domains = ["albany.edu"]
    start_urls = ["http://albany.edu", ]
    handle_httpstatus_list = [404]
    rules = (Rule(SgmlLinkExtractor(), callback='parse_item', follow=True),)

    def parse_item(self, response):
        if response.status == 404:
            item = LinkItem()
            item['url'] = response.url
            item['referer'] = response.request.headers.get('Referer')
            item['status'] = response.status

            return item