Scrapy - submit form to set up cookies, then scrape known URLs...

submitthenscrape.py

from scrapy.http import FormRequest, Request
from scrapy.selector import HtmlXPathSelector
from scrapy.spider import BaseSpider

BASE_URL = 'http://www.url.com/'

class ThingSpider(BaseSpider):
    name = 'thing'
    start_urls = ['%ssearchThing.php' % BASE_URL]

    def parse(self, response):
        """
        Simulates submission of the initial search form.
        """
        return [FormRequest.from_response(response,
                    callback=self.after_initial_search)]

    def after_initial_search(self, response):
        """
        Determines if we got a page of results and yields requests for all the
        results pages, if so.
        """
        if 'Search The Things' in response.body:
            raise ValueError('We appear to still be on the search form after submitting it.')
        for url in ['%ssearchThing.php?searching=true&pageNum=%s' % (BASE_URL, p*20)
                    for p in range(71)]:
            yield Request(url, self.parse_search_results)