Run for ever spyder, modifying start_requests() method. (private)

forever_spider.py

# Se coloca el inicio de la ejecucion en start_request()
class Foo(Spider):
    name = 'foo'
    allowed_domains = ['foo.com']

    def start_requests(self):
        while True:
            data = self.coll.find({'status': 'unscraped'}).limit(5000)

            if not data:
                break

            for row in data:
                pin = row['pin']
                url = 'http://foo.com/Pages/PIN-Results.aspx?PIN={}'.format(pin)
                yield Request(url, meta={'pin': pin})