Skip to content

Instantly share code, notes, and snippets.

@osya
Last active August 29, 2015 14:12
Show Gist options
  • Save osya/e56ded82e1e7b2ae4896 to your computer and use it in GitHub Desktop.
Save osya/e56ded82e1e7b2ae4896 to your computer and use it in GitHub Desktop.
How to make delay for one certain Request in Scrapy
@defer.inlineCallbacks
def parse_next_page(self, response):
log.msg('Test: in parse_next_page, status=%s' % response.status, level=log.WARNING)
res = []
if response.status != 302:
text = ''
try:
text = response.xpath('./component/text()').extract()[0]
except:
pass
table = Selector(text=text).xpath('.//table[starts-with(@id, "id")]/tbody/tr')
for resp in self.parse_lot_list(table):
r = yield resp
res.append(r)
self.page_number += 1
# Вычисление начальной страницы
o = urlparse(response.url)
url = o.scheme + '://' + o.netloc + o.path + '?bidKindId=2'
req = Request(url, callback=self.goto_next_page, dont_filter=True,)
if response.status != 302:
r = yield req
else:
r = yield self.later(req, timeout=20)
res.append(r)
defer.returnValue(res)
def later(self, result, timeout):
d = defer.Deferred()
reactor.callLater(timeout, d.callback, result)
return d
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment