Skip to content

Instantly share code, notes, and snippets.

@dchaplinsky
Created January 4, 2014 00:09
Show Gist options
  • Save dchaplinsky/8249328 to your computer and use it in GitHub Desktop.
Save dchaplinsky/8249328 to your computer and use it in GitHub Desktop.
Simple scrapy spider to utilize bindaddress meta param
from scrapy.spider import BaseSpider
from scrapy.conf import settings
from scrapy.http import Request
from random import choice
class DoubleIPSpider(BaseSpider):
name = "double_ip_test"
allowed_domains = ["httpbin.org"]
def start_requests(self):
yield self.make_requests_from_url("http://httpbin.org/ip")
yield self.make_requests_from_url("http://httpbin.org/ip")
yield self.make_requests_from_url("http://httpbin.org/ip")
yield self.make_requests_from_url("http://httpbin.org/ip")
yield self.make_requests_from_url("http://httpbin.org/ip")
def make_requests_from_url(self, url):
if settings.get("BIND_TO_IP"):
ip = choice(settings["BIND_TO_IP"])
print(ip)
return Request(url, dont_filter=True,
meta={'bindaddress': (
ip,
0)})
else:
return super(DoubleIPSpider, self).make_requests_from_url(url)
def parse(self, response):
print(response.body)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment