Skip to content

Instantly share code, notes, and snippets.

@pricejn2
Forked from dchaplinsky/double_ip_test.py
Created February 26, 2017 05:56
Show Gist options
  • Save pricejn2/99d35fbf20797e8dbd0682c32e91a617 to your computer and use it in GitHub Desktop.
Save pricejn2/99d35fbf20797e8dbd0682c32e91a617 to your computer and use it in GitHub Desktop.
Simple scrapy spider to utilize bindaddress meta param
from scrapy.spider import BaseSpider
from scrapy.conf import settings
from scrapy.http import Request
from random import choice
class DoubleIPSpider(BaseSpider):
name = "double_ip_test"
allowed_domains = ["httpbin.org"]
def start_requests(self):
yield self.make_requests_from_url("http://httpbin.org/ip")
yield self.make_requests_from_url("http://httpbin.org/ip")
yield self.make_requests_from_url("http://httpbin.org/ip")
yield self.make_requests_from_url("http://httpbin.org/ip")
yield self.make_requests_from_url("http://httpbin.org/ip")
def make_requests_from_url(self, url):
if settings.get("BIND_TO_IP"):
ip = choice(settings["BIND_TO_IP"])
print(ip)
return Request(url, dont_filter=True,
meta={'bindaddress': (
ip,
0)})
else:
return super(DoubleIPSpider, self).make_requests_from_url(url)
def parse(self, response):
print(response.body)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment