basic scrapy login

simple_login_spider.py

from scrapy.http import Request
from scrapy.exceptions import CloseSpider
from scrapy.selector import Selector


class MySpider(Spider):
	name = ''
	allowed_domains = [
	]
	start_urls = [
	]

	# Login
	login_user = ''
	login_pass = ''

	login_url  = ''

	# ... the usual (Rules or something)

	def start_requests(self):
		self.log('Starting up with login...', level=log.INFO)
		return [Request(url=self.login_url, callback=self.login)]

	def login(self, response):
		self.log('Attempting to login', level=log.DEBUG)
		sel = Selector(response)
		loginform = sel.xpath('//form[@name="loginForm"]') # your form name or whatever
		formname = loginform.xpath('@name').extract()[0]
		username = loginform.xpath('//input[@type="text"]/@name').extract()[0]
		password = loginform.xpath('//input[@type="password"]/@name').extract()[0]
		return [FormRequest.from_response(response, callback=self.check_login,
			#formxpath='//form[@name="loginForm"]',
			formname=formname,
			formdata={
				username: self.login_user,
				password: self.login_pass,
			},
		)]

	def check_login(self, response):
		if 'You are logged in // Welcome, Scraper // Whatever' not in response.body:
			self.log("Login failed", level=log.ERROR)
			print response.body
			raise CloseSpider('Login failed')
			return

		self.log('Logged in', level=log.INFO)
		# don't care for this response, switch URL now
		for url in self.start_urls:
			#yield self.make_requests_from_url(url)
			yield Request(url, dont_filter=True) # callback=self.parse

	def parse(self, response):
		# do something with logged in session
		# possibly re-check at some times, that we are still logged in
		pass