[scrapy site crawler] crawls a whole site

crawl-site.py

# run with
# `scrapy runspider crawl-site.py`

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class MySpider(CrawlSpider):
    name = 'domain.com'
    allowed_domains = ['domain.com']
    start_urls = ['https://www.domain.com']

    rules = (
        Rule(LinkExtractor(), callback='parse_item', follow=True),
    )

    # save to file
    """
    def parse_item(self, response):
        filename = response.url.split("/")[-2] + '.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
    """