link_follower.py

import scrapy


class PageItem(scrapy.Item):
    title = scrapy.Field()


class LinkReaderSpider(scrapy.Spider):
    name = 'link_follower'
    start_urls = ['http://gabrielfv.com']

    def parse(self, response):
        data = PageItem()
        data['title'] = response.css('title::text').extract_first()
        yield data

        links = response.css('a')
        for link in links:
            yield scrapy.Request(
                link.xpath('./@href').extract_first(),
                callback=self.parse_links
            )

    def parse_links(self, response):
        data = PageItem()
        data['title'] = response.css('title::text').extract_first()
        return data