Scrap Bio's from Facebook Research

spider-research.py

import scrapy
from research.items import ResearchItem

class ResearchSpider(scrapy.Spider):
    start_urls = [
        'https://research.fb.com/people/'
    ]
    name = 'research'



    def parse(self, response):
        for talk in response.css('div.person-panel'):
            href = talk.css('a::attr(href)').extract_first()
            full_url = response.urljoin(href)

            yield scrapy.Request(full_url, callback=self.parse_book)

        for href in response.css('a.btn--white'):
            yield response.follow(href, self.parse)


    def parse_book(self, response):
        name = response.css('.person-header h1::text').extract_first()
        tagline = response.css('.person-header p::text').extract_first()
        bio = response.css('.content-wrapper p::text').extract()
        website = response.css('.col-6-lrg a::attr(href)').extract_first()
        avatar = response.css('.header > div > div > div > div.col-12.col-3-lrg > figure::attr(style)').extract_first()
        intrests = response.css('div.page-content > div > div.col-12.col-8-lrg > section:nth-child(2) > div > p::text').extract_first()

        yield ResearchItem(name=name, tagline=tagline, bio=bio, website=website,avatar=avatar,intrests=intrests)