mvper

azurer.py

import re
import scrapy
import textwrap


class MvperSpider(scrapy.Spider):
    name = "mvper"
    start_urls = ['https://mvp.microsoft.com/en-us/MvpSearch?ex=Microsoft+Azure&sc=s']

    def parse(self, response):
        # follow links to mvp user profile pages
        #list_profiles = BeautifulSoup(response.body, 'lxml').findAll('div', {'class':'profileListItemFullName'})

        list_profiles = response.selector.css('.profileListItemFullName')

        for user in list_profiles:

            str_link = user.css('a::attr(href)').extract_first().replace(' ', '%20')
            profile_url = u'https://mvp.microsoft.com{}'.format(unicode(str_link))
        
            yield scrapy.Request(response.urljoin(profile_url),
                                 callback=self.parse_mvp_profile)

        # # follow pagination links
        # has_next = BeautifulSoup(response.body, 'lxml').findAll('div', {'class':'pager_items'})[0].img['title'] == 'Next'
        # if has_next is True:
        #     next_page = 'https://mvp.microsoft.com' + BeautifulSoup(response.body, 'lxml').findAll('div', {'class':'pager_items'})[0].findAll('a')[1]['href']
        #     yield scrapy.Request(next_page, callback=self.parse)

    def parse_mvp_profile(self, response):
        print(response.url)