airbnb_get_pdp - before reviews.py

"""
python3.6
Scrapy + Selenium
"""
for profile_url in profile_urls_distinct:
    self.logger.info('Home #' + str(q))
    self.driver.get(profile_url)
    q = q+1
    sleep(10)
    link_to_home = profile_url
    profile_scrapy_selector = Selector(text = self.driver.page_source)
    property_name = profile_scrapy_selector.xpath('//*[@itemprop="name"]//h1/span/text()').extract()
    price_night = profile_scrapy_selector.xpath('//*[@class ="_doc79r"]/text()').extract_first()
    rating_overall = profile_scrapy_selector.xpath('//*[@itemprop="ratingValue"]/@content').extract_first()
    rating_split = {}
    rating_categories = profile_scrapy_selector.xpath('//*[@class="_iq8x9is"]/span/text()').extract()
    rating_stars = profile_scrapy_selector.xpath('//*[@class="_iq8x9is"]/div/span//@aria-label').extract()
    i = 0
    for i in range(len(rating_categories)):
        rating_split[rating_categories[i]] = rating_stars[i]
        i = i+1
    home_neighborhood_short = profile_scrapy_selector.xpath('//*[@class="_6z3til"]//*[@class="_czm8crp"]/span/text()').extract()