Coin Selenium

sp.py

# -*- coding: utf-8 -*-
from time import sleep
import scrapy
from scrapy.selector import Selector
from selenium.common.exceptions import ElementClickInterceptedException
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from shutil import which
 
 
class CoinSpiderSelenium(scrapy.Spider):
    name = 'coin_selenium'
    allowed_domains = ['www.livecoin.net/en'] 
    start_urls = [
        'https://www.livecoin.net/en'
    ]
 
    def __init__(self):
        chrome_options = Options()
        chrome_options.add_argument("--headless")
 
        chrome_path = which("chromedriver")
 
        # driver = webdriver.Chrome(executable_path=chrome_path, options=chrome_options)
        driver = webdriver.Chrome(executable_path=chrome_path)
        driver.set_window_size(1920, 1080)
        driver.get("https://www.livecoin.net/en")
 
        # rur_tab = driver.find_elements_by_class_name("filterPanelItem___2z5Gb")
        # rur_tab[4].click()
 
        #code added after
        # try to scroll to the end of page but it works wrong. 
        # To troubleshoot the promblem can't click on button.
        show_more_button = driver.find_element_by_xpath("//div[@class='showMoreContainer___2HlS0']/button")
        # sleep(30)
        try:
            show_more_button.click()
        except ElementClickInterceptedException:
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            show_more_button.click()
        
 
        self.html = driver.page_source
        driver.close()
 
    def parse(self, response):
        resp = Selector(text=self.html)
        for currency in resp.xpath("//div[contains(@class, 'ReactVirtualized__Table__row tableRow___3EtiS ')]"):
            yield {
                'currency pair': currency.xpath(".//div[1]/div/text()").get(),
                'volume(24h)': currency.xpath(".//div[2]/span/text()").get()
            }