scrapy-selenium-chrome-configuration.py

from selenium import webdriver
from scrapy.http import HtmlResponse
from selenium.webdriver.chrome.options import Options

"""
  whatch chromedriver version supporting the chrome browser version in system
"""

class SeleniumMiddleware(object):
  def process_request(self,request,spider):
    options = Options()
    options.add_argument('--headless')
    options.add_argument('--disable-gpu')
    options.add_argument('--silent')      # faster
    options.add_argument('--log-level=3') # less verbosity
    # options.binary_location = ''

    capabilities = {}
    # capabilities['platform'] = 'Linux'
    # capabilities['version'] = '16.04'
      
    driver = webdriver.Chrome(
      executable_path='/usr/local/bin/chromedriver',
      chrome_options=options,
      desired_capabilities=capabilities)
    try:
      driver.get(request.url)
      driver.switch_to.frame('g_iframe')
      body = driver.page_source
      return HtmlResponse(driver.current_url,body=body,encoding='utf-8',request=request)
    except:
      driver.quit()