DynamicFetcher.py

import os
import time
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

REMOTE_DRIVER = os.environ.get('REMOTE_DRIVER')

class DynamicFetcher:
    def __init__(self):
        self.driver = None

    def prepare(self):
        if REMOTE_DRIVER is not None:
            driver = webdriver.Remote(
                command_executor=REMOTE_DRIVER,
                desired_capabilities=DesiredCapabilities.CHROME,
            )
        else:
            driver = webdriver.Chrome()

        driver.get('about:blank')
        self.driver = driver

    def is_prepared(self):
        return self.driver is not None

    def fetch(self, url):
        assert self.is_prepared()
        driver = self.driver

        driver.get(url)
        time.sleep(1) # FIXME: wait for loading

        return {
            'target_url': url,
            'url': driver.current_url,
            'title': driver.title,
            'html': driver.page_source,
        }

    def release(self):
        driver = self.driver
        if driver is not None:
            driver.quit()
        self.driver = None