Last active
December 16, 2017 03:17
-
-
Save wecsam/729901dd722955b49441fb3c3fcf0e11 to your computer and use it in GitHub Desktop.
This class is derived from requests.Session. It supports everything that requests.Session supports. Upon instantiation, this class opens Microsoft Edge, loads the specified URL, and copies the cookies, user agent string, and HTTP Referer header from the browser into this session. This allows your code to make requests as that browser.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import os, requests, threading, time, winreg | |
| from selenium import webdriver | |
| # Module-wide lock to prevent multiple WebDriver instances from running at the same time | |
| _webdriver_lock = threading.Lock() | |
| def get_windows_build_number(): | |
| key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Microsoft\Windows NT\CurrentVersion") | |
| try: | |
| return int(winreg.QueryValueEx(key, "CurrentBuild")[0]) | |
| finally: | |
| winreg.CloseKey(key) | |
| def os_path_prepend(path): | |
| os.putenv("PATH", path + os.pathsep + os.getenv("PATH")) | |
| def remove_hash(url): | |
| offset = url.find("#") | |
| if offset < 0: | |
| return url | |
| return url[:offset] | |
| # Check whether there is a WebDriver for Edge for this build of Windows 10. | |
| _windows_build_number = get_windows_build_number() | |
| _webdriver_path = os.path.join(os.path.dirname(__file__), "Edge", str(_windows_build_number), "MicrosoftWebDriver.exe") | |
| _webdriver_present = os.path.isfile(_webdriver_path) | |
| class UserAborted(Exception): | |
| def __init__(self): | |
| super().__init__("This action was aborted by the user.") | |
| class WebDriverUnavailable(Exception): | |
| def __init__(self, windows_build_number, webdriver_path): | |
| super().__init__( | |
| "Error: the Microsoft Edge WebDriver for your build of Windows was not found.\n" | |
| "You can download it at the following link and place it in the following path:\n" | |
| "Download page: https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/\n", | |
| "Your Windows build number: ", windows_build_number, "\n", | |
| "Save in folder: ", webdriver_path | |
| ) | |
| class SessionFromEdge(requests.Session): | |
| def __init__(self, url): | |
| ''' | |
| Description: | |
| This class is derived from requests.Session. It supports everything that requests.Session | |
| supports. Upon instantiation, this class opens Microsoft Edge, loads the specified URL, | |
| and copies the cookies, user agent string, and HTTP Referer header from the browser into | |
| this session. This allows your code to make requests as that browser. | |
| You can override certain methods in a derivative class; please see the descriptions for | |
| the other methods. | |
| Arguments: | |
| url: | |
| The URL that you want Microsoft Edge to load | |
| ''' | |
| if _webdriver_present: | |
| # Confirm with the user that Edge will be closed. | |
| if self.show_edge_close_warning(): | |
| super().__init__() | |
| with _webdriver_lock: | |
| # Launch Edge. This will close Edge if it is already open. | |
| driver = webdriver.Edge(_webdriver_path) | |
| try: | |
| # Navigate to the requested URL. | |
| driver.get(url) | |
| # Copy the full user agent string to the requests session. | |
| self.headers["User-Agent"] = driver.execute_script("return navigator.userAgent") | |
| # Wait for the user to get logged in or whatever. | |
| while self._waiting_for_session(driver): | |
| time.sleep(0.2) | |
| # Copy the URL of the page to the HTTP Referer header. | |
| self.headers["Referer"] = remove_hash(driver.current_url) | |
| # Copy cookies to the requests session. | |
| for cookie in driver.get_cookies(): | |
| # Translate the webdriver cookie to a requests cookie. | |
| if "expiry" in cookie: | |
| cookie["expires"] = cookie["expiry"] | |
| del cookie["expiry"] | |
| if "httpOnly" in cookie: | |
| cookie["rest"] = { | |
| "HttpOnly": cookie["httpOnly"] | |
| } | |
| del cookie["httpOnly"] | |
| # Put the cookie in the cookie jar. | |
| self.cookies.set(**cookie) | |
| finally: | |
| # Close Edge. | |
| threading.Thread(target=driver.quit).start() | |
| else: | |
| raise UserAborted() | |
| else: | |
| raise WebDriverUnavailable(_windows_build_number, _webdriver_path) | |
| def _waiting_for_session(self, driver): | |
| ''' | |
| This method is called periodically in __init__ after the URL loads and before the cookies | |
| and HTTP Referer header are copied to the session. While this method returns True, __init__ | |
| will continue to wait. | |
| You can override this method to change what __init__ waits for. | |
| ''' | |
| return False | |
| @staticmethod | |
| def show_edge_close_warning(): | |
| ''' | |
| This method is called in __init__ to warn the user that Edge will be closed. The user can | |
| cancel this operation, which will cause this method to return False. If this method returns | |
| True, then __init__ starts the WebDriver and begins loading. | |
| You can override this method to change the prompt. | |
| ''' | |
| print("Warning: if Microsoft Edge is running, it will be closed.") | |
| return input("Do you want to continue? Type Y and press Enter to continue: ").upper().startswith("Y") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment