Skip to content

Instantly share code, notes, and snippets.

@wecsam
Last active December 16, 2017 03:17
Show Gist options
  • Select an option

  • Save wecsam/729901dd722955b49441fb3c3fcf0e11 to your computer and use it in GitHub Desktop.

Select an option

Save wecsam/729901dd722955b49441fb3c3fcf0e11 to your computer and use it in GitHub Desktop.
This class is derived from requests.Session. It supports everything that requests.Session supports. Upon instantiation, this class opens Microsoft Edge, loads the specified URL, and copies the cookies, user agent string, and HTTP Referer header from the browser into this session. This allows your code to make requests as that browser.
import os, requests, threading, time, winreg
from selenium import webdriver
# Module-wide lock to prevent multiple WebDriver instances from running at the same time
_webdriver_lock = threading.Lock()
def get_windows_build_number():
key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Microsoft\Windows NT\CurrentVersion")
try:
return int(winreg.QueryValueEx(key, "CurrentBuild")[0])
finally:
winreg.CloseKey(key)
def os_path_prepend(path):
os.putenv("PATH", path + os.pathsep + os.getenv("PATH"))
def remove_hash(url):
offset = url.find("#")
if offset < 0:
return url
return url[:offset]
# Check whether there is a WebDriver for Edge for this build of Windows 10.
_windows_build_number = get_windows_build_number()
_webdriver_path = os.path.join(os.path.dirname(__file__), "Edge", str(_windows_build_number), "MicrosoftWebDriver.exe")
_webdriver_present = os.path.isfile(_webdriver_path)
class UserAborted(Exception):
def __init__(self):
super().__init__("This action was aborted by the user.")
class WebDriverUnavailable(Exception):
def __init__(self, windows_build_number, webdriver_path):
super().__init__(
"Error: the Microsoft Edge WebDriver for your build of Windows was not found.\n"
"You can download it at the following link and place it in the following path:\n"
"Download page: https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/\n",
"Your Windows build number: ", windows_build_number, "\n",
"Save in folder: ", webdriver_path
)
class SessionFromEdge(requests.Session):
def __init__(self, url):
'''
Description:
This class is derived from requests.Session. It supports everything that requests.Session
supports. Upon instantiation, this class opens Microsoft Edge, loads the specified URL,
and copies the cookies, user agent string, and HTTP Referer header from the browser into
this session. This allows your code to make requests as that browser.
You can override certain methods in a derivative class; please see the descriptions for
the other methods.
Arguments:
url:
The URL that you want Microsoft Edge to load
'''
if _webdriver_present:
# Confirm with the user that Edge will be closed.
if self.show_edge_close_warning():
super().__init__()
with _webdriver_lock:
# Launch Edge. This will close Edge if it is already open.
driver = webdriver.Edge(_webdriver_path)
try:
# Navigate to the requested URL.
driver.get(url)
# Copy the full user agent string to the requests session.
self.headers["User-Agent"] = driver.execute_script("return navigator.userAgent")
# Wait for the user to get logged in or whatever.
while self._waiting_for_session(driver):
time.sleep(0.2)
# Copy the URL of the page to the HTTP Referer header.
self.headers["Referer"] = remove_hash(driver.current_url)
# Copy cookies to the requests session.
for cookie in driver.get_cookies():
# Translate the webdriver cookie to a requests cookie.
if "expiry" in cookie:
cookie["expires"] = cookie["expiry"]
del cookie["expiry"]
if "httpOnly" in cookie:
cookie["rest"] = {
"HttpOnly": cookie["httpOnly"]
}
del cookie["httpOnly"]
# Put the cookie in the cookie jar.
self.cookies.set(**cookie)
finally:
# Close Edge.
threading.Thread(target=driver.quit).start()
else:
raise UserAborted()
else:
raise WebDriverUnavailable(_windows_build_number, _webdriver_path)
def _waiting_for_session(self, driver):
'''
This method is called periodically in __init__ after the URL loads and before the cookies
and HTTP Referer header are copied to the session. While this method returns True, __init__
will continue to wait.
You can override this method to change what __init__ waits for.
'''
return False
@staticmethod
def show_edge_close_warning():
'''
This method is called in __init__ to warn the user that Edge will be closed. The user can
cancel this operation, which will cause this method to return False. If this method returns
True, then __init__ starts the WebDriver and begins loading.
You can override this method to change the prompt.
'''
print("Warning: if Microsoft Edge is running, it will be closed.")
return input("Do you want to continue? Type Y and press Enter to continue: ").upper().startswith("Y")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment