Skip to content

Instantly share code, notes, and snippets.

@SaBenBurra
Created September 2, 2023 12:01
Show Gist options
  • Save SaBenBurra/29dbb7e64a26e5328ce853e491ddeb03 to your computer and use it in GitHub Desktop.
Save SaBenBurra/29dbb7e64a26e5328ce853e491ddeb03 to your computer and use it in GitHub Desktop.
web scraping with selenium faster
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
start_time = time.time()
url = ""
query = "" # xpath query
options = Options()
options.add_argument("--disable-extensions")
options.add_argument("--disable-javascript")
options.add_argument("--blink-settings=imagesEnabled=false")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option("useAutomationExtension", False)
options.add_argument("--disable-gpu")
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--disable-extensions")
options.add_argument("--disable-infobars")
options.add_argument("--disable-popup-blocking")
options.page_load_strategy = "none"
service = Service("./chromedriver")
driver = webdriver.Chrome(service=service, options=options)
wait = WebDriverWait(driver, 20)
driver.get(url)
wait.until(EC.presence_of_element_located((By.XPATH, query)))
driver.execute_script("window.stop();")
print(
driver.find_element(
By.XPATH,
query,
).text
)
print("Elapsed Time: " + str(time.time() - start_time))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment