JasonCrowe

from selenium import webdriver
from selenium.webdriver.chrome.options import Options

options = webdriver.ChromeOptions()
options.add_extension('./extension_6_0_0_0.zip')

###############
# ReCaptcha Solver CRX 6.0 
# https://stackoverflow.com/questions/34222412/load-chrome-extension-using-selenium/52420595#52420595
###############

JasonCrowe

from selenium import webdriver
from selenium.webdriver import DesiredCapabilities

profile = webdriver.FirefoxProfile()
profile.set_preference("dom.webdriver.enabled", False)
profile.set_preference('useAutomationExtension', False)
profile.set_preference("general.useragent.override","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36")
profile.update_preferences()
desired = DesiredCapabilities.FIREFOX

JasonCrowe

manifest_json = """
{
    "version": "1.0.0",
    "manifest_version": 2,
    "name": "Chrome Proxy",
    "permissions": [
        "proxy",
        "tabs",
        "unlimitedStorage",
        "storage",

JasonCrowe

window1 = driver.current_window_handle
driver.execute_script('''window.open("https://gmail.com","_blank");''')
window2 = [w for w in driver.window_handles if w != window1][0]
driver.switch_to.window(window2)

JasonCrowe

from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
from time import sleep

lum_session = random.random()
zone = 'my_lum_zone'
lum_password = 'my_lum_password'

# Amazon.com will show my location
# my_proxy = f'{zone}{lum_session}:{lum_password}@zproxy.luminati.io:22225'

JasonCrowe

def get_proxy():
    api_key = ''
    proxy_url = f'https://api.getproxylist.com/proxy?apiKey={api_key}&lastTested=600&protocol[]=http&allowsHttps=1&minDownloadSpeed=500&maxConnectTime=1'
    p = requests.get(proxy_url).json()
    return f"{p['protocol']}://{p['ip']}:{p['port']}"
      
def get_lender():
    while True:
        try:
            r = requests.get(

JasonCrowe

#infinite scroll
last_height = driver.execute_script("return document.body.scrollHeight")
SCROLL_PAUSE_TIME = 7
while True:
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    sleep(SCROLL_PAUSE_TIME)
    new_height = driver.execute_script("return document.body.scrollHeight")
    if new_height == last_height:
        break
    last_height = new_height

JasonCrowe

# -*- coding: utf-8 -*-
import scrapy
import dataset

# db = dataset.connect('sqlite:///:memory:')
db = dataset.connect('sqlite:///database.db')


class FullSiteSpider(scrapy.Spider):
    name = 'full_site'

JasonCrowe

import sqlite3
import pandas as pd

conn = sqlite3.connect("database.db")

writer = pd.ExcelWriter('order_inv_output.xlsx', engine='xlsxwriter')
pd.read_sql_query('select * from tbl_1', conn).to_excel(writer, sheet_name='Sheet 1', index=False)
pd.read_sql_query('select * from tbl_2', conn).to_excel(writer, sheet_name='Sheet 2', index=False)
pd.read_sql_query('select * from tbl_3', conn).to_excel(writer, sheet_name='Sheet 3', index=False)
writer.save()

JasonCrowe

from subprocess import call
from pkgutil import iter_modules


def install_package(name):
    if name not in [tuple_[1] for tuple_ in iter_modules()]:
        print(f"Installing {name} Library for Python")
        call(["pip", "install", name])