scaper.py

#!/usr/bin/env python3

import json
import os
import signal
import sqlite3
import sys
import threading
import time
from urllib.parse import urlparse

import requests
from selenium import webdriver

DB_FILE = 'all_tw.db'
MEDIA_DIR = 'media'
USE_FIREFOX = False

conn = None

def signal_handler(sig, frame):
    if conn:
        print("Closing database...")
        conn.close()
    print('Quitting!')
    sys.exit(0)

signal.signal(signal.SIGINT, signal_handler)

last_something_wrong_time = None
last_something_wrong_count = 0

def update_db(line):
    with open(DB_FILE, mode='r', encoding='utf-8') as dbr:
        existing_lines = dbr.readlines()

    with open(DB_FILE, mode='a', encoding='utf-8') as dbw:
        line = line + '\n'
        if line not in existing_lines:
            print('Write new pair ', line)
            dbw.write(line)
            return True
        else:
            print('Pair already exists ', line)
            return False


def download(url, sleep_seconds=0, media_dir=MEDIA_DIR):
    if not url:
        print('🔴 Empty url!')
        return

    uri_object = urlparse(url)

    dir_prefix = os.path.dirname(uri_object.path)[1:]
    compound_dir = os.path.join(media_dir, dir_prefix)

    os.makedirs(compound_dir, exist_ok=True)

    file_name = os.path.basename(uri_object.path)

    abs_file_name = os.path.join(compound_dir, file_name)
    time.sleep(sleep_seconds)

    if not os.path.exists(abs_file_name):
        resp = requests.get(url, headers={'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'})

        if resp.status_code == requests.codes.ok:
            with open(abs_file_name, 'wb') as aud:
                aud.write(resp.content)
                print(f'💾 Success writing {abs_file_name}!')
        else:
            return '<err>'
    else:
        print('✅ file already exists', abs_file_name)


if __name__ == '__main__':

    if not os.path.exists(MEDIA_DIR):
        os.mkdir(MEDIA_DIR)

    if USE_FIREFOX:
        # firefoxOptions = webdriver.FirefoxOptions()
        profile = webdriver.FirefoxProfile('/Users/io/Library/Application Support/Firefox/Profiles/7dvwm88t.socks')
        # Set proxy settings to manual
        profile.set_preference('network.proxy.type', 1)
        # Set proxy to Tor client on localhost
        profile.set_preference('network.proxy.socks', 'localhost')
        profile.set_preference('network.proxy.socks_port', 1080)
        profile.set_preference("network.proxy.socks_version", 5)
        profile.update_preferences()
        # Disable all images from loading, speeds page loading
        # http://kb.mozillazine.org/Permissions.default.image
        # profile.set_preference('permissions.default.image', 2)
        # profile.profile_dir='/Users/io/Library/Application Support/Firefox/Profiles/7dvwm88t.socks'

        driver = webdriver.Firefox(firefox_profile=profile)
    else:
        options = webdriver.ChromeOptions()
        # options.add_argument("user-data-dir=/Users/vio/Library/Application Support/Google/Chrome/Profile 3")  # Path to your chrome profile
        driver = webdriver.Chrome(options=options)

    driver.get('https://xhumanoid.com/ip')
    time.sleep(2)
    driver.get('https://ai.glossika.com/')

    try:
        driver.find_element_by_link_text('Log In').click()
        time.sleep(5)
        driver.find_element_by_css_selector('.glossika-menu-item.special').click()
        driver.find_element_by_name('email').send_keys('biozhel@gmail.com')
        time.sleep(5)
        pass_input = driver.find_element_by_name('password')
        pass_input.send_keys('kaffert02.hograccol@gmail.com')
        driver.find_element_by_css_selector('.remember-check i').click()

        # pass_input.send_keys(Keys.ENTER)
        driver.find_element_by_css_selector('.glossika-login-btn').click()
        time.sleep(4)

        # CLICK START LESSON 🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽🌽
        # driver.execute_script("""$('#bodyDom button')[2].click()""")
    except Exception as e:
        print(e)

    time.sleep(5)

    try:
        # driver.find_element_by_css_selector('#tabDom button i.fa-caret-right').click()
        pass # TEMP DB
        # driver.execute_script('''$("#tabDom button i.fa-caret-right").click()''')
    except Exception as e:
        print(e)

    conn = sqlite3.connect(DB_FILE)
    cursor = conn.cursor()

    cursor.execute("""
    CREATE TABLE IF NOT EXISTS tbl1 (
        id integer PRIMARY KEY,
        srcText text NOT NULL,
        tarText text,
        tarTranscription text,
        tarAudio text,
        srcAudio text,
        fnameChin text,
        fnameEng text
         )""")

    # api.Progress.sync,,,,,,,fetchLvlSentencesRequest,,,, currentLevelSentences
    last_seen_local_storage = {'reduxPersist:languageReducer': None}

    while True:
        try:

            driver.execute_script("""$('[name=pageNumberInput]').parent().find('i.fa-chevron-right').click()""")
            time.sleep(2)

            for storage_key in last_seen_local_storage:
                new_local_storage = driver.execute_script(f"""return localStorage.getItem('{storage_key}')""")

                if new_local_storage != last_seen_local_storage.get(storage_key):
                    print("Local storage changed")
                    last_seen_local_storage[storage_key] = new_local_storage
                    storage = json.loads(last_seen_local_storage.get(storage_key))

                    recordingSentence = storage.get('recordingSentence', None)
                    if recordingSentence:
                        id = recordingSentence.get('sentenceId')
                        audio_url = recordingSentence.get('tarAudio')
                        download(audio_url)

                        print(f'Got recordingSentence {recordingSentence}.\nTrying to store...', end='')

                        try:
                            cursor.execute('''INSERT INTO tbl1 values (?,?,?,?,?,?,?,?)
                                                            ''',
                                           [id,
                                            recordingSentence.get('srcLanguage').get('text'),
                                            recordingSentence.get('tarLanguage').get('text'),
                                            recordingSentence.get('tarLanguage').get('RECORDING'),
                                            audio_url,
                                            "urlAudioSourceLang",
                                            '>>>recordingSentence',
                                            ''
                                            ]
                                           )
                            print('Success.')
                        except Exception as e:
                            print(e, id)

                    for node in ['overviewMemorySentences', 'memorySentences']:
                        memorySentences = storage[node]

                        for idx, item in enumerate(memorySentences):
                            urlAudioTargetLang = item['tarAudio']
                            urlAudioSourceLang = item.get('srcAudio', '')
                            print(item)

                            parsed_url_obj = urlparse(urlAudioTargetLang)

                            try:
                                # ret = cursor.execute('''INSERT OR IGNORE INTO tbl1 values (?,?,?,?,?,?,?,?)
                                ret = cursor.execute('''INSERT INTO tbl1 values (?,?,?,?,?,?,?,?)
                                ''',
                                                     [item['sentenceId'],
                                                      item['srcText'],
                                                      item['tarText'],
                                                      item['tarTranscription'] or "",
                                                      urlAudioTargetLang,
                                                      urlAudioSourceLang,
                                                      parsed_url_obj.path,
                                                      ''
                                                      ]
                                                     )
                                print(ret.fetchall())
                            except Exception as e:
                                print(e)
                            file_name = os.path.basename(parsed_url_obj.path)
                            if os.path.exists(os.path.join(MEDIA_DIR, file_name)):
                                print('✅')
                            else:
                                if urlAudioTargetLang:
                                    print('🚀')
                                    threading.Thread(target=download, args=(urlAudioTargetLang, idx)).start()
                                if urlAudioSourceLang:
                                    print('🚀')
                                    threading.Thread(target=download, args=(urlAudioSourceLang, idx)).start()

                    conn.commit()
                    last_something_wrong_count = 0

                else:
                    print("storage unchanged")

                time.sleep(1.2)
                driver.execute_script('''$('[data-tip="Skip Easy"] img').click()''')
                last_something_wrong_count = 0
                last_something_wrong_time = None
        except Exception as e:
            print(e)