archive_org_pdf_uploader_v3.py

archive_org_pdf_uploader_v3.py

from __future__ import annotations

import argparse
import configparser
import os
import re
import shutil
import sqlite3
import sys
import tempfile
import threading
import time
import tkinter as tk
import unicodedata
from pathlib import Path
from tkinter import filedialog, messagebox, scrolledtext, ttk
from urllib.parse import quote, urlparse

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait


DEFAULT_FOLDER = Path(r"G:\De pus pe FTP 2")
FIREFOX_BINARY = Path(r"C:\Program Files\Mozilla Firefox\firefox.exe")
GECKODRIVER_BINARY = Path(r"C:\WINDOWS\geckodriver.exe")
YEAR_RE = re.compile(r"(?<!\d)((?:18|19|20)\d{2})(?:\s*[-–]\s*((?:18|19|20)\d{2}))?(?!\d)")
OPEN_FIREFOX_SESSIONS: list[tuple[webdriver.Firefox, Path]] = []


def archive_identifier(value: str) -> str:
    value = value.strip()
    parsed = urlparse(value if "://" in value else "https://archive.org/details/" + value)
    parts = [part for part in parsed.path.split("/") if part]
    if len(parts) >= 2 and parts[0] in {"details", "edit", "download"}:
        identifier = parts[1]
    elif parsed.path == "/upload/":
        from urllib.parse import parse_qs
        identifier = parse_qs(parsed.query).get("identifier", [""])[0]
    elif len(parts) == 1:
        identifier = parts[0]
    else:
        identifier = ""
    if not re.fullmatch(r"[A-Za-z0-9._-]+", identifier):
        raise ValueError("Link sau identifier Archive.org invalid.")
    return identifier


def is_new_upload_url(value: str) -> bool:
    parsed = urlparse(value.strip())
    return parsed.netloc.casefold() in {"archive.org", "www.archive.org"} and parsed.path.rstrip("/") == "/upload" and not parsed.query


def fill_new_item_metadata(driver: webdriver.Firefox, log) -> str:
    log("Astept 30 de secunde generarea titlului si a URL-ului nou...")
    time.sleep(30)
    title_element = WebDriverWait(driver, 30).until(lambda d: d.find_element(By.ID, "page_title"))
    title = (title_element.get_attribute("textContent") or "").strip()
    if not title:
        raise RuntimeError("Archive.org nu a generat titlul itemului nou.")
    log(f"Titlu generat: {title}")

    driver.find_element(By.ID, "description").click()
    frame = WebDriverWait(driver, 15).until(
        lambda d: next((item for item in d.find_elements(By.ID, "description_editor-wysiwyg-iframe") if item.is_displayed()), False)
    )
    driver.switch_to.frame(frame)
    body = driver.find_element(By.TAG_NAME, "body")
    body.clear()
    body.send_keys(title)
    driver.switch_to.default_content()

    driver.find_element(By.ID, "subjects").click()
    subject_input = WebDriverWait(driver, 15).until(
        lambda d: d.find_element(By.CSS_SELECTOR, 'input[aria-label="Edit Subject Tags"]')
    )
    subject_input.clear()
    subject_input.send_keys(title)
    subject_input.send_keys(Keys.TAB)

    WebDriverWait(driver, 15).until(
        lambda d: (d.find_element(By.ID, "description").get_attribute("textContent") or "").strip() == title
        and (d.find_element(By.ID, "subjects").get_attribute("textContent") or "").strip() == title
    )
    log("Description si Subject Tags au fost completate cu titlul generat.")
    return title


def click_final_upload(driver: webdriver.Firefox, log) -> None:
    button = WebDriverWait(driver, 60).until(
        lambda d: next((item for item in d.find_elements(By.ID, "upload_button") if item.is_displayed() and item.is_enabled()), False)
    )
    driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", button)
    time.sleep(1)
    button.click()
    log(f"Buton apasat: {(button.get_attribute('textContent') or '').strip()}")
    log("Astept finalizarea transferului; nu inchide Firefox sau aplicatia.")
    WebDriverWait(driver, 6 * 60 * 60, poll_frequency=2).until(
        lambda d: "/upload/" not in d.current_url
        or any(text in (d.find_element(By.TAG_NAME, "body").get_attribute("textContent") or "").lower()
               for text in ("upload complete", "files added", "item updated"))
    )
    log(f"Transfer finalizat. Pagina curenta: {driver.current_url}")


def pdf_sort_key(path: Path) -> tuple[int, int, str]:
    match = YEAR_RE.search(path.name)
    if match:
        return (-int(match.group(1)), -int(match.group(2) or match.group(1)), path.name.casefold())
    return (1, 1, path.name.casefold())


def find_pdfs(folder: Path) -> list[Path]:
    return sorted((p.resolve() for p in folder.rglob("*.pdf") if p.is_file()), key=pdf_sort_key)


def filename_key(name: str) -> str:
    return unicodedata.normalize("NFC", name).casefold()


def unique_local_pdfs(files: list[Path]) -> tuple[list[Path], list[Path]]:
    unique: list[Path] = []
    duplicates: list[Path] = []
    seen: set[str] = set()
    for path in files:
        key = filename_key(path.name)
        if key in seen:
            duplicates.append(path)
        else:
            seen.add(key)
            unique.append(path)
    return unique, duplicates


def read_existing_pdf_names(driver: webdriver.Firefox, identifier: str, log) -> set[str]:
    edit_url = f"https://archive.org/edit.php?edit-files=1&identifier={quote(identifier)}"
    log(f"Verific fisierele existente: {edit_url}")
    driver.get(edit_url)
    WebDriverWait(driver, 90).until(
        lambda d: d.find_elements(By.CSS_SELECTOR, ".x-tree-node-icon.file-pdf")
        or "File editor for" in (d.find_element(By.TAG_NAME, "body").get_attribute("textContent") or "")
    )
    time.sleep(2)
    names = driver.execute_script(
        """
        return Array.from(document.querySelectorAll('.x-tree-node-icon.file-pdf'))
          .map(img => {
            const node = img.closest('.x-tree-node-el');
            const span = node ? node.querySelector('a span') : null;
            return span ? span.textContent.trim() : '';
          })
          .filter(Boolean);
        """
    )
    result = {filename_key(name) for name in names}
    log(f"Archive.org contine deja {len(result)} fisiere PDF cu nume distincte.")
    return result


def normalized_words(value: str) -> set[str]:
    value = "".join(c for c in unicodedata.normalize("NFKD", value) if not unicodedata.combining(c)).lower()
    return set(re.findall(r"[a-z0-9]+", value))


def relevant_pdfs(folder: Path, url: str) -> list[Path]:
    identifier = archive_identifier(url)
    ignored = {"buletinul", "institutului", "politehnic", "din", "iasi", "bucuresti", "seria"}
    keywords = normalized_words(identifier) - ignored
    threshold = max(1, (len(keywords) + 1) // 2)
    files = find_pdfs(folder)
    matches = [p for p in files if len(normalized_words(p.stem) & keywords) >= threshold]
    return matches or files


def default_firefox_profile() -> Path:
    base = Path(os.environ["APPDATA"]) / "Mozilla" / "Firefox"
    ini = configparser.ConfigParser()
    ini.read(base / "profiles.ini", encoding="utf-8")
    install_sections = [s for s in ini.sections() if s.startswith("Install")]
    rel = ini[install_sections[0]].get("Default", "") if install_sections else ""
    if not rel:
        for section in ini.sections():
            if section.startswith("Profile") and ini[section].getboolean("Default", fallback=False):
                rel = ini[section].get("Path", "")
                break
    profile = base / rel
    if not profile.is_dir():
        raise FileNotFoundError("Nu am găsit profilul Firefox implicit.")
    return profile


def clone_firefox_profile(source: Path, target: Path) -> None:
    target.mkdir(parents=True, exist_ok=True)
    cookies = source / "cookies.sqlite"
    if cookies.exists():
        source_db = sqlite3.connect(f"file:{cookies.as_posix()}?mode=ro", uri=True, timeout=10)
        target_db = sqlite3.connect(target / "cookies.sqlite")
        try:
            source_db.backup(target_db)
        finally:
            target_db.close()
            source_db.close()
    for name in ("permissions.sqlite", "cert9.db", "key4.db", "pkcs11.txt"):
        item = source / name
        if item.is_file():
            shutil.copy2(item, target / name)


def upload_files(driver: webdriver.Firefox, files: list[Path], delay: float, log, finalize: bool) -> None:
    wait = WebDriverWait(driver, 60)
    def visible_row_name(row) -> str:
        value = row.find_element(By.CSS_SELECTOR, "span.file").get_attribute("textContent")
        return (value or "").strip()

    expected_names = [path.name for path in files]
    if len(expected_names) != len(set(expected_names)):
        raise RuntimeError("Lista locala contine nume PDF duplicate. Operatia a fost oprita.")
    log(f"Adaug {len(files)} PDF-uri individual, la interval de {delay:.1f} secunde.")
    files_to_add_individually = files
    for index, path in enumerate(files_to_add_individually, 1):
        rows_before = driver.find_elements(By.CSS_SELECTOR, "#file_table tbody tr")
        before_count = len(rows_before)
        if before_count != index - 1:
            raise RuntimeError(
                f"Protectie anti-duplicate: tabelul are {before_count} randuri in loc de {index - 1}."
            )
        inputs = driver.find_elements(By.CSS_SELECTOR, "input[type='file']")
        if not inputs:
            raise RuntimeError("Nu găsesc controlul de selectare a fișierelor pe pagina Archive.org.")
        control_id = "file_input_initial" if index == 1 else "file_input_add"
        control = wait.until(lambda d: d.find_element(By.ID, control_id))
        if index > 1:
            driver.execute_script("arguments[0].value = '';", control)
        control.send_keys(str(path))
        wait.until(lambda d: len(d.find_elements(By.CSS_SELECTOR, "#file_table tbody tr")) >= before_count + 1)
        rows_after = driver.find_elements(By.CSS_SELECTOR, "#file_table tbody tr")
        if len(rows_after) != before_count + 1:
            raise RuntimeError(
                f"Protectie anti-duplicate: fisierul {index} a produs {len(rows_after) - before_count} randuri."
            )
        displayed_names = [visible_row_name(row) for row in rows_after]
        if len(displayed_names) != len(set(displayed_names)):
            raise RuntimeError("Protectie anti-duplicate: tabelul contine nume repetate.")
        log(f"[{index}/{len(files)}] Adăugat: {path.name}")
        if index != len(files):
            time.sleep(delay)

    def completed_unique_table(d):
        rows = d.find_elements(By.CSS_SELECTOR, "#file_table tbody tr")
        if len(rows) != len(files):
            return False
        names = [visible_row_name(row) for row in rows]
        if any(not name for name in names):
            return False
        if len(names) != len(set(names)):
            return False
        return names

    try:
        final_names = WebDriverWait(driver, 180, poll_frequency=0.5).until(completed_unique_table)
    except TimeoutException:
        final_rows = driver.find_elements(By.CSS_SELECTOR, "#file_table tbody tr")
        visible_names = [visible_row_name(row) for row in final_rows]
        empty_count = sum(not name for name in visible_names)
        duplicate_count = len(visible_names) - len(set(name for name in visible_names if name)) - empty_count
        raise RuntimeError(
            f"Verificarea finala a expirat: {len(final_rows)} randuri, "
            f"{empty_count} nume necompletate, {max(0, duplicate_count)} duplicate reale."
        )
    log(f"Verificare reusita: {len(final_names)} randuri, toate numele sunt completate si unice.")
    if finalize:
        button = wait.until(lambda d: d.find_element(By.ID, "upload_button"))
        log("Lista este completă; trimit fișierele către item...")
        driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", button)
        time.sleep(1)
        button.click()
        log("Butonul «Add files to existing item» a fost apăsat.")
        log("Aștept finalizarea transferului; nu închide fereastra Firefox...")
        WebDriverWait(driver, 6 * 60 * 60, poll_frequency=2).until(
            lambda d: "/upload/" not in d.current_url
            or any(text in d.find_element(By.TAG_NAME, "body").text.lower() for text in ("upload complete", "files added", "item updated"))
        )
        log(f"Transfer finalizat. Pagina curentă: {driver.current_url}")
    else:
        log("Mod verificare: lista a fost pregătită, dar butonul final NU a fost apăsat.")


def run_browser(url: str, folder: Path, delay: float, finalize: bool, log) -> None:
    new_mode = is_new_upload_url(url)
    identifier = "" if new_mode else archive_identifier(url)
    all_files = find_pdfs(folder)
    if not all_files:
        raise FileNotFoundError(f"Nu există PDF-uri în {folder}")
    files, local_duplicates = unique_local_pdfs(all_files)
    log(f"Am gasit {len(all_files)} PDF-uri locale si {len(local_duplicates)} dubluri locale dupa nume.")
    for path in local_duplicates:
        log(f"SKIP dublura locala: {path}")
    temp_root = Path(tempfile.mkdtemp(prefix="archive-uploader-firefox-"))
    profile = temp_root / "profile"
    clone_firefox_profile(default_firefox_profile(), profile)
    options = Options()
    options.binary_location = str(FIREFOX_BINARY)
    options.profile = str(profile)
    options.add_argument("-no-remote")
    driver = webdriver.Firefox(options=options, service=Service(executable_path=str(GECKODRIVER_BINARY)))
    completed = False
    try:
        if new_mode:
            log("Mod V3: creez un item Archive.org nou.")
            driver.get("https://archive.org/upload/")
            WebDriverWait(driver, 60).until(lambda d: d.find_elements(By.ID, "file_input_initial"))
            if "/account/login" in driver.current_url:
                raise RuntimeError("Sesiunea Firefox nu este autentificata pe Archive.org.")
            upload_files(driver, files, delay, log, False)
            fill_new_item_metadata(driver, log)
            if finalize:
                click_final_upload(driver, log)
            else:
                log("Mod verificare: metadatele sunt completate, dar itemul NU a fost creat.")
            completed = True
            return
        existing_names = read_existing_pdf_names(driver, identifier, log)
        upload_paths = [path for path in files if filename_key(path.name) not in existing_names]
        skipped_online = [path for path in files if filename_key(path.name) in existing_names]
        for path in skipped_online:
            log(f"SKIP exista deja pe Archive.org: {path.name}")
        log(
            f"Rezumat V2: {len(all_files)} locale, {len(skipped_online)} deja online, "
            f"{len(local_duplicates)} dubluri locale, {len(upload_paths)} de incarcat."
        )
        if not upload_paths:
            log("Nu exista fisiere noi de incarcat. Operatia s-a incheiat fara upload.")
            completed = True
            driver.quit()
            shutil.rmtree(temp_root, ignore_errors=True)
            return
        upload_url = f"https://archive.org/upload/?identifier={quote(identifier)}"
        log(f"Deschid: {upload_url}")
        driver.get(upload_url)
        WebDriverWait(driver, 60).until(lambda d: d.find_elements(By.CSS_SELECTOR, "#file_input_initial, input[type='file']"))
        if "/account/login" in driver.current_url:
            raise RuntimeError("Sesiunea clonată nu este autentificată. Autentifică-te în fereastra Firefox deschisă de script și pornește din nou.")
        upload_files(driver, upload_paths, delay, log, finalize)
        completed = True
    finally:
        if completed and finalize:
            if driver.session_id:
                OPEN_FIREFOX_SESSIONS.append((driver, temp_root))
                log("Firefox ramane activ pana la terminarea transferului.")
        else:
            if driver.session_id:
                OPEN_FIREFOX_SESSIONS.append((driver, temp_root))
            if completed:
                log("Lista este pregatita. Firefox ramane deschis pentru verificare manuala.")
            else:
                log("A aparut o eroare. Firefox ramane deschis in starea curenta pentru verificare.")


class App(tk.Tk):
    def __init__(self) -> None:
        super().__init__()
        self.title("Archive.org – uploader PDF V3")
        self.geometry("900x620")
        self.url = tk.StringVar(value="https://archive.org/upload/")
        self.folder = tk.StringVar(value=str(DEFAULT_FOLDER))
        self.delay = tk.DoubleVar(value=1.0)
        self.finalize = tk.BooleanVar(value=True)
        self._build()
        self.preview()

    def _build(self) -> None:
        frame = ttk.Frame(self, padding=12)
        frame.pack(fill="both", expand=True)
        ttk.Label(frame, text="Linkul principal Archive.org:").grid(row=0, column=0, sticky="w")
        ttk.Entry(frame, textvariable=self.url).grid(row=1, column=0, columnspan=3, sticky="ew", pady=(2, 10))
        ttk.Label(frame, text="Folderul cu PDF-uri (include toate subfolderele):").grid(row=2, column=0, sticky="w")
        ttk.Entry(frame, textvariable=self.folder).grid(row=3, column=0, sticky="ew", pady=(2, 10))
        ttk.Button(frame, text="Alege folder", command=self.choose_folder).grid(row=3, column=1, padx=6)
        ttk.Button(frame, text="Actualizează lista", command=self.preview).grid(row=3, column=2)
        ttk.Label(frame, text="Pauză între fișiere (secunde):").grid(row=4, column=0, sticky="w")
        ttk.Spinbox(frame, from_=1.0, to=30.0, increment=0.5, textvariable=self.delay, width=8).grid(row=4, column=1, sticky="w")
        ttk.Checkbutton(frame, text="Apasă automat butonul final de upload", variable=self.finalize).grid(row=5, column=0, columnspan=3, sticky="w", pady=8)
        self.output = scrolledtext.ScrolledText(frame, wrap="none", height=24)
        self.output.grid(row=6, column=0, columnspan=3, sticky="nsew")
        self.start_button = ttk.Button(frame, text="Pornește upload-ul", command=self.start)
        self.start_button.grid(row=7, column=0, columnspan=3, pady=12)
        frame.columnconfigure(0, weight=1)
        frame.rowconfigure(6, weight=1)

    def choose_folder(self) -> None:
        chosen = filedialog.askdirectory(initialdir=self.folder.get())
        if chosen:
            self.folder.set(chosen)
            self.preview()

    def log(self, message: str) -> None:
        self.after(0, lambda: (self.output.insert("end", message + "\n"), self.output.see("end")))

    def preview(self) -> None:
        self.output.delete("1.0", "end")
        folder = Path(self.folder.get())
        if not folder.is_dir():
            self.output.insert("end", "Folder inexistent.\n")
            return
        try:
            files = find_pdfs(folder)
        except ValueError as exc:
            self.output.insert("end", str(exc) + "\n")
            return
        self.output.insert("end", f"{len(files)} PDF-uri, în ordinea upload-ului:\n\n")
        for i, path in enumerate(files, 1):
            self.output.insert("end", f"{i:03}. {path}\n")

    def start(self) -> None:
        try:
            new_mode = is_new_upload_url(self.url.get())
            identifier = "item nou" if new_mode else archive_identifier(self.url.get())
            folder = Path(self.folder.get()).resolve()
            files = find_pdfs(folder)
            if not files:
                raise ValueError("Folderul nu conține PDF-uri.")
        except Exception as exc:
            messagebox.showerror("Date invalide", str(exc))
            return
        action = "încărca și finaliza" if self.finalize.get() else "pregăti fără finalizare"
        if not messagebox.askyesno("Confirmare", f"Scriptul va {action} {len(files)} PDF-uri în:\n{identifier}\n\nContinui?"):
            return
        self.start_button.config(state="disabled")
        def worker() -> None:
            try:
                run_browser(self.url.get(), folder, max(1.0, self.delay.get()), self.finalize.get(), self.log)
                self.after(0, lambda: messagebox.showinfo("Gata", "Operația s-a încheiat."))
            except Exception as exc:
                error_message = str(exc)
                self.log(f"EROARE: {error_message}")
                self.after(0, lambda message=error_message: messagebox.showerror("Eroare", message))
            finally:
                self.after(0, lambda: self.start_button.config(state="normal"))
        threading.Thread(target=worker, daemon=True).start()


def self_test() -> None:
    assert archive_identifier("https://archive.org/details/test-item") == "test-item"
    assert archive_identifier("https://archive.org/edit/test-item") == "test-item"
    assert archive_identifier("https://archive.org/upload/?identifier=test-item") == "test-item"
    assert is_new_upload_url("https://archive.org/upload/")
    assert not is_new_upload_url("https://archive.org/upload/?identifier=test-item")
    files = find_pdfs(DEFAULT_FOLDER)
    assert files and all(p.suffix.lower() == ".pdf" for p in files)
    unique, duplicates = unique_local_pdfs(files)
    assert len(unique) + len(duplicates) == len(files)
    assert all(filename_key(path.name) == filename_key(path.name) for path in files)
    print(f"OK V3: {len(files)} PDF-uri locale; {len(unique)} nume unice; {len(duplicates)} dubluri locale.")


def browser_test(url: str) -> None:
    identifier = archive_identifier(url)
    temp_root = Path(tempfile.mkdtemp(prefix="archive-uploader-test-"))
    try:
        profile = temp_root / "profile"
        clone_firefox_profile(default_firefox_profile(), profile)
        options = Options()
        options.binary_location = str(FIREFOX_BINARY)
        options.profile = str(profile)
        options.add_argument("-no-remote")
        driver = webdriver.Firefox(options=options, service=Service(executable_path=str(GECKODRIVER_BINARY)))
        try:
            target = f"https://archive.org/upload/?identifier={quote(identifier)}"
            driver.get(target)
            WebDriverWait(driver, 60).until(lambda d: d.find_elements(By.CSS_SELECTOR, "#file_input_initial, input[type='file']"))
            print(f"OK browser: {driver.current_url}; selectorul de upload există; niciun fișier trimis.")
        finally:
            driver.quit()
    finally:
        shutil.rmtree(temp_root, ignore_errors=True)


def visible_new_metadata_test() -> None:
    folder = Path(r"G:\De pus pe FTP 2")
    names = [
        "Alexandru, Mihai - Policentralitate si coerenta metropolitana.pdf",
        "Drimba, Ovidiu - Istoria literaturii universale (Vol.3) anul 1971.pdf",
        "Drimba, Ovidiu - Istoria literaturii universale (Vol.2) anul 1970.pdf",
        "Dobos, Mihaela - Start Pentru Succes La Evaluarea Nationala. Limba Si Literatura Romana - Clasele 5-7.pdf",
    ]
    files = [folder / name for name in names]
    missing = [path for path in files if not path.is_file()]
    if missing:
        raise FileNotFoundError(f"Lipsesc fisierele de test: {missing}")
    temp_root = Path(tempfile.mkdtemp(prefix="archive-v3-visible-test-"))
    profile = temp_root / "profile"
    clone_firefox_profile(default_firefox_profile(), profile)
    options = Options()
    options.binary_location = str(FIREFOX_BINARY)
    options.profile = str(profile)
    options.add_argument("-no-remote")
    driver = webdriver.Firefox(options=options, service=Service(executable_path=str(GECKODRIVER_BINARY)))
    try:
        driver.get("https://archive.org/upload/")
        upload_files(driver, files, 1.0, print, False)
        fill_new_item_metadata(driver, print)
        print("TEST VIZIBIL REUSIT. Verifica Firefox; butonul final NU va fi apasat.")
        while driver.window_handles:
            time.sleep(1)
    except Exception as exc:
        print(f"EROARE TEST VIZIBIL: {exc}")
        while driver.window_handles:
            time.sleep(1)
    finally:
        try:
            driver.quit()
        except Exception:
            pass
        shutil.rmtree(temp_root, ignore_errors=True)


if __name__ == "__main__":
    if hasattr(sys.stdout, "reconfigure"):
        sys.stdout.reconfigure(encoding="utf-8")
    parser = argparse.ArgumentParser()
    parser.add_argument("--self-test", action="store_true")
    parser.add_argument("--browser-test", metavar="URL")
    parser.add_argument("--test-new-metadata", action="store_true")
    args = parser.parse_args()
    if args.self_test:
        self_test()
    elif args.browser_test:
        browser_test(args.browser_test)
    elif args.test_new_metadata:
        visible_new_metadata_test()
    else:
        App().mainloop()