archive_org_pdf_uploader.py

archive_org_pdf_uploader.py

from __future__ import annotations

import argparse
import configparser
import os
import re
import shutil
import sqlite3
import sys
import tempfile
import threading
import time
import tkinter as tk
import unicodedata
from pathlib import Path
from tkinter import filedialog, messagebox, scrolledtext, ttk
from urllib.parse import quote, urlparse

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.support.ui import WebDriverWait


DEFAULT_FOLDER = Path(r"G:\Buletinul Institutului Politehnic Bucureşti")
FIREFOX_BINARY = Path(r"C:\Program Files\Mozilla Firefox\firefox.exe")
GECKODRIVER_BINARY = Path(r"C:\WINDOWS\geckodriver.exe")
YEAR_RE = re.compile(r"(?<!\d)((?:18|19|20)\d{2})(?:\s*[-–]\s*((?:18|19|20)\d{2}))?(?!\d)")
OPEN_FIREFOX_SESSIONS: list[tuple[webdriver.Firefox, Path, Path]] = []


def archive_identifier(value: str) -> str:
    value = value.strip()
    parsed = urlparse(value if "://" in value else "https://archive.org/details/" + value)
    parts = [part for part in parsed.path.split("/") if part]
    if len(parts) >= 2 and parts[0] in {"details", "edit", "download"}:
        identifier = parts[1]
    elif parsed.path == "/upload/":
        from urllib.parse import parse_qs
        identifier = parse_qs(parsed.query).get("identifier", [""])[0]
    elif len(parts) == 1:
        identifier = parts[0]
    else:
        identifier = ""
    if not re.fullmatch(r"[A-Za-z0-9._-]+", identifier):
        raise ValueError("Link sau identifier Archive.org invalid.")
    return identifier


def pdf_sort_key(path: Path) -> tuple[int, int, str]:
    match = YEAR_RE.search(path.name)
    if match:
        return (-int(match.group(1)), -int(match.group(2) or match.group(1)), path.name.casefold())
    return (1, 1, path.name.casefold())


def find_pdfs(folder: Path) -> list[Path]:
    return sorted((p.resolve() for p in folder.rglob("*.pdf") if p.is_file()), key=pdf_sort_key)


def normalized_words(value: str) -> set[str]:
    value = "".join(c for c in unicodedata.normalize("NFKD", value) if not unicodedata.combining(c)).lower()
    return set(re.findall(r"[a-z0-9]+", value))


def relevant_pdfs(folder: Path, url: str) -> list[Path]:
    identifier = archive_identifier(url)
    ignored = {"buletinul", "institutului", "politehnic", "din", "iasi", "bucuresti", "seria"}
    keywords = normalized_words(identifier) - ignored
    threshold = max(1, (len(keywords) + 1) // 2)
    files = find_pdfs(folder)
    matches = [p for p in files if len(normalized_words(p.stem) & keywords) >= threshold]
    return matches or files


def default_firefox_profile() -> Path:
    base = Path(os.environ["APPDATA"]) / "Mozilla" / "Firefox"
    ini = configparser.ConfigParser()
    ini.read(base / "profiles.ini", encoding="utf-8")
    install_sections = [s for s in ini.sections() if s.startswith("Install")]
    rel = ini[install_sections[0]].get("Default", "") if install_sections else ""
    if not rel:
        for section in ini.sections():
            if section.startswith("Profile") and ini[section].getboolean("Default", fallback=False):
                rel = ini[section].get("Path", "")
                break
    profile = base / rel
    if not profile.is_dir():
        raise FileNotFoundError("Nu am găsit profilul Firefox implicit.")
    return profile


def clone_firefox_profile(source: Path, target: Path) -> None:
    target.mkdir(parents=True, exist_ok=True)
    cookies = source / "cookies.sqlite"
    if cookies.exists():
        source_db = sqlite3.connect(f"file:{cookies.as_posix()}?mode=ro", uri=True, timeout=10)
        target_db = sqlite3.connect(target / "cookies.sqlite")
        try:
            source_db.backup(target_db)
        finally:
            target_db.close()
            source_db.close()
    for name in ("permissions.sqlite", "cert9.db", "key4.db", "pkcs11.txt"):
        item = source / name
        if item.is_file():
            shutil.copy2(item, target / name)


def stage_files_with_short_paths(files: list[Path]) -> tuple[Path, list[Path]]:
    """Create same-volume hard links with short paths, preserving upload names."""
    if not files:
        raise ValueError("Lista de fisiere este goala.")
    drive_root = Path(files[0].anchor)
    staging = Path(tempfile.mkdtemp(prefix="ia-upload-", dir=drive_root))
    staged: list[Path] = []
    try:
        for index, source in enumerate(files, 1):
            if source.anchor.casefold() != files[0].anchor.casefold():
                raise ValueError("Toate PDF-urile trebuie sa fie pe aceeasi unitate de disc.")
            short_folder = staging / f"{index:03}"
            short_folder.mkdir()
            destination = short_folder / source.name
            os.link(source, destination)
            staged.append(destination)
        return staging, staged
    except Exception:
        shutil.rmtree(staging, ignore_errors=True)
        raise


def upload_files(driver: webdriver.Firefox, files: list[Path], delay: float, log, finalize: bool) -> None:
    wait = WebDriverWait(driver, 60)
    def visible_row_name(row) -> str:
        value = row.find_element(By.CSS_SELECTOR, "span.file").get_attribute("textContent")
        return (value or "").strip()

    expected_names = [path.name for path in files]
    if len(expected_names) != len(set(expected_names)):
        raise RuntimeError("Lista locala contine nume PDF duplicate. Operatia a fost oprita.")
    log(f"Adaug {len(files)} PDF-uri individual, la interval de {delay:.1f} secunde.")
    files_to_add_individually = files
    for index, path in enumerate(files_to_add_individually, 1):
        rows_before = driver.find_elements(By.CSS_SELECTOR, "#file_table tbody tr")
        before_count = len(rows_before)
        if before_count != index - 1:
            raise RuntimeError(
                f"Protectie anti-duplicate: tabelul are {before_count} randuri in loc de {index - 1}."
            )
        inputs = driver.find_elements(By.CSS_SELECTOR, "input[type='file']")
        if not inputs:
            raise RuntimeError("Nu găsesc controlul de selectare a fișierelor pe pagina Archive.org.")
        control_id = "file_input_initial" if index == 1 else "file_input_add"
        control = wait.until(lambda d: d.find_element(By.ID, control_id))
        if index > 1:
            driver.execute_script("arguments[0].value = '';", control)
        control.send_keys(str(path))
        wait.until(lambda d: len(d.find_elements(By.CSS_SELECTOR, "#file_table tbody tr")) >= before_count + 1)
        rows_after = driver.find_elements(By.CSS_SELECTOR, "#file_table tbody tr")
        if len(rows_after) != before_count + 1:
            raise RuntimeError(
                f"Protectie anti-duplicate: fisierul {index} a produs {len(rows_after) - before_count} randuri."
            )
        displayed_names = [visible_row_name(row) for row in rows_after]
        if len(displayed_names) != len(set(displayed_names)):
            raise RuntimeError("Protectie anti-duplicate: tabelul contine nume repetate.")
        log(f"[{index}/{len(files)}] Adăugat: {path.name}")
        if index != len(files):
            time.sleep(delay)

    def completed_unique_table(d):
        rows = d.find_elements(By.CSS_SELECTOR, "#file_table tbody tr")
        if len(rows) != len(files):
            return False
        names = [visible_row_name(row) for row in rows]
        if any(not name for name in names):
            return False
        if len(names) != len(set(names)):
            return False
        return names

    try:
        final_names = WebDriverWait(driver, 180, poll_frequency=0.5).until(completed_unique_table)
    except TimeoutException:
        final_rows = driver.find_elements(By.CSS_SELECTOR, "#file_table tbody tr")
        visible_names = [visible_row_name(row) for row in final_rows]
        empty_count = sum(not name for name in visible_names)
        duplicate_count = len(visible_names) - len(set(name for name in visible_names if name)) - empty_count
        raise RuntimeError(
            f"Verificarea finala a expirat: {len(final_rows)} randuri, "
            f"{empty_count} nume necompletate, {max(0, duplicate_count)} duplicate reale."
        )
    log(f"Verificare reusita: {len(final_names)} randuri, toate numele sunt completate si unice.")
    if finalize:
        button = wait.until(lambda d: d.find_element(By.ID, "upload_button"))
        log("Lista este completă; trimit fișierele către item...")
        driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", button)
        time.sleep(1)
        button.click()
        log("Butonul «Add files to existing item» a fost apăsat.")
        log("Aștept finalizarea transferului; nu închide fereastra Firefox...")
        WebDriverWait(driver, 6 * 60 * 60, poll_frequency=2).until(
            lambda d: "/upload/" not in d.current_url
            or any(text in d.find_element(By.TAG_NAME, "body").text.lower() for text in ("upload complete", "files added", "item updated"))
        )
        log(f"Transfer finalizat. Pagina curentă: {driver.current_url}")
    else:
        log("Mod verificare: lista a fost pregătită, dar butonul final NU a fost apăsat.")


def run_browser(url: str, folder: Path, delay: float, finalize: bool, log) -> None:
    identifier = archive_identifier(url)
    files = find_pdfs(folder)
    if not files:
        raise FileNotFoundError(f"Nu există PDF-uri în {folder}")
    log(f"Am găsit {len(files)} PDF-uri. Primul: {files[0].name}")
    staging_root, upload_paths = stage_files_with_short_paths(files)
    log("Am pregatit cai temporare scurte; numele originale ale PDF-urilor sunt pastrate.")
    temp_root = Path(tempfile.mkdtemp(prefix="archive-uploader-firefox-"))
    profile = temp_root / "profile"
    clone_firefox_profile(default_firefox_profile(), profile)
    options = Options()
    options.binary_location = str(FIREFOX_BINARY)
    options.profile = str(profile)
    options.add_argument("-no-remote")
    driver = webdriver.Firefox(options=options, service=Service(executable_path=str(GECKODRIVER_BINARY)))
    completed = False
    try:
        upload_url = f"https://archive.org/upload/?identifier={quote(identifier)}"
        log(f"Deschid: {upload_url}")
        driver.get(upload_url)
        WebDriverWait(driver, 60).until(lambda d: d.find_elements(By.CSS_SELECTOR, "#file_input_initial, input[type='file']"))
        if "/account/login" in driver.current_url:
            raise RuntimeError("Sesiunea clonată nu este autentificată. Autentifică-te în fereastra Firefox deschisă de script și pornește din nou.")
        upload_files(driver, upload_paths, delay, log, finalize)
        completed = True
    finally:
        if completed and finalize:
            OPEN_FIREFOX_SESSIONS.append((driver, temp_root, staging_root))
            log("Firefox si fisierele temporare raman active pana la terminarea transferului.")
        else:
            OPEN_FIREFOX_SESSIONS.append((driver, temp_root, staging_root))
            if completed:
                log("Lista este pregatita. Firefox ramane deschis pentru verificare manuala.")
            else:
                log("A aparut o eroare. Firefox ramane deschis in starea curenta pentru verificare.")


class App(tk.Tk):
    def __init__(self) -> None:
        super().__init__()
        self.title("Archive.org – adăugare PDF-uri")
        self.geometry("900x620")
        self.url = tk.StringVar(value="https://archive.org/details/buletinul-institutului-politehnic-din-iasi-electrotehnica-energetica-electronica")
        self.folder = tk.StringVar(value=str(DEFAULT_FOLDER))
        self.delay = tk.DoubleVar(value=1.0)
        self.finalize = tk.BooleanVar(value=True)
        self._build()
        self.preview()

    def _build(self) -> None:
        frame = ttk.Frame(self, padding=12)
        frame.pack(fill="both", expand=True)
        ttk.Label(frame, text="Linkul principal Archive.org:").grid(row=0, column=0, sticky="w")
        ttk.Entry(frame, textvariable=self.url).grid(row=1, column=0, columnspan=3, sticky="ew", pady=(2, 10))
        ttk.Label(frame, text="Folderul cu PDF-uri (include toate subfolderele):").grid(row=2, column=0, sticky="w")
        ttk.Entry(frame, textvariable=self.folder).grid(row=3, column=0, sticky="ew", pady=(2, 10))
        ttk.Button(frame, text="Alege folder", command=self.choose_folder).grid(row=3, column=1, padx=6)
        ttk.Button(frame, text="Actualizează lista", command=self.preview).grid(row=3, column=2)
        ttk.Label(frame, text="Pauză între fișiere (secunde):").grid(row=4, column=0, sticky="w")
        ttk.Spinbox(frame, from_=1.0, to=30.0, increment=0.5, textvariable=self.delay, width=8).grid(row=4, column=1, sticky="w")
        ttk.Checkbutton(frame, text="Apasă automat butonul final «Add files to existing item»", variable=self.finalize).grid(row=5, column=0, columnspan=3, sticky="w", pady=8)
        self.output = scrolledtext.ScrolledText(frame, wrap="none", height=24)
        self.output.grid(row=6, column=0, columnspan=3, sticky="nsew")
        self.start_button = ttk.Button(frame, text="Pornește upload-ul", command=self.start)
        self.start_button.grid(row=7, column=0, columnspan=3, pady=12)
        frame.columnconfigure(0, weight=1)
        frame.rowconfigure(6, weight=1)

    def choose_folder(self) -> None:
        chosen = filedialog.askdirectory(initialdir=self.folder.get())
        if chosen:
            self.folder.set(chosen)
            self.preview()

    def log(self, message: str) -> None:
        self.after(0, lambda: (self.output.insert("end", message + "\n"), self.output.see("end")))

    def preview(self) -> None:
        self.output.delete("1.0", "end")
        folder = Path(self.folder.get())
        if not folder.is_dir():
            self.output.insert("end", "Folder inexistent.\n")
            return
        try:
            files = find_pdfs(folder)
        except ValueError as exc:
            self.output.insert("end", str(exc) + "\n")
            return
        self.output.insert("end", f"{len(files)} PDF-uri, în ordinea upload-ului:\n\n")
        for i, path in enumerate(files, 1):
            self.output.insert("end", f"{i:03}. {path}\n")

    def start(self) -> None:
        try:
            archive_identifier(self.url.get())
            folder = Path(self.folder.get()).resolve()
            files = find_pdfs(folder)
            if not files:
                raise ValueError("Folderul nu conține PDF-uri.")
        except Exception as exc:
            messagebox.showerror("Date invalide", str(exc))
            return
        action = "încărca și finaliza" if self.finalize.get() else "pregăti fără finalizare"
        if not messagebox.askyesno("Confirmare", f"Scriptul va {action} {len(files)} PDF-uri în itemul:\n{archive_identifier(self.url.get())}\n\nContinui?"):
            return
        self.start_button.config(state="disabled")
        def worker() -> None:
            try:
                run_browser(self.url.get(), folder, max(1.0, self.delay.get()), self.finalize.get(), self.log)
                self.after(0, lambda: messagebox.showinfo("Gata", "Operația s-a încheiat."))
            except Exception as exc:
                error_message = str(exc)
                self.log(f"EROARE: {error_message}")
                self.after(0, lambda message=error_message: messagebox.showerror("Eroare", message))
            finally:
                self.after(0, lambda: self.start_button.config(state="normal"))
        threading.Thread(target=worker, daemon=True).start()


def self_test() -> None:
    assert archive_identifier("https://archive.org/details/test-item") == "test-item"
    assert archive_identifier("https://archive.org/edit/test-item") == "test-item"
    assert archive_identifier("https://archive.org/upload/?identifier=test-item") == "test-item"
    files = find_pdfs(DEFAULT_FOLDER)
    assert files and all(p.suffix.lower() == ".pdf" for p in files)
    target = relevant_pdfs(DEFAULT_FOLDER, "https://archive.org/details/buletinul-institutului-politehnic-din-iasi-electrotehnica-energetica-electronica")
    assert len(target) == 15 and "1985" in target[0].name and "1971" in target[-1].name
    staging, staged = stage_files_with_short_paths(target[:2])
    try:
        assert len(staged) == 2
        assert staged[0].name == target[0].name
        assert staged[0].samefile(target[0])
        assert len(str(staged[0])) < len(str(target[0]))
    finally:
        shutil.rmtree(staging, ignore_errors=True)
    print(f"OK: {len(files)} PDF-uri total; {len(target)} selectate; primul={target[0].name}; ultimul={target[-1].name}")


def browser_test(url: str) -> None:
    identifier = archive_identifier(url)
    temp_root = Path(tempfile.mkdtemp(prefix="archive-uploader-test-"))
    try:
        profile = temp_root / "profile"
        clone_firefox_profile(default_firefox_profile(), profile)
        options = Options()
        options.binary_location = str(FIREFOX_BINARY)
        options.profile = str(profile)
        options.add_argument("-no-remote")
        driver = webdriver.Firefox(options=options, service=Service(executable_path=str(GECKODRIVER_BINARY)))
        try:
            target = f"https://archive.org/upload/?identifier={quote(identifier)}"
            driver.get(target)
            WebDriverWait(driver, 60).until(lambda d: d.find_elements(By.CSS_SELECTOR, "#file_input_initial, input[type='file']"))
            print(f"OK browser: {driver.current_url}; selectorul de upload există; niciun fișier trimis.")
        finally:
            driver.quit()
    finally:
        shutil.rmtree(temp_root, ignore_errors=True)


if __name__ == "__main__":
    if hasattr(sys.stdout, "reconfigure"):
        sys.stdout.reconfigure(encoding="utf-8")
    parser = argparse.ArgumentParser()
    parser.add_argument("--self-test", action="store_true")
    parser.add_argument("--browser-test", metavar="URL")
    args = parser.parse_args()
    if args.self_test:
        self_test()
    elif args.browser_test:
        browser_test(args.browser_test)
    else:
        App().mainloop()