chrisb13 · July 29, 2026 00:58
diff --git a/verify_uploads.py b/verify_uploads.py
 #!/usr/bin/env python3
 """
 verify_uploads.py

 Read-only check: for every experiment under BASE, compares the local
 rendered .ipynb/.png files against what's actually on Figshare, using the
 same article-title scheme and MD5-completeness logic as
 mkfigs/configdoc.py. Makes no changes -- GET requests only.

 Works before publishing: Figshare's account/* endpoints return private
 (draft) articles for the token owner, same as public ones, so this is
 accurate whether or not you've published anything yet.

 Per file, one of:
  OK        -- exists remotely (in at least one entry), matches local content
  MISMATCH  -- exists remotely, fully uploaded, but content differs
  PENDING   -- exists remotely but not yet fully processed (no computed_md5)
  MISSING   -- no remote file with this name at all
  NO ARTICLE -- the experiment's article doesn't exist on Figshare yet
               (every local file for it is implicitly missing)

 Also detects and (with --confirm) cleans up duplicate file entries --
 Figshare allows more than one file with the same name in one article, and
 an earlier dict-based version of this script silently hid broken entries
 behind working ones sharing a name. Per duplicate name:
  DUP-STUB      -- broken/incomplete entries alongside a working copy;
                   safe to delete regardless of content (--confirm)
  DUP-OLD       -- 2+ identical complete copies matching the local file;
                   keeps the newest, deletes the rest (--confirm)
  DUP-STALE     -- 2+ identical complete copies, but NONE match the
                   current local file -- flagged only, nothing deleted
  DUP-CONFLICT  -- 2+ complete copies that disagree with each other --
                   flagged only, needs manual review

 Usage:
    export FIGSHARE_TOKEN=xxxx
    python3 verify_uploads.py --base /home/narky/VBoxSHARED/access-om3-paper-1-runs
    python3 verify_uploads.py --base ... --confirm   # also clean up safe duplicates
 """

 import argparse
 import hashlib
 import json
 import os
 import sys
 from pathlib import Path

 import requests

 FIGSHARE_BASE_URL = "https://api.figshare.com/v2/{endpoint}"


 def _headers(token):
    return {"Authorization": f"token {token}"}


 def _request(method, url, token, data=None):
    headers = _headers(token)
    if data is not None:
        data = json.dumps(data)
    resp = requests.request(method, url, headers=headers, data=data, timeout=60)
    resp.raise_for_status()
    try:
        return json.loads(resp.content)
    except ValueError:
        return resp.content


 def md5_of(path):
    h = hashlib.md5()
    with open(path, "rb") as f:
        for block in iter(lambda: f.read(65536), b""):
            h.update(block)
    return h.hexdigest()


 def list_all_articles(token, page_size=100):
    """Full paginated listing of every account article, building a
    title -> [id, ...] map (a list, not a single id -- duplicates matter
    here and must not be silently collapsed). Avoids the search endpoint
    entirely -- it turned out to be unreliable for titles containing
    literal '+' or '-' characters (e.g.
    'MC_100km_jra_ryf+wombatlite-1e74abf-11f9df5c', or even a plain
    hyphen-heavy title like '25km-iaf-test-for-AK-expt-7df5ef4c'), almost
    certainly because Figshare's search parses '+'/'-' as query-syntax
    operators (require/exclude) rather than literal text. A plain listing
    sidesteps that -- no query string, no parsing, just exact client-side
    matching."""
    titles = {}
    page = 1
    while True:
        url = FIGSHARE_BASE_URL.format(
            endpoint=f"account/articles?page={page}&page_size={page_size}")
        batch = _request("GET", url, token)
        if not batch:
            break
        for art in batch:
            titles.setdefault(art["title"], []).append(
                {"id": art["id"], "created_date": art.get("created_date")}
            )
        if len(batch) < page_size:
            break
        page += 1
    return titles


 def fetch_article_files_raw(token, article_id, page_size=100):
    """Full paginated listing of an article's files, WITHOUT collapsing by
    name. The endpoint defaults to page_size=10 if unspecified -- always
    paginate explicitly (confirmed directly: a real 93-file article
    returned only 10 files with no page_size set).

    Returns the raw list. Collapsing this into a name-keyed dict (an
    earlier version of this script did exactly that) turned out to
    silently hide real problems: Figshare allows more than one file entry
    under the same name in one article, and a plain dict just keeps
    whichever one pagination returns last -- hiding a broken sibling
    entry behind a working one with the same name. Confirmed against real
    data: files the Figshare web UI flagged as "Something went wrong"
    were invisible to a dict-based version of this check because a
    working duplicate with the same name existed too."""
    files = []
    page = 1
    while True:
        url = FIGSHARE_BASE_URL.format(
            endpoint=f"account/articles/{article_id}/files?page={page}&page_size={page_size}")
        batch = _request("GET", url, token)
        if not batch:
            break
        files.extend(batch)
        if len(batch) < page_size:
            break
        page += 1
    return files


 def index_by_name(raw_files):
    """{filename: [entry, entry, ...]} -- may have more than one entry
    per name; see fetch_article_files_raw's docstring for why that
    matters."""
    by_name = {}
    for f in raw_files:
        by_name.setdefault(f["name"], []).append(f)
    return by_name


 def file_status(entries, local_md5):
    """Status for a local file against every remote entry sharing its
    name (there may be more than one -- see fetch_article_files_raw).
    Checking all of them, not just one, is what makes this
    duplicate-safe: a working entry anywhere in the list is enough for
    OK, even if a broken stub with the same name also exists."""
    if not entries:
        return "MISSING"
    if any(e.get("computed_md5") == local_md5 for e in entries):
        return "OK"
    if any(e.get("computed_md5") for e in entries):
        return "MISMATCH"
    return "PENDING"


 def classify_duplicates(entries, local_md5):
    """Given 2+ remote entries sharing one filename, decide what (if
    anything) is safe to clean up automatically.

    Returns (action, to_delete, note):
      "delete_stubs"      -- broken/incomplete entries with a genuine
                              complete sibling present. Safe regardless
                              of content, since Figshare's own UI text
                              confirms unsuccessful uploads never appear
                              publicly -- a stub adds nothing.
      "delete_older_dupes"-- 2+ complete entries, all sharing the SAME
                              md5, AND that md5 matches local_md5. Only
                              this case is safe to trim automatically;
                              keeps the highest file id (files don't
                              expose a timestamp, but ids are assigned in
                              creation order, same as articles).
      "stale_duplicates"   -- complete entries agree with each other but
                              NOT with the current local file. Both/all
                              copies are outdated -- flagged, nothing
                              deleted, since which one (if any) should
                              survive isn't this script's call.
      "conflicting"        -- complete entries disagree with EACH OTHER.
                              Flagged only, needs manual review.
      (None, [], "")       -- fewer than 2 entries, nothing to do.
    """
    if len(entries) < 2:
        return (None, [], "")

    complete = [e for e in entries if e.get("status") == "available" and e.get("computed_md5")]
    stubs = [e for e in entries if e not in complete]

    if complete and stubs:
        return ("delete_stubs", stubs,
                f"{len(stubs)} broken/incomplete entr{'y' if len(stubs)==1 else 'ies'} "
                f"alongside {len(complete)} working cop{'y' if len(complete)==1 else 'ies'}")

    if len(complete) >= 2:
        md5s = {e["computed_md5"] for e in complete}
        if len(md5s) == 1:
            shared_md5 = next(iter(md5s))
            if shared_md5 == local_md5:
                newest = max(complete, key=lambda e: e["id"])
                older = [e for e in complete if e["id"] != newest["id"]]
                return ("delete_older_dupes", older,
                        f"{len(complete)} identical complete copies, matches local file "
                        f"-- keeping newest (id {newest['id']})")
            else:
                return ("stale_duplicates", [],
                        f"{len(complete)} identical complete copies, but content does NOT "
                        f"match the current local file -- needs manual review")
        else:
            return ("conflicting", [],
                    f"{len(complete)} complete copies with DIFFERING content "
                    f"({len(md5s)} distinct md5s) -- needs manual review")

    # All entries are incomplete stubs, none complete -- not a duplicate
    # cleanup case, just an ordinary PENDING file (handled elsewhere).
    return (None, [], "")


 def discover_experiments(base: Path):
    """Find every experiment dir under base with an mkfigs_output_* dir,
    without relying on a hardcoded RUNS list (mkfigs.sh's own list is known
    to drift out of sync with what actually exists on disk).

    Returns (exp_name, ofol) pairs. Rendered notebooks live directly in
    ofol (mkfigs_output_<experiment>/<nb>_rendered.ipynb); PNGs and
    markdown live in ofol/mkmd -- confirmed directly against pushit.py's
    own source (`rendered = ofol / f"{nb}_rendered.ipynb"`,
    `pngs = mdfol.glob(f"{nb}_*.png")` where `mdfol = ofol / "mkmd"`).
    Scanning only mkmd/ (an earlier version of this script's bug) finds
    every PNG but zero .ipynb files, which makes every real notebook
    upload look like an orphaned EXTRA."""
    found = []
    for exp_dir in sorted(base.iterdir()):
        if not exp_dir.is_dir():
            continue
        for ofol in exp_dir.glob("notebooks/mkfigs_output_*"):
            if ofol.is_dir():
                found.append((exp_dir.name, ofol))
    return found


 def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                  formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--base", required=True, help="Path containing all experiment directories")
    ap.add_argument("--token-env", default="FIGSHARE_TOKEN")
    ap.add_argument("--only-missing", action="store_true",
                     help="Only print files that aren't fully OK (skip printing every success)")
    ap.add_argument("--confirm", action="store_true",
                     help="Actually delete safe-to-remove duplicates (broken stubs, and "
                          "older copies of an exact duplicate that matches the local file). "
                          "Without this, everything is dry-run: reported but not deleted. "
                          "Ambiguous duplicates (stale or conflicting) are NEVER auto-deleted "
                          "regardless of this flag.")
    args = ap.parse_args()

    token = os.environ.get(args.token_env)
    if not token:
        sys.exit(f"Set {args.token_env} in the environment before running.")

    base = Path(args.base)
    if not base.is_dir():
        sys.exit(f"--base {base} is not a directory")

    experiments = discover_experiments(base)
    if not experiments:
        sys.exit(f"No experiments with a notebooks/mkfigs_output_* dir found under {base}")

    print(f"Found {len(experiments)} experiment(s) under {base}")
    print("Fetching full article listing from Figshare (one paginated scan)...\n")
    article_titles = list_all_articles(token)

    totals = {"OK": 0, "MISMATCH": 0, "PENDING": 0, "MISSING": 0, "EXTRA": 0}
    experiments_with_gaps = []

    for exp_name, ofol in experiments:
        mdfol = ofol / "mkmd"

        # Mirror pushit.py's own success gate exactly: a rendered.ipynb
        # file existing on disk does NOT mean the notebook succeeded --
        # nbconvert/papermill can leave a rendered.ipynb behind even when
        # the notebook errored mid-execution. pushit.py only counts a
        # notebook as OK if it also produced PNGs or markdown; otherwise
        # it's FAILED and was never uploaded, so it shouldn't be expected
        # on Figshare either. Only include ipynb+pngs for notebooks that
        # pass this same gate.
        #
        # PNG ownership: a naive f"{nb}_*.png" glob per notebook (what
        # pushit.py itself does) double-counts when one notebook's name is
        # a prefix of another's -- e.g. "MLD" matches "MLD_max_01.png" too,
        # since "MLD_*.png" doesn't care what's inside the wildcard. This
        # is a confirmed real bug in pushit.py itself (not just here): it
        # causes files like MLD_max's PNGs to be redundantly reprocessed
        # under MLD's own upload step. Fix here: assign each PNG to
        # exactly one notebook -- the longest (most specific) matching
        # name -- so ownership is unambiguous.
        notebook_names = sorted(
            (r.name[: -len("_rendered.ipynb")] for r in ofol.glob("*_rendered.ipynb")),
            key=len, reverse=True,
        )
        all_pngs = sorted(mdfol.glob("*.png")) if mdfol.exists() else []
        pngs_by_nb = {nb: [] for nb in notebook_names}
        for png in all_pngs:
            for nb in notebook_names:  # longest names checked first
                if png.name.startswith(f"{nb}_"):
                    pngs_by_nb[nb].append(png)
                    break

        local_files = []
        for rendered in sorted(ofol.glob("*_rendered.ipynb")):
            nb = rendered.name[: -len("_rendered.ipynb")]
            pngs = pngs_by_nb.get(nb, [])
            nb_md = mdfol / f"{nb}.md"
            if not pngs and not nb_md.exists():
                continue  # FAILED per pushit.py's own gate -- never uploaded, skip
            local_files.append(rendered)
            local_files.extend(pngs)
        local_files = sorted(set(local_files))

        if not local_files:
            continue

        title = f"ACCESS-OM3 evaluation figures – {exp_name}"
        matches = article_titles.get(title, [])

        if not matches:
            article_id = None
        elif len(matches) == 1:
            article_id = matches[0]["id"]
        else:
            # Duplicate articles under the same title. Don't silently pick
            # one -- check file counts on each and flag this loudly, since
            # it likely means a run couldn't find the real article (title
            # search failing on +/- characters) and created a fresh one.
            print(f"  *** WARNING: {len(matches)} articles share this title! ***")
            best_id, best_count = None, -1
            for m in matches:
                fcount = len(fetch_article_files_raw(token, m["id"]))
                print(f"      article {m['id']}  created {m.get('created_date')}  "
                      f"{fcount} file(s)")
                if fcount > best_count:
                    best_id, best_count = m["id"], fcount
            print(f"      -> using {best_id} (most files) for comparison below, "
                  f"but this needs manual cleanup on Figshare.")
            article_id = best_id

        print(f"=== {exp_name} ({len(local_files)} local files) ===")
        if article_id is None:
            print(f"  NO ARTICLE -- '{title}' not found on Figshare")
            totals["MISSING"] += len(local_files)
            experiments_with_gaps.append(exp_name)
            print()
            continue

        raw_files = fetch_article_files_raw(token, article_id)
        by_name = index_by_name(raw_files)
        exp_gap = False
        local_names = {p.name for p in local_files}
        local_md5s = {}  # filled in below, reused by duplicate classification

        for local_path in local_files:
            fname = local_path.name
            local_md5 = md5_of(local_path)
            local_md5s[fname] = local_md5
            entries = by_name.get(fname, [])

            status = file_status(entries, local_md5)

            totals[status] += 1
            if status != "OK":
                exp_gap = True
            if status != "OK" or not args.only_missing:
                print(f"  {status:9s} {fname}")

        # Reverse direction: files on Figshare with no local counterpart at
        # all. Could be stale leftovers from a notebook that used to
        # produce more images (e.g. a renumbered PNG series), or a genuine
        # duplicate/orphaned upload -- either way, worth knowing about
        # before publishing, not just gaps in the other direction.
        extra_names = sorted(set(by_name) - local_names)
        for fname in extra_names:
            totals["EXTRA"] += 1
            exp_gap = True
            print(f"  EXTRA     {fname}  (on Figshare, no local file with this name)")

        # Duplicate detection/cleanup -- only for names with a real local
        # file to check against (see classify_duplicates' docstring for
        # why "must match the file on disk" matters: two remote copies
        # agreeing with each other is NOT enough on its own, they could
        # both be stale).
        for local_path in local_files:
            fname = local_path.name
            entries = by_name.get(fname, [])
            if len(entries) < 2:
                continue
            action, to_delete, note = classify_duplicates(entries, local_md5s[fname])
            if action is None:
                continue
            exp_gap = True
            totals["DUP_" + action.upper()] = totals.get("DUP_" + action.upper(), 0) + 1
            tag = {
                "delete_stubs": "DUP-STUB",
                "delete_older_dupes": "DUP-OLD ",
                "stale_duplicates": "DUP-STALE",
                "conflicting": "DUP-CONFLICT",
            }[action]
            print(f"  {tag:12s} {fname}  ({note})")
            if action in ("delete_stubs", "delete_older_dupes"):
                for e in to_delete:
                    if args.confirm:
                        del_url = FIGSHARE_BASE_URL.format(
                            endpoint=f"account/articles/{article_id}/files/{e['id']}")
                        requests.delete(del_url, headers=_headers(token), timeout=30).raise_for_status()
                        print(f"               -> deleted id {e['id']}")
                    else:
                        print(f"               -> would delete id {e['id']} "
                              f"(pass --confirm to actually delete)")

        if exp_gap:
            experiments_with_gaps.append(exp_name)
        print()

    print("=" * 60)
    print("SUMMARY")
    print("=" * 60)
    for status in ("OK", "PENDING", "MISMATCH", "MISSING", "EXTRA"):
        print(f"  {status:9s} {totals[status]}")
    dup_keys = [k for k in totals if k.startswith("DUP_")]
    if dup_keys:
        print()
        for k in sorted(dup_keys):
            print(f"  {k:22s} {totals[k]}")
        if not args.confirm:
            print("\n  (dry run -- re-run with --confirm to actually delete "
                  "DUP_DELETE_STUBS / DUP_DELETE_OLDER_DUPES entries)")
    print()
    if experiments_with_gaps:
        print(f"{len(experiments_with_gaps)} experiment(s) have gaps -- NOT ready to publish:")
        for e in experiments_with_gaps:
            print(f"  - {e}")
        sys.exit(1)
    else:
        print("All local .ipynb/.png files are fully and correctly uploaded, "
              "and nothing extra is sitting on Figshare. Safe to publish.")
        sys.exit(0)


 if __name__ == "__main__":
    main()
	#!/usr/bin/env python3
	"""
	verify_uploads.py

	Read-only check: for every experiment under BASE, compares the local
	rendered .ipynb/.png files against what's actually on Figshare, using the
	same article-title scheme and MD5-completeness logic as
	mkfigs/configdoc.py. Makes no changes -- GET requests only.

	Works before publishing: Figshare's account/* endpoints return private
	(draft) articles for the token owner, same as public ones, so this is
	accurate whether or not you've published anything yet.

	Per file, one of:
	OK -- exists remotely (in at least one entry), matches local content
	MISMATCH -- exists remotely, fully uploaded, but content differs
	PENDING -- exists remotely but not yet fully processed (no computed_md5)
	MISSING -- no remote file with this name at all
	NO ARTICLE -- the experiment's article doesn't exist on Figshare yet
	(every local file for it is implicitly missing)

	Also detects and (with --confirm) cleans up duplicate file entries --
	Figshare allows more than one file with the same name in one article, and
	an earlier dict-based version of this script silently hid broken entries
	behind working ones sharing a name. Per duplicate name:
	DUP-STUB -- broken/incomplete entries alongside a working copy;
	safe to delete regardless of content (--confirm)
	DUP-OLD -- 2+ identical complete copies matching the local file;
	keeps the newest, deletes the rest (--confirm)
	DUP-STALE -- 2+ identical complete copies, but NONE match the
	current local file -- flagged only, nothing deleted
	DUP-CONFLICT -- 2+ complete copies that disagree with each other --
	flagged only, needs manual review

	Usage:
	export FIGSHARE_TOKEN=xxxx
	python3 verify_uploads.py --base /home/narky/VBoxSHARED/access-om3-paper-1-runs
	python3 verify_uploads.py --base ... --confirm # also clean up safe duplicates
	"""

	import argparse
	import hashlib
	import json
	import os
	import sys
	from pathlib import Path

	import requests

	FIGSHARE_BASE_URL = "https://api.figshare.com/v2/{endpoint}"


	def _headers(token):
	return {"Authorization": f"token {token}"}


	def _request(method, url, token, data=None):
	headers = _headers(token)
	if data is not None:
	data = json.dumps(data)
	resp = requests.request(method, url, headers=headers, data=data, timeout=60)
	resp.raise_for_status()
	try:
	return json.loads(resp.content)
	except ValueError:
	return resp.content


	def md5_of(path):
	h = hashlib.md5()
	with open(path, "rb") as f:
	for block in iter(lambda: f.read(65536), b""):
	h.update(block)
	return h.hexdigest()


	def list_all_articles(token, page_size=100):
	"""Full paginated listing of every account article, building a
	title -> [id, ...] map (a list, not a single id -- duplicates matter
	here and must not be silently collapsed). Avoids the search endpoint
	entirely -- it turned out to be unreliable for titles containing
	literal '+' or '-' characters (e.g.
	'MC_100km_jra_ryf+wombatlite-1e74abf-11f9df5c', or even a plain
	hyphen-heavy title like '25km-iaf-test-for-AK-expt-7df5ef4c'), almost
	certainly because Figshare's search parses '+'/'-' as query-syntax
	operators (require/exclude) rather than literal text. A plain listing
	sidesteps that -- no query string, no parsing, just exact client-side
	matching."""
	titles = {}
	page = 1
	while True:
	url = FIGSHARE_BASE_URL.format(
	endpoint=f"account/articles?page={page}&page_size={page_size}")
	batch = _request("GET", url, token)
	if not batch:
	break
	for art in batch:
	titles.setdefault(art["title"], []).append(
	{"id": art["id"], "created_date": art.get("created_date")}
	)
	if len(batch) < page_size:
	break
	page += 1
	return titles


	def fetch_article_files_raw(token, article_id, page_size=100):
	"""Full paginated listing of an article's files, WITHOUT collapsing by
	name. The endpoint defaults to page_size=10 if unspecified -- always
	paginate explicitly (confirmed directly: a real 93-file article
	returned only 10 files with no page_size set).

	Returns the raw list. Collapsing this into a name-keyed dict (an
	earlier version of this script did exactly that) turned out to
	silently hide real problems: Figshare allows more than one file entry
	under the same name in one article, and a plain dict just keeps
	whichever one pagination returns last -- hiding a broken sibling
	entry behind a working one with the same name. Confirmed against real
	data: files the Figshare web UI flagged as "Something went wrong"
	were invisible to a dict-based version of this check because a
	working duplicate with the same name existed too."""
	files = []
	page = 1
	while True:
	url = FIGSHARE_BASE_URL.format(
	endpoint=f"account/articles/{article_id}/files?page={page}&page_size={page_size}")
	batch = _request("GET", url, token)
	if not batch:
	break
	files.extend(batch)
	if len(batch) < page_size:
	break
	page += 1
	return files


	def index_by_name(raw_files):
	"""{filename: [entry, entry, ...]} -- may have more than one entry
	per name; see fetch_article_files_raw's docstring for why that
	matters."""
	by_name = {}
	for f in raw_files:
	by_name.setdefault(f["name"], []).append(f)
	return by_name


	def file_status(entries, local_md5):
	"""Status for a local file against every remote entry sharing its
	name (there may be more than one -- see fetch_article_files_raw).
	Checking all of them, not just one, is what makes this
	duplicate-safe: a working entry anywhere in the list is enough for
	OK, even if a broken stub with the same name also exists."""
	if not entries:
	return "MISSING"
	if any(e.get("computed_md5") == local_md5 for e in entries):
	return "OK"
	if any(e.get("computed_md5") for e in entries):
	return "MISMATCH"
	return "PENDING"


	def classify_duplicates(entries, local_md5):
	"""Given 2+ remote entries sharing one filename, decide what (if
	anything) is safe to clean up automatically.

	Returns (action, to_delete, note):
	"delete_stubs" -- broken/incomplete entries with a genuine
	complete sibling present. Safe regardless
	of content, since Figshare's own UI text
	confirms unsuccessful uploads never appear
	publicly -- a stub adds nothing.
	"delete_older_dupes"-- 2+ complete entries, all sharing the SAME
	md5, AND that md5 matches local_md5. Only
	this case is safe to trim automatically;
	keeps the highest file id (files don't
	expose a timestamp, but ids are assigned in
	creation order, same as articles).
	"stale_duplicates" -- complete entries agree with each other but
	NOT with the current local file. Both/all
	copies are outdated -- flagged, nothing
	deleted, since which one (if any) should
	survive isn't this script's call.
	"conflicting" -- complete entries disagree with EACH OTHER.
	Flagged only, needs manual review.
	(None, [], "") -- fewer than 2 entries, nothing to do.
	"""
	if len(entries) < 2:
	return (None, [], "")

	complete = [e for e in entries if e.get("status") == "available" and e.get("computed_md5")]
	stubs = [e for e in entries if e not in complete]

	if complete and stubs:
	return ("delete_stubs", stubs,
	f"{len(stubs)} broken/incomplete entr{'y' if len(stubs)==1 else 'ies'} "
	f"alongside {len(complete)} working cop{'y' if len(complete)==1 else 'ies'}")

	if len(complete) >= 2:
	md5s = {e["computed_md5"] for e in complete}
	if len(md5s) == 1:
	shared_md5 = next(iter(md5s))
	if shared_md5 == local_md5:
	newest = max(complete, key=lambda e: e["id"])
	older = [e for e in complete if e["id"] != newest["id"]]
	return ("delete_older_dupes", older,
	f"{len(complete)} identical complete copies, matches local file "
	f"-- keeping newest (id {newest['id']})")
	else:
	return ("stale_duplicates", [],
	f"{len(complete)} identical complete copies, but content does NOT "
	f"match the current local file -- needs manual review")
	else:
	return ("conflicting", [],
	f"{len(complete)} complete copies with DIFFERING content "
	f"({len(md5s)} distinct md5s) -- needs manual review")

	# All entries are incomplete stubs, none complete -- not a duplicate
	# cleanup case, just an ordinary PENDING file (handled elsewhere).
	return (None, [], "")


	def discover_experiments(base: Path):
	"""Find every experiment dir under base with an mkfigs_output_* dir,
	without relying on a hardcoded RUNS list (mkfigs.sh's own list is known
	to drift out of sync with what actually exists on disk).

	Returns (exp_name, ofol) pairs. Rendered notebooks live directly in
	ofol (mkfigs_output_<experiment>/<nb>_rendered.ipynb); PNGs and
	markdown live in ofol/mkmd -- confirmed directly against pushit.py's
	own source (`rendered = ofol / f"{nb}_rendered.ipynb"`,
	`pngs = mdfol.glob(f"{nb}_*.png")` where `mdfol = ofol / "mkmd"`).
	Scanning only mkmd/ (an earlier version of this script's bug) finds
	every PNG but zero .ipynb files, which makes every real notebook
	upload look like an orphaned EXTRA."""
	found = []
	for exp_dir in sorted(base.iterdir()):
	if not exp_dir.is_dir():
	continue
	for ofol in exp_dir.glob("notebooks/mkfigs_output_*"):
	if ofol.is_dir():
	found.append((exp_dir.name, ofol))
	return found


	def main():
	ap = argparse.ArgumentParser(description=__doc__,
	formatter_class=argparse.RawDescriptionHelpFormatter)
	ap.add_argument("--base", required=True, help="Path containing all experiment directories")
	ap.add_argument("--token-env", default="FIGSHARE_TOKEN")
	ap.add_argument("--only-missing", action="store_true",
	help="Only print files that aren't fully OK (skip printing every success)")
	ap.add_argument("--confirm", action="store_true",
	help="Actually delete safe-to-remove duplicates (broken stubs, and "
	"older copies of an exact duplicate that matches the local file). "
	"Without this, everything is dry-run: reported but not deleted. "
	"Ambiguous duplicates (stale or conflicting) are NEVER auto-deleted "
	"regardless of this flag.")
	args = ap.parse_args()

	token = os.environ.get(args.token_env)
	if not token:
	sys.exit(f"Set {args.token_env} in the environment before running.")

	base = Path(args.base)
	if not base.is_dir():
	sys.exit(f"--base {base} is not a directory")

	experiments = discover_experiments(base)
	if not experiments:
	sys.exit(f"No experiments with a notebooks/mkfigs_output_* dir found under {base}")

	print(f"Found {len(experiments)} experiment(s) under {base}")
	print("Fetching full article listing from Figshare (one paginated scan)...\n")
	article_titles = list_all_articles(token)

	totals = {"OK": 0, "MISMATCH": 0, "PENDING": 0, "MISSING": 0, "EXTRA": 0}
	experiments_with_gaps = []

	for exp_name, ofol in experiments:
	mdfol = ofol / "mkmd"

	# Mirror pushit.py's own success gate exactly: a rendered.ipynb
	# file existing on disk does NOT mean the notebook succeeded --
	# nbconvert/papermill can leave a rendered.ipynb behind even when
	# the notebook errored mid-execution. pushit.py only counts a
	# notebook as OK if it also produced PNGs or markdown; otherwise
	# it's FAILED and was never uploaded, so it shouldn't be expected
	# on Figshare either. Only include ipynb+pngs for notebooks that
	# pass this same gate.
	#
	# PNG ownership: a naive f"{nb}_*.png" glob per notebook (what
	# pushit.py itself does) double-counts when one notebook's name is
	# a prefix of another's -- e.g. "MLD" matches "MLD_max_01.png" too,
	# since "MLD_*.png" doesn't care what's inside the wildcard. This
	# is a confirmed real bug in pushit.py itself (not just here): it
	# causes files like MLD_max's PNGs to be redundantly reprocessed
	# under MLD's own upload step. Fix here: assign each PNG to
	# exactly one notebook -- the longest (most specific) matching
	# name -- so ownership is unambiguous.
	notebook_names = sorted(
	(r.name[: -len("_rendered.ipynb")] for r in ofol.glob("*_rendered.ipynb")),
	key=len, reverse=True,
	)
	all_pngs = sorted(mdfol.glob("*.png")) if mdfol.exists() else []
	pngs_by_nb = {nb: [] for nb in notebook_names}
	for png in all_pngs:
	for nb in notebook_names: # longest names checked first
	if png.name.startswith(f"{nb}_"):
	pngs_by_nb[nb].append(png)
	break

	local_files = []
	for rendered in sorted(ofol.glob("*_rendered.ipynb")):
	nb = rendered.name[: -len("_rendered.ipynb")]
	pngs = pngs_by_nb.get(nb, [])
	nb_md = mdfol / f"{nb}.md"
	if not pngs and not nb_md.exists():
	continue # FAILED per pushit.py's own gate -- never uploaded, skip
	local_files.append(rendered)
	local_files.extend(pngs)
	local_files = sorted(set(local_files))

	if not local_files:
	continue

	title = f"ACCESS-OM3 evaluation figures – {exp_name}"
	matches = article_titles.get(title, [])

	if not matches:
	article_id = None
	elif len(matches) == 1:
	article_id = matches[0]["id"]
	else:
	# Duplicate articles under the same title. Don't silently pick
	# one -- check file counts on each and flag this loudly, since
	# it likely means a run couldn't find the real article (title
	# search failing on +/- characters) and created a fresh one.
	print(f" * WARNING: {len(matches)} articles share this title! *")
	best_id, best_count = None, -1
	for m in matches:
	fcount = len(fetch_article_files_raw(token, m["id"]))
	print(f" article {m['id']} created {m.get('created_date')} "
	f"{fcount} file(s)")
	if fcount > best_count:
	best_id, best_count = m["id"], fcount
	print(f" -> using {best_id} (most files) for comparison below, "
	f"but this needs manual cleanup on Figshare.")
	article_id = best_id

	print(f"=== {exp_name} ({len(local_files)} local files) ===")
	if article_id is None:
	print(f" NO ARTICLE -- '{title}' not found on Figshare")
	totals["MISSING"] += len(local_files)
	experiments_with_gaps.append(exp_name)
	print()
	continue

	raw_files = fetch_article_files_raw(token, article_id)
	by_name = index_by_name(raw_files)
	exp_gap = False
	local_names = {p.name for p in local_files}
	local_md5s = {} # filled in below, reused by duplicate classification

	for local_path in local_files:
	fname = local_path.name
	local_md5 = md5_of(local_path)
	local_md5s[fname] = local_md5
	entries = by_name.get(fname, [])

	status = file_status(entries, local_md5)

	totals[status] += 1
	if status != "OK":
	exp_gap = True
	if status != "OK" or not args.only_missing:
	print(f" {status:9s} {fname}")

	# Reverse direction: files on Figshare with no local counterpart at
	# all. Could be stale leftovers from a notebook that used to
	# produce more images (e.g. a renumbered PNG series), or a genuine
	# duplicate/orphaned upload -- either way, worth knowing about
	# before publishing, not just gaps in the other direction.
	extra_names = sorted(set(by_name) - local_names)
	for fname in extra_names:
	totals["EXTRA"] += 1
	exp_gap = True
	print(f" EXTRA {fname} (on Figshare, no local file with this name)")

	# Duplicate detection/cleanup -- only for names with a real local
	# file to check against (see classify_duplicates' docstring for
	# why "must match the file on disk" matters: two remote copies
	# agreeing with each other is NOT enough on its own, they could
	# both be stale).
	for local_path in local_files:
	fname = local_path.name
	entries = by_name.get(fname, [])
	if len(entries) < 2:
	continue
	action, to_delete, note = classify_duplicates(entries, local_md5s[fname])
	if action is None:
	continue
	exp_gap = True
	totals["DUP_" + action.upper()] = totals.get("DUP_" + action.upper(), 0) + 1
	tag = {
	"delete_stubs": "DUP-STUB",
	"delete_older_dupes": "DUP-OLD ",
	"stale_duplicates": "DUP-STALE",
	"conflicting": "DUP-CONFLICT",
	}[action]
	print(f" {tag:12s} {fname} ({note})")
	if action in ("delete_stubs", "delete_older_dupes"):
	for e in to_delete:
	if args.confirm:
	del_url = FIGSHARE_BASE_URL.format(
	endpoint=f"account/articles/{article_id}/files/{e['id']}")
	requests.delete(del_url, headers=_headers(token), timeout=30).raise_for_status()
	print(f" -> deleted id {e['id']}")
	else:
	print(f" -> would delete id {e['id']} "
	f"(pass --confirm to actually delete)")

	if exp_gap:
	experiments_with_gaps.append(exp_name)
	print()

	print("=" * 60)
	print("SUMMARY")
	print("=" * 60)
	for status in ("OK", "PENDING", "MISMATCH", "MISSING", "EXTRA"):
	print(f" {status:9s} {totals[status]}")
	dup_keys = [k for k in totals if k.startswith("DUP_")]
	if dup_keys:
	print()
	for k in sorted(dup_keys):
	print(f" {k:22s} {totals[k]}")
	if not args.confirm:
	print("\n (dry run -- re-run with --confirm to actually delete "
	"DUP_DELETE_STUBS / DUP_DELETE_OLDER_DUPES entries)")
	print()
	if experiments_with_gaps:
	print(f"{len(experiments_with_gaps)} experiment(s) have gaps -- NOT ready to publish:")
	for e in experiments_with_gaps:
	print(f" - {e}")
	sys.exit(1)
	else:
	print("All local .ipynb/.png files are fully and correctly uploaded, "
	"and nothing extra is sitting on Figshare. Safe to publish.")
	sys.exit(0)


	if __name__ == "__main__":
	main()
No results found