From 2687e5277a429b0bf85b67a052b488eece4c0e1d Mon Sep 17 00:00:00 2001 From: Gabriel Spadon Date: Mon, 26 Oct 2026 10:44:03 -0300 Subject: [PATCH] feat: scope the monthly refresh to the website author The monthly refresh enriched all 64 enabled authors in data/input.csv and took 40 to 100 minutes. Its only consumer, MAPS-Lab/mapslab-website import-publications.yml, sparse-checks out `output/Spadon (bfdGsGUAAAAJ)` and aggregates that directory alone. Add data/website-authors.csv (the input.csv header and the Gabriel Spadon row) and pass it with --input to every `python3 main.py` call in monthly-refresh.yml. The manifest reads the same file, so enabled_authors and authors_with_bib describe one census. A subset input was not safe against the standing output tree. finalize_run walked every directory for the orphan sweep, year-window deletion, post-run fixup and rename, superseded-preprint removal and the DOI conflict gate, and run_all forced re-enrichment over conflicts it could not settle. All of them now read only the directories of the given records through iter_output_dirs(only=...). baseline.json still mirrors the whole tree. build_a2i2_folder rebuilds a populated folder only when every member resolved, where it used to accept half. The committed corpus already holds DOI conflicts in Haque, Oore and Trappenberg that only a full run's forced re-enrichment closes, so an unscoped one-author run ended in FinalizationError. --- .github/workflows/monthly-refresh.yml | 26 ++++- README.md | 6 + citeforge/fsscan.py | 10 +- citeforge/io_utils.py | 36 +++--- citeforge/pipeline/postrun.py | 24 ++-- citeforge/pipeline/scheduler.py | 9 +- data/website-authors.csv | 2 + tests/test_finalize_run.py | 160 +++++++++++++++++++++++++- tests/test_fsscan.py | 10 ++ tests/test_io_csv.py | 10 +- tests/test_regression.py | 19 ++- tests/test_scheduler_parallel.py | 29 ++++- tests/test_workflow_contracts.py | 44 +++++++ 13 files changed, 346 insertions(+), 39 deletions(-) create mode 100644 data/website-authors.csv diff --git a/.github/workflows/monthly-refresh.yml b/.github/workflows/monthly-refresh.yml index e2382580..eb8f710c 100644 --- a/.github/workflows/monthly-refresh.yml +++ b/.github/workflows/monthly-refresh.yml @@ -35,6 +35,16 @@ on: permissions: contents: read +# The refresh covers the authors of one census file, not data/input.csv. Its only +# consumer, MAPS-Lab/mapslab-website import-publications.yml, sparse-checks out +# `output/Spadon (bfdGsGUAAAAJ)` and aggregates nothing else, so enriching the +# other authors cost 40 to 100 minutes of provider quota per run for output no +# one reads. The post-run tail is confined to the census authors' directories +# (see finalize_run), which leaves every other committed author untouched. Every +# gate below reads this same file, so the counts compare like with like. +env: + MONTHLY_CENSUS: data/website-authors.csv + # One refresh at a time. Two overlapping runs race on the same output tree and # the same publication branch, and the loser force-pushes over the winner. # Never cancel in flight: a cancelled run loses its corpus progress. @@ -124,7 +134,7 @@ jobs: for i in $(seq 1 $max_runs); do echo "::group::Pipeline run $i/$max_runs" rc=0 - python3 main.py || rc=$? + python3 main.py --input "$MONTHLY_CENSUS" || rc=$? echo "::endgroup::" if [ "$rc" -ne 0 ]; then @@ -194,9 +204,17 @@ jobs: run: | set -euo pipefail mkdir -p /tmp/refresh-manifest - enabled_authors="$(python -c 'import csv; f=open("data/input.csv", encoding="utf-8", newline=""); print(sum(row["Enabled"].strip().lower() == "true" for row in csv.DictReader(f)))')" - authors_with_bib="$(find output -mindepth 2 -maxdepth 2 -type f -name '*.bib' ! -path 'output/a2i2/*' -print0 \ - | xargs -0 -r -n1 dirname | LC_ALL=C sort -u | wc -l)" + # Census authors only: the output tree holds every committed author, so + # a whole-tree directory count could never be compared with the census. + read -r enabled_authors authors_with_bib < <(python -c ' + import sys + from pathlib import Path + from citeforge.io_utils import read_records + from citeforge.text_utils import format_author_dirname + records = read_records(sys.argv[1]) + dirs = [Path("output", format_author_dirname(r.name, r.scholar_id or r.dblp or "")) for r in records] + print(len(records), sum(any(d.glob("*.bib")) for d in dirs)) + ' "$MONTHLY_CENSUS") { echo "run_id=${{ github.run_id }}" echo "converged=${{ steps.loop.outputs.converged }}" diff --git a/README.md b/README.md index 2ed9ca63..3dcab611 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,12 @@ A scheduled workflow re-runs the pipeline until the corpus digest stops changing a pull request. Publication always goes through review and Required CI; nothing is pushed to `main` directly, and the website sync only fires once the corpus has actually merged. +It covers the authors in `data/website-authors.csv`, passed with `--input`, rather than all of +`data/input.csv`: the website imports a single author's directory, and enriching the rest cost +40 to 100 minutes of provider quota per run. A run over part of the census touches no other author's +directory, and leaves `output/a2i2/` alone unless every a2i2 member is in its input. Run `citeforge` with the full census to refresh +the other authors and the joint folder. + ### Refresh engine `citeforge/refresh/` holds a second, resumable implementation with a durable ledger, reachable diff --git a/citeforge/fsscan.py b/citeforge/fsscan.py index eb45628d..5355d3d2 100644 --- a/citeforge/fsscan.py +++ b/citeforge/fsscan.py @@ -16,7 +16,7 @@ import shutil import subprocess import unicodedata -from collections.abc import Callable, Iterator +from collections.abc import Callable, Collection, Iterator from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any @@ -273,10 +273,14 @@ def iter_parsed_author_bibs( yield filename, path, entry -def iter_output_dirs(out_dir: str) -> list[str]: +def iter_output_dirs(out_dir: str, *, only: Collection[str] | None = None) -> list[str]: """Return the immediate subdirectory names of ``out_dir``, sorted. Directory entry names only (not full paths); plain files are excluded. + ``only`` restricts the scan to those names, which is how a run limited to a + subset of authors keeps every pass off the directories it does not own. ``OSError`` from :func:`os.listdir` is not swallowed here. """ - return sorted(e for e in os.listdir(out_dir) if os.path.isdir(os.path.join(out_dir, e))) + return sorted( + e for e in os.listdir(out_dir) if (only is None or e in only) and os.path.isdir(os.path.join(out_dir, e)) + ) diff --git a/citeforge/io_utils.py b/citeforge/io_utils.py index 51bd47cc..3e6ad7c3 100644 --- a/citeforge/io_utils.py +++ b/citeforge/io_utils.py @@ -15,7 +15,7 @@ import re import tempfile import threading -from collections.abc import Callable +from collections.abc import Callable, Collection from pathlib import Path from typing import IO, Any @@ -84,10 +84,15 @@ def _citation_coherence_signature(entry: dict[str, Any]) -> _CitationCoherenceSi ) -def find_incoherent_doi_author_dirs(out_dir: str) -> frozenset[str]: - """Return author directories containing materially conflicting copies of one DOI.""" +def find_incoherent_doi_author_dirs(out_dir: str, author_dirs: Collection[str]) -> frozenset[str]: + """Return those of *author_dirs* holding materially conflicting copies of one DOI. + + Only the named directories are read, so a copy in a directory outside the run + can neither be reported nor drag its owner into a forced re-enrichment the + run cannot perform. + """ grouped: dict[str, list[tuple[str, _CitationCoherenceSignature]]] = {} - for dirname in iter_output_dirs(out_dir): + for dirname in iter_output_dirs(out_dir, only=author_dirs): if dirname == A2I2_OUTPUT_DIR: continue author_dir = os.path.join(out_dir, dirname) @@ -376,8 +381,8 @@ def flush_summary_csv(csv_path: str) -> None: writer.writerow(row) -def collect_orphan_files(csv_path: str, output_dir: str) -> list[str]: - """Return absolute paths of .bib files on disk that have no entry in the summary CSV. +def collect_orphan_files(csv_path: str, output_dir: str, author_dirs: Collection[str]) -> list[str]: + """Return absolute paths of .bib files in *author_dirs* that have no entry in the summary CSV. These are stale leftovers from previous runs where the same article received a different citation key (e.g. Gemini returned a different @@ -402,7 +407,9 @@ def collect_orphan_files(csv_path: str, output_dir: str) -> list[str]: # a2i2 is rebuilt from the author directories every run and is never # tracked in the CSV, so its files would all read as orphans and become # deletion candidates. The other whole-tree passes skip it too. - subdirs = [os.path.join(output_dir, d) for d in iter_output_dirs(output_dir) if d != A2I2_OUTPUT_DIR] + subdirs = [ + os.path.join(output_dir, d) for d in iter_output_dirs(output_dir, only=author_dirs) if d != A2I2_OUTPUT_DIR + ] except OSError: return [] @@ -646,13 +653,14 @@ def _pick_richer( a2i2_dir = os.path.join(out_dir, A2I2_OUTPUT_DIR) os.makedirs(a2i2_dir, exist_ok=True) existing = [f for f in sorted(os.listdir(a2i2_dir)) if os.path.isfile(os.path.join(a2i2_dir, f))] - # Never empty a populated folder because the member list mostly failed to - # resolve. `--input` is operator-settable against the standing output tree, - # so a subset CSV would otherwise wipe the aggregate and report only the - # small number it rebuilt. - if existing and resolved_members < len(a2i2_names) / 2: - log.error( - "a2i2 rebuild aborted: only %d of %d members resolved against the input records, " + # The folder is a joint view over every member, so a populated one is never + # rebuilt from a subset. `--input` is operator-settable against the standing + # output tree, and a subset CSV would otherwise replace the aggregate with + # the unresolved members' papers missing. A majority is no safer than one + # author: every unresolved member's contribution is dropped either way. + if existing and resolved_members < len(a2i2_names): + log.warning( + "a2i2 rebuild skipped: only %d of %d members resolved against the input records, " "which would delete %d existing file(s). Pass the full author list to rebuild.", resolved_members, len(a2i2_names), diff --git a/citeforge/pipeline/postrun.py b/citeforge/pipeline/postrun.py index baffaae9..58ad9aa8 100644 --- a/citeforge/pipeline/postrun.py +++ b/citeforge/pipeline/postrun.py @@ -15,6 +15,7 @@ import csv import os import re +from collections.abc import Collection from dataclasses import dataclass from typing import Any @@ -50,6 +51,7 @@ _is_preprint_fields, author_list_is_surname_initials, extract_year_from_any, + format_author_dirname, ) # The author prefix of a generated name: "Jin2026-AlteredNeuro.bib" and the @@ -142,9 +144,15 @@ def finalize_run( cleanup, post-run fixup, superseded-preprint removal, the a2i2 build and the baseline.json rewrite. Order is load-bearing. + Every step that deletes, rewrites or renames a ``.bib`` is confined to the + directories of *records*, so a run over a subset of the census leaves every + other author's committed output untouched. Only the baseline count, which + mirrors the whole tree, still reads outside that set. + Raises :class:`FinalizationError` when a step cannot read or write a file it is responsible for. """ + in_scope = frozenset(format_author_dirname(rec.name, rec.scholar_id or rec.dblp or "") for rec in records) counts = get_api_call_counts() logger.step("Run complete", category=LogCategory.PLAN) logger.info(f"Records processed: {processed}", category=LogCategory.PLAN) @@ -172,7 +180,7 @@ def finalize_run( logger.info(f"Reconciled summary CSV: removed {phantoms} phantom entries", category=LogCategory.CLEANUP) # Safe orphan removal (duplicates only) - orphans = collect_orphan_files(csv_path, out_dir) + orphans = collect_orphan_files(csv_path, out_dir, in_scope) if orphans: csv_entries = _load_csv_entries(csv_path) for orphan in orphans: @@ -216,7 +224,7 @@ def finalize_run( # Remove .bib files outside the contribution window window_min = get_min_year() window_removed = 0 - for entry in iter_output_dirs(out_dir): + for entry in iter_output_dirs(out_dir, only=in_scope): if entry == A2I2_OUTPUT_DIR: continue d = os.path.join(out_dir, entry) @@ -266,7 +274,7 @@ def finalize_run( renamed_count = 0 renames_blocked = 0 applied_renames: dict[str, str] = {} - for pr_entry_name in iter_output_dirs(out_dir): + for pr_entry_name in iter_output_dirs(out_dir, only=in_scope): if pr_entry_name == A2I2_OUTPUT_DIR: continue pr_dir = os.path.join(out_dir, pr_entry_name) @@ -338,7 +346,7 @@ def finalize_run( category=LogCategory.CLEANUP, ) - superseded = _remove_superseded_preprints(out_dir) + superseded = _remove_superseded_preprints(out_dir, in_scope) if superseded: logger.info( f"Removed {superseded} superseded preprint .bib files (published twin exists)", @@ -360,7 +368,7 @@ def finalize_run( category=LogCategory.CLEANUP, ) - unresolved_doi_conflicts = find_incoherent_doi_author_dirs(out_dir) + unresolved_doi_conflicts = find_incoherent_doi_author_dirs(out_dir, in_scope) if unresolved_doi_conflicts: affected = ", ".join(sorted(unresolved_doi_conflicts)) raise FinalizationError(f"conflicting copies of the same DOI remain in author directories: {affected}") @@ -442,10 +450,10 @@ def _looks_published(entry: dict[str, Any]) -> bool: return has_container or (bool(doi) and bool(str(fields.get("publisher") or "").strip())) -def _remove_superseded_preprints(out_dir: str) -> int: +def _remove_superseded_preprints(out_dir: str, author_dirs: Collection[str]) -> int: """Delete a preprint ``.bib`` when a published record of the same work exists. - Within each author directory, a preprint file is removed only when another + Within each directory of *author_dirs*, a preprint file is removed only when another file in that directory is a genuinely published record (see :func:`_looks_published`) whose title matches at or above ``SIM_MERGE_DUPLICATE_THRESHOLD``. This generalizes the "published outranks a @@ -457,7 +465,7 @@ def _remove_superseded_preprints(out_dir: str) -> int: Returns the number of preprint files removed. """ removed = 0 - for entry_name in iter_output_dirs(out_dir): + for entry_name in iter_output_dirs(out_dir, only=author_dirs): if entry_name == A2I2_OUTPUT_DIR: continue author_dir = os.path.join(out_dir, entry_name) diff --git a/citeforge/pipeline/scheduler.py b/citeforge/pipeline/scheduler.py index e6f9dfd4..d2fc0684 100644 --- a/citeforge/pipeline/scheduler.py +++ b/citeforge/pipeline/scheduler.py @@ -348,8 +348,13 @@ def run_all( can open a conflict the run then has to close. Authors drawn into a conflict by an earlier pass are forced in a further pass, each at most once, which bounds the loop by the number of authors. Returns (total_saved, processed). + + Conflicts are looked for only among the directories of *records*. A co-author + outside the run cannot be re-enriched by it, so counting that author's copy + would force a pass that can never settle the conflict. """ - forced = find_incoherent_doi_author_dirs(out_dir) + in_scope = frozenset(_author_dirname(rec) for rec in records) + forced = find_incoherent_doi_author_dirs(out_dir, in_scope) if forced: logger.warn( f"Forcing enrichment for {len(forced)} author output directories with conflicting copies of the same DOI", @@ -358,7 +363,7 @@ def run_all( args = (serpapi_key, serply_key, s2_api_key, or_creds, gemini_api_key) total_saved, processed = _run_pass(*args, records, out_dir, summary_csv_path, force_enrich, forced) while not force_enrich: - reopened = find_incoherent_doi_author_dirs(out_dir) - forced + reopened = find_incoherent_doi_author_dirs(out_dir, in_scope) - forced batch = [rec for rec in records if _author_dirname(rec) in reopened] if not batch: break diff --git a/data/website-authors.csv b/data/website-authors.csv new file mode 100644 index 00000000..a1403c26 --- /dev/null +++ b/data/website-authors.csv @@ -0,0 +1,2 @@ +Name,Scholar Link,DBLP Link,Enabled,Exclusion Reason +Gabriel Spadon,https://scholar.google.com/citations?user=bfdGsGUAAAAJ,https://dblp.org/pid/192/1659,true, diff --git a/tests/test_finalize_run.py b/tests/test_finalize_run.py index 79785813..66686923 100644 --- a/tests/test_finalize_run.py +++ b/tests/test_finalize_run.py @@ -32,6 +32,7 @@ import pytest +from citeforge import bibtex_utils as bt from citeforge.config import get_min_year from citeforge.io_utils import append_summary_to_csv, init_summary_csv from citeforge.models import Record @@ -804,8 +805,9 @@ def test_unresolved_same_doi_conflict_aborts_finalization(tmp_path: Path, no_a2i write_bib(first_dir, article(key="DoeStudy", author="Jane Doe and Richard Roe", **common), "Doe-Study.bib") write_bib(second_dir, article(key="DoeStudy", author="Jane Doe and Alice Poe", **common), "Doe-Study.bib") + both = [Record(name="Doe, Jane", scholar_id="abc123"), Record(name="Roe, Richard", scholar_id="def456")] with pytest.raises(FinalizationError, match="conflicting copies of the same DOI"): - finalize_run(str(out_dir), _records(), total_saved=2, processed=1, summary_csv_path=None) + finalize_run(str(out_dir), both, total_saved=2, processed=2, summary_csv_path=None) def test_failed_baseline_write_raises(tmp_path: Path, no_a2i2: None, monkeypatch: pytest.MonkeyPatch) -> None: @@ -823,3 +825,159 @@ def test_failed_baseline_write_raises(tmp_path: Path, no_a2i2: None, monkeypatch with pytest.raises(FinalizationError, match=r"baseline\.json"): finalize_run(str(out_dir), _records(), total_saved=1, processed=1, summary_csv_path=str(csv_path)) + + +# --- SUBSET RUN SCOPE ------------------------------------------------------- +# +# `--input` is operator-settable against the standing output tree, and the +# monthly refresh now names a one-author census. Every pass in the tail that +# deletes, rewrites or renames a file must stay inside the directories of the +# records it was given, or one author's refresh silently edits forty others'. + +_OUT_OF_SCOPE = ("Roe (r1)", "Poe (p1)") +_SHARED_DOI = "10.1000/shared" + + +def _tree_bytes(root: Path, *, skip: frozenset[str]) -> dict[str, bytes]: + """Map every file under *root* to its bytes, leaving out the top-level names in *skip*.""" + return { + str(p.relative_to(root)): p.read_bytes() + for p in sorted(root.rglob("*")) + if p.is_file() and p.relative_to(root).parts[0] not in skip + } + + +def _plant_hazards(out_dir: Path) -> list[Path]: + """Fill the two out-of-scope author directories with one trap per tail pass. + + Returns the tracked paths. Each trap is something finalize_run deletes, + rewrites or refuses to continue past when it can see it, so the byte-equality + assertion is not vacuous. + """ + roe, poe = (out_dir / name for name in _OUT_OF_SCOPE) + alpha = "Alpha Study of Vessel Tracking Systems" + shared_title = "Detecting Ongoing Events Using Contextual Word and Sentence Embeddings" + tracked = [ + write_bib(roe, article(key="RoeAlpha", title=alpha), f"Roe{_IN_WINDOW_YEAR}-Alpha.bib"), + # Out of the contribution window, so the year-window pass deletes it. + write_bib( + roe, + article(key="RoeOld", title="An Old Study of Harbour Currents", year=str(get_min_year() - 1)), + f"Roe{get_min_year() - 1}-Old.bib", + ), + # A preprint with a published twin, so the superseded-preprint pass deletes it. + write_bib( + roe, + misc( + key="RoePre", + title=shared_title, + year=str(_IN_WINDOW_YEAR - 2), + howpublished="arXiv", + doi="10.48550/arxiv.2007.01379", + ), + f"Roe{_IN_WINDOW_YEAR - 2}-Detecting.bib", + ), + write_bib( + roe, + article( + key="RoePub", + title=shared_title, + year=str(_IN_WINDOW_YEAR), + journal="Expert Systems with Applications", + doi="10.1016/j.eswa.2022.118257", + ), + f"Roe{_IN_WINDOW_YEAR}-Detecting.bib", + ), + # Two authors holding the same DOI with different metadata. + write_bib( + roe, + article(key="RoeShared", title="Gamma Study of Reef Acoustics", year=str(_IN_WINDOW_YEAR), doi=_SHARED_DOI), + f"Roe{_IN_WINDOW_YEAR}-Gamma.bib", + ), + write_bib( + poe, + article( + key="PoeShared", + title="Gamma Revisited, a Reef Acoustics Survey", + year=str(_IN_WINDOW_YEAR + 1), + doi=_SHARED_DOI, + ), + f"Poe{_IN_WINDOW_YEAR + 1}-Gamma.bib", + ), + ] + # Valid BibTeX that is not in the serializer's form, so the fixup pass rewrites it. + loose = poe / f"Poe{_IN_WINDOW_YEAR}-Delta.bib" + loose.write_text( + f"@article{{PoeDelta,\n\ttitle={{Delta Study of Harbour Currents}}, author={{Poe, Peter}}," + f"year={{{_IN_WINDOW_YEAR}}},journal={{Ocean Dynamics}}}}\n", + encoding="utf-8", + ) + parsed = bt.parse_bibtex_to_dict(loose.read_text(encoding="utf-8")) + assert parsed is not None and bt.bibtex_from_dict(parsed) != loose.read_text(encoding="utf-8") + tracked.append(loose) + # An untracked duplicate of a tracked file, so the orphan sweep deletes it. + write_bib(roe, article(key="RoeAlphaCopy", title=alpha), f"Roe{_IN_WINDOW_YEAR}-AlphaCopy.bib") + return tracked + + +def test_a_subset_run_leaves_every_other_authors_output_byte_identical( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """One author's finalization deletes, rewrites and renames nothing outside its directory. + + The in-scope author holds a duplicate orphan and a DOI that two out-of-scope + authors also hold with different metadata. The out-of-scope directories hold + one trap per pass. Before the tail was scoped, this run deleted three of their + files, rewrote four more, and then raised on the DOI conflict it could not + resolve, because those two authors are not in the run to be re-enriched. + """ + out_dir = tmp_path / "out" + doe = _author_dir(out_dir) + alpha = "Alpha Study of Vessel Tracking Systems" + doe_tracked = [ + write_bib(doe, article(key="DoeAlpha", title=alpha), f"Doe{_IN_WINDOW_YEAR}-Alpha.bib"), + write_bib( + doe, + article(key="DoeShared", title="Gamma and the Reef", year=str(_IN_WINDOW_YEAR), doi=_SHARED_DOI), + f"Doe{_IN_WINDOW_YEAR}-Gamma.bib", + ), + ] + doe_orphan = write_bib(doe, article(key="DoeAlphaCopy", title=alpha), f"Doe{_IN_WINDOW_YEAR}-AlphaCopy.bib") + tracked = sorted([*doe_tracked, *_plant_hazards(out_dir)], key=str) + a2i2_input = tmp_path / "a2i2.csv" + a2i2_input.write_text("Name,Scholar Link,DBLP Link\nJane Doe,,\nRichard Roe,,\nPeter Poe,,\n", encoding="utf-8") + monkeypatch.setattr(postrun, "DEFAULT_A2I2_INPUT", str(a2i2_input)) + write_bib(out_dir / "a2i2", article(key="Joint", title="A Joint A2I2 Paper"), f"Joint{_IN_WINDOW_YEAR}-Paper.bib") + csv_path = tmp_path / "summary.csv" + _track_in_csv(csv_path, tracked) + untouchable = frozenset({doe.name, "baseline.json"}) + before = _tree_bytes(out_dir, skip=untouchable) + csv_before = csv_path.read_bytes() + records = [Record(name="Jane Doe", scholar_id="abc123")] + + finalize_run(str(out_dir), records, total_saved=1, processed=1, summary_csv_path=str(csv_path)) + + assert _tree_bytes(out_dir, skip=untouchable) == before + assert csv_path.read_bytes() == csv_before + assert not doe_orphan.exists(), "the in-scope author must still be swept" + mirror = json.loads((out_dir / "baseline.json").read_text(encoding="utf-8"))["authors"] + assert {name: mirror[name] for name in _OUT_OF_SCOPE} == {"Roe (r1)": 6, "Poe (p1)": 2} + + +def test_a_run_over_two_of_three_a2i2_members_does_not_rebuild_the_folder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Dropping a member from the input must not silently drop its papers from a2i2.""" + out_dir = tmp_path / "out" + write_bib(_author_dir(out_dir, "Doe (abc123)"), article(key="D", title="Doe Paper"), f"Doe{_IN_WINDOW_YEAR}-D.bib") + write_bib(_author_dir(out_dir, "Roe (r1)"), article(key="R", title="Roe Paper"), f"Roe{_IN_WINDOW_YEAR}-R.bib") + joint = write_bib(out_dir / "a2i2", article(key="P", title="Poe Paper"), f"Poe{_IN_WINDOW_YEAR}-P.bib") + a2i2_input = tmp_path / "a2i2.csv" + a2i2_input.write_text("Name,Scholar Link,DBLP Link\nJane Doe,,\nRichard Roe,,\nPeter Poe,,\n", encoding="utf-8") + monkeypatch.setattr(postrun, "DEFAULT_A2I2_INPUT", str(a2i2_input)) + records = [Record(name="Jane Doe", scholar_id="abc123"), Record(name="Richard Roe", scholar_id="r1")] + + report = finalize_run(str(out_dir), records, total_saved=2, processed=2, summary_csv_path=None) + + assert report.a2i2_files == 0 + assert [p.name for p in (out_dir / "a2i2").iterdir()] == [joint.name] diff --git a/tests/test_fsscan.py b/tests/test_fsscan.py index 91a0045c..4c184141 100644 --- a/tests/test_fsscan.py +++ b/tests/test_fsscan.py @@ -41,6 +41,16 @@ def test_iter_output_dirs_returns_sorted_subdir_names_excluding_files(tmp_path: assert iter_output_dirs(str(tmp_path)) == ["alpha", "zeta"] +def test_iter_output_dirs_only_keeps_the_named_directories(tmp_path: pathlib.Path) -> None: + for name in ("alpha", "beta", "zeta"): + (tmp_path / name).mkdir() + (tmp_path / "gamma").write_text("", encoding="utf-8") + + # A name that is absent, or is a file, is not invented. + assert iter_output_dirs(str(tmp_path), only={"zeta", "alpha", "gamma", "missing"}) == ["alpha", "zeta"] + assert iter_output_dirs(str(tmp_path), only=set()) == [] + + def _write_scan_fixture(tmp_path: pathlib.Path) -> None: """Author dir with parseable, unparseable, non-bib, and unreadable entries.""" (tmp_path / "c.bib").write_text(_BIB_C, encoding="utf-8") diff --git a/tests/test_io_csv.py b/tests/test_io_csv.py index 89565557..00e50c31 100644 --- a/tests/test_io_csv.py +++ b/tests/test_io_csv.py @@ -244,7 +244,10 @@ def test_incoherent_doi_scan_returns_every_affected_author_directory(tmp_path: P _write_bib(dir_a / "Lowe2021.bib", "inproceedings", "Lowe2021", {**common, "year": "2021"}) _write_bib(dir_b / "Lowe2022.bib", "inproceedings", "Lowe2022", {**common, "year": "2022"}) - assert io_utils.find_incoherent_doi_author_dirs(str(out)) == frozenset({"Oore (A1)", "Trappenberg (B1)"}) + both = {"Oore (A1)", "Trappenberg (B1)"} + assert io_utils.find_incoherent_doi_author_dirs(str(out), both) == frozenset(both) + # A copy outside the named directories is not read, so it can conflict with nothing. + assert io_utils.find_incoherent_doi_author_dirs(str(out), {"Oore (A1)"}) == frozenset() def test_incoherent_doi_scan_compares_author_identity_not_only_count(tmp_path: Path) -> None: @@ -262,7 +265,8 @@ def test_incoherent_doi_scan_compares_author_identity_not_only_count(tmp_path: P _write_bib(dir_a / "Hasan2025.bib", "article", "Hasan2025", {**common, "author": "Tariq Hasan and Evan Frick"}) _write_bib(dir_b / "Hasan2025.bib", "article", "Hasan2025", {**common, "author": "Tariq Hasan and Mahdi Haque"}) - assert io_utils.find_incoherent_doi_author_dirs(str(out)) == frozenset({"Hasan (A1)", "Haque (B1)"}) + both = {"Hasan (A1)", "Haque (B1)"} + assert io_utils.find_incoherent_doi_author_dirs(str(out), both) == frozenset(both) def test_incoherent_doi_scan_accepts_equivalent_author_renderings(tmp_path: Path) -> None: @@ -280,7 +284,7 @@ def test_incoherent_doi_scan_accepts_equivalent_author_renderings(tmp_path: Path _write_bib(dir_a / "Oore2025.bib", "article", "Oore2025", {**common, "author": "Sageev Oore and Jason D'eon"}) _write_bib(dir_b / "Oore2025.bib", "article", "Oore2025", {**common, "author": "Oore, Sageev and D'eon, Jason"}) - assert io_utils.find_incoherent_doi_author_dirs(str(out)) == frozenset() + assert io_utils.find_incoherent_doi_author_dirs(str(out), {"Oore (A1)", "Rudzicz (B1)"}) == frozenset() class TestBuildA2i2Folder: diff --git a/tests/test_regression.py b/tests/test_regression.py index c671f172..4ecaf78c 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -2402,7 +2402,7 @@ def test_orphan_detected(self, tmp_path: Any) -> None: row.update({k: "0" for k in _SUMMARY_CSV_FIELDNAMES if k not in row}) writer.writerow(row) - orphans = collect_orphan_files(csv_path, str(out_dir)) + orphans = collect_orphan_files(csv_path, str(out_dir), {"Author (ID)"}) assert len(orphans) == 1 assert os.path.basename(orphans[0]) == "Orphan2023-OldKey.bib" @@ -2426,9 +2426,24 @@ def test_no_orphans(self, tmp_path: Any) -> None: row.update({k: "0" for k in _SUMMARY_CSV_FIELDNAMES if k not in row}) writer.writerow(row) - orphans = collect_orphan_files(csv_path, str(out_dir)) + orphans = collect_orphan_files(csv_path, str(out_dir), {"Author (ID)"}) assert len(orphans) == 0 + def test_only_the_named_author_directories_are_scanned(self, tmp_path: Any) -> None: + """An untracked file in a directory outside the run is not a deletion candidate.""" + from citeforge.io_utils import collect_orphan_files, init_summary_csv + + out_dir = tmp_path / "output" + for name in ("Author (ID)", "Other (ID2)"): + (out_dir / name).mkdir(parents=True) + (out_dir / name / "Untracked2024-Paper.bib").write_text("@article{k, title={U}}") + csv_path = str(tmp_path / "summary.csv") + init_summary_csv(csv_path) + + orphans = collect_orphan_files(csv_path, str(out_dir), {"Author (ID)"}) + + assert orphans == [str(out_dir / "Author (ID)" / "Untracked2024-Paper.bib")] + class TestIsKnownSummaryPath: """is_known_summary_path checks the in-memory set from init_summary_csv.""" diff --git a/tests/test_scheduler_parallel.py b/tests/test_scheduler_parallel.py index a80fe48d..dd969044 100644 --- a/tests/test_scheduler_parallel.py +++ b/tests/test_scheduler_parallel.py @@ -161,7 +161,7 @@ def capture_record( monkeypatch.setattr( scheduler, "find_incoherent_doi_author_dirs", - lambda _out_dir: frozenset({"Lovelace (a1)"}), + lambda _out_dir, _author_dirs: frozenset({"Lovelace (a1)"}), raising=False, ) monkeypatch.setattr(scheduler, "process_record", capture_record) @@ -231,5 +231,30 @@ def reenrich_when_forced( scheduler.run_all("key", None, None, None, None, records, str(tmp_path), None, False) - assert scheduler.find_incoherent_doi_author_dirs(str(tmp_path)) == frozenset() + assert scheduler.find_incoherent_doi_author_dirs(str(tmp_path), {p.name for p in tmp_path.iterdir()}) == frozenset() assert sorted(forced_calls) == ["Evangelos Milios", "Sageev Oore", "Thomas Trappenberg"] + + +def test_run_all_ignores_a_conflict_with_an_author_outside_the_run( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A co-author the run does not cover cannot be re-enriched by it. + + Forcing the in-scope author over such a conflict would repeat a pass that + can never settle it, and every later run would fail the same way. + """ + _write_doi_copy(tmp_path / "Lovelace (a1)", "10.1/x", "A Lowe and S Oore", 2021) + _write_doi_copy(tmp_path / "Hopper (h1)", "10.1/x", "A Lowe and S Oore", 2022) + forced: dict[str, bool] = {} + + def capture_record(_serpapi_key: str, _serply_key: str | None, rec: Record, _out_dir: str, **kwargs: Any) -> int: + forced[rec.name] = bool(kwargs["force_enrich"]) + return 0 + + monkeypatch.setattr(scheduler, "process_record", capture_record) + + scheduler.run_all( + "key", None, None, None, None, [Record("Ada Lovelace", scholar_id="a1")], str(tmp_path), None, False + ) + + assert forced == {"Ada Lovelace": False} diff --git a/tests/test_workflow_contracts.py b/tests/test_workflow_contracts.py index 74fc38eb..eb41a0df 100644 --- a/tests/test_workflow_contracts.py +++ b/tests/test_workflow_contracts.py @@ -13,9 +13,11 @@ import pytest from ruamel.yaml import YAML +from citeforge.io_utils import read_records from citeforge.refresh.census import AuthorCensus, AuthorCensusRow from citeforge.refresh.ledger import Ledger from citeforge.refresh.types import GenerationSpec, TaskDisposition +from citeforge.text_utils import format_author_dirname # Resolved rather than spelled, so the path is absolute on any host. _BASH = shutil.which("bash") or "/bin/bash" @@ -32,6 +34,12 @@ # dropping them. _SHADOW_WORKFLOWS = [path for path in _WORKFLOW_PATHS if "shadow" in path.stem] +# The only directory MAPS-Lab/mapslab-website import-publications.yml sparse-checks out +# of this repository and aggregates. The monthly census exists to refresh exactly it. +_WEBSITE_AUTHOR_DIRECTORY = "Spadon (bfdGsGUAAAAJ)" +_WEBSITE_CENSUS = _REPOSITORY_ROOT / "data/website-authors.csv" +_MONTHLY_CENSUS_VARIABLE = "MONTHLY_CENSUS" + # The one workflow allowed to hold a write token and mint the App credential. # An allowlist by name fails loudly on a rename, which is the correct outcome # for a privilege grant. @@ -192,6 +200,42 @@ def test_every_shell_block_in_every_workflow_is_balanced(self) -> None: f"{path.name}:{job_id}[{index}] {step.get('name', '')}: {result.stderr.strip()}" ) + def test_every_pipeline_run_and_gate_reads_the_website_census(self) -> None: + """The refresh covers the one author the website reads, not data/input.csv. + + Every pipeline invocation must name the census, and no gate may count + against another one, or the manifest would compare 1 author's run to 64 + authors' directories and no audit could trust the ratio. + """ + workflow = _load_workflow(_MONTHLY_REFRESH_WORKFLOW) + statements = _shell_statements(_workflow_run_commands(workflow)) + + assert workflow["env"] == {_MONTHLY_CENSUS_VARIABLE: "data/website-authors.csv"} + runs = [line for line in statements if "main.py" in line] + assert runs, "the pipeline invocation is gone" + for line in runs: + assert f'--input "${_MONTHLY_CENSUS_VARIABLE}"' in line, line + assert not any("data/input.csv" in line for line in statements) + manifest = next( + step["run"] for step in _workflow_steps(workflow) if step.get("name") == "Build the run manifest" + ) + assert f'"${_MONTHLY_CENSUS_VARIABLE}"' in manifest, "the manifest counts against a different census" + + def test_the_website_census_is_the_one_author_the_website_reads(self) -> None: + records = read_records(str(_WEBSITE_CENSUS)) + + directories = [format_author_dirname(r.name, r.scholar_id or r.dblp or "") for r in records] + assert directories == [_WEBSITE_AUTHOR_DIRECTORY] + + def test_the_website_census_row_is_copied_verbatim_from_the_input_census(self) -> None: + """Both files name the same person, so a corrected profile link in one cannot rot in the other.""" + website = _WEBSITE_CENSUS.read_text(encoding="utf-8").splitlines() + full = (_REPOSITORY_ROOT / "data/input.csv").read_text(encoding="utf-8").splitlines() + + assert len(website) == 2 + assert website[0] == full[0] + assert website[1] in full[1:] + def test_convergence_requires_a_confirming_pass(self) -> None: """An empty digest must never satisfy the convergence test.