diff --git a/.github/workflows/monthly-refresh.yml b/.github/workflows/monthly-refresh.yml index eb8f710c..e2382580 100644 --- a/.github/workflows/monthly-refresh.yml +++ b/.github/workflows/monthly-refresh.yml @@ -35,16 +35,6 @@ on: permissions: contents: read -# The refresh covers the authors of one census file, not data/input.csv. Its only -# consumer, MAPS-Lab/mapslab-website import-publications.yml, sparse-checks out -# `output/Spadon (bfdGsGUAAAAJ)` and aggregates nothing else, so enriching the -# other authors cost 40 to 100 minutes of provider quota per run for output no -# one reads. The post-run tail is confined to the census authors' directories -# (see finalize_run), which leaves every other committed author untouched. Every -# gate below reads this same file, so the counts compare like with like. -env: - MONTHLY_CENSUS: data/website-authors.csv - # One refresh at a time. Two overlapping runs race on the same output tree and # the same publication branch, and the loser force-pushes over the winner. # Never cancel in flight: a cancelled run loses its corpus progress. @@ -134,7 +124,7 @@ jobs: for i in $(seq 1 $max_runs); do echo "::group::Pipeline run $i/$max_runs" rc=0 - python3 main.py --input "$MONTHLY_CENSUS" || rc=$? + python3 main.py || rc=$? echo "::endgroup::" if [ "$rc" -ne 0 ]; then @@ -204,17 +194,9 @@ jobs: run: | set -euo pipefail mkdir -p /tmp/refresh-manifest - # Census authors only: the output tree holds every committed author, so - # a whole-tree directory count could never be compared with the census. - read -r enabled_authors authors_with_bib < <(python -c ' - import sys - from pathlib import Path - from citeforge.io_utils import read_records - from citeforge.text_utils import format_author_dirname - records = read_records(sys.argv[1]) - dirs = [Path("output", format_author_dirname(r.name, r.scholar_id or r.dblp or "")) for r in records] - print(len(records), sum(any(d.glob("*.bib")) for d in dirs)) - ' "$MONTHLY_CENSUS") + enabled_authors="$(python -c 'import csv; f=open("data/input.csv", encoding="utf-8", newline=""); print(sum(row["Enabled"].strip().lower() == "true" for row in csv.DictReader(f)))')" + authors_with_bib="$(find output -mindepth 2 -maxdepth 2 -type f -name '*.bib' ! -path 'output/a2i2/*' -print0 \ + | xargs -0 -r -n1 dirname | LC_ALL=C sort -u | wc -l)" { echo "run_id=${{ github.run_id }}" echo "converged=${{ steps.loop.outputs.converged }}" diff --git a/README.md b/README.md index 3dcab611..2ed9ca63 100644 --- a/README.md +++ b/README.md @@ -85,12 +85,6 @@ A scheduled workflow re-runs the pipeline until the corpus digest stops changing a pull request. Publication always goes through review and Required CI; nothing is pushed to `main` directly, and the website sync only fires once the corpus has actually merged. -It covers the authors in `data/website-authors.csv`, passed with `--input`, rather than all of -`data/input.csv`: the website imports a single author's directory, and enriching the rest cost -40 to 100 minutes of provider quota per run. A run over part of the census touches no other author's -directory, and leaves `output/a2i2/` alone unless every a2i2 member is in its input. Run `citeforge` with the full census to refresh -the other authors and the joint folder. - ### Refresh engine `citeforge/refresh/` holds a second, resumable implementation with a durable ledger, reachable diff --git a/citeforge/fsscan.py b/citeforge/fsscan.py index 5355d3d2..eb45628d 100644 --- a/citeforge/fsscan.py +++ b/citeforge/fsscan.py @@ -16,7 +16,7 @@ import shutil import subprocess import unicodedata -from collections.abc import Callable, Collection, Iterator +from collections.abc import Callable, Iterator from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any @@ -273,14 +273,10 @@ def iter_parsed_author_bibs( yield filename, path, entry -def iter_output_dirs(out_dir: str, *, only: Collection[str] | None = None) -> list[str]: +def iter_output_dirs(out_dir: str) -> list[str]: """Return the immediate subdirectory names of ``out_dir``, sorted. Directory entry names only (not full paths); plain files are excluded. - ``only`` restricts the scan to those names, which is how a run limited to a - subset of authors keeps every pass off the directories it does not own. ``OSError`` from :func:`os.listdir` is not swallowed here. """ - return sorted( - e for e in os.listdir(out_dir) if (only is None or e in only) and os.path.isdir(os.path.join(out_dir, e)) - ) + return sorted(e for e in os.listdir(out_dir) if os.path.isdir(os.path.join(out_dir, e))) diff --git a/citeforge/io_utils.py b/citeforge/io_utils.py index 3e6ad7c3..51bd47cc 100644 --- a/citeforge/io_utils.py +++ b/citeforge/io_utils.py @@ -15,7 +15,7 @@ import re import tempfile import threading -from collections.abc import Callable, Collection +from collections.abc import Callable from pathlib import Path from typing import IO, Any @@ -84,15 +84,10 @@ def _citation_coherence_signature(entry: dict[str, Any]) -> _CitationCoherenceSi ) -def find_incoherent_doi_author_dirs(out_dir: str, author_dirs: Collection[str]) -> frozenset[str]: - """Return those of *author_dirs* holding materially conflicting copies of one DOI. - - Only the named directories are read, so a copy in a directory outside the run - can neither be reported nor drag its owner into a forced re-enrichment the - run cannot perform. - """ +def find_incoherent_doi_author_dirs(out_dir: str) -> frozenset[str]: + """Return author directories containing materially conflicting copies of one DOI.""" grouped: dict[str, list[tuple[str, _CitationCoherenceSignature]]] = {} - for dirname in iter_output_dirs(out_dir, only=author_dirs): + for dirname in iter_output_dirs(out_dir): if dirname == A2I2_OUTPUT_DIR: continue author_dir = os.path.join(out_dir, dirname) @@ -381,8 +376,8 @@ def flush_summary_csv(csv_path: str) -> None: writer.writerow(row) -def collect_orphan_files(csv_path: str, output_dir: str, author_dirs: Collection[str]) -> list[str]: - """Return absolute paths of .bib files in *author_dirs* that have no entry in the summary CSV. +def collect_orphan_files(csv_path: str, output_dir: str) -> list[str]: + """Return absolute paths of .bib files on disk that have no entry in the summary CSV. These are stale leftovers from previous runs where the same article received a different citation key (e.g. Gemini returned a different @@ -407,9 +402,7 @@ def collect_orphan_files(csv_path: str, output_dir: str, author_dirs: Collection # a2i2 is rebuilt from the author directories every run and is never # tracked in the CSV, so its files would all read as orphans and become # deletion candidates. The other whole-tree passes skip it too. - subdirs = [ - os.path.join(output_dir, d) for d in iter_output_dirs(output_dir, only=author_dirs) if d != A2I2_OUTPUT_DIR - ] + subdirs = [os.path.join(output_dir, d) for d in iter_output_dirs(output_dir) if d != A2I2_OUTPUT_DIR] except OSError: return [] @@ -653,14 +646,13 @@ def _pick_richer( a2i2_dir = os.path.join(out_dir, A2I2_OUTPUT_DIR) os.makedirs(a2i2_dir, exist_ok=True) existing = [f for f in sorted(os.listdir(a2i2_dir)) if os.path.isfile(os.path.join(a2i2_dir, f))] - # The folder is a joint view over every member, so a populated one is never - # rebuilt from a subset. `--input` is operator-settable against the standing - # output tree, and a subset CSV would otherwise replace the aggregate with - # the unresolved members' papers missing. A majority is no safer than one - # author: every unresolved member's contribution is dropped either way. - if existing and resolved_members < len(a2i2_names): - log.warning( - "a2i2 rebuild skipped: only %d of %d members resolved against the input records, " + # Never empty a populated folder because the member list mostly failed to + # resolve. `--input` is operator-settable against the standing output tree, + # so a subset CSV would otherwise wipe the aggregate and report only the + # small number it rebuilt. + if existing and resolved_members < len(a2i2_names) / 2: + log.error( + "a2i2 rebuild aborted: only %d of %d members resolved against the input records, " "which would delete %d existing file(s). Pass the full author list to rebuild.", resolved_members, len(a2i2_names), diff --git a/citeforge/pipeline/postrun.py b/citeforge/pipeline/postrun.py index 58ad9aa8..baffaae9 100644 --- a/citeforge/pipeline/postrun.py +++ b/citeforge/pipeline/postrun.py @@ -15,7 +15,6 @@ import csv import os import re -from collections.abc import Collection from dataclasses import dataclass from typing import Any @@ -51,7 +50,6 @@ _is_preprint_fields, author_list_is_surname_initials, extract_year_from_any, - format_author_dirname, ) # The author prefix of a generated name: "Jin2026-AlteredNeuro.bib" and the @@ -144,15 +142,9 @@ def finalize_run( cleanup, post-run fixup, superseded-preprint removal, the a2i2 build and the baseline.json rewrite. Order is load-bearing. - Every step that deletes, rewrites or renames a ``.bib`` is confined to the - directories of *records*, so a run over a subset of the census leaves every - other author's committed output untouched. Only the baseline count, which - mirrors the whole tree, still reads outside that set. - Raises :class:`FinalizationError` when a step cannot read or write a file it is responsible for. """ - in_scope = frozenset(format_author_dirname(rec.name, rec.scholar_id or rec.dblp or "") for rec in records) counts = get_api_call_counts() logger.step("Run complete", category=LogCategory.PLAN) logger.info(f"Records processed: {processed}", category=LogCategory.PLAN) @@ -180,7 +172,7 @@ def finalize_run( logger.info(f"Reconciled summary CSV: removed {phantoms} phantom entries", category=LogCategory.CLEANUP) # Safe orphan removal (duplicates only) - orphans = collect_orphan_files(csv_path, out_dir, in_scope) + orphans = collect_orphan_files(csv_path, out_dir) if orphans: csv_entries = _load_csv_entries(csv_path) for orphan in orphans: @@ -224,7 +216,7 @@ def finalize_run( # Remove .bib files outside the contribution window window_min = get_min_year() window_removed = 0 - for entry in iter_output_dirs(out_dir, only=in_scope): + for entry in iter_output_dirs(out_dir): if entry == A2I2_OUTPUT_DIR: continue d = os.path.join(out_dir, entry) @@ -274,7 +266,7 @@ def finalize_run( renamed_count = 0 renames_blocked = 0 applied_renames: dict[str, str] = {} - for pr_entry_name in iter_output_dirs(out_dir, only=in_scope): + for pr_entry_name in iter_output_dirs(out_dir): if pr_entry_name == A2I2_OUTPUT_DIR: continue pr_dir = os.path.join(out_dir, pr_entry_name) @@ -346,7 +338,7 @@ def finalize_run( category=LogCategory.CLEANUP, ) - superseded = _remove_superseded_preprints(out_dir, in_scope) + superseded = _remove_superseded_preprints(out_dir) if superseded: logger.info( f"Removed {superseded} superseded preprint .bib files (published twin exists)", @@ -368,7 +360,7 @@ def finalize_run( category=LogCategory.CLEANUP, ) - unresolved_doi_conflicts = find_incoherent_doi_author_dirs(out_dir, in_scope) + unresolved_doi_conflicts = find_incoherent_doi_author_dirs(out_dir) if unresolved_doi_conflicts: affected = ", ".join(sorted(unresolved_doi_conflicts)) raise FinalizationError(f"conflicting copies of the same DOI remain in author directories: {affected}") @@ -450,10 +442,10 @@ def _looks_published(entry: dict[str, Any]) -> bool: return has_container or (bool(doi) and bool(str(fields.get("publisher") or "").strip())) -def _remove_superseded_preprints(out_dir: str, author_dirs: Collection[str]) -> int: +def _remove_superseded_preprints(out_dir: str) -> int: """Delete a preprint ``.bib`` when a published record of the same work exists. - Within each directory of *author_dirs*, a preprint file is removed only when another + Within each author directory, a preprint file is removed only when another file in that directory is a genuinely published record (see :func:`_looks_published`) whose title matches at or above ``SIM_MERGE_DUPLICATE_THRESHOLD``. This generalizes the "published outranks a @@ -465,7 +457,7 @@ def _remove_superseded_preprints(out_dir: str, author_dirs: Collection[str]) -> Returns the number of preprint files removed. """ removed = 0 - for entry_name in iter_output_dirs(out_dir, only=author_dirs): + for entry_name in iter_output_dirs(out_dir): if entry_name == A2I2_OUTPUT_DIR: continue author_dir = os.path.join(out_dir, entry_name) diff --git a/citeforge/pipeline/scheduler.py b/citeforge/pipeline/scheduler.py index d2fc0684..e6f9dfd4 100644 --- a/citeforge/pipeline/scheduler.py +++ b/citeforge/pipeline/scheduler.py @@ -348,13 +348,8 @@ def run_all( can open a conflict the run then has to close. Authors drawn into a conflict by an earlier pass are forced in a further pass, each at most once, which bounds the loop by the number of authors. Returns (total_saved, processed). - - Conflicts are looked for only among the directories of *records*. A co-author - outside the run cannot be re-enriched by it, so counting that author's copy - would force a pass that can never settle the conflict. """ - in_scope = frozenset(_author_dirname(rec) for rec in records) - forced = find_incoherent_doi_author_dirs(out_dir, in_scope) + forced = find_incoherent_doi_author_dirs(out_dir) if forced: logger.warn( f"Forcing enrichment for {len(forced)} author output directories with conflicting copies of the same DOI", @@ -363,7 +358,7 @@ def run_all( args = (serpapi_key, serply_key, s2_api_key, or_creds, gemini_api_key) total_saved, processed = _run_pass(*args, records, out_dir, summary_csv_path, force_enrich, forced) while not force_enrich: - reopened = find_incoherent_doi_author_dirs(out_dir, in_scope) - forced + reopened = find_incoherent_doi_author_dirs(out_dir) - forced batch = [rec for rec in records if _author_dirname(rec) in reopened] if not batch: break diff --git a/data/website-authors.csv b/data/website-authors.csv deleted file mode 100644 index a1403c26..00000000 --- a/data/website-authors.csv +++ /dev/null @@ -1,2 +0,0 @@ -Name,Scholar Link,DBLP Link,Enabled,Exclusion Reason -Gabriel Spadon,https://scholar.google.com/citations?user=bfdGsGUAAAAJ,https://dblp.org/pid/192/1659,true, diff --git a/tests/test_finalize_run.py b/tests/test_finalize_run.py index 66686923..79785813 100644 --- a/tests/test_finalize_run.py +++ b/tests/test_finalize_run.py @@ -32,7 +32,6 @@ import pytest -from citeforge import bibtex_utils as bt from citeforge.config import get_min_year from citeforge.io_utils import append_summary_to_csv, init_summary_csv from citeforge.models import Record @@ -805,9 +804,8 @@ def test_unresolved_same_doi_conflict_aborts_finalization(tmp_path: Path, no_a2i write_bib(first_dir, article(key="DoeStudy", author="Jane Doe and Richard Roe", **common), "Doe-Study.bib") write_bib(second_dir, article(key="DoeStudy", author="Jane Doe and Alice Poe", **common), "Doe-Study.bib") - both = [Record(name="Doe, Jane", scholar_id="abc123"), Record(name="Roe, Richard", scholar_id="def456")] with pytest.raises(FinalizationError, match="conflicting copies of the same DOI"): - finalize_run(str(out_dir), both, total_saved=2, processed=2, summary_csv_path=None) + finalize_run(str(out_dir), _records(), total_saved=2, processed=1, summary_csv_path=None) def test_failed_baseline_write_raises(tmp_path: Path, no_a2i2: None, monkeypatch: pytest.MonkeyPatch) -> None: @@ -825,159 +823,3 @@ def test_failed_baseline_write_raises(tmp_path: Path, no_a2i2: None, monkeypatch with pytest.raises(FinalizationError, match=r"baseline\.json"): finalize_run(str(out_dir), _records(), total_saved=1, processed=1, summary_csv_path=str(csv_path)) - - -# --- SUBSET RUN SCOPE ------------------------------------------------------- -# -# `--input` is operator-settable against the standing output tree, and the -# monthly refresh now names a one-author census. Every pass in the tail that -# deletes, rewrites or renames a file must stay inside the directories of the -# records it was given, or one author's refresh silently edits forty others'. - -_OUT_OF_SCOPE = ("Roe (r1)", "Poe (p1)") -_SHARED_DOI = "10.1000/shared" - - -def _tree_bytes(root: Path, *, skip: frozenset[str]) -> dict[str, bytes]: - """Map every file under *root* to its bytes, leaving out the top-level names in *skip*.""" - return { - str(p.relative_to(root)): p.read_bytes() - for p in sorted(root.rglob("*")) - if p.is_file() and p.relative_to(root).parts[0] not in skip - } - - -def _plant_hazards(out_dir: Path) -> list[Path]: - """Fill the two out-of-scope author directories with one trap per tail pass. - - Returns the tracked paths. Each trap is something finalize_run deletes, - rewrites or refuses to continue past when it can see it, so the byte-equality - assertion is not vacuous. - """ - roe, poe = (out_dir / name for name in _OUT_OF_SCOPE) - alpha = "Alpha Study of Vessel Tracking Systems" - shared_title = "Detecting Ongoing Events Using Contextual Word and Sentence Embeddings" - tracked = [ - write_bib(roe, article(key="RoeAlpha", title=alpha), f"Roe{_IN_WINDOW_YEAR}-Alpha.bib"), - # Out of the contribution window, so the year-window pass deletes it. - write_bib( - roe, - article(key="RoeOld", title="An Old Study of Harbour Currents", year=str(get_min_year() - 1)), - f"Roe{get_min_year() - 1}-Old.bib", - ), - # A preprint with a published twin, so the superseded-preprint pass deletes it. - write_bib( - roe, - misc( - key="RoePre", - title=shared_title, - year=str(_IN_WINDOW_YEAR - 2), - howpublished="arXiv", - doi="10.48550/arxiv.2007.01379", - ), - f"Roe{_IN_WINDOW_YEAR - 2}-Detecting.bib", - ), - write_bib( - roe, - article( - key="RoePub", - title=shared_title, - year=str(_IN_WINDOW_YEAR), - journal="Expert Systems with Applications", - doi="10.1016/j.eswa.2022.118257", - ), - f"Roe{_IN_WINDOW_YEAR}-Detecting.bib", - ), - # Two authors holding the same DOI with different metadata. - write_bib( - roe, - article(key="RoeShared", title="Gamma Study of Reef Acoustics", year=str(_IN_WINDOW_YEAR), doi=_SHARED_DOI), - f"Roe{_IN_WINDOW_YEAR}-Gamma.bib", - ), - write_bib( - poe, - article( - key="PoeShared", - title="Gamma Revisited, a Reef Acoustics Survey", - year=str(_IN_WINDOW_YEAR + 1), - doi=_SHARED_DOI, - ), - f"Poe{_IN_WINDOW_YEAR + 1}-Gamma.bib", - ), - ] - # Valid BibTeX that is not in the serializer's form, so the fixup pass rewrites it. - loose = poe / f"Poe{_IN_WINDOW_YEAR}-Delta.bib" - loose.write_text( - f"@article{{PoeDelta,\n\ttitle={{Delta Study of Harbour Currents}}, author={{Poe, Peter}}," - f"year={{{_IN_WINDOW_YEAR}}},journal={{Ocean Dynamics}}}}\n", - encoding="utf-8", - ) - parsed = bt.parse_bibtex_to_dict(loose.read_text(encoding="utf-8")) - assert parsed is not None and bt.bibtex_from_dict(parsed) != loose.read_text(encoding="utf-8") - tracked.append(loose) - # An untracked duplicate of a tracked file, so the orphan sweep deletes it. - write_bib(roe, article(key="RoeAlphaCopy", title=alpha), f"Roe{_IN_WINDOW_YEAR}-AlphaCopy.bib") - return tracked - - -def test_a_subset_run_leaves_every_other_authors_output_byte_identical( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """One author's finalization deletes, rewrites and renames nothing outside its directory. - - The in-scope author holds a duplicate orphan and a DOI that two out-of-scope - authors also hold with different metadata. The out-of-scope directories hold - one trap per pass. Before the tail was scoped, this run deleted three of their - files, rewrote four more, and then raised on the DOI conflict it could not - resolve, because those two authors are not in the run to be re-enriched. - """ - out_dir = tmp_path / "out" - doe = _author_dir(out_dir) - alpha = "Alpha Study of Vessel Tracking Systems" - doe_tracked = [ - write_bib(doe, article(key="DoeAlpha", title=alpha), f"Doe{_IN_WINDOW_YEAR}-Alpha.bib"), - write_bib( - doe, - article(key="DoeShared", title="Gamma and the Reef", year=str(_IN_WINDOW_YEAR), doi=_SHARED_DOI), - f"Doe{_IN_WINDOW_YEAR}-Gamma.bib", - ), - ] - doe_orphan = write_bib(doe, article(key="DoeAlphaCopy", title=alpha), f"Doe{_IN_WINDOW_YEAR}-AlphaCopy.bib") - tracked = sorted([*doe_tracked, *_plant_hazards(out_dir)], key=str) - a2i2_input = tmp_path / "a2i2.csv" - a2i2_input.write_text("Name,Scholar Link,DBLP Link\nJane Doe,,\nRichard Roe,,\nPeter Poe,,\n", encoding="utf-8") - monkeypatch.setattr(postrun, "DEFAULT_A2I2_INPUT", str(a2i2_input)) - write_bib(out_dir / "a2i2", article(key="Joint", title="A Joint A2I2 Paper"), f"Joint{_IN_WINDOW_YEAR}-Paper.bib") - csv_path = tmp_path / "summary.csv" - _track_in_csv(csv_path, tracked) - untouchable = frozenset({doe.name, "baseline.json"}) - before = _tree_bytes(out_dir, skip=untouchable) - csv_before = csv_path.read_bytes() - records = [Record(name="Jane Doe", scholar_id="abc123")] - - finalize_run(str(out_dir), records, total_saved=1, processed=1, summary_csv_path=str(csv_path)) - - assert _tree_bytes(out_dir, skip=untouchable) == before - assert csv_path.read_bytes() == csv_before - assert not doe_orphan.exists(), "the in-scope author must still be swept" - mirror = json.loads((out_dir / "baseline.json").read_text(encoding="utf-8"))["authors"] - assert {name: mirror[name] for name in _OUT_OF_SCOPE} == {"Roe (r1)": 6, "Poe (p1)": 2} - - -def test_a_run_over_two_of_three_a2i2_members_does_not_rebuild_the_folder( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Dropping a member from the input must not silently drop its papers from a2i2.""" - out_dir = tmp_path / "out" - write_bib(_author_dir(out_dir, "Doe (abc123)"), article(key="D", title="Doe Paper"), f"Doe{_IN_WINDOW_YEAR}-D.bib") - write_bib(_author_dir(out_dir, "Roe (r1)"), article(key="R", title="Roe Paper"), f"Roe{_IN_WINDOW_YEAR}-R.bib") - joint = write_bib(out_dir / "a2i2", article(key="P", title="Poe Paper"), f"Poe{_IN_WINDOW_YEAR}-P.bib") - a2i2_input = tmp_path / "a2i2.csv" - a2i2_input.write_text("Name,Scholar Link,DBLP Link\nJane Doe,,\nRichard Roe,,\nPeter Poe,,\n", encoding="utf-8") - monkeypatch.setattr(postrun, "DEFAULT_A2I2_INPUT", str(a2i2_input)) - records = [Record(name="Jane Doe", scholar_id="abc123"), Record(name="Richard Roe", scholar_id="r1")] - - report = finalize_run(str(out_dir), records, total_saved=2, processed=2, summary_csv_path=None) - - assert report.a2i2_files == 0 - assert [p.name for p in (out_dir / "a2i2").iterdir()] == [joint.name] diff --git a/tests/test_fsscan.py b/tests/test_fsscan.py index 4c184141..91a0045c 100644 --- a/tests/test_fsscan.py +++ b/tests/test_fsscan.py @@ -41,16 +41,6 @@ def test_iter_output_dirs_returns_sorted_subdir_names_excluding_files(tmp_path: assert iter_output_dirs(str(tmp_path)) == ["alpha", "zeta"] -def test_iter_output_dirs_only_keeps_the_named_directories(tmp_path: pathlib.Path) -> None: - for name in ("alpha", "beta", "zeta"): - (tmp_path / name).mkdir() - (tmp_path / "gamma").write_text("", encoding="utf-8") - - # A name that is absent, or is a file, is not invented. - assert iter_output_dirs(str(tmp_path), only={"zeta", "alpha", "gamma", "missing"}) == ["alpha", "zeta"] - assert iter_output_dirs(str(tmp_path), only=set()) == [] - - def _write_scan_fixture(tmp_path: pathlib.Path) -> None: """Author dir with parseable, unparseable, non-bib, and unreadable entries.""" (tmp_path / "c.bib").write_text(_BIB_C, encoding="utf-8") diff --git a/tests/test_io_csv.py b/tests/test_io_csv.py index 00e50c31..89565557 100644 --- a/tests/test_io_csv.py +++ b/tests/test_io_csv.py @@ -244,10 +244,7 @@ def test_incoherent_doi_scan_returns_every_affected_author_directory(tmp_path: P _write_bib(dir_a / "Lowe2021.bib", "inproceedings", "Lowe2021", {**common, "year": "2021"}) _write_bib(dir_b / "Lowe2022.bib", "inproceedings", "Lowe2022", {**common, "year": "2022"}) - both = {"Oore (A1)", "Trappenberg (B1)"} - assert io_utils.find_incoherent_doi_author_dirs(str(out), both) == frozenset(both) - # A copy outside the named directories is not read, so it can conflict with nothing. - assert io_utils.find_incoherent_doi_author_dirs(str(out), {"Oore (A1)"}) == frozenset() + assert io_utils.find_incoherent_doi_author_dirs(str(out)) == frozenset({"Oore (A1)", "Trappenberg (B1)"}) def test_incoherent_doi_scan_compares_author_identity_not_only_count(tmp_path: Path) -> None: @@ -265,8 +262,7 @@ def test_incoherent_doi_scan_compares_author_identity_not_only_count(tmp_path: P _write_bib(dir_a / "Hasan2025.bib", "article", "Hasan2025", {**common, "author": "Tariq Hasan and Evan Frick"}) _write_bib(dir_b / "Hasan2025.bib", "article", "Hasan2025", {**common, "author": "Tariq Hasan and Mahdi Haque"}) - both = {"Hasan (A1)", "Haque (B1)"} - assert io_utils.find_incoherent_doi_author_dirs(str(out), both) == frozenset(both) + assert io_utils.find_incoherent_doi_author_dirs(str(out)) == frozenset({"Hasan (A1)", "Haque (B1)"}) def test_incoherent_doi_scan_accepts_equivalent_author_renderings(tmp_path: Path) -> None: @@ -284,7 +280,7 @@ def test_incoherent_doi_scan_accepts_equivalent_author_renderings(tmp_path: Path _write_bib(dir_a / "Oore2025.bib", "article", "Oore2025", {**common, "author": "Sageev Oore and Jason D'eon"}) _write_bib(dir_b / "Oore2025.bib", "article", "Oore2025", {**common, "author": "Oore, Sageev and D'eon, Jason"}) - assert io_utils.find_incoherent_doi_author_dirs(str(out), {"Oore (A1)", "Rudzicz (B1)"}) == frozenset() + assert io_utils.find_incoherent_doi_author_dirs(str(out)) == frozenset() class TestBuildA2i2Folder: diff --git a/tests/test_regression.py b/tests/test_regression.py index 4ecaf78c..c671f172 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -2402,7 +2402,7 @@ def test_orphan_detected(self, tmp_path: Any) -> None: row.update({k: "0" for k in _SUMMARY_CSV_FIELDNAMES if k not in row}) writer.writerow(row) - orphans = collect_orphan_files(csv_path, str(out_dir), {"Author (ID)"}) + orphans = collect_orphan_files(csv_path, str(out_dir)) assert len(orphans) == 1 assert os.path.basename(orphans[0]) == "Orphan2023-OldKey.bib" @@ -2426,24 +2426,9 @@ def test_no_orphans(self, tmp_path: Any) -> None: row.update({k: "0" for k in _SUMMARY_CSV_FIELDNAMES if k not in row}) writer.writerow(row) - orphans = collect_orphan_files(csv_path, str(out_dir), {"Author (ID)"}) + orphans = collect_orphan_files(csv_path, str(out_dir)) assert len(orphans) == 0 - def test_only_the_named_author_directories_are_scanned(self, tmp_path: Any) -> None: - """An untracked file in a directory outside the run is not a deletion candidate.""" - from citeforge.io_utils import collect_orphan_files, init_summary_csv - - out_dir = tmp_path / "output" - for name in ("Author (ID)", "Other (ID2)"): - (out_dir / name).mkdir(parents=True) - (out_dir / name / "Untracked2024-Paper.bib").write_text("@article{k, title={U}}") - csv_path = str(tmp_path / "summary.csv") - init_summary_csv(csv_path) - - orphans = collect_orphan_files(csv_path, str(out_dir), {"Author (ID)"}) - - assert orphans == [str(out_dir / "Author (ID)" / "Untracked2024-Paper.bib")] - class TestIsKnownSummaryPath: """is_known_summary_path checks the in-memory set from init_summary_csv.""" diff --git a/tests/test_scheduler_parallel.py b/tests/test_scheduler_parallel.py index dd969044..a80fe48d 100644 --- a/tests/test_scheduler_parallel.py +++ b/tests/test_scheduler_parallel.py @@ -161,7 +161,7 @@ def capture_record( monkeypatch.setattr( scheduler, "find_incoherent_doi_author_dirs", - lambda _out_dir, _author_dirs: frozenset({"Lovelace (a1)"}), + lambda _out_dir: frozenset({"Lovelace (a1)"}), raising=False, ) monkeypatch.setattr(scheduler, "process_record", capture_record) @@ -231,30 +231,5 @@ def reenrich_when_forced( scheduler.run_all("key", None, None, None, None, records, str(tmp_path), None, False) - assert scheduler.find_incoherent_doi_author_dirs(str(tmp_path), {p.name for p in tmp_path.iterdir()}) == frozenset() + assert scheduler.find_incoherent_doi_author_dirs(str(tmp_path)) == frozenset() assert sorted(forced_calls) == ["Evangelos Milios", "Sageev Oore", "Thomas Trappenberg"] - - -def test_run_all_ignores_a_conflict_with_an_author_outside_the_run( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """A co-author the run does not cover cannot be re-enriched by it. - - Forcing the in-scope author over such a conflict would repeat a pass that - can never settle it, and every later run would fail the same way. - """ - _write_doi_copy(tmp_path / "Lovelace (a1)", "10.1/x", "A Lowe and S Oore", 2021) - _write_doi_copy(tmp_path / "Hopper (h1)", "10.1/x", "A Lowe and S Oore", 2022) - forced: dict[str, bool] = {} - - def capture_record(_serpapi_key: str, _serply_key: str | None, rec: Record, _out_dir: str, **kwargs: Any) -> int: - forced[rec.name] = bool(kwargs["force_enrich"]) - return 0 - - monkeypatch.setattr(scheduler, "process_record", capture_record) - - scheduler.run_all( - "key", None, None, None, None, [Record("Ada Lovelace", scholar_id="a1")], str(tmp_path), None, False - ) - - assert forced == {"Ada Lovelace": False} diff --git a/tests/test_workflow_contracts.py b/tests/test_workflow_contracts.py index eb41a0df..74fc38eb 100644 --- a/tests/test_workflow_contracts.py +++ b/tests/test_workflow_contracts.py @@ -13,11 +13,9 @@ import pytest from ruamel.yaml import YAML -from citeforge.io_utils import read_records from citeforge.refresh.census import AuthorCensus, AuthorCensusRow from citeforge.refresh.ledger import Ledger from citeforge.refresh.types import GenerationSpec, TaskDisposition -from citeforge.text_utils import format_author_dirname # Resolved rather than spelled, so the path is absolute on any host. _BASH = shutil.which("bash") or "/bin/bash" @@ -34,12 +32,6 @@ # dropping them. _SHADOW_WORKFLOWS = [path for path in _WORKFLOW_PATHS if "shadow" in path.stem] -# The only directory MAPS-Lab/mapslab-website import-publications.yml sparse-checks out -# of this repository and aggregates. The monthly census exists to refresh exactly it. -_WEBSITE_AUTHOR_DIRECTORY = "Spadon (bfdGsGUAAAAJ)" -_WEBSITE_CENSUS = _REPOSITORY_ROOT / "data/website-authors.csv" -_MONTHLY_CENSUS_VARIABLE = "MONTHLY_CENSUS" - # The one workflow allowed to hold a write token and mint the App credential. # An allowlist by name fails loudly on a rename, which is the correct outcome # for a privilege grant. @@ -200,42 +192,6 @@ def test_every_shell_block_in_every_workflow_is_balanced(self) -> None: f"{path.name}:{job_id}[{index}] {step.get('name', '')}: {result.stderr.strip()}" ) - def test_every_pipeline_run_and_gate_reads_the_website_census(self) -> None: - """The refresh covers the one author the website reads, not data/input.csv. - - Every pipeline invocation must name the census, and no gate may count - against another one, or the manifest would compare 1 author's run to 64 - authors' directories and no audit could trust the ratio. - """ - workflow = _load_workflow(_MONTHLY_REFRESH_WORKFLOW) - statements = _shell_statements(_workflow_run_commands(workflow)) - - assert workflow["env"] == {_MONTHLY_CENSUS_VARIABLE: "data/website-authors.csv"} - runs = [line for line in statements if "main.py" in line] - assert runs, "the pipeline invocation is gone" - for line in runs: - assert f'--input "${_MONTHLY_CENSUS_VARIABLE}"' in line, line - assert not any("data/input.csv" in line for line in statements) - manifest = next( - step["run"] for step in _workflow_steps(workflow) if step.get("name") == "Build the run manifest" - ) - assert f'"${_MONTHLY_CENSUS_VARIABLE}"' in manifest, "the manifest counts against a different census" - - def test_the_website_census_is_the_one_author_the_website_reads(self) -> None: - records = read_records(str(_WEBSITE_CENSUS)) - - directories = [format_author_dirname(r.name, r.scholar_id or r.dblp or "") for r in records] - assert directories == [_WEBSITE_AUTHOR_DIRECTORY] - - def test_the_website_census_row_is_copied_verbatim_from_the_input_census(self) -> None: - """Both files name the same person, so a corrected profile link in one cannot rot in the other.""" - website = _WEBSITE_CENSUS.read_text(encoding="utf-8").splitlines() - full = (_REPOSITORY_ROOT / "data/input.csv").read_text(encoding="utf-8").splitlines() - - assert len(website) == 2 - assert website[0] == full[0] - assert website[1] in full[1:] - def test_convergence_requires_a_confirming_pass(self) -> None: """An empty digest must never satisfy the convergence test.