Revert "feat: scope the monthly refresh to the website author (#67)"

.github/workflows/monthly-refresh.yml

@@ -35,16 +35,6 @@ on:
 permissions:
   contents: read
 
-# The refresh covers the authors of one census file, not data/input.csv. Its only
-# consumer, MAPS-Lab/mapslab-website import-publications.yml, sparse-checks out
-# `output/Spadon (bfdGsGUAAAAJ)` and aggregates nothing else, so enriching the
-# other authors cost 40 to 100 minutes of provider quota per run for output no
-# one reads. The post-run tail is confined to the census authors' directories
-# (see finalize_run), which leaves every other committed author untouched. Every
-# gate below reads this same file, so the counts compare like with like.
-env:
-  MONTHLY_CENSUS: data/website-authors.csv
-
 # One refresh at a time. Two overlapping runs race on the same output tree and
 # the same publication branch, and the loser force-pushes over the winner.
 # Never cancel in flight: a cancelled run loses its corpus progress.
@@ -134,7 +124,7 @@ jobs:
           for i in $(seq 1 $max_runs); do
             echo "::group::Pipeline run $i/$max_runs"
             rc=0
-            python3 main.py --input "$MONTHLY_CENSUS" || rc=$?
+            python3 main.py || rc=$?
             echo "::endgroup::"
 
             if [ "$rc" -ne 0 ]; then
@@ -204,17 +194,9 @@ jobs:
         run: |
           set -euo pipefail
           mkdir -p /tmp/refresh-manifest
-          # Census authors only: the output tree holds every committed author, so
-          # a whole-tree directory count could never be compared with the census.
-          read -r enabled_authors authors_with_bib < <(python -c '
-          import sys
-          from pathlib import Path
-          from citeforge.io_utils import read_records
-          from citeforge.text_utils import format_author_dirname
-          records = read_records(sys.argv[1])
-          dirs = [Path("output", format_author_dirname(r.name, r.scholar_id or r.dblp or "")) for r in records]
-          print(len(records), sum(any(d.glob("*.bib")) for d in dirs))
-          ' "$MONTHLY_CENSUS")
+          enabled_authors="$(python -c 'import csv; f=open("data/input.csv", encoding="utf-8", newline=""); print(sum(row["Enabled"].strip().lower() == "true" for row in csv.DictReader(f)))')"
+          authors_with_bib="$(find output -mindepth 2 -maxdepth 2 -type f -name '*.bib' ! -path 'output/a2i2/*' -print0 \
+            | xargs -0 -r -n1 dirname | LC_ALL=C sort -u | wc -l)"
           {
             echo "run_id=${{ github.run_id }}"
             echo "converged=${{ steps.loop.outputs.converged }}"

README.md

@@ -85,12 +85,6 @@ A scheduled workflow re-runs the pipeline until the corpus digest stops changing
 a pull request. Publication always goes through review and Required CI; nothing is pushed to
 `main` directly, and the website sync only fires once the corpus has actually merged.
 
-It covers the authors in `data/website-authors.csv`, passed with `--input`, rather than all of
-`data/input.csv`: the website imports a single author's directory, and enriching the rest cost
-40 to 100 minutes of provider quota per run. A run over part of the census touches no other author's
-directory, and leaves `output/a2i2/` alone unless every a2i2 member is in its input. Run `citeforge` with the full census to refresh
-the other authors and the joint folder.
-
 ### Refresh engine
 
 `citeforge/refresh/` holds a second, resumable implementation with a durable ledger, reachable

citeforge/fsscan.py

@@ -16,7 +16,7 @@
 import shutil
 import subprocess
 import unicodedata
-from collections.abc import Callable, Collection, Iterator
+from collections.abc import Callable, Iterator
 from dataclasses import dataclass
 from pathlib import Path, PurePosixPath
 from typing import Any
@@ -273,14 +273,10 @@ def iter_parsed_author_bibs(
         yield filename, path, entry
 
 
-def iter_output_dirs(out_dir: str, *, only: Collection[str] | None = None) -> list[str]:
+def iter_output_dirs(out_dir: str) -> list[str]:
     """Return the immediate subdirectory names of ``out_dir``, sorted.
 
     Directory entry names only (not full paths); plain files are excluded.
-    ``only`` restricts the scan to those names, which is how a run limited to a
-    subset of authors keeps every pass off the directories it does not own.
     ``OSError`` from :func:`os.listdir` is not swallowed here.
     """
-    return sorted(
-        e for e in os.listdir(out_dir) if (only is None or e in only) and os.path.isdir(os.path.join(out_dir, e))
-    )
+    return sorted(e for e in os.listdir(out_dir) if os.path.isdir(os.path.join(out_dir, e)))

citeforge/io_utils.py

@@ -15,7 +15,7 @@
 import re
 import tempfile
 import threading
-from collections.abc import Callable, Collection
+from collections.abc import Callable
 from pathlib import Path
 from typing import IO, Any
 
@@ -84,15 +84,10 @@ def _citation_coherence_signature(entry: dict[str, Any]) -> _CitationCoherenceSi
     )
 
 
-def find_incoherent_doi_author_dirs(out_dir: str, author_dirs: Collection[str]) -> frozenset[str]:
-    """Return those of *author_dirs* holding materially conflicting copies of one DOI.
-
-    Only the named directories are read, so a copy in a directory outside the run
-    can neither be reported nor drag its owner into a forced re-enrichment the
-    run cannot perform.
-    """
+def find_incoherent_doi_author_dirs(out_dir: str) -> frozenset[str]:
+    """Return author directories containing materially conflicting copies of one DOI."""
     grouped: dict[str, list[tuple[str, _CitationCoherenceSignature]]] = {}
-    for dirname in iter_output_dirs(out_dir, only=author_dirs):
+    for dirname in iter_output_dirs(out_dir):
         if dirname == A2I2_OUTPUT_DIR:
             continue
         author_dir = os.path.join(out_dir, dirname)
@@ -381,8 +376,8 @@ def flush_summary_csv(csv_path: str) -> None:
                 writer.writerow(row)
 
 
-def collect_orphan_files(csv_path: str, output_dir: str, author_dirs: Collection[str]) -> list[str]:
-    """Return absolute paths of .bib files in *author_dirs* that have no entry in the summary CSV.
+def collect_orphan_files(csv_path: str, output_dir: str) -> list[str]:
+    """Return absolute paths of .bib files on disk that have no entry in the summary CSV.
 
     These are stale leftovers from previous runs where the same article
     received a different citation key (e.g. Gemini returned a different
@@ -407,9 +402,7 @@ def collect_orphan_files(csv_path: str, output_dir: str, author_dirs: Collection
         # a2i2 is rebuilt from the author directories every run and is never
         # tracked in the CSV, so its files would all read as orphans and become
         # deletion candidates. The other whole-tree passes skip it too.
-        subdirs = [
-            os.path.join(output_dir, d) for d in iter_output_dirs(output_dir, only=author_dirs) if d != A2I2_OUTPUT_DIR
-        ]
+        subdirs = [os.path.join(output_dir, d) for d in iter_output_dirs(output_dir) if d != A2I2_OUTPUT_DIR]
     except OSError:
         return []
 
@@ -653,14 +646,13 @@ def _pick_richer(
     a2i2_dir = os.path.join(out_dir, A2I2_OUTPUT_DIR)
     os.makedirs(a2i2_dir, exist_ok=True)
     existing = [f for f in sorted(os.listdir(a2i2_dir)) if os.path.isfile(os.path.join(a2i2_dir, f))]
-    # The folder is a joint view over every member, so a populated one is never
-    # rebuilt from a subset. `--input` is operator-settable against the standing
-    # output tree, and a subset CSV would otherwise replace the aggregate with
-    # the unresolved members' papers missing. A majority is no safer than one
-    # author: every unresolved member's contribution is dropped either way.
-    if existing and resolved_members < len(a2i2_names):
-        log.warning(
-            "a2i2 rebuild skipped: only %d of %d members resolved against the input records, "
+    # Never empty a populated folder because the member list mostly failed to
+    # resolve. `--input` is operator-settable against the standing output tree,
+    # so a subset CSV would otherwise wipe the aggregate and report only the
+    # small number it rebuilt.
+    if existing and resolved_members < len(a2i2_names) / 2:
+        log.error(
+            "a2i2 rebuild aborted: only %d of %d members resolved against the input records, "
             "which would delete %d existing file(s). Pass the full author list to rebuild.",
             resolved_members,
             len(a2i2_names),

citeforge/pipeline/postrun.py

@@ -15,7 +15,6 @@
 import csv
 import os
 import re
-from collections.abc import Collection
 from dataclasses import dataclass
 from typing import Any
 
@@ -51,7 +50,6 @@
     _is_preprint_fields,
     author_list_is_surname_initials,
     extract_year_from_any,
-    format_author_dirname,
 )
 
 # The author prefix of a generated name: "Jin2026-AlteredNeuro.bib" and the
@@ -144,15 +142,9 @@ def finalize_run(
     cleanup, post-run fixup, superseded-preprint removal, the a2i2 build and the
     baseline.json rewrite. Order is load-bearing.
 
-    Every step that deletes, rewrites or renames a ``.bib`` is confined to the
-    directories of *records*, so a run over a subset of the census leaves every
-    other author's committed output untouched. Only the baseline count, which
-    mirrors the whole tree, still reads outside that set.
-
     Raises :class:`FinalizationError` when a step cannot read or write a file it
     is responsible for.
     """
-    in_scope = frozenset(format_author_dirname(rec.name, rec.scholar_id or rec.dblp or "") for rec in records)
     counts = get_api_call_counts()
     logger.step("Run complete", category=LogCategory.PLAN)
     logger.info(f"Records processed: {processed}", category=LogCategory.PLAN)
@@ -180,7 +172,7 @@ def finalize_run(
             logger.info(f"Reconciled summary CSV: removed {phantoms} phantom entries", category=LogCategory.CLEANUP)
 
         # Safe orphan removal (duplicates only)
-        orphans = collect_orphan_files(csv_path, out_dir, in_scope)
+        orphans = collect_orphan_files(csv_path, out_dir)
         if orphans:
             csv_entries = _load_csv_entries(csv_path)
             for orphan in orphans:
@@ -224,7 +216,7 @@ def finalize_run(
     # Remove .bib files outside the contribution window
     window_min = get_min_year()
     window_removed = 0
-    for entry in iter_output_dirs(out_dir, only=in_scope):
+    for entry in iter_output_dirs(out_dir):
         if entry == A2I2_OUTPUT_DIR:
             continue
         d = os.path.join(out_dir, entry)
@@ -274,7 +266,7 @@ def finalize_run(
     renamed_count = 0
     renames_blocked = 0
     applied_renames: dict[str, str] = {}
-    for pr_entry_name in iter_output_dirs(out_dir, only=in_scope):
+    for pr_entry_name in iter_output_dirs(out_dir):
         if pr_entry_name == A2I2_OUTPUT_DIR:
             continue
         pr_dir = os.path.join(out_dir, pr_entry_name)
@@ -346,7 +338,7 @@ def finalize_run(
             category=LogCategory.CLEANUP,
         )
 
-    superseded = _remove_superseded_preprints(out_dir, in_scope)
+    superseded = _remove_superseded_preprints(out_dir)
     if superseded:
         logger.info(
             f"Removed {superseded} superseded preprint .bib files (published twin exists)",
@@ -368,7 +360,7 @@ def finalize_run(
                 category=LogCategory.CLEANUP,
             )
 
-    unresolved_doi_conflicts = find_incoherent_doi_author_dirs(out_dir, in_scope)
+    unresolved_doi_conflicts = find_incoherent_doi_author_dirs(out_dir)
     if unresolved_doi_conflicts:
         affected = ", ".join(sorted(unresolved_doi_conflicts))
         raise FinalizationError(f"conflicting copies of the same DOI remain in author directories: {affected}")
@@ -450,10 +442,10 @@ def _looks_published(entry: dict[str, Any]) -> bool:
     return has_container or (bool(doi) and bool(str(fields.get("publisher") or "").strip()))
 
 
-def _remove_superseded_preprints(out_dir: str, author_dirs: Collection[str]) -> int:
+def _remove_superseded_preprints(out_dir: str) -> int:
     """Delete a preprint ``.bib`` when a published record of the same work exists.
 
-    Within each directory of *author_dirs*, a preprint file is removed only when another
+    Within each author directory, a preprint file is removed only when another
     file in that directory is a genuinely published record (see
     :func:`_looks_published`) whose title matches at or above
     ``SIM_MERGE_DUPLICATE_THRESHOLD``. This generalizes the "published outranks a
@@ -465,7 +457,7 @@ def _remove_superseded_preprints(out_dir: str, author_dirs: Collection[str]) ->
     Returns the number of preprint files removed.
     """
     removed = 0
-    for entry_name in iter_output_dirs(out_dir, only=author_dirs):
+    for entry_name in iter_output_dirs(out_dir):
         if entry_name == A2I2_OUTPUT_DIR:
             continue
         author_dir = os.path.join(out_dir, entry_name)

citeforge/pipeline/scheduler.py

@@ -348,13 +348,8 @@ def run_all(
     can open a conflict the run then has to close. Authors drawn into a conflict
     by an earlier pass are forced in a further pass, each at most once, which
     bounds the loop by the number of authors. Returns (total_saved, processed).
-
-    Conflicts are looked for only among the directories of *records*. A co-author
-    outside the run cannot be re-enriched by it, so counting that author's copy
-    would force a pass that can never settle the conflict.
     """
-    in_scope = frozenset(_author_dirname(rec) for rec in records)
-    forced = find_incoherent_doi_author_dirs(out_dir, in_scope)
+    forced = find_incoherent_doi_author_dirs(out_dir)
     if forced:
         logger.warn(
             f"Forcing enrichment for {len(forced)} author output directories with conflicting copies of the same DOI",
@@ -363,7 +358,7 @@ def run_all(
     args = (serpapi_key, serply_key, s2_api_key, or_creds, gemini_api_key)
     total_saved, processed = _run_pass(*args, records, out_dir, summary_csv_path, force_enrich, forced)
     while not force_enrich:
-        reopened = find_incoherent_doi_author_dirs(out_dir, in_scope) - forced
+        reopened = find_incoherent_doi_author_dirs(out_dir) - forced
         batch = [rec for rec in records if _author_dirname(rec) in reopened]
         if not batch:
             break