From d5969c323fcbba95a34f490dcafafe6859fdb60a Mon Sep 17 00:00:00 2001 From: John Pearson Date: Sat, 1 Aug 2026 18:29:16 -0400 Subject: [PATCH] Guard author lists and publisher URLs against Scholar regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first unattended run under the new merge logic (9a7b0bf) walked two fields backwards, both in places the existing guards didn't reach. URL: the container-title guard stops a published paper being relabelled as a preprint, but the link is a separate field, and Scholar replaced schreiner2026synaptic's Nature URL with a bioRxiv abstract while the venue correctly stayed "Nature". Reject a preprint-host URL when it would replace a publisher URL — but only for entries we believe are published. An entry whose venue is still bioRxiv genuinely is a preprint, and its bioRxiv link is the canonical one; guarding those would freeze the link at whichever mirror was seen first. wei2026dynamic is exactly that case, so its bioRxiv URL is left alone. Matching is on the URL host, not the whole string, so a publisher URL with "arxiv" somewhere in the path isn't caught. author: Scholar returned draelos2020online with Eva Naumann dropped, and an unconditional overwrite took it. Losing a co-author is worse than being slow to add one, so only accept a list at least as long as the stored one. Also repair the two entries the run damaged: restore the Nature URL (verified 200) and re-add Naumann in her original position. Left the escholarship URL change alone — publisher to publisher, not a regression — and wei2026dynamic's bioRxiv URL, which is correct for a preprint. Verified by replaying the exact payload from 9a7b0bf against the repaired data: neither field is damaged now, and both guards log why. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013CB1BSCz8YKpK4YyZiN81C --- _data/publications.yaml | 4 +++- scholar_scraper.py | 42 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/_data/publications.yaml b/_data/publications.yaml index ab0e261..7188739 100644 --- a/_data/publications.yaml +++ b/_data/publications.yaml @@ -17,7 +17,7 @@ container-title: Nature publisher: Nature Publishing Group UK page: 1-9 - URL: https://www.biorxiv.org/content/10.64898/2026.01.20.699515.abstract + URL: https://www.nature.com/articles/s41586-026-10510-x - id: subramanian2026selective type: article-journal author: @@ -783,6 +783,8 @@ author: - family: Draelos given: Anne + - family: Naumann + given: Eva A - family: Pearson given: John M issued: diff --git a/scholar_scraper.py b/scholar_scraper.py index 552e230..f74624c 100644 --- a/scholar_scraper.py +++ b/scholar_scraper.py @@ -8,6 +8,7 @@ import sys import re import time +import urllib.parse from datetime import date from scholarly import scholarly, ProxyGenerator @@ -358,6 +359,19 @@ def is_preprint_venue(venue): """True if a container-title names a preprint server, not a journal.""" return bool(PREPRINT_VENUE_RE.search(venue or '')) +# Hosts that serve preprints. Matched against the URL's host only, so a +# published paper whose title happens to contain "arxiv" isn't caught. +PREPRINT_HOST_RE = re.compile( + r'(^|\.)(biorxiv|medrxiv|arxiv|chemrxiv|psyarxiv|ssrn|osf)\.(org|io|com|net)$', re.I) + +def is_preprint_url(url): + """True if a URL points at a preprint server rather than a publisher.""" + try: + host = urllib.parse.urlsplit(url or '').hostname or '' + except ValueError: + return False + return bool(PREPRINT_HOST_RE.search(host)) + def looks_truncated(new, old): """ True if `new` is a shortened form of `old` rather than a real retitling. @@ -398,6 +412,34 @@ def merge_entry_fields(old, new): print(f" ! keeping '{old_venue}' over '{value}'", flush=True) continue + # The venue guard above stops a published paper being relabelled as a + # preprint, but the link is a separate field: Scholar has flipped a + # Nature URL back to a bioRxiv abstract while the venue stayed right. + # + # Only guard this for papers we believe are published. An entry whose + # venue is still bioRxiv genuinely is a preprint, and a bioRxiv link + # is the canonical one for it — blocking that would freeze the link at + # whatever mirror happened to be seen first. + if key == 'URL': + old_url = old.get('URL') or '' + venue_now = merged.get('container-title') or '' + published = bool(venue_now) and not is_preprint_venue(venue_now) + if published and old_url and not is_preprint_url(old_url) and is_preprint_url(value): + print(f" ! keeping publisher URL over preprint URL '{value}' " + f"(venue is '{venue_now}')", flush=True) + continue + + # Scholar sometimes returns a shortened author list for a paper it + # already had in full (it dropped a middle author from the NeurIPS + # entry). Losing a co-author is worse than missing a late addition, + # so only accept a list that is at least as long as the stored one. + if key == 'author' and isinstance(value, list): + old_authors = old.get('author') or [] + if len(value) < len(old_authors): + print(f" ! keeping {len(old_authors)} stored authors over " + f"{len(value)} fetched", flush=True) + continue + merged[key] = value return merged