diff --git a/_data/publications.yaml b/_data/publications.yaml index ab0e261..7188739 100644 --- a/_data/publications.yaml +++ b/_data/publications.yaml @@ -17,7 +17,7 @@ container-title: Nature publisher: Nature Publishing Group UK page: 1-9 - URL: https://www.biorxiv.org/content/10.64898/2026.01.20.699515.abstract + URL: https://www.nature.com/articles/s41586-026-10510-x - id: subramanian2026selective type: article-journal author: @@ -783,6 +783,8 @@ author: - family: Draelos given: Anne + - family: Naumann + given: Eva A - family: Pearson given: John M issued: diff --git a/scholar_scraper.py b/scholar_scraper.py index 552e230..f74624c 100644 --- a/scholar_scraper.py +++ b/scholar_scraper.py @@ -8,6 +8,7 @@ import sys import re import time +import urllib.parse from datetime import date from scholarly import scholarly, ProxyGenerator @@ -358,6 +359,19 @@ def is_preprint_venue(venue): """True if a container-title names a preprint server, not a journal.""" return bool(PREPRINT_VENUE_RE.search(venue or '')) +# Hosts that serve preprints. Matched against the URL's host only, so a +# published paper whose title happens to contain "arxiv" isn't caught. +PREPRINT_HOST_RE = re.compile( + r'(^|\.)(biorxiv|medrxiv|arxiv|chemrxiv|psyarxiv|ssrn|osf)\.(org|io|com|net)$', re.I) + +def is_preprint_url(url): + """True if a URL points at a preprint server rather than a publisher.""" + try: + host = urllib.parse.urlsplit(url or '').hostname or '' + except ValueError: + return False + return bool(PREPRINT_HOST_RE.search(host)) + def looks_truncated(new, old): """ True if `new` is a shortened form of `old` rather than a real retitling. @@ -398,6 +412,34 @@ def merge_entry_fields(old, new): print(f" ! keeping '{old_venue}' over '{value}'", flush=True) continue + # The venue guard above stops a published paper being relabelled as a + # preprint, but the link is a separate field: Scholar has flipped a + # Nature URL back to a bioRxiv abstract while the venue stayed right. + # + # Only guard this for papers we believe are published. An entry whose + # venue is still bioRxiv genuinely is a preprint, and a bioRxiv link + # is the canonical one for it — blocking that would freeze the link at + # whatever mirror happened to be seen first. + if key == 'URL': + old_url = old.get('URL') or '' + venue_now = merged.get('container-title') or '' + published = bool(venue_now) and not is_preprint_venue(venue_now) + if published and old_url and not is_preprint_url(old_url) and is_preprint_url(value): + print(f" ! keeping publisher URL over preprint URL '{value}' " + f"(venue is '{venue_now}')", flush=True) + continue + + # Scholar sometimes returns a shortened author list for a paper it + # already had in full (it dropped a middle author from the NeurIPS + # entry). Losing a co-author is worse than missing a late addition, + # so only accept a list that is at least as long as the stored one. + if key == 'author' and isinstance(value, list): + old_authors = old.get('author') or [] + if len(value) < len(old_authors): + print(f" ! keeping {len(old_authors)} stored authors over " + f"{len(value)} fetched", flush=True) + continue + merged[key] = value return merged