From 6f6e6bef6a096c064c1a55532d951b936fd074c0 Mon Sep 17 00:00:00 2001 From: thorntwig Date: Thu, 3 Sep 2026 11:53:13 +0200 Subject: [PATCH 1/2] feat: Add wiki entity linking and QID lookup methods --- asknews_sdk/api/wiki.py | 577 ++++++++++++++++++++++++++++++++++- asknews_sdk/dto/__init__.py | 22 +- asknews_sdk/dto/wiki.py | 212 ++++++++++++- tests/test_entity_linking.py | 415 +++++++++++++++++++++++++ 4 files changed, 1198 insertions(+), 28 deletions(-) create mode 100644 tests/test_entity_linking.py diff --git a/asknews_sdk/api/wiki.py b/asknews_sdk/api/wiki.py index bcc6814..4dfa15b 100644 --- a/asknews_sdk/api/wiki.py +++ b/asknews_sdk/api/wiki.py @@ -1,7 +1,17 @@ from typing import Dict, List, Optional +from urllib.parse import quote from asknews_sdk.api.base import BaseAPI -from asknews_sdk.dto.wiki import WikiSearchResponse +from asknews_sdk.dto.wiki import ( + WikiBatchEntityRequest, + WikiBatchEntityResponse, + WikiBatchLinkEntityRequest, + WikiBatchLinkEntityResponse, + WikiBatchSearchResponse, + WikiEntityResponse, + WikiLinkEntityResponse, + WikiSearchResponse, +) class WikiAPI(BaseAPI): @@ -19,8 +29,9 @@ def search_wiki( full_articles: bool = False, hybrid_search: bool = False, diversify: float = 0.0, - string_guarantee: List[str] = None, + string_guarantee: Optional[List[str]] = None, include_main_section: bool = False, + has_wikidata: Optional[bool] = None, *, http_headers: Optional[Dict] = None, ) -> WikiSearchResponse: @@ -32,9 +43,28 @@ def search_wiki( :param query: Query string that can be any phrase, keyword, question, or paragraph. :type query: str - + :param n_documents: Number of documents to return. + :type n_documents: int + :param neighbor_chunks: Number of neighbor chunks to attach and return. + :type neighbor_chunks: int + :param full_articles: If true, full articles will be returned. + :type full_articles: bool + :param hybrid_search: If true, hybrid search will be used. + :type hybrid_search: bool + :param diversify: Diversity factor for MMR re-ranking (0.0-1.0). + :type diversify: float + :param string_guarantee: List of strings that must be present in the results. + :type string_guarantee: Optional[List[str]] + :param include_main_section: If true, the main section of the article is + prepended to each chunk. + :type include_main_section: bool + :param has_wikidata: Filter results by whether the article has Wikidata. + If None, no filtering is applied. + :type has_wikidata: Optional[bool] + :param http_headers: Additional HTTP headers. + :type http_headers: Optional[Dict] :return: The search response. - :rtype: SearchResponse + :rtype: WikiSearchResponse """ response = self.client.request( method="GET", @@ -48,16 +78,269 @@ def search_wiki( "string_guarantee": string_guarantee, "diversify": diversify, "include_main_section": include_main_section, + "has_wikidata": has_wikidata, }, headers=http_headers, accept=[(WikiSearchResponse.__content_type__, 1.0)], ) return WikiSearchResponse.model_validate(response.content) + def search_wiki_batch( + self, + queries: List[str], + n_documents: int = 5, + neighbor_chunks: int = 1, + full_articles: bool = False, + hybrid_search: bool = False, + diversify: float = 0.0, + string_guarantee: Optional[List[str]] = None, + include_main_section: bool = False, + has_wikidata: Optional[bool] = None, + *, + http_headers: Optional[Dict] = None, + ) -> WikiBatchSearchResponse: + """ + Search for wiki articles for multiple queries in a single batch. + + https://docs.asknews.app/en/reference#post-/v1/wiki/search/batch + + :param queries: List of query strings to search for in parallel. + :type queries: List[str] + :param n_documents: Number of documents to return per query. + :type n_documents: int + :param neighbor_chunks: Number of neighbor chunks to attach and return. + :type neighbor_chunks: int + :param full_articles: If true, full articles will be returned. + :type full_articles: bool + :param hybrid_search: If true, hybrid search will be used. + :type hybrid_search: bool + :param diversify: Diversity factor for MMR re-ranking (0.0-1.0). + :type diversify: float + :param string_guarantee: List of strings that must be present in the results. + :type string_guarantee: Optional[List[str]] + :param include_main_section: If true, the main section of the article is + prepended to each chunk. + :type include_main_section: bool + :param has_wikidata: Filter results by whether the article has Wikidata. + If None, no filtering is applied. + :type has_wikidata: Optional[bool] + :param http_headers: Additional HTTP headers. + :type http_headers: Optional[Dict] + :return: The batch search response. + :rtype: WikiBatchSearchResponse + """ + response = self.client.request( + method="POST", + endpoint="/v1/wiki/search/batch", + query={ + "queries": queries, + "n_documents": n_documents, + "neighbor_chunks": neighbor_chunks, + "full_articles": full_articles, + "hybrid_search": hybrid_search, + "string_guarantee": string_guarantee, + "diversify": diversify, + "include_main_section": include_main_section, + "has_wikidata": has_wikidata, + }, + headers=http_headers, + accept=[(WikiBatchSearchResponse.__content_type__, 1.0)], + ) + return WikiBatchSearchResponse.model_validate(response.content) + + def link_entity( + self, + entity: str, + entity_type: Optional[str] = None, + entity_description: Optional[str] = None, + n_candidates: int = 5, + relevance_threshold: float = 0.40, + ambiguity_margin: float = 0.05, + allow_ambiguous: bool = True, + include_candidates: bool = True, + *, + http_headers: Optional[Dict] = None, + ) -> WikiLinkEntityResponse: + """ + Link an entity name to its Wikidata entity. + + https://docs.asknews.app/en/reference#get-/v1/wiki/link-entity + + :param entity: Name of the entity to link. + :type entity: str + :param entity_type: Optional type of the entity (e.g. 'person', 'location', + 'organization'). + :type entity_type: Optional[str] + :param entity_description: Optional description providing additional context + to improve entity matching. + :type entity_description: Optional[str] + :param n_candidates: Number of candidate entities to return alongside the + linked entity. Does not affect which entity is linked. + :type n_candidates: int + :param relevance_threshold: Accept/abstain gate: minimum relevance (0-1) the + linked entity must have. + :type relevance_threshold: float + :param ambiguity_margin: A different entity with the same name scoring within + this relative margin (0-1) of the winner marks the result 'ambiguous'. + :type ambiguity_margin: float + :param allow_ambiguous: If true, 'ambiguous' links still return their + entity; if false, only confident 'linked' matches are accepted. + :type allow_ambiguous: bool + :param include_candidates: If true, include the candidate list alongside the + linked entity. + :type include_candidates: bool + :param http_headers: Additional HTTP headers. + :type http_headers: Optional[Dict] + :return: The entity link response. + :rtype: WikiLinkEntityResponse + """ + response = self.client.request( + method="GET", + endpoint="/v1/wiki/link-entity", + query={ + "entity": entity, + "entity_type": entity_type, + "entity_description": entity_description, + "n_candidates": n_candidates, + "relevance_threshold": relevance_threshold, + "ambiguity_margin": ambiguity_margin, + "allow_ambiguous": allow_ambiguous, + "include_candidates": include_candidates, + }, + headers=http_headers, + accept=[(WikiLinkEntityResponse.__content_type__, 1.0)], + ) + return WikiLinkEntityResponse.model_validate(response.content) + + def link_entity_batch( + self, + entities: List[str], + entity_types: Optional[List[Optional[str]]] = None, + entity_descriptions: Optional[List[Optional[str]]] = None, + relevance_threshold: Optional[float] = None, + ambiguity_margin: Optional[float] = None, + allow_ambiguous: bool = True, + include_candidates: bool = True, + n_candidates: int = 5, + *, + http_headers: Optional[Dict] = None, + ) -> WikiBatchLinkEntityResponse: + """ + Link multiple entity names to their Wikidata entities in parallel. + + https://docs.asknews.app/en/reference#post-/v1/wiki/link-entity/batch + + :param entities: List of entity names to link. + :type entities: List[str] + :param entity_types: Optional types of the entities. Must be the same length + as entities if provided; individual elements may be None. + :type entity_types: Optional[List[Optional[str]]] + :param entity_descriptions: Optional descriptions for the entities. Must be the + same length as entities if provided; individual elements may be None. + :type entity_descriptions: Optional[List[Optional[str]]] + :param relevance_threshold: Accept/abstain gate: minimum relevance (0-1) the + linked entity must have. + :type relevance_threshold: Optional[float] + :param ambiguity_margin: A different entity with the same name scoring within + this relative margin (0-1) of the winner marks the result 'ambiguous'. + :type ambiguity_margin: Optional[float] + :param allow_ambiguous: If true, 'ambiguous' links still return their + entity; if false, only confident 'linked' matches are accepted. + :type allow_ambiguous: bool + :param include_candidates: If true, include the candidate list alongside the + linked entity. + :type include_candidates: bool + :param n_candidates: Number of candidate entities to return per entity alongside + the linked entity. Does not affect which entity is linked. + :type n_candidates: int + :param http_headers: Additional HTTP headers. + :type http_headers: Optional[Dict] + :return: The batch entity link response. + :rtype: WikiBatchLinkEntityResponse + """ + body = WikiBatchLinkEntityRequest( + entities=entities, + entity_types=entity_types, + entity_descriptions=entity_descriptions, + relevance_threshold=relevance_threshold, + ambiguity_margin=ambiguity_margin, + allow_ambiguous=allow_ambiguous, + include_candidates=include_candidates, + ) + response = self.client.request( + method="POST", + endpoint="/v1/wiki/link-entity/batch", + body=body.model_dump(mode="json"), + query={"n_candidates": n_candidates}, + headers=http_headers, + accept=[(WikiBatchLinkEntityResponse.__content_type__, 1.0)], + ) + return WikiBatchLinkEntityResponse.model_validate(response.content) + + def get_entity( + self, + qid: str, + *, + http_headers: Optional[Dict] = None, + ) -> WikiEntityResponse: + """ + Retrieve a Wikidata entity by its QID. + + https://docs.asknews.app/en/reference#get-/v1/wiki/entity/-qid- + + :param qid: Wikidata QID of the entity to retrieve, e.g. 'Q312'. + :type qid: str + :param http_headers: Additional HTTP headers. + :type http_headers: Optional[Dict] + :return: The entity response. A QID that is not in the collection comes back + with found=False rather than raising. + :rtype: WikiEntityResponse + """ + response = self.client.request( + method="GET", + endpoint="/v1/wiki/entity/{qid}", + # Escaped: QIDs are free-form input and a raw "/" would otherwise be + # normalized away into a different endpoint. + params={"qid": quote(qid, safe="")}, + headers=http_headers, + accept=[(WikiEntityResponse.__content_type__, 1.0)], + ) + return WikiEntityResponse.model_validate(response.content) + + def get_entity_batch( + self, + qids: List[str], + *, + http_headers: Optional[Dict] = None, + ) -> WikiBatchEntityResponse: + """ + Retrieve multiple Wikidata entities by QID in a single request. + + https://docs.asknews.app/en/reference#post-/v1/wiki/entity/batch + + :param qids: Wikidata QIDs to retrieve. + :type qids: List[str] + :param http_headers: Additional HTTP headers. + :type http_headers: Optional[Dict] + :return: The batch entity response. Results are in the order the QIDs were + supplied; a QID that is not in the collection comes back with found=False + rather than being omitted. + :rtype: WikiBatchEntityResponse + """ + body = WikiBatchEntityRequest(qids=qids) + response = self.client.request( + method="POST", + endpoint="/v1/wiki/entity/batch", + body=body.model_dump(mode="json"), + headers=http_headers, + accept=[(WikiBatchEntityResponse.__content_type__, 1.0)], + ) + return WikiBatchEntityResponse.model_validate(response.content) + class AsyncWikiAPI(BaseAPI): """ - News API + Wiki API https://docs.asknews.app/en/reference#tag--wiki """ @@ -70,27 +353,42 @@ async def search_wiki( full_articles: bool = False, hybrid_search: bool = False, diversify: float = 0.0, - string_guarantee: List[str] = None, + string_guarantee: Optional[List[str]] = None, include_main_section: bool = False, + has_wikidata: Optional[bool] = None, *, http_headers: Optional[Dict] = None, ) -> WikiSearchResponse: """ - Search for news articles given a query. + Search for wiki articles given a query. - https://docs.asknews.app/en/reference#get-/v1/news/search + https://docs.asknews.app/en/reference#get-/v1/wiki/search :param query: Query string that can be any phrase, keyword, question, or paragraph. - If method='nl', then this will be used as a natural language query. - If method='kw', then this will be used as a direct keyword query. :type query: str - :param n_articles: Number of articles to return, defaults to 10 - :type n_articles: Optional[int] + :param n_documents: Number of documents to return. + :type n_documents: int + :param neighbor_chunks: Number of neighbor chunks to attach and return. + :type neighbor_chunks: int + :param full_articles: If true, full articles will be returned. + :type full_articles: bool + :param hybrid_search: If true, hybrid search will be used. + :type hybrid_search: bool + :param diversify: Diversity factor for MMR re-ranking (0.0-1.0). + :type diversify: float + :param string_guarantee: List of strings that must be present in the results. + :type string_guarantee: Optional[List[str]] + :param include_main_section: If true, the main section of the article is + prepended to each chunk. + :type include_main_section: bool + :param has_wikidata: Filter results by whether the article has Wikidata. + If None, no filtering is applied. + :type has_wikidata: Optional[bool] :param http_headers: Additional HTTP headers. :type http_headers: Optional[Dict] :return: The search response. - :rtype: SearchResponse + :rtype: WikiSearchResponse """ response = await self.client.request( method="GET", @@ -104,8 +402,261 @@ async def search_wiki( "string_guarantee": string_guarantee, "diversify": diversify, "include_main_section": include_main_section, + "has_wikidata": has_wikidata, }, headers=http_headers, accept=[(WikiSearchResponse.__content_type__, 1.0)], ) return WikiSearchResponse.model_validate(response.content) + + async def search_wiki_batch( + self, + queries: List[str], + n_documents: int = 5, + neighbor_chunks: int = 1, + full_articles: bool = False, + hybrid_search: bool = False, + diversify: float = 0.0, + string_guarantee: Optional[List[str]] = None, + include_main_section: bool = False, + has_wikidata: Optional[bool] = None, + *, + http_headers: Optional[Dict] = None, + ) -> WikiBatchSearchResponse: + """ + Search for wiki articles for multiple queries in a single batch. + + https://docs.asknews.app/en/reference#post-/v1/wiki/search/batch + + :param queries: List of query strings to search for in parallel. + :type queries: List[str] + :param n_documents: Number of documents to return per query. + :type n_documents: int + :param neighbor_chunks: Number of neighbor chunks to attach and return. + :type neighbor_chunks: int + :param full_articles: If true, full articles will be returned. + :type full_articles: bool + :param hybrid_search: If true, hybrid search will be used. + :type hybrid_search: bool + :param diversify: Diversity factor for MMR re-ranking (0.0-1.0). + :type diversify: float + :param string_guarantee: List of strings that must be present in the results. + :type string_guarantee: Optional[List[str]] + :param include_main_section: If true, the main section of the article is + prepended to each chunk. + :type include_main_section: bool + :param has_wikidata: Filter results by whether the article has Wikidata. + If None, no filtering is applied. + :type has_wikidata: Optional[bool] + :param http_headers: Additional HTTP headers. + :type http_headers: Optional[Dict] + :return: The batch search response. + :rtype: WikiBatchSearchResponse + """ + response = await self.client.request( + method="POST", + endpoint="/v1/wiki/search/batch", + query={ + "queries": queries, + "n_documents": n_documents, + "neighbor_chunks": neighbor_chunks, + "full_articles": full_articles, + "hybrid_search": hybrid_search, + "string_guarantee": string_guarantee, + "diversify": diversify, + "include_main_section": include_main_section, + "has_wikidata": has_wikidata, + }, + headers=http_headers, + accept=[(WikiBatchSearchResponse.__content_type__, 1.0)], + ) + return WikiBatchSearchResponse.model_validate(response.content) + + async def link_entity( + self, + entity: str, + entity_type: Optional[str] = None, + entity_description: Optional[str] = None, + n_candidates: int = 5, + relevance_threshold: float = 0.40, + ambiguity_margin: float = 0.05, + allow_ambiguous: bool = True, + include_candidates: bool = True, + *, + http_headers: Optional[Dict] = None, + ) -> WikiLinkEntityResponse: + """ + Link an entity name to its Wikidata entity. + + https://docs.asknews.app/en/reference#get-/v1/wiki/link-entity + + :param entity: Name of the entity to link. + :type entity: str + :param entity_type: Optional type of the entity (e.g. 'person', 'location', + 'organization'). + :type entity_type: Optional[str] + :param entity_description: Optional description providing additional context + to improve entity matching. + :type entity_description: Optional[str] + :param n_candidates: Number of candidate entities to return alongside the + linked entity. Does not affect which entity is linked. + :type n_candidates: int + :param relevance_threshold: Accept/abstain gate: minimum relevance (0-1) the + linked entity must have. + :type relevance_threshold: float + :param ambiguity_margin: A different entity with the same name scoring within + this relative margin (0-1) of the winner marks the result 'ambiguous'. + :type ambiguity_margin: float + :param allow_ambiguous: If true, 'ambiguous' links still return their + entity; if false, only confident 'linked' matches are accepted. + :type allow_ambiguous: bool + :param include_candidates: If true, include the candidate list alongside the + linked entity. + :type include_candidates: bool + :param http_headers: Additional HTTP headers. + :type http_headers: Optional[Dict] + :return: The entity link response. + :rtype: WikiLinkEntityResponse + """ + response = await self.client.request( + method="GET", + endpoint="/v1/wiki/link-entity", + query={ + "entity": entity, + "entity_type": entity_type, + "entity_description": entity_description, + "n_candidates": n_candidates, + "relevance_threshold": relevance_threshold, + "ambiguity_margin": ambiguity_margin, + "allow_ambiguous": allow_ambiguous, + "include_candidates": include_candidates, + }, + headers=http_headers, + accept=[(WikiLinkEntityResponse.__content_type__, 1.0)], + ) + return WikiLinkEntityResponse.model_validate(response.content) + + async def link_entity_batch( + self, + entities: List[str], + entity_types: Optional[List[Optional[str]]] = None, + entity_descriptions: Optional[List[Optional[str]]] = None, + relevance_threshold: Optional[float] = None, + ambiguity_margin: Optional[float] = None, + allow_ambiguous: bool = True, + include_candidates: bool = True, + n_candidates: int = 5, + *, + http_headers: Optional[Dict] = None, + ) -> WikiBatchLinkEntityResponse: + """ + Link multiple entity names to their Wikidata entities in parallel. + + https://docs.asknews.app/en/reference#post-/v1/wiki/link-entity/batch + + :param entities: List of entity names to link. + :type entities: List[str] + :param entity_types: Optional types of the entities. Must be the same length + as entities if provided; individual elements may be None. + :type entity_types: Optional[List[Optional[str]]] + :param entity_descriptions: Optional descriptions for the entities. Must be the + same length as entities if provided; individual elements may be None. + :type entity_descriptions: Optional[List[Optional[str]]] + :param relevance_threshold: Accept/abstain gate: minimum relevance (0-1) the + linked entity must have. + :type relevance_threshold: Optional[float] + :param ambiguity_margin: A different entity with the same name scoring within + this relative margin (0-1) of the winner marks the result 'ambiguous'. + :type ambiguity_margin: Optional[float] + :param allow_ambiguous: If true, 'ambiguous' links still return their + entity; if false, only confident 'linked' matches are accepted. + :type allow_ambiguous: bool + :param include_candidates: If true, include the candidate list alongside the + linked entity. + :type include_candidates: bool + :param n_candidates: Number of candidate entities to return per entity alongside + the linked entity. Does not affect which entity is linked. + :type n_candidates: int + :param http_headers: Additional HTTP headers. + :type http_headers: Optional[Dict] + :return: The batch entity link response. + :rtype: WikiBatchLinkEntityResponse + """ + body = WikiBatchLinkEntityRequest( + entities=entities, + entity_types=entity_types, + entity_descriptions=entity_descriptions, + relevance_threshold=relevance_threshold, + ambiguity_margin=ambiguity_margin, + allow_ambiguous=allow_ambiguous, + include_candidates=include_candidates, + ) + response = await self.client.request( + method="POST", + endpoint="/v1/wiki/link-entity/batch", + body=body.model_dump(mode="json"), + query={"n_candidates": n_candidates}, + headers=http_headers, + accept=[(WikiBatchLinkEntityResponse.__content_type__, 1.0)], + ) + return WikiBatchLinkEntityResponse.model_validate(response.content) + + async def get_entity( + self, + qid: str, + *, + http_headers: Optional[Dict] = None, + ) -> WikiEntityResponse: + """ + Retrieve a Wikidata entity by its QID. + + https://docs.asknews.app/en/reference#get-/v1/wiki/entity/-qid- + + :param qid: Wikidata QID of the entity to retrieve, e.g. 'Q312'. + :type qid: str + :param http_headers: Additional HTTP headers. + :type http_headers: Optional[Dict] + :return: The entity response. A QID that is not in the collection comes back + with found=False rather than raising. + :rtype: WikiEntityResponse + """ + response = await self.client.request( + method="GET", + endpoint="/v1/wiki/entity/{qid}", + # Escaped: QIDs are free-form input and a raw "/" would otherwise be + # normalized away into a different endpoint. + params={"qid": quote(qid, safe="")}, + headers=http_headers, + accept=[(WikiEntityResponse.__content_type__, 1.0)], + ) + return WikiEntityResponse.model_validate(response.content) + + async def get_entity_batch( + self, + qids: List[str], + *, + http_headers: Optional[Dict] = None, + ) -> WikiBatchEntityResponse: + """ + Retrieve multiple Wikidata entities by QID in a single request. + + https://docs.asknews.app/en/reference#post-/v1/wiki/entity/batch + + :param qids: Wikidata QIDs to retrieve. + :type qids: List[str] + :param http_headers: Additional HTTP headers. + :type http_headers: Optional[Dict] + :return: The batch entity response. Results are in the order the QIDs were + supplied; a QID that is not in the collection comes back with found=False + rather than being omitted. + :rtype: WikiBatchEntityResponse + """ + body = WikiBatchEntityRequest(qids=qids) + response = await self.client.request( + method="POST", + endpoint="/v1/wiki/entity/batch", + body=body.model_dump(mode="json"), + headers=http_headers, + accept=[(WikiBatchEntityResponse.__content_type__, 1.0)], + ) + return WikiBatchEntityResponse.model_validate(response.content) diff --git a/asknews_sdk/dto/__init__.py b/asknews_sdk/dto/__init__.py index 4187618..339a6b0 100644 --- a/asknews_sdk/dto/__init__.py +++ b/asknews_sdk/dto/__init__.py @@ -34,7 +34,18 @@ FinanceResponseTimeSeriesData, ) from asknews_sdk.dto.stories import StoriesResponse, StoryResponse, StoryResponseUpdate -from asknews_sdk.dto.wiki import WikiSearchResponse +from asknews_sdk.dto.wiki import ( + WikiBatchEntityRequest, + WikiBatchEntityResponse, + WikiBatchLinkEntityRequest, + WikiBatchLinkEntityResponse, + WikiBatchSearchResponse, + WikidataResponseDictItem, + WikiEntityResponse, + WikiLinkEntityResponse, + WikiResponseDictItem, + WikiSearchResponse, +) __all__ = ( @@ -73,4 +84,13 @@ "URLIndexingRequest", "URLIndexingResponse", "WikiSearchResponse", + "WikiResponseDictItem", + "WikidataResponseDictItem", + "WikiBatchSearchResponse", + "WikiLinkEntityResponse", + "WikiBatchLinkEntityRequest", + "WikiBatchLinkEntityResponse", + "WikiEntityResponse", + "WikiBatchEntityRequest", + "WikiBatchEntityResponse", ) diff --git a/asknews_sdk/dto/wiki.py b/asknews_sdk/dto/wiki.py index 9e6ca18..a08e651 100644 --- a/asknews_sdk/dto/wiki.py +++ b/asknews_sdk/dto/wiki.py @@ -1,20 +1,135 @@ from datetime import datetime -from typing import List, Optional +from typing import Any, Dict, List, Literal, Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, field_validator from asknews_sdk.dto.base import BaseSchema class CirrusMetadata(BaseModel): - create_timestamp: datetime - wikibase_item: str - version: int - popularity_score: float - text_bytes: int + create_timestamp: Optional[datetime] = None + wikibase_item: Optional[str] = None + version: Optional[int] = None + popularity_score: Optional[float] = None + text_bytes: Optional[int] = None + heading: Optional[List[str]] = None + incoming_links: Optional[int] = None + outgoing_links: Optional[int] = None # other fields can be added as needed +class WikidataMetadata(BaseModel): + """Wikidata entity metadata. + """ + + model_config = ConfigDict(extra="allow") + + # {lang: article title} for every Wikipedia the entity links to. Build a URL + # with f"https://{lang}.wikipedia.org/wiki/{title}". Replaces the former + # enwiki-only `wikipedia_url`, which was exactly the "en" entry. + wikipedia_titles: Optional[Dict[str, str]] = None + # Count of ALL sitelinks, including sister projects and languages absent + # from wikipedia_titles. + sitelink_count: Optional[int] = None + # True when the entity is a class/type (it has P279 subclass_of). + is_class: Optional[bool] = None + # {lang: label} across the entity-resolution languages. + alt_labels: Optional[Dict[str, str]] = None + aliases: Optional[List[str]] = None + # Normalized registered domain of official_website (P856). + official_website_domain: Optional[str] = None + + # QID-valued fields list EVERY claim, not just the current one: `ceo` on a + # company carries former officeholders alongside the incumbent. Each entry + # is {qid, label, ...} plus whichever of start_time/end_time/point_in_time + # and rank that claim carries — an absent qualifier is an absent key, so + # read with .get(). Which entry is "current" is the caller's judgement from + # those fields; Wikidata does not always mark it. Times are ISO-ish + # strings; quantities are {amount, unit_qid, unit_label}. + # + # Keep as Any: a stricter annotation would fail the whole response. + + # Cross-cutting: present across most entity types. + instance_of: Optional[Any] = None + subclass_of: Optional[Any] = None + part_of: Optional[Any] = None + country: Optional[Any] = None + official_website: Optional[Any] = None + image: Optional[Any] = None + inception: Optional[Any] = None + dissolved: Optional[Any] = None + # Entities Wikidata explicitly marks as confusable with this one (P1889). + different_from: Optional[Any] = None + + # Person. + date_of_birth: Optional[Any] = None + date_of_death: Optional[Any] = None + place_of_birth: Optional[Any] = None + country_of_citizenship: Optional[Any] = None + occupation: Optional[Any] = None + employer: Optional[Any] = None + position_held: Optional[Any] = None + political_party: Optional[Any] = None + educated_at: Optional[Any] = None + notable_work: Optional[Any] = None + awards: Optional[Any] = None + member_of: Optional[Any] = None + + # Organization. + headquarters: Optional[Any] = None + ceo: Optional[Any] = None + chairperson: Optional[Any] = None + founded_by: Optional[Any] = None + parent_organization: Optional[Any] = None + owned_by: Optional[Any] = None + subsidiary: Optional[Any] = None + industry: Optional[Any] = None + number_of_employees: Optional[Any] = None + legal_form: Optional[Any] = None + + # Location. + located_in: Optional[Any] = None + coordinates: Optional[Any] = None + population: Optional[Any] = None + capital: Optional[Any] = None + official_language: Optional[Any] = None + head_of_government: Optional[Any] = None + head_of_state: Optional[Any] = None + currency: Optional[Any] = None + basic_form_of_government: Optional[Any] = None + + # Event / conflict. + participant: Optional[Any] = None + winner: Optional[Any] = None + conflict: Optional[Any] = None + significant_event: Optional[Any] = None + number_of_deaths: Optional[Any] = None + victim: Optional[Any] = None + perpetrator: Optional[Any] = None + + # Authority-control identifiers — the natural follow-on to entity linking: + # cross-walking a resolved QID into library and archive catalogues. + gnd_id: Optional[Any] = None + library_of_congress_authorities_id: Optional[Any] = None + viaf_cluster_id: Optional[Any] = None + isni: Optional[Any] = None + idref_id: Optional[Any] = None + + # Alternate names. Distinct from the core `aliases`/`alt_labels`: these are + # property-table entries carrying language-tagged values. + official_name: Optional[Any] = None + native_label: Optional[Any] = None + short_name: Optional[Any] = None + + # Common relations and social handles. + has_parts: Optional[Any] = None + named_after: Optional[Any] = None + replaces: Optional[Any] = None + x_twitter_username: Optional[Any] = None + subreddit: Optional[Any] = None + social_media_followers: Optional[Any] = None + + class WikiResponseDictItem(BaseModel): content: str title: str @@ -23,17 +138,86 @@ class WikiResponseDictItem(BaseModel): timestamp: datetime cirrus_metadata: Optional[CirrusMetadata] = None point_id: Optional[str] = None + has_main_section: Optional[bool] = None + + +class WikidataResponseDictItem(BaseModel): + title: str + description: Optional[str] = None + qid: str + relevance: Optional[float] = None + wikidata_metadata: Optional[WikidataMetadata] = None class WikiSearchResponse(BaseSchema): documents: List[WikiResponseDictItem] - # @classmethod - # def from_qdrant_records( - # cls, - # data: dict, - # ) -> "WikiSearchResponse": - # return cls( +class WikiBatchSearchResponse(BaseSchema): + results: List[WikiSearchResponse] + + +class WikiLinkEntityResponse(BaseSchema): + entity: str + entity_type: Optional[str] = None + linked_entity: Optional[WikidataResponseDictItem] = None + candidates: Optional[List[WikidataResponseDictItem]] = None + relevance_threshold: Optional[float] = None + link_status: Literal["linked", "ambiguous", "no_match"] + link_confidence: float + + +class WikiBatchLinkEntityRequest(BaseModel): + entities: List[str] + entity_types: Optional[List[Optional[str]]] = None + entity_descriptions: Optional[List[Optional[str]]] = None + relevance_threshold: Optional[float] = None + ambiguity_margin: Optional[float] = None + allow_ambiguous: bool = True + include_candidates: bool = True + + @field_validator("entities") + @classmethod + def validate_entities_not_empty(cls, v): + if any(e == "" for e in v): + raise ValueError("entities must not contain empty strings") + return v + + @field_validator("entity_types", "entity_descriptions") + @classmethod + def validate_lengths_match(cls, v, info): + entities = info.data.get("entities") + if v is not None and entities is not None and len(v) != len(entities): + field = info.field_name + raise ValueError( + f"{field} length ({len(v)}) must match entities length ({len(entities)})" + ) + return v + + +class WikiBatchLinkEntityResponse(BaseSchema): + results: List[WikiLinkEntityResponse] + + +class WikiEntityResponse(BaseSchema): + """Direct QID lookup. `entity` is None when the QID is not in the collection; + a miss is reported as found=False rather than as an error.""" + + qid: str + entity: Optional[WikidataResponseDictItem] = None + found: bool + + +class WikiBatchEntityRequest(BaseModel): + qids: List[str] + + @field_validator("qids") + @classmethod + def validate_qids_not_empty(cls, v): + if any(not q.strip() for q in v): + raise ValueError("qids must not contain empty strings") + return v + - # ) +class WikiBatchEntityResponse(BaseSchema): + results: List[WikiEntityResponse] diff --git a/tests/test_entity_linking.py b/tests/test_entity_linking.py new file mode 100644 index 0000000..fdc4d70 --- /dev/null +++ b/tests/test_entity_linking.py @@ -0,0 +1,415 @@ +"""Tests for `link_entity` / `link_entity_batch`: their DTOs and the requests they send.""" +import inspect +from urllib.parse import parse_qs, quote, urlparse + +import pytest +from pydantic import ValidationError + +from asknews_sdk.api.wiki import AsyncWikiAPI, WikiAPI +from asknews_sdk.dto.wiki import ( + WikiBatchEntityRequest, + WikiBatchEntityResponse, + WikiBatchLinkEntityRequest, + WikiBatchLinkEntityResponse, + WikidataMetadata, + WikiEntityResponse, + WikiLinkEntityResponse, + WikiSearchResponse, +) +from asknews_sdk.utils import build_url + + +def build_linked_payload(**overrides): + """A minimal, valid `linked` entity link payload.""" + payload = { + "entity": "Apple", + "entity_type": "organization", + "linked_entity": { + "title": "Apple Inc.", + "description": "American technology company", + "qid": "Q312", + "relevance": 0.91, + }, + "candidates": [ + {"title": "Apple Inc.", "qid": "Q312", "relevance": 0.91}, + {"title": "Apple", "qid": "Q89", "relevance": 0.42}, + ], + "relevance_threshold": 0.40, + "link_status": "linked", + "link_confidence": 0.91, + } + payload.update(overrides) + return payload + + +def test_linked_payload_parses(): + response = WikiLinkEntityResponse.model_validate(build_linked_payload()) + + assert response.link_status == "linked" + assert response.linked_entity.qid == "Q312" + assert response.linked_entity.relevance == 0.91 + assert [c.qid for c in response.candidates] == ["Q312", "Q89"] + + +def test_no_match_has_null_entity(): + """An abstained link carries a status but no entity.""" + response = WikiLinkEntityResponse.model_validate( + { + "entity": "Zzzz Not A Real Entity", + "entity_type": None, + "link_status": "no_match", + "link_confidence": 0.0, + } + ) + + assert response.link_status == "no_match" + assert response.linked_entity is None + assert response.candidates is None + + +def test_ambiguous_status_is_accepted(): + response = WikiLinkEntityResponse.model_validate( + build_linked_payload(link_status="ambiguous", link_confidence=0.51) + ) + + assert response.link_status == "ambiguous" + assert response.linked_entity.qid == "Q312" + + +def test_unknown_link_status_is_rejected(): + with pytest.raises(ValidationError): + WikiLinkEntityResponse.model_validate( + build_linked_payload(link_status="maybe") + ) + + +def test_candidates_omitted_when_not_requested(): + payload = build_linked_payload() + payload.pop("candidates") + + response = WikiLinkEntityResponse.model_validate(payload) + + assert response.candidates is None + assert response.linked_entity.qid == "Q312" + + +def test_unknown_wikidata_metadata_fields_are_preserved(): + """`wikidata_metadata` allows extras so new upstream properties survive the SDK.""" + response = WikiLinkEntityResponse.model_validate( + build_linked_payload( + linked_entity={ + "title": "Apple Inc.", + "qid": "Q312", + "relevance": 0.91, + "wikidata_metadata": { + "ceo": {"label": "Tim Cook"}, + "some_future_property": 42, + }, + } + ) + ) + + metadata = response.linked_entity.wikidata_metadata + assert metadata.ceo == {"label": "Tim Cook"} + assert metadata.model_extra == {"some_future_property": 42} + + +def test_batch_response_preserves_order_and_per_entity_status(): + response = WikiBatchLinkEntityResponse.model_validate( + { + "results": [ + build_linked_payload(), + { + "entity": "Nowhere", + "entity_type": None, + "link_status": "no_match", + "link_confidence": 0.0, + }, + ] + } + ) + + assert [r.entity for r in response.results] == ["Apple", "Nowhere"] + assert [r.link_status for r in response.results] == ["linked", "no_match"] + assert response.results[1].linked_entity is None + + +def test_batch_request_defaults_match_the_api(): + request = WikiBatchLinkEntityRequest(entities=["Apple", "Paris"]) + + assert request.allow_ambiguous is True + assert request.include_candidates is True + assert request.relevance_threshold is None + assert request.ambiguity_margin is None + + +def test_batch_request_allows_per_element_none_types(): + """Types align positionally, and an unknown type for one entity is expressed as None.""" + request = WikiBatchLinkEntityRequest( + entities=["Apple", "Paris"], + entity_types=["organization", None], + entity_descriptions=[None, "the French capital"], + ) + + assert request.entity_types == ["organization", None] + assert request.entity_descriptions == [None, "the French capital"] + + +@pytest.mark.parametrize("field", ["entity_types", "entity_descriptions"]) +def test_batch_request_rejects_length_mismatch(field): + with pytest.raises(ValidationError, match="must match entities length"): + WikiBatchLinkEntityRequest(entities=["Apple", "Paris"], **{field: ["organization"]}) + + +def test_batch_request_rejects_empty_entity_names(): + with pytest.raises(ValidationError, match="must not contain empty strings"): + WikiBatchLinkEntityRequest(entities=["Apple", ""]) + + +def test_batch_request_round_trips_to_json_body(): + """The request is serialized as the POST body, so it must survive a JSON round trip.""" + request = WikiBatchLinkEntityRequest( + entities=["Apple", "Paris"], + entity_types=["organization", None], + relevance_threshold=0.5, + allow_ambiguous=False, + ) + + dumped = request.model_dump(mode="json") + + assert dumped["entities"] == ["Apple", "Paris"] + assert dumped["entity_types"] == ["organization", None] + assert dumped["relevance_threshold"] == 0.5 + assert dumped["allow_ambiguous"] is False + assert WikiBatchLinkEntityRequest.model_validate(dumped) == request + + +# --- request shape sent to the API -------------------------------------------- + + +@pytest.mark.parametrize("api", [WikiAPI, AsyncWikiAPI]) +@pytest.mark.parametrize("method", ["link_entity", "link_entity_batch"]) +def test_candidate_count_is_named_n_candidates(api, method): + """Linking counts candidates, not documents: the API names this `n_candidates`.""" + parameters = inspect.signature(getattr(api, method)).parameters + + assert "n_candidates" in parameters + assert "n_documents" not in parameters + assert parameters["n_candidates"].default == 5 + + +@pytest.mark.parametrize("api", [WikiAPI, AsyncWikiAPI]) +@pytest.mark.parametrize("method", ["search_wiki", "search_wiki_batch"]) +def test_search_keeps_n_documents(api, method): + """The rename was scoped to entity linking; wiki search still counts documents.""" + parameters = inspect.signature(getattr(api, method)).parameters + + assert "n_documents" in parameters + assert "n_candidates" not in parameters + + +@pytest.mark.parametrize("api", [WikiAPI, AsyncWikiAPI]) +@pytest.mark.parametrize("method", ["search_wiki", "search_wiki_batch"]) +def test_search_does_not_return_entity_documents(api, method): + """Wiki search returns article chunks only; entities come from the link and + QID methods, so there is no toggle for including them here.""" + parameters = inspect.signature(getattr(api, method)).parameters + + assert "include_entities" not in parameters + # The surviving Wikidata-adjacent filter is unaffected by the removal. + assert "has_wikidata" in parameters + + +def test_search_response_holds_only_article_documents(): + """Search can no longer return entity documents, so a QID payload must not validate.""" + response = WikiSearchResponse.model_validate( + { + "documents": [ + { + "content": "Apple Inc. is an American technology company.", + "title": "Apple Inc.", + "url": "https://en.wikipedia.org/wiki/Apple_Inc.", + "categories": ["Technology companies"], + "timestamp": "2026-08-10T12:00:00+00:00", + } + ] + } + ) + + assert response.documents[0].title == "Apple Inc." + assert not hasattr(response.documents[0], "qid") + + with pytest.raises(ValidationError): + WikiSearchResponse.model_validate( + {"documents": [{"title": "Apple Inc.", "qid": "Q312", "relevance": 0.91}]} + ) + + +def test_link_entity_sends_n_candidates_on_the_query_string(): + """`build_url` drops None and stringifies, so assert on the URL the API receives.""" + url = build_url( + base_url="https://api.asknews.dev", + endpoint="/v1/wiki/link-entity", + query={ + "entity": "Apple", + "entity_type": None, + "n_candidates": 3, + "relevance_threshold": 0.40, + }, + ) + + parsed = parse_qs(urlparse(url).query) + assert parsed["n_candidates"] == ["3"] + assert "n_documents" not in parsed + # Unset optional filters must not be sent, or they would override server defaults. + assert "entity_type" not in parsed + + +# --- QID lookup --------------------------------------------------------------- + + +def test_entity_response_parses_a_hit(): + response = WikiEntityResponse.model_validate( + { + "qid": "Q312", + "entity": {"title": "Apple Inc.", "qid": "Q312", "description": "tech company"}, + "found": True, + } + ) + + assert response.found is True + assert response.entity.qid == "Q312" + + +def test_entity_response_reports_a_miss_without_erroring(): + """A QID that is not in the collection is a found=False result, not an error.""" + response = WikiEntityResponse.model_validate({"qid": "Q_missing", "found": False}) + + assert response.found is False + assert response.entity is None + + +def test_batch_entity_response_preserves_order_and_misses(): + response = WikiBatchEntityResponse.model_validate( + { + "results": [ + {"qid": "Q312", "entity": {"title": "Apple Inc.", "qid": "Q312"}, "found": True}, + {"qid": "Q_missing", "found": False}, + ] + } + ) + + assert [r.qid for r in response.results] == ["Q312", "Q_missing"] + assert [r.found for r in response.results] == [True, False] + assert response.results[1].entity is None + + +def test_batch_entity_request_rejects_blank_qids(): + with pytest.raises(ValidationError, match="must not contain empty strings"): + WikiBatchEntityRequest(qids=["Q312", " "]) + + +@pytest.mark.parametrize("api", [WikiAPI, AsyncWikiAPI]) +@pytest.mark.parametrize("method", ["get_entity", "get_entity_batch"]) +def test_qid_lookup_methods_exist(api, method): + assert callable(getattr(api, method)) + + +def test_qid_is_escaped_into_the_path(): + """`build_url` interpolates raw and then normalizes, so a '/' in a QID would + otherwise silently retarget the request at a different endpoint.""" + safe = build_url( + base_url="https://api.asknews.dev", + endpoint="/v1/wiki/entity/{qid}", + params={"qid": quote("Q1/../admin", safe="")}, + ) + unsafe = build_url( + base_url="https://api.asknews.dev", + endpoint="/v1/wiki/entity/{qid}", + params={"qid": "Q1/../admin"}, + ) + + assert urlparse(safe).path == "/v1/wiki/entity/Q1%2F..%2Fadmin" + # Demonstrates why the escaping is required. + assert urlparse(unsafe).path == "/v1/wiki/entity/admin" + + +# --- WikidataMetadata shape tolerance ----------------------------------------- + + +@pytest.mark.parametrize( + "field,value", + [ + # The shapes the API sends today: lists of dicts, with the key set + # varying by property. + # + # A QID-valued property lists EVERY claim, not just the current one. + # Both entries here are rank "normal" — Wikidata does not reliably mark + # the current one — and the second carries no end_time qualifier at all, + # so the key is simply absent rather than null. + ( + "ceo", + [ + { + "qid": "Q5820", + "label": "Steve Jobs", + "start_time": "+1997-09-16T00:00:00Z", + "end_time": "+2011-08-24T00:00:00Z", + "rank": "normal", + }, + { + "qid": "Q312556", + "label": "Tim Cook", + "start_time": "+2011-08-24T00:00:00Z", + "rank": "normal", + }, + ], + ), + ("place_of_birth", [{"qid": "Q60", "label": "New York City"}]), + ("date_of_birth", [{"time": "+1955-02-24T00:00:00Z"}]), + ("gnd_id", ["118637347"]), + ("official_name", [{"text": "Apple Inc.", "language": "en"}]), + ("social_media_followers", [{"amount": 5000000, "platform": "x_twitter"}]), + # Absent when the entity has no such claim. + ("ceo", None), + ], +) +def test_metadata_accepts_property_shapes(field, value): + """Property-derived fields must parse whatever arrives, expected or not. + + The metadata is nested inside the response model, so a raise here fails the + WHOLE call — including for callers that never read the offending field. That + blast radius is why these stay `Any` rather than being typed to today's + shapes. + """ + metadata = WikidataMetadata(**{field: value}) + + assert getattr(metadata, field) == value + + +def test_metadata_passes_through_undeclared_fields(): + """Wikidata exposes far more properties than this model declares. + + Undeclared ones must still reach the caller, otherwise a newly served + property would be silently dropped until the SDK caught up. + """ + metadata = WikidataMetadata(some_future_property=[{"qid": "Q1"}]) + + assert metadata.model_dump(exclude_none=True) == { + "some_future_property": [{"qid": "Q1"}] + } + + +def test_metadata_uses_the_served_field_names(): + """A declared field whose name is not the served one never populates. + + It is an invisible failure: it does not raise, it just reads as None + forever. These singular/plural pairs are the easy ones to get wrong. + """ + declared = set(WikidataMetadata.model_fields) + + assert {"occupations", "employers", "positions_held"}.isdisjoint(declared) + assert {"occupation", "employer", "position_held"} <= declared + # `wikipedia_titles` carries every language, not just en. + assert "wikipedia_url" not in declared + assert "wikipedia_titles" in declared From cfa4c8fd57ca0bb5f8e02b2d7b9b3279fb35184c Mon Sep 17 00:00:00 2001 From: thorntwig Date: Fri, 4 Sep 2026 12:12:55 +0200 Subject: [PATCH 2/2] fix: Declare and type every Wikidata property in the wiki DTOs --- asknews_sdk/dto/wiki.py | 468 +++++++++++++++++++++++++++-------- tests/test_entity_linking.py | 36 ++- 2 files changed, 396 insertions(+), 108 deletions(-) diff --git a/asknews_sdk/dto/wiki.py b/asknews_sdk/dto/wiki.py index a08e651..cb2b176 100644 --- a/asknews_sdk/dto/wiki.py +++ b/asknews_sdk/dto/wiki.py @@ -1,7 +1,7 @@ from datetime import datetime -from typing import Any, Dict, List, Literal, Optional +from typing import Dict, List, Literal, Optional -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, field_validator, model_serializer from asknews_sdk.dto.base import BaseSchema @@ -18,116 +18,390 @@ class CirrusMetadata(BaseModel): # other fields can be added as needed +class _WikidataValue(BaseModel): + """Base for the nested property-value shapes. + + Two settings, both load-bearing: + + * ``extra="allow"`` -- the API attaches per-property context keys to these + entries (``for_work`` on awards, ``kinship`` on relatives, + ``character_role``/``character_name`` on cast members), and new ones can + appear without warning. Keeping extras means a newly served qualifier + reaches you as an attribute on ``model_extra`` instead of raising. + * ``_drop_none`` -- the API omits absent qualifiers rather than sending + nulls, and re-serializing one of these models keeps that shape, so a + round trip does not invent keys the API never sent. + """ + + model_config = ConfigDict(extra="allow") + + @model_serializer(mode="wrap") + def _drop_none(self, handler): + return {k: v for k, v in handler(self).items() if v is not None} + + +class WikidataQidRef(_WikidataValue): + """A QID-valued claim -- the element type of most metadata fields. + + Most QID fields list EVERY claim, not just the current one: ``ceo`` carries + former officeholders alongside the incumbent. Which entry is current is + yours to decide -- ``rank`` marks it when an editor has set one, but is + often left "normal" throughout, and a missing ``end_time`` does not imply + currency (an announced successor has none either). + + A few fields are list-shaped but store only ONE entry; they are marked + ``[one entry]`` on the field below. Do not read a list's length as a count + without checking. + """ + + qid: str + label: str + # Temporal qualifiers, present on time-bounded roles such as ceo, + # position_held or member_of. Dating rates vary enormously by property, so + # `end_time is None` means "current OR undated". + start_time: Optional[str] = None + end_time: Optional[str] = None + # The point-in-time counterpart to the start/end interval -- the only date + # a point event carries (an award has a year, not a range). + point_in_time: Optional[str] = None + # Wikidata claim rank: "preferred" | "normal". + rank: Optional[str] = None + + +class WikidataQuantity(_WikidataValue): + """A quantity claim, with its unit and attribution qualifiers. + + The unit is not decoration -- revenue in EUR and USD are otherwise silently + mixed in one field. ``platform``/``account_id``/``point_in_time`` attribute + the value: ``social_media_followers`` carries one claim per account per + snapshot, so a bare number cannot be attributed, ordered or summed without + double-counting the same account at two dates. + """ + + amount: float + unit_qid: Optional[str] = None + unit_label: Optional[str] = None + # A plain platform name ("X (Twitter)", "YouTube"), not a QID. + platform: Optional[str] = None + account_id: Optional[str] = None + point_in_time: Optional[str] = None + + +class WikidataMonolingualText(_WikidataValue): + """A language-tagged text value (e.g. official_name).""" + + text: str + language: Optional[str] = None + + +class WikidataCoordinate(_WikidataValue): + """A globe-coordinate value.""" + + latitude: Optional[float] = None + longitude: Optional[float] = None + + class WikidataMetadata(BaseModel): """Wikidata entity metadata. + + A typed core plus every Wikidata property the API returns. ``extra="allow"`` + is kept so a newly added property still reaches you (via ``model_extra``) + before this SDK is updated to name it. + + Almost every field is absent on any given entity -- a person has no + ``capital``, a city no ``date_of_birth`` -- so read defensively and expect + ``None``. The trailing comment on each line is the Wikidata property id, + for lookup at wikidata.org/wiki/Property:. """ model_config = ConfigDict(extra="allow") + # ---- Core: not property-table entries ---------------------------------- # {lang: article title} for every Wikipedia the entity links to. Build a URL - # with f"https://{lang}.wikipedia.org/wiki/{title}". Replaces the former - # enwiki-only `wikipedia_url`, which was exactly the "en" entry. + # with f"https://{lang}.wikipedia.org/wiki/{title}". wikipedia_titles: Optional[Dict[str, str]] = None # Count of ALL sitelinks, including sister projects and languages absent # from wikipedia_titles. sitelink_count: Optional[int] = None - # True when the entity is a class/type (it has P279 subclass_of). + # True when the entity is a class/type (it has subclass_of). is_class: Optional[bool] = None - # {lang: label} across the entity-resolution languages. alt_labels: Optional[Dict[str, str]] = None aliases: Optional[List[str]] = None - # Normalized registered domain of official_website (P856). + # Normalized registered domain of official_website, for exact-match lookup + # -- scheme/www/trailing-slash variants make the raw URL an unreliable key. official_website_domain: Optional[str] = None - - # QID-valued fields list EVERY claim, not just the current one: `ceo` on a - # company carries former officeholders alongside the incumbent. Each entry - # is {qid, label, ...} plus whichever of start_time/end_time/point_in_time - # and rank that claim carries — an absent qualifier is an absent key, so - # read with .get(). Which entry is "current" is the caller's judgement from - # those fields; Wikidata does not always mark it. Times are ISO-ish - # strings; quantities are {amount, unit_qid, unit_label}. - # - # Keep as Any: a stricter annotation would fail the whole response. - - # Cross-cutting: present across most entity types. - instance_of: Optional[Any] = None - subclass_of: Optional[Any] = None - part_of: Optional[Any] = None - country: Optional[Any] = None - official_website: Optional[Any] = None - image: Optional[Any] = None - inception: Optional[Any] = None - dissolved: Optional[Any] = None - # Entities Wikidata explicitly marks as confusable with this one (P1889). - different_from: Optional[Any] = None - - # Person. - date_of_birth: Optional[Any] = None - date_of_death: Optional[Any] = None - place_of_birth: Optional[Any] = None - country_of_citizenship: Optional[Any] = None - occupation: Optional[Any] = None - employer: Optional[Any] = None - position_held: Optional[Any] = None - political_party: Optional[Any] = None - educated_at: Optional[Any] = None - notable_work: Optional[Any] = None - awards: Optional[Any] = None - member_of: Optional[Any] = None - - # Organization. - headquarters: Optional[Any] = None - ceo: Optional[Any] = None - chairperson: Optional[Any] = None - founded_by: Optional[Any] = None - parent_organization: Optional[Any] = None - owned_by: Optional[Any] = None - subsidiary: Optional[Any] = None - industry: Optional[Any] = None - number_of_employees: Optional[Any] = None - legal_form: Optional[Any] = None - - # Location. - located_in: Optional[Any] = None - coordinates: Optional[Any] = None - population: Optional[Any] = None - capital: Optional[Any] = None - official_language: Optional[Any] = None - head_of_government: Optional[Any] = None - head_of_state: Optional[Any] = None - currency: Optional[Any] = None - basic_form_of_government: Optional[Any] = None - - # Event / conflict. - participant: Optional[Any] = None - winner: Optional[Any] = None - conflict: Optional[Any] = None - significant_event: Optional[Any] = None - number_of_deaths: Optional[Any] = None - victim: Optional[Any] = None - perpetrator: Optional[Any] = None - - # Authority-control identifiers — the natural follow-on to entity linking: - # cross-walking a resolved QID into library and archive catalogues. - gnd_id: Optional[Any] = None - library_of_congress_authorities_id: Optional[Any] = None - viaf_cluster_id: Optional[Any] = None - isni: Optional[Any] = None - idref_id: Optional[Any] = None - - # Alternate names. Distinct from the core `aliases`/`alt_labels`: these are - # property-table entries carrying language-tagged values. - official_name: Optional[Any] = None - native_label: Optional[Any] = None - short_name: Optional[Any] = None - - # Common relations and social handles. - has_parts: Optional[Any] = None - named_after: Optional[Any] = None - replaces: Optional[Any] = None - x_twitter_username: Optional[Any] = None - subreddit: Optional[Any] = None - social_media_followers: Optional[Any] = None + # Revision metadata for the source Wikidata item. + lastrevid: Optional[int] = None + last_modified: Optional[str] = None + + # ---- Every extracted Wikidata property --------------------------------- + + # List[WikidataQidRef] -- normally EVERY claim, best-ranked first + # (`ceo` carries former officeholders alongside the incumbent). + # NOTE: the fields marked [one entry] below are list-shaped but + # currently store a SINGLE entry -- the shape is held open so + # restoring the full history stays non-breaking. Do not read + # their length as a count. + instance_of: Optional[List[WikidataQidRef]] = None # P31 + subclass_of: Optional[List[WikidataQidRef]] = None # P279 + spouse: Optional[List[WikidataQidRef]] = None # P26 + child: Optional[List[WikidataQidRef]] = None # P40 + sibling: Optional[List[WikidataQidRef]] = None # P3373 + relative: Optional[List[WikidataQidRef]] = None # P1038 + student_of: Optional[List[WikidataQidRef]] = None # P1066 + doctoral_advisor: Optional[List[WikidataQidRef]] = None # P184 + country_of_citizenship: Optional[List[WikidataQidRef]] = None # P27 + gender: Optional[List[WikidataQidRef]] = None # P21 + occupation: Optional[List[WikidataQidRef]] = None # P106 + academic_degree: Optional[List[WikidataQidRef]] = None # P512 + noble_title: Optional[List[WikidataQidRef]] = None # P97 + medical_condition: Optional[List[WikidataQidRef]] = None # P1050 + employer: Optional[List[WikidataQidRef]] = None # P108 + educated_at: Optional[List[WikidataQidRef]] = None # P69 + position_held: Optional[List[WikidataQidRef]] = None # P39 + member_of: Optional[List[WikidataQidRef]] = None # P463 + awards: Optional[List[WikidataQidRef]] = None # P166 + notable_work: Optional[List[WikidataQidRef]] = None # P800 + field_of_work: Optional[List[WikidataQidRef]] = None # P101 + political_party: Optional[List[WikidataQidRef]] = None # P102 + religion: Optional[List[WikidataQidRef]] = None # P140 + native_language: Optional[List[WikidataQidRef]] = None # P103 + cause_of_death: Optional[List[WikidataQidRef]] = None # P509 + legislative_body: Optional[List[WikidataQidRef]] = None # P194 + electoral_district: Optional[List[WikidataQidRef]] = None # P768 + candidacy_in_election: Optional[List[WikidataQidRef]] = None # P3602 + appointed_by: Optional[List[WikidataQidRef]] = None # P748 + replaced_by: Optional[List[WikidataQidRef]] = None # P1366 + replaces: Optional[List[WikidataQidRef]] = None # P1365 + diplomatic_relation: Optional[List[WikidataQidRef]] = None # P530 + executive_body: Optional[List[WikidataQidRef]] = None # P208 + judicial_branch: Optional[List[WikidataQidRef]] = None # P209 + flag: Optional[List[WikidataQidRef]] = None # P163 + territory_claimed_by: Optional[List[WikidataQidRef]] = None # P1336 + political_alignment: Optional[List[WikidataQidRef]] = None # P1387 + political_ideology: Optional[List[WikidataQidRef]] = None # P1142 + headquarters: Optional[List[WikidataQidRef]] = None # P159 + ceo: Optional[List[WikidataQidRef]] = None # P169 + chief_operating_officer: Optional[List[WikidataQidRef]] = None # P1789 + founded_by: Optional[List[WikidataQidRef]] = None # P112 + parent_organization: Optional[List[WikidataQidRef]] = None # P749 + industry: Optional[List[WikidataQidRef]] = None # P452 + stock_exchange: Optional[List[WikidataQidRef]] = None # P414 + legal_form: Optional[List[WikidataQidRef]] = None # P1454 + owned_by: Optional[List[WikidataQidRef]] = None # P127 + subsidiary: Optional[List[WikidataQidRef]] = None # P355 + chairperson: Optional[List[WikidataQidRef]] = None # P488 + board_member: Optional[List[WikidataQidRef]] = None # P3320 + merged_into: Optional[List[WikidataQidRef]] = None # P7888 + partnership_with: Optional[List[WikidataQidRef]] = None # P1327 + beneficial_owner: Optional[List[WikidataQidRef]] = None # P12621 + significant_person: Optional[List[WikidataQidRef]] = None # P3342 + country: Optional[List[WikidataQidRef]] = None # P17 + located_in: Optional[List[WikidataQidRef]] = None # P131 + capital: Optional[List[WikidataQidRef]] = None # P36 + official_language: Optional[List[WikidataQidRef]] = None # P37 + basic_form_of_government: Optional[List[WikidataQidRef]] = None # P122 + language_used: Optional[List[WikidataQidRef]] = None # P2936 + head_of_government: Optional[List[WikidataQidRef]] = None # P6 + head_of_state: Optional[List[WikidataQidRef]] = None # P35 + part_of: Optional[List[WikidataQidRef]] = None # P361 + author: Optional[List[WikidataQidRef]] = None # P50 + director: Optional[List[WikidataQidRef]] = None # P57 + genre: Optional[List[WikidataQidRef]] = None # P136 + cast_member: Optional[List[WikidataQidRef]] = None # P161 + original_language: Optional[List[WikidataQidRef]] = None # P364 + crew: Optional[List[WikidataQidRef]] = None # P1029 + operator: Optional[List[WikidataQidRef]] = None # P137 + launch_vehicle: Optional[List[WikidataQidRef]] = None # P375 [one entry] + manufacturer: Optional[List[WikidataQidRef]] = None # P176 + brand: Optional[List[WikidataQidRef]] = None # P1716 + product_or_material_produced: Optional[List[WikidataQidRef]] = None # P1056 + part_of_series: Optional[List[WikidataQidRef]] = None # P179 + developer: Optional[List[WikidataQidRef]] = None # P178 + platform: Optional[List[WikidataQidRef]] = None # P400 + material_used: Optional[List[WikidataQidRef]] = None # P186 + operating_system: Optional[List[WikidataQidRef]] = None # P306 + license: Optional[List[WikidataQidRef]] = None # P275 + cpu: Optional[List[WikidataQidRef]] = None # P880 + gpu: Optional[List[WikidataQidRef]] = None # P2560 + director_manager: Optional[List[WikidataQidRef]] = None # P1037 + sponsor: Optional[List[WikidataQidRef]] = None # P859 + participant_in: Optional[List[WikidataQidRef]] = None # P1344 + significant_event: Optional[List[WikidataQidRef]] = None # P793 + sport: Optional[List[WikidataQidRef]] = None # P641 + league: Optional[List[WikidataQidRef]] = None # P118 + member_of_sports_team: Optional[List[WikidataQidRef]] = None # P54 + coach: Optional[List[WikidataQidRef]] = None # P286 + sports_season: Optional[List[WikidataQidRef]] = None # P3450 + winner: Optional[List[WikidataQidRef]] = None # P1346 + participant: Optional[List[WikidataQidRef]] = None # P710 + country_of_origin: Optional[List[WikidataQidRef]] = None # P495 + conflict: Optional[List[WikidataQidRef]] = None # P607 + ethnic_group: Optional[List[WikidataQidRef]] = None # P172 + currency: Optional[List[WikidataQidRef]] = None # P38 + convicted_of: Optional[List[WikidataQidRef]] = None # P1399 + penalty: Optional[List[WikidataQidRef]] = None # P1596 + charge: Optional[List[WikidataQidRef]] = None # P1595 + victim: Optional[List[WikidataQidRef]] = None # P8032 + perpetrator: Optional[List[WikidataQidRef]] = None # P8031 + jurisdiction: Optional[List[WikidataQidRef]] = None # P1001 + investigated_by: Optional[List[WikidataQidRef]] = None # P1840 + discoverer_or_inventor: Optional[List[WikidataQidRef]] = None # P61 + location_of_discovery: Optional[List[WikidataQidRef]] = None # P189 + approved_by: Optional[List[WikidataQidRef]] = None # P790 + military_branch: Optional[List[WikidataQidRef]] = None # P241 + military_rank: Optional[List[WikidataQidRef]] = None # P410 + armament: Optional[List[WikidataQidRef]] = None # P520 + commanded_by: Optional[List[WikidataQidRef]] = None # P4791 + military_unit: Optional[List[WikidataQidRef]] = None # P7779 + producer: Optional[List[WikidataQidRef]] = None # P162 + screenwriter: Optional[List[WikidataQidRef]] = None # P58 + composer: Optional[List[WikidataQidRef]] = None # P86 + performer: Optional[List[WikidataQidRef]] = None # P175 + record_label: Optional[List[WikidataQidRef]] = None # P264 + distributor: Optional[List[WikidataQidRef]] = None # P750 + production_company: Optional[List[WikidataQidRef]] = None # P272 + collection: Optional[List[WikidataQidRef]] = None # P195 + creator: Optional[List[WikidataQidRef]] = None # P170 + iucn_conservation_status: Optional[List[WikidataQidRef]] = None # P141 [one entry] + climate_classification: Optional[List[WikidataQidRef]] = None # P2564 + source_of_energy: Optional[List[WikidataQidRef]] = None # P618 + seismic_classification: Optional[List[WikidataQidRef]] = None # P9235 + named_after: Optional[List[WikidataQidRef]] = None # P138 + different_from: Optional[List[WikidataQidRef]] = None # P1889 + has_parts: Optional[List[WikidataQidRef]] = None # P527 + follows: Optional[List[WikidataQidRef]] = None # P155 + followed_by: Optional[List[WikidataQidRef]] = None # P156 + + # WikidataQidRef -- rival accounts of ONE fact, so only the + # best-ranked claim is stored. + place_of_birth: Optional[WikidataQidRef] = None # P19 + place_of_death: Optional[WikidataQidRef] = None # P20 + + # List[WikidataQuantity] -- read unit_qid/unit_label before + # comparing, and platform/account_id/point_in_time before summing. + # NOTE: the fields marked [one entry] below are list-shaped but + # currently store a SINGLE entry -- the shape is held open so + # restoring the full history stays non-breaking. Do not read + # their length as a count. + social_media_followers: Optional[List[WikidataQuantity]] = None # P8687 + number_of_subscribers: Optional[List[WikidataQuantity]] = None # P3744 [one entry] + number_of_employees: Optional[List[WikidataQuantity]] = None # P1128 [one entry] + total_revenue: Optional[List[WikidataQuantity]] = None # P2139 [one entry] + total_assets: Optional[List[WikidataQuantity]] = None # P2403 + population: Optional[List[WikidataQuantity]] = None # P1082 [one entry] + payload_mass: Optional[List[WikidataQuantity]] = None # P4519 + mass: Optional[List[WikidataQuantity]] = None # P2067 + battery_capacity: Optional[List[WikidataQuantity]] = None # P4140 + data_transfer_speed: Optional[List[WikidataQuantity]] = None # P6711 + frequency: Optional[List[WikidataQuantity]] = None # P2144 + number_of_processor_cores: Optional[List[WikidataQuantity]] = None # P1141 + storage_capacity: Optional[List[WikidataQuantity]] = None # P2928 + engine_displacement: Optional[List[WikidataQuantity]] = None # P8628 [one entry] + wheelbase: Optional[List[WikidataQuantity]] = None # P3039 + width: Optional[List[WikidataQuantity]] = None # P2049 + height: Optional[List[WikidataQuantity]] = None # P2048 + length: Optional[List[WikidataQuantity]] = None # P2043 + speed: Optional[List[WikidataQuantity]] = None # P2052 + torque: Optional[List[WikidataQuantity]] = None # P2230 + power: Optional[List[WikidataQuantity]] = None # P2109 + duration: Optional[List[WikidataQuantity]] = None # P2047 + richter_magnitude: Optional[List[WikidataQuantity]] = None # P2528 + number_of_deaths: Optional[List[WikidataQuantity]] = None # P1120 + number_of_casualties: Optional[List[WikidataQuantity]] = None # P1590 + number_of_injured: Optional[List[WikidataQuantity]] = None # P1339 + range: Optional[List[WikidataQuantity]] = None # P2073 + number_produced: Optional[List[WikidataQuantity]] = None # P1092 [one entry] + box_office: Optional[List[WikidataQuantity]] = None # P2142 + budget: Optional[List[WikidataQuantity]] = None # P2130 + carbon_footprint: Optional[List[WikidataQuantity]] = None # P5991 + + # str -- an ISO-ish time string; rival precisions of ONE date + # collapse to the best-ranked claim. + date_of_birth: Optional[str] = None # P569 + date_of_death: Optional[str] = None # P570 + dissolved: Optional[str] = None # P576 + inception: Optional[str] = None # P571 + date_of_official_opening: Optional[str] = None # P1619 + + # List[str] -- ISO-ish time strings. + # NOTE: the fields marked [one entry] below are list-shaped but + # currently store a SINGLE entry -- the shape is held open so + # restoring the full history stays non-breaking. Do not read + # their length as a count. + publication_date: Optional[List[str]] = None # P577 + launch_date: Optional[List[str]] = None # P619 [one entry] + time_of_discovery: Optional[List[str]] = None # P575 + service_entry: Optional[List[str]] = None # P729 [one entry] + service_retirement: Optional[List[str]] = None # P730 [one entry] + + # List[WikidataMonolingualText] -- language-tagged values. + short_name: Optional[List[WikidataMonolingualText]] = None # P1813 + nickname: Optional[List[WikidataMonolingualText]] = None # P1449 + native_label: Optional[List[WikidataMonolingualText]] = None # P1705 + name_in_native_language: Optional[List[WikidataMonolingualText]] = None # P1559 + official_name: Optional[List[WikidataMonolingualText]] = None # P1448 + + # List[str] -- ALWAYS a list: an identifier is a join key and + # entities legitimately carry two (Alphabet has two SEC CIKs). + x_twitter_username: Optional[List[str]] = None # P2002 + facebook_id: Optional[List[str]] = None # P2013 + instagram_username: Optional[List[str]] = None # P2003 + youtube_channel_id: Optional[List[str]] = None # P2397 + linkedin_company_or_organization_id: Optional[List[str]] = None # P4264 + linkedin_personal_profile_id: Optional[List[str]] = None # P6634 + telegram_username: Optional[List[str]] = None # P3789 + tiktok_username: Optional[List[str]] = None # P7085 + youtube_handle: Optional[List[str]] = None # P11245 + mastodon_address: Optional[List[str]] = None # P4033 + threads_username: Optional[List[str]] = None # P11892 + vk_username: Optional[List[str]] = None # P3185 + weibo_user_id: Optional[List[str]] = None # P3579 + bluesky_handle: Optional[List[str]] = None # P12361 + pinterest_username: Optional[List[str]] = None # P3836 + twitch_username: Optional[List[str]] = None # P5797 + tumblr_username: Optional[List[str]] = None # P3943 + snapchat_username: Optional[List[str]] = None # P2984 + reddit_username: Optional[List[str]] = None # P4265 + bilibili_uid: Optional[List[str]] = None # P6455 + discord_server_numeric_id: Optional[List[str]] = None # P9345 + whatsapp_channel_id: Optional[List[str]] = None # P12542 + wechat_id: Optional[List[str]] = None # P7650 + truth_social_username: Optional[List[str]] = None # P10858 + quora_username: Optional[List[str]] = None # P4411 + douyin_username: Optional[List[str]] = None # P7120 + gab_username: Optional[List[str]] = None # P8919 + rednote_profile_id: Optional[List[str]] = None # P12038 + rumble_channel: Optional[List[str]] = None # P11962 + discord_username: Optional[List[str]] = None # P9101 + subreddit: Optional[List[str]] = None # P3984 + viaf_cluster_id: Optional[List[str]] = None # P214 + isni: Optional[List[str]] = None # P213 + gnd_id: Optional[List[str]] = None # P227 + library_of_congress_authorities_id: Optional[List[str]] = None # P244 + idref_id: Optional[List[str]] = None # P269 + grid_id: Optional[List[str]] = None # P2427 + sec_cik: Optional[List[str]] = None # P5531 + duns_number: Optional[List[str]] = None # P2771 + opensanctions_id: Optional[List[str]] = None # P10632 + isin: Optional[List[str]] = None # P946 + lei_code: Optional[List[str]] = None # P1278 + swift_bic_code: Optional[List[str]] = None # P2627 + imdb_id: Optional[List[str]] = None # P345 + + # str -- locale variants collapse to ONE site; the normalized + # `official_website_domain` sibling in the core above is the match key. + official_website: Optional[str] = None # P856 + + # List[str] -- Wikimedia Commons file URLs. + image: Optional[List[str]] = None # P18 + + # WikidataCoordinate -- rival measurements of ONE point. + coordinates: Optional[WikidataCoordinate] = None # P625 + + # List[str]. + ticker_symbol: Optional[List[str]] = None # P249 + version: Optional[List[str]] = None # P348 class WikiResponseDictItem(BaseModel): diff --git a/tests/test_entity_linking.py b/tests/test_entity_linking.py index fdc4d70..7e02194 100644 --- a/tests/test_entity_linking.py +++ b/tests/test_entity_linking.py @@ -3,7 +3,7 @@ from urllib.parse import parse_qs, quote, urlparse import pytest -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from asknews_sdk.api.wiki import AsyncWikiAPI, WikiAPI from asknews_sdk.dto.wiki import ( @@ -102,7 +102,7 @@ def test_unknown_wikidata_metadata_fields_are_preserved(): "qid": "Q312", "relevance": 0.91, "wikidata_metadata": { - "ceo": {"label": "Tim Cook"}, + "ceo": [{"qid": "Q312556", "label": "Tim Cook"}], "some_future_property": 42, }, } @@ -110,7 +110,9 @@ def test_unknown_wikidata_metadata_fields_are_preserved(): ) metadata = response.linked_entity.wikidata_metadata - assert metadata.ceo == {"label": "Tim Cook"} + # A declared property parses into its typed shape... + assert metadata.ceo[0].label == "Tim Cook" + # ...while one this SDK does not know about still reaches the caller. assert metadata.model_extra == {"some_future_property": 42} @@ -365,8 +367,13 @@ def test_qid_is_escaped_into_the_path(): }, ], ), - ("place_of_birth", [{"qid": "Q60", "label": "New York City"}]), - ("date_of_birth", [{"time": "+1955-02-24T00:00:00Z"}]), + # A "single-best-value" property: rival accounts of ONE fact, so the API + # sends the best-ranked claim alone rather than a list. + ("place_of_birth", {"qid": "Q60", "label": "New York City"}), + # Times are plain ISO-ish strings. + ("date_of_birth", "+1955-02-24T00:00:00Z"), + # External identifiers are ALWAYS lists -- an identifier is a join key, + # and entities legitimately carry two. ("gnd_id", ["118637347"]), ("official_name", [{"text": "Apple Inc.", "language": "en"}]), ("social_media_followers", [{"amount": 5000000, "platform": "x_twitter"}]), @@ -375,16 +382,23 @@ def test_qid_is_escaped_into_the_path(): ], ) def test_metadata_accepts_property_shapes(field, value): - """Property-derived fields must parse whatever arrives, expected or not. + """Property-derived fields parse the shapes the API actually sends. - The metadata is nested inside the response model, so a raise here fails the - WHOLE call — including for callers that never read the offending field. That - blast radius is why these stay `Any` rather than being typed to today's - shapes. + These fields are typed, so the parsed value is a model (or list of models) + rather than the raw dict — compare via model_dump(exclude_none=True), which + also pins that absent qualifiers stay absent rather than becoming nulls. """ metadata = WikidataMetadata(**{field: value}) + parsed = getattr(metadata, field) - assert getattr(metadata, field) == value + def _plain(v): + if isinstance(v, list): + return [_plain(x) for x in v] + if isinstance(v, BaseModel): + return v.model_dump(exclude_none=True) + return v + + assert _plain(parsed) == value def test_metadata_passes_through_undeclared_fields():