diff --git a/MSMetaEnhancer/libs/converters/web/Wikidata.py b/MSMetaEnhancer/libs/converters/web/Wikidata.py new file mode 100644 index 0000000..6d6a34b --- /dev/null +++ b/MSMetaEnhancer/libs/converters/web/Wikidata.py @@ -0,0 +1,179 @@ +import json +from urllib.parse import urlencode + +from frozendict import frozendict + +from MSMetaEnhancer.libs.converters.web.WebConverter import WebConverter + + +class Wikidata(WebConverter): + """ + Convert compound names, identifiers, and structures using Wikidata. + + Wikidata Query Service: https://query.wikidata.org/ + """ + + attributes = ( + "compound_name", + "casno", + "inchi", + "inchikey", + "canonical_smiles", + "pubchemid", + "wikidataid", + ) + identifier_properties = { + "pubchemid": "P662", + "inchi": "P234", + "inchikey": "P235", + } + + def __init__(self, session): + super().__init__(session) + self.endpoints = {"Wikidata": "https://query.wikidata.org/sparql?"} + source_methods = { + "compound_name": "from_name", + "casno": "from_casno", + "wikidataid": "from_wikidataid", + "pubchemid": "from_pubchemid", + "inchi": "from_inchi", + "inchikey": "from_inchikey", + } + conversions = [ + (source, target, method) + for source, method in source_methods.items() + for target in self.attributes + if source != target + ] + self.create_top_level_conversion_methods(conversions) + + @staticmethod + def result_variables(): + return " ".join(f"?{attribute}" for attribute in Wikidata.attributes) + + @staticmethod + def item_attributes_query(): + return """ + OPTIONAL { + ?item rdfs:label ?compound_name . + FILTER(LANG(?compound_name) = "en") + } + OPTIONAL { ?item wdt:P231 ?casno . } + OPTIONAL { ?item wdt:P234 ?inchi . } + OPTIONAL { ?item wdt:P235 ?inchikey . } + OPTIONAL { ?item wdt:P233 ?canonical_smiles . } + OPTIONAL { ?item wdt:P662 ?pubchemid . } + BIND(STRAFTER(STR(?item), "http://www.wikidata.org/entity/") + AS ?wikidataid) + """ + + async def from_name(self, name): + query = f""" + PREFIX bd: + PREFIX mwapi: + PREFIX rdfs: + PREFIX wikibase: + PREFIX wdt: + + SELECT {self.result_variables()} WHERE {{ + SERVICE wikibase:mwapi {{ + bd:serviceParam wikibase:api "EntitySearch" ; + wikibase:endpoint "www.wikidata.org" ; + mwapi:search {self.sparql_literal(name)} ; + mwapi:language "en" ; + mwapi:limit "1" . + ?item wikibase:apiOutputItem mwapi:item . + }} + {self.item_attributes_query()} + }} + LIMIT 1 + """ + return await self.call_service(query) + + async def from_casno(self, casno): + query = f""" + PREFIX rdfs: + PREFIX wdt: + + SELECT {self.result_variables()} WHERE {{ + ?item wdt:P231 ?casno . + FILTER(?casno = {self.sparql_literal(casno)}) + {self.item_attributes_query()} + }} + LIMIT 1 + """ + return await self.call_service(query) + + async def from_wikidataid(self, wikidataid): + if not wikidataid.startswith("Q") or not wikidataid[1:].isdigit(): + return {} + + query = f""" + PREFIX rdfs: + PREFIX wd: + PREFIX wdt: + + SELECT {self.result_variables()} WHERE {{ + BIND(wd:{wikidataid} AS ?item) + {self.item_attributes_query()} + }} + LIMIT 1 + """ + return await self.call_service(query) + + async def from_pubchemid(self, pubchemid): + return await self.from_identifier("pubchemid", pubchemid) + + async def from_inchi(self, inchi): + return await self.from_identifier("inchi", inchi) + + async def from_inchikey(self, inchikey): + return await self.from_identifier("inchikey", inchikey) + + async def from_identifier(self, attribute, value): + property_id = self.identifier_properties[attribute] + query = f""" + PREFIX rdfs: + PREFIX wdt: + + SELECT {self.result_variables()} WHERE {{ + ?item wdt:{property_id} ?identifier . + FILTER(STR(?identifier) = {self.sparql_literal(value)}) + {self.item_attributes_query()} + }} + LIMIT 1 + """ + return await self.call_service(query) + + async def call_service(self, query): + args = urlencode({"query": query, "format": "json"}) + response = await self.query_the_service( + "Wikidata", + args, + headers=frozendict( + { + "User-Agent": ( + "MSMetaEnhancer (https://github.com/RECETOX/MSMetaEnhancer)" + ) + } + ), + ) + if response: + return self.parse_attributes(response) + return {} + + @staticmethod + def sparql_literal(value): + return json.dumps(str(value), ensure_ascii=False) + + @staticmethod + def parse_attributes(response): + bindings = json.loads(response).get("results", {}).get("bindings", []) + if not bindings: + return {} + + return { + attribute: bindings[0][attribute]["value"] + for attribute in Wikidata.attributes + if attribute in bindings[0] + } diff --git a/MSMetaEnhancer/libs/converters/web/__init__.py b/MSMetaEnhancer/libs/converters/web/__init__.py index 76e1e31..8b86930 100644 --- a/MSMetaEnhancer/libs/converters/web/__init__.py +++ b/MSMetaEnhancer/libs/converters/web/__init__.py @@ -3,5 +3,6 @@ from MSMetaEnhancer.libs.converters.web.CIR import CIR from MSMetaEnhancer.libs.converters.web.PubChem import PubChem from MSMetaEnhancer.libs.converters.web.BridgeDb import BridgeDb +from MSMetaEnhancer.libs.converters.web.Wikidata import Wikidata -__all__ = ["IDSM", "CTS", "CIR", "PubChem", "BridgeDb"] +__all__ = ["IDSM", "CTS", "CIR", "PubChem", "BridgeDb", "Wikidata"] diff --git a/README.md b/README.md index bac190f..3db0baa 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![DOI](https://joss.theoj.org/papers/10.21105/joss.04494/status.svg)](https://doi.org/10.21105/joss.04494) **MSMetaEnhancer** is a tool used for `.msp` files annotation. -It adds metadata like SMILES, InChI, and CAS number fetched from the following services: [CIR](https://cactus.nci.nih.gov/chemical/structure_documentation), [CTS](https://cts.fiehnlab.ucdavis.edu/), [PubChem](https://pubchem.ncbi.nlm.nih.gov/), [IDSM](https://idsm.elixir-czech.cz/), and [BridgeDb](https://bridgedb.github.io/). +It adds metadata like SMILES, InChI, and CAS number fetched from the following services: [CIR](https://cactus.nci.nih.gov/chemical/structure_documentation), [CTS](https://cts.fiehnlab.ucdavis.edu/), [PubChem](https://pubchem.ncbi.nlm.nih.gov/), [IDSM](https://idsm.elixir-czech.cz/), [BridgeDb](https://bridgedb.github.io/), and [Wikidata](https://www.wikidata.org/). The app uses asynchronous implementation of annotation process allowing for optimal fetching speed. If you use MSMetaEnhancer in your work, please cite the following publication: @@ -20,11 +20,18 @@ Troják et al., (2022). MSMetaEnhancer: A Python package for mass spectra metada import asyncio from MSMetaEnhancer import Application -from MSMetaEnhancer.libs.converters.web import CTS, CIR, IDSM, PubChem, BridgeDb +from MSMetaEnhancer.libs.converters.web import ( + CTS, + CIR, + IDSM, + PubChem, + BridgeDb, + Wikidata, +) from MSMetaEnhancer.libs.converters.compute import RDKit from MSMetaEnhancer.libs.utils.ConverterBuilder import ConverterBuilder -ConverterBuilder.register([CTS, CIR, IDSM, PubChem, BridgeDb, RDKit]) +ConverterBuilder.register([CTS, CIR, IDSM, PubChem, BridgeDb, Wikidata, RDKit]) app = Application() @@ -35,7 +42,7 @@ app.load_data('sample.msp', file_format='msp') app.curate_metadata() # specify requested services (these are supported) -services = ['CTS', 'CIR', 'IDSM', 'PubChem', 'BridgeDb', 'RDKit'] +services = ['CTS', 'CIR', 'IDSM', 'PubChem', 'BridgeDb', 'Wikidata', 'RDKit'] # specify requested jobs jobs = [('name', 'inchi', 'IDSM'), ('inchi', 'formula', 'IDSM'), ('inchi', 'inchikey', 'IDSM'), diff --git a/docs/source/MSMetaEnhancer.libs.webConverters.rst b/docs/source/MSMetaEnhancer.libs.webConverters.rst index ba23c56..26ee3d5 100644 --- a/docs/source/MSMetaEnhancer.libs.webConverters.rst +++ b/docs/source/MSMetaEnhancer.libs.webConverters.rst @@ -41,6 +41,14 @@ BridgeDb :undoc-members: :show-inheritance: +Wikidata +-------- + +.. automodule:: MSMetaEnhancer.libs.converters.web.Wikidata + :members: + :undoc-members: + :show-inheritance: + General Web Converter --------------------- diff --git a/galaxy/generate_options.py b/galaxy/generate_options.py index dbed8d6..5474747 100644 --- a/galaxy/generate_options.py +++ b/galaxy/generate_options.py @@ -4,7 +4,14 @@ # this add to path the home dir, so it can be called from anywhere sys.path.append(os.path.split(sys.path[0])[0]) -from MSMetaEnhancer.libs.converters.web import CTS, CIR, IDSM, PubChem, BridgeDb +from MSMetaEnhancer.libs.converters.web import ( + CTS, + CIR, + IDSM, + PubChem, + BridgeDb, + Wikidata, +) from MSMetaEnhancer.libs.converters.compute import RDKit from MSMetaEnhancer.libs.utils.ConverterBuilder import ConverterBuilder @@ -13,7 +20,7 @@ def generate_options(): - ConverterBuilder.register([CTS, CIR, IDSM, PubChem, BridgeDb, RDKit]) + ConverterBuilder.register([CTS, CIR, IDSM, PubChem, BridgeDb, Wikidata, RDKit]) jobs = [] converters = web_converters + compute_converters diff --git a/tests/test_Wikidata.py b/tests/test_Wikidata.py new file mode 100644 index 0000000..fad080b --- /dev/null +++ b/tests/test_Wikidata.py @@ -0,0 +1,224 @@ +import asyncio +import json +from urllib.parse import parse_qs + +import mock +import pytest + +from MSMetaEnhancer.libs.converters.web import Wikidata +from MSMetaEnhancer.libs.utils.Errors import UnknownResponse +from tests.utils import wrap_with_session + + +def test_get_conversions(): + conversions = Wikidata(None).get_conversion_functions() + + assert ("compound_name", "casno", "Wikidata") in conversions + assert ("compound_name", "inchi", "Wikidata") in conversions + assert ("compound_name", "inchikey", "Wikidata") in conversions + assert ("compound_name", "canonical_smiles", "Wikidata") in conversions + assert ("compound_name", "pubchemid", "Wikidata") in conversions + assert ("casno", "compound_name", "Wikidata") in conversions + assert ("casno", "inchi", "Wikidata") in conversions + assert ("casno", "inchikey", "Wikidata") in conversions + assert ("casno", "canonical_smiles", "Wikidata") in conversions + assert ("casno", "pubchemid", "Wikidata") in conversions + assert ("wikidataid", "casno", "Wikidata") in conversions + assert ("pubchemid", "inchi", "Wikidata") in conversions + assert ("inchi", "inchikey", "Wikidata") in conversions + assert ("inchikey", "compound_name", "Wikidata") in conversions + + +def test_from_name_queries_wikidata_and_parses_attributes(): + converter = Wikidata(None) + converter.query_the_service = mock.AsyncMock( + return_value=json.dumps( + { + "results": { + "bindings": [ + { + "compound_name": {"value": "Aspirin"}, + "casno": {"value": "50-78-2"}, + "inchi": {"value": "InChI=1S/C9H8O4"}, + "inchikey": {"value": "BSYNRYMUTXBXSQ-UHFFFAOYSA-N"}, + "canonical_smiles": { + "value": "CC(=O)OC1=CC=CC=C1C(=O)O" + }, + "pubchemid": {"value": "2244"}, + } + ] + } + } + ) + ) + + result = asyncio.run(converter.from_name("aspirin")) + + converter.query_the_service.assert_called_once() + service, args = converter.query_the_service.call_args.args[:2] + query_params = parse_qs(args) + assert service == "Wikidata" + assert query_params["format"] == ["json"] + assert 'mwapi:search "aspirin"' in query_params["query"][0] + assert result == { + "compound_name": "Aspirin", + "casno": "50-78-2", + "inchi": "InChI=1S/C9H8O4", + "inchikey": "BSYNRYMUTXBXSQ-UHFFFAOYSA-N", + "canonical_smiles": "CC(=O)OC1=CC=CC=C1C(=O)O", + "pubchemid": "2244", + } + + +def test_multiple_name_conversions_reuse_cached_service_request(): + converter = Wikidata(None) + converter.loop_request = mock.AsyncMock( + return_value=json.dumps( + { + "results": { + "bindings": [ + { + "compound_name": {"value": "Aspirin"}, + "casno": {"value": "50-78-2"}, + } + ] + } + } + ) + ) + + async def convert_multiple_attributes(): + name_result = await converter.from_name("aspirin") + casno_result = await converter.compound_name_to_casno("aspirin") + return name_result, casno_result + + name_result, casno_result = asyncio.run(convert_multiple_attributes()) + + converter.loop_request.assert_awaited_once() + assert name_result["compound_name"] == "Aspirin" + assert casno_result["casno"] == "50-78-2" + + +def test_from_casno_queries_cas_property(): + converter = Wikidata(None) + converter.query_the_service = mock.AsyncMock( + return_value=json.dumps( + { + "results": { + "bindings": [ + { + "compound_name": {"value": "Aspirin"}, + "casno": {"value": "50-78-2"}, + } + ] + } + } + ) + ) + + result = asyncio.run(converter.from_casno("50-78-2")) + + _, args = converter.query_the_service.call_args.args[:2] + query = parse_qs(args)["query"][0] + assert "?item wdt:P231 ?casno ." in query + assert 'FILTER(?casno = "50-78-2")' in query + assert result == {"compound_name": "Aspirin", "casno": "50-78-2"} + + +@pytest.mark.parametrize( + "method, identifier, query_fragment", + [ + ("from_wikidataid", "Q18216", "BIND(wd:Q18216 AS ?item)"), + ( + "from_pubchemid", + "2244", + 'wdt:P662 ?identifier', + ), + ( + "from_inchi", + "InChI=1S/C9H8O4", + "wdt:P234 ?identifier", + ), + ( + "from_inchikey", + "BSYNRYMUTXBXSQ-UHFFFAOYSA-N", + "wdt:P235 ?identifier", + ), + ], +) +def test_identifier_lookups(method, identifier, query_fragment): + converter = Wikidata(None) + converter.query_the_service = mock.AsyncMock( + return_value=json.dumps( + { + "results": { + "bindings": [ + { + "compound_name": {"value": "aspirin"}, + "pubchemid": {"value": "2244"}, + "wikidataid": {"value": "Q18216"}, + } + ] + } + } + ) + ) + + result = asyncio.run(getattr(converter, method)(identifier)) + + query = parse_qs(converter.query_the_service.call_args.args[1])["query"][0] + assert query_fragment in query + if method != "from_wikidataid": + assert f'FILTER(STR(?identifier) = "{identifier}")' in query + assert result == { + "compound_name": "aspirin", + "pubchemid": "2244", + "wikidataid": "Q18216", + } + + +def test_from_wikidataid_rejects_invalid_ids(): + converter = Wikidata(None) + converter.query_the_service = mock.AsyncMock() + + result = asyncio.run(converter.from_wikidataid("Q1 } SERVICE")) + + assert result == {} + converter.query_the_service.assert_not_called() + + +@pytest.mark.dependency() +def test_aspirin_name_lookup_live(): + result = asyncio.run(wrap_with_session(Wikidata, "from_name", ["aspirin"])) + + assert result["casno"] == "50-78-2" + assert result["canonical_smiles"] == "CC(=O)OC1=CC=CC=C1C(=O)O" + assert result["pubchemid"] == "2244" + + +@pytest.mark.dependency(depends=["test_aspirin_name_lookup_live"]) +def test_aspirin_cas_lookup_live(): + result = asyncio.run(wrap_with_session(Wikidata, "from_casno", ["50-78-2"])) + + assert result["compound_name"].casefold() == "aspirin" + + +@pytest.mark.dependency(depends=["test_aspirin_name_lookup_live"]) +@pytest.mark.xfail(raises=UnknownResponse) +@pytest.mark.parametrize( + "method, identifier", + [ + ("from_wikidataid", "Q18216"), + ("from_pubchemid", "2244"), + ( + "from_inchi", + "InChI=1S/C9H8O4/c1-6(10)13-8-5-3-2-4-7(8)9(11)12/h2-5H,1H3,(H,11,12)", + ), + ("from_inchikey", "BSYNRYMUTXBXSQ-UHFFFAOYSA-N"), + ], +) +def test_aspirin_identifier_lookup_live(method, identifier): + result = asyncio.run(wrap_with_session(Wikidata, method, [identifier])) + + assert result["casno"] == "50-78-2" + assert result["canonical_smiles"] == "CC(=O)OC1=CC=CC=C1C(=O)O"