From 58acf34df0615f536421cde523370c94c27f3d4b Mon Sep 17 00:00:00 2001 From: "aiste.grigaliunaite@decodo.com" Date: Fri, 23 Oct 2026 17:01:27 +0300 Subject: [PATCH] chore: add ecommerce and serp tools --- README.md | 26 ++++++++ src/decodo_crewai/__init__.py | 4 +- src/decodo_crewai/_base.py | 7 ++ src/decodo_crewai/ecommerce_tool.py | 95 +++++++++++++++++++++++++++ src/decodo_crewai/serp_tool.py | 67 +++++++++++++++++++ tests/test_tools.py | 99 ++++++++++++++++++++++++++++- 6 files changed, 296 insertions(+), 2 deletions(-) create mode 100644 src/decodo_crewai/ecommerce_tool.py create mode 100644 src/decodo_crewai/serp_tool.py diff --git a/README.md b/README.md index 7d0f35c..72dccc4 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ | --- | --- | | `DecodoWebScraperTool` | Scrapes any public URL and returns the page as Markdown. | | `DecodoSearchTool` | Searches Google or Amazon and returns the parsed results as JSON. | +| `DecodoEcommerceTool` | Gets product details or search results from Amazon, Walmart or Target as JSON. | +| `DecodoSERPTool` | Gets Google or Bing results pages, with paging, or a Google AI Mode answer as JSON. | ## Requirements @@ -99,6 +101,30 @@ Arguments the agent fills in: | `source` | `"google"` or `"amazon"`, optional | `google` (default) for web results, `amazon` for product listings. | | `geo` | `str`, optional | Google: a country name, e.g. `United States`. Amazon: a US ZIP code, e.g. `10001`, or a two-letter country code for delivery outside the US, e.g. `DE`. | +### DecodoEcommerceTool + +Arguments the agent fills in. Pass either `product_id` or `query`: + +| Argument | Type | Description | +| --- | --- | --- | +| `store` | `"amazon"`, `"walmart"` or `"target"` | Store to look up. | +| `product_id` | `str`, optional | One product's details. Amazon: the ASIN, e.g. `B09H74FXNW`. Walmart or Target: the product ID from the product URL. | +| `query` | `str`, optional | Search the store's products instead. | +| `zip_code` | `str`, optional | US ZIP code for prices and availability at that delivery location, e.g. `10001`. | + +### DecodoSERPTool + +Arguments the agent fills in: + +| Argument | Type | Description | +| --- | --- | --- | +| `query` | `str` | Search query. | +| `engine` | `"google"`, `"bing"` or `"google_ai_mode"`, optional | `google` (default) or `bing` for a results page; `google_ai_mode` for Google's AI-written answer and the sources it cites. | +| `page` | `int`, optional | Results page to fetch, starting at 1. Ignored by `google_ai_mode`. | +| `geo` | `str`, optional | Country to search from, e.g. `United States`. | + +Bing results are rendered in a headless browser and can take over a minute. + ## Errors When a request fails, for example because of an invalid API key, a rate limit or a page that returns `404`, the tools return a CrewAI `ToolFailure` instead of raising. The agent sees what went wrong and can retry or change course, and CrewAI records the failure according to the agent's `tool_failure_policy`. diff --git a/src/decodo_crewai/__init__.py b/src/decodo_crewai/__init__.py index efba8fe..5360d1f 100644 --- a/src/decodo_crewai/__init__.py +++ b/src/decodo_crewai/__init__.py @@ -1,5 +1,7 @@ from ._base import DecodoBaseTool +from .ecommerce_tool import DecodoEcommerceTool from .search_tool import DecodoSearchTool +from .serp_tool import DecodoSERPTool from .web_scraper_tool import DecodoWebScraperTool -__all__ = ["DecodoBaseTool", "DecodoSearchTool", "DecodoWebScraperTool"] +__all__ = ["DecodoBaseTool", "DecodoEcommerceTool", "DecodoSERPTool", "DecodoSearchTool", "DecodoWebScraperTool"] diff --git a/src/decodo_crewai/_base.py b/src/decodo_crewai/_base.py index 4f18c29..6289416 100644 --- a/src/decodo_crewai/_base.py +++ b/src/decodo_crewai/_base.py @@ -77,6 +77,13 @@ def _scrape(self, params: ScrapeRequest) -> Any: results = response.get("results") or [] if not results: + failed: dict[str, Any] = dict(response) + if failed.get("message"): + return ToolFailure( + message=f"Decodo could not scrape the target: {failed['message']}", + code=str(failed.get("status_code", "scrape_failed")), + retryable=True, + ) return ToolFailure(message="Decodo returned no results.", code="empty_response") result = results[0] diff --git a/src/decodo_crewai/ecommerce_tool.py b/src/decodo_crewai/ecommerce_tool.py new file mode 100644 index 0000000..4f455c5 --- /dev/null +++ b/src/decodo_crewai/ecommerce_tool.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import json +from typing import Literal + +from crewai.tools.tool_failure import ToolFailure +from decodo import ( + AmazonProductParams, + AmazonSearchParams, + ScrapeRequest, + TargetProductParams, + TargetSearchParams, + WalmartProductParams, + WalmartSearchParams, +) +from pydantic import BaseModel, Field, model_validator + +from ._base import DecodoBaseTool + +Store = Literal["amazon", "walmart", "target"] + + +class DecodoEcommerceArgs(BaseModel): + store: Store = Field(description="Store to look up: 'amazon', 'walmart' or 'target'") + product_id: str | None = Field( + default=None, + description=( + "Get one product's details. Amazon: the ASIN, e.g. 'B09H74FXNW'. " + "Walmart or Target: the product ID from the product URL, e.g. '15296401808'" + ), + ) + query: str | None = Field(default=None, description="Search the store's products for this query instead") + zip_code: str | None = Field( + default=None, + description="US ZIP code to see prices and availability for that delivery location, e.g. '10001'", + ) + + @model_validator(mode="after") + def _one_of_product_id_or_query(self) -> DecodoEcommerceArgs: + if bool(self.product_id) == bool(self.query): + raise ValueError("Pass either product_id or query, not both.") + return self + + +class DecodoEcommerceTool(DecodoBaseTool): + """Get product details or search results from Amazon, Walmart or Target, parsed as JSON. + + Args: + api_key: Decodo Web Scraping API key. Defaults to the ``DECODO_API_KEY`` environment variable. + """ + + name: str = "Decodo Ecommerce" + description: str = ( + "Look up products on Amazon, Walmart or Target and return structured JSON. " + "Pass product_id for one product's details: title, price, rating, seller, availability. " + "Pass query to search the store and get matching products with prices." + ) + args_schema: type[BaseModel] = DecodoEcommerceArgs + + def _run( + self, + store: Store, + product_id: str | None = None, + query: str | None = None, + zip_code: str | None = None, + ) -> str | ToolFailure: + if bool(product_id) == bool(query): + return ToolFailure(message="Pass either product_id or query, not both.", code="invalid_params") + + zip_code = zip_code or None + params: ScrapeRequest + if store == "amazon": + params = ( + AmazonProductParams(query=product_id, geo=zip_code, parse=True) + if product_id + else AmazonSearchParams(query=query, geo=zip_code, parse=True) + ) + elif store == "walmart": + params = ( + WalmartProductParams(product_id=product_id, delivery_zip=zip_code, parse=True) + if product_id + else WalmartSearchParams(query=query, delivery_zip=zip_code, parse=True) + ) + else: + # Target pages need JavaScript rendering to load product data. + params = ( + TargetProductParams(product_id=product_id, delivery_zip=zip_code, headless="html", parse=True) + if product_id + else TargetSearchParams(query=query, delivery_zip=zip_code, headless="html", parse=True) + ) + + content = self._scrape(params) + if isinstance(content, ToolFailure | str): + return content + return json.dumps(content) diff --git a/src/decodo_crewai/serp_tool.py b/src/decodo_crewai/serp_tool.py new file mode 100644 index 0000000..38a806a --- /dev/null +++ b/src/decodo_crewai/serp_tool.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import json +from typing import Literal + +from crewai.tools.tool_failure import ToolFailure +from decodo import BingSearchParams, GoogleAiModeParams, GoogleSearchParams, ScrapeRequest +from pydantic import BaseModel, Field + +from ._base import DecodoBaseTool + +Engine = Literal["google", "bing", "google_ai_mode"] + + +class DecodoSERPArgs(BaseModel): + query: str = Field(description="Search query") + engine: Engine = Field( + default="google", + description=( + "'google' or 'bing' for a results page; 'google_ai_mode' for Google's AI-written answer " + "with the links it cites" + ), + ) + page: int = Field(default=1, ge=1, description="Results page to fetch, starting at 1. Ignored by google_ai_mode") + geo: str | None = Field( + default=None, + description="Country to search from, as a country name, e.g. 'United States'", + ) + + +class DecodoSERPTool(DecodoBaseTool): + """Search Google or Bing, or ask Google AI Mode, and return the parsed results as JSON. + + Args: + api_key: Decodo Web Scraping API key. Defaults to the ``DECODO_API_KEY`` environment variable. + """ + + name: str = "Decodo SERP" + description: str = ( + "Get search engine results as structured JSON. " + "Use engine='google' or engine='bing' for a full results page, with page for deeper results. " + "Use engine='google_ai_mode' for Google's AI-written answer and the sources it cites." + ) + args_schema: type[BaseModel] = DecodoSERPArgs + + def _run( + self, + query: str, + engine: Engine = "google", + page: int = 1, + geo: str | None = None, + ) -> str | ToolFailure: + geo = geo or None + page_from = page if page > 1 else None + params: ScrapeRequest + if engine == "google_ai_mode": + params = GoogleAiModeParams(query=query, geo=geo, parse=True) + elif engine == "bing": + # Without JavaScript rendering, Bing fails with status 613 (target could not be scraped). + params = BingSearchParams(query=query, geo=geo, page_from=page_from, headless="html", parse=True) + else: + params = GoogleSearchParams(query=query, geo=geo, page_from=page_from, parse=True) + + content = self._scrape(params) + if isinstance(content, ToolFailure | str): + return content + return json.dumps(content) diff --git a/tests/test_tools.py b/tests/test_tools.py index 894fddf..acb8d57 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -8,7 +8,8 @@ import pytest from crewai.tools.tool_failure import ToolFailure -from decodo_crewai import DecodoSearchTool, DecodoWebScraperTool +from decodo_crewai import DecodoEcommerceTool, DecodoSearchTool, DecodoSERPTool, DecodoWebScraperTool +from decodo_crewai.ecommerce_tool import DecodoEcommerceArgs class FakeApi: @@ -140,6 +141,96 @@ def test_empty_results_become_tool_failure(fake_api: FakeApi) -> None: assert result.code == "empty_response" +def test_failed_task_becomes_retryable_tool_failure(fake_api: FakeApi) -> None: + fake_api.body = {"status": "failed", "status_code": 613, "message": "We were not able to scrape the target."} + + result = DecodoWebScraperTool()._run(url="https://example.com") + + assert isinstance(result, ToolFailure) + assert result.code == "613" + assert result.retryable is True + assert "We were not able to scrape the target." in result.message + + +@pytest.mark.parametrize( + ("kwargs", "payload"), + [ + ( + {"store": "amazon", "product_id": "B09H74FXNW", "zip_code": "10001"}, + {"target": "amazon_product", "query": "B09H74FXNW", "geo": "10001", "parse": True}, + ), + ( + {"store": "amazon", "query": "usb c cable"}, + {"target": "amazon_search", "query": "usb c cable", "parse": True}, + ), + ( + {"store": "walmart", "product_id": "15296401808"}, + {"target": "walmart_product", "product_id": "15296401808", "parse": True}, + ), + ( + {"store": "walmart", "query": "camping tent", "zip_code": "10001"}, + {"target": "walmart_search", "query": "camping tent", "delivery_zip": "10001", "parse": True}, + ), + ( + {"store": "target", "product_id": "1003921355"}, + {"target": "target_product", "product_id": "1003921355", "headless": "html", "parse": True}, + ), + ( + {"store": "target", "query": "camping tent"}, + {"target": "target_search", "query": "camping tent", "headless": "html", "parse": True}, + ), + ], +) +def test_ecommerce_requests(fake_api: FakeApi, kwargs: dict[str, Any], payload: dict[str, Any]) -> None: + content = {"title": "A product", "price": 9.99} + fake_api.body = {"results": [{"status_code": 200, "content": content}]} + + result = DecodoEcommerceTool().run(**kwargs) + + assert fake_api.payload == payload + assert json.loads(result) == content + + +@pytest.mark.parametrize("kwargs", [{}, {"product_id": "B09H74FXNW", "query": "usb c cable"}]) +def test_ecommerce_needs_exactly_one_of_product_id_or_query(fake_api: FakeApi, kwargs: dict[str, Any]) -> None: + with pytest.raises(ValueError, match="either product_id or query"): + DecodoEcommerceArgs(store="amazon", **kwargs) + + result = DecodoEcommerceTool()._run(store="amazon", **kwargs) + + assert isinstance(result, ToolFailure) + assert result.code == "invalid_params" + assert not fake_api.calls + + +@pytest.mark.parametrize( + ("kwargs", "payload"), + [ + ({"query": "batteries"}, {"target": "google_search", "query": "batteries", "parse": True}), + ( + {"query": "batteries", "page": 2, "geo": "Germany"}, + {"target": "google_search", "query": "batteries", "geo": "Germany", "page_from": 2, "parse": True}, + ), + ( + {"query": "batteries", "engine": "bing", "page": 3}, + {"target": "bing_search", "query": "batteries", "page_from": 3, "headless": "html", "parse": True}, + ), + ( + {"query": "what are batteries", "engine": "google_ai_mode", "page": 2}, + {"target": "google_ai_mode", "query": "what are batteries", "parse": True}, + ), + ], +) +def test_serp_requests(fake_api: FakeApi, kwargs: dict[str, Any], payload: dict[str, Any]) -> None: + content = {"results": {"organic": [{"title": "A"}]}} + fake_api.body = {"results": [{"status_code": 200, "content": content}]} + + result = DecodoSERPTool().run(**kwargs) + + assert fake_api.payload == payload + assert json.loads(result) == content + + @pytest.mark.live @pytest.mark.skipif(not os.environ.get("DECODO_LIVE_API_KEY"), reason="DECODO_LIVE_API_KEY not set") def test_live_scrape_and_search() -> None: @@ -150,3 +241,9 @@ def test_live_scrape_and_search() -> None: results = DecodoSearchTool(api_key=api_key).run(query="decodo") assert isinstance(results, str) and json.loads(results) + + product = DecodoEcommerceTool(api_key=api_key).run(store="walmart", product_id="15296401808") + assert isinstance(product, str) and json.loads(product) + + answer = DecodoSERPTool(api_key=api_key).run(query="what are solid state batteries", engine="google_ai_mode") + assert isinstance(answer, str) and json.loads(answer)