chore: add ecommerce and serp tools

README.md

@@ -9,6 +9,8 @@
 | --- | --- |
 | `DecodoWebScraperTool` | Scrapes any public URL and returns the page as Markdown. |
 | `DecodoSearchTool` | Searches Google or Amazon and returns the parsed results as JSON. |
+| `DecodoEcommerceTool` | Gets product details or search results from Amazon, Walmart or Target as JSON. |
+| `DecodoSERPTool` | Gets Google or Bing results pages, with paging, or a Google AI Mode answer as JSON. |
 
 ## Requirements
 
@@ -99,6 +101,30 @@ Arguments the agent fills in:
 | `source` | `"google"` or `"amazon"`, optional | `google` (default) for web results, `amazon` for product listings. |
 | `geo` | `str`, optional | Google: a country name, e.g. `United States`. Amazon: a US ZIP code, e.g. `10001`, or a two-letter country code for delivery outside the US, e.g. `DE`. |
 
+### DecodoEcommerceTool
+
+Arguments the agent fills in. Pass either `product_id` or `query`:
+
+| Argument | Type | Description |
+| --- | --- | --- |
+| `store` | `"amazon"`, `"walmart"` or `"target"` | Store to look up. |
+| `product_id` | `str`, optional | One product's details. Amazon: the ASIN, e.g. `B09H74FXNW`. Walmart or Target: the product ID from the product URL. |
+| `query` | `str`, optional | Search the store's products instead. |
+| `zip_code` | `str`, optional | US ZIP code for prices and availability at that delivery location, e.g. `10001`. |
+
+### DecodoSERPTool
+
+Arguments the agent fills in:
+
+| Argument | Type | Description |
+| --- | --- | --- |
+| `query` | `str` | Search query. |
+| `engine` | `"google"`, `"bing"` or `"google_ai_mode"`, optional | `google` (default) or `bing` for a results page; `google_ai_mode` for Google's AI-written answer and the sources it cites. |
+| `page` | `int`, optional | Results page to fetch, starting at 1. Ignored by `google_ai_mode`. |
+| `geo` | `str`, optional | Country to search from, e.g. `United States`. |
+
+Bing results are rendered in a headless browser and can take over a minute.
+
 ## Errors
 
 When a request fails, for example because of an invalid API key, a rate limit or a page that returns `404`, the tools return a CrewAI `ToolFailure` instead of raising. The agent sees what went wrong and can retry or change course, and CrewAI records the failure according to the agent's `tool_failure_policy`.

src/decodo_crewai/__init__.py

@@ -1,5 +1,7 @@
 from ._base import DecodoBaseTool
+from .ecommerce_tool import DecodoEcommerceTool
 from .search_tool import DecodoSearchTool
+from .serp_tool import DecodoSERPTool
 from .web_scraper_tool import DecodoWebScraperTool
 
-__all__ = ["DecodoBaseTool", "DecodoSearchTool", "DecodoWebScraperTool"]
+__all__ = ["DecodoBaseTool", "DecodoEcommerceTool", "DecodoSERPTool", "DecodoSearchTool", "DecodoWebScraperTool"]

src/decodo_crewai/_base.py

@@ -77,6 +77,13 @@ def _scrape(self, params: ScrapeRequest) -> Any:
 
         results = response.get("results") or []
         if not results:
+            failed: dict[str, Any] = dict(response)
+            if failed.get("message"):
+                return ToolFailure(
+                    message=f"Decodo could not scrape the target: {failed['message']}",
+                    code=str(failed.get("status_code", "scrape_failed")),
+                    retryable=True,
+                )
             return ToolFailure(message="Decodo returned no results.", code="empty_response")
 
         result = results[0]

src/decodo_crewai/ecommerce_tool.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import json
+from typing import Literal
+
+from crewai.tools.tool_failure import ToolFailure
+from decodo import (
+    AmazonProductParams,
+    AmazonSearchParams,
+    ScrapeRequest,
+    TargetProductParams,
+    TargetSearchParams,
+    WalmartProductParams,
+    WalmartSearchParams,
+)
+from pydantic import BaseModel, Field, model_validator
+
+from ._base import DecodoBaseTool
+
+Store = Literal["amazon", "walmart", "target"]
+
+
+class DecodoEcommerceArgs(BaseModel):
+    store: Store = Field(description="Store to look up: 'amazon', 'walmart' or 'target'")
+    product_id: str | None = Field(
+        default=None,
+        description=(
+            "Get one product's details. Amazon: the ASIN, e.g. 'B09H74FXNW'. "
+            "Walmart or Target: the product ID from the product URL, e.g. '15296401808'"
+        ),
+    )
+    query: str | None = Field(default=None, description="Search the store's products for this query instead")
+    zip_code: str | None = Field(
+        default=None,
+        description="US ZIP code to see prices and availability for that delivery location, e.g. '10001'",
+    )
+
+    @model_validator(mode="after")
+    def _one_of_product_id_or_query(self) -> DecodoEcommerceArgs:
+        if bool(self.product_id) == bool(self.query):
+            raise ValueError("Pass either product_id or query, not both.")
+        return self
+
+
+class DecodoEcommerceTool(DecodoBaseTool):
+    """Get product details or search results from Amazon, Walmart or Target, parsed as JSON.
+
+    Args:
+        api_key: Decodo Web Scraping API key. Defaults to the ``DECODO_API_KEY`` environment variable.
+    """
+
+    name: str = "Decodo Ecommerce"
+    description: str = (
+        "Look up products on Amazon, Walmart or Target and return structured JSON. "
+        "Pass product_id for one product's details: title, price, rating, seller, availability. "
+        "Pass query to search the store and get matching products with prices."
+    )
+    args_schema: type[BaseModel] = DecodoEcommerceArgs
+
+    def _run(
+        self,
+        store: Store,
+        product_id: str | None = None,
+        query: str | None = None,
+        zip_code: str | None = None,
+    ) -> str | ToolFailure:
+        if bool(product_id) == bool(query):
+            return ToolFailure(message="Pass either product_id or query, not both.", code="invalid_params")
+
+        zip_code = zip_code or None
+        params: ScrapeRequest
+        if store == "amazon":
+            params = (
+                AmazonProductParams(query=product_id, geo=zip_code, parse=True)
+                if product_id
+                else AmazonSearchParams(query=query, geo=zip_code, parse=True)
+            )
+        elif store == "walmart":
+            params = (
+                WalmartProductParams(product_id=product_id, delivery_zip=zip_code, parse=True)
+                if product_id
+                else WalmartSearchParams(query=query, delivery_zip=zip_code, parse=True)
+            )
+        else:
+            # Target pages need JavaScript rendering to load product data.
+            params = (
+                TargetProductParams(product_id=product_id, delivery_zip=zip_code, headless="html", parse=True)
+                if product_id
+                else TargetSearchParams(query=query, delivery_zip=zip_code, headless="html", parse=True)
+            )
+
+        content = self._scrape(params)
+        if isinstance(content, ToolFailure | str):
+            return content
+        return json.dumps(content)

src/decodo_crewai/serp_tool.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import json
+from typing import Literal
+
+from crewai.tools.tool_failure import ToolFailure
+from decodo import BingSearchParams, GoogleAiModeParams, GoogleSearchParams, ScrapeRequest
+from pydantic import BaseModel, Field
+
+from ._base import DecodoBaseTool
+
+Engine = Literal["google", "bing", "google_ai_mode"]
+
+
+class DecodoSERPArgs(BaseModel):
+    query: str = Field(description="Search query")
+    engine: Engine = Field(
+        default="google",
+        description=(
+            "'google' or 'bing' for a results page; 'google_ai_mode' for Google's AI-written answer "
+            "with the links it cites"
+        ),
+    )
+    page: int = Field(default=1, ge=1, description="Results page to fetch, starting at 1. Ignored by google_ai_mode")
+    geo: str | None = Field(
+        default=None,
+        description="Country to search from, as a country name, e.g. 'United States'",
+    )
+
+
+class DecodoSERPTool(DecodoBaseTool):
+    """Search Google or Bing, or ask Google AI Mode, and return the parsed results as JSON.
+
+    Args:
+        api_key: Decodo Web Scraping API key. Defaults to the ``DECODO_API_KEY`` environment variable.
+    """
+
+    name: str = "Decodo SERP"
+    description: str = (
+        "Get search engine results as structured JSON. "
+        "Use engine='google' or engine='bing' for a full results page, with page for deeper results. "
+        "Use engine='google_ai_mode' for Google's AI-written answer and the sources it cites."
+    )
+    args_schema: type[BaseModel] = DecodoSERPArgs
+
+    def _run(
+        self,
+        query: str,
+        engine: Engine = "google",
+        page: int = 1,
+        geo: str | None = None,
+    ) -> str | ToolFailure:
+        geo = geo or None
+        page_from = page if page > 1 else None
+        params: ScrapeRequest
+        if engine == "google_ai_mode":
+            params = GoogleAiModeParams(query=query, geo=geo, parse=True)
+        elif engine == "bing":
+            # Without JavaScript rendering, Bing fails with status 613 (target could not be scraped).
+            params = BingSearchParams(query=query, geo=geo, page_from=page_from, headless="html", parse=True)
+        else:
+            params = GoogleSearchParams(query=query, geo=geo, page_from=page_from, parse=True)
+
+        content = self._scrape(params)
+        if isinstance(content, ToolFailure | str):
+            return content
+        return json.dumps(content)

tests/test_tools.py

@@ -8,7 +8,8 @@
 import pytest
 from crewai.tools.tool_failure import ToolFailure
 
-from decodo_crewai import DecodoSearchTool, DecodoWebScraperTool
+from decodo_crewai import DecodoEcommerceTool, DecodoSearchTool, DecodoSERPTool, DecodoWebScraperTool
+from decodo_crewai.ecommerce_tool import DecodoEcommerceArgs
 
 
 class FakeApi:
@@ -140,6 +141,96 @@ def test_empty_results_become_tool_failure(fake_api: FakeApi) -> None:
     assert result.code == "empty_response"
 
 
+def test_failed_task_becomes_retryable_tool_failure(fake_api: FakeApi) -> None:
+    fake_api.body = {"status": "failed", "status_code": 613, "message": "We were not able to scrape the target."}
+
+    result = DecodoWebScraperTool()._run(url="https://example.com")
+
+    assert isinstance(result, ToolFailure)
+    assert result.code == "613"
+    assert result.retryable is True
+    assert "We were not able to scrape the target." in result.message
+
+
+@pytest.mark.parametrize(
+    ("kwargs", "payload"),
+    [
+        (
+            {"store": "amazon", "product_id": "B09H74FXNW", "zip_code": "10001"},
+            {"target": "amazon_product", "query": "B09H74FXNW", "geo": "10001", "parse": True},
+        ),
+        (
+            {"store": "amazon", "query": "usb c cable"},
+            {"target": "amazon_search", "query": "usb c cable", "parse": True},
+        ),
+        (
+            {"store": "walmart", "product_id": "15296401808"},
+            {"target": "walmart_product", "product_id": "15296401808", "parse": True},
+        ),
+        (
+            {"store": "walmart", "query": "camping tent", "zip_code": "10001"},
+            {"target": "walmart_search", "query": "camping tent", "delivery_zip": "10001", "parse": True},
+        ),
+        (
+            {"store": "target", "product_id": "1003921355"},
+            {"target": "target_product", "product_id": "1003921355", "headless": "html", "parse": True},
+        ),
+        (
+            {"store": "target", "query": "camping tent"},
+            {"target": "target_search", "query": "camping tent", "headless": "html", "parse": True},
+        ),
+    ],
+)
+def test_ecommerce_requests(fake_api: FakeApi, kwargs: dict[str, Any], payload: dict[str, Any]) -> None:
+    content = {"title": "A product", "price": 9.99}
+    fake_api.body = {"results": [{"status_code": 200, "content": content}]}
+
+    result = DecodoEcommerceTool().run(**kwargs)
+
+    assert fake_api.payload == payload
+    assert json.loads(result) == content
+
+
+@pytest.mark.parametrize("kwargs", [{}, {"product_id": "B09H74FXNW", "query": "usb c cable"}])
+def test_ecommerce_needs_exactly_one_of_product_id_or_query(fake_api: FakeApi, kwargs: dict[str, Any]) -> None:
+    with pytest.raises(ValueError, match="either product_id or query"):
+        DecodoEcommerceArgs(store="amazon", **kwargs)
+
+    result = DecodoEcommerceTool()._run(store="amazon", **kwargs)
+
+    assert isinstance(result, ToolFailure)
+    assert result.code == "invalid_params"
+    assert not fake_api.calls
+
+
+@pytest.mark.parametrize(
+    ("kwargs", "payload"),
+    [
+        ({"query": "batteries"}, {"target": "google_search", "query": "batteries", "parse": True}),
+        (
+            {"query": "batteries", "page": 2, "geo": "Germany"},
+            {"target": "google_search", "query": "batteries", "geo": "Germany", "page_from": 2, "parse": True},
+        ),
+        (
+            {"query": "batteries", "engine": "bing", "page": 3},
+            {"target": "bing_search", "query": "batteries", "page_from": 3, "headless": "html", "parse": True},
+        ),
+        (
+            {"query": "what are batteries", "engine": "google_ai_mode", "page": 2},
+            {"target": "google_ai_mode", "query": "what are batteries", "parse": True},
+        ),
+    ],
+)
+def test_serp_requests(fake_api: FakeApi, kwargs: dict[str, Any], payload: dict[str, Any]) -> None:
+    content = {"results": {"organic": [{"title": "A"}]}}
+    fake_api.body = {"results": [{"status_code": 200, "content": content}]}
+
+    result = DecodoSERPTool().run(**kwargs)
+
+    assert fake_api.payload == payload
+    assert json.loads(result) == content
+
+
 @pytest.mark.live
 @pytest.mark.skipif(not os.environ.get("DECODO_LIVE_API_KEY"), reason="DECODO_LIVE_API_KEY not set")
 def test_live_scrape_and_search() -> None:
@@ -150,3 +241,9 @@ def test_live_scrape_and_search() -> None:
 
     results = DecodoSearchTool(api_key=api_key).run(query="decodo")
     assert isinstance(results, str) and json.loads(results)
+
+    product = DecodoEcommerceTool(api_key=api_key).run(store="walmart", product_id="15296401808")
+    assert isinstance(product, str) and json.loads(product)
+
+    answer = DecodoSERPTool(api_key=api_key).run(query="what are solid state batteries", engine="google_ai_mode")
+    assert isinstance(answer, str) and json.loads(answer)