diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py index 279c27708..c5e1d86fe 100644 --- a/crawl4ai/utils.py +++ b/crawl4ai/utils.py @@ -1,3 +1,5 @@ +import ipaddress +import socket import time from concurrent.futures import ThreadPoolExecutor, as_completed from bs4 import BeautifulSoup, Comment, element, Tag, NavigableString @@ -50,6 +52,46 @@ import inspect +def _validate_public_http_url(url: str) -> tuple[str, int | None]: + """Return hostname/port only when the URL resolves to a public address.""" + parsed = urlparse(url) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname: + raise ValueError("robots URL must use http(s) and include a hostname") + if parsed.username is not None or parsed.password is not None: + raise ValueError("robots URL must not contain user information") + + try: + port = parsed.port + except ValueError as exc: + raise ValueError("robots URL contains an invalid port") from exc + + try: + addresses = [ipaddress.ip_address(parsed.hostname)] + except ValueError: + try: + addresses = [ + ipaddress.ip_address(info[4][0]) + for info in socket.getaddrinfo( + parsed.hostname, port, type=socket.SOCK_STREAM + ) + ] + except (OSError, ValueError) as exc: + raise ValueError("robots hostname could not be resolved") from exc + + if not addresses or any( + address.is_private + or address.is_loopback + or address.is_link_local + or address.is_reserved + or address.is_multicast + or address.is_unspecified + for address in addresses + ): + raise ValueError("robots URL resolves to a private or reserved address") + + return parsed.hostname, port + + # Monkey patch to fix wildcard handling in urllib.robotparser from urllib.robotparser import RuleLine import re @@ -327,7 +369,8 @@ async def can_fetch(self, url: str, user_agent: str = "*") -> bool: domain = parsed.netloc if not domain: return True - except Exception as _ex: + hostname, port = _validate_public_http_url(url) + except (TypeError, ValueError): return True # Fast path - check cache first @@ -336,12 +379,18 @@ async def can_fetch(self, url: str, user_agent: str = "*") -> bool: # If rules not found or stale, fetch new ones if not is_fresh: try: - # Ensure we use the same scheme as the input URL - scheme = parsed.scheme or 'http' - robots_url = f"{scheme}://{domain}/robots.txt" - + # Use the validated hostname/port and never follow a + # redirect to an address that was not checked above. + scheme = parsed.scheme.lower() + netloc = f"[{hostname}]" if ":" in hostname else hostname + if port: + netloc = f"{netloc}:{port}" + robots_url = f"{scheme}://{netloc}/robots.txt" + async with aiohttp.ClientSession() as session: - async with session.get(robots_url, timeout=2, ssl=False) as response: + async with session.get( + robots_url, timeout=2, allow_redirects=False + ) as response: if response.status == 200: rules = await response.text() self._cache_rules(domain, rules) diff --git a/docs/md_v2/marketplace/backend/database.py b/docs/md_v2/marketplace/backend/database.py index 8ccfbaf46..c4974f479 100644 --- a/docs/md_v2/marketplace/backend/database.py +++ b/docs/md_v2/marketplace/backend/database.py @@ -77,14 +77,44 @@ def _create_or_update_table(self, table_name: str, columns: Dict): self.conn.commit() - def get_all(self, table: str, limit: int = 100, offset: int = 0, where: str = None) -> List[Dict]: - cursor = self.conn.cursor() - query = f"SELECT * FROM {table}" - if where: - query += f" WHERE {where}" - query += f" LIMIT {limit} OFFSET {offset}" + def get_all( + self, + table: str, + limit: int = 100, + offset: int = 0, + where: Dict[str, Any] | None = None, + ) -> List[Dict]: + """Return rows using schema-validated equality filters. + + SQL identifiers cannot be bound as parameters, so table and filter + column names are validated against the loaded schema before they are + interpolated. Filter values, limits, and offsets remain bound + parameters; callers must not pass raw SQL fragments. + """ + if table not in self.schema["tables"]: + raise ValueError(f"Unknown table: {table}") + + filters = where or {} + if not isinstance(filters, dict): + raise TypeError("where must be a mapping of column names to values") + + allowed_columns = set(self.schema["tables"][table]["columns"]) + clauses = [] + params = [] + for column, value in filters.items(): + if column not in allowed_columns: + raise ValueError(f"Unknown filter column for {table}: {column}") + clauses.append(f'"{column}" = ?') + params.append(value) + + query = f'SELECT * FROM "{table}"' + if clauses: + query += " WHERE " + " AND ".join(clauses) + query += " LIMIT ? OFFSET ?" + params.extend([max(0, int(limit)), max(0, int(offset))]) - cursor.execute(query) + cursor = self.conn.cursor() + cursor.execute(query, params) rows = cursor.fetchall() return [dict(row) for row in rows] @@ -114,4 +144,4 @@ def search(self, query: str, tables: List[str] = None) -> Dict[str, List[Dict]]: def close(self): if self.conn: - self.conn.close() \ No newline at end of file + self.conn.close() diff --git a/docs/md_v2/marketplace/backend/server.py b/docs/md_v2/marketplace/backend/server.py index f4935eb6d..7a5f6bfe6 100644 --- a/docs/md_v2/marketplace/backend/server.py +++ b/docs/md_v2/marketplace/backend/server.py @@ -95,18 +95,17 @@ async def get_apps( offset: int = Query(default=0) ): """Get apps with optional filters""" - where_clauses = [] + filters = {} if category: - where_clauses.append(f"category = '{category}'") + filters["category"] = category if type: - where_clauses.append(f"type = '{type}'") + filters["type"] = type if featured is not None: - where_clauses.append(f"featured = {1 if featured else 0}") + filters["featured"] = int(featured) if sponsored is not None: - where_clauses.append(f"sponsored = {1 if sponsored else 0}") + filters["sponsored"] = int(sponsored) - where = " AND ".join(where_clauses) if where_clauses else None - apps = db.get_all('apps', limit=limit, offset=offset, where=where) + apps = db.get_all("apps", limit=limit, offset=offset, where=filters or None) # Parse JSON fields for app in apps: @@ -118,7 +117,7 @@ async def get_apps( @router.get("/apps/{slug}") async def get_app(slug: str): """Get single app by slug""" - apps = db.get_all('apps', where=f"slug = '{slug}'", limit=1) + apps = db.get_all("apps", where={"slug": slug}, limit=1) if not apps: raise HTTPException(status_code=404, detail="App not found") @@ -135,8 +134,8 @@ async def get_articles( offset: int = Query(default=0) ): """Get articles with optional category filter""" - where = f"category = '{category}'" if category else None - articles = db.get_all('articles', limit=limit, offset=offset, where=where) + filters = {"category": category} if category else None + articles = db.get_all("articles", limit=limit, offset=offset, where=filters) # Parse JSON fields for article in articles: @@ -150,7 +149,7 @@ async def get_articles( @router.get("/articles/{slug}") async def get_article(slug: str): """Get single article by slug""" - articles = db.get_all('articles', where=f"slug = '{slug}'", limit=1) + articles = db.get_all("articles", where={"slug": slug}, limit=1) if not articles: raise HTTPException(status_code=404, detail="Article not found") @@ -174,8 +173,8 @@ async def get_categories(): @router.get("/sponsors") async def get_sponsors(active: Optional[bool] = True): """Get sponsors, default active only""" - where = f"active = {1 if active else 0}" if active is not None else None - sponsors = db.get_all('sponsors', where=where, limit=20) + filters = {"active": int(active)} if active is not None else None + sponsors = db.get_all("sponsors", where=filters, limit=20) # Filter by date if active if active: @@ -211,10 +210,10 @@ async def search(q: str = Query(min_length=2)): async def get_stats(): """Get marketplace statistics""" stats = { - "total_apps": len(db.get_all('apps', limit=10000)), - "total_articles": len(db.get_all('articles', limit=10000)), - "total_categories": len(db.get_all('categories', limit=1000)), - "active_sponsors": len(db.get_all('sponsors', where="active = 1", limit=1000)) + "total_apps": len(db.get_all("apps", limit=10000)), + "total_articles": len(db.get_all("articles", limit=10000)), + "total_categories": len(db.get_all("categories", limit=1000)), + "active_sponsors": len(db.get_all("sponsors", where={"active": 1}, limit=1000)), } return json_response(stats, cache_time=1800) @@ -279,13 +278,13 @@ async def get_admin_stats(): stats = { "apps": { "total": len(db.get_all('apps', limit=10000)), - "featured": len(db.get_all('apps', where="featured = 1", limit=10000)), - "sponsored": len(db.get_all('apps', where="sponsored = 1", limit=10000)) + "featured": len(db.get_all("apps", where={"featured": 1}, limit=10000)), + "sponsored": len(db.get_all("apps", where={"sponsored": 1}, limit=10000)) }, "articles": len(db.get_all('articles', limit=10000)), "categories": len(db.get_all('categories', limit=1000)), "sponsors": { - "active": len(db.get_all('sponsors', where="active = 1", limit=1000)), + "active": len(db.get_all("sponsors", where={"active": 1}, limit=1000)), "total": len(db.get_all('sponsors', limit=10000)) }, "total_views": sum(app.get('views', 0) for app in db.get_all('apps', limit=10000)) @@ -494,4 +493,4 @@ async def root(): if __name__ == "__main__": import uvicorn - uvicorn.run(app, host="127.0.0.1", port=8100) \ No newline at end of file + uvicorn.run(app, host="127.0.0.1", port=8100)