Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 17 additions & 14 deletions src/crawlee/_request.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from typing import TYPE_CHECKING, Annotated, Any, TypedDict, cast

from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, PlainSerializer, PlainValidator, TypeAdapter
from pydantic.alias_generators import to_camel
from yarl import URL

from crawlee._types import EnqueueStrategy, HttpHeaders, HttpMethod, HttpPayload, JsonSerializable
Expand Down Expand Up @@ -34,28 +35,28 @@ class RequestState(IntEnum):
class CrawleeRequestData(BaseModel):
"""Crawlee-specific configuration stored in the `user_data`."""

max_retries: Annotated[int | None, Field(alias='maxRetries', frozen=True)] = None
max_retries: Annotated[int | None, Field(frozen=True)] = None
"""Maximum number of retries for this request. Allows to override the global `max_request_retries` option of
`BasicCrawler`."""

enqueue_strategy: Annotated[EnqueueStrategy | None, Field(alias='enqueueStrategy')] = None
enqueue_strategy: Annotated[EnqueueStrategy | None, Field()] = None
"""The strategy that was used for enqueuing the request."""

state: RequestState = RequestState.UNPROCESSED
"""Describes the request's current lifecycle state."""

session_rotation_count: Annotated[int | None, Field(alias='sessionRotationCount')] = None
session_rotation_count: Annotated[int | None, Field()] = None
"""The number of finished session rotations for this request."""

skip_navigation: Annotated[bool, Field(alias='skipNavigation')] = False
skip_navigation: Annotated[bool, Field()] = False

last_proxy_tier: Annotated[int | None, Field(alias='lastProxyTier')] = None
last_proxy_tier: Annotated[int | None, Field()] = None
"""The last proxy tier used to process the request."""

forefront: Annotated[bool, Field()] = False
"""Indicate whether the request should be enqueued at the front of the queue."""

crawl_depth: Annotated[int, Field(alias='crawlDepth')] = 0
crawl_depth: Annotated[int, Field()] = 0
"""The depth of the request in the crawl tree."""

session_id: Annotated[str | None, Field()] = None
Expand All @@ -69,7 +70,7 @@ class UserData(BaseModel, MutableMapping[str, JsonSerializable]):
values.
"""

model_config = ConfigDict(extra='allow')
model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True, extra='allow')
__pydantic_extra__: dict[str, JsonSerializable] = Field(init=False)

crawlee_data: Annotated[CrawleeRequestData | None, Field(alias='__crawlee')] = None
Expand Down Expand Up @@ -166,9 +167,11 @@ class Request(BaseModel):
```
"""

model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)

unique_key: Annotated[str, Field(alias='uniqueKey', frozen=True)]
unique_key: Annotated[str, Field(frozen=True)]
"""A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing
to the same URL.

Expand Down Expand Up @@ -228,16 +231,16 @@ class Request(BaseModel):
request's scope, keeping them accessible on retries, failures etc.
"""

retry_count: Annotated[int, Field(alias='retryCount')] = 0
retry_count: Annotated[int, Field()] = 0
"""Number of times the request has been retried."""

no_retry: Annotated[bool, Field(alias='noRetry')] = False
no_retry: Annotated[bool, Field()] = False
"""If set to `True`, the request will not be retried in case of failure."""

loaded_url: Annotated[str | None, BeforeValidator(validate_http_url), Field(alias='loadedUrl')] = None
loaded_url: Annotated[str | None, BeforeValidator(validate_http_url), Field()] = None
"""URL of the web page that was loaded. This can differ from the original URL in case of redirects."""

handled_at: Annotated[datetime | None, Field(alias='handledAt')] = None
handled_at: Annotated[datetime | None, Field()] = None
"""Timestamp when the request was handled."""

@classmethod
Expand Down Expand Up @@ -432,5 +435,5 @@ def was_already_handled(self) -> bool:
class RequestWithLock(Request):
"""A crawling request with information about locks."""

lock_expires_at: Annotated[datetime, Field(alias='lockExpiresAt')]
lock_expires_at: Annotated[datetime, Field()]
"""The timestamp when the lock expires."""
15 changes: 11 additions & 4 deletions src/crawlee/_utils/system.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import psutil
from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PlainValidator
from pydantic.alias_generators import to_camel

from crawlee._utils.byte_size import ByteSize
from crawlee._utils.log import LoggerOnce
Expand Down Expand Up @@ -124,9 +125,11 @@ def _get_child_used_memory(child: psutil.Process) -> int:
class CpuInfo(BaseModel):
"""Information about the CPU usage."""

model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)

used_ratio: Annotated[float, Field(alias='usedRatio')]
used_ratio: Annotated[float, Field()]
"""The ratio of CPU currently in use, represented as a float between 0 and 1."""

# Workaround for Pydantic and type checkers when using Annotated with default_factory
Expand All @@ -147,7 +150,9 @@ class CpuInfo(BaseModel):
class MemoryUsageInfo(BaseModel):
"""Information about the memory usage."""

model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)

current_size: Annotated[
ByteSize,
Expand Down Expand Up @@ -180,7 +185,9 @@ class MemoryUsageInfo(BaseModel):
class MemoryInfo(MemoryUsageInfo):
"""Information about system memory."""

model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)

total_size: Annotated[
ByteSize, PlainValidator(ByteSize.validate), PlainSerializer(lambda size: size.bytes), Field(alias='totalSize')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from jaro import jaro_winkler_metric
from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PlainValidator
from pydantic.alias_generators import to_camel
from sklearn.linear_model import LogisticRegression
from typing_extensions import override

Expand All @@ -32,7 +33,9 @@


class RenderingTypePredictorState(BaseModel):
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)

model: Annotated[
LogisticRegression,
Expand All @@ -41,7 +44,7 @@ class RenderingTypePredictorState(BaseModel):
PlainSerializer(sklearn_model_serializer),
]

labels_coefficients: Annotated[defaultdict[str, float], Field(alias='labelsCoefficients')]
labels_coefficients: Annotated[defaultdict[str, float], Field()]


@docs_group('Other')
Expand Down
29 changes: 21 additions & 8 deletions src/crawlee/events/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing import Annotated, Any, TypeVar

from pydantic import BaseModel, ConfigDict, Field
from pydantic.alias_generators import to_camel

from crawlee._utils.docs import docs_group
from crawlee._utils.models import timedelta_secs
Expand Down Expand Up @@ -40,18 +41,22 @@ class Event(str, Enum):
class EventPersistStateData(BaseModel):
"""Data for the persist state event."""

model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)

is_migrating: Annotated[bool, Field(alias='isMigrating')]
is_migrating: Annotated[bool, Field()]


@docs_group('Event data')
class EventSystemInfoData(BaseModel):
"""Data for the system info event."""

model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)

cpu_info: Annotated[CpuInfo, Field(alias='cpuInfo')]
cpu_info: Annotated[CpuInfo, Field()]
memory_info: Annotated[
MemoryUsageInfo,
Field(alias='memoryInfo'),
Expand All @@ -62,7 +67,9 @@ class EventSystemInfoData(BaseModel):
class EventMigratingData(BaseModel):
"""Data for the migrating event."""

model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)

# The remaining time in seconds before the migration is forced and the process is killed
# Optional because it's not present when the event handler is called manually
Expand All @@ -73,21 +80,27 @@ class EventMigratingData(BaseModel):
class EventAbortingData(BaseModel):
"""Data for the aborting event."""

model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)


@docs_group('Event data')
class EventExitData(BaseModel):
"""Data for the exit event."""

model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)


@docs_group('Event data')
class EventCrawlerStatusData(BaseModel):
"""Data for the crawler status event."""

model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)

message: str
"""A message describing the current status of the crawler."""
Expand Down
21 changes: 13 additions & 8 deletions src/crawlee/fingerprint_suite/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from typing import Annotated, Literal

from pydantic import BaseModel, ConfigDict, Field
from pydantic.alias_generators import to_camel

SupportedOperatingSystems = Literal['windows', 'macos', 'linux', 'android', 'ios']
SupportedDevices = Literal['desktop', 'mobile']
Expand All @@ -11,32 +12,36 @@


class ScreenOptions(BaseModel):
model_config = ConfigDict(extra='forbid', validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, extra='forbid', validate_by_name=True, validate_by_alias=True
)

"""Defines the screen constrains for the fingerprint generator."""

min_width: Annotated[float | None, Field(alias='minWidth')] = None
min_width: Annotated[float | None, Field()] = None
"""Minimal screen width constraint for the fingerprint generator."""

max_width: Annotated[float | None, Field(alias='maxWidth')] = None
max_width: Annotated[float | None, Field()] = None
"""Maximal screen width constraint for the fingerprint generator."""

min_height: Annotated[float | None, Field(alias='minHeight')] = None
min_height: Annotated[float | None, Field()] = None
"""Minimal screen height constraint for the fingerprint generator."""

max_height: Annotated[float | None, Field(alias='maxHeight')] = None
max_height: Annotated[float | None, Field()] = None
"""Maximal screen height constraint for the fingerprint generator."""


class HeaderGeneratorOptions(BaseModel):
"""Collection of header related attributes that can be used by the fingerprint generator."""

model_config = ConfigDict(extra='forbid', validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, extra='forbid', validate_by_name=True, validate_by_alias=True
)

browsers: list[SupportedBrowserType] | None = None
"""List of BrowserSpecifications to generate the headers for."""

operating_systems: Annotated[list[SupportedOperatingSystems] | None, Field(alias='operatingSystems')] = None
operating_systems: Annotated[list[SupportedOperatingSystems] | None, Field()] = None
"""List of operating systems to generate the headers for."""

devices: list[SupportedDevices] | None = None
Expand All @@ -47,7 +52,7 @@ class HeaderGeneratorOptions(BaseModel):
(https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) request header
in the language format accepted by that header, for example `en`, `en-US` or `de`."""

http_version: Annotated[SupportedHttpVersion | None, Field(alias='httpVersion')] = None
http_version: Annotated[SupportedHttpVersion | None, Field()] = None
"""HTTP version to be used for header generation (the headers differ depending on the version)."""

strict: bool | None = None
Expand Down
11 changes: 7 additions & 4 deletions src/crawlee/request_loaders/_request_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from typing import Annotated

from pydantic import BaseModel, ConfigDict, Field, ValidationError
from pydantic.alias_generators import to_camel
from typing_extensions import override

from crawlee._request import Request
Expand All @@ -17,11 +18,13 @@


class RequestListState(BaseModel):
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)

next_index: Annotated[int, Field(alias='nextIndex')] = 0
next_unique_key: Annotated[str | None, Field(alias='nextUniqueKey')] = None
in_progress: Annotated[set[str], Field(alias='inProgress')] = set()
next_index: Annotated[int, Field()] = 0
next_unique_key: Annotated[str | None, Field()] = None
in_progress: Annotated[set[str], Field()] = set()


class RequestListData(BaseModel):
Expand Down
23 changes: 13 additions & 10 deletions src/crawlee/request_loaders/_sitemap_request_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from typing import TYPE_CHECKING, Annotated, Any

from pydantic import BaseModel, ConfigDict, Field
from pydantic.alias_generators import to_camel
from typing_extensions import override
from yarl import URL

Expand Down Expand Up @@ -67,36 +68,38 @@ class SitemapRequestLoaderState(BaseModel):
`in_progress` is cleared.
"""

model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
model_config = ConfigDict(
alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True
)

url_queue: Annotated[deque[str], Field(alias='urlQueue')]
url_queue: Annotated[deque[str], Field()]
"""Queue of URLs extracted from sitemaps and ready for processing."""

in_progress: Annotated[set[str], Field(alias='inProgress')] = set()
in_progress: Annotated[set[str], Field()] = set()
"""Set of request URLs currently being processed."""

pending_sitemap_urls: Annotated[deque[str], Field(alias='pendingSitemapUrls')]
pending_sitemap_urls: Annotated[deque[str], Field()]
"""Queue of sitemap URLs that need to be fetched and processed."""

in_progress_sitemap_url: Annotated[str | None, Field(alias='inProgressSitemapUrl')] = None
in_progress_sitemap_url: Annotated[str | None, Field()] = None
"""The sitemap URL currently being processed."""

current_sitemap_processed_urls: Annotated[set[str], Field(alias='currentSitemapProcessedUrls')] = set()
current_sitemap_processed_urls: Annotated[set[str], Field()] = set()
"""URLs from the current sitemap that have been added to the queue."""

processed_sitemap_urls: Annotated[set[str], Field(alias='processedSitemapUrls')] = set()
processed_sitemap_urls: Annotated[set[str], Field()] = set()
"""Set of processed sitemap URLs."""

sitemap_depths: Annotated[dict[str, int], Field(alias='sitemapDepths')] = {}
sitemap_depths: Annotated[dict[str, int], Field()] = {}
"""Nesting depth of each known sitemap URL, used to bound how far nested sitemap chains are followed."""

completed: Annotated[bool, Field(alias='sitemapCompleted')] = False
"""Whether all sitemaps have been fully processed."""

total_count: Annotated[int, Field(alias='totalCount')] = 0
total_count: Annotated[int, Field()] = 0
"""Total number of URLs found and added to the queue from all processed sitemaps."""

handled_count: Annotated[int, Field(alias='handledCount')] = 0
handled_count: Annotated[int, Field()] = 0
"""Number of URLs that have been successfully handled."""


Expand Down
Loading