diff --git a/src/crawlee/_request.py b/src/crawlee/_request.py index 326e8ddce9..2ff22b459c 100644 --- a/src/crawlee/_request.py +++ b/src/crawlee/_request.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Annotated, Any, TypedDict, cast from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, PlainSerializer, PlainValidator, TypeAdapter +from pydantic.alias_generators import to_camel from yarl import URL from crawlee._types import EnqueueStrategy, HttpHeaders, HttpMethod, HttpPayload, JsonSerializable @@ -34,28 +35,28 @@ class RequestState(IntEnum): class CrawleeRequestData(BaseModel): """Crawlee-specific configuration stored in the `user_data`.""" - max_retries: Annotated[int | None, Field(alias='maxRetries', frozen=True)] = None + max_retries: Annotated[int | None, Field(frozen=True)] = None """Maximum number of retries for this request. Allows to override the global `max_request_retries` option of `BasicCrawler`.""" - enqueue_strategy: Annotated[EnqueueStrategy | None, Field(alias='enqueueStrategy')] = None + enqueue_strategy: Annotated[EnqueueStrategy | None, Field()] = None """The strategy that was used for enqueuing the request.""" state: RequestState = RequestState.UNPROCESSED """Describes the request's current lifecycle state.""" - session_rotation_count: Annotated[int | None, Field(alias='sessionRotationCount')] = None + session_rotation_count: Annotated[int | None, Field()] = None """The number of finished session rotations for this request.""" - skip_navigation: Annotated[bool, Field(alias='skipNavigation')] = False + skip_navigation: Annotated[bool, Field()] = False - last_proxy_tier: Annotated[int | None, Field(alias='lastProxyTier')] = None + last_proxy_tier: Annotated[int | None, Field()] = None """The last proxy tier used to process the request.""" forefront: Annotated[bool, Field()] = False """Indicate whether the request should be enqueued at the front of the queue.""" - crawl_depth: Annotated[int, Field(alias='crawlDepth')] = 0 + crawl_depth: Annotated[int, Field()] = 0 """The depth of the request in the crawl tree.""" session_id: Annotated[str | None, Field()] = None @@ -69,7 +70,7 @@ class UserData(BaseModel, MutableMapping[str, JsonSerializable]): values. """ - model_config = ConfigDict(extra='allow') + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True, extra='allow') __pydantic_extra__: dict[str, JsonSerializable] = Field(init=False) crawlee_data: Annotated[CrawleeRequestData | None, Field(alias='__crawlee')] = None @@ -166,9 +167,11 @@ class Request(BaseModel): ``` """ - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) - unique_key: Annotated[str, Field(alias='uniqueKey', frozen=True)] + unique_key: Annotated[str, Field(frozen=True)] """A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing to the same URL. @@ -228,16 +231,16 @@ class Request(BaseModel): request's scope, keeping them accessible on retries, failures etc. """ - retry_count: Annotated[int, Field(alias='retryCount')] = 0 + retry_count: Annotated[int, Field()] = 0 """Number of times the request has been retried.""" - no_retry: Annotated[bool, Field(alias='noRetry')] = False + no_retry: Annotated[bool, Field()] = False """If set to `True`, the request will not be retried in case of failure.""" - loaded_url: Annotated[str | None, BeforeValidator(validate_http_url), Field(alias='loadedUrl')] = None + loaded_url: Annotated[str | None, BeforeValidator(validate_http_url), Field()] = None """URL of the web page that was loaded. This can differ from the original URL in case of redirects.""" - handled_at: Annotated[datetime | None, Field(alias='handledAt')] = None + handled_at: Annotated[datetime | None, Field()] = None """Timestamp when the request was handled.""" @classmethod @@ -432,5 +435,5 @@ def was_already_handled(self) -> bool: class RequestWithLock(Request): """A crawling request with information about locks.""" - lock_expires_at: Annotated[datetime, Field(alias='lockExpiresAt')] + lock_expires_at: Annotated[datetime, Field()] """The timestamp when the lock expires.""" diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py index 45d0483679..1fcd8a29fc 100644 --- a/src/crawlee/_utils/system.py +++ b/src/crawlee/_utils/system.py @@ -8,6 +8,7 @@ import psutil from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PlainValidator +from pydantic.alias_generators import to_camel from crawlee._utils.byte_size import ByteSize from crawlee._utils.log import LoggerOnce @@ -124,9 +125,11 @@ def _get_child_used_memory(child: psutil.Process) -> int: class CpuInfo(BaseModel): """Information about the CPU usage.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) - used_ratio: Annotated[float, Field(alias='usedRatio')] + used_ratio: Annotated[float, Field()] """The ratio of CPU currently in use, represented as a float between 0 and 1.""" # Workaround for Pydantic and type checkers when using Annotated with default_factory @@ -147,7 +150,9 @@ class CpuInfo(BaseModel): class MemoryUsageInfo(BaseModel): """Information about the memory usage.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) current_size: Annotated[ ByteSize, @@ -180,7 +185,9 @@ class MemoryUsageInfo(BaseModel): class MemoryInfo(MemoryUsageInfo): """Information about system memory.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) total_size: Annotated[ ByteSize, PlainValidator(ByteSize.validate), PlainSerializer(lambda size: size.bytes), Field(alias='totalSize') diff --git a/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py b/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py index e03572515b..c2d9ef2ac9 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py @@ -11,6 +11,7 @@ from jaro import jaro_winkler_metric from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PlainValidator +from pydantic.alias_generators import to_camel from sklearn.linear_model import LogisticRegression from typing_extensions import override @@ -32,7 +33,9 @@ class RenderingTypePredictorState(BaseModel): - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) model: Annotated[ LogisticRegression, @@ -41,7 +44,7 @@ class RenderingTypePredictorState(BaseModel): PlainSerializer(sklearn_model_serializer), ] - labels_coefficients: Annotated[defaultdict[str, float], Field(alias='labelsCoefficients')] + labels_coefficients: Annotated[defaultdict[str, float], Field()] @docs_group('Other') diff --git a/src/crawlee/events/_types.py b/src/crawlee/events/_types.py index 090758a29b..e7b0d4890f 100644 --- a/src/crawlee/events/_types.py +++ b/src/crawlee/events/_types.py @@ -5,6 +5,7 @@ from typing import Annotated, Any, TypeVar from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel from crawlee._utils.docs import docs_group from crawlee._utils.models import timedelta_secs @@ -40,18 +41,22 @@ class Event(str, Enum): class EventPersistStateData(BaseModel): """Data for the persist state event.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) - is_migrating: Annotated[bool, Field(alias='isMigrating')] + is_migrating: Annotated[bool, Field()] @docs_group('Event data') class EventSystemInfoData(BaseModel): """Data for the system info event.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) - cpu_info: Annotated[CpuInfo, Field(alias='cpuInfo')] + cpu_info: Annotated[CpuInfo, Field()] memory_info: Annotated[ MemoryUsageInfo, Field(alias='memoryInfo'), @@ -62,7 +67,9 @@ class EventSystemInfoData(BaseModel): class EventMigratingData(BaseModel): """Data for the migrating event.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) # The remaining time in seconds before the migration is forced and the process is killed # Optional because it's not present when the event handler is called manually @@ -73,21 +80,27 @@ class EventMigratingData(BaseModel): class EventAbortingData(BaseModel): """Data for the aborting event.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) @docs_group('Event data') class EventExitData(BaseModel): """Data for the exit event.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) @docs_group('Event data') class EventCrawlerStatusData(BaseModel): """Data for the crawler status event.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) message: str """A message describing the current status of the crawler.""" diff --git a/src/crawlee/fingerprint_suite/_types.py b/src/crawlee/fingerprint_suite/_types.py index 2e09cf8a55..ce4a88974d 100644 --- a/src/crawlee/fingerprint_suite/_types.py +++ b/src/crawlee/fingerprint_suite/_types.py @@ -3,6 +3,7 @@ from typing import Annotated, Literal from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel SupportedOperatingSystems = Literal['windows', 'macos', 'linux', 'android', 'ios'] SupportedDevices = Literal['desktop', 'mobile'] @@ -11,32 +12,36 @@ class ScreenOptions(BaseModel): - model_config = ConfigDict(extra='forbid', validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, extra='forbid', validate_by_name=True, validate_by_alias=True + ) """Defines the screen constrains for the fingerprint generator.""" - min_width: Annotated[float | None, Field(alias='minWidth')] = None + min_width: Annotated[float | None, Field()] = None """Minimal screen width constraint for the fingerprint generator.""" - max_width: Annotated[float | None, Field(alias='maxWidth')] = None + max_width: Annotated[float | None, Field()] = None """Maximal screen width constraint for the fingerprint generator.""" - min_height: Annotated[float | None, Field(alias='minHeight')] = None + min_height: Annotated[float | None, Field()] = None """Minimal screen height constraint for the fingerprint generator.""" - max_height: Annotated[float | None, Field(alias='maxHeight')] = None + max_height: Annotated[float | None, Field()] = None """Maximal screen height constraint for the fingerprint generator.""" class HeaderGeneratorOptions(BaseModel): """Collection of header related attributes that can be used by the fingerprint generator.""" - model_config = ConfigDict(extra='forbid', validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, extra='forbid', validate_by_name=True, validate_by_alias=True + ) browsers: list[SupportedBrowserType] | None = None """List of BrowserSpecifications to generate the headers for.""" - operating_systems: Annotated[list[SupportedOperatingSystems] | None, Field(alias='operatingSystems')] = None + operating_systems: Annotated[list[SupportedOperatingSystems] | None, Field()] = None """List of operating systems to generate the headers for.""" devices: list[SupportedDevices] | None = None @@ -47,7 +52,7 @@ class HeaderGeneratorOptions(BaseModel): (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) request header in the language format accepted by that header, for example `en`, `en-US` or `de`.""" - http_version: Annotated[SupportedHttpVersion | None, Field(alias='httpVersion')] = None + http_version: Annotated[SupportedHttpVersion | None, Field()] = None """HTTP version to be used for header generation (the headers differ depending on the version).""" strict: bool | None = None diff --git a/src/crawlee/request_loaders/_request_list.py b/src/crawlee/request_loaders/_request_list.py index c4523ba2c8..57092e5a7e 100644 --- a/src/crawlee/request_loaders/_request_list.py +++ b/src/crawlee/request_loaders/_request_list.py @@ -7,6 +7,7 @@ from typing import Annotated from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic.alias_generators import to_camel from typing_extensions import override from crawlee._request import Request @@ -17,11 +18,13 @@ class RequestListState(BaseModel): - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) - next_index: Annotated[int, Field(alias='nextIndex')] = 0 - next_unique_key: Annotated[str | None, Field(alias='nextUniqueKey')] = None - in_progress: Annotated[set[str], Field(alias='inProgress')] = set() + next_index: Annotated[int, Field()] = 0 + next_unique_key: Annotated[str | None, Field()] = None + in_progress: Annotated[set[str], Field()] = set() class RequestListData(BaseModel): diff --git a/src/crawlee/request_loaders/_sitemap_request_loader.py b/src/crawlee/request_loaders/_sitemap_request_loader.py index 230c50affc..56a72d0e5d 100644 --- a/src/crawlee/request_loaders/_sitemap_request_loader.py +++ b/src/crawlee/request_loaders/_sitemap_request_loader.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Annotated, Any from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel from typing_extensions import override from yarl import URL @@ -67,36 +68,38 @@ class SitemapRequestLoaderState(BaseModel): `in_progress` is cleared. """ - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) - url_queue: Annotated[deque[str], Field(alias='urlQueue')] + url_queue: Annotated[deque[str], Field()] """Queue of URLs extracted from sitemaps and ready for processing.""" - in_progress: Annotated[set[str], Field(alias='inProgress')] = set() + in_progress: Annotated[set[str], Field()] = set() """Set of request URLs currently being processed.""" - pending_sitemap_urls: Annotated[deque[str], Field(alias='pendingSitemapUrls')] + pending_sitemap_urls: Annotated[deque[str], Field()] """Queue of sitemap URLs that need to be fetched and processed.""" - in_progress_sitemap_url: Annotated[str | None, Field(alias='inProgressSitemapUrl')] = None + in_progress_sitemap_url: Annotated[str | None, Field()] = None """The sitemap URL currently being processed.""" - current_sitemap_processed_urls: Annotated[set[str], Field(alias='currentSitemapProcessedUrls')] = set() + current_sitemap_processed_urls: Annotated[set[str], Field()] = set() """URLs from the current sitemap that have been added to the queue.""" - processed_sitemap_urls: Annotated[set[str], Field(alias='processedSitemapUrls')] = set() + processed_sitemap_urls: Annotated[set[str], Field()] = set() """Set of processed sitemap URLs.""" - sitemap_depths: Annotated[dict[str, int], Field(alias='sitemapDepths')] = {} + sitemap_depths: Annotated[dict[str, int], Field()] = {} """Nesting depth of each known sitemap URL, used to bound how far nested sitemap chains are followed.""" completed: Annotated[bool, Field(alias='sitemapCompleted')] = False """Whether all sitemaps have been fully processed.""" - total_count: Annotated[int, Field(alias='totalCount')] = 0 + total_count: Annotated[int, Field()] = 0 """Total number of URLs found and added to the queue from all processed sitemaps.""" - handled_count: Annotated[int, Field(alias='handledCount')] = 0 + handled_count: Annotated[int, Field()] = 0 """Number of URLs that have been successfully handled.""" diff --git a/src/crawlee/sessions/_models.py b/src/crawlee/sessions/_models.py index fa0b64042c..54ab29afc9 100644 --- a/src/crawlee/sessions/_models.py +++ b/src/crawlee/sessions/_models.py @@ -13,6 +13,7 @@ PlainSerializer, computed_field, ) +from pydantic.alias_generators import to_camel from crawlee._types import JsonSerializable @@ -23,27 +24,31 @@ class SessionModel(BaseModel): """Model for a Session object.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) - id: Annotated[str, Field(alias='id')] - max_age: Annotated[timedelta, Field(alias='maxAge')] - user_data: Annotated[MutableMapping[str, JsonSerializable], Field(alias='userData')] - max_error_score: Annotated[float, Field(alias='maxErrorScore')] - error_score_decrement: Annotated[float, Field(alias='errorScoreDecrement')] - created_at: Annotated[datetime, Field(alias='createdAt')] - usage_count: Annotated[int, Field(alias='usageCount')] - max_usage_count: Annotated[int, Field(alias='maxUsageCount')] - error_score: Annotated[float, Field(alias='errorScore')] - cookies: Annotated[list[CookieParam], Field(alias='cookies')] - blocked_status_codes: Annotated[list[int], Field(alias='blockedStatusCodes')] + id: Annotated[str, Field()] + max_age: Annotated[timedelta, Field()] + user_data: Annotated[MutableMapping[str, JsonSerializable], Field()] + max_error_score: Annotated[float, Field()] + error_score_decrement: Annotated[float, Field()] + created_at: Annotated[datetime, Field()] + usage_count: Annotated[int, Field()] + max_usage_count: Annotated[int, Field()] + error_score: Annotated[float, Field()] + cookies: Annotated[list[CookieParam], Field()] + blocked_status_codes: Annotated[list[int], Field()] class SessionPoolModel(BaseModel): """Model for a SessionPool object.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, validate_by_name=True, validate_by_alias=True + ) - max_pool_size: Annotated[int, Field(alias='maxPoolSize')] + max_pool_size: Annotated[int, Field()] sessions: Annotated[ dict[ diff --git a/src/crawlee/statistics/_models.py b/src/crawlee/statistics/_models.py index 428332f23e..f1d7f3396e 100644 --- a/src/crawlee/statistics/_models.py +++ b/src/crawlee/statistics/_models.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Annotated, Any from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PlainValidator, computed_field +from pydantic.alias_generators import to_camel from typing_extensions import override from crawlee._utils.console import make_table @@ -58,14 +59,20 @@ def __str__(self) -> str: class StatisticsState(BaseModel): """Statistic data about a crawler run.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, ser_json_inf_nan='constants') - stats_id: Annotated[int | None, Field(alias='statsId')] = None + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, + ser_json_inf_nan='constants', + ) + stats_id: Annotated[int | None, Field()] = None - requests_finished: Annotated[int, Field(alias='requestsFinished')] = 0 - requests_failed: Annotated[int, Field(alias='requestsFailed')] = 0 - requests_retries: Annotated[int, Field(alias='requestsRetries')] = 0 - requests_failed_per_minute: Annotated[float, Field(alias='requestsFailedPerMinute')] = 0 - requests_finished_per_minute: Annotated[float, Field(alias='requestsFinishedPerMinute')] = 0 + requests_finished: Annotated[int, Field()] = 0 + requests_failed: Annotated[int, Field()] = 0 + requests_retries: Annotated[int, Field()] = 0 + requests_failed_per_minute: Annotated[float, Field()] = 0 + requests_finished_per_minute: Annotated[float, Field()] = 0 request_min_duration: Annotated[timedelta_ms | None, Field(alias='requestMinDurationMillis')] = None request_max_duration: Annotated[timedelta_ms | None, Field(alias='requestMaxDurationMillis')] = None request_total_failed_duration: Annotated[timedelta_ms, Field(alias='requestTotalFailedDurationMillis')] = ( @@ -74,9 +81,9 @@ class StatisticsState(BaseModel): request_total_finished_duration: Annotated[timedelta_ms, Field(alias='requestTotalFinishedDurationMillis')] = ( timedelta() ) - crawler_started_at: Annotated[datetime | None, Field(alias='crawlerStartedAt')] = None + crawler_started_at: Annotated[datetime | None, Field()] = None crawler_last_started_at: Annotated[datetime | None, Field(alias='crawlerLastStartTimestamp')] = None - crawler_finished_at: Annotated[datetime | None, Field(alias='crawlerFinishedAt')] = None + crawler_finished_at: Annotated[datetime | None, Field()] = None # Workaround for Pydantic and type checkers when using Annotated with default_factory if TYPE_CHECKING: @@ -85,7 +92,7 @@ class StatisticsState(BaseModel): requests_with_status_code: dict[str, int] = {} else: errors: Annotated[dict[str, Any], Field(default_factory=dict)] - retry_errors: Annotated[dict[str, Any], Field(alias='retryErrors', default_factory=dict)] + retry_errors: Annotated[dict[str, Any], Field(default_factory=dict)] requests_with_status_code: Annotated[ dict[str, int], Field(alias='requestsWithStatusCode', default_factory=dict), diff --git a/src/crawlee/storage_clients/models.py b/src/crawlee/storage_clients/models.py index 1725340a53..9dd3bbd284 100644 --- a/src/crawlee/storage_clients/models.py +++ b/src/crawlee/storage_clients/models.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Annotated, Any, Generic from pydantic import BaseModel, BeforeValidator, ConfigDict, Field +from pydantic.alias_generators import to_camel from typing_extensions import TypeVar from crawlee._types import HttpMethod, JsonSerializable @@ -23,21 +24,28 @@ class StorageMetadata(BaseModel): It contains common fields shared across all specific storage types. """ - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, extra='allow', from_attributes=True) + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, + extra='allow', + from_attributes=True, + ) - id: Annotated[str, Field(alias='id')] + id: Annotated[str, Field()] """The unique identifier of the storage.""" - name: Annotated[str | None, Field(alias='name', default=None)] + name: Annotated[str | None, Field(default=None)] """The name of the storage.""" - accessed_at: Annotated[datetime, Field(alias='accessedAt')] + accessed_at: Annotated[datetime, Field()] """The timestamp when the storage was last accessed.""" - created_at: Annotated[datetime, Field(alias='createdAt')] + created_at: Annotated[datetime, Field()] """The timestamp when the storage was created.""" - modified_at: Annotated[datetime, Field(alias='modifiedAt')] + modified_at: Annotated[datetime, Field()] """The timestamp when the storage was last modified.""" @@ -45,9 +53,15 @@ class StorageMetadata(BaseModel): class DatasetMetadata(StorageMetadata): """Model for a dataset metadata.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True) + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, + from_attributes=True, + ) - item_count: Annotated[int, Field(alias='itemCount')] + item_count: Annotated[int, Field()] """The number of items in the dataset.""" @@ -55,25 +69,37 @@ class DatasetMetadata(StorageMetadata): class KeyValueStoreMetadata(StorageMetadata): """Model for a key-value store metadata.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True) + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, + from_attributes=True, + ) @docs_group('Storage data') class RequestQueueMetadata(StorageMetadata): """Model for a request queue metadata.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True) + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, + from_attributes=True, + ) - had_multiple_clients: Annotated[bool, Field(alias='hadMultipleClients')] + had_multiple_clients: Annotated[bool, Field()] """Indicates whether the queue has been accessed by multiple clients (consumers).""" - handled_request_count: Annotated[int, Field(alias='handledRequestCount')] + handled_request_count: Annotated[int, Field()] """The number of requests that have been handled from the queue.""" - pending_request_count: Annotated[int, Field(alias='pendingRequestCount')] + pending_request_count: Annotated[int, Field()] """The number of requests that are still pending in the queue.""" - total_request_count: Annotated[int, Field(alias='totalRequestCount')] + total_request_count: Annotated[int, Field()] """The total number of requests that have been added to the queue.""" @@ -81,21 +107,27 @@ class RequestQueueMetadata(StorageMetadata): class KeyValueStoreRecordMetadata(BaseModel): """Model for a key-value store record metadata.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True) + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, + from_attributes=True, + ) - key: Annotated[str, Field(alias='key')] + key: Annotated[str, Field()] """The key of the record. A unique identifier for the record in the key-value store. """ - content_type: Annotated[str, Field(alias='contentType')] + content_type: Annotated[str, Field()] """The MIME type of the record. Describe the format and type of data stored in the record, following the MIME specification. """ - size: Annotated[int | None, Field(alias='size', default=None)] = None + size: Annotated[int | None, Field(default=None)] = None """The size of the record in bytes.""" @@ -103,9 +135,15 @@ class KeyValueStoreRecordMetadata(BaseModel): class KeyValueStoreRecord(KeyValueStoreRecordMetadata, Generic[KvsValueType]): """Model for a key-value store record.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True) + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, + from_attributes=True, + ) - value: Annotated[KvsValueType, Field(alias='value')] + value: Annotated[KvsValueType, Field()] """The value of the record.""" @@ -113,7 +151,13 @@ class KeyValueStoreRecord(KeyValueStoreRecordMetadata, Generic[KvsValueType]): class DatasetItemsListPage(BaseModel): """Model for a single page of dataset items returned from a collection list method.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True) + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, + from_attributes=True, + ) count: Annotated[int, Field(default=0)] """The number of objects returned on this page.""" @@ -143,23 +187,35 @@ class DatasetItemsListPage(BaseModel): class ProcessedRequest(BaseModel): """Represents a processed request.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True) + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, + from_attributes=True, + ) id: Annotated[str | None, Field(alias='requestId', default=None)] = None """Internal representation of the request by the storage client. Only some clients use id.""" - unique_key: Annotated[str, Field(alias='uniqueKey')] - was_already_present: Annotated[bool, Field(alias='wasAlreadyPresent')] - was_already_handled: Annotated[bool, Field(alias='wasAlreadyHandled')] + unique_key: Annotated[str, Field()] + was_already_present: Annotated[bool, Field()] + was_already_handled: Annotated[bool, Field()] @docs_group('Storage data') class UnprocessedRequest(BaseModel): """Represents an unprocessed request.""" - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True) + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, + from_attributes=True, + ) - unique_key: Annotated[str, Field(alias='uniqueKey')] + unique_key: Annotated[str, Field()] url: Annotated[str, BeforeValidator(validate_http_url), Field()] method: Annotated[HttpMethod | None, Field()] = None @@ -173,11 +229,17 @@ class AddRequestsResponse(BaseModel): encountered issues during processing. """ - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True) + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + validate_by_name=True, + validate_by_alias=True, + from_attributes=True, + ) - processed_requests: Annotated[list[ProcessedRequest], Field(alias='processedRequests')] + processed_requests: Annotated[list[ProcessedRequest], Field()] """Successfully processed requests, including information about whether they were already present in the queue and whether they had been handled previously.""" - unprocessed_requests: Annotated[list[UnprocessedRequest], Field(alias='unprocessedRequests')] + unprocessed_requests: Annotated[list[UnprocessedRequest], Field()] """Requests that could not be processed, typically due to validation errors or other issues."""