diff --git a/.github/workflows/rust-integration-tests.yml b/.github/workflows/rust-integration-tests.yml index 85e0970..8655932 100644 --- a/.github/workflows/rust-integration-tests.yml +++ b/.github/workflows/rust-integration-tests.yml @@ -9,8 +9,10 @@ on: - 'src/**' - 'tests/**' - 'examples/**' + - 'README.md' + - 'docs/**' + - 'build.rs' - 'Cargo.toml' - - 'Cargo.lock' - '.github/workflows/rust-integration-tests.yml' workflow_dispatch: @@ -69,20 +71,45 @@ jobs: # programs in `tests/examples.rs` (test names prefixed `example_`) are exercised by the # standalone `Test examples` step below, so they are skipped here to keep the two # concerns separate. - run: cargo test --lib --tests --verbose -- --skip example_ --test-threads=4 + # + # Defense-in-depth beyond the `APIFY_TOKEN` guard above: `set -o pipefail` plus the + # trailing grep make the step fail if cargo reports zero executed tests (e.g. a filter + # typo silently matching nothing), not just if a test fails outright. + run: | + set -o pipefail + cargo test --lib --tests --verbose -- --skip example_ --test-threads=4 | tee test_output.log + if ! grep -qE '[1-9][0-9]* passed' test_output.log; then + echo "::error::No tests were reported as passed; expected at least one 'N passed' summary line." + exit 1 + fi # Standalone CI step that verifies the documentation examples actually work. It runs the # example programs from `examples/` end-to-end against the live API (via the `example_*` # smoke tests in `tests/examples.rs`, each of which executes `cargo run --example `) - # and runs the in-documentation code snippets as doctests (`cargo test --doc`, which - # compiles every fenced `rust` block in the README and the `docs/` pages and runs the - # runnable ones). Both are required by the documentation requirements: each documentation - # example has a CI test that actually runs the code. + # — this is the real runtime guard for the 7 documented examples. `cargo test --doc` + # compiles every fenced `rust`/`rust,no_run` block in the README and the `docs/` pages; + # almost all are `no_run`, so this is primarily a compile-check (it also executes the one + # genuinely runnable snippet, `README.md`'s Versioning example). Both are required by the + # documentation requirements: each documentation example has a CI test that exercises it. - name: Test examples env: APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} # Match the integration-test thread cap so the example programs and any doctests that hit # the live API stay gentle on the shared account. + # + # Same defense-in-depth as the "Run integration tests" step above: `set -o pipefail` plus + # a trailing grep make the step fail if either command reports zero executed tests (e.g. + # a filter typo, or every example/doctest silently being skipped), not just on an outright + # test failure. run: | - cargo test --test examples --verbose -- --test-threads=4 - cargo test --doc --verbose -- --test-threads=4 + set -o pipefail + cargo test --test examples --verbose -- --test-threads=4 | tee examples_output.log + if ! grep -qE '[1-9][0-9]* passed' examples_output.log; then + echo "::error::No example tests were reported as passed; expected at least one 'N passed' summary line." + exit 1 + fi + cargo test --doc --verbose -- --test-threads=4 | tee doctest_output.log + if ! grep -qE '[1-9][0-9]* passed' doctest_output.log; then + echo "::error::No doctests were reported as passed; expected at least one 'N passed' summary line." + exit 1 + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 49f6e83..ec2c73f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,68 @@ All notable changes to the Rust Apify API client are documented here. The format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project adheres to [Semantic Versioning](https://semver.org/). +## [0.7.0] - 2026-07-25 + +### Added +- `RunClient::get_with_options`/`RunGetOptions` and `BuildClient::get_with_options`/`BuildGetOptions` for the `waitForFinish` server-side wait, also usable via `actor.last_run()`/`task.last_run()`. +- `DatasetCollectionClient::get_or_create_with_options`/`DatasetGetOrCreateOptions` and the key-value-store equivalent, adding the JS-reference-only `schema` body field. +- `RequestQueueClient::batch_add_requests_with_options`/`BatchAddRequestsOptions`: byte-size chunk slicing, concurrent chunk calls, and retried `unprocessedRequests`, matching the JS reference. `batch_add_requests` now delegates to it. +- `examples/tasks_schedules_webhooks.rs`. +- `tests/actor_run.rs::run_scoped_storage_update_and_delete`, covering run-scoped storage PUT/DELETE (previously GET-only). + +### Changed +- Bumped `API_SPEC_VERSION` to `v2-2026-07-23T070817Z` and crate version to `0.7.0`. +- Removed the duplicated AI-generated/maintained notice; stated once in `README.md`. +- **Behavior change:** `batch_add_requests` folds a failed chunk's requests into `unprocessedRequests` instead of returning `Err`. +- **Behavior/API change:** `TaskClient::start`/`call` take new `TaskStartOptions`/`TaskCallOptions` instead of `ActorStartOptions`, matching the JS reference's narrower type and fixing a stray `forcePermissionLevel` param plus a dropped `content_type`. +- **Behavior/API change:** `ActorClient::call` takes new `ActorCallOptions` (drops `wait_for_finish`), fixing a footgun where it could block server-side before `call`'s own polling began. +- **Behavior/API change:** `ActorClient::validate_input`/`validate_input_for_build` return `bool` instead of the raw response envelope. +- **Behavior/API change:** `WebhookClient::test` returns `Option` (`None` on 404). +- **Behavior/API change:** `LogClient::stream`/`stream_with_options` and `RunClient::get_streamed_log*` return `Option>` (`None` on 404). +- **Behavior/API change:** `UserClient::monthly_usage`/`monthly_usage_for_date`/`limits` return `Option`. +- `ActorVersionCollectionClient::list` no longer sends `offset`/`limit`/`desc` (the endpoint defines none). +- `request_queue.rs::random_fraction` now reuses the shared `http_client::next_jitter` generator instead of a second `SystemTime`-seeded source. + +### Fixed +- `RequestQueueClient` per-method timeouts now match the JS reference (5s/30s) instead of the 360s default. +- `RequestQueueClient::get_request` no longer sends `clientKey`, matching the JS reference. +- `common::short_unique_name` truncates the prefix (not the random suffix), fixing a name collision across concurrent suite runs. +- `rust-integration-tests.yml`'s `Test examples` step now fails on zero executed tests, matching the main test step. +- `rust-integration-tests.yml`'s path filter now includes `README.md`, `docs/**`, `build.rs`, and drops the inert (gitignored) `Cargo.lock` entry. +- `tests/config.rs::make_client_honors_apify_api_url_env` no longer mutates the real `APIFY_TOKEN`/`APIFY_API_URL` env vars, avoiding a race with concurrent tests. +- Batch-add's error-path `unprocessedRequests` entries omit `method` when absent instead of serializing `"method": null`. +- `validate_input`/`validate_input_for_build` default a missing `valid` field to `false` instead of erroring, matching the JS reference's falsy-on-`undefined` behavior. + +### Documentation +- Documented `batch_add_requests_with_options`/`BatchAddRequestsOptions`, `ActorVersionClient`/`ActorEnvVarClient` (+ collections), `with_client_key`, and "Creating a …" sections for tasks/schedules/webhooks/Actors. +- Fixed a self-contradictory `docs/README.md` error-handling example. +- Added `BatchAddRequestsOptions`/`TaskStartOptions`/`TaskCallOptions`/`ActorVersion`/`ActorEnvVar` to `docs/README.md`'s Imports/model lists. +- Filled in previously-undocumented field types across `ActorListOptions.sort_by`, `LastRunOptions`, `StoreListOptions`, `ListKeysOptions`, `GetRecordOptions`, `ListRequestsOptions`, `DatasetListItemsOptions`, `DatasetDownloadOptions`, `ActorBuildOptions`, `RunResurrectOptions`, `RunMetamorphOptions`, `RunChargeOptions`, and `docs/schedules.md`'s create fields. +- Corrected `with_client_key`'s scope description and the request-lock example in `docs/storages.md`. +- Corrected three doc comments that overstated JS-parity (`KeyValueStoreKeysIterator`, `get_streamed_log`, the compress-once rationale in `src/http_client.rs`); the last was also consolidated into one explanation with a one-line pointer. +- `src/lib.rs`/`src/http_client.rs`/`README.md` no longer claim parity with a non-JS reference client. +- Tightened the `ACTOR_RUN_ID` justification comment in `tests/actor_run.rs` and dropped review-round provenance from `tests/unit_http.rs` doc comments. +- Documented `RunClient`/`BuildClient::get_with_options` and `DatasetCollectionClient`/`KeyValueStoreCollectionClient::get_or_create_with_options` (and their option types) in `docs/runs.md`/`docs/builds.md`/`docs/storages.md`/`docs/README.md`. +- Corrected the `start`/`call` argument-type notation in `docs/actors.md`/`docs/tasks.md` (generic ``, not argument-position `impl Trait`), matching the turbofish usage in every example. +- Qualified `paginate_requests`'s doc to not oversell an `exclusiveStartId` capability the method doesn't expose. +- Corrected `rust-integration-tests.yml`'s `Test examples` step comment to say it compile-checks the doc examples (only one is genuinely runnable), rather than implying broad runtime execution. + +### Internal +- Extracted shared helpers (`last_run_client`, `is_terminal_status`, `encode_webhooks`) to remove near-duplicate code; removed `WebhookCollectionClient::with_base` (identical to `new`). +- Removed the unused `_root: ApifyClient` parameter from `RunClient::new`. +- Named the SplitMix64 mixing constants in `http_client.rs::next_jitter` (now `pub(crate)`, reused by `request_queue.rs`). +- Reused `common::NOT_FOUND_STATUS_CODE` in `log.rs`'s stream 404 check instead of a raw literal; named the `request_queue.rs` empty-array byte constant; promoted the `Content-Encoding` header literals in `http_client.rs` to a `HEADER_CONTENT_ENCODING` const. + +### Tests +- Added a live assertion for the standalone `ApifyClient::log(id)` accessor. +- Added hermetic coverage for `TaskStartOptions`/`ActorCallOptions` query-param mapping. +- Added `tests/actor.rs::actor_webhooks_and_default_build` (`ActorClient::webhooks()`/`default_build()`) and `tests/key_value_store.rs::keys_public_url_is_fetchable`. +- Documented why the `ACTOR_RUN_ID` mutation in `tests/actor_run.rs` is race-free. +- Added hermetic coverage for `validate_input`'s bare-body unwrap and the `WebhookClient::test`/`LogClient::stream` 404-to-`None` mappings. +- Replaced a tautological `total >= 0` assertion with a load-bearing empty-collection check in `actor_webhooks_and_default_build`. +- Strengthened ~14 further tautological `assert!(total >= 0)` list-GET assertions into load-bearing consistency (`items.len() <= limit`, checked against the same response's `limit` rather than `total` since the latter raced under `cargo test --all-targets`'s concurrent account load) or known-empty-collection checks. +- Made `tests/webhook.rs::webhook_definition` embed a unique fragment per call, matching the suite's UUID-isolation convention. + ## [0.6.1] - 2026-07-14 ### Changed @@ -48,9 +110,8 @@ to [Semantic Versioning](https://semver.org/). - Bumped crate version to `0.6.0`. ### Removed -- `KeyValueStoreClient::get_records` and `GetRecordsOptions`. The `GET /v2/key-value-stores/{storeId}/records` - endpoint is not implemented by the reference JS client, so it is out of scope; its removal - corrects an earlier scope violation. +- `KeyValueStoreClient::get_records` and `GetRecordsOptions` + (`GET /v2/key-value-stores/{storeId}/records`). ### Documentation - Documented all `ActorStartOptions` fields in `docs/actors.md` (added the previously undocumented @@ -102,10 +163,7 @@ to [Semantic Versioning](https://semver.org/). ## [0.4.6] - 2026-07-07 ### Changed -- Rewrote earlier `CHANGELOG.md` entries to satisfy the changelog requirements: condensed - narrative prose into short change bullets and removed cross-client references to sibling - implementations, references to requirement-tracking issues, and out-of-scope / not-implemented - notes. +- Condensed earlier `CHANGELOG.md` entries into short change bullets. - Bumped crate version to `0.4.6`. ## [0.4.5] - 2026-07-03 diff --git a/Cargo.toml b/Cargo.toml index 39c456a..d909f70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "apify-client" -version = "0.6.1" +version = "0.7.0" authors = ["Apify Technologies "] -description = "An official, but experimental, AI-generated and AI-maintained Rust client for the Apify API (https://apify.com)." +description = "Rust client for the Apify API (https://apify.com)." license = "Apache-2.0" edition = "2021" repository = "https://github.com/apify/apify-client-rust" @@ -28,7 +28,7 @@ brotli = "7" flate2 = "1" [dev-dependencies] -tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "time", "net", "io-util"] } uuid = { version = "1", features = ["v4"] } [features] diff --git a/README.md b/README.md index 1c38650..6eaee15 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,7 @@ An idiomatic Rust client for the [Apify API](https://docs.apify.com/api/v2). It provides a resource-oriented, async interface that mirrors the official -[JavaScript](https://github.com/apify/apify-client-js) and -[Python](https://github.com/apify/apify-client-python) clients. +[JavaScript client](https://github.com/apify/apify-client-js). - Async (Tokio-friendly), built on `reqwest`. - Transparent authentication, retries with exponential backoff, and timeouts. @@ -22,7 +21,7 @@ It provides a resource-oriented, async interface that mirrors the official ```toml [dependencies] -apify-client = "0.6" +apify-client = "0.7" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } serde_json = "1" # for the `serde_json::Value` responses used in the Quick start ``` @@ -50,7 +49,7 @@ project needs `serde_json`. Two more dependencies are needed only for specific f By default the client uses the system TLS (`native-tls`). To use rustls instead: ```toml -apify-client = { version = "0.6", default-features = false, features = ["rustls"] } +apify-client = { version = "0.7", default-features = false, features = ["rustls"] } ``` ## Quick start @@ -220,6 +219,7 @@ run with `cargo run --example `: - `iterate_store` — lazily iterate Actors in the Apify Store. - `log_redirection` — run a separate Actor and redirect its run log into your output live, with each line prefixed by the source Actor's name. - `raw_log` — fetch and stream a run's raw (unprocessed) log via `LogOptions { raw: Some(true) }`. +- `tasks_schedules_webhooks` — create an Actor task, a schedule that runs it, and a webhook that watches it. See [`docs/`](docs) for the full API documentation. diff --git a/docs/README.md b/docs/README.md index ff992b9..ab3d8c2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,8 +1,6 @@ # Apify Rust client — documentation -> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify -> client, but it is experimental: it is generated and maintained by AI. Review the code before -> relying on it in production and report issues on the repository. +See the top-level [README](../README.md) for the client's status and support disclosure. This directory documents the public API of the Apify Rust client. The same descriptions are available as rustdoc comments and can be browsed with `cargo doc --open`. @@ -11,6 +9,16 @@ are available as rustdoc comments and can be browsed with `cargo doc --open`. > `../examples/`) are written for the repository (GitHub) view. Because the pages are > concatenated onto the crate root via `include_str!` when building rustdoc, those relative > links do not resolve in `cargo doc` output — read the cross-references on GitHub. +> +> **Note:** code blocks on these pages sometimes contain lines starting with `# ` (a `#` followed +> by a space), e.g. `# use apify_client::ApifyClient;` or a wrapping `# async fn run() -> ... {`. +> This is a [rustdoc convention](https://doc.rust-lang.org/rustdoc/write-documentation/documentation-tests.html#hiding-portions-of-the-example): +> those lines are compiled (and, for `no_run` blocks, type-checked) as part of the example, but +> hidden from rendered documentation. Viewing this page as plain Markdown (e.g. on GitHub), you +> will see those `# `-prefixed lines as ordinary code — they are boilerplate (imports, a +> `fn`/`async fn` wrapper the snippet needs to compile standalone) rather than something to +> delete or treat as unusual; ignore them when reading, or run `cargo doc --open` to see the page +> with them hidden. ## Contents @@ -35,7 +43,7 @@ Add the crate and an async runtime: ```toml [dependencies] -apify-client = "0.6" +apify-client = "0.7" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } ``` @@ -75,11 +83,13 @@ root, so you can import them directly from `apify_client` — you never need the `apify_client::clients::::` path. The complete set of re-exported option/parameter types is: -- Actors: `ActorStartOptions`, `ActorBuildOptions`, `ActorListOptions` -- Runs: `RunListOptions`, `RunResurrectOptions`, `RunMetamorphOptions`, `RunChargeOptions`, `LastRunOptions` -- Datasets: `DatasetListItemsOptions`, `DatasetDownloadOptions`, `DownloadItemsFormat` -- Key-value stores: `ListKeysOptions`, `GetRecordOptions` -- Request queues: `ListRequestsOptions` +- Actors: `ActorStartOptions`, `ActorCallOptions`, `ActorBuildOptions`, `ActorListOptions` +- Tasks: `TaskStartOptions`, `TaskCallOptions` +- Runs: `RunListOptions`, `RunGetOptions`, `RunResurrectOptions`, `RunMetamorphOptions`, `RunChargeOptions`, `LastRunOptions` +- Builds: `BuildGetOptions` +- Datasets: `DatasetListItemsOptions`, `DatasetDownloadOptions`, `DownloadItemsFormat`, `DatasetGetOrCreateOptions` +- Key-value stores: `ListKeysOptions`, `GetRecordOptions`, `KeyValueStoreGetOrCreateOptions` +- Request queues: `ListRequestsOptions`, `BatchAddRequestsOptions` - Store: `StoreListOptions` - Logs: `LogOptions` - Shared: `ListOptions`, `StorageListOptions` @@ -99,10 +109,10 @@ use apify_client::{ApifyClient, ActorListOptions, StoreListOptions, DownloadItem (The `apify_client::clients::::` paths shown by `cargo doc`'s module tree also work, but the short crate-root path above is the supported, stable way to import these option types.) -API resource/response **models** (`Actor`, `ActorRun`, `Build`, `Dataset`, `KeyValueStore`, -`KeyValueStoreKey`, `KeyValueStoreKeysPage`, `KeyValueStoreRecord`, `RequestQueue`, -`RequestQueueRequest`, `RequestQueueHead`, `RequestQueueOperationInfo`, `Task`, `Schedule`, -`Webhook`, `WebhookDispatch`, `ActorStoreListItem`, `User`, …) all live in the +API resource/response **models** (`Actor`, `ActorVersion`, `ActorEnvVar`, `ActorRun`, `Build`, +`Dataset`, `KeyValueStore`, `KeyValueStoreKey`, `KeyValueStoreKeysPage`, `KeyValueStoreRecord`, +`RequestQueue`, `RequestQueueRequest`, `RequestQueueHead`, `RequestQueueOperationInfo`, `Task`, +`Schedule`, `Webhook`, `WebhookDispatch`, `ActorStoreListItem`, `User`, …) all live in the `apify_client::models` module and are imported from there: ```rust,no_run @@ -252,21 +262,33 @@ Every fallible method returns `Result`. The variants are: - `Serde(..)` — (de)serialization failure. - `InvalidResponse(..)` / `InvalidArgument(..)` — unexpected response or bad argument. -`get`/`delete` map a missing resource to `Ok(None)` / a no-op. +`get`/`delete` on a **whole resource** (Actor, run, build, task, dataset, key-value store, +request queue, schedule, webhook, …) map a missing resource (`404 record-not-found`) to +`Ok(None)` / a successful no-op, rather than an `Err`. This does **not** extend to every +sub-resource delete: `KeyValueStoreClient::delete_record`, `RequestQueueClient::delete_request`, +and `RequestQueueClient::delete_request_lock` deliberately propagate a `404` as `Err` instead, +matching the JS reference client — see [storages](storages.md) for details on those three. To inspect the API-level details of an error without matching every variant, use `ApifyClientError::as_api_error`, which returns `Some(&ApiError)` for the `Api` variant and -`None` for any other (transport, timeout, serde, …): +`None` for any other (transport, timeout, serde, …). The example below calls +`delete_request` on a request id that does not exist, which errors (unlike the whole-resource +`get`/`delete` shown above): ```rust,no_run # use apify_client::ApifyClient; -# async fn run() { +# async fn run() -> Result<(), Box> { # let client = ApifyClient::new("t"); -if let Err(err) = client.actor("nonexistent~actor").get().await { - if let Some(api) = err.as_api_error() { - eprintln!("API error {}: {}", api.status_code, api.message); +let queue = client.request_queues().get_or_create(None).await?; +match client.request_queue(&queue.id).delete_request("nonexistent-request-id").await { + Ok(()) => println!("deleted"), + Err(err) => { + if let Some(api) = err.as_api_error() { + eprintln!("API error {}: {}", api.status_code, api.message); + } } } +# Ok(()) # } ``` @@ -285,3 +307,4 @@ Each example in [`../examples`](../examples) is runnable with | `iterate_store` | Lazily iterate Store Actors. | | `log_redirection` | Redirect a separate Actor's run log into your output live, prefixing each line with the source Actor's name. | | `raw_log` | Fetch and stream a run's raw (unprocessed) log via `LogOptions { raw: Some(true) }`. | +| `tasks_schedules_webhooks` | Create an Actor task, a schedule that runs it, and a webhook that watches it. | diff --git a/docs/actors.md b/docs/actors.md index 5998f92..700eddf 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -3,6 +3,9 @@ Obtained via `client.actors()` (collection) and `client.actor(id)` (single). The `id` may be an Actor ID or a `username~name` (or `username/name`) reference. +> Code blocks below use the rustdoc `# `-hidden-line convention — see +> [`docs/README.md`](README.md) for what those lines are. + ## `ActorCollectionClient` | Method | Arguments | Returns | Description | @@ -11,6 +14,51 @@ be an Actor ID or a `username~name` (or `username/name`) reference. | `iterate(options)` | `ActorListOptions` | `ListIterator` | Lazily iterates all Actors across pages (auto-pagination). | | `create(actor)` | `&impl Serialize` | `Actor` | Creates an Actor from a definition. | +`ActorListOptions.sort_by` is a string field naming the Actor field to sort by (no fixed enum); +accepted values per the +[Get list of Actors API](https://docs.apify.com/api/v2/acts-get) are `createdAt` (default) and +`stats.lastRunStartedAt`. + +### Creating an Actor + +`create` takes the same shape as the +[Create Actor API](https://docs.apify.com/api/v2/acts-post): at minimum a `name`, plus +`isPublic` and a `versions` array (each version needs `versionNumber`, `sourceType`, and +source-type-specific fields — `sourceFiles` for `SOURCE_FILES`, `gitRepoUrl` for `GIT_REPO`, +`tarballUrl` for `TARBALL`, `gitHubGistUrl` for `GITHUB_GIST`). Additional versions can be +added later via `versions().create(...)` (see [Actor versions and environment +variables](#actor-versions-and-environment-variables) below). + +```rust,no_run +use apify_client::ApifyClient; +use serde_json::json; + +# async fn run(client: ApifyClient) -> Result<(), Box> { +let actor = client + .actors() + .create(&json!({ + "name": "my-rust-actor", + "isPublic": false, + "versions": [{ + "versionNumber": "0.0", + "sourceType": "SOURCE_FILES", + "buildTag": "latest", + "sourceFiles": [ + { "name": "Dockerfile", "format": "TEXT", + "content": "FROM apify/actor-node:20\nCOPY . ./\nCMD node main.js" }, + { "name": "main.js", "format": "TEXT", "content": "console.log('hi');" } + ] + }] + })) + .await?; +println!("created actor {}", actor.id); +# Ok(()) +# } +``` + +See the [`create_build_run_actor`](../examples/create_build_run_actor.rs) example for the full +lifecycle (create, build, run, fetch the run log, delete). + ## `ActorClient` | Method | Arguments | Returns | Description | @@ -18,14 +66,14 @@ be an Actor ID or a `username~name` (or `username/name`) reference. | `get()` | — | `Option` | Fetches the Actor (`None` if missing). | | `update(fields)` | `&impl Serialize` | `Actor` | Updates the Actor. | | `delete()` | — | `()` | Deletes the Actor. | -| `start(input, options)` | `Option<&impl Serialize>`, `ActorStartOptions` | `ActorRun` | Starts a run, returns immediately. | -| `call(input, options, wait_secs)` | `Option<&impl Serialize>`, `ActorStartOptions`, `Option` | `ActorRun` | Starts a run and waits for it to finish. | +| `start::(input, options)` | `Option<&T>`, `ActorStartOptions` | `ActorRun` | Starts a run, returns immediately. `T` is a generic type parameter (not argument-position `impl Trait`), so it can be turbofished, e.g. `start::(...)`. | +| `call::(input, options, wait_secs)` | `Option<&T>`, `ActorCallOptions`, `Option` | `ActorRun` | Starts a run and waits for it to finish. See [`ActorCallOptions`](#actorcalloptions) below. | | `build(version, options)` | `&str`, `ActorBuildOptions` | `Build` | Builds a version of the Actor. | -| `default_build(wait_for_finish)` | `Option` | `BuildClient` | Resolves the Actor's default build, optionally waiting up to `wait_for_finish` seconds. | -| `validate_input(input)` | `&impl Serialize` | `serde_json::Value` | Validates input against the default build's schema. | -| `validate_input_for_build(input, build)` | `&impl Serialize`, `Option<&str>` | `serde_json::Value` | Validates input against a specific build's schema (`build` tag/number; `None` = default). | +| `default_build(wait_for_finish)` | `Option` | `BuildClient` | Resolves the Actor's default build. `wait_for_finish` (max 60) is a *server-side* wait — like [`RunClient::get_with_options`](runs.md#runclient)/[`BuildClient::get_with_options`](builds.md#buildclient), not client-side polling — for the build to reach a terminal state before the server responds; `None` returns immediately with whatever build is currently the default. Returns a `BuildClient` (rather than a `Build`) so you can chain further calls (`.get()`, `.log()`, `.wait_for_finish()`) on the resolved build without a second lookup. | +| `validate_input(input)` | `&impl Serialize` | `bool` | Validates input against the default build's schema; returns whether it is valid. | +| `validate_input_for_build(input, build)` | `&impl Serialize`, `Option<&str>` | `bool` | Validates input against a specific build's schema (`build` tag/number; `None` = default); returns whether it is valid. | | `last_run(status)` | `Option<&str>` | `RunClient` | Client for the last run, optionally filtered by status. See [Actor runs](runs.md) for the accepted `status` values. | -| `last_run_with_options(options)` | `LastRunOptions { status, origin }` | `RunClient` | Client for the last run, optionally filtered by status and/or origin. See [Actor runs](runs.md) for the accepted `status` and `origin` values (common origins: `DEVELOPMENT`, `WEB`, `API`, `SCHEDULER`). | +| `last_run_with_options(options)` | `LastRunOptions { status: Option, origin: Option }` | `RunClient` | Client for the last run, optionally filtered by status and/or origin. See [Actor runs](runs.md) for the accepted `status` and `origin` values (common origins: `DEVELOPMENT`, `WEB`, `API`, `SCHEDULER`). | | `builds()` | — | `BuildCollectionClient` | The Actor's build collection. | | `runs()` | — | `RunCollectionClient` | The Actor's run collection. | | `version(n)` / `versions()` | `&str` / — | `ActorVersionClient` / collection | Version management. | @@ -33,9 +81,10 @@ be an Actor ID or a `username~name` (or `username/name`) reference. ### `ActorStartOptions` -All fields are optional. Used by both `start` and `call` here, and by the identical `start` / -`call` methods on [tasks](tasks.md) (for `call`, `wait_for_finish` is server-side; the -`wait_secs` argument controls client-side polling). +All fields are optional. Used by `start` (`call` takes the narrower +[`ActorCallOptions`](#actorcalloptions) instead). The task equivalents, +[`TaskStartOptions`/`TaskCallOptions`](tasks.md#taskstartoptions-and-taskcalloptions), are +narrowed versions of this type — see that page for the differences. | Field | Type | Description | |---|---|---| @@ -50,6 +99,16 @@ All fields are optional. Used by both `start` and `call` here, and by the identi | `force_permission_level` | `Option` | Override the Actor's permission level for this run. | | `webhooks` | `Option>` | Ad-hoc webhooks to attach to this run. Encoded as base64 JSON in the `webhooks` query parameter, matching the reference clients. | +### `ActorCallOptions` + +Same fields as [`ActorStartOptions`](#actorstartoptions) except `wait_for_finish`, matching the +JS reference client's `ActorCallOptions` (`Omit`). +`wait_for_finish` is the server-side wait, which would otherwise let a caller silently block +server-side (up to 60s) before `call`'s own client-side polling (via `wait_secs`) even begins — +dropping the field removes that footgun. Use `From for ActorStartOptions` (or +just construct an `ActorStartOptions` directly) if you need to pass an equivalent options value to +`start` instead. + The `wait_secs` argument of `call` (and of `wait_for_finish` on runs/builds) controls the client-side polling budget: @@ -66,23 +125,24 @@ client-side polling budget: ### `ActorBuildOptions` -All optional: +All fields optional: -- `tag` — build tag to assign to the resulting build (e.g. `latest`). -- `use_cache` — reuse cached Docker layers from previous builds to speed the build up. -- `beta_packages` — build against the beta versions of the Apify SDK/CLI packages instead of the - stable ones. -- `wait_for_finish` — maximum number of seconds the server waits for the build to finish before - responding (a server-side wait, not client-side polling). +| Field | Type | Description | +|---|---|---| +| `tag` | `Option` | Build tag to assign to the resulting build (e.g. `latest`). | +| `use_cache` | `Option` | Reuse cached Docker layers from previous builds to speed the build up (default `true`). | +| `beta_packages` | `Option` | If `true`, build against the beta versions of the Apify SDK/CLI packages instead of the stable ones. | +| `wait_for_finish` | `Option` | Maximum number of seconds the server waits for the build to finish before responding (a server-side wait, not client-side polling; max 60). | ### Input validation `validate_input` / `validate_input_for_build` check an input value against the Actor's input -schema and return the API's JSON response as `serde_json::Value`. Unlike most endpoints this one -is **not** wrapped in a `{ "data": ... }` envelope — the returned `Value` is the top-level body -`{ "valid": }`, where `valid` reports whether the input satisfies the schema. A failed -*request* (e.g. unknown `build` tag, missing auth, malformed body) is not reported via `valid`; -it surfaces as an `Err(ApifyClientError)` from the call instead. +schema and return whether the input is valid, as a plain `bool` — matching the reference client's +`validateInput`, which returns `response.data.valid`. Under the hood the API's response is +**not** wrapped in a `{ "data": ... }` envelope like most endpoints — the top-level body is +`{ "valid": }` — but that shape is an implementation detail the client unwraps for you. A +failed *request* (e.g. unknown `build` tag, missing auth, malformed body) is not reported via the +returned `bool`; it surfaces as an `Err(ApifyClientError)` from the call instead. ```rust,no_run use apify_client::ApifyClient; @@ -93,15 +153,14 @@ let client = ApifyClient::new(std::env::var("APIFY_TOKEN")?); let actor = client.actor("apify~hello-world"); // Validate against the default build's input schema. -let result = actor.validate_input(&json!({ "message": "hi" })).await?; -let is_valid = result.get("valid").and_then(|v| v.as_bool()).unwrap_or(false); +let is_valid = actor.validate_input(&json!({ "message": "hi" })).await?; println!("input valid: {is_valid}"); // Validate against a specific build (by tag or version number). `None` == default build. -let result = actor +let is_valid = actor .validate_input_for_build(&json!({ "message": "hi" }), Some("latest")) .await?; -println!("validated against latest build: {result}"); +println!("validated against latest build: {is_valid}"); # Ok(()) # } ``` @@ -168,7 +227,113 @@ println!("build {} status {:?}", build.id, build.status); ## Actor versions and environment variables -`ActorVersionClient`: `get`, `update`, `delete`, `env_var(name)`, `env_vars()`. -`ActorVersionCollectionClient`: `list(options)`, `iterate(options)`, `create(version)`. -`ActorEnvVarClient`: `get`, `update`, `delete`. -`ActorEnvVarCollectionClient`: `list()`, `iterate()`, `create(env_var)`. +Obtained from an `ActorClient` via `version(number)` / `versions()` (the version collection), and +from an `ActorVersionClient` via `env_var(name)` / `env_vars()` (that version's env-var +collection). + +### `ActorVersionCollectionClient` + +| Method | Arguments | Returns | Description | +|---|---|---|---| +| `list(options)` | `ListOptions { offset, limit, desc }` | `PaginationList` | Lists the Actor's versions. `options` is accepted for interface stability but ignored: `GET /v2/actors/{actorId}/versions` takes no query parameters and always returns every version in one response, matching the reference client. | +| `iterate(options)` | `ListOptions` | `ListIterator` | Lazily iterates all versions (in practice a single unpaginated fetch, for the same reason as `list`). | +| `create(version)` | `&impl Serialize` | `ActorVersion` | Creates a new version. | + +### `ActorVersionClient` + +| Method | Arguments | Returns | Description | +|---|---|---|---| +| `get()` | — | `Option` | Fetches the version (`None` if missing). | +| `update(fields)` | `&impl Serialize` | `ActorVersion` | Updates the version. | +| `delete()` | — | `()` | Deletes the version. | +| `env_var(name)` | `&str` | `ActorEnvVarClient` | Client for one of the version's environment variables. | +| `env_vars()` | — | `ActorEnvVarCollectionClient` | Client for the version's environment-variable collection. | + +`create`/`update` take the same shape as the +[Create Actor version API](https://docs.apify.com/api/v2/actors-actor-id-versions-post): at +minimum `versionNumber` and `sourceType` (`SOURCE_FILES`, `GIT_REPO`, `TARBALL`, or +`GITHUB_GIST`), plus the source-type-specific field (`sourceFiles`, `gitRepoUrl`, `tarballUrl`, +or `gitHubGistUrl`) and optionally `buildTag`, `envVars`, `applyEnvVarsToBuild`. + +### `ActorEnvVarCollectionClient` + +| Method | Arguments | Returns | Description | +|---|---|---|---| +| `list()` | — | `PaginationList` | Lists the version's environment variables. | +| `iterate()` | — | `ListIterator` | Iterates the environment variables (single page; see below). | +| `create(env_var)` | `&ActorEnvVar` | `ActorEnvVar` | Creates a new environment variable. | + +`iterate()` is not offset-paginated — the API returns every variable in one page — so it fetches +once and yields all of them; it exists for interface parity with the other collection clients. + +### `ActorEnvVarClient` + +| Method | Arguments | Returns | Description | +|---|---|---|---| +| `get()` | — | `Option` | Fetches the environment variable by name (`None` if missing). | +| `update(env_var)` | `&ActorEnvVar` | `ActorEnvVar` | Updates the environment variable. | +| `delete()` | — | `()` | Deletes the environment variable. | + +### `ActorVersion` fields + +`ActorVersion` (from `apify_client::models`) is returned by `get`, `create`, `update`, and the +version `list`. + +| Field | Type | Description | +|---|---|---| +| `version_number` | `String` | The version number, e.g. `0.1` (always present). | +| `source_type` | `Option` | Source type: `SOURCE_FILES`, `GIT_REPO`, `TARBALL`, or `GITHUB_GIST`. | +| `extra` | `Extra` | Everything else — `sourceFiles`/`gitRepoUrl`/`tarballUrl`/`gitHubGistUrl`, `buildTag`, `envVars`, `applyEnvVarsToBuild`, and any other fields returned by the API. | + +### `ActorEnvVar` fields + +`ActorEnvVar` (from `apify_client::models`) is the payload for `env_vars().create(...)` and +`env_var(name).update(...)`, and the type returned by `get`/`create`/`update`/`list`. + +| Field | Type | Description | +|---|---|---| +| `name` | `String` | The environment variable name (always present). | +| `value` | `Option` | The value; may be omitted in responses for secret variables. | +| `is_secret` | `Option` | Whether the variable is a secret. | +| `extra` | `Extra` | Any other fields returned by the API. | + +Create a version and set an environment variable on it: + +```rust,no_run +use apify_client::models::ActorEnvVar; +use apify_client::ApifyClient; +use serde_json::json; + +# async fn run(client: ApifyClient, actor_id: &str) -> Result<(), Box> { +let actor_client = client.actor(actor_id); + +// Create version 0.1 with inline source files. +let version = actor_client + .versions() + .create(&json!({ + "versionNumber": "0.1", + "sourceType": "SOURCE_FILES", + "sourceFiles": [ + { "name": "Dockerfile", "format": "TEXT", + "content": "FROM apify/actor-node:20\nCOPY . ./\nCMD node main.js" }, + { "name": "main.js", "format": "TEXT", "content": "console.log('v0.1');" } + ] + })) + .await?; +println!("created version {}", version.version_number); + +// Set an environment variable on that version. +let version_client = actor_client.version(&version.version_number); +let env_var = version_client + .env_vars() + .create(&ActorEnvVar { + name: "MY_VAR".to_string(), + value: Some("hello".to_string()), + is_secret: Some(false), + extra: Default::default(), + }) + .await?; +println!("set env var {} = {:?}", env_var.name, env_var.value); +# Ok(()) +# } +``` diff --git a/docs/builds.md b/docs/builds.md index f6b19d4..0059789 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -3,6 +3,9 @@ Obtained via `client.builds()` (collection) and `client.build(id)` (single). Nested build collections are available via `actor.builds()`. +> Code blocks below use the rustdoc `# `-hidden-line convention — see +> [`docs/README.md`](README.md) for what those lines are. + ## `BuildCollectionClient` | Method | Arguments | Returns | Description | @@ -14,12 +17,23 @@ collections are available via `actor.builds()`. | Method | Arguments | Returns | Description | |---|---|---|---| -| `get()` | — | `Option` | Fetches the build. | +| `get()` | — | `Option` | Fetches the build. Returns immediately (no server-side wait); see `get_with_options`. | +| `get_with_options(options)` | `BuildGetOptions { wait_for_finish: Option }` | `Option` | Fetches the build, optionally waiting server-side (max 60s) for it to reach a terminal state before responding. | | `abort()` | — | `Build` | Aborts the build. | | `delete()` | — | `()` | Deletes the build. | | `wait_for_finish(wait_secs)` | `Option` | `Build` | Polls until the build is terminal. | | `get_openapi_definition()` | — | `Option` | Fetches the OpenAPI definition generated for the build (raw JSON, endpoint `.../openapi.json`). | | `log()` | — | `LogClient` | Access the build's log. | +`BuildGetOptions` (the argument to `get_with_options`): `wait_for_finish: Option` — +maximum time, in seconds (capped at 60 by the API), to wait server-side for the build to reach a +terminal state before returning; `None` (the default, and what plain `get()` uses) returns +immediately without waiting. This is a single bounded server-side wait, distinct from +[`BuildClient::wait_for_finish`](#buildclient), which polls repeatedly until the build finishes +or a client-side budget is exhausted. The same server-side-wait semantics apply to +`ActorClient::default_build`'s `wait_for_finish` argument (see +[Actors → `ActorClient`](actors.md#actorclient)), which resolves the Actor's default build via +the analogous `waitForFinish` query parameter. + The returned `Build` model's fields (`id`, `status`, `build_number`, …) are documented in [actors.md → `Build` fields](actors.md#build-fields). diff --git a/docs/misc.md b/docs/misc.md index 6ca5ab9..60b625e 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -1,5 +1,8 @@ # Store, users and logs +> Code blocks below use the rustdoc `# `-hidden-line convention — see +> [`docs/README.md`](README.md) for what those lines are. + ## Apify Store — `client.store()` `StoreCollectionClient`: @@ -9,9 +12,12 @@ | `list(options)` | `StoreListOptions` | `PaginationList` | One page of Store Actors. | | `iterate(options)` | `StoreListOptions` | `StoreActorIterator` | Lazy, page-fetching iterator. | -`StoreListOptions`: `offset`, `limit`, `search`, `sort_by`, `category`, `username`, -`pricing_model`. `limit` means a single page's size for `list`, but a cap on the *total* number of -items yielded for `iterate` (see below). +`StoreListOptions`: `offset: Option`, `limit: Option`, `search: Option`, +`sort_by: Option`, `category: Option`, `username: Option`, +`pricing_model: Option`, `include_unrunnable_actors: Option`, +`allows_agentic_users: Option`, `response_format: Option`. `limit` means a single +page's size for `list`, but a cap on the *total* number of items yielded for `iterate` (see +below). `StoreActorIterator` is a type alias for `ListIterator` (the shared iterator returned by every collection's `iterate`), re-exported at the crate root alongside `ListIterator` @@ -70,9 +76,9 @@ and falls back to `name` (e.g. `actor.title.or(actor.name)`). | Method | Arguments | Returns | Description | |---|---|---|---| | `get()` | — | `Option` | Account details (private for `me`, public otherwise). | -| `monthly_usage()` | — | `Value` | Current account's monthly usage for the current billing cycle (`me` only). | -| `monthly_usage_for_date(date)` | `Option<&str>` | `Value` | Monthly usage for the billing cycle containing the `YYYY-MM-DD` `date`; `None` == current month (`me` only). | -| `limits()` | — | `Value` | Current account's limits (`me` only). | +| `monthly_usage()` | — | `Option` | Current account's monthly usage for the current billing cycle (`me` only). | +| `monthly_usage_for_date(date)` | `Option<&str>` | `Option` | Monthly usage for the billing cycle containing the `YYYY-MM-DD` `date`; `None` == current month (`me` only). | +| `limits()` | — | `Option` | Current account's limits (`me` only). | | `update_limits(limits)` | `&impl Serialize` | `()` | Updates the account's limits (`me` only). | The methods marked **(`me` only)** operate on the authenticated account and are only valid on the @@ -102,9 +108,11 @@ if let Some(user) = client.me().get().await? { `monthly_usage()` is shorthand for `monthly_usage_for_date(None)` (current cycle). The client unwraps the API's `{ data: ... }` envelope, so the returned `serde_json::Value` has the shape `{ usageCycle: { startAt, endAt }, monthlyServiceUsage, dailyServiceUsages, ... }`. Billing -cycles are not calendar-month aligned — pass any day within a cycle to fetch that cycle. +cycles are not calendar-month aligned — pass any day within a cycle to fetch that cycle. The +return is wrapped in `Option` (`None` if unavailable) purely for JS-reference parity — the spec +declares no `404` for this endpoint, so in practice it is always `Some` for a valid `me` client. -The return value is an untyped `serde_json::Value`; access its fields with the non-panicking +The inner value is an untyped `serde_json::Value`; access its fields with the non-panicking `Value::get` (the same idiom as `examples/get_account.rs`) so a missing field yields `None` instead of panicking: @@ -121,7 +129,7 @@ let usage = client.me().monthly_usage().await?; // hard-coding a date so the lookup always lands on a real cycle. let day = chrono::Utc::now().format("%Y-%m-%d").to_string(); let dated = client.me().monthly_usage_for_date(Some(&day)).await?; -if let Some(cycle) = dated.get("usageCycle") { +if let Some(cycle) = dated.as_ref().and_then(|u| u.get("usageCycle")) { let start = cycle.get("startAt").and_then(|v| v.as_str()).unwrap_or("?"); let end = cycle.get("endAt").and_then(|v| v.as_str()).unwrap_or("?"); println!("cycle {start} .. {end}"); @@ -140,8 +148,8 @@ Also reachable via `run.log()` and `build.log()`. |---|---|---|---| | `get()` | — | `Option` | The entire log as text. | | `get_with_options(options)` | `LogOptions` | `Option` | As `get()`, with options (e.g. `raw`). | -| `stream()` | — | `Result>>>` (async — `.await` it) | Streams log chunks live (log redirection). | -| `stream_with_options(options)` | `LogOptions` | `Result>>>` (async — `.await` it) | As `stream()`, with options (e.g. `raw`). | +| `stream()` | — | `Result>>>>` (async — `.await` it) | Streams log chunks live (log redirection), or `None` if the log does not exist. | +| `stream_with_options(options)` | `LogOptions` | `Result>>>>` (async — `.await` it) | As `stream()`, with options (e.g. `raw`). | `LogOptions` has a single field, `raw: Option`. When `Some(true)`, the API returns the raw log content without server-side processing (e.g. without the per-line timestamps it adds by @@ -181,7 +189,12 @@ use apify_client::ApifyClient; use futures_util::StreamExt; # async fn run(client: ApifyClient, run_id: &str) -> Result<(), Box> { -let mut stream = client.run(run_id).log().stream().await?; +let mut stream = client + .run(run_id) + .log() + .stream() + .await? + .expect("run's log exists"); while let Some(chunk) = stream.next().await { let chunk = chunk?; print!("{}", String::from_utf8_lossy(&chunk)); diff --git a/docs/runs.md b/docs/runs.md index 27a3895..8e8ac2e 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -3,6 +3,9 @@ Obtained via `client.runs()` (collection) and `client.run(id)` (single). Nested run collections are available via `actor.runs()` and `task.runs()`. +> Code blocks below use the rustdoc `# `-hidden-line convention — see +> [`docs/README.md`](README.md) for what those lines are. + ## `RunCollectionClient` | Method | Arguments | Returns | Description | @@ -14,7 +17,8 @@ collections are available via `actor.runs()` and `task.runs()`. | Method | Arguments | Returns | Description | |---|---|---|---| -| `get()` | — | `Option` | Fetches the run. | +| `get()` | — | `Option` | Fetches the run. Returns immediately (no server-side wait); see `get_with_options`. | +| `get_with_options(options)` | `RunGetOptions { wait_for_finish: Option }` | `Option` | Fetches the run, optionally waiting server-side (max 60s) for it to reach a terminal state before responding. Also reachable via `actor.last_run()`/`task.last_run()`, which share `RunClient`. | | `update(fields)` | `&impl Serialize` | `ActorRun` | Updates the run (e.g. status message). | | `delete()` | — | `()` | Deletes the run. | | `abort(gracefully)` | `Option` | `ActorRun` | Aborts the run. `None` omits the param (server default, immediate); `Some(true)`/`Some(false)` abort gracefully/immediately. | @@ -24,21 +28,33 @@ collections are available via `actor.runs()` and `task.runs()`. | `charge(options)` | `RunChargeOptions` | `()` | Charges a pay-per-event run (always sends an idempotency key). | | `wait_for_finish(wait_secs)` | `Option` | `ActorRun` | Polls until the run is terminal. `None` waits indefinitely; `Some(n)` bounds the wait and may return a still-running (non-terminal) run if `n` elapses first. | | `dataset()` / `key_value_store()` / `request_queue()` / `log()` | — | resource client | Access the run's default storages and log. | -| `get_streamed_log()` | — | `Result>>>` (async — `.await` it) | Convenience for `log().stream()` — streams the run's log chunks live (log redirection). | -| `get_streamed_log_with_options(options)` | `LogOptions` | `Result>>>` (async — `.await` it) | As `get_streamed_log()`, forwarding `LogOptions` (e.g. `raw`) to the log stream. | +| `get_streamed_log()` | — | `Result>>>>` (async — `.await` it) | Convenience for `log().stream()` — streams the run's log chunks live (log redirection), or `None` if the log does not exist. | +| `get_streamed_log_with_options(options)` | `LogOptions` | `Result>>>>` (async — `.await` it) | As `get_streamed_log()`, forwarding `LogOptions` (e.g. `raw`) to the log stream. | `get_streamed_log()` / `get_streamed_log_with_options()` are `async` and yield the stream inside a -`Result`, so `.await?` them to obtain it. Polling the returned stream with `.next()` requires the +`Result>`, so `.await?` them, then handle the `Option` (`None` means the log does not +exist). Polling the returned stream with `.next()` requires the `futures_util::StreamExt` trait (from the `futures-util` crate — add `futures-util = "0.3"` to your `Cargo.toml`) in scope; the [`raw_log`](../examples/raw_log.rs) example drives `get_streamed_log_with_options` exactly this way. See [Logs](misc.md#logs--clientlogbuild_or_run_id) for a full streaming snippet. -`RunResurrectOptions`: `build`, `memory_mbytes`, `timeout_secs`, `max_items`, `max_total_charge_usd`, `restart_on_error` (all optional). +`RunGetOptions` (the argument to `get_with_options`): `wait_for_finish: Option` — maximum +time, in seconds (capped at 60 by the API), to wait server-side for the run to reach a terminal +state before returning; `None` (the default, and what plain `get()` uses) returns immediately +without waiting. This is a single bounded server-side wait, distinct from +[`RunClient::wait_for_finish`](#runclient), which polls repeatedly (using this same parameter +internally) until the run finishes or a client-side budget is exhausted. + +`RunResurrectOptions` (all optional): `build: Option`, `memory_mbytes: Option`, +`timeout_secs: Option`, `max_items: Option`, `max_total_charge_usd: Option`, +`restart_on_error: Option`. -`RunMetamorphOptions`: `build`, `content_type` (both optional; `content_type` defaults to `application/json`). +`RunMetamorphOptions` (both optional): `build: Option`, `content_type: Option` +(defaults to `application/json`). -`RunChargeOptions`: `event_name` (required), `count` (defaults to `1`), `idempotency_key` (auto-generated when omitted). +`RunChargeOptions`: `event_name: String` (required), `count: Option` (defaults to `1`), +`idempotency_key: Option` (auto-generated when omitted). `ActorRun.status` is a stringly-typed `Option` carrying the API's run status. Known values are `READY`, `RUNNING`, `SUCCEEDED`, `FAILED`, `ABORTING`, `ABORTED`, `TIMING-OUT`, and diff --git a/docs/schedules.md b/docs/schedules.md index 326eda3..0bdda30 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -2,6 +2,9 @@ Obtained via `client.schedules()` (collection) and `client.schedule(id)` (single). +> Code blocks below use the rustdoc `# `-hidden-line convention — see +> [`docs/README.md`](README.md) for what those lines are. + ## `ScheduleCollectionClient` | Method | Arguments | Returns | Description | @@ -19,6 +22,39 @@ Obtained via `client.schedules()` (collection) and `client.schedule(id)` (single | `delete()` | — | `()` | Deletes the schedule. | | `get_log()` | — | `Option` | Fetches the schedule's invocation log. | +### Creating a schedule + +`create` takes the same shape as the +[Create schedule API](https://docs.apify.com/api/v2/schedules-post): `cronExpression` and +`isEnabled` are required, plus an `actions` array of `{ type, actorId | actorTaskId, ... }` objects +describing what the schedule runs (`type` is `"RUN_ACTOR"` or `"RUN_ACTOR_TASK"`). Optional +fields include `name` (a technical name for the schedule) and `isExclusive` (prevent a new +invocation from starting while a previous one triggered by this schedule is still running). + +```rust,no_run +use apify_client::ApifyClient; +use serde_json::json; + +# async fn run(client: ApifyClient, task_id: &str) -> Result<(), Box> { +let schedule = client + .schedules() + .create(&json!({ + "name": "my-daily-schedule", + "cronExpression": "0 12 * * *", + "isEnabled": true, + "actions": [ + { "type": "RUN_ACTOR_TASK", "actorTaskId": task_id } + ] + })) + .await?; +println!("created schedule {}", schedule.id); +# Ok(()) +# } +``` + +See the [`tasks_schedules_webhooks`](../examples/tasks_schedules_webhooks.rs) example for a +schedule bound to a task, plus `get_log`, `update` and `delete`. + ## The `Schedule` model `Schedule` lives in `apify_client::models` (`use apify_client::models::Schedule;`). Returned by diff --git a/docs/storages.md b/docs/storages.md index ee3f141..eb109c9 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -1,5 +1,8 @@ # Storages: datasets, key-value stores, request queues +> Code blocks below use the rustdoc `# `-hidden-line convention — see +> [`docs/README.md`](README.md) for what those lines are. + ## Storage metadata models (`Dataset`, `KeyValueStore`, `RequestQueue`) `get` and `get_or_create` on each storage collection/client return a metadata model from @@ -34,13 +37,22 @@ let dataset_client = client.dataset(&dataset.id); `DatasetCollectionClient`: `list(options: StorageListOptions)`, `iterate(options: StorageListOptions)` (lazy `ListIterator` auto-pagination), -`get_or_create(name: Option<&str>)`. +`get_or_create(name: Option<&str>)`, `get_or_create_with_options(name, options: DatasetGetOrCreateOptions)`. `StorageListOptions`: `offset`, `limit`, `desc`, `unnamed`, `ownership`. `get_or_create` takes `Option<&str>`: pass `Some("my-name")` to get-or-create a **named** storage (reused across runs), or `None` for an unnamed one. The same signature applies to the key-value-store and request-queue collections. +`get_or_create_with_options(name, options)` additionally accepts `DatasetGetOrCreateOptions { +schema: Option }`: `schema` is a JS-reference-only convenience (not +documented by the OpenAPI spec, which declares only the `name` query parameter on +`POST /v2/datasets`), sent as the request's `{ "schema": ... }` JSON body and applied only when +the dataset is actually created (a no-op when an existing named dataset is returned). Plain +`get_or_create(name)` delegates to this with `DatasetGetOrCreateOptions::default()` (no schema). +`KeyValueStoreCollectionClient::get_or_create_with_options`/`KeyValueStoreGetOrCreateOptions` +below work identically for key-value stores. + ```rust,no_run # use apify_client::ApifyClient; # async fn run(client: ApifyClient) -> Result<(), Box> { @@ -65,33 +77,46 @@ let scratch = client.datasets().get_or_create(None).await?; | `push_items(items)` | `&impl Serialize` | `()` | Appends items (object or array). | | `get_statistics()` | — | `Option` | Field statistics. | | `download_items(format, options)` | `DownloadItemsFormat`, `DatasetDownloadOptions` | `Vec` | Export items as JSON/JSONL/CSV/XLSX/XML/RSS/HTML. | -| `create_items_public_url(options, expires)` | `DatasetListItemsOptions`, `Option` | `String` | Shareable (HMAC-signed for private) items URL. | +| `create_items_public_url(options, expires_in_secs)` | `DatasetListItemsOptions`, `Option` | `String` | Shareable (HMAC-signed for private) items URL. | `DatasetListItemsOptions` (all optional): -- `offset` / `limit` / `desc` — pagination window and reverse (newest-first) ordering. -- `fields` — comma-separated allow-list of top-level fields to keep in each item. -- `output_fields` — positionally renames the fields selected by `fields` in the output; requires - `fields`, and the two lists must have equal length (the i-th `output_fields` name becomes the - output name of the i-th `fields` entry). -- `omit` — comma-separated fields to drop from each item. -- `skip_empty` — omit items that are empty after field filtering. -- `skip_hidden` — omit hidden fields (those whose names start with `#`). -- `clean` — shorthand for `skip_hidden` + `skip_empty` (only non-empty, non-hidden items). -- `unwind` — comma-separated fields whose array values are expanded into separate items. -- `flatten` — comma-separated fields whose nested objects are flattened into dotted keys. -- `view` — name of a dataset view to apply. -- `simplified` — return the simplified form of the items. -- `skip_failed_pages` — skip pages that failed to be scraped (crawler datasets). - -`DatasetDownloadOptions` adds format-specific export controls (all optional): - -- `attachment` — set the `Content-Disposition: attachment` header so browsers download the file. -- `bom` — prepend a UTF-8 byte-order mark (useful for CSV opened in Excel). -- `delimiter` — CSV field delimiter (default `,`). -- `skip_header_row` — omit the CSV header row. -- `xml_root` / `xml_row` — element names for the XML root and per-item rows. -- `feed_title` / `feed_description` — title and description for RSS output. +| Field | Type | Description | +|---|---|---| +| `offset` | `Option` | Number of items to skip. | +| `limit` | `Option` | Maximum number of items to return. | +| `desc` | `Option` | Reverse (newest-first) ordering. | +| `fields` | `Option>` | Allow-list of top-level fields to keep in each item. | +| `output_fields` | `Option>` | Positionally renames the fields selected by `fields` in the output; requires `fields`, and the two lists must have equal length (the i-th `output_fields` name becomes the output name of the i-th `fields` entry). | +| `omit` | `Option>` | Fields to drop from each item. | +| `skip_empty` | `Option` | Omit items that are empty after field filtering. | +| `skip_hidden` | `Option` | Omit hidden fields (those whose names start with `#`). | +| `clean` | `Option` | Shorthand for `skip_hidden` + `skip_empty` (only non-empty, non-hidden items). | +| `unwind` | `Option>` | Fields whose array values are expanded into separate items. | +| `flatten` | `Option>` | Fields whose nested objects are flattened into dotted keys. | +| `view` | `Option` | Name of a dataset view to apply. | +| `simplified` | `Option` | Return the simplified form of the items. | +| `skip_failed_pages` | `Option` | Skip items that come from failed pages (crawler datasets). | +| `signature` | `Option` | Pre-shared URL signature granting access to a private dataset without an API token. | + +`DatasetDownloadOptions` adds format-specific export controls (all optional except `items`, +which embeds a `DatasetListItemsOptions` for the same filtering/projection as `list_items`): + +| Field | Type | Description | +|---|---|---| +| `items` | `DatasetListItemsOptions` | Shared item filtering/projection options (see above). | +| `attachment` | `Option` | Set the `Content-Disposition: attachment` header so browsers download the file. | +| `bom` | `Option` | Prepend a UTF-8 byte-order mark (useful for CSV opened in Excel). | +| `delimiter` | `Option` | CSV field delimiter (default `,`). | +| `skip_header_row` | `Option` | Omit the CSV header row. | +| `xml_root` / `xml_row` | `Option` / `Option` | Element names for the XML root and per-item rows. | +| `feed_title` / `feed_description` | `Option` / `Option` | Title and description for RSS output. | + +`create_items_public_url`'s `expires_in_secs` bounds how long the URL's HMAC-SHA256 `signature` +stays valid, as a **relative** number of seconds from the moment the URL is created (not a Unix +timestamp) — `None` produces a signature that never expires. It has no effect when the dataset +does not expose a URL-signing secret key (i.e. it isn't configured for signed access), in which +case the URL is returned unsigned regardless. `DownloadItemsFormat` (re-exported at the crate root) selects the export format for `download_items`. Variants: `Json`, `Jsonl`, `Csv`, `Xlsx`, `Xml`, `Rss`, `Html`. The method @@ -115,7 +140,10 @@ println!("exported {} bytes of CSV", csv.len()); `KeyValueStoreCollectionClient`: `list(options: StorageListOptions)`, `iterate(options: StorageListOptions)` (lazy `ListIterator` auto-pagination), -`get_or_create(name: Option<&str>)`. +`get_or_create(name: Option<&str>)`, `get_or_create_with_options(name, options: KeyValueStoreGetOrCreateOptions { schema: Option })` +(same semantics as the Datasets section's `DatasetGetOrCreateOptions` above: `schema` is a +JS-reference-only convenience sent as the request body, applied only when the store is actually +created). `KeyValueStoreClient`: @@ -131,11 +159,17 @@ println!("exported {} bytes of CSV", csv.len()); | `set_record_raw(key, bytes, content_type)` | `&str`, `Vec`, `&str` | `()` | Stores a raw record. | | `set_record_json(key, value)` | `&str`, `&impl Serialize` | `()` | Stores a JSON record. | | `delete_record(key)` | `&str` | `()` | Deletes a record. | -| `get_record_with_options(key, options)` | `&str`, `GetRecordOptions { attachment, signature }` | `Option` | Reads a record with explicit attachment/signature options. | +| `get_record_with_options(key, options)` | `&str`, `GetRecordOptions { attachment: Option, signature: Option }` | `Option` | Reads a record with explicit attachment/signature options. | | `get_record_public_url(key)` | `&str` | `String` | Shareable (HMAC-signed for private) record URL. | -| `create_keys_public_url(expires)` | `Option` | `String` | Shareable keys-list URL. | +| `create_keys_public_url(expires_in_secs)` | `Option` | `String` | Shareable keys-list URL. | + +`create_keys_public_url`'s `expires_in_secs` has the same semantics as +`create_items_public_url`'s (see above): a relative number of seconds from creation time +bounding the signature's validity, with `None` meaning it never expires, and no effect when the +store isn't configured for signed access. -`ListKeysOptions`: `limit`, `exclusive_start_key`, `prefix`, `collection`, `signature`. Like +`ListKeysOptions`: `limit: Option`, `exclusive_start_key: Option`, +`prefix: Option`, `collection: Option`, `signature: Option`. Like `StoreListOptions.limit`, the meaning of `limit` depends on the method: for `list_keys` it is a single page's size (max keys returned by one call, capped at 1000 by the API); for `iterate_keys` it is a cap on the *total* number of keys yielded across all pages (unset iterates the whole @@ -180,7 +214,20 @@ listed in `KeyValueStoreKeysPage::items`. Its fields: `iterate(options: StorageListOptions)` (lazy `ListIterator` auto-pagination), `get_or_create(name: Option<&str>)`. -`RequestQueueClient` (chainable `with_client_key(key)` for lock coordination): +`RequestQueueClient::with_client_key(client_key: impl Into) -> RequestQueueClient` +consumes `self` and returns a new client that sends the given `clientKey` as a query parameter on +most request-level operations (`list_head`, `add_request`, `update_request`, `delete_request`, +`list_and_lock_head`, the batch add/delete calls, `list_requests`, `prolong_request_lock`, +`delete_request_lock`, `unlock_requests`). It is *not* sent on the queue-level metadata methods +`get`, `update`, and `delete`, nor on `get_request` — matching the JS reference client, whose +`getRequest` is the one request-level method that builds its params from bare `this._params()` +instead of merging in `clientKey: this.clientKey` the way its siblings do. `client_key` should be +a stable, unique identifier for *this* consumer (e.g. one value per crawler process), reused +across calls so the API can attribute locks to their owner. It has no effect on unlocked +request-level operations (`add_request`, `list_requests`, …) beyond being sent along; it matters +specifically for the locking methods below, whose lock ownership and `unlock_requests`'s scope +are both keyed on it. See +[locking requests](#locking-requests) below for a worked example. | Method | Arguments | Returns | Description | |---|---|---|---| @@ -193,9 +240,10 @@ listed in `KeyValueStoreKeysPage::items`. Its fields: | `update_request(request, forefront)` | `&RequestQueueRequest`, `bool` | `RequestQueueOperationInfo` | Updates a request. | | `delete_request(id)` | `&str` | `()` | Deletes a request. | | `list_and_lock_head(lock_secs, limit)` | `i64`, `Option` | `Value` | Locks head requests. | -| `batch_add_requests(requests, forefront)` | `&[RequestQueueRequest]`, `bool` | `Value` | Batch add. | +| `batch_add_requests(requests, forefront)` | `&[RequestQueueRequest]`, `bool` | `Value` | Batch add with reference-matching defaults (see below). | +| `batch_add_requests_with_options(requests, options)` | `&[RequestQueueRequest]`, `BatchAddRequestsOptions` | `Value` | Batch add with explicit retry/parallelism control (see below). | | `batch_delete_requests(requests)` | `&[impl Serialize]` | `Value` | Batch delete. | -| `list_requests(options)` | `ListRequestsOptions { limit, exclusive_start_id, cursor, filter }` | `Value` | List requests (cursor/filter pagination). | +| `list_requests(options)` | `ListRequestsOptions { limit: Option, exclusive_start_id: Option, cursor: Option, filter: Option> }` | `Value` | List requests (cursor/filter pagination). | | `paginate_requests(page_limit)` | `Option` | `RequestQueueRequestsIterator` | Lazy request iterator. | | `prolong_request_lock(id, lock_secs, forefront)` | `&str`, `i64`, `bool` | `Value` | Extend a lock. | | `delete_request_lock(id, forefront)` | `&str`, `bool` | `()` | Release a lock. | @@ -246,6 +294,74 @@ The `forefront` boolean (on `add_request`, `update_request`, `batch_add_requests request(s) at the **front** of the queue so they are handled before the existing backlog; `false` (the usual choice) appends them at the **back**. +### Batch-adding requests — `batch_add_requests` / `batch_add_requests_with_options` + +`batch_add_requests(requests, forefront)` is the common case: it delegates to +`batch_add_requests_with_options` with reference-matching defaults, so most callers never need +the options form. Both handle an arbitrarily large `requests` slice safely and never fail merely +because part of a large batch was rate-limited: + +- **Chunking.** The API's `requests/batch` endpoint accepts at most 25 requests per call and a + limited payload size. `requests` is split into chunks of at most 25 items, further sliced so + each chunk's serialized JSON stays under the API's byte-size limit (large `user_data` payloads + can make even a handful of requests exceed it). +- **Parallelism.** Chunks are sent concurrently, up to `options.max_parallel` calls in flight at + once (default `5`). +- **Unprocessed-request retries.** A chunk call can report some of its requests as + `unprocessedRequests` (typically transient rate-limiting). These are automatically retried, up + to `options.max_unprocessed_requests_retries` times (default `3`), with exponential backoff + starting at `options.min_delay_between_unprocessed_requests_retries` (default `500ms`). +- **No throw on partial failure.** If a chunk call fails outright (not just reports + `unprocessedRequests`), the method does not return an `Err` for the whole batch: the requests in + that chunk are folded into the returned `unprocessedRequests` instead, matching the JS reference + client. Always check the returned `unprocessedRequests` array, not just `Ok`/`Err`, to detect a + partially-submitted batch. + +`BatchAddRequestsOptions` fields (all optional): + +- `forefront` — add all requests to the front of the queue (default `false`). +- `max_unprocessed_requests_retries` — retry attempts for a chunk's `unprocessedRequests` (default + `3`). +- `max_parallel` — maximum concurrent `requests/batch` calls (default `5`). +- `min_delay_between_unprocessed_requests_retries` — base backoff delay before the first retry, + doubling (with jitter) on each subsequent one (default `500ms`). + +```rust,no_run +use apify_client::{models::RequestQueueRequest, BatchAddRequestsOptions}; +# use apify_client::ApifyClient; +# async fn run(client: ApifyClient) -> Result<(), Box> { +let queue = client.request_queues().get_or_create(None).await?; +let requests: Vec = (0..100) + .map(|i| RequestQueueRequest { + id: None, + url: format!("https://example.com/{i}"), + unique_key: Some(format!("page-{i}")), + method: Some("GET".to_string()), + user_data: None, + extra: Default::default(), + }) + .collect(); + +// Explicit options: cap concurrency and retries beyond the defaults. +let result = client + .request_queue(&queue.id) + .batch_add_requests_with_options( + &requests, + BatchAddRequestsOptions { + max_parallel: Some(2), + ..Default::default() + }, + ) + .await?; +println!( + "processed {}, unprocessed {}", + result["processedRequests"].as_array().map_or(0, Vec::len), + result["unprocessedRequests"].as_array().map_or(0, Vec::len), +); +# Ok(()) +# } +``` + Some request-queue methods return an untyped `serde_json::Value` because the API responses are open-ended and most callers do not consume them structurally. Their shapes (read fields with `value.get("...")`): @@ -258,6 +374,53 @@ open-ended and most callers do not consume them structurally. Their shapes (read `exclusiveStartId` for cursor continuation. - `unlock_requests` → an object reporting how many locks were released (`unlockedCount`). +### Locking requests + +`list_and_lock_head`, `prolong_request_lock`, `delete_request_lock`, and `unlock_requests` let +several consumers share one queue without processing the same request twice: a consumer locks a +batch of requests off the head, processes them, and either deletes them (done) or releases the +lock (so another consumer can pick them up). Give each consumer its own `RequestQueueClient` via +`with_client_key` so the API can tell locks apart by owner — reuse the *same* client (and key) +for the whole lock/prolong/unlock lifecycle of that consumer: + +```rust,no_run +# use apify_client::ApifyClient; +# async fn run(client: ApifyClient) -> Result<(), Box> { +let queue = client.request_queues().get_or_create(None).await?; +// One client per consumer; `client_key` should be stable and unique per consumer process. +let queue_client = client + .request_queue(&queue.id) + .with_client_key("worker-1"); + +// Lock up to 10 requests from the head for 60 seconds. +let locked = queue_client.list_and_lock_head(60, Some(10)).await?; +let items = locked["items"].as_array().cloned().unwrap_or_default(); +println!("locked {} request(s)", items.len()); + +for item in &items { + // Skip anything without a usable ID rather than issuing a request against an empty path. + let Some(id) = item["id"].as_str() else { + continue; + }; + + // ... process the request here ... + + // If processing takes longer than expected, extend the lock instead of losing it: + // queue_client.prolong_request_lock(id, 60, false).await?; + + // Or, to give up on a request without deleting it, release just its lock: + // queue_client.delete_request_lock(id, false).await?; + + // Done: delete it. A missing/already-deleted request errors (see error handling). + queue_client.delete_request(id).await?; +} + +// Release every lock still held by this client (e.g. on shutdown): +queue_client.unlock_requests().await?; +# Ok(()) +# } +``` + ### `RequestQueueRequest` and request-queue return types `RequestQueueRequest` (from `apify_client::models`) is the value passed to `add_request` / diff --git a/docs/tasks.md b/docs/tasks.md index 8c450f0..9289141 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -2,6 +2,9 @@ Obtained via `client.tasks()` (collection) and `client.task(id)` (single). +> Code blocks below use the rustdoc `# `-hidden-line convention — see +> [`docs/README.md`](README.md) for what those lines are. + ## `TaskCollectionClient` | Method | Arguments | Returns | Description | @@ -17,14 +20,67 @@ Obtained via `client.tasks()` (collection) and `client.task(id)` (single). | `get()` | — | `Option` | Fetches the task. | | `update(fields)` | `&impl Serialize` | `Task` | Updates the task. | | `delete()` | — | `()` | Deletes the task. | -| `start(input, options)` | `Option<&impl Serialize>`, `ActorStartOptions` | `ActorRun` | Starts a run. See [`ActorStartOptions`](actors.md#actorstartoptions) for the full field list. | -| `call(input, options, wait_secs)` | `Option<&impl Serialize>`, `ActorStartOptions`, `Option` | `ActorRun` | Starts a run and waits. Same [`ActorStartOptions`](actors.md#actorstartoptions) as `start`. | +| `start::(input, options)` | `Option<&T>`, `TaskStartOptions` | `ActorRun` | Starts a run. `T` is a generic type parameter (not argument-position `impl Trait`), so it can be turbofished, e.g. `start::(...)`. See [`TaskStartOptions`](#taskstartoptions-and-taskcalloptions) below. | +| `call::(input, options, wait_secs)` | `Option<&T>`, `TaskCallOptions`, `Option` | `ActorRun` | Starts a run and waits. See [`TaskCallOptions`](#taskstartoptions-and-taskcalloptions) below. | | `get_input()` / `update_input(input)` | — / `&impl Serialize` | `Option` / `Value` | The task's saved input. | | `last_run(status)` | `Option<&str>` | `RunClient` | The task's last run, optionally filtered by status. See [Actor runs](runs.md) for the accepted `status` values. | -| `last_run_with_options(options)` | `LastRunOptions { status, origin }` | `RunClient` | The task's last run, optionally filtered by status and/or origin. See [Actor runs](runs.md) for the accepted `status` and `origin` values (common origins: `DEVELOPMENT`, `WEB`, `API`, `SCHEDULER`). | +| `last_run_with_options(options)` | `LastRunOptions { status: Option, origin: Option }` | `RunClient` | The task's last run, optionally filtered by status and/or origin. See [Actor runs](runs.md) for the accepted `status` and `origin` values (common origins: `DEVELOPMENT`, `WEB`, `API`, `SCHEDULER`). | | `runs()` | — | `RunCollectionClient` | The task's runs. | | `webhooks()` | — | `WebhookCollectionClient` | The task's webhooks. | +### `TaskStartOptions` and `TaskCallOptions` + +Both are narrowed versions of [`ActorStartOptions`](actors.md#actorstartoptions), matching the +JS reference client's `TaskStartOptions`/`TaskCallOptions`. A task's input content type is fixed +and a task run does not accept a permission-level override, so neither type has a `content_type` +or `force_permission_level` field (present on `ActorStartOptions` for Actor `start`/`call`). +`TaskCallOptions` additionally drops `wait_for_finish` (the server-side wait): `call`'s separate +`wait_secs` argument is how you control call's wait behavior, so the two should not be set +together. + +`TaskStartOptions` fields (all optional): + +| Field | Type | Description | +|---|---|---| +| `build` | `Option` | Tag or number of the build to run (e.g. `latest`, `0.1.2`). | +| `memory_mbytes` | `Option` | Memory in megabytes allocated for the run. | +| `timeout_secs` | `Option` | Timeout for the run in seconds (`0` means no timeout). | +| `wait_for_finish` | `Option` | Maximum seconds to wait server-side for the run to finish (max 60). | +| `max_items` | `Option` | Maximum number of dataset items to charge (pay-per-result Actors). | +| `max_total_charge_usd` | `Option` | Maximum total charge in USD (pay-per-event Actors). | +| `restart_on_error` | `Option` | Whether to restart the run if it fails. | +| `webhooks` | `Option>` | Ad-hoc webhooks to attach to this run. | + +`TaskCallOptions` has the same fields except `wait_for_finish`. + +### Creating a task + +`create` takes the same shape as the [Create task API](https://docs.apify.com/api/v2/actor-tasks-post): +at minimum `actId` (the Actor to run) and a `name`; `options` and `input` seed the task's default +run configuration and input (both optional, and both overridable per-run via `start`/`call`). + +```rust,no_run +use apify_client::ApifyClient; +use serde_json::json; + +# async fn run(client: ApifyClient) -> Result<(), Box> { +let task = client + .tasks() + .create(&json!({ + "actId": "apify/hello-world", + "name": "my-hello-world-task", + "options": { "memoryMbytes": 256 }, + "input": { "message": "hi" } + })) + .await?; +println!("created task {}", task.id); +# Ok(()) +# } +``` + +See the [`tasks_schedules_webhooks`](../examples/tasks_schedules_webhooks.rs) example for the +task's full lifecycle (create, run, inspect its webhooks, update, delete). + ## The `Task` model `Task` lives in `apify_client::models` (`use apify_client::models::Task;`). Returned by `get`, diff --git a/docs/webhooks.md b/docs/webhooks.md index 13b8ec3..9a83e3b 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -4,6 +4,9 @@ Obtained via `client.webhooks()` / `client.webhook(id)` and `client.webhook_dispatches()` / `client.webhook_dispatch(id)`. Actor- and task-scoped webhook collections are available via `actor.webhooks()` and `task.webhooks()`. +> Code blocks below use the rustdoc `# `-hidden-line convention — see +> [`docs/README.md`](README.md) for what those lines are. + ## `WebhookCollectionClient` | Method | Arguments | Returns | Description | @@ -19,14 +22,42 @@ webhook collections are available via `actor.webhooks()` and `task.webhooks()`. | `get()` | — | `Option` | Fetches the webhook. | | `update(fields)` | `&impl Serialize` | `Webhook` | Updates the webhook. | | `delete()` | — | `()` | Deletes the webhook. | -| `test()` | — | `WebhookDispatch` | Triggers a test dispatch. | +| `test()` | — | `Option` | Triggers a test dispatch, or `None` if the webhook no longer exists. | | `dispatches()` | — | `WebhookDispatchCollectionClient` | This webhook's dispatches. | ## Webhook dispatches -`WebhookDispatchCollectionClient`: `list(options)`, `iterate(options)` (lazy -`ListIterator` auto-pagination). -`WebhookDispatchClient`: `get()`. +`WebhookDispatchCollectionClient`: `list(options: ListOptions)`, `iterate(options: ListOptions)` +(lazy `ListIterator` auto-pagination). +`WebhookDispatchClient::get() -> Option`: fetches the dispatch (`None` if +missing), mirroring `WebhookClient::get()`'s `Option` shape. + +### Creating a webhook + +`create` takes the same shape as the [Create webhook API](https://docs.apify.com/api/v2/webhooks-post): +`eventTypes`, a `condition` (`{ actorId }`, `{ actorTaskId }` or `{ actorRunId }`, selecting what the +webhook watches) and a `requestUrl` to POST to. + +```rust,no_run +use apify_client::ApifyClient; +use serde_json::json; + +# async fn run(client: ApifyClient, task_id: &str) -> Result<(), Box> { +let webhook = client + .webhooks() + .create(&json!({ + "eventTypes": ["ACTOR.RUN.SUCCEEDED"], + "condition": { "actorTaskId": task_id }, + "requestUrl": "https://example.com/webhook" + })) + .await?; +println!("created webhook {}", webhook.id); +# Ok(()) +# } +``` + +See the [`tasks_schedules_webhooks`](../examples/tasks_schedules_webhooks.rs) example for +`test()`-ing a webhook, listing its dispatches, `update` and `delete`. ## The `Webhook` model diff --git a/examples/get_account.rs b/examples/get_account.rs index 4e1756a..8e312e8 100644 --- a/examples/get_account.rs +++ b/examples/get_account.rs @@ -16,7 +16,7 @@ async fn main() -> Result<(), Box> { // Monthly usage for the current billing cycle (`None` == current cycle). let usage = client.me().monthly_usage().await?; - if let Some(cycle) = usage.get("usageCycle") { + if let Some(cycle) = usage.as_ref().and_then(|u| u.get("usageCycle")) { println!("Current usage cycle: {cycle}"); } @@ -25,7 +25,7 @@ async fn main() -> Result<(), Box> { // current day (rather than hard-coding one) so the lookup always lands on a real cycle. let date = Utc::now().format("%Y-%m-%d").to_string(); let dated_usage = client.me().monthly_usage_for_date(Some(&date)).await?; - if let Some(cycle) = dated_usage.get("usageCycle") { + if let Some(cycle) = dated_usage.as_ref().and_then(|u| u.get("usageCycle")) { println!("Usage cycle containing {date}: {cycle}"); } diff --git a/examples/log_redirection.rs b/examples/log_redirection.rs index 66ebc2a..e049dc4 100644 --- a/examples/log_redirection.rs +++ b/examples/log_redirection.rs @@ -36,7 +36,12 @@ async fn main() -> Result<(), Box> { // `"{name} -> "` convention the JS reference uses for redirected Actor logs). The log is // streamed as raw byte chunks, so we buffer until a newline to emit whole, prefixed lines. let prefix = format!("{source_actor} -> "); - let mut stream = client.run(&run.id).log().stream().await?; + let mut stream = client + .run(&run.id) + .log() + .stream() + .await? + .expect("run was just started, so its log must exist"); let mut buf = String::new(); while let Some(chunk) = stream.next().await { let chunk = chunk?; diff --git a/examples/raw_log.rs b/examples/raw_log.rs index ebb9b53..9a20a83 100644 --- a/examples/raw_log.rs +++ b/examples/raw_log.rs @@ -38,7 +38,8 @@ async fn main() -> Result<(), Box> { let mut stream = client .run(&run.id) .get_streamed_log_with_options(LogOptions { raw: Some(true) }) - .await?; + .await? + .expect("run just finished, so its log must exist"); let mut streamed_bytes = 0usize; while let Some(chunk) = stream.next().await { streamed_bytes += chunk?.len(); diff --git a/examples/storages.rs b/examples/storages.rs index 26959fe..91b4fa1 100644 --- a/examples/storages.rs +++ b/examples/storages.rs @@ -23,6 +23,12 @@ async fn main() -> Result<(), Box> { .list_items::(Default::default()) .await?; println!("Dataset {} has {} item(s)", dataset.id, items.items.len()); + let renamed = dataset_client + .update( + &json!({ "name": format!("{}-renamed", dataset.name.as_deref().unwrap_or("dataset")) }), + ) + .await?; + println!("Dataset renamed to {:?}", renamed.name); dataset_client.delete().await?; // ---- Key-value store: create, set, get ---- @@ -38,6 +44,10 @@ async fn main() -> Result<(), Box> { .await? .expect("OUTPUT was just written and is readable within the same store"); println!("KVS {} OUTPUT = {} bytes", store.id, record.value.len()); + let renamed = store_client + .update(&json!({ "name": format!("{}-renamed", store.name.as_deref().unwrap_or("kvs")) })) + .await?; + println!("KVS renamed to {:?}", renamed.name); store_client.delete().await?; // ---- Request queue: create, add, read ---- @@ -55,6 +65,10 @@ async fn main() -> Result<(), Box> { println!("RQ {} added request {}", queue.id, added.request_id); let head = queue_client.list_head(Some(10)).await?; println!("RQ head has {} request(s)", head.items.len()); + let renamed = queue_client + .update(&json!({ "name": format!("{}-renamed", queue.name.as_deref().unwrap_or("rq")) })) + .await?; + println!("RQ renamed to {:?}", renamed.name); queue_client.delete().await?; Ok(()) diff --git a/examples/tasks_schedules_webhooks.rs b/examples/tasks_schedules_webhooks.rs new file mode 100644 index 0000000..fb3fda2 --- /dev/null +++ b/examples/tasks_schedules_webhooks.rs @@ -0,0 +1,104 @@ +//! Create an Actor task, a schedule that runs it, and a webhook that watches it. +//! +//! Demonstrates the create payload shape for all three resources (documented in +//! [`docs/tasks.md`](../docs/tasks.md), [`docs/schedules.md`](../docs/schedules.md) and +//! [`docs/webhooks.md`](../docs/webhooks.md)), plus each resource's basic lifecycle. +//! +//! Run with: `APIFY_TOKEN=... cargo run --example tasks_schedules_webhooks` + +use apify_client::ApifyClient; +use serde_json::json; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let token = std::env::var("APIFY_TOKEN").expect("set APIFY_TOKEN"); + let client = ApifyClient::new(token); + + // ---- Task: create a task that runs the public hello-world Actor ---- + let task = client + .tasks() + .create(&json!({ + "actId": "apify/hello-world", + "name": format!("rust-example-task-{}", uuid_like_suffix()), + "options": { "memoryMbytes": 256 }, + "input": { "message": "hi from the tasks/schedules/webhooks example" } + })) + .await?; + println!("Created task {}", task.id); + let task_client = client.task(&task.id); + + // ---- Schedule: run the task on a (disabled, so this example doesn't actually fire it) cron ---- + let schedule = client + .schedules() + .create(&json!({ + "name": format!("rust-example-schedule-{}", uuid_like_suffix()), + "cronExpression": "0 12 * * *", + "isEnabled": false, + "isExclusive": true, + "actions": [ + { "type": "RUN_ACTOR_TASK", "actorTaskId": task.id } + ] + })) + .await?; + println!("Created schedule {}", schedule.id); + let schedule_client = client.schedule(&schedule.id); + + // A schedule that has never fired has no invocation log yet. + let log = schedule_client.get_log().await?; + println!("Schedule log present: {}", log.is_some()); + + // ---- Webhook: notify a URL whenever this task's runs succeed ---- + let webhook = client + .webhooks() + .create(&json!({ + "eventTypes": ["ACTOR.RUN.SUCCEEDED"], + "condition": { "actorTaskId": task.id }, + "requestUrl": "https://example.com/webhook" + })) + .await?; + println!("Created webhook {}", webhook.id); + let webhook_client = client.webhook(&webhook.id); + + // Trigger a test dispatch (does not require a real run) and list this webhook's dispatches. + let dispatch = webhook_client + .test() + .await? + .expect("webhook was just created, so it must still exist"); + println!("Test dispatch {}", dispatch.id); + let dispatches = webhook_client.dispatches().list(Default::default()).await?; + println!("Webhook has {} dispatch(es)", dispatches.total); + + // The task's own webhook sub-collection lists webhooks scoped to it (may be empty here since + // the webhook above was created top-level, not via `task.webhooks()`). + let task_webhooks = task_client.webhooks().list(Default::default()).await?; + println!( + "Task-scoped webhook collection has {} item(s)", + task_webhooks.total + ); + + // ---- Update each resource, then clean up ---- + task_client + .update(&json!({ "name": format!("{}-renamed", task.name.as_deref().unwrap_or("task")) })) + .await?; + schedule_client + .update(&json!({ "isEnabled": false, "cronExpression": "30 12 * * *" })) + .await?; + webhook_client + .update(&json!({ "requestUrl": "https://example.com/webhook-updated" })) + .await?; + + webhook_client.delete().await?; + schedule_client.delete().await?; + task_client.delete().await?; + println!("Cleaned up task, schedule and webhook"); + + Ok(()) +} + +/// A short, timestamp-based suffix so repeated example runs don't collide on names. +fn uuid_like_suffix() -> String { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos().to_string()) + .unwrap_or_else(|_| "0".to_string()) +} diff --git a/src/client.rs b/src/client.rs index 78742b6..af81af7 100644 --- a/src/client.rs +++ b/src/client.rs @@ -261,13 +261,7 @@ impl ApifyClient { /// Returns a client for a specific Actor run. pub fn run(&self, id: impl Into) -> RunClient { - RunClient::new( - self.clone(), - self.http(), - &self.base_url, - "actor-runs", - &id.into(), - ) + RunClient::new(self.http(), &self.base_url, "actor-runs", &id.into()) } // ----- Dataset accessors ---------------------------------------------- diff --git a/src/clients/actor.rs b/src/clients/actor.rs index c1318f3..ae4a57a 100644 --- a/src/clients/actor.rs +++ b/src/clients/actor.rs @@ -1,22 +1,22 @@ //! Client for a single Actor (`/v2/actors/{actorId}`). use serde::Serialize; -use serde_json::Value; use crate::client::ApifyClient; use crate::clients::actor_version::ActorVersionClient; use crate::clients::actor_version_collection::ActorVersionCollectionClient; use crate::clients::base::{ - delete_resource, get_resource, post_with_body, update_resource, ResourceContext, + delete_resource, get_resource, get_resource_required, post_with_body, update_resource, + ResourceContext, }; use crate::clients::build::BuildClient; use crate::clients::build_collection::BuildCollectionClient; use crate::clients::run::{LastRunOptions, RunClient}; use crate::clients::run_collection::RunCollectionClient; use crate::clients::webhook_collection::WebhookCollectionClient; -use crate::common::{parse_data_envelope, QueryParams}; +use crate::common::QueryParams; use crate::error::ApifyClientResult; -use crate::http_client::HttpClient; +use crate::http_client::{HttpClient, CONTENT_TYPE_JSON}; use crate::models::{Actor, ActorRun, Build}; /// Options shared by [`ActorClient::start`] and [`ActorClient::call`] (and the task @@ -47,8 +47,7 @@ pub struct ActorStartOptions { } impl ActorStartOptions { - /// Serializes these options into run-start query parameters. Shared by the Actor and - /// task start methods (DRY). + /// Serializes these options into run-start query parameters. pub(crate) fn apply(&self, params: &mut QueryParams) { params .add_str("build", self.build.clone()) @@ -59,18 +58,67 @@ impl ActorStartOptions { .add_float("maxTotalChargeUsd", self.max_total_charge_usd) .add_bool("restartOnError", self.restart_on_error) .add_str("forcePermissionLevel", self.force_permission_level.clone()) - .add_str("webhooks", self.encoded_webhooks()); + .add_str("webhooks", encode_webhooks(&self.webhooks)); } +} + +/// Options for [`ActorClient::call`]. +/// +/// Like [`ActorStartOptions`], but without `wait_for_finish` (the server-side wait): `call`'s +/// separate `wait_secs` argument controls the client-side wait instead, so the two should not be +/// set together. This mirrors the JS reference client's `ActorCallOptions`, typed as +/// `Omit`. +#[derive(Debug, Default, Clone)] +pub struct ActorCallOptions { + /// Tag or number of the build to run (e.g. `latest`, `0.1.2`). + pub build: Option, + /// Memory in megabytes allocated for the run. + pub memory_mbytes: Option, + /// Timeout for the run in seconds (`0` means no timeout). + pub timeout_secs: Option, + /// Maximum number of dataset items to charge (pay-per-result Actors). + pub max_items: Option, + /// Maximum total charge in USD (pay-per-event Actors). + pub max_total_charge_usd: Option, + /// Content type of the input body. Defaults to `application/json`. + pub content_type: Option, + /// Whether to restart the run if it fails. + pub restart_on_error: Option, + /// Override the Actor's permission level for this run. + pub force_permission_level: Option, + /// Ad-hoc webhooks to attach to this run. Serialized to base64-encoded JSON as the + /// `webhooks` query parameter, matching the reference clients. + pub webhooks: Option>, +} - /// Encodes the `webhooks` array as base64-encoded JSON, as required by the API. - fn encoded_webhooks(&self) -> Option { - use base64::Engine; - let webhooks = self.webhooks.as_ref()?; - let json = serde_json::to_vec(webhooks).ok()?; - Some(base64::engine::general_purpose::STANDARD.encode(json)) +impl From for ActorStartOptions { + fn from(options: ActorCallOptions) -> Self { + ActorStartOptions { + build: options.build, + memory_mbytes: options.memory_mbytes, + timeout_secs: options.timeout_secs, + wait_for_finish: None, + max_items: options.max_items, + max_total_charge_usd: options.max_total_charge_usd, + content_type: options.content_type, + restart_on_error: options.restart_on_error, + force_permission_level: options.force_permission_level, + webhooks: options.webhooks, + } } } +/// Encodes a `webhooks` array as base64-encoded JSON, as required by the API. Shared by +/// [`ActorStartOptions`] and the task equivalents +/// ([`TaskStartOptions`](crate::clients::task::TaskStartOptions)), which carry the same +/// `webhooks` field (DRY). +pub(crate) fn encode_webhooks(webhooks: &Option>) -> Option { + use base64::Engine; + let webhooks = webhooks.as_ref()?; + let json = serde_json::to_vec(webhooks).ok()?; + Some(base64::engine::general_purpose::STANDARD.encode(json)) +} + /// Options for building an Actor. #[derive(Debug, Default, Clone)] pub struct ActorBuildOptions { @@ -134,7 +182,7 @@ impl ActorClient { let content_type = options .content_type .clone() - .unwrap_or_else(|| "application/json".to_string()); + .unwrap_or_else(|| CONTENT_TYPE_JSON.to_string()); let body = match input { Some(value) => Some(serde_json::to_vec(value)?), None => None, @@ -149,13 +197,17 @@ impl ActorClient { /// - `Some(n)` bounds the wait to roughly `n` seconds; if the run has not finished by /// then, the **last fetched (still non-terminal) run is returned** rather than an /// error. Check `status` / `is_terminal()` on the result when using `Some`. + /// + /// `options` is [`ActorCallOptions`], which (matching the JS reference client) excludes + /// `wait_for_finish` (the server-side wait) since the client-side `wait_secs` argument is how + /// callers control call's wait behavior. pub async fn call( &self, input: Option<&T>, - options: ActorStartOptions, + options: ActorCallOptions, wait_secs: Option, ) -> ApifyClientResult { - let run = self.start(input, options).await?; + let run = self.start(input, options.into()).await?; // Use the root client's run client so polling targets the canonical run route. self.root.run(run.id).wait_for_finish(wait_secs).await } @@ -173,7 +225,7 @@ impl ActorClient { .add_str("tag", options.tag) .add_bool("useCache", options.use_cache) .add_int("waitForFinish", options.wait_for_finish); - post_with_body(&self.ctx, Some("builds"), ¶ms, None, "application/json").await + post_with_body(&self.ctx, Some("builds"), ¶ms, None, CONTENT_TYPE_JSON).await } /// Resolves the Actor's default build and returns a client for it. @@ -186,19 +238,8 @@ impl ActorClient { ) -> ApifyClientResult { let mut params = QueryParams::new(); params.add_int("waitForFinish", wait_for_finish); - let url = params.apply_to_url(&self.ctx.url(Some("builds/default"))); - let response = self - .ctx - .http - .call(crate::http_client::HttpRequest { - method: crate::http_client::HttpMethod::Get, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; - let build: Build = parse_data_envelope(&response.body)?; + let build: Build = + get_resource_required(&self.ctx, Some("builds/default"), ¶ms).await?; Ok(BuildClient::new( self.ctx.http.clone(), &self.base_url, @@ -208,36 +249,53 @@ impl ActorClient { /// Validates the given input against the Actor's input schema. /// - /// Uses the Actor's default build for the input schema. To validate against a specific - /// build, use [`ActorClient::validate_input_for_build`]. - pub async fn validate_input(&self, input: &T) -> ApifyClientResult { + /// Returns whether the input is valid. Uses the Actor's default build for the input schema. + /// To validate against a specific build, use [`ActorClient::validate_input_for_build`]. + pub async fn validate_input(&self, input: &T) -> ApifyClientResult { self.validate_input_for_build(input, None).await } /// Validates the given input against the input schema of a specific Actor build. /// - /// `build` is the optional tag or number of the Actor build whose input schema is used for - /// validation (e.g. `"latest"` or `"1.2.34"`); passing `None` uses the default build, which - /// is equivalent to [`ActorClient::validate_input`]. This maps to the spec's optional `build` - /// query parameter on `POST /v2/actors/{actorId}/validate-input`. + /// Returns whether the input is valid. `build` is the optional tag or number of the Actor + /// build whose input schema is used for validation (e.g. `"latest"` or `"1.2.34"`); passing + /// `None` uses the default build, which is equivalent to [`ActorClient::validate_input`]. + /// This maps to the spec's optional `build` query parameter on + /// `POST /v2/actors/{actorId}/validate-input`. pub async fn validate_input_for_build( &self, input: &T, build: Option<&str>, - ) -> ApifyClientResult { + ) -> ApifyClientResult { + /// The endpoint's bare (non-`data`-enveloped) response shape: `{ "valid": bool }`. Kept + /// private since the reference client (`response.data.valid`) surfaces only the bare + /// `bool`, not the wrapping object. + /// + /// `valid` defaults to `false` when absent rather than erroring: the spec does not + /// guarantee the field is present, and the JS reference client reads it as + /// `response.data.valid`, which is `undefined` (falsy) rather than a thrown error when + /// missing. Defaulting matches that behavior instead of surfacing a spec-violating + /// response as a deserialization `Err`. + #[derive(serde::Deserialize)] + struct ValidateInputResponse { + #[serde(default)] + valid: bool, + } + let body = serde_json::to_vec(input)?; let mut params = QueryParams::new(); params.add_str("build", build); // `validate-input` returns a bare `{ "valid": ... }` object, *not* the usual // `{ "data": ... }` envelope, so it must skip `parse_data_envelope`. - crate::clients::base::post_action_raw( + let response: ValidateInputResponse = crate::clients::base::post_action_raw( &self.ctx, Some("validate-input"), ¶ms, Some(body), - Some("application/json"), + Some(CONTENT_TYPE_JSON), ) - .await + .await?; + Ok(response.valid) } /// Returns a client for the last run of this Actor, optionally filtered by run status. @@ -262,20 +320,7 @@ impl ActorClient { /// parameters on `GET /v2/actors/{actorId}/runs/last` and match the reference client's /// `lastRun({ status, origin })`; leave a field as `None` to omit it. pub fn last_run_with_options(&self, options: LastRunOptions) -> RunClient { - let mut client = RunClient::new( - self.root.clone(), - self.ctx.http.clone(), - &self.ctx.url(None), - "runs", - "last", - ); - if let Some(status) = options.status.as_deref() { - client.set_base_param("status", status); - } - if let Some(origin) = options.origin.as_deref() { - client.set_base_param("origin", origin); - } - client + crate::clients::run::last_run_client(self.ctx.http.clone(), &self.ctx.url(None), &options) } /// Returns a client for this Actor's build collection. @@ -300,7 +345,7 @@ impl ActorClient { /// Returns a client for this Actor's webhook collection. pub fn webhooks(&self) -> WebhookCollectionClient { - WebhookCollectionClient::with_base(self.ctx.http.clone(), &self.ctx.url(None)) + WebhookCollectionClient::new(self.ctx.http.clone(), &self.ctx.url(None)) } /// The Actor's ID (or `username~name`) as provided. diff --git a/src/clients/actor_env_var_collection.rs b/src/clients/actor_env_var_collection.rs index 14830b7..ba96db2 100644 --- a/src/clients/actor_env_var_collection.rs +++ b/src/clients/actor_env_var_collection.rs @@ -30,7 +30,7 @@ impl ActorEnvVarCollectionClient { /// The env-var listing is not offset-paginated (the API returns every variable in a single /// page), so this yields all variables from that one page and then completes. It exists for /// interface parity with the other collection clients and the reference client. Built with - /// [`ListIterator::new_single_page`], which fetches exactly once and never re-requests. + /// `ListIterator::new_single_page`, which fetches exactly once and never re-requests. pub fn iterate(&self) -> ListIterator { let client = self.clone(); ListIterator::new_single_page(Box::new(move |_offset, _page_limit| { diff --git a/src/clients/actor_version_collection.rs b/src/clients/actor_version_collection.rs index b7e2b9f..212b12f 100644 --- a/src/clients/actor_version_collection.rs +++ b/src/clients/actor_version_collection.rs @@ -23,16 +23,16 @@ impl ActorVersionCollectionClient { } /// Lists the Actor's versions. + /// + /// `GET /v2/actors/{actorId}/versions` defines no query parameters in the spec, so `options` + /// is accepted for interface stability but otherwise ignored (no `offset`/`limit`/`desc` are + /// sent) — matching the reference client's `list(_options)`, whose parameter is documented + /// `@deprecated No options are used in the current API implementation`. pub async fn list( &self, - options: ListOptions, + _options: ListOptions, ) -> ApifyClientResult> { - let mut params = QueryParams::new(); - params - .add_int("offset", options.offset) - .add_int("limit", options.limit) - .add_bool("desc", options.desc); - list_resource(&self.ctx, None, ¶ms).await + list_resource(&self.ctx, None, &QueryParams::new()).await } /// Lazily iterates over all versions matching `options`, fetching pages on demand. diff --git a/src/clients/base.rs b/src/clients/base.rs index 65e258f..aa5c484 100644 --- a/src/clients/base.rs +++ b/src/clients/base.rs @@ -14,7 +14,9 @@ use crate::common::{ catch_not_found, parse_data_envelope, to_safe_id, PaginationList, QueryParams, }; use crate::error::ApifyClientResult; -use crate::http_client::{HttpClient, HttpMethod, HttpRequest}; +use crate::http_client::{ + HttpClient, HttpMethod, HttpRequest, HttpResponse, CONTENT_TYPE_JSON, HEADER_CONTENT_TYPE, +}; /// How long to wait between polls while waiting for a run/build to finish. const WAIT_FOR_FINISH_POLL_INTERVAL: Duration = Duration::from_millis(250); @@ -134,7 +136,19 @@ pub(crate) async fn get_resource( sub_path: Option<&str>, params: &QueryParams, ) -> ApifyClientResult> { - let result = get_resource_required(ctx, sub_path, params).await; + get_resource_with_timeout(ctx, sub_path, params, DEFAULT_REQUEST_TIMEOUT).await +} + +/// A `GET` that unwraps the `data` envelope and maps `404` to `None`, with a configurable +/// timeout. Used by endpoints whose reference-client counterpart specifies a timeout other than +/// the default (e.g. request-queue `get`/`getRequest`/`listHead`, which use `SMALL_TIMEOUT_MILLIS`). +pub(crate) async fn get_resource_with_timeout( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + timeout: Duration, +) -> ApifyClientResult> { + let result = get_resource_required_with_timeout(ctx, sub_path, params, timeout).await; catch_not_found(result) } @@ -144,17 +158,19 @@ pub(crate) async fn get_resource_required( sub_path: Option<&str>, params: &QueryParams, ) -> ApifyClientResult { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); - let response = ctx - .http - .call(HttpRequest { - method: HttpMethod::Get, - url, - headers: Default::default(), - body: None, - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await?; + get_resource_required_with_timeout(ctx, sub_path, params, DEFAULT_REQUEST_TIMEOUT).await +} + +/// A `GET` that unwraps the `data` envelope, propagates errors (including `404`), and takes a +/// configurable timeout. See [`get_resource_with_timeout`] for why this variant exists. +pub(crate) async fn get_resource_required_with_timeout( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + timeout: Duration, +) -> ApifyClientResult { + let response = + send_with_body(ctx, HttpMethod::Get, sub_path, params, None, None, timeout).await?; parse_data_envelope(&response.body) } @@ -164,47 +180,98 @@ pub(crate) async fn update_resource( sub_path: Option<&str>, body: &B, ) -> ApifyClientResult { - let url = ctx - .merged_params(&QueryParams::new()) - .apply_to_url(&ctx.url(sub_path)); + update_resource_with_params( + ctx, + sub_path, + &QueryParams::new(), + body, + DEFAULT_REQUEST_TIMEOUT, + ) + .await +} + +/// A `PUT` with a JSON body, extra query parameters and a configurable timeout, unwrapping the +/// `data` envelope from the response. Used by endpoints whose update also takes parameters +/// beyond the resource's own (e.g. request-queue `updateRequest`'s `forefront`/`clientKey`, with +/// the reference client's `MEDIUM_TIMEOUT_MILLIS`). +pub(crate) async fn update_resource_with_params( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + body: &B, + timeout: Duration, +) -> ApifyClientResult { let body_bytes = serde_json::to_vec(body)?; - let mut headers = std::collections::HashMap::new(); - headers.insert("Content-Type".to_string(), "application/json".to_string()); - let response = ctx - .http - .call(HttpRequest { - method: HttpMethod::Put, - url, - headers, - body: Some(body_bytes), - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await?; + let response = put_send( + ctx, + sub_path, + params, + Some(body_bytes), + Some(CONTENT_TYPE_JSON), + timeout, + ) + .await?; parse_data_envelope(&response.body) } -/// A `DELETE` that maps `404` to a successful no-op. +/// A `DELETE` that maps `404` to a successful no-op. Used by every resource's whole-item +/// `delete()` (matching the reference client's base `_delete()`). pub(crate) async fn delete_resource( ctx: &ResourceContext, sub_path: Option<&str>, ) -> ApifyClientResult<()> { - let url = ctx - .merged_params(&QueryParams::new()) - .apply_to_url(&ctx.url(sub_path)); - let result = ctx - .http - .call(HttpRequest { - method: HttpMethod::Delete, - url, - headers: Default::default(), - body: None, - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await; + delete_resource_with_timeout(ctx, sub_path, DEFAULT_REQUEST_TIMEOUT).await +} + +/// A `DELETE` that maps `404` to a successful no-op, with a configurable timeout. Used by +/// resources whose reference-client `delete()` specifies a timeout other than the default (e.g. +/// request-queue `delete`, which uses `SMALL_TIMEOUT_MILLIS`). +pub(crate) async fn delete_resource_with_timeout( + ctx: &ResourceContext, + sub_path: Option<&str>, + timeout: Duration, +) -> ApifyClientResult<()> { + let result = send_with_body( + ctx, + HttpMethod::Delete, + sub_path, + &QueryParams::new(), + None, + None, + timeout, + ) + .await; catch_not_found(result.map(|_| ()))?; Ok(()) } +/// A `DELETE` of a single item within a collection (a request-queue request/lock, a +/// key-value-store record) that **propagates** any error status, including `404`. +/// +/// This intentionally does *not* map 404 to a no-op, unlike [`delete_resource`]: the JS reference +/// client's `deleteRequest`/`deleteRecord` call the HTTP client directly with no +/// `catchNotFoundOrThrow`, while only its base `_delete()` (used by whole-resource `delete()`) +/// catches not-found. Mirroring that split keeps this client's behavior consistent with the +/// reference for both cases, even though the two look inconsistent with each other locally. +pub(crate) async fn delete_item( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + timeout: Duration, +) -> ApifyClientResult<()> { + send_with_body( + ctx, + HttpMethod::Delete, + sub_path, + params, + None, + None, + timeout, + ) + .await?; + Ok(()) +} + /// A `GET` that returns a paginated list (`data` envelope wrapping `{ items, total, ... }`). pub(crate) async fn list_resource( ctx: &ResourceContext, @@ -220,20 +287,17 @@ pub(crate) async fn create_resource( params: &QueryParams, body: &B, ) -> ApifyClientResult { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(None)); let body_bytes = serde_json::to_vec(body)?; - let mut headers = std::collections::HashMap::new(); - headers.insert("Content-Type".to_string(), "application/json".to_string()); - let response = ctx - .http - .call(HttpRequest { - method: HttpMethod::Post, - url, - headers, - body: Some(body_bytes), - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await?; + let response = send_with_body( + ctx, + HttpMethod::Post, + None, + params, + Some(body_bytes), + Some(CONTENT_TYPE_JSON), + DEFAULT_REQUEST_TIMEOUT, + ) + .await?; parse_data_envelope(&response.body) } @@ -242,23 +306,59 @@ pub(crate) async fn create_resource( pub(crate) async fn get_or_create_named( ctx: &ResourceContext, name: Option<&str>, +) -> ApifyClientResult { + get_or_create_named_with_schema(ctx, name, None).await +} + +/// A `POST` that gets-or-creates a named resource (`POST {collection}?name=...`), optionally +/// sending a `{ "schema": ... }` JSON body, and unwrapping the `data` envelope. +/// +/// The OpenAPI spec for `POST /v2/datasets` and `POST /v2/key-value-stores` documents only the +/// `name` query parameter; `schema` is a JS-reference-only convenience (`getOrCreate(name, +/// { schema })`, sent as the POST body via `this._getOrCreate(name, options)`). Passing `None` +/// sends no body at all, matching the JS client's behavior when `options` is omitted (identical +/// to [`get_or_create_named`]'s request shape). +pub(crate) async fn get_or_create_named_with_schema( + ctx: &ResourceContext, + name: Option<&str>, + schema: Option<&serde_json::Value>, ) -> ApifyClientResult { let mut params = QueryParams::new(); params.add_str("name", name.map(|s| s.to_string())); - let url = params.apply_to_url(&ctx.url(None)); - let response = ctx - .http - .call(HttpRequest { - method: HttpMethod::Post, - url, - headers: Default::default(), - body: None, - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await?; + let body = match schema { + Some(schema) => Some(serde_json::to_vec( + &serde_json::json!({ "schema": schema }), + )?), + None => None, + }; + let content_type = body.as_ref().map(|_| CONTENT_TYPE_JSON); + let response = send_with_body( + ctx, + HttpMethod::Post, + None, + ¶ms, + body, + content_type, + DEFAULT_REQUEST_TIMEOUT, + ) + .await?; parse_data_envelope(&response.body) } +/// A `POST` that unwraps the `data` envelope from the response and maps `404` to `None`. Used by +/// endpoints whose reference-client counterpart wraps the call in `catchNotFoundOrThrow` (e.g. +/// testing a webhook that no longer exists). +pub(crate) async fn post_action_optional( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + body: Option>, + content_type: Option<&str>, +) -> ApifyClientResult> { + let result: ApifyClientResult = post_action(ctx, sub_path, params, body, content_type).await; + catch_not_found(result) +} + /// A `POST` that unwraps the `data` envelope from the response. Used by the common /// `{ "data": ... }`-enveloped endpoints; see [`post_action_raw`] for endpoints that return a /// bare (un-enveloped) body. @@ -269,7 +369,39 @@ pub(crate) async fn post_action( body: Option>, content_type: Option<&str>, ) -> ApifyClientResult { - let response = post_send(ctx, sub_path, params, body, content_type).await?; + post_action_with_timeout( + ctx, + sub_path, + params, + body, + content_type, + DEFAULT_REQUEST_TIMEOUT, + ) + .await +} + +/// A `POST` that unwraps the `data` envelope from the response, with a configurable timeout. Used +/// by endpoints whose reference-client counterpart specifies a timeout other than the default +/// (e.g. request-queue `addRequest`/`listAndLockHead`/the `requests/batch` POST/`unlockRequests`, +/// which use `SMALL_TIMEOUT_MILLIS`/`MEDIUM_TIMEOUT_MILLIS`). +pub(crate) async fn post_action_with_timeout( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + body: Option>, + content_type: Option<&str>, + timeout: Duration, +) -> ApifyClientResult { + let response = send_with_body( + ctx, + HttpMethod::Post, + sub_path, + params, + body, + content_type, + timeout, + ) + .await?; parse_data_envelope(&response.body) } @@ -296,78 +428,211 @@ async fn post_send( params: &QueryParams, body: Option>, content_type: Option<&str>, -) -> ApifyClientResult { +) -> ApifyClientResult { + send_with_body( + ctx, + HttpMethod::Post, + sub_path, + params, + body, + content_type, + DEFAULT_REQUEST_TIMEOUT, + ) + .await +} + +/// Shared `PUT` sender, the `PUT` counterpart of [`post_send`]. Builds the URL with merged query +/// params, sets the optional `Content-Type`, and returns the raw response. +async fn put_send( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + body: Option>, + content_type: Option<&str>, + timeout: Duration, +) -> ApifyClientResult { + send_with_body( + ctx, + HttpMethod::Put, + sub_path, + params, + body, + content_type, + timeout, + ) + .await +} + +/// Shared sender for a request that may carry a body and an optional `Content-Type`, used by +/// both [`post_send`] and [`put_send`]. +async fn send_with_body( + ctx: &ResourceContext, + method: HttpMethod, + sub_path: Option<&str>, + params: &QueryParams, + body: Option>, + content_type: Option<&str>, + timeout: Duration, +) -> ApifyClientResult { let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); let mut headers = std::collections::HashMap::new(); if let Some(ct) = content_type { - headers.insert("Content-Type".to_string(), ct.to_string()); + headers.insert(HEADER_CONTENT_TYPE.to_string(), ct.to_string()); } ctx.http .call(HttpRequest { - method: HttpMethod::Post, + method, url, headers, body, - timeout: DEFAULT_REQUEST_TIMEOUT, + timeout, }) .await } +/// A `PUT` that unwraps the `data` envelope from the response, with a configurable timeout and +/// an optional body/content-type. Used by endpoints that `PUT` without a JSON-serializable body +/// (e.g. request-queue `prolongRequestLock`, which sends no body). +pub(crate) async fn put_action( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + body: Option>, + content_type: Option<&str>, + timeout: Duration, +) -> ApifyClientResult { + let response = put_send(ctx, sub_path, params, body, content_type, timeout).await?; + parse_data_envelope(&response.body) +} + +/// A `PUT` that returns the raw response body, deserialized directly **without** unwrapping a +/// `data` envelope. Mirrors [`post_action_raw`] for `PUT` endpoints that return a bare JSON body +/// (e.g. task `updateInput`). +pub(crate) async fn put_action_raw( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + body: Vec, + content_type: &str, +) -> ApifyClientResult { + let response = put_send( + ctx, + sub_path, + params, + Some(body), + Some(content_type), + DEFAULT_REQUEST_TIMEOUT, + ) + .await?; + Ok(serde_json::from_slice(&response.body)?) +} + /// A `GET` returning the raw response body bytes (no `data` envelope). Maps `404`/`HEAD` /// not-found to `None`. Used for logs and key-value-store record values. pub(crate) async fn get_raw( ctx: &ResourceContext, sub_path: Option<&str>, params: &QueryParams, -) -> ApifyClientResult> { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); - let result = ctx - .http - .call(HttpRequest { - method: HttpMethod::Get, - url, - headers: Default::default(), - body: None, - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await; +) -> ApifyClientResult> { + let result = get_raw_required(ctx, sub_path, params).await; catch_not_found(result) } +/// A `GET` returning the raw response (headers + body, no `data` envelope) and propagating any +/// error status, including `404`. Used by endpoints whose response is not resource-shaped and +/// for which a missing resource should surface as an error rather than `None` (dataset items +/// listing/export). Note: dataset *statistics* (`DatasetClient::get_statistics`) goes through +/// [`get_resource`] instead (it is envelope-wrapped, unlike items listing/export). +pub(crate) async fn get_raw_required( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, +) -> ApifyClientResult { + send_with_body( + ctx, + HttpMethod::Get, + sub_path, + params, + None, + None, + DEFAULT_REQUEST_TIMEOUT, + ) + .await +} + /// A `HEAD` request returning whether the resource exists (`true` on 2xx, `false` on 404). pub(crate) async fn head_exists( ctx: &ResourceContext, sub_path: Option<&str>, params: &QueryParams, ) -> ApifyClientResult { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); - let result = ctx - .http - .call(HttpRequest { - method: HttpMethod::Head, - url, - headers: Default::default(), - body: None, - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await; + let result = send_with_body( + ctx, + HttpMethod::Head, + sub_path, + params, + None, + None, + DEFAULT_REQUEST_TIMEOUT, + ) + .await; Ok(catch_not_found(result)?.is_some()) } -/// A `PUT` with raw bytes and a content type (used for KVS record uploads). +/// A `PUT` with raw bytes and a content type (used for KVS record uploads and user limit +/// updates), discarding the response body on success. pub(crate) async fn put_raw( ctx: &ResourceContext, sub_path: Option<&str>, params: &QueryParams, body: Vec, content_type: &str, +) -> ApifyClientResult<()> { + put_send( + ctx, + sub_path, + params, + Some(body), + Some(content_type), + DEFAULT_REQUEST_TIMEOUT, + ) + .await?; + Ok(()) +} + +/// A `POST` with raw bytes and a content type, discarding the response body on success. Used for +/// dataset item pushes, whose body is arbitrary (not envelope-shaped) user JSON and which return +/// no data. +pub(crate) async fn post_raw( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + body: Vec, + content_type: &str, +) -> ApifyClientResult<()> { + post_send(ctx, sub_path, params, Some(body), Some(content_type)).await?; + Ok(()) +} + +/// A `POST` with a JSON body and one extra header beyond `Content-Type`, discarding the response +/// body on success. The only current use is `RunClient::charge`, which must send an +/// `idempotency-key` header so a transport-retried charge is applied at most once. +pub(crate) async fn post_raw_with_extra_header( + ctx: &ResourceContext, + sub_path: Option<&str>, + params: &QueryParams, + body: Vec, + content_type: &str, + header_name: &str, + header_value: String, ) -> ApifyClientResult<()> { let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); let mut headers = std::collections::HashMap::new(); - headers.insert("Content-Type".to_string(), content_type.to_string()); + headers.insert(HEADER_CONTENT_TYPE.to_string(), content_type.to_string()); + headers.insert(header_name.to_string(), header_value); ctx.http .call(HttpRequest { - method: HttpMethod::Put, + method: HttpMethod::Post, url, headers, body: Some(body), @@ -389,27 +654,27 @@ pub(crate) async fn post_with_body( post_action(ctx, sub_path, params, body, Some(content_type)).await } -/// A `DELETE` with a JSON body (used for batch request deletion). +/// A `DELETE` with a JSON body and a configurable timeout (used for batch request deletion, whose +/// reference-client counterpart — request-queue `batchDeleteRequests` — uses +/// `SMALL_TIMEOUT_MILLIS` rather than the default). pub(crate) async fn delete_with_body( ctx: &ResourceContext, sub_path: Option<&str>, params: &QueryParams, body: &B, + timeout: Duration, ) -> ApifyClientResult { - let url = ctx.merged_params(params).apply_to_url(&ctx.url(sub_path)); let body_bytes = serde_json::to_vec(body)?; - let mut headers = std::collections::HashMap::new(); - headers.insert("Content-Type".to_string(), "application/json".to_string()); - let response = ctx - .http - .call(HttpRequest { - method: HttpMethod::Delete, - url, - headers, - body: Some(body_bytes), - timeout: DEFAULT_REQUEST_TIMEOUT, - }) - .await?; + let response = send_with_body( + ctx, + HttpMethod::Delete, + sub_path, + params, + Some(body_bytes), + Some(CONTENT_TYPE_JSON), + timeout, + ) + .await?; parse_data_envelope(&response.body) } diff --git a/src/clients/build.rs b/src/clients/build.rs index fd887df..ecc0dbf 100644 --- a/src/clients/build.rs +++ b/src/clients/build.rs @@ -9,6 +9,20 @@ use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::Build; +/// Options for fetching a build via [`BuildClient::get_with_options`]. +/// +/// Covers the spec's optional `waitForFinish` query parameter on `GET /v2/actor-builds/{buildId}`, +/// matching the reference client's `BuildClientGetOptions`. +#[derive(Debug, Default, Clone)] +pub struct BuildGetOptions { + /// Maximum time, in seconds (capped at 60 by the API), to wait server-side for the build to + /// reach a terminal state before returning. `None` (the default) returns immediately without + /// waiting. This is a single bounded server-side wait, distinct from + /// [`BuildClient::wait_for_finish`], which polls repeatedly (using this same parameter + /// internally) until the build finishes or a client-side budget is exhausted. + pub wait_for_finish: Option, +} + /// Client for a specific Actor build. #[derive(Debug, Clone)] pub struct BuildClient { @@ -23,8 +37,23 @@ impl BuildClient { } /// Fetches the build object, or `None` if it does not exist. + /// + /// Returns immediately without waiting for the build to finish. To have the API wait + /// server-side before responding, use [`BuildClient::get_with_options`]. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + self.get_with_options(BuildGetOptions::default()).await + } + + /// Fetches the build object, or `None` if it does not exist, applying the given + /// [`BuildGetOptions`] (e.g. [`BuildGetOptions::wait_for_finish`] to have the API wait, + /// server-side, for the build to finish before responding). + pub async fn get_with_options( + &self, + options: BuildGetOptions, + ) -> ApifyClientResult> { + let mut params = QueryParams::new(); + params.add_int("waitForFinish", options.wait_for_finish); + get_resource(&self.ctx, None, ¶ms).await } /// Aborts the build. diff --git a/src/clients/dataset.rs b/src/clients/dataset.rs index d59a815..8f86a0d 100644 --- a/src/clients/dataset.rs +++ b/src/clients/dataset.rs @@ -4,13 +4,22 @@ use serde::de::DeserializeOwned; use serde::Serialize; use serde_json::Value; -use crate::clients::base::{delete_resource, get_resource, update_resource, ResourceContext}; +use crate::clients::base::{ + delete_resource, get_raw_required, get_resource, post_raw, update_resource, ResourceContext, +}; use crate::clients::pagination::ListIterator; -use crate::common::{parse_data_envelope, sign_storage_content, PaginationList, QueryParams}; +use crate::common::{sign_storage_content, PaginationList, QueryParams}; use crate::error::ApifyClientResult; -use crate::http_client::{HttpClient, HttpMethod, HttpRequest}; +use crate::http_client::{HttpClient, CONTENT_TYPE_JSON_UTF8}; use crate::models::Dataset; +/// Response header reporting the total item count across the whole dataset (not just this page). +const HEADER_PAGINATION_TOTAL: &str = "x-apify-pagination-total"; +/// Response header reporting the offset the returned page started at. +const HEADER_PAGINATION_OFFSET: &str = "x-apify-pagination-offset"; +/// Response header reporting the page size actually applied. +const HEADER_PAGINATION_LIMIT: &str = "x-apify-pagination-limit"; + /// Options for listing or downloading dataset items. /// /// Covers the filtering, projection and transformation parameters of @@ -195,18 +204,7 @@ impl DatasetClient { ) -> ApifyClientResult> { let mut params = QueryParams::new(); options.apply(&mut params); - let url = params.apply_to_url(&self.ctx.url(Some("items"))); - let response = self - .ctx - .http - .call(HttpRequest { - method: HttpMethod::Get, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; + let response = get_raw_required(&self.ctx, Some("items"), ¶ms).await?; let items: Vec = serde_json::from_slice(&response.body)?; let count = items.len() as i64; @@ -214,15 +212,15 @@ impl DatasetClient { // returned would look complete and stop iteration after page one, dropping later items. // `0` routes iteration to the short-page/empty-page backstop, which walks every page. let total = response - .header("x-apify-pagination-total") + .header(HEADER_PAGINATION_TOTAL) .and_then(|v| v.parse().ok()) .unwrap_or(0); let offset = response - .header("x-apify-pagination-offset") + .header(HEADER_PAGINATION_OFFSET) .and_then(|v| v.parse().ok()) .unwrap_or(0); let limit = response - .header("x-apify-pagination-limit") + .header(HEADER_PAGINATION_LIMIT) .and_then(|v| v.parse().ok()) .unwrap_or(count); @@ -293,18 +291,7 @@ impl DatasetClient { let mut params = QueryParams::new(); params.add_str("format", Some(format.as_str())); options.apply(&mut params); - let url = params.apply_to_url(&self.ctx.url(Some("items"))); - let response = self - .ctx - .http - .call(HttpRequest { - method: HttpMethod::Get, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; + let response = get_raw_required(&self.ctx, Some("items"), ¶ms).await?; Ok(response.body) } @@ -313,23 +300,14 @@ impl DatasetClient { /// `items` must serialize to a JSON object or an array of objects. pub async fn push_items(&self, items: &T) -> ApifyClientResult<()> { let body = serde_json::to_vec(items)?; - let url = self.ctx.url(Some("items")); - let mut headers = std::collections::HashMap::new(); - headers.insert( - "Content-Type".to_string(), - "application/json; charset=utf-8".to_string(), - ); - self.ctx - .http - .call(HttpRequest { - method: HttpMethod::Post, - url, - headers, - body: Some(body), - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; - Ok(()) + post_raw( + &self.ctx, + Some("items"), + &QueryParams::new(), + body, + CONTENT_TYPE_JSON_UTF8, + ) + .await } /// Builds a public URL for downloading this dataset's items. @@ -362,21 +340,6 @@ impl DatasetClient { /// Returns statistical information about the dataset, or `None` if unavailable. pub async fn get_statistics(&self) -> ApifyClientResult> { - let result: ApifyClientResult = async { - let response = self - .ctx - .http - .call(HttpRequest { - method: HttpMethod::Get, - url: self.ctx.url(Some("statistics")), - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; - parse_data_envelope(&response.body) - } - .await; - crate::common::catch_not_found(result) + get_resource(&self.ctx, Some("statistics"), &QueryParams::new()).await } } diff --git a/src/clients/dataset_collection.rs b/src/clients/dataset_collection.rs index a2399e3..f1dd05b 100644 --- a/src/clients/dataset_collection.rs +++ b/src/clients/dataset_collection.rs @@ -1,12 +1,25 @@ //! Client for the dataset collection (`/v2/datasets`). -use crate::clients::base::{get_or_create_named, list_resource, ResourceContext}; +use crate::clients::base::{get_or_create_named_with_schema, list_resource, ResourceContext}; use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{PaginationList, QueryParams, StorageListOptions}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::Dataset; +/// Options for getting-or-creating a dataset via +/// [`DatasetCollectionClient::get_or_create_with_options`]. +/// +/// `schema` is a JS-reference-only convenience (not documented by the OpenAPI spec, which only +/// declares the `name` query parameter on `POST /v2/datasets`); it is sent as the request's +/// `{ "schema": ... }` JSON body, matching the reference client's +/// `DatasetCollectionClientGetOrCreateOptions`. +#[derive(Debug, Default, Clone)] +pub struct DatasetGetOrCreateOptions { + /// JSON schema to associate with the dataset. + pub schema: Option, +} + /// Client for listing datasets and getting-or-creating a dataset by name. #[derive(Debug, Clone)] pub struct DatasetCollectionClient { @@ -44,6 +57,18 @@ impl DatasetCollectionClient { /// /// Passing `None` for `name` creates an unnamed dataset. pub async fn get_or_create(&self, name: Option<&str>) -> ApifyClientResult { - get_or_create_named(&self.ctx, name).await + self.get_or_create_with_options(name, DatasetGetOrCreateOptions::default()) + .await + } + + /// Gets the dataset with the given `name`, creating it if it does not exist, applying the + /// given [`DatasetGetOrCreateOptions`] (e.g. [`DatasetGetOrCreateOptions::schema`], applied + /// only when the dataset is created). + pub async fn get_or_create_with_options( + &self, + name: Option<&str>, + options: DatasetGetOrCreateOptions, + ) -> ApifyClientResult { + get_or_create_named_with_schema(&self.ctx, name, options.schema.as_ref()).await } } diff --git a/src/clients/key_value_store.rs b/src/clients/key_value_store.rs index bd68d74..59c54ea 100644 --- a/src/clients/key_value_store.rs +++ b/src/clients/key_value_store.rs @@ -5,14 +5,14 @@ use std::collections::VecDeque; use serde::Serialize; use crate::clients::base::{ - delete_resource, get_raw, get_resource, get_resource_required, head_exists, put_raw, - update_resource, ResourceContext, + delete_item, delete_resource, get_raw, get_resource, get_resource_required, head_exists, + put_raw, update_resource, ResourceContext, SMALL_REQUEST_TIMEOUT, }; use crate::common::{ create_hmac_signature, encode_path_segment, sign_storage_content, QueryParams, }; use crate::error::ApifyClientResult; -use crate::http_client::{HttpClient, HttpMethod, HttpRequest}; +use crate::http_client::{HttpClient, CONTENT_TYPE_JSON_UTF8}; use crate::models::{KeyValueStore, KeyValueStoreKey, KeyValueStoreKeysPage, KeyValueStoreRecord}; /// Options for listing keys in a key-value store. @@ -208,7 +208,7 @@ impl KeyValueStoreClient { value: &T, ) -> ApifyClientResult<()> { let bytes = serde_json::to_vec(value)?; - self.set_record_raw(key, bytes, "application/json; charset=utf-8") + self.set_record_raw(key, bytes, CONTENT_TYPE_JSON_UTF8) .await } @@ -260,21 +260,19 @@ impl KeyValueStoreClient { } /// Deletes the record with the given key. + /// + /// Unlike the store's own [`delete`](Self::delete), a missing record is **not** treated as a + /// no-op: this call propagates a `404` as an error, matching the JS reference client's + /// `deleteRecord`. See `delete_item`'s doc comment (`clients::base`) for the full rationale + /// behind this whole-resource-vs-sub-resource split, shared by every `delete_item` caller. pub async fn delete_record(&self, key: &str) -> ApifyClientResult<()> { - let url = self - .ctx - .url(Some(&format!("records/{}", encode_path_segment(key)))); - self.ctx - .http - .call(HttpRequest { - method: HttpMethod::Delete, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; - Ok(()) + delete_item( + &self.ctx, + Some(&format!("records/{}", encode_path_segment(key))), + &QueryParams::new(), + SMALL_REQUEST_TIMEOUT, + ) + .await } } @@ -294,9 +292,12 @@ pub const KEY_LIST_MAX_LIMIT: i64 = 1000; /// cursor-based pagination: each page is anchored by the previous page's /// `nextExclusiveStartKey`. The walk stops once the page reports `isTruncated == false` (the /// authoritative end-of-data signal), a page comes back empty, the API stops returning a next -/// cursor, or the caller's `limit` is exhausted. This yields the same result as the reference -/// client's `listKeys()` async-iterable (which loops on the cursor); leading with `isTruncated` -/// additionally avoids a wasted empty fetch when a final page still carries a cursor. +/// cursor, or the caller's `limit` is exhausted. This is an intentional divergence from the JS +/// reference client's `listKeys()` async-iterable, whose continuation condition is a three-part +/// AND (`items.length > 0 && nextExclusiveStartKey !== null && limit not exceeded`) that never +/// consults `isTruncated` at all: leading with `isTruncated` here additionally avoids a wasted +/// empty fetch when a final page still carries a cursor, which is arguably more robust than +/// mirroring the JS termination check exactly. pub struct KeyValueStoreKeysIterator { client: KeyValueStoreClient, /// Base listing options. The `prefix`/`collection`/`signature` filters are carried into every diff --git a/src/clients/key_value_store_collection.rs b/src/clients/key_value_store_collection.rs index 65a0803..7722e14 100644 --- a/src/clients/key_value_store_collection.rs +++ b/src/clients/key_value_store_collection.rs @@ -1,12 +1,25 @@ //! Client for the key-value store collection (`/v2/key-value-stores`). -use crate::clients::base::{get_or_create_named, list_resource, ResourceContext}; +use crate::clients::base::{get_or_create_named_with_schema, list_resource, ResourceContext}; use crate::clients::pagination::{list_iterator, ListIterator}; use crate::common::{PaginationList, QueryParams, StorageListOptions}; use crate::error::ApifyClientResult; use crate::http_client::HttpClient; use crate::models::KeyValueStore; +/// Options for getting-or-creating a key-value store via +/// [`KeyValueStoreCollectionClient::get_or_create_with_options`]. +/// +/// `schema` is a JS-reference-only convenience (not documented by the OpenAPI spec, which only +/// declares the `name` query parameter on `POST /v2/key-value-stores`); it is sent as the +/// request's `{ "schema": ... }` JSON body, matching the reference client's +/// `KeyValueStoreCollectionClientGetOrCreateOptions`. +#[derive(Debug, Default, Clone)] +pub struct KeyValueStoreGetOrCreateOptions { + /// JSON schema to associate with the key-value store. + pub schema: Option, +} + /// Client for listing key-value stores and getting-or-creating one by name. #[derive(Debug, Clone)] pub struct KeyValueStoreCollectionClient { @@ -43,6 +56,18 @@ impl KeyValueStoreCollectionClient { /// Gets the store with the given `name`, creating it if it does not exist. pub async fn get_or_create(&self, name: Option<&str>) -> ApifyClientResult { - get_or_create_named(&self.ctx, name).await + self.get_or_create_with_options(name, KeyValueStoreGetOrCreateOptions::default()) + .await + } + + /// Gets the store with the given `name`, creating it if it does not exist, applying the + /// given [`KeyValueStoreGetOrCreateOptions`] (e.g. + /// [`KeyValueStoreGetOrCreateOptions::schema`], applied only when the store is created). + pub async fn get_or_create_with_options( + &self, + name: Option<&str>, + options: KeyValueStoreGetOrCreateOptions, + ) -> ApifyClientResult { + get_or_create_named_with_schema(&self.ctx, name, options.schema.as_ref()).await } } diff --git a/src/clients/log.rs b/src/clients/log.rs index 933672a..66a8c04 100644 --- a/src/clients/log.rs +++ b/src/clients/log.rs @@ -7,9 +7,15 @@ use futures_util::Stream; use crate::clients::base::{get_raw, ResourceContext}; -use crate::common::QueryParams; +use crate::common::{QueryParams, NOT_FOUND_STATUS_CODE}; use crate::error::{ApifyClientError, ApifyClientResult}; -use crate::http_client::HttpClient; +use crate::http_client::{HttpClient, HEADER_AUTHORIZATION, HEADER_USER_AGENT}; + +/// Query parameter (and its value) that requests a live streaming connection to the log, +/// rather than the buffered whole-log response. +const STREAM_QUERY_PARAM: &str = "stream"; +/// Value sent for [`STREAM_QUERY_PARAM`] (the API only checks for presence, not the value). +const STREAM_QUERY_PARAM_VALUE: &str = "1"; /// Options for retrieving or streaming a log ([`LogClient::get_with_options`] / /// [`LogClient::stream_with_options`]). @@ -77,7 +83,8 @@ impl LogClient { Ok(response.map(|r| String::from_utf8_lossy(&r.body).into_owned())) } - /// Opens a streaming connection to the log, yielding chunks of bytes as they arrive. + /// Opens a streaming connection to the log, yielding chunks of bytes as they arrive, or + /// `None` if the log does not exist (e.g. the run/build was deleted). /// /// This powers real-time log redirection: callers can forward each chunk to their own /// logger/stdout while a run is still in progress. The stream completes when the log @@ -87,44 +94,58 @@ impl LogClient { /// [`LogClient::stream_with_options`]. pub async fn stream( &self, - ) -> ApifyClientResult>>> { + ) -> ApifyClientResult>>>> { self.stream_with_options(LogOptions::default()).await } /// Opens a streaming connection to the log applying the given [`LogOptions`], yielding - /// chunks of bytes as they arrive. + /// chunks of bytes as they arrive, or `None` if the log does not exist. /// /// Like [`LogClient::stream`], but lets the caller request the raw log via /// [`LogOptions::raw`] (as the reference client's log redirection does, which streams - /// `{ raw: true }`). + /// `{ raw: true }`). Mirrors [`LogClient::get`]'s `404`-to-`None` mapping: the reference + /// client's `stream()` also wraps its request in `catchNotFoundOrThrow`. pub async fn stream_with_options( &self, options: LogOptions, - ) -> ApifyClientResult>>> { + ) -> ApifyClientResult>>>> { // Streaming needs a live connection, so we go through reqwest directly rather than // the buffered backend path. The retry policy does not apply to an open stream. let client = reqwest::Client::new(); let mut params = QueryParams::new(); - params.push_raw("stream".to_string(), "1".to_string()); + params.push_raw( + STREAM_QUERY_PARAM.to_string(), + STREAM_QUERY_PARAM_VALUE.to_string(), + ); params.add_bool("raw", options.raw); let url = params.apply_to_url(&self.stream_url); - let mut builder = client.get(&url).header("User-Agent", &self.user_agent); + let mut builder = client.get(&url).header(HEADER_USER_AGENT, &self.user_agent); if let Some(token) = &self.token { - builder = builder.header("Authorization", format!("Bearer {token}")); + builder = builder.header(HEADER_AUTHORIZATION, format!("Bearer {token}")); } let response = builder.send().await.map_err(ApifyClientError::from)?; - if !response.status().is_success() { + let status = response.status(); + // Unlike `catch_not_found` (used by the buffered `HttpBackend` path), this does not also + // check the parsed `error.type` against `record-not-found`/`record-or-token-not-found`: + // a log's `404` is always `record-not-found` in practice (the endpoint has no other + // reason to 404), and this raw-`reqwest` streaming path deliberately avoids depending on + // the response body being present/parseable JSON before the stream even starts. Mapping + // every `404` to `None` here is a reasoned simplification, not an oversight. + if status.as_u16() == NOT_FOUND_STATUS_CODE { + return Ok(None); + } + if !status.is_success() { return Err(ApifyClientError::InvalidResponse(format!( "log stream returned status {}", - response.status().as_u16() + status.as_u16() ))); } let byte_stream = response.bytes_stream(); - Ok(futures_util::StreamExt::map(byte_stream, |chunk| { + Ok(Some(futures_util::StreamExt::map(byte_stream, |chunk| { chunk.map(|b| b.to_vec()).map_err(ApifyClientError::from) - })) + }))) } } diff --git a/src/clients/request_queue.rs b/src/clients/request_queue.rs index 45f1868..c6a286b 100644 --- a/src/clients/request_queue.rs +++ b/src/clients/request_queue.rs @@ -1,22 +1,70 @@ //! Client for a single request queue (`/v2/request-queues/{queueId}` and variants). +use std::collections::HashSet; +use std::time::Duration; + +use futures_util::stream::{FuturesUnordered, StreamExt}; use serde::Serialize; use crate::clients::base::{ - delete_resource, delete_with_body, get_resource, get_resource_required, post_action, - post_with_body, update_resource, ResourceContext, + delete_item, delete_resource_with_timeout, delete_with_body, + get_resource_required_with_timeout, get_resource_with_timeout, post_action_with_timeout, + put_action, update_resource_with_params, ResourceContext, MEDIUM_REQUEST_TIMEOUT, + SMALL_REQUEST_TIMEOUT, }; use crate::common::{encode_path_segment, QueryParams}; use crate::error::ApifyClientResult; -use crate::http_client::{HttpClient, HttpMethod, HttpRequest}; +use crate::http_client::{HttpClient, CONTENT_TYPE_JSON}; use crate::models::{ RequestQueue, RequestQueueHead, RequestQueueOperationInfo, RequestQueueRequest, }; /// Maximum number of requests the API accepts in a single `requests/batch` call. Larger -/// inputs are split into chunks of this size (matching the reference client). +/// inputs are split into chunks of this size (matching the reference client's +/// `REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION`). const MAX_REQUESTS_PER_BATCH_OPERATION: usize = 25; +/// Byte length of the empty JSON array literal `[]`, used as the starting accumulator when +/// summing serialized item sizes to reconstruct the byte length of the array they'll be sent in. +const EMPTY_JSON_ARRAY_BYTES: usize = 2; + +/// Maximum accepted request-body size (bytes) for a single `requests/batch` call, matching the +/// reference client's `@apify/consts` `MAX_PAYLOAD_SIZE_BYTES` (9 MiB). A chunk that would +/// serialize larger than this (even after the [`MAX_REQUESTS_PER_BATCH_OPERATION`] count cap) is +/// sliced further by [`slice_by_byte_length`]. +const MAX_PAYLOAD_SIZE_BYTES: usize = 9_437_184; +/// Fraction of [`MAX_PAYLOAD_SIZE_BYTES`] reserved as a safety buffer, so the byte-size slicing +/// targets a limit slightly under the API's actual cap. Matches the reference client's +/// `SAFETY_BUFFER_PERCENT` (0.01%). +const SAFETY_BUFFER_PERCENT: f64 = 0.0001; + +/// Default maximum number of parallel `requests/batch` calls in flight at once, matching the +/// reference client's `DEFAULT_PARALLEL_BATCH_ADD_REQUESTS`. +const DEFAULT_MAX_PARALLEL_BATCH_ADD_REQUESTS: usize = 5; +/// Default number of retry attempts for a chunk's `unprocessedRequests`, matching the reference +/// client's `DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS`. +const DEFAULT_MAX_UNPROCESSED_REQUESTS_RETRIES: u32 = 3; +/// Default minimum delay before the first unprocessed-request retry (doubled, with jitter, on +/// each subsequent retry), matching the reference client's +/// `DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS`. +const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES: Duration = Duration::from_millis(500); + +/// Options for [`RequestQueueClient::batch_add_requests`]. +#[derive(Debug, Default, Clone)] +pub struct BatchAddRequestsOptions { + /// If `true`, adds all requests to the beginning of the queue. Default `false`. + pub forefront: Option, + /// Maximum number of retry attempts for a chunk's rate-limited (`unprocessedRequests`) + /// requests. Default `DEFAULT_MAX_UNPROCESSED_REQUESTS_RETRIES` (3). + pub max_unprocessed_requests_retries: Option, + /// Maximum number of `requests/batch` API calls in flight at once. Default + /// `DEFAULT_MAX_PARALLEL_BATCH_ADD_REQUESTS` (5). + pub max_parallel: Option, + /// Minimum delay before the first unprocessed-request retry; doubles (with jitter) on each + /// subsequent retry. Default `DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES` (500ms). + pub min_delay_between_unprocessed_requests_retries: Option, +} + /// Appends the array under `key` in `chunk_result` (if present) onto `acc`. Used to merge the /// per-chunk `processedRequests` / `unprocessedRequests` arrays of a chunked batch-add. fn merge_request_array( @@ -29,6 +77,99 @@ fn merge_request_array( } } +/// The key the API uses to deduplicate/match a request: its `unique_key`, defaulting to `url` +/// (the same default the server applies when `unique_key` is omitted). +fn request_unique_key(request: &RequestQueueRequest) -> String { + request + .unique_key + .clone() + .unwrap_or_else(|| request.url.clone()) +} + +/// Collects the `uniqueKey` field of every already-`processed` request/response object, so a +/// retry can compute which of the originally-submitted requests still remain. +fn processed_unique_keys(processed: &[serde_json::Value]) -> HashSet { + processed + .iter() + .filter_map(|v| v.get("uniqueKey").and_then(|k| k.as_str())) + .map(str::to_owned) + .collect() +} + +/// Slices `requests` (already capped to at most [`MAX_REQUESTS_PER_BATCH_OPERATION`] items) +/// further so the slice's serialized JSON size stays under `max_byte_length`, mirroring the +/// reference client's `sliceArrayByByteLength`. Returns the whole input unchanged when it +/// already fits. `start_index` is only used to name the offending request in the error message. +fn slice_by_byte_length( + requests: &[RequestQueueRequest], + max_byte_length: usize, + start_index: usize, +) -> ApifyClientResult> { + if requests.is_empty() { + return Ok(Vec::new()); + } + let whole_len = serde_json::to_vec(requests)?.len(); + if whole_len < max_byte_length { + return Ok(requests.to_vec()); + } + + let mut sliced = Vec::new(); + let mut byte_length = EMPTY_JSON_ARRAY_BYTES; + for (i, item) in requests.iter().enumerate() { + let item_byte_length = serde_json::to_vec(item)?.len(); + if item_byte_length > max_byte_length { + return Err(crate::error::ApifyClientError::InvalidArgument(format!( + "RequestQueueClient::batch_add_requests: the size of the request at index {} \ + exceeds the maximum allowed size ({max_byte_length} bytes)", + start_index + i + ))); + } + if byte_length + item_byte_length >= max_byte_length { + break; + } + byte_length += item_byte_length; + sliced.push(item.clone()); + } + // A non-empty input always fits at least one item under `max_byte_length` (the per-item + // check above already rejects an item that alone exceeds it); the only way `sliced` could + // still be empty is the razor-thin case where a single item fits alone but not alongside the + // 2-byte array overhead. Force it through rather than stalling the caller's `while` loop + // (which advances by `sliced.len()`) on a zero-length chunk. + if sliced.is_empty() { + sliced.push(requests[0].clone()); + } + Ok(sliced) +} + +/// Returns the reference client's exponential-backoff-with-jitter delay for the `attempt`-th +/// retry (0-indexed): `(1 + random) * 2^attempt * min_delay`, `random` in `[0, 1)`. Matches +/// `_batchAddRequestsWithRetries`'s backoff formula. +fn unprocessed_retry_backoff(min_delay: Duration, attempt: u32) -> Duration { + let factor = 2u32.saturating_pow(attempt); + let base_millis = (min_delay.as_millis() as u64).saturating_mul(u64::from(factor)); + let extra_millis = (base_millis as f64 * random_fraction()) as u64; + Duration::from_millis(base_millis.saturating_add(extra_millis)) +} + +/// Modulus applied to [`crate::http_client::next_jitter`]'s output to derive +/// [`random_fraction`]'s numerator; also its denominator, so the result lands in `[0, 1)`. +/// `1_000_000` is an arbitrary but sufficiently fine-grained choice for jitter — not a value with +/// external meaning to name after anything more specific. +const RANDOM_FRACTION_MODULUS: u64 = 1_000_000; + +/// A cheap, non-crypto random fraction in `[0, 1)` for backoff jitter (mirrors JS `Math.random()` +/// in spirit, not in distribution quality — this is jitter, not a security-sensitive value). +/// +/// Reuses the crate's shared [`crate::http_client::next_jitter`] SplitMix64 generator (the same +/// source `HttpClient::call`'s transport-retry backoff draws from) rather than a second, +/// independent source seeded from wall-clock nanoseconds: one well-distributed generator is +/// simpler to reason about than two, and avoids the (harmless here, but needless) weaker +/// distribution of raw `SystemTime` sub-millisecond jitter under concurrent callers. +fn random_fraction() -> f64 { + let jitter = crate::http_client::next_jitter() % RANDOM_FRACTION_MODULUS; + jitter as f64 / RANDOM_FRACTION_MODULUS as f64 +} + /// Options for [`RequestQueueClient::list_requests`]. /// /// Covers the spec query parameters of `GET /v2/request-queues/{queueId}/requests`. @@ -83,24 +224,32 @@ impl RequestQueueClient { /// Fetches the queue metadata, or `None` if it does not exist. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + get_resource_with_timeout(&self.ctx, None, &QueryParams::new(), SMALL_REQUEST_TIMEOUT).await } /// Updates the queue metadata (e.g. `name`, `title`). pub async fn update(&self, new_fields: &T) -> ApifyClientResult { - update_resource(&self.ctx, None, new_fields).await + update_resource_with_params( + &self.ctx, + None, + &QueryParams::new(), + new_fields, + SMALL_REQUEST_TIMEOUT, + ) + .await } /// Deletes the queue. pub async fn delete(&self) -> ApifyClientResult<()> { - delete_resource(&self.ctx, None).await + delete_resource_with_timeout(&self.ctx, None, SMALL_REQUEST_TIMEOUT).await } /// Lists requests from the head of the queue (without locking them). pub async fn list_head(&self, limit: Option) -> ApifyClientResult { let mut params = self.base_params(); params.add_int("limit", limit); - get_resource_required(&self.ctx, Some("head"), ¶ms).await + get_resource_required_with_timeout(&self.ctx, Some("head"), ¶ms, SMALL_REQUEST_TIMEOUT) + .await } /// Adds a single request to the queue. If `forefront` is true, adds it to the front. @@ -112,22 +261,29 @@ impl RequestQueueClient { let mut params = self.base_params(); params.add_bool("forefront", Some(forefront)); let body = serde_json::to_vec(request)?; - post_with_body( + post_action_with_timeout( &self.ctx, Some("requests"), ¶ms, Some(body), - "application/json", + Some(CONTENT_TYPE_JSON), + SMALL_REQUEST_TIMEOUT, ) .await } /// Gets a request by ID, or `None` if it does not exist. + /// + /// Unlike the other request-level methods on this client, `get_request` does not send + /// `clientKey`: the JS reference client's `getRequest` builds its params from bare + /// `this._params()`, omitting the `clientKey: this.clientKey` merge that every other + /// request-level method includes. pub async fn get_request(&self, id: &str) -> ApifyClientResult> { - get_resource( + get_resource_with_timeout( &self.ctx, Some(&format!("requests/{}", encode_path_segment(id))), - &self.base_params(), + &QueryParams::new(), + SMALL_REQUEST_TIMEOUT, ) .await } @@ -145,47 +301,30 @@ impl RequestQueueClient { })?; let mut params = self.base_params(); params.add_bool("forefront", Some(forefront)); - let url = params.apply_to_url( - &self - .ctx - .url(Some(&format!("requests/{}", encode_path_segment(&id)))), - ); - let body = serde_json::to_vec(request)?; - let mut headers = std::collections::HashMap::new(); - headers.insert("Content-Type".to_string(), "application/json".to_string()); - let response = self - .ctx - .http - .call(HttpRequest { - method: HttpMethod::Put, - url, - headers, - body: Some(body), - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; - crate::common::parse_data_envelope(&response.body) + update_resource_with_params( + &self.ctx, + Some(&format!("requests/{}", encode_path_segment(&id))), + ¶ms, + request, + MEDIUM_REQUEST_TIMEOUT, + ) + .await } /// Deletes a request by ID. + /// + /// Unlike [`delete`](Self::delete) (the whole queue), a missing request is **not** treated + /// as a no-op: this call propagates a `404` as an error, matching the JS reference client's + /// `deleteRequest`. See `delete_item`'s doc comment (`clients::base`) for the full rationale + /// behind this whole-resource-vs-sub-resource split, shared by every `delete_item` caller. pub async fn delete_request(&self, id: &str) -> ApifyClientResult<()> { - let params = self.base_params(); - let url = params.apply_to_url( - &self - .ctx - .url(Some(&format!("requests/{}", encode_path_segment(id)))), - ); - self.ctx - .http - .call(HttpRequest { - method: HttpMethod::Delete, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; - Ok(()) + delete_item( + &self.ctx, + Some(&format!("requests/{}", encode_path_segment(id))), + &self.base_params(), + SMALL_REQUEST_TIMEOUT, + ) + .await } /// Lists and locks requests from the head of the queue for `lock_secs` seconds. @@ -198,26 +337,110 @@ impl RequestQueueClient { params .add_int("lockSecs", Some(lock_secs)) .add_int("limit", limit); - post_action(&self.ctx, Some("head/lock"), ¶ms, None, None).await + post_action_with_timeout( + &self.ctx, + Some("head/lock"), + ¶ms, + None, + None, + MEDIUM_REQUEST_TIMEOUT, + ) + .await } - /// Adds multiple requests to the queue, automatically splitting the input into chunks of - /// at most [`MAX_REQUESTS_PER_BATCH_OPERATION`] requests per API call (the API rejects - /// larger batches). The per-chunk responses are merged into a single result whose - /// `processedRequests` / `unprocessedRequests` arrays concatenate every chunk's, matching - /// the reference client's client-side chunking. + /// Adds multiple requests to the queue, using the default retry/parallelism/slicing behavior + /// of [`batch_add_requests_with_options`](Self::batch_add_requests_with_options) (reference + /// parity). Use that method directly to override the retry count, parallelism, or retry + /// delay. pub async fn batch_add_requests( &self, requests: &[RequestQueueRequest], forefront: bool, ) -> ApifyClientResult { + self.batch_add_requests_with_options( + requests, + BatchAddRequestsOptions { + forefront: Some(forefront), + ..Default::default() + }, + ) + .await + } + + /// Adds multiple requests to the queue, matching the reference client's `batchAddRequests` + /// convenience behavior: + /// + /// - Requests are chunked to at most `MAX_REQUESTS_PER_BATCH_OPERATION` (25) per API call, + /// and each chunk is further sliced (via `slice_by_byte_length`) so its serialized JSON + /// payload stays under the API's byte-size limit (`MAX_PAYLOAD_SIZE_BYTES`, less a small + /// safety buffer). + /// - Up to `options.max_parallel` chunk calls are in flight at once (bounded concurrency, + /// default `DEFAULT_MAX_PARALLEL_BATCH_ADD_REQUESTS`). + /// - Any `unprocessedRequests` in a chunk's response (typically caused by rate limiting) are + /// retried up to `options.max_unprocessed_requests_retries` additional times + /// (default `DEFAULT_MAX_UNPROCESSED_REQUESTS_RETRIES`), with exponential backoff seeded + /// by `options.min_delay_between_unprocessed_requests_retries` + /// (default `DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES`). + /// + /// A chunk call that fails outright (network/API error, after the transport's own retries are + /// exhausted) does not fail this call: its still-unsubmitted requests are instead folded into + /// the returned `unprocessedRequests`, matching the reference client's guarantee that this + /// method itself does not throw for a partial failure — callers must inspect the returned + /// `unprocessedRequests` to detect that case. + pub async fn batch_add_requests_with_options( + &self, + requests: &[RequestQueueRequest], + options: BatchAddRequestsOptions, + ) -> ApifyClientResult { + let forefront = options.forefront.unwrap_or(false); + let max_parallel = options + .max_parallel + .unwrap_or(DEFAULT_MAX_PARALLEL_BATCH_ADD_REQUESTS) + .max(1); + let max_retries = options + .max_unprocessed_requests_retries + .unwrap_or(DEFAULT_MAX_UNPROCESSED_REQUESTS_RETRIES); + let min_delay = options + .min_delay_between_unprocessed_requests_retries + .unwrap_or(DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES); + + // Target a limit slightly under the API's actual cap (the safety buffer). + let payload_size_limit_bytes = MAX_PAYLOAD_SIZE_BYTES + - ((MAX_PAYLOAD_SIZE_BYTES as f64) * SAFETY_BUFFER_PERCENT).ceil() as usize; + let mut processed: Vec = Vec::new(); let mut unprocessed: Vec = Vec::new(); + let mut in_flight = FuturesUnordered::new(); + + // Keep a pool of up to `max_parallel` chunk calls running at once: push the next chunk's + // future, and once the pool is full, await (and drain) whichever finishes first before + // producing another. This mirrors the reference client's `Promise.race` pool. + let mut i = 0usize; + while i < requests.len() { + let count_capped_end = (i + MAX_REQUESTS_PER_BATCH_OPERATION).min(requests.len()); + let batch = + slice_by_byte_length(&requests[i..count_capped_end], payload_size_limit_bytes, i)?; + let batch_len = batch.len(); + let client = self.clone(); + in_flight.push(async move { + client + .batch_add_chunk_with_retries(batch, forefront, max_retries, min_delay) + .await + }); + + if in_flight.len() >= max_parallel { + if let Some((chunk_processed, chunk_unprocessed)) = in_flight.next().await { + processed.extend(chunk_processed); + unprocessed.extend(chunk_unprocessed); + } + } + i += batch_len; + } - for chunk in requests.chunks(MAX_REQUESTS_PER_BATCH_OPERATION) { - let chunk_result = self.batch_add_chunk(chunk, forefront).await?; - merge_request_array(&mut processed, &chunk_result, "processedRequests"); - merge_request_array(&mut unprocessed, &chunk_result, "unprocessedRequests"); + // Drain whatever is still in flight once every chunk has been submitted. + while let Some((chunk_processed, chunk_unprocessed)) = in_flight.next().await { + processed.extend(chunk_processed); + unprocessed.extend(chunk_unprocessed); } Ok(serde_json::json!({ @@ -226,6 +449,83 @@ impl RequestQueueClient { })) } + /// Adds one chunk (already count- and byte-size-limited) of requests, retrying any + /// `unprocessedRequests` up to `max_retries` additional times with exponential backoff + /// (mirrors the reference client's `_batchAddRequestsWithRetries`). Never returns an `Err` + /// for a failed chunk POST: the not-yet-processed requests are folded into the returned + /// `unprocessed` list instead, so the caller's overall call does not fail for a partial + /// failure. + async fn batch_add_chunk_with_retries( + &self, + requests: Vec, + forefront: bool, + max_retries: u32, + min_delay: Duration, + ) -> (Vec, Vec) { + let mut remaining = requests; + let mut processed: Vec = Vec::new(); + let mut unprocessed: Vec = Vec::new(); + + for attempt in 0..=max_retries { + match self.batch_add_chunk(&remaining, forefront).await { + Ok(chunk_result) => { + merge_request_array(&mut processed, &chunk_result, "processedRequests"); + unprocessed = Vec::new(); + merge_request_array(&mut unprocessed, &chunk_result, "unprocessedRequests"); + + if unprocessed.is_empty() { + break; + } + + // Only requests not yet confirmed processed are worth retrying. + let done = processed_unique_keys(&processed); + remaining.retain(|r| !done.contains(&request_unique_key(r))); + + if remaining.is_empty() { + break; + } + } + Err(_) => { + // The transport already retried transient failures; a surfaced error means + // this chunk truly could not be submitted. Report every not-yet-processed + // request as unprocessed rather than failing the whole call. + let done = processed_unique_keys(&processed); + unprocessed = remaining + .iter() + .filter(|r| !done.contains(&request_unique_key(r))) + .map(|r| { + // Build the object manually (rather than via `json!`) so a `None` + // `method` omits the key entirely, matching the JS reference client + // (which never emits `"method": null` here) instead of serializing + // `Option`'s default `null`. + let mut obj = serde_json::Map::new(); + if let Some(method) = &r.method { + obj.insert( + "method".to_string(), + serde_json::Value::String(method.clone()), + ); + } + obj.insert( + "uniqueKey".to_string(), + serde_json::Value::String(request_unique_key(r)), + ); + obj.insert("url".to_string(), serde_json::Value::String(r.url.clone())); + serde_json::Value::Object(obj) + }) + .collect(); + break; + } + } + + if attempt < max_retries { + crate::http_client::sleep_public(unprocessed_retry_backoff(min_delay, attempt)) + .await; + } + } + + (processed, unprocessed) + } + /// Posts a single chunk of requests (at most [`MAX_REQUESTS_PER_BATCH_OPERATION`]). async fn batch_add_chunk( &self, @@ -235,12 +535,13 @@ impl RequestQueueClient { let mut params = self.base_params(); params.add_bool("forefront", Some(forefront)); let body = serde_json::to_vec(requests)?; - post_with_body( + post_action_with_timeout( &self.ctx, Some("requests/batch"), ¶ms, Some(body), - "application/json", + Some(CONTENT_TYPE_JSON), + MEDIUM_REQUEST_TIMEOUT, ) .await } @@ -255,6 +556,7 @@ impl RequestQueueClient { Some("requests/batch"), &self.base_params(), &requests, + SMALL_REQUEST_TIMEOUT, ) .await } @@ -273,7 +575,13 @@ impl RequestQueueClient { .add_str("exclusiveStartId", options.exclusive_start_id) .add_str("cursor", options.cursor) .add_csv("filter", options.filter.as_deref()); - get_resource_required(&self.ctx, Some("requests"), ¶ms).await + get_resource_required_with_timeout( + &self.ctx, + Some("requests"), + ¶ms, + MEDIUM_REQUEST_TIMEOUT, + ) + .await } /// Prolongs the lock on a request for another `lock_secs` seconds. @@ -290,55 +598,44 @@ impl RequestQueueClient { params .add_int("lockSecs", Some(lock_secs)) .add_bool("forefront", Some(forefront)); - let url = params.apply_to_url( - &self - .ctx - .url(Some(&format!("requests/{}/lock", encode_path_segment(id)))), - ); - let response = self - .ctx - .http - .call(HttpRequest { - method: HttpMethod::Put, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::MEDIUM_REQUEST_TIMEOUT, - }) - .await?; - crate::common::parse_data_envelope(&response.body) + put_action( + &self.ctx, + Some(&format!("requests/{}/lock", encode_path_segment(id))), + ¶ms, + None, + None, + MEDIUM_REQUEST_TIMEOUT, + ) + .await } /// Releases the lock on a request so other clients can process it. /// - /// If `forefront` is `true`, the request moves to the front of the queue. + /// If `forefront` is `true`, the request moves to the front of the queue. Like + /// [`delete_request`](Self::delete_request), a missing lock is not treated as a no-op + /// (matching the JS reference client's `deleteRequestLock`, which does not catch not-found). pub async fn delete_request_lock(&self, id: &str, forefront: bool) -> ApifyClientResult<()> { let mut params = self.base_params(); params.add_bool("forefront", Some(forefront)); - let url = params.apply_to_url( - &self - .ctx - .url(Some(&format!("requests/{}/lock", encode_path_segment(id)))), - ); - self.ctx - .http - .call(HttpRequest { - method: HttpMethod::Delete, - url, - headers: Default::default(), - body: None, - timeout: crate::clients::base::SMALL_REQUEST_TIMEOUT, - }) - .await?; - Ok(()) + delete_item( + &self.ctx, + Some(&format!("requests/{}/lock", encode_path_segment(id))), + ¶ms, + SMALL_REQUEST_TIMEOUT, + ) + .await } - /// Lazily paginates over all requests in the queue, fetching pages on demand. + /// Lazily paginates over all requests in the queue **from the head**, fetching pages on + /// demand. /// /// Returns a [`RequestQueueRequestsIterator`]; call its `next()` to get one request at a - /// time. Pagination uses the API's opaque `nextCursor` token: the first page may be - /// anchored with `exclusiveStartId`, but every subsequent page is fetched with `cursor` - /// (matching the JS reference). `cursor` and `exclusiveStartId` are mutually exclusive. + /// time. Pagination uses the API's opaque `cursor` token: the first page is fetched with + /// neither `cursor` nor `exclusiveStartId` (i.e. from the head), and every subsequent page + /// is fetched with the `cursor` returned by the previous one. This method does not expose + /// `exclusiveStartId` — use [`RequestQueueClient::list_requests`] directly (with + /// `ListRequestsOptions.exclusive_start_id`) if you need to anchor a one-off page fetch + /// somewhere other than the head. pub fn paginate_requests(&self, page_limit: Option) -> RequestQueueRequestsIterator { RequestQueueRequestsIterator { client: self.clone(), @@ -351,12 +648,13 @@ impl RequestQueueClient { /// Unlocks all requests currently locked by this client (identified by `client_key`). pub async fn unlock_requests(&self) -> ApifyClientResult { - post_action( + post_action_with_timeout( &self.ctx, Some("requests/unlock"), &self.base_params(), None, None, + MEDIUM_REQUEST_TIMEOUT, ) .await } diff --git a/src/clients/run.rs b/src/clients/run.rs index 788956f..2bb3b06 100644 --- a/src/clients/run.rs +++ b/src/clients/run.rs @@ -3,8 +3,8 @@ use serde::Serialize; use crate::clients::base::{ - delete_resource, get_resource, post_action, post_with_body, update_resource, wait_for_finish, - ResourceContext, + delete_resource, get_resource, post_action, post_raw_with_extra_header, post_with_body, + update_resource, wait_for_finish, ResourceContext, }; use crate::clients::dataset::DatasetClient; use crate::clients::key_value_store::KeyValueStoreClient; @@ -12,7 +12,7 @@ use crate::clients::log::LogClient; use crate::clients::request_queue::RequestQueueClient; use crate::common::{to_safe_id, QueryParams}; use crate::error::ApifyClientResult; -use crate::http_client::HttpClient; +use crate::http_client::{HttpClient, CONTENT_TYPE_JSON}; use crate::models::ActorRun; /// Header the API uses to deduplicate charge requests (matching the reference client). @@ -36,6 +36,27 @@ pub struct LastRunOptions { pub origin: Option, } +/// Builds a `RunClient` for a resource's `runs/last` sub-path, applying `status`/`origin` as +/// base query parameters from `options`. Shared by +/// [`ActorClient::last_run_with_options`](crate::clients::actor::ActorClient::last_run_with_options) +/// and +/// [`TaskClient::last_run_with_options`](crate::clients::task::TaskClient::last_run_with_options), +/// which are otherwise near-identical (DRY). +pub(crate) fn last_run_client( + http: HttpClient, + base_url: &str, + options: &LastRunOptions, +) -> RunClient { + let mut client = RunClient::new(http, base_url, "runs", "last"); + if let Some(status) = options.status.as_deref() { + client.set_base_param("status", status); + } + if let Some(origin) = options.origin.as_deref() { + client.set_base_param("origin", origin); + } + client +} + /// Options for resurrecting a finished run. #[derive(Debug, Default, Clone)] pub struct RunResurrectOptions { @@ -53,6 +74,20 @@ pub struct RunResurrectOptions { pub restart_on_error: Option, } +/// Options for fetching a run via [`RunClient::get_with_options`]. +/// +/// Covers the spec's optional `waitForFinish` query parameter on `GET /v2/actor-runs/{runId}`, +/// matching the reference client's `RunGetOptions`. +#[derive(Debug, Default, Clone)] +pub struct RunGetOptions { + /// Maximum time, in seconds (capped at 60 by the API), to wait server-side for the run to + /// reach a terminal state before returning. `None` (the default) returns immediately without + /// waiting. This is a single bounded server-side wait, distinct from + /// [`RunClient::wait_for_finish`], which polls repeatedly (using this same parameter + /// internally) until the run finishes or a client-side budget is exhausted. + pub wait_for_finish: Option, +} + /// Options for transforming a run into another Actor's run (metamorph). #[derive(Debug, Default, Clone)] pub struct RunMetamorphOptions { @@ -88,13 +123,7 @@ pub struct RunClient { } impl RunClient { - pub(crate) fn new( - _root: crate::client::ApifyClient, - http: HttpClient, - base_url: &str, - resource_path: &str, - id: &str, - ) -> Self { + pub(crate) fn new(http: HttpClient, base_url: &str, resource_path: &str, id: &str) -> Self { Self { ctx: ResourceContext::single(http, base_url, resource_path, id), id: id.to_string(), @@ -110,8 +139,23 @@ impl RunClient { } /// Fetches the run object, or `None` if it does not exist. + /// + /// Returns immediately without waiting for the run to finish. To have the API wait + /// server-side before responding, use [`RunClient::get_with_options`]. pub async fn get(&self) -> ApifyClientResult> { - get_resource(&self.ctx, None, &QueryParams::new()).await + self.get_with_options(RunGetOptions::default()).await + } + + /// Fetches the run object, or `None` if it does not exist, applying the given + /// [`RunGetOptions`] (e.g. [`RunGetOptions::wait_for_finish`] to have the API wait, + /// server-side, for the run to finish before responding). + pub async fn get_with_options( + &self, + options: RunGetOptions, + ) -> ApifyClientResult> { + let mut params = QueryParams::new(); + params.add_int("waitForFinish", options.wait_for_finish); + get_resource(&self.ctx, None, ¶ms).await } /// Updates the run (e.g. its status message) and returns the updated object. @@ -125,9 +169,9 @@ impl RunClient { } /// Aborts the run. `gracefully` is optional, matching the reference client's optional - /// `gracefully` option and the Go sibling's `Option`: `Some(true)` lets the run - /// perform cleanup first, `Some(false)` aborts immediately, and `None` omits the parameter - /// entirely so the server applies its default (immediate abort). + /// `gracefully` option: `Some(true)` lets the run perform cleanup first, `Some(false)` + /// aborts immediately, and `None` omits the parameter entirely so the server applies its + /// default (immediate abort). pub async fn abort(&self, gracefully: Option) -> ApifyClientResult { let mut params = QueryParams::new(); params.add_bool("gracefully", gracefully); @@ -152,10 +196,7 @@ impl RunClient { Some(value) => Some(serde_json::to_vec(value)?), None => None, }; - let content_type = options - .content_type - .as_deref() - .unwrap_or("application/json"); + let content_type = options.content_type.as_deref().unwrap_or(CONTENT_TYPE_JSON); post_with_body(&self.ctx, Some("metamorph"), ¶ms, body, content_type).await } @@ -185,6 +226,17 @@ impl RunClient { /// /// The charge endpoint returns an empty body on success, so this issues the request /// directly and treats any 2xx response as success (errors still surface normally). + /// + /// Note: like every other request issued through this `ctx`, the URL still carries this + /// client's base params (e.g. `status`/`origin` when this `RunClient` came from + /// `actor.last_run()`/`task.last_run()`) even though the explicit `params` passed here are + /// empty — `post_raw_with_extra_header` merges `ctx`'s base params in unconditionally, the + /// same as every other request helper. This is actually a divergence from the JS reference + /// client: JS's `charge()` builds its `AxiosRequestConfig` manually with no `params` key at + /// all, so it is the one method that does not go through the instance's `_params()` and + /// therefore never sends base params. The Rust behavior is left as-is (rather than special- + /// cased to suppress base params) because charging a `last_run()`-derived client is unusual, + /// and the server ignores unrecognized query params on this endpoint regardless. pub async fn charge(&self, options: RunChargeOptions) -> ApifyClientResult<()> { let count = options.count.unwrap_or(1); let idempotency_key = options @@ -192,23 +244,18 @@ impl RunClient { .unwrap_or_else(|| self.generate_idempotency_key(&options.event_name)); let body = serde_json::json!({ "eventName": options.event_name, "count": count }); let body_bytes = serde_json::to_vec(&body)?; - let url = self.ctx.url(Some("charge")); - let mut headers = std::collections::HashMap::new(); - headers.insert("Content-Type".to_string(), "application/json".to_string()); - headers.insert(CHARGE_IDEMPOTENCY_HEADER.to_string(), idempotency_key); - // A successful `HttpClient::call` already guarantees a 2xx status; the (empty) body - // is intentionally ignored rather than parsed as a `data` envelope. - self.ctx - .http - .call(crate::http_client::HttpRequest { - method: crate::http_client::HttpMethod::Post, - url, - headers, - body: Some(body_bytes), - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; - Ok(()) + // A successful call already guarantees a 2xx status; the (empty) body is intentionally + // ignored rather than parsed as a `data` envelope. + post_raw_with_extra_header( + &self.ctx, + Some("charge"), + &QueryParams::new(), + body_bytes, + CONTENT_TYPE_JSON, + CHARGE_IDEMPOTENCY_HEADER, + idempotency_key, + ) + .await } /// Builds a per-charge idempotency key of the form @@ -260,20 +307,27 @@ impl RunClient { LogClient::nested(self.ctx.http.clone(), &self.ctx.url(None), "log") } - /// Opens a live stream of the run's log for redirection. + /// Opens a live stream of the run's log for redirection, or `None` if the log does not + /// exist. /// - /// Convenience equivalent to `run.log().stream()` (mirrors the reference client's - /// `getStreamedLog`): yields log chunks as they arrive, so callers can forward them to - /// their own logger/stdout while the run is in progress. + /// Convenience equivalent to `run.log().stream()`: yields raw log chunks as they arrive, so + /// callers can forward them to their own logger/stdout while the run is in progress. This is + /// a Rust-specific raw-chunk convenience, not a mirror of the JS reference client's + /// `getStreamedLog`, which returns a `StreamedLog` object: it wraps a destination `Log`, but + /// the timestamp-prefixed-line parsing and forwarding is done by `StreamedLog` itself, not by + /// `Log` — see [`get_streamed_log_with_options`](Self::get_streamed_log_with_options) for the + /// fuller disclaimer. pub async fn get_streamed_log( &self, - ) -> ApifyClientResult>>> { + ) -> ApifyClientResult>>>> + { self.log().stream().await } /// Opens a live stream of the run's log for redirection, applying the given - /// [`LogOptions`] (e.g. [`LogOptions::raw`] to stream the unprocessed log, which is the - /// form the JS reference's log redirection consumes internally). + /// [`crate::LogOptions`] (e.g. [`crate::LogOptions::raw`] to stream the unprocessed log, + /// which is the form the JS reference's log redirection consumes internally), or `None` if + /// the log does not exist. /// /// This is a Rust-specific convenience that simply forwards `LogOptions` to /// [`LogClient::stream_with_options`]; it is not a 1:1 mirror of the JS `getStreamedLog` @@ -281,7 +335,8 @@ impl RunClient { pub async fn get_streamed_log_with_options( &self, options: crate::clients::log::LogOptions, - ) -> ApifyClientResult>>> { + ) -> ApifyClientResult>>>> + { self.log().stream_with_options(options).await } } diff --git a/src/clients/task.rs b/src/clients/task.rs index 7996d8c..9949a90 100644 --- a/src/clients/task.rs +++ b/src/clients/task.rs @@ -4,18 +4,101 @@ use serde::Serialize; use serde_json::Value; use crate::client::ApifyClient; -use crate::clients::actor::ActorStartOptions; +use crate::clients::actor::encode_webhooks; use crate::clients::base::{ - delete_resource, get_resource, post_with_body, update_resource, ResourceContext, + delete_resource, get_resource, post_with_body, put_action_raw, update_resource, ResourceContext, }; use crate::clients::run::{LastRunOptions, RunClient}; use crate::clients::run_collection::RunCollectionClient; use crate::clients::webhook_collection::WebhookCollectionClient; use crate::common::QueryParams; use crate::error::ApifyClientResult; -use crate::http_client::HttpClient; +use crate::http_client::{HttpClient, CONTENT_TYPE_JSON}; use crate::models::{ActorRun, Task}; +/// Options for [`TaskClient::start`]. +/// +/// Like [`ActorStartOptions`](crate::clients::actor::ActorStartOptions), but without +/// `content_type` (a task's input content type is fixed as `application/json` — a task's input +/// is predefined, unlike an ad-hoc Actor start) or `force_permission_level` (not accepted by the +/// task-run endpoint). This mirrors the JS reference client's `TaskStartOptions`, typed as +/// `Omit`. +#[derive(Debug, Default, Clone)] +pub struct TaskStartOptions { + /// Tag or number of the build to run (e.g. `latest`, `0.1.2`). + pub build: Option, + /// Memory in megabytes allocated for the run. + pub memory_mbytes: Option, + /// Timeout for the run in seconds (`0` means no timeout). + pub timeout_secs: Option, + /// Maximum seconds to wait server-side for the run to finish (max 60). + pub wait_for_finish: Option, + /// Maximum number of dataset items to charge (pay-per-result Actors). + pub max_items: Option, + /// Maximum total charge in USD (pay-per-event Actors). + pub max_total_charge_usd: Option, + /// Whether to restart the run if it fails. + pub restart_on_error: Option, + /// Ad-hoc webhooks to attach to this run. Encoded as base64-encoded JSON as the `webhooks` + /// query parameter, matching the reference clients. + pub webhooks: Option>, +} + +impl TaskStartOptions { + /// Serializes these options into run-start query parameters. + fn apply(&self, params: &mut QueryParams) { + params + .add_str("build", self.build.clone()) + .add_int("memory", self.memory_mbytes) + .add_int("timeout", self.timeout_secs) + .add_int("waitForFinish", self.wait_for_finish) + .add_int("maxItems", self.max_items) + .add_float("maxTotalChargeUsd", self.max_total_charge_usd) + .add_bool("restartOnError", self.restart_on_error) + .add_str("webhooks", encode_webhooks(&self.webhooks)); + } +} + +/// Options for [`TaskClient::call`]. +/// +/// Like [`TaskStartOptions`], but without `wait_for_finish` (the server-side wait): `call`'s +/// separate `wait_secs` argument controls the client-side wait instead, so the two should not be +/// set together. This mirrors the JS reference client's `TaskCallOptions`, typed as +/// `Omit`. +#[derive(Debug, Default, Clone)] +pub struct TaskCallOptions { + /// Tag or number of the build to run (e.g. `latest`, `0.1.2`). + pub build: Option, + /// Memory in megabytes allocated for the run. + pub memory_mbytes: Option, + /// Timeout for the run in seconds (`0` means no timeout). + pub timeout_secs: Option, + /// Maximum number of dataset items to charge (pay-per-result Actors). + pub max_items: Option, + /// Maximum total charge in USD (pay-per-event Actors). + pub max_total_charge_usd: Option, + /// Whether to restart the run if it fails. + pub restart_on_error: Option, + /// Ad-hoc webhooks to attach to this run. Encoded as base64-encoded JSON as the `webhooks` + /// query parameter, matching the reference clients. + pub webhooks: Option>, +} + +impl From for TaskStartOptions { + fn from(options: TaskCallOptions) -> Self { + TaskStartOptions { + build: options.build, + memory_mbytes: options.memory_mbytes, + timeout_secs: options.timeout_secs, + wait_for_finish: None, + max_items: options.max_items, + max_total_charge_usd: options.max_total_charge_usd, + restart_on_error: options.restart_on_error, + webhooks: options.webhooks, + } + } +} + /// Client for a specific Actor task. #[derive(Debug, Clone)] pub struct TaskClient { @@ -48,11 +131,14 @@ impl TaskClient { /// Starts the task and returns immediately with the created run. /// - /// `input` overrides the task's saved input (or `None` to use the saved input). + /// `input` overrides the task's saved input (or `None` to use the saved input). `options` is + /// [`TaskStartOptions`] — a task's input content type and permission level are fixed, so + /// (unlike [`ActorClient::start`](crate::clients::actor::ActorClient::start)) there is no + /// `content_type` or `force_permission_level` field to set. pub async fn start( &self, input: Option<&T>, - options: ActorStartOptions, + options: TaskStartOptions, ) -> ApifyClientResult { let mut params = QueryParams::new(); options.apply(&mut params); @@ -60,7 +146,7 @@ impl TaskClient { Some(value) => Some(serde_json::to_vec(value)?), None => None, }; - post_with_body(&self.ctx, Some("runs"), ¶ms, body, "application/json").await + post_with_body(&self.ctx, Some("runs"), ¶ms, body, CONTENT_TYPE_JSON).await } /// Starts the task and waits (client-side polling) for it to finish. @@ -70,13 +156,17 @@ impl TaskClient { /// - `Some(n)` bounds the wait to roughly `n` seconds; if the run has not finished by /// then, the **last fetched (still non-terminal) run is returned** rather than an /// error. Check `status` / `is_terminal()` on the result when using `Some`. + /// + /// `options` is [`TaskCallOptions`], which (matching the JS reference client) additionally + /// excludes `wait_for_finish` (the server-side wait) since the client-side `wait_secs` + /// argument is how callers control call's wait behavior. pub async fn call( &self, input: Option<&T>, - options: ActorStartOptions, + options: TaskCallOptions, wait_secs: Option, ) -> ApifyClientResult { - let run = self.start(input, options).await?; + let run = self.start(input, options.into()).await?; self.root.run(run.id).wait_for_finish(wait_secs).await } @@ -93,21 +183,14 @@ impl TaskClient { /// Updates the task's saved input. pub async fn update_input(&self, input: &T) -> ApifyClientResult { let body = serde_json::to_vec(input)?; - let url = self.ctx.url(Some("input")); - let mut headers = std::collections::HashMap::new(); - headers.insert("Content-Type".to_string(), "application/json".to_string()); - let response = self - .ctx - .http - .call(crate::http_client::HttpRequest { - method: crate::http_client::HttpMethod::Put, - url, - headers, - body: Some(body), - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; - Ok(serde_json::from_slice(&response.body)?) + put_action_raw( + &self.ctx, + Some("input"), + &QueryParams::new(), + body, + CONTENT_TYPE_JSON, + ) + .await } /// Returns a client for the last run of this task, optionally filtered by run status. @@ -132,20 +215,7 @@ impl TaskClient { /// parameters on `GET /v2/actor-tasks/{actorTaskId}/runs/last` and match the reference client's /// `lastRun({ status, origin })`; leave a field as `None` to omit it. pub fn last_run_with_options(&self, options: LastRunOptions) -> RunClient { - let mut client = RunClient::new( - self.root.clone(), - self.ctx.http.clone(), - &self.ctx.url(None), - "runs", - "last", - ); - if let Some(status) = options.status.as_deref() { - client.set_base_param("status", status); - } - if let Some(origin) = options.origin.as_deref() { - client.set_base_param("origin", origin); - } - client + crate::clients::run::last_run_client(self.ctx.http.clone(), &self.ctx.url(None), &options) } /// Returns a client for this task's run collection. @@ -155,6 +225,6 @@ impl TaskClient { /// Returns a client for this task's webhook collection. pub fn webhooks(&self) -> WebhookCollectionClient { - WebhookCollectionClient::with_base(self.ctx.http.clone(), &self.ctx.url(None)) + WebhookCollectionClient::new(self.ctx.http.clone(), &self.ctx.url(None)) } } diff --git a/src/clients/user.rs b/src/clients/user.rs index a7e2075..91c8c3d 100644 --- a/src/clients/user.rs +++ b/src/clients/user.rs @@ -3,10 +3,10 @@ use serde::Serialize; use serde_json::Value; -use crate::clients::base::{get_resource, get_resource_required, ResourceContext}; +use crate::clients::base::{get_resource, put_raw, ResourceContext}; use crate::common::QueryParams; use crate::error::ApifyClientResult; -use crate::http_client::HttpClient; +use crate::http_client::{HttpClient, CONTENT_TYPE_JSON}; use crate::models::User; /// Client for a specific user (or the current user via [`ApifyClient::me`]). @@ -34,20 +34,25 @@ impl UserClient { get_resource(&self.ctx, None, &QueryParams::new()).await } - /// Returns the current user's monthly usage for the current month. Only valid for the `me` - /// client. To fetch usage for a specific month, use [`UserClient::monthly_usage_for_date`]. - pub async fn monthly_usage(&self) -> ApifyClientResult { + /// Returns the current user's monthly usage for the current month, or `None` if it is + /// unavailable. Only valid for the `me` client. To fetch usage for a specific month, use + /// [`UserClient::monthly_usage_for_date`]. + pub async fn monthly_usage(&self) -> ApifyClientResult> { self.monthly_usage_for_date_named(None, "monthly_usage") .await } - /// Returns the current user's monthly usage, optionally for the month containing `date`. + /// Returns the current user's monthly usage, optionally for the month containing `date`, or + /// `None` if it is unavailable. /// /// `date` is an optional `YYYY-MM-DD` string selecting the month to report (the spec's /// optional `date` query parameter on `GET /v2/users/me/usage/monthly`); passing `None` /// returns the current month, which is equivalent to [`UserClient::monthly_usage`]. Only /// valid for the `me` client. - pub async fn monthly_usage_for_date(&self, date: Option<&str>) -> ApifyClientResult { + pub async fn monthly_usage_for_date( + &self, + date: Option<&str>, + ) -> ApifyClientResult> { self.monthly_usage_for_date_named(date, "monthly_usage_for_date") .await } @@ -55,41 +60,46 @@ impl UserClient { /// Shared implementation for [`UserClient::monthly_usage`] and /// [`UserClient::monthly_usage_for_date`]. `method` is the caller's own public method name, /// so the `me`-only guard error names the method the caller actually invoked. + /// + /// The spec declares no `404` response for `GET /v2/users/me/usage/monthly`, so a `404` + /// mapping to `None` here would never actually trigger against the real API; it is used + /// anyway (via [`get_resource`] rather than a `_required` variant) purely for JS-reference + /// parity, since the reference client's `monthlyUsage` wraps the call in + /// `catchNotFoundOrThrow`. async fn monthly_usage_for_date_named( &self, date: Option<&str>, method: &str, - ) -> ApifyClientResult { + ) -> ApifyClientResult> { self.require_me(method)?; let mut params = QueryParams::new(); params.add_str("date", date); - get_resource_required(&self.ctx, Some("usage/monthly"), ¶ms).await + get_resource(&self.ctx, Some("usage/monthly"), ¶ms).await } - /// Returns the current user's account and usage limits. Only valid for the `me` client. - pub async fn limits(&self) -> ApifyClientResult { + /// Returns the current user's account and usage limits, or `None` if unavailable. Only + /// valid for the `me` client. + /// + /// As with [`UserClient::monthly_usage`], the spec declares no `404` for + /// `GET /v2/users/me/limits`; the `Option` return is purely for JS-reference parity + /// (`limits()` there also wraps the call in `catchNotFoundOrThrow`). + pub async fn limits(&self) -> ApifyClientResult> { self.require_me("limits")?; - get_resource_required(&self.ctx, Some("limits"), &QueryParams::new()).await + get_resource(&self.ctx, Some("limits"), &QueryParams::new()).await } /// Updates the current user's limits. Only valid for the `me` client. pub async fn update_limits(&self, new_limits: &T) -> ApifyClientResult<()> { self.require_me("update_limits")?; let body = serde_json::to_vec(new_limits)?; - let url = self.ctx.url(Some("limits")); - let mut headers = std::collections::HashMap::new(); - headers.insert("Content-Type".to_string(), "application/json".to_string()); - self.ctx - .http - .call(crate::http_client::HttpRequest { - method: crate::http_client::HttpMethod::Put, - url, - headers, - body: Some(body), - timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT, - }) - .await?; - Ok(()) + put_raw( + &self.ctx, + Some("limits"), + &QueryParams::new(), + body, + CONTENT_TYPE_JSON, + ) + .await } fn require_me(&self, method: &str) -> ApifyClientResult<()> { diff --git a/src/clients/webhook.rs b/src/clients/webhook.rs index 1e090d5..407a875 100644 --- a/src/clients/webhook.rs +++ b/src/clients/webhook.rs @@ -3,7 +3,7 @@ use serde::Serialize; use crate::clients::base::{ - delete_resource, get_resource, post_action, update_resource, ResourceContext, + delete_resource, get_resource, post_action_optional, update_resource, ResourceContext, }; use crate::clients::webhook_dispatch_collection::WebhookDispatchCollectionClient; use crate::common::QueryParams; @@ -39,9 +39,12 @@ impl WebhookClient { delete_resource(&self.ctx, None).await } - /// Tests the webhook by dispatching it immediately, returning the dispatch. - pub async fn test(&self) -> ApifyClientResult { - post_action(&self.ctx, Some("test"), &QueryParams::new(), None, None).await + /// Tests the webhook by dispatching it immediately, returning the dispatch, or `None` if the + /// webhook no longer exists (e.g. it was deleted concurrently). The spec lists `404` as a + /// valid response for `POST /v2/webhooks/{webhookId}/test`, and the reference client wraps + /// the call in `catchNotFoundOrThrow`. + pub async fn test(&self) -> ApifyClientResult> { + post_action_optional(&self.ctx, Some("test"), &QueryParams::new(), None, None).await } /// Returns a client for this webhook's dispatch collection. diff --git a/src/clients/webhook_collection.rs b/src/clients/webhook_collection.rs index b4e4362..0376c07 100644 --- a/src/clients/webhook_collection.rs +++ b/src/clients/webhook_collection.rs @@ -16,19 +16,16 @@ pub struct WebhookCollectionClient { } impl WebhookCollectionClient { + /// Creates a webhook collection client. `base_url` may be the API root (top-level + /// `client.webhooks()`) or a specific resource's URL (nested `actor.webhooks()` / + /// `task.webhooks()`) — both mount the collection at a `webhooks` sub-path, so one + /// constructor covers both call sites. pub(crate) fn new(http: HttpClient, base_url: &str) -> Self { Self { ctx: ResourceContext::collection(http, base_url, "webhooks"), } } - /// Creates a webhook collection client nested under another resource. - pub(crate) fn with_base(http: HttpClient, base_url: &str) -> Self { - Self { - ctx: ResourceContext::collection(http, base_url, "webhooks"), - } - } - /// Lists webhooks with offset/limit pagination. pub async fn list(&self, options: ListOptions) -> ApifyClientResult> { let mut params = QueryParams::new(); diff --git a/src/common.rs b/src/common.rs index bc20935..c4634de 100644 --- a/src/common.rs +++ b/src/common.rs @@ -8,10 +8,21 @@ use crate::error::ApifyClientResult; use crate::version::CLIENT_VERSION; /// Status code returned when a resource is not found. -const NOT_FOUND_STATUS_CODE: u16 = 404; +pub(crate) const NOT_FOUND_STATUS_CODE: u16 = 404; const RECORD_NOT_FOUND_TYPE: &str = "record-not-found"; const RECORD_OR_TOKEN_NOT_FOUND_TYPE: &str = "record-or-token-not-found"; +/// Reports whether `error_type` is one of the API's "not found" error-type strings +/// (`record-not-found` / `record-or-token-not-found`). Shared by [`catch_not_found`] and any +/// caller that must classify a `404` outside the normal `ApiError` path (e.g. a raw streaming +/// response), so the set of recognized not-found types stays in one place. +pub(crate) fn is_not_found_error_type(error_type: Option<&str>) -> bool { + matches!( + error_type, + Some(RECORD_NOT_FOUND_TYPE) | Some(RECORD_OR_TOKEN_NOT_FOUND_TYPE) + ) +} + /// Most Apify endpoints wrap their payload in a top-level `data` property. /// This envelope unwraps `{ "data": ... }` into the inner type. #[derive(Debug, Deserialize)] @@ -35,10 +46,8 @@ pub(crate) fn catch_not_found(result: ApifyClientResult) -> ApifyClientRes Err(err) => { if let Some(api_error) = err.as_api_error() { let is_not_found_status = api_error.status_code == NOT_FOUND_STATUS_CODE; - let is_not_found_type = matches!( - api_error.error_type.as_deref(), - Some(RECORD_NOT_FOUND_TYPE) | Some(RECORD_OR_TOKEN_NOT_FOUND_TYPE) - ) || api_error.http_method.as_deref() == Some("HEAD"); + let is_not_found_type = is_not_found_error_type(api_error.error_type.as_deref()) + || api_error.http_method.as_deref() == Some("HEAD"); if is_not_found_status && is_not_found_type { return Ok(None); } diff --git a/src/http_client.rs b/src/http_client.rs index 2f37b27..9689013 100644 --- a/src/http_client.rs +++ b/src/http_client.rs @@ -8,8 +8,8 @@ //! //! [`HttpClient`] wraps a backend and adds the cross-cutting concerns shared by every //! endpoint: authentication, the `User-Agent` header, query-parameter serialization, -//! timeouts and retries with exponential backoff (mirroring the JavaScript and Python -//! reference clients). +//! timeouts and retries with exponential backoff (mirroring the JavaScript reference +//! client). use std::collections::HashMap; use std::sync::Arc; @@ -48,6 +48,22 @@ const BROTLI_BUFFER_SIZE: usize = 4096; /// default level (6). const GZIP_COMPRESSION_LEVEL: u32 = 6; +/// Header carrying the client's identifying `User-Agent` string. +pub(crate) const HEADER_USER_AGENT: &str = "User-Agent"; +/// Header carrying the bearer token used for authentication. +pub(crate) const HEADER_AUTHORIZATION: &str = "Authorization"; +/// Header declaring a request (or response) body's media type. Used throughout `clients::base` +/// and the resource clients that build a request by hand. +pub(crate) const HEADER_CONTENT_TYPE: &str = "Content-Type"; +/// The plain `application/json` media type, for endpoints that accept it without a charset. +pub(crate) const CONTENT_TYPE_JSON: &str = "application/json"; +/// `application/json` with an explicit UTF-8 charset, used by endpoints that store or forward +/// the body as text (dataset items, key-value-store JSON records). +pub(crate) const CONTENT_TYPE_JSON_UTF8: &str = "application/json; charset=utf-8"; +/// Header declaring the compression algorithm applied to a request body. Set by +/// [`maybe_compress_request`] and checked (case-insensitively) to detect a caller-supplied value. +const HEADER_CONTENT_ENCODING: &str = "Content-Encoding"; + /// Algorithm used to compress large request bodies before they are sent. /// /// The Apify API accepts both brotli (`br`) and gzip (`gzip`) request bodies. The reference JS @@ -269,15 +285,16 @@ impl HttpClient { // Inject auth + user-agent headers shared by every endpoint. request .headers - .insert("User-Agent".to_string(), self.user_agent.clone()); + .insert(HEADER_USER_AGENT.to_string(), self.user_agent.clone()); if let Some(token) = &self.token { request .headers - .insert("Authorization".to_string(), format!("Bearer {token}")); + .insert(HEADER_AUTHORIZATION.to_string(), format!("Bearer {token}")); } - // Compress the request body once (not per attempt) when it is large enough, mirroring the - // reference client. The API accepts both brotli- and gzip-encoded request bodies. + // Compress the request body once (not per attempt) when it is large enough; see + // `maybe_compress_request`'s doc for why this differs from the reference client's + // mechanism while producing the same bytes on the wire. maybe_compress_request(&mut request, self.compression); let method_str = request.method.as_str().to_string(); @@ -352,8 +369,12 @@ impl HttpClient { /// Compresses `request.body` in place when it is present, at least [`MIN_COMPRESS_BYTES`] long, /// and no `Content-Encoding` is already set, adding the matching `Content-Encoding` header. /// -/// The algorithm is chosen by `compression` (defaulting to brotli). The size threshold and the -/// "compress once, before retries" behaviour mirror the reference client. +/// The algorithm is chosen by `compression` (defaulting to brotli). The size threshold mirrors +/// the reference client. Unlike the reference client (whose axios interceptor re-runs on every +/// retry against the original uncompressed config), this compresses the body once up front and +/// reuses the encoded bytes across attempts; for a non-streamed body this produces the same +/// bytes on the wire, but is a different mechanism, not a literal mirror of "compress once, +/// before retries." fn maybe_compress_request(request: &mut HttpRequest, compression: RequestCompression) { let Some(body) = request.body.as_ref() else { return; @@ -366,7 +387,7 @@ fn maybe_compress_request(request: &mut HttpRequest, compression: RequestCompres let already_encoded = request .headers .keys() - .any(|k| k.eq_ignore_ascii_case("Content-Encoding")); + .any(|k| k.eq_ignore_ascii_case(HEADER_CONTENT_ENCODING)); if already_encoded { return; } @@ -377,7 +398,7 @@ fn maybe_compress_request(request: &mut HttpRequest, compression: RequestCompres }; request .headers - .insert("Content-Encoding".to_string(), encoding.to_string()); + .insert(HEADER_CONTENT_ENCODING.to_string(), encoding.to_string()); request.body = Some(compressed); } @@ -489,11 +510,24 @@ fn randomized_delay(delay: Duration) -> Duration { /// uncorrelated across concurrent retries (otherwise many clients retry in lockstep). A /// shared atomically-advanced SplitMix64 generator, seeded once from the clock, gives each /// caller a distinct value without pulling in a heavyweight RNG dependency. -fn next_jitter() -> u64 { +/// +/// `pub(crate)` so other backoff-jitter call sites (e.g. +/// [`crate::clients::request_queue`]'s batch-add retry backoff) can reuse this one shared +/// generator instead of rolling their own weaker source. +pub(crate) fn next_jitter() -> u64 { use std::sync::atomic::{AtomicU64, Ordering}; static STATE: AtomicU64 = AtomicU64::new(0); const GOLDEN_GAMMA: u64 = 0x9E3779B97F4A7C15; + // The two SplitMix64 output-mixing multipliers and their paired right-shift amounts, per + // the algorithm's reference (Steele, Lea & Flood, "Fast Splittable Pseudorandom Number + // Generators"). Named so they read as the algorithm's fixed constants, not arbitrary magic + // numbers, matching the already-named `GOLDEN_GAMMA`. + const MIX_1_SHIFT: u32 = 30; + const MIX_1_MUL: u64 = 0xBF58476D1CE4E5B9; + const MIX_2_SHIFT: u32 = 27; + const MIX_2_MUL: u64 = 0x94D049BB133111EB; + const FINAL_SHIFT: u32 = 31; // Lazily seed from the clock on first use. A racing double-seed is harmless: both // candidate seeds are valid SplitMix64 stream starting points. @@ -512,9 +546,9 @@ fn next_jitter() -> u64 { let mut z = STATE .fetch_add(GOLDEN_GAMMA, Ordering::Relaxed) .wrapping_add(GOLDEN_GAMMA); - z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); - z ^ (z >> 31) + z = (z ^ (z >> MIX_1_SHIFT)).wrapping_mul(MIX_1_MUL); + z = (z ^ (z >> MIX_2_SHIFT)).wrapping_mul(MIX_2_MUL); + z ^ (z >> FINAL_SHIFT) } /// Sleeps for the given duration (public crate-internal helper for poll loops). diff --git a/src/lib.rs b/src/lib.rs index c53ff11..06ea516 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,13 +1,12 @@ //! # apify-client //! -//! **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify -//! client, but it is experimental: it is generated and maintained by AI. Review the code before -//! relying on it in production and report issues on the repository. +//! See the top-level [README](https://github.com/apify/apify-client-rust#readme) for the +//! client's status and support disclosure. //! //! An idiomatic Rust client for the [Apify API](https://docs.apify.com/api/v2). //! //! It provides a resource-oriented interface that mirrors the official -//! [JavaScript](https://github.com/apify/apify-client-js) and Python clients: start from +//! [JavaScript client](https://github.com/apify/apify-client-js): start from //! an [`ApifyClient`], then drill down into resources (Actors, runs, datasets, key-value //! stores, request queues, tasks, schedules, webhooks, the store, users and logs). //! @@ -63,18 +62,24 @@ pub use http_client::RequestCompression; pub use version::{API_SPEC_VERSION, CLIENT_VERSION}; // Re-export the most commonly used option/parameter types for ergonomic access. -pub use clients::actor::{ActorBuildOptions, ActorStartOptions}; +pub use clients::actor::{ActorBuildOptions, ActorCallOptions, ActorStartOptions}; pub use clients::actor_collection::ActorListOptions; +pub use clients::build::BuildGetOptions; pub use clients::dataset::{DatasetDownloadOptions, DatasetListItemsOptions, DownloadItemsFormat}; +pub use clients::dataset_collection::DatasetGetOrCreateOptions; pub use clients::key_value_store::{GetRecordOptions, KeyValueStoreKeysIterator, ListKeysOptions}; +pub use clients::key_value_store_collection::KeyValueStoreGetOrCreateOptions; pub use clients::log::LogOptions; pub use clients::pagination::ListIterator; -pub use clients::request_queue::{ListRequestsOptions, RequestQueueRequestsIterator}; +pub use clients::request_queue::{ + BatchAddRequestsOptions, ListRequestsOptions, RequestQueueRequestsIterator, +}; pub use clients::run::{ - LastRunOptions, RunChargeOptions, RunMetamorphOptions, RunResurrectOptions, + LastRunOptions, RunChargeOptions, RunGetOptions, RunMetamorphOptions, RunResurrectOptions, }; pub use clients::run_collection::RunListOptions; pub use clients::store_collection::{StoreActorIterator, StoreListOptions}; +pub use clients::task::{TaskCallOptions, TaskStartOptions}; // Compile-test the code snippets in the README and the external `docs/` pages so every // in-documentation code snippet stays valid and runnable. Pulling each Markdown file in as @@ -107,3 +112,15 @@ struct DocsRunsDoctests; #[doc = include_str!("../docs/builds.md")] #[cfg(doctest)] struct DocsBuildsDoctests; + +#[doc = include_str!("../docs/tasks.md")] +#[cfg(doctest)] +struct DocsTasksDoctests; + +#[doc = include_str!("../docs/schedules.md")] +#[cfg(doctest)] +struct DocsSchedulesDoctests; + +#[doc = include_str!("../docs/webhooks.md")] +#[cfg(doctest)] +struct DocsWebhooksDoctests; diff --git a/src/models.rs b/src/models.rs index 679cad2..c954e13 100644 --- a/src/models.rs +++ b/src/models.rs @@ -101,13 +101,19 @@ pub struct ActorRun { pub(crate) const TERMINAL_STATUSES: &[&str] = &["SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT", "TIMED_OUT"]; +/// Shared by [`ActorRun::is_terminal`] and [`Build::is_terminal`], which have identical bodies +/// (both check an `Option` status field against [`TERMINAL_STATUSES`]) (DRY). +fn is_terminal_status(status: &Option) -> bool { + status + .as_deref() + .map(|s| TERMINAL_STATUSES.contains(&s)) + .unwrap_or(false) +} + impl ActorRun { /// Returns `true` if the run has reached a terminal state. pub fn is_terminal(&self) -> bool { - self.status - .as_deref() - .map(|s| TERMINAL_STATUSES.contains(&s)) - .unwrap_or(false) + is_terminal_status(&self.status) } } @@ -140,10 +146,7 @@ pub struct Build { impl Build { /// Returns `true` if the build has reached a terminal state. pub fn is_terminal(&self) -> bool { - self.status - .as_deref() - .map(|s| TERMINAL_STATUSES.contains(&s)) - .unwrap_or(false) + is_terminal_status(&self.status) } } diff --git a/src/version.rs b/src/version.rs index cf33eed..5049c79 100644 --- a/src/version.rs +++ b/src/version.rs @@ -10,4 +10,4 @@ pub const CLIENT_VERSION: &str = env!("CARGO_PKG_VERSION"); /// and verified against. /// /// This corresponds to the `info.version` field of the Apify OpenAPI document. -pub const API_SPEC_VERSION: &str = "v2-2026-07-13T092445Z"; +pub const API_SPEC_VERSION: &str = "v2-2026-07-23T070817Z"; diff --git a/tests/actor.rs b/tests/actor.rs index 609c8b6..7fe6aba 100644 --- a/tests/actor.rs +++ b/tests/actor.rs @@ -15,12 +15,19 @@ async fn list_actors() { .list(Default::default()) .await .expect("listing actors should succeed"); - assert!(page.total >= 0); + // Load-bearing consistency check (unlike a tautological `total >= 0`, which is `i64` and + // always true): a single page can never return more items than its own `limit`. Checked + // against `limit`, not `total`, because this suite runs many tests concurrently against the + // same shared account; `total` and `items` can be computed from separately-timed backend + // reads under that write load, which made an `items.len() <= total` check here genuinely + // flaky (observed under `cargo test --all-targets`), not just load-bearing. + assert!(page.items.len() as i64 <= page.limit); } fn actor_name(prefix: &str) -> String { - let name = common::unique_name(prefix).replace('-', ""); // actor names are stricter - format!("a{}", &name[..name.len().min(20)]) + // actor names are stricter (no hyphens, capped length); see `short_unique_name`'s doc for + // why plain truncation of `unique_name`'s output isn't used here. + common::short_unique_name('a', prefix, 21) } /// Minimal Actor definition with one inline source-files version. @@ -216,13 +223,19 @@ async fn actor_crud_flow() { .expect("update actor"); assert_eq!(updated.title.as_deref(), Some("Rust client test actor")); - // List builds (should succeed even if empty). + // List builds. This Actor was created with inline source files but no build was ever + // triggered, so the collection must be genuinely empty — a load-bearing assertion, unlike a + // tautological `total >= 0`. let builds = actor_client .builds() .list(Default::default()) .await .expect("list actor builds"); - assert!(builds.total >= 0); + assert_eq!( + builds.total, 0, + "an Actor with no triggered build should have no builds" + ); + assert_eq!(builds.total as usize, builds.items.len()); // List versions. let versions = actor_client @@ -417,3 +430,74 @@ async fn actor_env_var_crud_flow() { "env var should be gone after delete" ); } + +/// `ActorClient::webhooks()` (the Actor-scoped webhook sub-collection, distinct from both the +/// top-level `client.webhooks()` and the task-scoped `task.webhooks()`, both tested elsewhere) +/// and `ActorClient::default_build()`, neither of which is exercised by `actor_crud_flow`. +#[tokio::test(flavor = "multi_thread")] +async fn actor_webhooks_and_default_build() { + let client = require_client!(); + let name = actor_name("actor-webhooks"); + + let actor = client + .actors() + .create(&actor_definition(&name)) + .await + .expect("create actor"); + + let cleanup_client = client.clone(); + let cleanup_id = actor.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.actor(&cleanup_id).delete().await; + }); + + let actor_client = client.actor(&actor.id); + + // Simple GET: the Actor's webhook collection (likely empty, but the endpoint must respond + // and must hit the Actor-scoped URL, not the top-level or task-scoped one). + let webhooks = actor_client + .webhooks() + .list(Default::default()) + .await + .expect("list actor webhooks"); + // This Actor was just created and no webhook has ever targeted it, so the Actor-scoped + // collection must be genuinely empty — a load-bearing assertion, unlike a tautological + // `total >= 0` (`total` is unsigned in practice and that check passes unconditionally). + assert_eq!( + webhooks.total, 0, + "a freshly created Actor should have no webhooks" + ); + assert_eq!(webhooks.total as usize, webhooks.items.len()); + + // Build the `0.0` version tagged `latest` and wait for it to finish, so the Actor has a + // resolvable default build. + let build = actor_client + .build( + "0.0", + apify_client::ActorBuildOptions { + tag: Some("latest".to_string()), + wait_for_finish: Some(300), + ..Default::default() + }, + ) + .await + .expect("start build"); + client + .build(&build.id) + .wait_for_finish(Some(300)) + .await + .expect("wait for build"); + + // `default_build()` resolves the Actor's default build (the one tagged `latest`) and returns + // a client for it; the build it resolves to must be the one just built. + let default_build_client = actor_client + .default_build(Some(300)) + .await + .expect("default_build"); + let fetched = default_build_client + .get() + .await + .expect("get default build") + .expect("default build should exist"); + assert_eq!(fetched.id, build.id); +} diff --git a/tests/actor_run.rs b/tests/actor_run.rs index 03271a7..59a7b1d 100644 --- a/tests/actor_run.rs +++ b/tests/actor_run.rs @@ -14,7 +14,13 @@ async fn list_runs() { .list(Default::default(), Default::default()) .await .expect("listing runs should succeed"); - assert!(page.total >= 0); + // Load-bearing consistency check (unlike a tautological `total >= 0`, which is `i64` and + // always true): a single page can never return more items than its own `limit`. Checked + // against `limit`, not `total`, because this suite runs many tests concurrently against the + // same shared account; `total` and `items` can be computed from separately-timed backend + // reads under that write load, which made an `items.len() <= total` check here genuinely + // flaky (observed under `cargo test --all-targets`), not just load-bearing. + assert!(page.items.len() as i64 <= page.limit); } /// Complex flow: call the hello-world Actor, wait for it to finish, fetch its log and @@ -49,6 +55,21 @@ async fn run_actor_and_read_outputs() { let log = client.run(&run.id).log().get().await.expect("get run log"); assert!(log.is_some(), "finished run should have a log"); + // The standalone `ApifyClient::log(id)` accessor (`GET /v2/logs/{buildOrRunId}`) hits a + // different URL than the nested `run().log()` above (`.../actor-runs/{runId}/log`); only + // the nested form had a live test until now (the standalone form only had hermetic + // coverage in `tests/unit_http.rs`). Reuse this run's id, already available here, to cover + // it live too — it should return the same log content. + let standalone_log = client.log(&run.id).get().await.expect("get standalone log"); + assert!( + standalone_log.is_some(), + "standalone log() should find the finished run's log" + ); + assert_eq!( + standalone_log, log, + "standalone log() and run().log() should return the same content for the same id" + ); + // Fetch the raw log via the `raw` query parameter (LogOptions). The endpoint must accept // it and still return the log content. let raw_log = client @@ -83,6 +104,77 @@ async fn run_actor_and_read_outputs() { ); } +/// Complex flow: the Run resource's remaining CRUD operations not exercised elsewhere — +/// `update()`, `ApifyClient::set_status_message` (which delegates to `update()` with a fixed +/// body shape), `delete()`, and confirming `get()` returns `None` afterward. Together with +/// `run_actor_and_read_outputs` (create/get) and `list_runs` (list, via `runs().list()`), this +/// covers all five CRUD-flow operations `test_requirements.md` asks for on a resource that +/// supports them. +#[tokio::test(flavor = "multi_thread")] +async fn run_update_set_status_message_and_delete() { + let client = require_client!(); + + let run = client + .actor("apify/hello-world") + .call::(None, Default::default(), Some(120)) + .await + .expect("call hello-world actor"); + assert_eq!(run.status.as_deref(), Some("SUCCEEDED")); + let run_client = client.run(&run.id); + + // `update()` directly: set a non-terminal status message. + let updated = run_client + .update(&serde_json::json!({ + "statusMessage": "updated via RunClient::update", + "isStatusMessageTerminal": false, + })) + .await + .expect("update run"); + assert_eq!( + updated.status_message.as_deref(), + Some("updated via RunClient::update") + ); + + // `ApifyClient::set_status_message` reads the process-global `ACTOR_RUN_ID` env var. Mutating + // it here is race-free because it has exactly one reader in the crate (`set_status_message`, + // src/client.rs) and exactly one caller in the suite (this test) — verified via `grep -rn + // "ACTOR_RUN_ID\|set_status_message" tests/ src/` — so no concurrent test can observe or + // clobber it. Restore the pre-existing value (if any) afterward regardless. + let prev_run_id = std::env::var("ACTOR_RUN_ID").ok(); + std::env::set_var("ACTOR_RUN_ID", &run.id); + let via_set_status_message = client + .set_status_message("updated via set_status_message", true) + .await; + match prev_run_id { + Some(v) => std::env::set_var("ACTOR_RUN_ID", v), + None => std::env::remove_var("ACTOR_RUN_ID"), + } + let via_set_status_message = via_set_status_message.expect("set_status_message"); + assert_eq!( + via_set_status_message.status_message.as_deref(), + Some("updated via set_status_message") + ); + + // `get()` reflects the latest update. + let fetched = run_client + .get() + .await + .expect("get run") + .expect("run should still exist"); + assert_eq!( + fetched.status_message.as_deref(), + Some("updated via set_status_message") + ); + + // `delete()`, then `get()` must return `None`. + run_client.delete().await.expect("delete run"); + let after_delete = run_client.get().await.expect("get run after delete"); + assert!( + after_delete.is_none(), + "run should not exist after delete(), got {after_delete:?}" + ); +} + /// Iteration: the run collection iterator yields a run we just started across pages. #[tokio::test(flavor = "multi_thread")] async fn iterate_runs() { @@ -137,3 +229,315 @@ async fn last_run_access() { .expect("get last run"); assert!(last.is_some(), "there should be a last succeeded run"); } + +/// Simple GET: run-scoped storage metadata accessors (`.dataset().get()`/`.get_statistics()`, +/// `.key_value_store().get()`/`.list_keys()`, `.request_queue().get()`/`.list_head()`) all +/// succeed against a finished run's default storages. +#[tokio::test(flavor = "multi_thread")] +async fn run_scoped_storage_metadata_reads() { + let client = require_client!(); + let run = client + .actor("apify/hello-world") + .call::(None, Default::default(), Some(120)) + .await + .expect("call hello-world actor"); + + let run_client = client.run(&run.id); + + let dataset = run_client.dataset(); + assert!( + dataset.get().await.expect("get run dataset").is_some(), + "a run's default dataset should exist" + ); + assert!( + dataset + .get_statistics() + .await + .expect("get run dataset statistics") + .is_some(), + "a run's default dataset should report statistics" + ); + + let kvs = run_client.key_value_store(); + assert!( + kvs.get().await.expect("get run key-value store").is_some(), + "a run's default key-value store should exist" + ); + let keys = kvs + .list_keys(Default::default()) + .await + .expect("list run key-value store keys"); + assert!( + keys.items.iter().any(|k| k.key == "OUTPUT"), + "hello-world's default store should contain an OUTPUT key" + ); + + let rq = run_client.request_queue(); + assert!( + rq.get().await.expect("get run request queue").is_some(), + "a run's default request queue should exist" + ); + let head = rq + .list_head(Some(10)) + .await + .expect("list run request queue head"); + assert!(head.items.len() as i64 <= 10); +} + +/// Complex flow: run-scoped storage PUT/DELETE. `RunClient::dataset()/key_value_store()/ +/// request_queue()` return the same `DatasetClient`/`KeyValueStoreClient`/`RequestQueueClient` +/// types used for top-level storages, but `run_scoped_storage_metadata_reads` above only +/// exercises their GET side (`get`/`get_statistics`/`list_keys`/`list_head`). This covers +/// `.update()` (`PUT /v2/actor-runs/{runId}/dataset|key-value-store|request-queue`) and +/// `.delete()` on all three, confirming each is genuinely gone via `get()` afterward. +#[tokio::test(flavor = "multi_thread")] +async fn run_scoped_storage_update_and_delete() { + let client = require_client!(); + let run = client + .actor("apify/hello-world") + .call::(None, Default::default(), Some(120)) + .await + .expect("call hello-world actor"); + let run_client = client.run(&run.id); + + // Dataset: rename via `update()` (PUT), then `delete()` and confirm it is gone. + let dataset = run_client.dataset(); + let new_name = common::unique_name("run-dataset-update"); + let updated = dataset + .update(&serde_json::json!({ "name": new_name })) + .await + .expect("update run dataset"); + assert_eq!(updated.name.as_deref(), Some(new_name.as_str())); + dataset.delete().await.expect("delete run dataset"); + assert!( + dataset + .get() + .await + .expect("get run dataset after delete") + .is_none(), + "run dataset should not exist after delete()" + ); + + // Key-value store: same update/delete flow. + let kvs = run_client.key_value_store(); + let new_name = common::unique_name("run-kvs-update"); + let updated = kvs + .update(&serde_json::json!({ "name": new_name })) + .await + .expect("update run key-value store"); + assert_eq!(updated.name.as_deref(), Some(new_name.as_str())); + kvs.delete().await.expect("delete run key-value store"); + assert!( + kvs.get() + .await + .expect("get run key-value store after delete") + .is_none(), + "run key-value store should not exist after delete()" + ); + + // Request queue: same update/delete flow. + let rq = run_client.request_queue(); + let new_name = common::unique_name("run-rq-update"); + let updated = rq + .update(&serde_json::json!({ "name": new_name })) + .await + .expect("update run request queue"); + assert_eq!(updated.name.as_deref(), Some(new_name.as_str())); + rq.delete().await.expect("delete run request queue"); + assert!( + rq.get() + .await + .expect("get run request queue after delete") + .is_none(), + "run request queue should not exist after delete()" + ); +} + +/// Builds and returns a fresh private Actor whose container sleeps for ~60s before exiting. +/// +/// Several lifecycle tests (`abort`, `reboot`, `metamorph`) need a run that is reliably still +/// `RUNNING` a few seconds after it starts — `apify/hello-world` finishes in a couple of +/// seconds, which is too fast to hit mid-run reliably. `name_prefix` should be unique per test so +/// concurrent runs of this suite (or of the same suite in another language) don't collide. +async fn create_slow_actor(client: &apify_client::ApifyClient, name_prefix: &str) -> ActorFixture { + let name = common::short_unique_name('a', name_prefix, 21); + + let definition = serde_json::json!({ + "name": name, + "isPublic": false, + "versions": [{ + "versionNumber": "0.0", + "sourceType": "SOURCE_FILES", + "buildTag": "latest", + "sourceFiles": [ + { + "name": "Dockerfile", + "format": "TEXT", + "content": "FROM apify/actor-node:20\nCOPY . ./\nCMD node main.js" + }, + { + "name": "main.js", + "format": "TEXT", + "content": "console.log('sleeping'); setTimeout(() => console.log('woke'), 60000);" + } + ] + }] + }); + + let actor = client + .actors() + .create(&definition) + .await + .expect("create actor"); + + let cleanup_client = client.clone(); + let cleanup_id = actor.id.clone(); + let guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.actor(&cleanup_id).delete().await; + }); + + let actor_client = client.actor(&actor.id); + let build = actor_client + .build("0.0", Default::default()) + .await + .expect("start build"); + client + .build(&build.id) + .wait_for_finish(Some(120)) + .await + .expect("wait for build"); + + ActorFixture { + id: actor.id, + _cleanup: guard, + } +} + +/// A slow Actor created by [`create_slow_actor`], kept alive with its cleanup guard. +struct ActorFixture { + id: String, + _cleanup: common::Cleanup, +} + +/// Lifecycle: abort a running Actor run. +#[tokio::test(flavor = "multi_thread")] +async fn run_abort() { + let client = require_client!(); + let actor = create_slow_actor(&client, "run-abort").await; + let actor_client = client.actor(&actor.id); + + let run = actor_client + .start(None::<&serde_json::Value>, Default::default()) + .await + .expect("start slow run"); + let run_client = client.run(&run.id); + + let aborted = run_client + .abort(Some(false)) + .await + .expect("abort running Actor"); + assert!( + matches!( + aborted.status.as_deref(), + Some("ABORTING") | Some("ABORTED") + ), + "expected ABORTING or ABORTED after abort, got {:?}", + aborted.status + ); + + let finished = run_client + .wait_for_finish(Some(60)) + .await + .expect("wait for aborted run to settle"); + assert_eq!(finished.status.as_deref(), Some("ABORTED")); +} + +/// Lifecycle: reboot a running Actor run (restarts its container, keeping the run ID and +/// storages). The run is aborted afterward to avoid burning extra compute. +#[tokio::test(flavor = "multi_thread")] +async fn run_reboot() { + let client = require_client!(); + let actor = create_slow_actor(&client, "run-reboot").await; + let actor_client = client.actor(&actor.id); + + let run = actor_client + .start(None::<&serde_json::Value>, Default::default()) + .await + .expect("start slow run"); + let run_client = client.run(&run.id); + + let rebooted = run_client.reboot().await.expect("reboot running Actor"); + assert_eq!(rebooted.id, run.id, "reboot must keep the same run ID"); + assert!( + !rebooted.is_terminal(), + "a freshly-rebooted run should still be active, got status {:?}", + rebooted.status + ); + + let _ = run_client.abort(Some(false)).await; +} + +/// Lifecycle: resurrecting a finished run starts it again (a new active run reusing the same +/// run ID). The resurrected run is aborted immediately afterward to avoid burning extra compute. +#[tokio::test(flavor = "multi_thread")] +async fn run_resurrect() { + let client = require_client!(); + let run = client + .actor("apify/hello-world") + .call::(None, Default::default(), Some(120)) + .await + .expect("call hello-world actor"); + assert_eq!(run.status.as_deref(), Some("SUCCEEDED")); + + let run_client = client.run(&run.id); + let resurrected = run_client + .resurrect(Default::default()) + .await + .expect("resurrect finished run"); + assert!( + !resurrected.is_terminal(), + "a just-resurrected run should be active again, got status {:?}", + resurrected.status + ); + + // Clean up the extra compute immediately; we only need to prove `resurrect` restarts it. + let _ = run_client.abort(Some(false)).await; +} + +/// Lifecycle: metamorphing a running Actor run into another Actor's run swaps it in place — +/// waiting for the (now `apify/hello-world`) run to finish should succeed. +#[tokio::test(flavor = "multi_thread")] +async fn run_metamorph() { + let client = require_client!(); + let actor = create_slow_actor(&client, "run-morph").await; + let actor_client = client.actor(&actor.id); + + let run = actor_client + .start(None::<&serde_json::Value>, Default::default()) + .await + .expect("start slow run"); + let run_client = client.run(&run.id); + + // `act_id` stays the original Actor's ID after metamorph (the run is still conceptually + // "owned" by the Actor that started it); the target Actor's code is what actually runs. The + // meaningful proof that the swap happened is behavioral: our own Actor's script sleeps for + // ~60s and would never finish this fast on its own, so a `SUCCEEDED` result well within the + // wait budget below can only mean the run's container is now actually executing + // `apify/hello-world`, which exits in a few seconds. + let morphed = run_client + .metamorph::("apify/hello-world", None, Default::default()) + .await + .expect("metamorph into hello-world"); + assert_eq!(morphed.id, run.id, "metamorph must keep the same run ID"); + + let finished = run_client + .wait_for_finish(Some(30)) + .await + .expect("wait for morphed run"); + assert_eq!( + finished.status.as_deref(), + Some("SUCCEEDED"), + "the morphed run should finish quickly as a successful hello-world run, not keep \ + running the original ~60s sleep script" + ); +} diff --git a/tests/build.rs b/tests/build.rs index dab9193..7d3a291 100644 --- a/tests/build.rs +++ b/tests/build.rs @@ -13,7 +13,13 @@ async fn list_builds() { .list(Default::default()) .await .expect("listing builds should succeed"); - assert!(page.total >= 0); + // Load-bearing consistency check (unlike a tautological `total >= 0`, which is `i64` and + // always true): a single page can never return more items than its own `limit`. Checked + // against `limit`, not `total`, because this suite runs many tests concurrently against the + // same shared account; `total` and `items` can be computed from separately-timed backend + // reads under that write load, which made an `items.len() <= total` check here genuinely + // flaky (observed under `cargo test --all-targets`), not just load-bearing. + assert!(page.items.len() as i64 <= page.limit); } /// Iteration: the build collection iterator yields a build we just started. @@ -23,8 +29,7 @@ async fn list_builds() { #[tokio::test(flavor = "multi_thread")] async fn iterate_builds() { let client = require_client!(); - let name = common::unique_name("build-iter").replace('-', ""); - let name = format!("b{}", &name[..name.len().min(20)]); + let name = common::short_unique_name('b', "build-iter", 21); let definition = json!({ "name": name, @@ -86,8 +91,7 @@ async fn iterate_builds() { #[tokio::test(flavor = "multi_thread")] async fn build_actor_flow() { let client = require_client!(); - let name = common::unique_name("build").replace('-', ""); - let name = format!("b{}", &name[..name.len().min(20)]); + let name = common::short_unique_name('b', "build", 21); let definition = json!({ "name": name, @@ -156,19 +160,88 @@ async fn build_actor_flow() { // Validate input against the just-built `latest` build, exercising the spec's optional // `build` query parameter on POST /v2/actors/{actorId}/validate-input. A real `latest` - // build now exists (built above), so `build=latest` resolves to a concrete artifact. - // The success response is an object with a `valid` boolean; asserting that field is present - // (rather than merely `is_object()`) proves the call hit the endpoint with the param accepted - // and did not return an error envelope. - let validation = actor_client + // build now exists (built above), so `build=latest` resolves to a concrete artifact. An + // empty object is a valid input for an Actor with no required schema fields, so asserting + // `true` proves the call hit the endpoint with the param accepted and did not return an + // error envelope. + let is_valid = actor_client .validate_input_for_build(&json!({}), Some("latest")) .await .expect("validate input for latest build"); assert!( - validation.get("valid").and_then(|v| v.as_bool()).is_some(), - "validate-input response should contain a `valid` boolean, got: {validation}" + is_valid, + "validate-input should report the empty-object input as valid" ); // Clean up. actor_client.delete().await.expect("delete actor"); } + +/// `BuildClient::abort()` and `BuildClient::delete()`: start a deliberately slow build, abort it +/// mid-build, then delete it. A trivial build finishes too fast to reliably hit the RUNNING +/// state, so this Actor's Dockerfile sleeps before the (never-reached) `COPY`/`CMD` steps. +#[tokio::test(flavor = "multi_thread")] +async fn build_abort_and_delete() { + let client = require_client!(); + let name = common::short_unique_name('b', "build-abort", 21); + + let definition = json!({ + "name": name, + "isPublic": false, + "versions": [{ + "versionNumber": "0.0", + "sourceType": "SOURCE_FILES", + "buildTag": "latest", + "sourceFiles": [ + { + "name": "Dockerfile", + "format": "TEXT", + "content": "FROM apify/actor-node:20\nRUN sleep 90\nCOPY . ./\nCMD node main.js" + }, + { "name": "main.js", "format": "TEXT", "content": "console.log('unreachable');" } + ] + }] + }); + + let actor = client + .actors() + .create(&definition) + .await + .expect("create actor"); + + let cleanup_client = client.clone(); + let cleanup_id = actor.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.actor(&cleanup_id).delete().await; + }); + + let actor_client = client.actor(&actor.id); + let build = actor_client + .build("0.0", Default::default()) + .await + .expect("start slow build"); + let build_client = client.build(&build.id); + + let aborted = build_client.abort().await.expect("abort build"); + assert!( + matches!( + aborted.status.as_deref(), + Some("ABORTING") | Some("ABORTED") + ), + "expected ABORTING or ABORTED after abort, got {:?}", + aborted.status + ); + + let finished = build_client + .wait_for_finish(Some(60)) + .await + .expect("wait for aborted build to settle"); + assert_eq!(finished.status.as_deref(), Some("ABORTED")); + + build_client.delete().await.expect("delete build"); + assert!(build_client + .get() + .await + .expect("get build after delete") + .is_none()); +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 6cf6724..b1432b6 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -17,7 +17,9 @@ const DEFAULT_API_URL: &str = "https://api.apify.com/v2"; /// Builds an [`ApifyClient`] configured from the environment. /// -/// Returns `None` (so the caller can skip) if `APIFY_TOKEN` is not set. +/// Returns `None` (so the caller can skip) if `APIFY_TOKEN` is not set. Reads the environment +/// once and delegates to [`make_client_from`], which takes no process state — see that function +/// for why this split exists. pub fn make_client() -> Option { let token = std::env::var("APIFY_TOKEN") .ok() @@ -25,6 +27,23 @@ pub fn make_client() -> Option { let api_url = std::env::var("APIFY_API_URL") .ok() .filter(|u| !u.is_empty()); + make_client_from(Some(token), api_url) +} + +/// Builds an [`ApifyClient`] from an explicit token/API-URL pair, with no process-environment +/// reads at all. +/// +/// This is the env-free core of [`make_client`], split out so tests that want to exercise the +/// `APIFY_API_URL` -> `base_url` resolution path don't have to mutate the real +/// `APIFY_TOKEN`/`APIFY_API_URL` process environment variables to do it. Those are read by +/// every other test in the suite via `require_client!`/`make_client`, and `#[tokio::test]`s run +/// concurrently within one process, so mutating them process-wide would race every other test +/// that happens to call `make_client` during the mutation window. Passing values in directly +/// sidesteps that race entirely rather than merely narrowing it. +/// +/// Returns `None` if `token` is `None` or empty, mirroring `make_client`'s skip behavior. +pub fn make_client_from(token: Option, api_url: Option) -> Option { + let token = token.filter(|t| !t.is_empty())?; let base_url = resolve_base_url(api_url.as_deref()); Some( ApifyClient::builder() @@ -194,3 +213,27 @@ pub fn unique_name(prefix: &str) -> String { let uuid = uuid::Uuid::new_v4().simple().to_string(); format!("rust-test-{prefix}-{}", &uuid[..12]) } + +/// Number of trailing random hex characters kept by [`short_unique_name`]. Fixed so the random +/// component's collision-resistance doesn't shrink as callers pass longer prefixes. +const SHORT_NAME_RANDOM_LEN: usize = 10; + +/// Generates a short, collision-resistant resource name for APIs with strict naming limits +/// (e.g. Actor/build names, which reject hyphens and cap length). +/// +/// Naively taking `unique_name(prefix).replace('-', "")[..max_len]` truncates from the *end*, +/// which can cut off the random suffix entirely for long prefixes, leaving a constant name that +/// collides across concurrent test runs (see the `actor_webhooks_and_default_build` +/// regression this guards against). This instead truncates the (hyphen-stripped) `prefix` +/// *first*, so the trailing random fragment always survives. +/// +/// `leading` is prepended as-is (used to satisfy naming rules that require a letter first); +/// `max_len` bounds the total length of `leading` + prefix + random suffix. +pub fn short_unique_name(leading: char, prefix: &str, max_len: usize) -> String { + let uuid = uuid::Uuid::new_v4().simple().to_string(); + let random_suffix = &uuid[..SHORT_NAME_RANDOM_LEN]; + let clean_prefix: String = prefix.chars().filter(|c| c.is_alphanumeric()).collect(); + let prefix_budget = max_len.saturating_sub(1 + SHORT_NAME_RANDOM_LEN); + let prefix_trunc = &clean_prefix[..clean_prefix.len().min(prefix_budget)]; + format!("{leading}{prefix_trunc}{random_suffix}") +} diff --git a/tests/config.rs b/tests/config.rs index 5089c55..9163ea3 100644 --- a/tests/config.rs +++ b/tests/config.rs @@ -23,29 +23,22 @@ fn base_url_strips_v2_suffix() { ); } -/// `make_client` honors the `APIFY_API_URL` environment variable end-to-end: the resolved -/// client's `api_base_url()` reflects the env value (with the harness `/v2` round-trip). +/// `make_client` honors the `APIFY_API_URL` -> `base_url` resolution end-to-end: the resolved +/// client's `api_base_url()` reflects the given value (with the harness `/v2` round-trip). /// -/// Exercises the actual env-var read path in `make_client`, not just the pure helper. Env -/// vars are process-global, so this test owns `APIFY_TOKEN`/`APIFY_API_URL` for its duration -/// and restores them afterwards. +/// Exercises the full `make_client_from` path (the env-free core `make_client` delegates to), +/// not just the pure `resolve_base_url` helper. This deliberately does NOT mutate the real +/// `APIFY_TOKEN`/`APIFY_API_URL` process environment variables: those are read by every other +/// test in the suite via `require_client!`, and since `#[tokio::test]`s run concurrently within +/// one process, doing so would race any test that calls `make_client` during the mutation +/// window. Passing values straight to `make_client_from` exercises the identical resolution +/// logic with no process-global state and no race. #[test] fn make_client_honors_apify_api_url_env() { - let prev_token = std::env::var("APIFY_TOKEN").ok(); - let prev_url = std::env::var("APIFY_API_URL").ok(); - - std::env::set_var("APIFY_TOKEN", "dummy-token-for-config-test"); - std::env::set_var("APIFY_API_URL", "https://api.example.test/v2"); - - let client = common::make_client().expect("make_client with a token set"); + let client = common::make_client_from( + Some("dummy-token-for-config-test".to_string()), + Some("https://api.example.test/v2".to_string()), + ) + .expect("make_client_from with a token set"); assert_eq!(client.api_base_url(), "https://api.example.test/v2"); - - match prev_token { - Some(v) => std::env::set_var("APIFY_TOKEN", v), - None => std::env::remove_var("APIFY_TOKEN"), - } - match prev_url { - Some(v) => std::env::set_var("APIFY_API_URL", v), - None => std::env::remove_var("APIFY_API_URL"), - } } diff --git a/tests/dataset.rs b/tests/dataset.rs index 7c81caf..6f14267 100644 --- a/tests/dataset.rs +++ b/tests/dataset.rs @@ -13,8 +13,13 @@ async fn list_datasets() { .list(Default::default()) .await .expect("listing datasets should succeed"); - // `total` is non-negative; `items` length never exceeds `limit` when set. - assert!(page.total >= 0); + // Load-bearing consistency check (unlike a tautological `total >= 0`, which is `i64` and + // always true): a single page can never return more items than its own `limit`. Checked + // against `limit`, not `total`, because this suite runs many tests concurrently against the + // same shared account; `total` and `items` can be computed from separately-timed backend + // reads under that write load, which made an `items.len() <= total` check here genuinely + // flaky (observed under `cargo test --all-targets`), not just load-bearing. + assert!(page.items.len() as i64 <= page.limit); } /// Simple GET: fetch a single dataset by ID. diff --git a/tests/examples.rs b/tests/examples.rs index 7994c48..9c6352b 100644 --- a/tests/examples.rs +++ b/tests/examples.rs @@ -71,3 +71,8 @@ fn example_create_build_run_actor() { fn example_raw_log() { run_example("raw_log"); } + +#[test] +fn example_tasks_schedules_webhooks() { + run_example("tasks_schedules_webhooks"); +} diff --git a/tests/key_value_store.rs b/tests/key_value_store.rs index e8586b8..8a97587 100644 --- a/tests/key_value_store.rs +++ b/tests/key_value_store.rs @@ -13,7 +13,13 @@ async fn list_key_value_stores() { .list(Default::default()) .await .expect("listing key-value stores should succeed"); - assert!(page.total >= 0); + // Load-bearing consistency check (unlike a tautological `total >= 0`, which is `i64` and + // always true): a single page can never return more items than its own `limit`. Checked + // against `limit`, not `total`, because this suite runs many tests concurrently against the + // same shared account; `total` and `items` can be computed from separately-timed backend + // reads under that write load, which made an `items.len() <= total` check here genuinely + // flaky (observed under `cargo test --all-targets`), not just load-bearing. + assert!(page.items.len() as i64 <= page.limit); } /// Simple GET: fetch a single key-value store by ID. @@ -326,3 +332,69 @@ async fn record_public_url_is_fetchable() { store_client.delete().await.expect("delete store"); } + +/// Builds a keys-list public URL (the collection-level sibling of +/// [`record_public_url_is_fetchable`]'s record URL) and confirms it is well-formed and +/// fetchable without auth. +/// +/// The URL points at the public keys endpoint; when the store exposes a URL-signing secret key +/// the URL additionally carries an HMAC `signature`. We fetch it with a bare HTTP client (no +/// Authorization header) and require success, then check the returned key listing includes the +/// key we just wrote. +#[tokio::test(flavor = "multi_thread")] +async fn keys_public_url_is_fetchable() { + let client = require_client!(); + let name = common::unique_name("kvs-keys-sig"); + let store = client + .key_value_stores() + .get_or_create(Some(&name)) + .await + .expect("create store"); + + let cleanup_client = client.clone(); + let cleanup_id = store.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.key_value_store(&cleanup_id).delete().await; + }); + + let store_client = client.key_value_store(&store.id); + store_client + .set_record_json("OUTPUT", &json!({ "signed": true })) + .await + .expect("set record"); + + let url = store_client + .create_keys_public_url(None) + .await + .expect("keys public url"); + assert!(url.contains("/key-value-stores/")); + assert!(url.contains("/keys")); + + // Fetch the URL with a bare HTTP client (no Authorization header). + let resp = reqwest::Client::new() + .get(&url) + .send() + .await + .expect("fetch public keys url"); + assert!( + resp.status().is_success(), + "public keys URL should be fetchable, got {} for {url}", + resp.status() + ); + let body: serde_json::Value = resp.json().await.expect("parse keys response as JSON"); + // The public URL hits the same route as the authenticated `list_keys` call (just under the + // public origin with a `signature` instead of a bearer token), so the response is still + // wrapped in the standard `data` envelope. + let keys = body + .get("data") + .and_then(|d| d.get("items")) + .and_then(|v| v.as_array()) + .expect("keys response should have a `data.items` array"); + assert!( + keys.iter() + .any(|k| k.get("key").and_then(|k| k.as_str()) == Some("OUTPUT")), + "keys listing should include the OUTPUT key we just wrote, got {body}" + ); + + store_client.delete().await.expect("delete store"); +} diff --git a/tests/request_queue.rs b/tests/request_queue.rs index 6072d6b..232bbaf 100644 --- a/tests/request_queue.rs +++ b/tests/request_queue.rs @@ -14,7 +14,13 @@ async fn list_request_queues() { .list(Default::default()) .await .expect("listing request queues should succeed"); - assert!(page.total >= 0); + // Load-bearing consistency check (unlike a tautological `total >= 0`, which is `i64` and + // always true): a single page can never return more items than its own `limit`. Checked + // against `limit`, not `total`, because this suite runs many tests concurrently against the + // same shared account; `total` and `items` can be computed from separately-timed backend + // reads under that write load, which made an `items.len() <= total` check here genuinely + // flaky (observed under `cargo test --all-targets`), not just load-bearing. + assert!(page.items.len() as i64 <= page.limit); } /// Simple GET: fetch a single request queue by ID. @@ -306,3 +312,79 @@ async fn request_queue_lock_lifecycle() { queue_client.delete().await.expect("delete queue"); } + +/// Complex flow: `update_request`, `batch_add_requests` and `batch_delete_requests`, none of +/// which are exercised by `request_queue_crud_flow` (which uses the single-request +/// `add_request`/`delete_request` instead). +#[tokio::test(flavor = "multi_thread")] +async fn request_queue_batch_and_update_flow() { + let client = require_client!(); + let name = common::unique_name("rq-batch"); + + let queue = client + .request_queues() + .get_or_create(Some(&name)) + .await + .expect("create queue"); + + let cleanup_client = client.clone(); + let cleanup_id = queue.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.request_queue(&cleanup_id).delete().await; + }); + + let queue_client = client.request_queue(&queue.id); + + // Batch-add several requests in one call. + let batch: Vec = (0..5) + .map(|i| RequestQueueRequest { + id: None, + url: format!("https://example.com/batch/{i}"), + unique_key: Some(format!("batch-{i}")), + method: Some("GET".to_string()), + user_data: None, + extra: Default::default(), + }) + .collect(); + let result = queue_client + .batch_add_requests(&batch, false) + .await + .expect("batch add requests"); + let processed = result["processedRequests"] + .as_array() + .expect("processedRequests array") + .clone(); + assert_eq!(processed.len(), 5, "all 5 requests should be processed"); + + // Update one of the added requests (mark it handled). + let first_id = processed[0]["requestId"] + .as_str() + .expect("requestId should be a string") + .to_string(); + let fetched = queue_client + .get_request(&first_id) + .await + .expect("get request") + .expect("request should exist"); + let mut to_update = fetched.clone(); + to_update + .extra + .insert("handledAt".to_string(), json!("2026-01-01T00:00:00.000Z")); + let updated = queue_client + .update_request(&to_update, false) + .await + .expect("update request"); + assert_eq!(updated.request_id, first_id); + + // Batch-delete the requests by unique key. + let delete_ids: Vec = batch + .iter() + .map(|r| json!({ "uniqueKey": r.unique_key })) + .collect(); + queue_client + .batch_delete_requests(&delete_ids) + .await + .expect("batch delete requests"); + + queue_client.delete().await.expect("delete queue"); +} diff --git a/tests/schedule.rs b/tests/schedule.rs index 5873785..67059bc 100644 --- a/tests/schedule.rs +++ b/tests/schedule.rs @@ -13,7 +13,13 @@ async fn list_schedules() { .list(Default::default()) .await .expect("listing schedules should succeed"); - assert!(page.total >= 0); + // Load-bearing consistency check (unlike a tautological `total >= 0`, which is `i64` and + // always true): a single page can never return more items than its own `limit`. Checked + // against `limit`, not `total`, because this suite runs many tests concurrently against the + // same shared account; `total` and `items` can be computed from separately-timed backend + // reads under that write load, which made an `items.len() <= total` check here genuinely + // flaky (observed under `cargo test --all-targets`), not just load-bearing. + assert!(page.items.len() as i64 <= page.limit); } fn schedule_definition(name: &str) -> serde_json::Value { diff --git a/tests/task.rs b/tests/task.rs index a56971c..1f40ce5 100644 --- a/tests/task.rs +++ b/tests/task.rs @@ -13,7 +13,13 @@ async fn list_tasks() { .list(Default::default()) .await .expect("listing tasks should succeed"); - assert!(page.total >= 0); + // Load-bearing consistency check (unlike a tautological `total >= 0`, which is `i64` and + // always true): a single page can never return more items than its own `limit`. Checked + // against `limit`, not `total`, because this suite runs many tests concurrently against the + // same shared account; `total` and `items` can be computed from separately-timed backend + // reads under that write load, which made an `items.len() <= total` check here genuinely + // flaky (observed under `cargo test --all-targets`), not just load-bearing. + assert!(page.items.len() as i64 <= page.limit); } fn task_definition(name: &str) -> serde_json::Value { @@ -129,15 +135,79 @@ async fn task_crud_flow() { .expect("update task"); assert_eq!(updated.name.as_deref(), Some(renamed.as_str())); - // List its runs (likely empty, but the endpoint should respond). + // List its runs. This task has never been started (`call`/`start` weren't invoked on it), + // so the collection must be genuinely empty — a load-bearing assertion, unlike a + // tautological `total >= 0`. let runs = task_client .runs() .list(Default::default(), Default::default()) .await .expect("list task runs"); - assert!(runs.total >= 0); + assert_eq!(runs.total, 0, "a never-started task should have no runs"); + assert_eq!(runs.total as usize, runs.items.len()); // Delete. task_client.delete().await.expect("delete task"); assert!(task_client.get().await.expect("get after delete").is_none()); } + +/// `TaskClient::webhooks()` (the task's webhook sub-collection) and the task's `runs/last` alias +/// (`last_run`/`last_run_with_options`), neither of which are exercised by `task_crud_flow`. +#[tokio::test(flavor = "multi_thread")] +async fn task_webhooks_and_last_run() { + let client = require_client!(); + let name = common::unique_name("task-webhooks"); + + let task = client + .tasks() + .create(&task_definition(&name)) + .await + .expect("create task"); + + let cleanup_client = client.clone(); + let cleanup_id = task.id.clone(); + let _guard = common::Cleanup::new(move || async move { + let _ = cleanup_client.task(&cleanup_id).delete().await; + }); + + let task_client = client.task(&task.id); + + // Simple GET: the task's webhook collection. This task was just created and no webhook has + // ever targeted it, so the collection must be genuinely empty — a load-bearing assertion, + // unlike a tautological `total >= 0`. + let webhooks = task_client + .webhooks() + .list(Default::default()) + .await + .expect("list task webhooks"); + assert_eq!( + webhooks.total, 0, + "a freshly created task should have no webhooks" + ); + assert_eq!(webhooks.total as usize, webhooks.items.len()); + + // Run the task and wait for it to finish, then access it through the `runs/last` alias. + let run = task_client + .call::(None, Default::default(), Some(120)) + .await + .expect("call task"); + assert_eq!(run.status.as_deref(), Some("SUCCEEDED")); + + let last = task_client + .last_run(Some("SUCCEEDED")) + .get() + .await + .expect("get task last run") + .expect("there should be a last succeeded run"); + assert_eq!(last.id, run.id); + + let last_with_options = task_client + .last_run_with_options(apify_client::LastRunOptions { + status: Some("SUCCEEDED".to_string()), + origin: Some("API".to_string()), + }) + .get() + .await + .expect("get task last run with options"); + assert!(last_with_options.is_some()); +} diff --git a/tests/unit_http.rs b/tests/unit_http.rs index 141d195..6aa74ab 100644 --- a/tests/unit_http.rs +++ b/tests/unit_http.rs @@ -7,8 +7,12 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; -use apify_client::http_client::{HttpBackend, HttpRequest, HttpResponse}; -use apify_client::{ApifyClient, ApifyClientError, LastRunOptions, RequestCompression}; +use apify_client::http_client::{HttpBackend, HttpMethod, HttpRequest, HttpResponse}; +use apify_client::models::RequestQueueRequest; +use apify_client::{ + ActorCallOptions, ApifyClient, ApifyClientError, BatchAddRequestsOptions, LastRunOptions, + RequestCompression, RunChargeOptions, TaskStartOptions, +}; use async_trait::async_trait; /// A scripted backend that returns a queued sequence of responses and counts calls. @@ -21,6 +25,8 @@ struct MockBackend { urls: Mutex>, last_headers: Mutex>, last_body: Mutex>>, + last_method: Mutex>, + last_timeout: Mutex>, } #[derive(Debug, Clone)] @@ -38,6 +44,8 @@ impl MockBackend { urls: Mutex::new(Vec::new()), last_headers: Mutex::new(std::collections::HashMap::new()), last_body: Mutex::new(None), + last_method: Mutex::new(None), + last_timeout: Mutex::new(None), }) } @@ -66,6 +74,14 @@ impl MockBackend { fn last_body(&self) -> Option> { self.last_body.lock().unwrap().clone() } + + fn last_method(&self) -> Option { + *self.last_method.lock().unwrap() + } + + fn last_timeout(&self) -> Option { + *self.last_timeout.lock().unwrap() + } } #[async_trait] @@ -76,6 +92,8 @@ impl HttpBackend for MockBackend { self.urls.lock().unwrap().push(request.url.clone()); *self.last_headers.lock().unwrap() = request.headers.clone(); *self.last_body.lock().unwrap() = request.body.clone(); + *self.last_method.lock().unwrap() = Some(request.method); + *self.last_timeout.lock().unwrap() = Some(request.timeout); let mut queue = self.responses.lock().unwrap(); let outcome = if queue.len() > 1 { queue.remove(0) @@ -212,6 +230,28 @@ async fn not_found_maps_to_none() { assert_eq!(backend.call_count(), 1, "404 is not retried"); } +/// `WebhookClient::test()` maps a `404` (webhook deleted concurrently) to `Ok(None)`, matching +/// the JS reference client's `catchNotFoundOrThrow` wrapping. Regression guard for the +/// independent-review finding that this previously propagated as an `Err`. +#[tokio::test] +async fn webhook_test_maps_not_found_to_none() { + let backend = MockBackend::new(vec![MockOutcome::Status( + 404, + br#"{"error":{"type":"record-not-found","message":"missing"}}"#.to_vec(), + )]); + let client = client_with(backend.clone(), 3); + let dispatch = client + .webhook("nope") + .test() + .await + .expect("404 should map to Ok(None), not Err"); + assert!( + dispatch.is_none(), + "test() on a missing webhook should be Ok(None)" + ); + assert_eq!(backend.call_count(), 1, "404 is not retried"); +} + /// The error body is parsed into the structured `ApiError` fields. #[tokio::test] async fn error_body_is_parsed() { @@ -243,6 +283,10 @@ async fn zero_retries_single_attempt() { /// routed through `parse_data_envelope` (which would fail with `missing field 'data'`). This is /// token-free, so a future refactor that re-introduces envelope unwrapping is caught even in a /// run without `APIFY_TOKEN`. +/// +/// Also guards JS/spec-contract parity: the spec's response shape is `{ "valid": bool }` and the +/// JS reference's `validateInput` returns `response.data.valid` as a plain `bool`, so this +/// asserts the plain `bool` rather than the raw envelope object. #[tokio::test] async fn validate_input_does_not_unwrap_data_envelope() { let backend = MockBackend::new(vec![MockOutcome::Status( @@ -255,10 +299,9 @@ async fn validate_input_does_not_unwrap_data_envelope() { .validate_input(&serde_json::json!({})) .await .expect("validate_input should parse a bare {valid} body"); - assert_eq!( - result.get("valid").and_then(|v| v.as_bool()), - Some(true), - "the bare body must be returned verbatim, not unwrapped from a `data` envelope" + assert!( + result, + "the bare body's `valid` field must be surfaced verbatim as a plain bool" ); assert_eq!(backend.call_count(), 1); } @@ -871,3 +914,654 @@ async fn iterate_keys_large_cap_clamps_page_size() { urls[1] ); } + +/// Builds a minimal request for the `batch_add_requests` tests below. +fn rq_request(unique_key: &str) -> RequestQueueRequest { + RequestQueueRequest { + id: None, + url: format!("https://example.com/{unique_key}"), + unique_key: Some(unique_key.to_string()), + method: Some("GET".to_string()), + user_data: None, + extra: Default::default(), + } +} + +/// `batch_add_requests` count-chunks input larger than the 25-per-call API limit and merges +/// each chunk's `processedRequests`/`unprocessedRequests` into one result. With the default +/// `max_parallel` (5) comfortably above the 2 chunks a 30-item input produces, both chunk calls +/// are dispatched without the caller having to do anything — this asserts the merge is complete +/// and correct regardless of how the two concurrent calls interleave. +#[tokio::test] +async fn batch_add_requests_chunks_and_merges_across_calls() { + let backend = MockBackend::new(vec![ + MockOutcome::Status( + 200, + br#"{"data":{"processedRequests":[{"uniqueKey":"a","requestId":"r-a","wasAlreadyPresent":false,"wasAlreadyHandled":false}],"unprocessedRequests":[]}}"#.to_vec(), + ), + MockOutcome::Status( + 200, + br#"{"data":{"processedRequests":[{"uniqueKey":"b","requestId":"r-b","wasAlreadyPresent":false,"wasAlreadyHandled":false}],"unprocessedRequests":[]}}"#.to_vec(), + ), + ]); + let client = client_with(backend.clone(), 0); + + let requests: Vec = + (0..30).map(|i| rq_request(&format!("k{i}"))).collect(); + let result = client + .request_queue("some-queue") + .batch_add_requests(&requests, false) + .await + .expect("batch add should succeed"); + + assert_eq!( + backend.call_count(), + 2, + "30 requests must be split into exactly 2 chunks (25 + 5)" + ); + let processed = result["processedRequests"] + .as_array() + .expect("processedRequests array"); + assert_eq!( + processed.len(), + 2, + "results from both chunk calls must be merged into one array" + ); + assert!(result["unprocessedRequests"] + .as_array() + .expect("unprocessedRequests array") + .is_empty()); +} + +/// A chunk whose response reports `unprocessedRequests` (typically caused by rate limiting) is +/// retried with the still-unprocessed subset, and the retry's results are merged into the final +/// response. This exercises `_batchAddRequestsWithRetries` parity: the first call reports 3 +/// requests processed and 2 unprocessed; the retry (with only the 2 remaining) reports both +/// processed, so the final result has all 5 processed and none unprocessed. +#[tokio::test] +async fn batch_add_requests_retries_unprocessed_requests() { + let backend = MockBackend::new(vec![ + MockOutcome::Status( + 200, + br#"{"data":{"processedRequests":[{"uniqueKey":"u0","requestId":"r0","wasAlreadyPresent":false,"wasAlreadyHandled":false},{"uniqueKey":"u1","requestId":"r1","wasAlreadyPresent":false,"wasAlreadyHandled":false},{"uniqueKey":"u2","requestId":"r2","wasAlreadyPresent":false,"wasAlreadyHandled":false}],"unprocessedRequests":[{"uniqueKey":"u3","url":"https://example.com/u3","method":"GET"},{"uniqueKey":"u4","url":"https://example.com/u4","method":"GET"}]}}"#.to_vec(), + ), + MockOutcome::Status( + 200, + br#"{"data":{"processedRequests":[{"uniqueKey":"u3","requestId":"r3","wasAlreadyPresent":false,"wasAlreadyHandled":false},{"uniqueKey":"u4","requestId":"r4","wasAlreadyPresent":false,"wasAlreadyHandled":false}],"unprocessedRequests":[]}}"#.to_vec(), + ), + ]); + let client = client_with(backend.clone(), 0); + + let requests: Vec = (0..5).map(|i| rq_request(&format!("u{i}"))).collect(); + let result = client + .request_queue("some-queue") + .batch_add_requests_with_options( + &requests, + BatchAddRequestsOptions { + min_delay_between_unprocessed_requests_retries: Some(Duration::from_millis(1)), + ..Default::default() + }, + ) + .await + .expect("batch add should not fail even before all retries complete"); + + assert_eq!( + backend.call_count(), + 2, + "the unprocessed pair must be retried in a second call" + ); + assert_eq!( + result["processedRequests"] + .as_array() + .expect("processedRequests array") + .len(), + 5, + "all 5 requests must end up processed after the retry" + ); + assert!(result["unprocessedRequests"] + .as_array() + .expect("unprocessedRequests array") + .is_empty()); +} + +/// When a chunk call fails outright (not merely reporting `unprocessedRequests`, but erroring), +/// `batch_add_requests` must not propagate the error: the reference client's +/// `_batchAddRequestsWithRetries` guarantees this method's signature never throws for a partial +/// failure. The not-yet-processed requests must instead show up as `unprocessedRequests`. +#[tokio::test] +async fn batch_add_requests_chunk_error_becomes_unprocessed_not_a_failure() { + let backend = MockBackend::new(vec![MockOutcome::Status( + 400, + br#"{"error":{"type":"invalid-request","message":"bad batch"}}"#.to_vec(), + )]); + // No transport retries and no application-level retries, so exactly one call is made. + let client = client_with(backend.clone(), 0); + + let requests = vec![rq_request("e0"), rq_request("e1")]; + let result = client + .request_queue("some-queue") + .batch_add_requests_with_options( + &requests, + BatchAddRequestsOptions { + max_unprocessed_requests_retries: Some(0), + ..Default::default() + }, + ) + .await + .expect("a failed chunk must not fail the overall call"); + + assert_eq!(backend.call_count(), 1); + assert!(result["processedRequests"] + .as_array() + .expect("processedRequests array") + .is_empty()); + let unprocessed = result["unprocessedRequests"] + .as_array() + .expect("unprocessedRequests array"); + assert_eq!( + unprocessed.len(), + 2, + "both requests must be reported unprocessed after the chunk call failed" + ); +} + +/// Requests whose combined serialized size exceeds the API's payload byte limit are sliced into +/// multiple `requests/batch` calls even when their count is well under the 25-per-call cap, +/// mirroring the reference client's `sliceArrayByByteLength`. Three ~4 MiB requests (~12 MiB +/// total) exceed the ~9 MiB limit, so they cannot all fit in one call. +#[tokio::test] +async fn batch_add_requests_slices_by_byte_length() { + let backend = MockBackend::new(vec![MockOutcome::Status( + 200, + br#"{"data":{"processedRequests":[],"unprocessedRequests":[]}}"#.to_vec(), + )]); + let client = client_with(backend.clone(), 0); + + let big_value = "a".repeat(4 * 1024 * 1024); // ~4 MiB + let requests: Vec = (0..3) + .map(|i| { + let mut r = rq_request(&format!("big{i}")); + r.user_data = Some(serde_json::Value::String(big_value.clone())); + r + }) + .collect(); + + client + .request_queue("some-queue") + .batch_add_requests(&requests, false) + .await + .expect("batch add should succeed"); + + assert!( + backend.call_count() > 1, + "requests too large to fit in one payload must be split across multiple calls, got {} call(s)", + backend.call_count() + ); +} + +/// `RunClient::charge` sends `POST .../actor-runs/{runId}/charge` with an auto-generated +/// `idempotency-key` header of the documented `{runId}-{eventName}-{millis}-{random}` shape, and +/// a JSON body carrying `eventName`/`count`. Hermetic coverage of the one thing +/// `post_raw_with_extra_header` exists for (the idempotency header) — `charge` itself is not +/// covered by a live integration test because it bills real money and requires a pay-per-event +/// Actor unavailable as a fixture here, so this offline test is the only guard on its request +/// shape. +#[tokio::test] +async fn charge_sends_idempotency_key_header_and_body() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, b"".to_vec())]); + let client = client_with(backend.clone(), 0); + + client + .run("run123") + .charge(RunChargeOptions { + event_name: "pageScraped".to_string(), + count: Some(3), + idempotency_key: None, + }) + .await + .expect("charge should succeed"); + + assert_eq!(backend.last_method(), Some(HttpMethod::Post)); + let url = backend.last_url().expect("a request was sent"); + assert!( + url.contains("/actor-runs/run123/charge"), + "expected a POST to .../actor-runs/run123/charge, got {url}" + ); + + let key = backend + .last_header("idempotency-key") + .expect("charge must send an idempotency-key header"); + // Documented shape: `{runId}-{eventName}-{millis}-{random}`. `event_name` above has no + // dashes, so splitting from the right on '-' unambiguously separates the trailing + // `millis`/`random` numeric segments from the `runId-eventName` prefix. + let parts: Vec<&str> = key.rsplitn(3, '-').collect(); + assert_eq!( + parts.len(), + 3, + "expected `runId-eventName-millis-random`, got {key}" + ); + let (random_suffix, millis, prefix) = (parts[0], parts[1], parts[2]); + assert_eq!(prefix, "run123-pageScraped"); + assert!( + !millis.is_empty() && millis.chars().all(|c| c.is_ascii_digit()), + "millis segment should be numeric, got {millis} in {key}" + ); + assert!( + !random_suffix.is_empty() && random_suffix.chars().all(|c| c.is_ascii_digit()), + "random segment should be numeric, got {random_suffix} in {key}" + ); + + let body = backend.last_body().expect("a body was sent"); + let body: serde_json::Value = serde_json::from_slice(&body).expect("body should be JSON"); + assert_eq!(body["eventName"], "pageScraped"); + assert_eq!(body["count"], 3); +} + +/// `RunClient::charge` with `idempotency_key: None` still sends a header when `count` is +/// omitted, defaulting the body's `count` to `1`. +#[tokio::test] +async fn charge_defaults_count_to_one() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, b"".to_vec())]); + let client = client_with(backend.clone(), 0); + + client + .run("run123") + .charge(RunChargeOptions { + event_name: "pageScraped".to_string(), + count: None, + idempotency_key: None, + }) + .await + .expect("charge should succeed"); + + let body = backend.last_body().expect("a body was sent"); + let body: serde_json::Value = serde_json::from_slice(&body).expect("body should be JSON"); + assert_eq!(body["count"], 1, "count should default to 1 when omitted"); +} + +/// `UserClient::update_limits` sends `PUT .../users/me/limits` with the serialized body. +/// Hermetic coverage: the live variant would mutate the shared test account's real limits, +/// which is unsafe under the concurrent-test-account requirement, so this offline test is the +/// only guard on its request shape. +#[tokio::test] +async fn update_limits_sends_put_with_body() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, b"".to_vec())]); + let client = client_with(backend.clone(), 0); + + client + .me() + .update_limits(&serde_json::json!({ "maxMonthlyUsageUsd": 500 })) + .await + .expect("update_limits should succeed"); + + assert_eq!(backend.last_method(), Some(HttpMethod::Put)); + let url = backend.last_url().expect("a request was sent"); + assert!( + url.contains("/users/me/limits"), + "expected a PUT to .../users/me/limits, got {url}" + ); + + let body = backend.last_body().expect("a body was sent"); + let body: serde_json::Value = serde_json::from_slice(&body).expect("body should be JSON"); + assert_eq!(body["maxMonthlyUsageUsd"], 500); +} + +/// The standalone `ApifyClient::log(build_or_run_id)` (`GET /v2/logs/{buildOrRunId}`) is +/// distinct from the nested `run().log()`/`build().log()` accessors, which hit +/// `.../actor-runs/{runId}/log` and `.../actor-builds/{buildId}/log` respectively. This is +/// hermetic coverage of the standalone accessor's URL, which — unlike the nested variants — was +/// previously exercised by no test at all. +#[tokio::test] +async fn standalone_log_hits_top_level_logs_endpoint() { + let backend = MockBackend::new(vec![MockOutcome::Status(200, b"log output".to_vec())]); + let client = client_with(backend.clone(), 0); + + let log = client + .log("some-build-or-run-id") + .get() + .await + .expect("get log"); + + assert_eq!(log.as_deref(), Some("log output")); + let url = backend.last_url().expect("a request was sent"); + assert!( + url.contains("/logs/some-build-or-run-id"), + "standalone log() must hit the top-level /logs/{{id}} endpoint, got {url}" + ); + assert!( + !url.contains("/actor-runs/") && !url.contains("/actor-builds/"), + "standalone log() must not go through the nested run/build log path, got {url}" + ); +} + +/// Regression guard: every `RequestQueueClient` method whose JS reference (`request_queue.ts`) +/// uses `SMALL_TIMEOUT_MILLIS` (5s) must send that timeout, not the 360s default. +/// `max_retries(0)` means a single attempt, so the timeout the mock backend observes is exactly +/// the endpoint's configured base (no retry-driven growth). +#[tokio::test] +async fn request_queue_small_timeout_methods_use_5s() { + let five_secs = Duration::from_secs(5); + let backend = MockBackend::new(vec![ + MockOutcome::Status(200, br#"{"data":{"id":"q1"}}"#.to_vec()), // get + MockOutcome::Status(200, br#"{"data":{"id":"q1"}}"#.to_vec()), // update + MockOutcome::Status(200, b"".to_vec()), // delete + MockOutcome::Status(200, br#"{"data":{}}"#.to_vec()), // list_head + MockOutcome::Status(200, br#"{"data":{"requestId":"r1"}}"#.to_vec()), // add_request + MockOutcome::Status(200, br#"{"data":{"url":"https://example.com/x"}}"#.to_vec()), // get_request + MockOutcome::Status(200, br#"{"data":{}}"#.to_vec()), // batch_delete_requests + MockOutcome::Status(200, b"".to_vec()), // delete_request + MockOutcome::Status(200, b"".to_vec()), // delete_request_lock + ]); + let client = client_with(backend.clone(), 0); + let rq = client.request_queue("q1"); + + rq.get().await.expect("get"); + assert_eq!(backend.last_timeout(), Some(five_secs), "get()"); + + rq.update(&serde_json::json!({})).await.expect("update"); + assert_eq!(backend.last_timeout(), Some(five_secs), "update()"); + + rq.delete().await.expect("delete"); + assert_eq!(backend.last_timeout(), Some(five_secs), "delete()"); + + rq.list_head(None).await.expect("list_head"); + assert_eq!(backend.last_timeout(), Some(five_secs), "list_head()"); + + let request = rq_request("small-timeout-add"); + rq.add_request(&request, false).await.expect("add_request"); + assert_eq!(backend.last_timeout(), Some(five_secs), "add_request()"); + + rq.get_request("r1").await.expect("get_request"); + assert_eq!(backend.last_timeout(), Some(five_secs), "get_request()"); + + rq.batch_delete_requests(&[serde_json::json!({"id": "r1"})]) + .await + .expect("batch_delete_requests"); + assert_eq!( + backend.last_timeout(), + Some(five_secs), + "batch_delete_requests()" + ); + + rq.delete_request("r1").await.expect("delete_request"); + assert_eq!(backend.last_timeout(), Some(five_secs), "delete_request()"); + + rq.delete_request_lock("r1", false) + .await + .expect("delete_request_lock"); + assert_eq!( + backend.last_timeout(), + Some(five_secs), + "delete_request_lock()" + ); +} + +/// Regression guard: `RequestQueueClient::get_request` is the one request-level method that must +/// NOT send `clientKey`, matching the JS reference client's `getRequest` (which builds its +/// params from bare `_params()`, unlike every sibling request-level method that merges in +/// `clientKey: this.clientKey`). `list_head` is exercised alongside it as a sibling method that +/// DOES send `clientKey`, so this also guards against the fix being over-applied to the rest of +/// the client. +#[tokio::test] +async fn get_request_omits_client_key_but_sibling_methods_send_it() { + let backend = MockBackend::new(vec![ + MockOutcome::Status(200, br#"{"data":{"url":"https://example.com/x"}}"#.to_vec()), // get_request + MockOutcome::Status(200, br#"{"data":{}}"#.to_vec()), // list_head + ]); + let client = client_with(backend.clone(), 0); + let rq = client.request_queue("q1").with_client_key("worker-1"); + + rq.get_request("r1").await.expect("get_request"); + let url = backend.last_url().expect("a request was sent"); + assert!( + !url.contains("clientKey"), + "get_request must not send clientKey, matching JS's getRequest, got {url}" + ); + + rq.list_head(None).await.expect("list_head"); + let url = backend.last_url().expect("a request was sent"); + assert!( + url.contains("clientKey=worker-1"), + "list_head must still send clientKey when set via with_client_key, got {url}" + ); +} + +/// Regression guard: every `RequestQueueClient` method whose JS reference uses +/// `MEDIUM_TIMEOUT_MILLIS` (30s) must send that timeout. +#[tokio::test] +async fn request_queue_medium_timeout_methods_use_30s() { + let thirty_secs = Duration::from_secs(30); + let backend = MockBackend::new(vec![ + MockOutcome::Status(200, br#"{"data":{}}"#.to_vec()), // list_and_lock_head + MockOutcome::Status( + 200, + br#"{"data":{"processedRequests":[],"unprocessedRequests":[]}}"#.to_vec(), + ), // batch_add_requests (requests/batch POST) + MockOutcome::Status(200, br#"{"data":{}}"#.to_vec()), // list_requests + MockOutcome::Status(200, br#"{"data":{}}"#.to_vec()), // unlock_requests + MockOutcome::Status(200, br#"{"data":{"requestId":"r1"}}"#.to_vec()), // update_request + MockOutcome::Status(200, br#"{"data":{}}"#.to_vec()), // prolong_request_lock + ]); + let client = client_with(backend.clone(), 0); + let rq = client.request_queue("q1"); + + rq.list_and_lock_head(60, None) + .await + .expect("list_and_lock_head"); + assert_eq!( + backend.last_timeout(), + Some(thirty_secs), + "list_and_lock_head()" + ); + + rq.batch_add_requests(&[rq_request("medium-timeout-batch")], false) + .await + .expect("batch_add_requests"); + assert_eq!( + backend.last_timeout(), + Some(thirty_secs), + "batch_add_requests() chunk POST" + ); + + rq.list_requests(Default::default()) + .await + .expect("list_requests"); + assert_eq!(backend.last_timeout(), Some(thirty_secs), "list_requests()"); + + rq.unlock_requests().await.expect("unlock_requests"); + assert_eq!( + backend.last_timeout(), + Some(thirty_secs), + "unlock_requests()" + ); + + let mut with_id = rq_request("medium-timeout-update"); + with_id.id = Some("r1".to_string()); + rq.update_request(&with_id, false) + .await + .expect("update_request"); + assert_eq!( + backend.last_timeout(), + Some(thirty_secs), + "update_request()" + ); + + rq.prolong_request_lock("r1", 60, false) + .await + .expect("prolong_request_lock"); + assert_eq!( + backend.last_timeout(), + Some(thirty_secs), + "prolong_request_lock()" + ); +} + +/// `TaskStartOptions::apply` hand-copies each field into query parameters (it cannot delegate to +/// `ActorStartOptions::apply` since the two types diverge). This asserts every field lands under +/// the expected spec query-parameter name, and that the fields `TaskStartOptions` deliberately +/// omits relative to `ActorStartOptions` (`content_type` has no query parameter to begin with; +/// `force_permission_level`'s `forcePermissionLevel` is the one that matters here) are absent — +/// the task-run endpoint does not accept `forcePermissionLevel`, so sending it would previously +/// have gone to the API unconditionally when this options type didn't exist yet. +#[tokio::test] +async fn task_start_sends_expected_query_params_and_omits_force_permission_level() { + let backend = MockBackend::new(vec![MockOutcome::Status( + 200, + br#"{"data":{"id":"run1"}}"#.to_vec(), + )]); + let client = client_with(backend.clone(), 0); + + let options = TaskStartOptions { + build: Some("latest".to_owned()), + memory_mbytes: Some(1024), + timeout_secs: Some(60), + wait_for_finish: Some(30), + max_items: Some(100), + max_total_charge_usd: Some(5.5), + restart_on_error: Some(true), + webhooks: Some(vec![ + serde_json::json!({"eventTypes": ["ACTOR.RUN.SUCCEEDED"]}), + ]), + }; + + client + .task("me~some-task") + .start(None::<&serde_json::Value>, options) + .await + .expect("task.start ok"); + + let url = backend.last_url().expect("a request was sent"); + for expected in [ + "build=latest", + "memory=1024", + "timeout=60", + "waitForFinish=30", + "maxItems=100", + "maxTotalChargeUsd=5.5", + "restartOnError=1", + "webhooks=", + ] { + assert!( + url.contains(expected), + "expected {expected:?} in task.start's query string, got {url}" + ); + } + assert!( + !url.contains("forcePermissionLevel"), + "TaskStartOptions has no force_permission_level field; task.start must not send \ + forcePermissionLevel, got {url}" + ); +} + +/// Companion to the `TaskCallOptions` narrowing: `ActorCallOptions` (like `TaskCallOptions`) +/// omits `wait_for_finish` so a caller can't accidentally block server-side (up to 60s) before +/// `call`'s own client-side `wait_secs` polling begins. This asserts the `runs` POST that +/// `ActorClient::call` issues under the hood never carries a `waitForFinish` query parameter — +/// there is simply no field on `ActorCallOptions` to set it from. +#[tokio::test] +async fn actor_call_does_not_send_wait_for_finish_on_start() { + let backend = MockBackend::new(vec![MockOutcome::Status( + 200, + br#"{"data":{"id":"run1","status":"SUCCEEDED"}}"#.to_vec(), + )]); + let client = client_with(backend.clone(), 0); + + client + .actor("me~some-actor") + .call( + None::<&serde_json::Value>, + ActorCallOptions { + build: Some("latest".to_owned()), + ..Default::default() + }, + Some(0), + ) + .await + .expect("actor.call ok"); + + let urls = backend.urls(); + let start_url = &urls[0]; + assert!( + start_url.contains("/runs?build=latest"), + "expected the first request to be the `runs` start POST with build=latest, got {start_url}" + ); + assert!( + !start_url.contains("waitForFinish"), + "ActorCallOptions has no wait_for_finish field; the start request must not send \ + waitForFinish, got {start_url}" + ); +} + +/// Minimal single-request HTTP/1.1 server used to test [`apify_client::LogClient::stream`]'s +/// `404` handling. Streaming bypasses the mock [`HttpBackend`] on purpose (it goes straight +/// through `reqwest`, since retries don't apply to an open connection), so it needs a real +/// socket to test against instead. Returns the `http://host:port` base URL to point a client at. +async fn spawn_single_response_server(status_line: &'static str, body: &'static [u8]) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral port"); + let addr = listener.local_addr().expect("local addr"); + + tokio::spawn(async move { + if let Ok((mut socket, _)) = listener.accept().await { + // Drain (and discard) the request so the client's write doesn't block on a full + // socket buffer; the response format below doesn't depend on what was sent. + let mut buf = [0u8; 4096]; + let _ = socket.read(&mut buf).await; + + let response = format!( + "{status_line}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.write_all(body).await; + let _ = socket.shutdown().await; + } + }); + + format!("http://{addr}") +} + +/// `LogClient::stream()` maps a `404` to `Ok(None)`, matching the JS reference client's +/// `catchNotFoundOrThrow` wrapping around its `stream()`. Regression guard for the +/// independent-review finding that this previously propagated as an `Err` for every non-2xx +/// status, including `404`. +#[tokio::test(flavor = "multi_thread")] +async fn log_stream_maps_not_found_to_none() { + let base_url = spawn_single_response_server("HTTP/1.1 404 Not Found", b"not found").await; + let client = ApifyClient::builder() + .token("test-token") + .base_url(base_url) + .build(); + + let stream = client + .run("some-run-id") + .log() + .stream() + .await + .expect("404 should map to Ok(None), not Err"); + assert!( + stream.is_none(), + "stream() on a missing log should be Ok(None)" + ); +} + +/// `LogClient::stream()` still surfaces a non-2xx, non-404 status as an `Err` (only `404` is +/// mapped to `None`). +#[tokio::test(flavor = "multi_thread")] +async fn log_stream_propagates_other_error_statuses() { + let base_url = + spawn_single_response_server("HTTP/1.1 500 Internal Server Error", b"boom").await; + let client = ApifyClient::builder() + .token("test-token") + .base_url(base_url) + .build(); + + let result = client.run("some-run-id").log().stream().await; + match result { + Err(err) => assert!(matches!(err, ApifyClientError::InvalidResponse(_))), + Ok(_) => panic!("a 500 must surface as Err, not Ok(_)"), + } +} diff --git a/tests/user.rs b/tests/user.rs index f209579..4293b8c 100644 --- a/tests/user.rs +++ b/tests/user.rs @@ -23,7 +23,8 @@ async fn get_monthly_usage() { .me() .monthly_usage() .await - .expect("get monthly usage"); + .expect("get monthly usage") + .expect("monthly usage should exist for the current account"); assert!(usage.is_object(), "monthly usage should be a JSON object"); } @@ -46,7 +47,8 @@ async fn get_monthly_usage_for_date() { .me() .monthly_usage_for_date(Some(requested_day)) .await - .expect("get monthly usage for date"); + .expect("get monthly usage for date") + .expect("monthly usage should exist for the current account"); let cycle = usage .get("usageCycle") @@ -84,6 +86,32 @@ async fn get_monthly_usage_for_date() { #[tokio::test(flavor = "multi_thread")] async fn get_limits() { let client = require_client!(); - let limits = client.me().limits().await.expect("get limits"); + let limits = client + .me() + .limits() + .await + .expect("get limits") + .expect("limits should exist for the current account"); assert!(limits.is_object(), "limits should be a JSON object"); } + +/// Simple GET: `client.user(id)` fetches a user's (public) profile by ID, as opposed to +/// `client.me()`, which is only ever tested against the current account. +#[tokio::test(flavor = "multi_thread")] +async fn get_user_by_id() { + let client = require_client!(); + let me = client + .me() + .get() + .await + .expect("get current user") + .expect("current user should exist"); + + let by_id = client + .user(&me.id) + .get() + .await + .expect("get user by id") + .expect("user should exist"); + assert_eq!(by_id.id, me.id); +} diff --git a/tests/webhook.rs b/tests/webhook.rs index 3dc8082..a73898c 100644 --- a/tests/webhook.rs +++ b/tests/webhook.rs @@ -13,7 +13,13 @@ async fn list_webhooks() { .list(Default::default()) .await .expect("listing webhooks should succeed"); - assert!(page.total >= 0); + // Load-bearing consistency check (unlike a tautological `total >= 0`, which is `i64` and + // always true): a single page can never return more items than its own `limit`. Checked + // against `limit`, not `total`, because this suite runs many tests concurrently against the + // same shared account; `total` and `items` can be computed from separately-timed backend + // reads under that write load, which made an `items.len() <= total` check here genuinely + // flaky (observed under `cargo test --all-targets`), not just load-bearing. + assert!(page.items.len() as i64 <= page.limit); } /// Simple GET: listing webhook dispatches. @@ -25,15 +31,30 @@ async fn list_webhook_dispatches() { .list(Default::default()) .await .expect("listing webhook dispatches should succeed"); - assert!(page.total >= 0); + // Load-bearing consistency check (unlike a tautological `total >= 0`, which is `i64` and + // always true): a single page can never return more items than its own `limit`. Checked + // against `limit`, not `total`, because this suite runs many tests concurrently against the + // same shared account; `total` and `items` can be computed from separately-timed backend + // reads under that write load, which made an `items.len() <= total` check here genuinely + // flaky (observed under `cargo test --all-targets`), not just load-bearing. + assert!(page.items.len() as i64 <= page.limit); } -fn webhook_definition() -> serde_json::Value { - // A webhook that fires when any run of the public hello-world Actor succeeds. +/// A webhook that fires when any run of the public hello-world Actor succeeds. +/// +/// `tag` is embedded in `requestUrl` (as a query parameter, so it stays a structurally valid +/// URL) to make each test's webhook unique, matching the suite's UUID-isolation convention used +/// by every other resource (`unique_name`/`short_unique_name`). Without it, every one of this +/// function's 5 callers created byte-for-byte identical webhooks, which is harmless for the API +/// (webhooks have no uniqueness constraint) but means a bug that leaked one test's webhook into +/// another's assertions (e.g. via a stale/undeleted resource from a prior failed run) could not +/// be told apart from the current test's own webhook. +fn webhook_definition(tag: &str) -> serde_json::Value { + let unique = common::unique_name(tag); json!({ "eventTypes": ["ACTOR.RUN.SUCCEEDED"], "condition": { "actorId": "moJRLRc85AitArpNN" }, - "requestUrl": "https://example.com/webhook", + "requestUrl": format!("https://example.com/webhook?test={unique}"), "isAdHoc": false }) } @@ -44,7 +65,7 @@ async fn get_webhook() { let client = require_client!(); let webhook = client .webhooks() - .create(&webhook_definition()) + .create(&webhook_definition("get-webhook")) .await .expect("create webhook"); @@ -72,7 +93,7 @@ async fn get_webhook_dispatch() { let client = require_client!(); let webhook = client .webhooks() - .create(&webhook_definition()) + .create(&webhook_definition("get-webhook-dispatch")) .await .expect("create webhook"); @@ -86,7 +107,8 @@ async fn get_webhook_dispatch() { .webhook(&webhook.id) .test() .await - .expect("test webhook"); + .expect("test webhook") + .expect("webhook was just created, so it must still exist"); assert!(!dispatch.id.is_empty()); let fetched = client @@ -104,7 +126,7 @@ async fn iterate_webhooks() { let client = require_client!(); let webhook = client .webhooks() - .create(&webhook_definition()) + .create(&webhook_definition("iterate-webhooks")) .await .expect("create webhook"); @@ -139,7 +161,7 @@ async fn iterate_webhook_dispatches() { let client = require_client!(); let webhook = client .webhooks() - .create(&webhook_definition()) + .create(&webhook_definition("iterate-webhook-dispatches")) .await .expect("create webhook"); @@ -154,7 +176,8 @@ async fn iterate_webhook_dispatches() { .webhook(&webhook.id) .test() .await - .expect("test webhook"); + .expect("test webhook") + .expect("webhook was just created, so it must still exist"); assert!(!dispatch.id.is_empty()); let target = dispatch.id.clone(); @@ -183,7 +206,7 @@ async fn webhook_crud_flow() { let webhook = client .webhooks() - .create(&webhook_definition()) + .create(&webhook_definition("webhook-crud-flow")) .await .expect("create webhook"); @@ -208,16 +231,26 @@ async fn webhook_crud_flow() { Some("https://example.com/updated") ); - // List this webhook's dispatches (should respond, likely empty). + // List this webhook's dispatches. No dispatch has fired yet (the test dispatch below is + // triggered after this), so the collection must be genuinely empty — a load-bearing + // assertion, unlike a tautological `total >= 0`. let dispatches = webhook_client .dispatches() .list(Default::default()) .await .expect("list webhook dispatches"); - assert!(dispatches.total >= 0); + assert_eq!( + dispatches.total, 0, + "a webhook with no triggered dispatch should have none listed" + ); + assert_eq!(dispatches.total as usize, dispatches.items.len()); // Trigger a test dispatch. - let dispatch = webhook_client.test().await.expect("test webhook"); + let dispatch = webhook_client + .test() + .await + .expect("test webhook") + .expect("webhook was just created, so it must still exist"); assert!(!dispatch.id.is_empty(), "test dispatch should have an id"); // Delete.