From c74cb317022fb56f46b46178de3f7171f7a94081 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 16:17:06 +0000 Subject: [PATCH 1/6] feat: sync Java client with Apify OpenAPI spec v2-2026-10-29T100357Z and port the apify-client-js v3 changes Spec sync: deployment filter on run lists, includeInputSchema on the Store list, user limits and monthly usage by user id, Run.standby (Server Actor tenancy) and the other new model fields. Ported from apify-client-js (v3 branch): timeout tiers (#1046), configurable request compression (#1065), default headers (#1058), normalized format:uri fields (#1045), pushItems shape check (#1066), raw byte[]/String Actor input (#1078), path-segment encoding of resource ids (#1011), serialize-once batchAddRequests (#1051), RunClient.iterateDatasetItems (#1085) and the listing replica-lag wait of the integration tests (#1090). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAKY9wkdABFznxvsCXt --- CHANGELOG.md | 52 ++++ README.md | 89 +++++- docs/README.md | 10 +- docs/actors.md | 2 +- docs/builds.md | 2 +- docs/misc.md | 10 +- docs/runs.md | 15 +- docs/storages.md | 18 +- pom.xml | 2 +- spotbugs-exclude.xml | 19 +- .../java/com/apify/client/ApifyClient.java | 19 ++ .../com/apify/client/ApifyClientBuilder.java | 111 ++++++- src/main/java/com/apify/client/Version.java | 4 +- .../com/apify/client/actor/ActorClient.java | 38 ++- .../client/actor/ActorCollectionClient.java | 3 +- .../client/actor/ActorDefaultRunOptions.java | 12 + .../com/apify/client/actor/ActorEnvVar.java | 6 + .../actor/ActorEnvVarCollectionClient.java | 3 +- .../com/apify/client/actor/ActorStandby.java | 21 ++ .../com/apify/client/actor/ActorVersion.java | 30 ++ .../actor/ActorVersionCollectionClient.java | 3 +- .../java/com/apify/client/build/Build.java | 30 ++ .../com/apify/client/build/BuildClient.java | 17 +- .../com/apify/client/build/BuildOptions.java | 12 + .../com/apify/client/dataset/Dataset.java | 27 ++ .../apify/client/dataset/DatasetClient.java | 71 ++++- .../dataset/DatasetListItemsOptions.java | 41 +++ .../client/http/BrotliHttpCompressor.java | 73 +++++ .../apify/client/http/GzipHttpCompressor.java | 53 ++++ .../com/apify/client/http/HttpCompressor.java | 28 ++ .../com/apify/client/http/RetryConfig.java | 49 ++- .../java/com/apify/client/http/Timeout.java | 110 +++++++ .../apify/client/internal/HttpClientCore.java | 285 ++++++++++-------- .../java/com/apify/client/internal/Json.java | 25 ++ .../internal/NormalizedUrlDeserializer.java | 22 ++ .../apify/client/internal/QueryParams.java | 10 + .../client/internal/ResourceContext.java | 230 +++++++++----- .../client/internal/RunStartSupport.java | 53 +++- .../apify/client/internal/UrlNormalizer.java | 162 ++++++++++ .../apify/client/keyvalue/KeyValueStore.java | 37 +++ .../client/keyvalue/KeyValueStoreClient.java | 18 +- .../client/keyvalue/KeyValueStoreKey.java | 10 + .../client/keyvalue/SetRecordOptions.java | 26 +- .../java/com/apify/client/log/LogClient.java | 5 +- .../client/requestqueue/RequestQueue.java | 16 + .../requestqueue/RequestQueueClient.java | 129 +++++--- .../java/com/apify/client/run/ActorRun.java | 62 ++++ .../com/apify/client/run/ActorRunMeta.java | 24 ++ .../com/apify/client/run/ActorRunOptions.java | 6 + .../com/apify/client/run/ActorRunStandby.java | 18 ++ .../java/com/apify/client/run/RunClient.java | 87 +++++- .../com/apify/client/run/RunDatasetItems.java | 179 +++++++++++ .../com/apify/client/run/RunListOptions.java | 17 +- .../com/apify/client/schedule/Schedule.java | 15 + .../apify/client/schedule/ScheduleClient.java | 3 +- .../client/store/ActorStoreListItem.java | 7 + .../apify/client/store/StoreListOptions.java | 14 +- src/main/java/com/apify/client/task/Task.java | 10 + .../com/apify/client/task/TaskClient.java | 6 +- .../client/task/TaskCollectionClient.java | 3 +- .../com/apify/client/task/TaskOptions.java | 12 + .../java/com/apify/client/task/TaskStats.java | 7 + .../com/apify/client/user/UserClient.java | 30 +- .../com/apify/client/user/UserProfile.java | 8 + .../com/apify/client/webhook/Webhook.java | 55 ++++ .../apify/client/webhook/WebhookClient.java | 4 +- .../webhook/WebhookDispatchWebhookInfo.java | 4 + .../client/ClientBehaviourRegressionTest.java | 36 +-- .../com/apify/client/ClientInputTest.java | 277 +++++++++++++++++ .../com/apify/client/CompressionTest.java | 133 ++++++-- .../java/com/apify/client/MockTransport.java | 20 +- .../com/apify/client/RunDatasetItemsTest.java | 170 +++++++++++ .../com/apify/client/TimeoutTierTest.java | 269 +++++++++++++++++ .../apify/client/UrlNormalizationTest.java | 114 +++++++ .../integration/ActorRunIntegrationTest.java | 51 ++++ .../integration/DatasetIntegrationTest.java | 20 ++ .../client/integration/IntegrationBase.java | 14 +- .../integration/StoreIntegrationTest.java | 12 + .../integration/UserIntegrationTest.java | 11 + 79 files changed, 3226 insertions(+), 480 deletions(-) create mode 100644 src/main/java/com/apify/client/http/BrotliHttpCompressor.java create mode 100644 src/main/java/com/apify/client/http/GzipHttpCompressor.java create mode 100644 src/main/java/com/apify/client/http/HttpCompressor.java create mode 100644 src/main/java/com/apify/client/http/Timeout.java create mode 100644 src/main/java/com/apify/client/internal/NormalizedUrlDeserializer.java create mode 100644 src/main/java/com/apify/client/internal/UrlNormalizer.java create mode 100644 src/main/java/com/apify/client/run/ActorRunStandby.java create mode 100644 src/main/java/com/apify/client/run/RunDatasetItems.java create mode 100644 src/test/java/com/apify/client/ClientInputTest.java create mode 100644 src/test/java/com/apify/client/RunDatasetItemsTest.java create mode 100644 src/test/java/com/apify/client/TimeoutTierTest.java create mode 100644 src/test/java/com/apify/client/UrlNormalizationTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d6050f7..a12f2ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,58 @@ All notable changes to the Apify Java client are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.12.0] - 2026-10-30 + +### Added + +- Timeout tiers: `Timeout` (`SHORT`, `MEDIUM`, `LONG`, `NO_TIMEOUT`, `of(Duration)`), the builder + setters `timeoutShort`, `timeoutMedium`, `timeoutLong` (defaults 5s, 30s, 360s) and `timeoutMax` + (cap of a single attempt, default 360s), and `ApifyClient.withTimeout(Timeout)` to replace the tier + of every request a client sends. Every method now picks its tier as the reference client does + (metadata reads and writes `SHORT`; listing, batch and trigger calls `MEDIUM`; downloads, uploads + and streaming `LONG`), a request with `waitForFinish` adds the hold to its tier, and the polling of + `waitForFinish`/`call` has no timeout. `SetRecordOptions.timeout(Timeout)`. +- Configurable request compression: `HttpCompressor`, `BrotliHttpCompressor` (quality 0-11) and + `GzipHttpCompressor` (level 1-9) with `ApifyClientBuilder.compression(...)`. +- `ApifyClientBuilder.header(name, value)`: default headers sent with every request. +- `RunClient.iterateDatasetItems(...)`: iterate the items of a run's default dataset while the run + is still producing them. +- `RunListOptions.deployment(...)` and `StoreListOptions.includeInputSchema(...)`. +- `UserClient.limits()`, `updateLimits(...)` and `monthlyUsage(...)` work for any user id. +- Typed fields: `ActorRun.getStandby()` (`ActorRunStandby`), `getConsoleUrl()`, `getOutput()`, + `getEventUsage()`, `getAccountedChargedEventCounts()`, `getExternalChargedTotalUsd()`, + `getIntegrationTracking()`, `getCreatedByOrganizationMemberUserId()`; `ActorRunMeta` origin ids; + `title`, `consoleUrl` and public URLs on `Dataset`, `KeyValueStore`, `KeyValueStoreKey` and + `RequestQueue`; `Webhook` integration fields; `Schedule.getUiState()`; `Task.getStandbyUrl()`; + `TaskStats.getLastRunStartedAt()`; `Build` and `BuildOptions` fields; `ActorVersion` webhook ids + and source URLs; `ActorEnvVar.getValueHash()`; `ActorStandby` tenancy and auth flags; the + `maxTotalChargeUsd` run options. + +### Changed + +- Updated to Apify OpenAPI specification `v2-2026-10-29T100357Z`. +- The `format: uri` fields of the models are returned in the normalized WHATWG form (empty path `/`, + lowercase punycode host, default port dropped, unsafe characters percent-encoded). +- A resource client's id is percent-encoded into a single path segment, and an empty id or a bare + `.`/`..` is rejected with `IllegalArgumentException` instead of addressing the collection. +- `ApifyClientBuilder.timeout(Duration)` is deprecated: it is an alias of `timeoutMax(Duration)`. +- The silent fallback from a failing brotli encode to gzip is gone: the configured compressor is the + only one that runs, and its failure fails the request. Brotli stays the default (quality 6) where + its native library loads, gzip elsewhere. +- `batchAddRequests` serializes each request once instead of twice. +- A `waitForFinish` hold is no longer clamped below the request timeout; the timeout of the request + is extended by the hold instead. +- The integration tests wait up to 90 s for a just-created resource to appear in a listing. + +### Fixed + +- A timed-out Actor or task start, build trigger or `pushItems` request is no longer retried, so it + cannot start a second billed run or store items twice. +- `pushItems` rejects what the API rejects (anything but an object or an array of objects) before a + request is sent, and sends a pre-serialized `String` as it is. +- An Actor or task input given as `byte[]` or `String` is sent as it is instead of being wrapped in + a second layer of JSON. + ## [0.11.0] - 2026-10-26 ### Added diff --git a/README.md b/README.md index b3b3f8a..5ab1b29 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Maven (Maven Central is a default repository, so no extra configuration is neede com.apify apify-client - 0.11.0 + 0.12.0 ``` @@ -35,7 +35,7 @@ repositories { } dependencies { - implementation 'com.apify:apify-client:0.11.0' + implementation 'com.apify:apify-client:0.12.0' } ``` @@ -133,7 +133,12 @@ ApifyClient configured = .baseUrl("https://api.apify.com") // /v2 is appended automatically .maxRetries(8) .minDelayBetweenRetries(Duration.ofMillis(500)) - .timeout(Duration.ofSeconds(360)) + .timeoutShort(Duration.ofSeconds(5)) + .timeoutMedium(Duration.ofSeconds(30)) + .timeoutLong(Duration.ofSeconds(360)) + .timeoutMax(Duration.ofSeconds(360)) + .compression(new GzipHttpCompressor(6)) + .header("X-Integration", "my-platform") .userAgentSuffix("MyTool/1.0") .build(); ``` @@ -152,15 +157,70 @@ validated at call time (see below), not deferred to `build()`. | `publicBaseUrl(String)` | `String` | same as `baseUrl` | non-null, non-blank | | `maxRetries(int)` | `int` | `8` | `>= 0` | | `minDelayBetweenRetries(Duration)` | `Duration` | `500ms` | non-null, `>= 0` | -| `maxDelayBetweenRetries(Duration)` | `Duration` | same as `timeout` (`360s`) | non-null, `>= 0` | -| `timeout(Duration)` | `Duration` | `360s` | non-null, strictly `> 0` | +| `maxDelayBetweenRetries(Duration)` | `Duration` | same as `timeoutMax` (`360s`) | non-null, `>= 0` | +| `timeoutShort(Duration)` | `Duration` | `5s` | non-null, strictly `> 0` | +| `timeoutMedium(Duration)` | `Duration` | `30s` | non-null, strictly `> 0` | +| `timeoutLong(Duration)` | `Duration` | `360s` | non-null, strictly `> 0` | +| `timeoutMax(Duration)` | `Duration` | `360s` | non-null, strictly `> 0` | +| `compression(HttpCompressor)` | `HttpCompressor` | `BrotliHttpCompressor` (quality 6), or `GzipHttpCompressor` where the brotli native library is unavailable | non-null | +| `header(String, String)` | `String`, `String` | none | non-blank name, non-null value | | `userAgentSuffix(String)` | `String` | none (no suffix appended) | any string | | `httpTransport(HttpTransport)` | `HttpTransport` | `new DefaultHttpTransport()` | non-null | -A violated range throws `IllegalArgumentException` from the setter itself. `timeout` is the ceiling -each retry attempt's per-request (socket) timeout grows toward — not a wall-clock bound on the -cumulative time across all retries; the connection-establishment timeout is a separate, -transport-level setting (see `DefaultHttpTransport`'s constructors below). +A violated range throws `IllegalArgumentException` from the setter itself. The deprecated +`timeout(Duration)` is an alias of `timeoutMax(Duration)`. + +### Timeouts + +Every method picks the timeout tier that fits the expected duration of its request: `short` (5s) for +simple metadata reads and writes, `medium` (30s) for listing, batch and trigger operations, `long` +(360s) for downloads, uploads and streaming. The three durations are set with `timeoutShort`, +`timeoutMedium` and `timeoutLong`. A method that asks the API to hold the response with +`waitForFinish` (`getWithWait`, `start`, building an Actor, ...) gets its tier plus that hold (at +most 60s), so the client does not abort a request the API is still holding. The polling of +`waitForFinish(...)`/`call(...)` and the start inside `call(...)` have no timeout. + +`timeoutMax` caps the timeout of a single request attempt. The first attempt gets the timeout of the +method's tier, each retry doubles it, and none exceeds `timeoutMax`; a tier or timeout above the cap +is capped and logged as a warning. It bounds each attempt, not the wall-clock time across all +retries. The connection-establishment timeout is a separate, transport-level setting (see +`DefaultHttpTransport`'s constructors below). + +A call that legitimately needs a different timeout gets it from `ApifyClient.withTimeout(Timeout)`, +which returns a client that sends every request with that timeout instead of the method's tier. A +`Timeout` is a tier (`Timeout.SHORT`, `MEDIUM`, `LONG`), an exact duration (`Timeout.ofSeconds(120)`, +`Timeout.of(Duration)`) or `Timeout.NO_TIMEOUT` (the request is never aborted, so a stalled connection +is not retried either): + +```java +ApifyClient client = ApifyClient.create("my-api-token"); +List items = + client + .withTimeout(Timeout.ofSeconds(900)) + .dataset("my-dataset-id") + .listItems(new DatasetListItemsOptions()) + .join() + .getItems(); +``` + +An override is used as given, so for a `waitForFinish` request it has to cover the hold too. +`KeyValueStoreClient.setRecord(...)` additionally takes the timeout in its `SetRecordOptions`. + +A request that creates a billed resource (an Actor or task run, a build) or pushes dataset items is +not retried when it times out, since the API may already have processed it. + +### Request compression and default headers + +Request bodies of at least 1 KiB with a compressible content type are compressed with the +configured `HttpCompressor`: `BrotliHttpCompressor` (quality 0-11, default 6; used by default where +its native library is available) or `GzipHttpCompressor` (level 1-9, default 9), or your own +implementation of the interface. A failing compressor fails the request; the client does not fall +back to another coding. A body that already carries a `Content-Encoding`, and already-compressed +content types (images, archives, ...), are sent as they are. + +`header(name, value)` adds a header to every request, such as an integration platform marker. It +replaces a built-in header of the same name (`User-Agent`, `Authorization`) and is replaced by a +header a request sets itself. `publicBaseUrl` does not affect ordinary API requests (those always use `baseUrl`); it only changes the origin embedded in the handful of public, directly-fetchable URLs the client builds without a @@ -280,10 +340,11 @@ A few methods validate their own preconditions synchronously, before ever buildi future, and throw a plain JDK exception directly from the call site instead, not an `ApifyClientException` subtype (so no `.join()`/`.get()` unwrapping is involved for these): `ApifyClient.setStatusMessage(message, options)` throws `IllegalStateException` if the -`ACTOR_RUN_ID` environment variable is not set, `ApifyClient.me()`'s limits methods (`limits()`, -`updateLimits(...)`, `monthlyUsage(...)`) throw the same `IllegalStateException` if called on a -`UserClient` that is not `me()`, and `ApifyClientBuilder`'s setters throw `IllegalArgumentException` -on an invalid configuration value (e.g. a negative retry count). These are local, no-request-sent +`ACTOR_RUN_ID` environment variable is not set, `ApifyClientBuilder`'s setters throw +`IllegalArgumentException` on an invalid configuration value (e.g. a negative retry count), a +resource client's id is rejected with `IllegalArgumentException` when it is empty or a bare `.`/`..` +(any other id is percent-encoded into one path segment), and `DatasetClient.pushItems(...)` throws +it for items the API would reject (anything but an object or an array of objects). These are local, no-request-sent failures, not something the API responded with or the transport failed to deliver, which is why they stay outside the request/transport exception hierarchy above — catch `IllegalStateException`/`IllegalArgumentException` separately if you call any of them. @@ -317,7 +378,7 @@ try { The public `com.apify.client.Version` class (`import com.apify.client.Version;`) exposes two constants: -- `Version.CLIENT_VERSION` — the semantic version of this client (`0.11.0`). +- `Version.CLIENT_VERSION` — the semantic version of this client (`0.12.0`). - `Version.API_SPEC_VERSION` — the version of the [Apify OpenAPI specification](https://docs.apify.com/api/openapi.json) (its `info.version` field) that this client's endpoints, parameters and models were last generated and checked against (`v2-2026-10-26T112451Z`). It is a snapshot, not a live compatibility diff --git a/docs/README.md b/docs/README.md index 92fa938..f9a8294 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,8 +16,10 @@ ApifyClient client = ApifyClient.create("my-api-token"); ``` `ApifyClient.create` takes the token as an explicit argument — it does not read `APIFY_TOKEN` -automatically. Use `ApifyClient.builder()` for non-default settings (base URL, retries, timeout, -user-agent suffix, custom HTTP transport). +automatically. Use `ApifyClient.builder()` for non-default settings (base URL, retries, timeout tiers, +compression, default headers, user-agent suffix, custom HTTP transport). `client.withTimeout(...)` +returns a client that sends every request with a different timeout (see +[Timeouts](../README.md#timeouts)). Get your API token from the [Apify Console → Settings → API & Integrations](https://console.apify.com/settings/integrations). @@ -39,7 +41,7 @@ option types, so import each resource you use from its own package: | Package | Contains | |---|---| | `com.apify.client` (root) | `ApifyClient`, `ApifyClientBuilder`, `Version`, `PaginationList`, `ListOptions`, `StorageListOptions`, `ApifyResource` | -| `com.apify.client.http` | `ApifyClientException`, `ApifyApiException`, `ApifyTransportException`, `HttpTransport`, `DefaultHttpTransport`, `HttpTimeoutException` | +| `com.apify.client.http` | `ApifyClientException`, `ApifyApiException`, `ApifyTransportException`, `HttpTransport`, `DefaultHttpTransport`, `HttpTimeoutException`, `Timeout`, `HttpCompressor`, `BrotliHttpCompressor`, `GzipHttpCompressor` | | `com.apify.client.actor` | `Actor`, `ActorClient`, `ActorCollectionClient`, `ActorListOptions`, `ActorStartOptions`, `ActorCallOptions`, `ActorBuildOptions`, `ActorStandby`, `ActorVersion`, `SourceCodeFile`, `ActorVersionClient`, `ActorVersionCollectionClient`, `ActorEnvVar`, `ActorEnvVarClient`, `ActorEnvVarCollectionClient`, `ValidateInputOptions`, `ActorStats`, `ActorDefaultRunOptions` | | `com.apify.client.build` | `Build`, `BuildClient`, `BuildCollectionClient`, `BuildActVersion`, `ActorDefinition`, `ActorDefinitionMeta`, `ActorDefinitionStorages`, `BuildMeta`, `BuildStats`, `BuildOptions`, `BuildUsage` | | `com.apify.client.run` | `ActorRun`, `ActorRunStats`, `ActorRunOptions`, `ActorRunMeta`, `ActorRunUsage`, `RunClient`, `RunCollectionClient`, `RunListOptions`, `LastRunOptions`, `MetamorphOptions`, `RunChargeOptions`, `RunResurrectOptions`, `SetStatusMessageOptions` | @@ -77,7 +79,7 @@ A few methods return data whose shape is not modelled by this client and is inst Jackson `JsonNode` (or accept an arbitrary `Object` serialized to JSON): - Read, returning a required `JsonNode` (never absent; a 404 throws rather than returning empty — - see [error handling](../README.md#error-handling)): `me().monthlyUsage(...)`, `me().limits()`, + see [error handling](../README.md#error-handling)): `me().monthlyUsage(...)`, `me().limits()` (and the same on `user(id)`), `dataset(id).getStatistics()`, `task(id).getInput()`. - Read, returning `Optional` (empty when the underlying resource has none): `build(id).getOpenApiDefinition()`. diff --git a/docs/actors.md b/docs/actors.md index 43c1f3f..4655b21 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -46,7 +46,7 @@ Actor created = client.actors().create(Map.of( | `get()` | Fetch the Actor. Completes with `Optional`. | | `update(Object)` | Update the Actor with the given fields. Completes with `Actor`. | | `delete()` | Delete the Actor. | -| `start(Object input, ActorStartOptions)` | Start a run, returning immediately. Completes with `ActorRun`. | +| `start(Object input, ActorStartOptions)` | Start a run, returning immediately. Completes with `ActorRun`. The `input` is any JSON-serializable value, or a `byte[]`/`String` that is sent as it is (set the matching `ActorStartOptions.contentType(...)`), or `null`. A start that times out is not retried, so no second run is started. | | `call(Object input, ActorStartOptions, Long waitSecs)` | Start a run and poll until it finishes (`null` waits indefinitely); does **not** stream the run's log. Completes with `ActorRun`. | | `call(Object input, ActorCallOptions, Long waitSecs)` | As above, additionally streaming the run's log for the duration of the wait by default (matching the reference client's `call` defaulting `options.log` to `'default'`). Use `ActorCallOptions.disableLogStreaming()` to opt out, or `logOptions(StreamedLogOptions)` for a custom destination. | | `validateInput(Object input)` / `validateInput(Object input, ValidateInputOptions)` | Validate an input against the Actor's input schema. Completes with `boolean`. `ValidateInputOptions` fields (optional): `build` (`String`), `contentType` (`String`). | diff --git a/docs/builds.md b/docs/builds.md index 7255e2e..cdc606e 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -15,7 +15,7 @@ builds) and a single build with `client.build(id)`. | Method | Description | |---|---| | `get()` | Fetch the build. Completes with `Optional`. | -| `getWithWait(Long waitForFinishSecs)` | Fetch, optionally waiting server-side for the build to finish (clamped to the request timeout; the API caps server-side waiting at 60s). Completes with `Optional`. | +| `getWithWait(Long waitForFinishSecs)` | Fetch, optionally waiting server-side for the build to finish (the API caps server-side waiting at 60s; the request timeout is extended by that hold). Completes with `Optional`. | | `abort()` | Abort the build. Completes with `Build`. | | `delete()` | Delete the build. | | `waitForFinish(Long waitSecs)` | Poll until the build finishes (`null` waits indefinitely); see [the run equivalent](runs.md#runclient) for its 404 handling. Completes with `Build`. | diff --git a/docs/misc.md b/docs/misc.md index 63b8773..8db218d 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -39,12 +39,12 @@ still available via the inherited `getExtra()` (see | Method | Description | |---|---| | `get()` | Fetch the user. Completes with `Optional` (private details for `me()` via `getExtra()`). | -| `monthlyUsage()` / `monthlyUsage(String date)` | Account monthly usage (`me()` only). `date` (any day within the target month, formatted `YYYY-MM-DD`) reports that month; `null`/empty reports the current month. Completes with `JsonNode`. | -| `limits()` | Account resource limits (`me()` only). Completes with `JsonNode`. | -| `updateLimits(Object)` | Update account limits (`me()` only). Completes with no value (`CompletableFuture`). | +| `monthlyUsage()` / `monthlyUsage(String date)` | The user's monthly usage. `date` (any day within the target month, formatted `YYYY-MM-DD`) reports that month; `null`/empty reports the current month. Completes with `JsonNode`. | +| `limits()` | The user's account and usage limits. Completes with `JsonNode`. | +| `updateLimits(Object)` | Update the user's account and usage limits. Completes with no value (`CompletableFuture`). | -The usage/limits methods are only available for `me()`; calling them on `user(id)` throws -`IllegalStateException`. +The usage/limits methods work on `me()` and on `user(id)` (`/v2/users/{userId}/usage/monthly` and +`/v2/users/{userId}/limits`), subject to what the API token may access. ```java Optional me = client.me().get().join(); diff --git a/docs/runs.md b/docs/runs.md index c19d478..de5a4f0 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -11,6 +11,9 @@ Access the run collection with `client.runs()` (or `client.actor(id).runs()` / | `iterate(ListOptions, RunListOptions, Long chunkSize)` | Lazy `Flow.Publisher` over all matching runs; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | `RunListOptions` adds `status(List)` (e.g. `SUCCEEDED`, `RUNNING`; sent comma-separated), +`deployment(List)` (filter Actor Standby runs by serving pool: `SINGLE_TENANT`, +`MULTI_TENANT_FREE_USERS`, `MULTI_TENANT_PAYING_USERS`, `MULTI_TENANT_OWNER_PAID`; needs a `status` +filter with only non-terminal statuses, declared by `client.runs()` and `client.actor(id).runs()`), declared by every scope's endpoint (`client.runs()`, `client.actor(id).runs()`, `client.task(id).runs()`), and `startedAfter(String)` / `startedBefore(String)` (ISO-8601), declared only by the account-wide `client.runs()` and the Actor-scoped `client.actor(id).runs()` endpoints — @@ -29,7 +32,7 @@ PaginationList runs = client.runs().list( | Method | Description | |---|---| | `get()` | Fetch the run. Completes with `Optional`. | -| `getWithWait(Long waitForFinishSecs)` | Fetch, optionally waiting server-side for the run to finish (clamped to the request timeout; the API caps server-side waiting at 60s). Completes with `Optional`. | +| `getWithWait(Long waitForFinishSecs)` | Fetch, optionally waiting server-side for the run to finish (the API caps server-side waiting at 60s; the request timeout is extended by that hold). Completes with `Optional`. | | `update(Object)` | Update the run. Completes with `ActorRun`. | | `delete()` | Delete the run. | | `abort(Boolean gracefully)` | Abort the run (`null` = server default). Completes with `ActorRun`. | @@ -39,6 +42,7 @@ PaginationList runs = client.runs().list( | `charge(RunChargeOptions)` | Charge a pay-per-event run for a named event. Completes with no value (`CompletableFuture`). | | `waitForFinish(Long waitSecs)` | Poll until the run finishes (`null` waits indefinitely). Completes with `ActorRun`. | | `dataset()` / `keyValueStore()` / `requestQueue()` | Clients for the run's default storages. | +| `iterateDatasetItems(DatasetListItemsOptions)` / `iterateDatasetItems(DatasetListItemsOptions, Long chunkSize, Long pollIntervalSecs, Class)` | A lazy `Flow.Publisher` over the items of the run's default dataset **while the run is still producing them**; it ends once the run has finished and every item has been read. While the run is active, each poll yields the rows below the dataset's `itemCount` and then waits up to `pollIntervalSecs` (default 5) for the run to finish; afterwards the rows past the lagging `itemCount` are read a page at a time. The options' `offset`/`limit` bound the dataset *rows* scanned (`desc` is ignored), `chunkSize` is the rows per request (default 1000). On a `lastRun()` client it sticks to the run its first request resolves to. | | `log()` | A `LogClient` for the run's log (see [Store, users & logs](misc.md#logs--clientlogid)). | | `getStreamedLog()` | Completes with a `StreamedLog` that redirects the live log to a default per-run logger. | | `getStreamedLog(StreamedLogOptions)` | As above, with a custom destination / options. | @@ -115,6 +119,15 @@ The status is one of `READY`, `RUNNING`, `SUCCEEDED`, `FAILED`, `TIMING-OUT`, `T `ABORTING`, `ABORTED`; `isTerminal()` is true for the finished states (`SUCCEEDED`, `FAILED`, `TIMED-OUT`, `ABORTED`). +`ActorRun` also exposes the Server Actor deployment that served the run, `getStandby()` +(`ActorRunStandby`; `getDeployment()` is `SINGLE_TENANT` or one of the `MULTI_TENANT_*` pools, `null` +for a run not started by a Server Actor request; charges in a multi-tenant pool need +`RunChargeOptions.requestId(...)`), and `getConsoleUrl()`, `getOutput()` (`Map`), +`getEventUsage()` (`Map`), `getAccountedChargedEventCounts()`, +`getExternalChargedTotalUsd()`, `getIntegrationTracking()` and +`getCreatedByOrganizationMemberUserId()`; `getMeta()` adds `getActorRunId()`, +`getWebhookDispatchId()`, `getScheduledAct2Id()` and `getScheduledActorTaskId()`. + `ActorRun` additionally exposes: `getGeneralAccess()` (`String`; who can access the run without owning it, e.g. `"ANYONE_WITH_ID_CAN_READ"`, `"RESTRICTED"`, or `null` to follow the account/Actor default), `getChargedEventCounts()` (`Map`; per-event-type charge counts for diff --git a/docs/storages.md b/docs/storages.md index f4d6802..56c940c 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -13,9 +13,17 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and | Model | Common getters | Type-specific | |---|---|---| -| `Dataset` | `getId`, `getName`, `getUserId`, `getCreatedAt`, `getModifiedAt` | `getItemCount()` (`long`) | -| `KeyValueStore` | `getId`, `getName`, `getUserId`, `getCreatedAt`, `getModifiedAt` | — | -| `RequestQueue` | `getId`, `getName`, `getUserId`, `getCreatedAt`, `getModifiedAt` | `getTotalRequestCount()` (`long`) | +| `Dataset` | `getId`, `getName`, `getTitle`, `getUserId`, `getCreatedAt`, `getModifiedAt`, `getConsoleUrl` | `getItemCount()` (`long`), `getItemsPublicUrl()` | +| `KeyValueStore` | `getId`, `getName`, `getTitle`, `getUserId`, `getCreatedAt`, `getModifiedAt`, `getConsoleUrl` | `getKeysPublicUrl()`, `getRecordsPublicUrl()` | +| `RequestQueue` | `getId`, `getName`, `getTitle`, `getUserId`, `getCreatedAt`, `getModifiedAt`, `getConsoleUrl` | `getTotalRequestCount()` (`long`) | + +The URL fields of every model (`getConsoleUrl()`, `getItemsPublicUrl()`, `getContainerUrl()`, a +webhook's `getRequestUrl()`, ...) are the fields the OpenAPI spec marks `format: uri`. They are +returned normalized, the way a WHATWG URL serializer writes them, so every Apify client returns the +same string: an empty path becomes `/`, the scheme and host are lowercased (a non-ASCII host is +punycoded), a default port is dropped and unsafe characters are percent-encoded. Compare such a field +against a stored URL after normalizing the stored one too. A value that is not an absolute URL is +returned as it is. `Dataset`, `KeyValueStore`, `RequestQueue` and `RequestQueueRequest` all extend `ApifyResource`, so any field not covered by a typed getter above is still available via the inherited `getExtra()` @@ -45,7 +53,7 @@ any field not covered by a typed getter above is still available via the inherit | `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Flow.Publisher`. For typed iteration at the server-default page size, pass a `null` chunk size: `iterateItems(opts, null, T.class)`. | | `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes. `DownloadItemsFormat` is one of `JSON`, `JSONL`, `CSV`, `XLSX`, `XML`, `RSS`, `HTML`. Completes with `byte[]`. | | `streamItems(DownloadItemsFormat, DatasetDownloadOptions)` | As above, but resolves to a live `InputStream` over the export instead of fully buffering it, so a large export is processed in chunks. The caller closes the stream (or reads it to the end). Throws on any error response, including a 404. | -| `pushItems(Object)` | Push a single item or a list of items. Completes with no value (`CompletableFuture`). | +| `pushItems(Object)` | Push a single item (a `Map`, a POJO, a `JsonNode` object, ...), a list/array of items, or a `String` that already is the JSON serialization of either (sent as it is). The API accepts only an object or an array of objects, so anything else (a number, a plain string, an array of strings) throws `IllegalArgumentException` before a request is sent. A push that times out is not retried, so items cannot be stored twice. Completes with no value (`CompletableFuture`). | | `getStatistics()` | Dataset statistics. Completes with `JsonNode`; throws `NotFoundError` if the dataset is gone (no separate "statistics absent" state). | | `createItemsPublicUrl(DatasetListItemsOptions, Long expiresInSecs)` | A public (optionally signed) items URL, served as `json`. Completes with `String`. | | `createItemsPublicUrl(DatasetListItemsOptions, Long expiresInSecs, DownloadItemsFormat format)` | As above, with the URL's serialization `format` set explicitly. | @@ -127,7 +135,7 @@ datasets. | `recordExists(String key)` | Whether a record exists. Completes with `boolean`. | | `getRecord(String key)` / `getRecord(String key, GetRecordOptions)` | Fetch a record. Completes with `Optional`. | | `setRecord(String key, byte[] value, String contentType)` | Store raw bytes. Completes with no value (`CompletableFuture`). | -| `setRecord(String key, byte[] value, String contentType, SetRecordOptions)` | Store raw bytes with write options (`SetRecordOptions`: `timeoutSecs` (`Long`), `doNotRetryTimeouts` (`Boolean`), `contentEncoding` (`String`, see below)). Completes with no value. | +| `setRecord(String key, byte[] value, String contentType, SetRecordOptions)` | Store raw bytes with write options (`SetRecordOptions`: `timeoutSecs` (`Long`) or `timeout` (`Timeout`: a tier, an exact duration or `NO_TIMEOUT`; the default is the `LONG` tier), `doNotRetryTimeouts` (`Boolean`), `contentEncoding` (`String`, see below)). Completes with no value. | | `setRecordJson(String key, Object value)` | Store JSON. Completes with no value. | | `deleteRecord(String key)` | Delete a record. Completes with no value. | | `getRecordPublicUrl(String key)` | A public (optionally signed) record URL. Completes with `String`. | diff --git a/pom.xml b/pom.xml index fbe46db..bbb34ab 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.apify apify-client - 0.11.0 + 0.12.0 jar Apify Java Client diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml index 39df467..4c5a74c 100644 --- a/spotbugs-exclude.xml +++ b/spotbugs-exclude.xml @@ -18,7 +18,7 @@ 2. UWF_UNWRITTEN_FIELD / UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD on Jackson-populated DTOs/response shapes ({@code KeyValueStoreKey}, {@code DeletedRequestInfo}, {@code ActorRunMeta}, {@code - TaskOptions}, {@code WebhookLastDispatch}, {@code ActorDefaultRunOptions}, {@code BuildActVersion}, + TaskOptions}, {@code WebhookLastDispatch}, {@code ActorDefaultRunOptions}, {@code ActorRunStandby}, {@code TaskStats}, {@code BuildActVersion}, {@code ActorDefinitionMeta}, {@code ActorDefinitionStorages}, {@code BuildMeta}, {@code BuildOptions}, {@code BuildStats}, {@code PricingInfo}, {@code ProxyGroup}, {@code UserProxy}, {@code WebhookDispatchEventData}, {@code WebhookDispatchWebhookInfo}, the internal @@ -26,13 +26,12 @@ deserializer via reflection, which SpotBugs cannot see - a known false positive for data-binding models. - 3. EI_EXPOSE_REP / EI_EXPOSE_REP2 on {@code ApiResponse} and {@code HttpClientCore$Compressed}'s - {@code body} component: both are {@code record}s whose {@code byte[] body} intentionally holds - the raw payload without a defensive copy (the payload can be large, e.g. dataset exports, so + 3. EI_EXPOSE_REP / EI_EXPOSE_REP2 on {@code ApiResponse}'s {@code body} component: a {@code + record} whose {@code byte[] body} intentionally holds the raw payload without a defensive copy (the payload can be large, e.g. dataset exports, so copying it on every construction/access would be wasteful). A plain field of the same shape is not flagged (SpotBugs' constructor/getter analysis does not trace a raw public field the way it - traces a record's synthesized accessor and canonical constructor), so this pair of exclusions is - specific to the record form. + traces a record's synthesized accessor and canonical constructor), so this exclusion is specific + to the record form. 4. EI_EXPOSE_REP on the specific getters listed below, each returning a field whose type extends the shared {@code ApifyResource} forward-compatibility base (an {@code extra} catch-all map + @@ -75,6 +74,8 @@ + + @@ -95,10 +96,7 @@ - - - - + @@ -139,6 +137,7 @@ + diff --git a/src/main/java/com/apify/client/ApifyClient.java b/src/main/java/com/apify/client/ApifyClient.java index 7dc81e3..d0260df 100644 --- a/src/main/java/com/apify/client/ApifyClient.java +++ b/src/main/java/com/apify/client/ApifyClient.java @@ -8,6 +8,7 @@ import com.apify.client.dataset.DatasetCollectionClient; import com.apify.client.http.DefaultHttpTransport; import com.apify.client.http.HttpTransport; +import com.apify.client.http.Timeout; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.keyvalue.KeyValueStoreClient; @@ -99,6 +100,24 @@ public static ApifyClientBuilder builder() { return new ApifyClientBuilder(); } + /** + * Returns a client that sends every request with the given {@link Timeout} instead of the tier + * each method would pick, for a call that legitimately needs more (or less) time than its + * default. The returned client shares this client's transport, token and retry policy, and so do + * all the resource clients obtained from it: + * + * 
{@code
+   * client.withTimeout(Timeout.ofSeconds(120)).dataset(id).listItems(options);
+   * }
+ * + * A timeout above the builder's {@code timeoutMax} is capped at it. For a method that asks the + * API to hold the response with {@code waitForFinish}, the given timeout is used as it is, so it + * has to cover the hold too. + */ + public ApifyClient withTimeout(Timeout timeout) { + return new ApifyClient(http.withTimeout(timeout), baseUrl, publicBaseUrl); + } + /** * Returns the {@code User-Agent} header value this client sends. Not part of the public API * (there is no supported use for an end user to read it back); package-private only so the test diff --git a/src/main/java/com/apify/client/ApifyClientBuilder.java b/src/main/java/com/apify/client/ApifyClientBuilder.java index 5f572c9..d020d00 100644 --- a/src/main/java/com/apify/client/ApifyClientBuilder.java +++ b/src/main/java/com/apify/client/ApifyClientBuilder.java @@ -1,13 +1,17 @@ package com.apify.client; import com.apify.client.http.DefaultHttpTransport; +import com.apify.client.http.HttpCompressor; import com.apify.client.http.HttpTransport; import com.apify.client.http.RetryConfig; +import com.apify.client.http.Timeout; import com.apify.client.internal.HttpClientCore; import java.net.URI; import java.net.URISyntaxException; import java.time.Duration; +import java.util.LinkedHashMap; import java.util.Locale; +import java.util.Map; import java.util.function.BooleanSupplier; /** @@ -21,7 +25,7 @@ public final class ApifyClientBuilder { static final int DEFAULT_MAX_RETRIES = 8; static final Duration DEFAULT_MIN_DELAY = Duration.ofMillis(500); - static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(360); + static final Duration DEFAULT_TIMEOUT = RetryConfig.DEFAULT_TIMEOUT_LONG; /** Environment variable that signals the client is running on the Apify platform. */ static final String ENV_IS_AT_HOME = "APIFY_IS_AT_HOME"; @@ -32,9 +36,14 @@ public final class ApifyClientBuilder { private int maxRetries = DEFAULT_MAX_RETRIES; private Duration minDelayBetweenRetries = DEFAULT_MIN_DELAY; private Duration maxDelayBetweenRetries = DEFAULT_TIMEOUT; - private Duration timeout = DEFAULT_TIMEOUT; + private Duration timeoutShort = RetryConfig.DEFAULT_TIMEOUT_SHORT; + private Duration timeoutMedium = RetryConfig.DEFAULT_TIMEOUT_MEDIUM; + private Duration timeoutLong = RetryConfig.DEFAULT_TIMEOUT_LONG; + private Duration timeoutMax = DEFAULT_TIMEOUT; private String userAgentSuffix; private HttpTransport httpTransport; + private HttpCompressor compression; + private final Map defaultHeaders = new LinkedHashMap<>(); private BooleanSupplier isAtHomeFn = ApifyClientBuilder::defaultIsAtHome; ApifyClientBuilder() {} @@ -74,7 +83,10 @@ public ApifyClientBuilder minDelayBetweenRetries(Duration minDelayBetweenRetries return this; } - /** Sets the maximum (exponentially-grown) delay between retries (default equals the timeout). */ + /** + * Sets the maximum (exponentially-grown) delay between retries (default 360s, as {@link + * #timeoutMax(Duration)}). + */ public ApifyClientBuilder maxDelayBetweenRetries(Duration maxDelayBetweenRetries) { this.maxDelayBetweenRetries = requireNonNegative(maxDelayBetweenRetries, "maxDelayBetweenRetries"); @@ -82,13 +94,54 @@ public ApifyClientBuilder maxDelayBetweenRetries(Duration maxDelayBetweenRetries } /** - * Sets the maximum per-attempt request (socket) timeout (default 360s): the ceiling each retry - * attempt's timeout grows toward, not a wall-clock bound on the cumulative time across retries. - * The connection-establishment timeout is separate and configured on the {@link HttpTransport} - * (see {@link DefaultHttpTransport#DefaultHttpTransport(Duration)}). + * Sets the duration of the {@link Timeout#SHORT} tier (default 5s), used by simple metadata reads + * and writes. + */ + public ApifyClientBuilder timeoutShort(Duration timeoutShort) { + this.timeoutShort = requirePositive(timeoutShort, "timeoutShort"); + return this; + } + + /** + * Sets the duration of the {@link Timeout#MEDIUM} tier (default 30s), used by listing, batch and + * trigger operations. + */ + public ApifyClientBuilder timeoutMedium(Duration timeoutMedium) { + this.timeoutMedium = requirePositive(timeoutMedium, "timeoutMedium"); + return this; + } + + /** + * Sets the duration of the {@link Timeout#LONG} tier (default 360s), used by downloads, uploads + * and streaming. */ + public ApifyClientBuilder timeoutLong(Duration timeoutLong) { + this.timeoutLong = requirePositive(timeoutLong, "timeoutLong"); + return this; + } + + /** + * Sets the cap on the timeout of a single request attempt (default 360s). The first attempt gets + * the timeout of its tier (or the one given with {@link ApifyClient#withTimeout(Timeout)}), each + * retry doubles it, and none exceeds this cap; a tier or timeout above it is capped, with a + * logged warning. It bounds each attempt, not the cumulative time across retries. The + * connection-establishment timeout is separate and configured on the {@link HttpTransport} (see + * {@link DefaultHttpTransport#DefaultHttpTransport(Duration)}). + */ + public ApifyClientBuilder timeoutMax(Duration timeoutMax) { + this.timeoutMax = requirePositive(timeoutMax, "timeoutMax"); + return this; + } + + /** + * Sets the cap on the timeout of a single request attempt. + * + * @deprecated the same as {@link #timeoutMax(Duration)}, which names what it does now that the + * timeout of a request comes from a tier; use that instead. + */ + @Deprecated public ApifyClientBuilder timeout(Duration timeout) { - this.timeout = requirePositive(timeout, "timeout"); + this.timeoutMax = requirePositive(timeout, "timeout"); return this; } @@ -107,6 +160,36 @@ public ApifyClientBuilder httpTransport(HttpTransport httpTransport) { return this; } + /** + * Sets how request bodies of at least 1 KiB are compressed (default {@link + * com.apify.client.http.BrotliHttpCompressor} where its native library is available, {@link + * com.apify.client.http.GzipHttpCompressor} elsewhere). Pass a configured {@code + * BrotliHttpCompressor}/{@code GzipHttpCompressor} to tune the quality, or your own {@link + * HttpCompressor}. + */ + public ApifyClientBuilder compression(HttpCompressor compression) { + if (compression == null) { + throw new IllegalArgumentException("compression must not be null"); + } + this.compression = compression; + return this; + } + + /** + * Adds a header sent with every request, such as an integration platform marker. It replaces a + * built-in header of the same name ({@code User-Agent}, {@code Authorization}) and is replaced by + * a header a request sets itself. Header names are compared case-insensitively. + */ + public ApifyClientBuilder header(String name, String value) { + requireNonBlank(name, "header name"); + if (value == null) { + throw new IllegalArgumentException("header value must not be null"); + } + defaultHeaders.keySet().removeIf(existing -> existing.equalsIgnoreCase(name)); + defaultHeaders.put(name, value); + return this; + } + /** Validates a required, non-blank string argument, returning it unchanged. */ private static String requireNonBlank(String value, String argName) { if (value == null || value.isBlank()) { @@ -160,8 +243,16 @@ public ApifyClient build() { HttpTransport transport = httpTransport != null ? httpTransport : new DefaultHttpTransport(); String userAgent = buildUserAgent(userAgentSuffix, isAtHomeFn); RetryConfig retry = - new RetryConfig(maxRetries, minDelayBetweenRetries, maxDelayBetweenRetries, timeout); - HttpClientCore http = new HttpClientCore(transport, token, userAgent, retry); + new RetryConfig( + maxRetries, + minDelayBetweenRetries, + maxDelayBetweenRetries, + timeoutShort, + timeoutMedium, + timeoutLong, + timeoutMax); + HttpClientCore http = + new HttpClientCore(transport, token, userAgent, retry, compression, defaultHeaders); String apiBase = normalizeApiUrl(baseUrl); String publicBase = normalizeApiUrl(publicBaseUrl != null ? publicBaseUrl : baseUrl); diff --git a/src/main/java/com/apify/client/Version.java b/src/main/java/com/apify/client/Version.java index 638cb83..2e3d187 100644 --- a/src/main/java/com/apify/client/Version.java +++ b/src/main/java/com/apify/client/Version.java @@ -13,13 +13,13 @@ public final class Version { * The semantic version of this client library (see SemVer). * Changes to the public interface other than additive ones are considered breaking changes. */ - public static final String CLIENT_VERSION = "0.11.0"; + public static final String CLIENT_VERSION = "0.12.0"; /** * The version of the Apify OpenAPI specification this client was generated and verified against. * Corresponds to the {@code info.version} field of the Apify OpenAPI document. */ - public static final String API_SPEC_VERSION = "v2-2026-10-26T112451Z"; + public static final String API_SPEC_VERSION = "v2-2026-10-29T100357Z"; private Version() {} } diff --git a/src/main/java/com/apify/client/actor/ActorClient.java b/src/main/java/com/apify/client/actor/ActorClient.java index ed427ff..dfd327e 100644 --- a/src/main/java/com/apify/client/actor/ActorClient.java +++ b/src/main/java/com/apify/client/actor/ActorClient.java @@ -4,6 +4,7 @@ import com.apify.client.build.Build; import com.apify.client.build.BuildClient; import com.apify.client.build.BuildCollectionClient; +import com.apify.client.http.Timeout; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; @@ -62,7 +63,8 @@ public CompletableFuture delete() { /** * Starts the Actor and completes with the created run as soon as it exists (no waiting). {@code - * input} is any JSON-serializable value (or {@code null} for no input). + * input} is any JSON-serializable value, a {@code byte[]} or {@link String} that is sent as it is + * (set the content type with the options' {@code contentType}), or {@code null} for no input. */ public CompletableFuture start(Object input, ActorStartOptions options) { return RunStartSupport.start( @@ -126,18 +128,24 @@ public CompletableFuture validateInput(Object input) { /** * Validates {@code input} against the Actor's input schema and completes with whether it is - * valid. {@code input} is any JSON-serializable value (or {@code null}). {@code options} may pin - * the build whose schema is used and the content type of the input body. + * valid. {@code input} is any JSON-serializable value, a {@code byte[]} or {@link String} that is + * sent as it is, or {@code null}. {@code options} may pin the build whose schema is used and the + * content type of the input body. */ public CompletableFuture validateInput(Object input, ValidateInputOptions options) { ValidateInputOptions opts = options == null ? new ValidateInputOptions() : options; QueryParams params = new QueryParams(); opts.apply(params); - byte[] body = input == null ? null : Json.toBytes(input); + byte[] body = Json.toRequestBody(input); // The validate-input endpoint returns a bare {"valid": } object, not the standard // {"data": ...} envelope, so parse it without unwrapping. return ctx.postWithBodyNoEnvelope( - "validate-input", params, body, opts.contentTypeOrDefault(), JsonNode.class) + "validate-input", + params, + body, + opts.contentTypeOrDefault(), + JsonNode.class, + Timeout.SHORT) .thenApply( result -> { JsonNode valid = result == null ? null : result.get("valid"); @@ -153,7 +161,15 @@ public CompletableFuture build(String versionNumber, ActorBuildOptions op QueryParams params = new QueryParams(); params.addString("version", versionNumber); options.apply(params); - return ctx.postWithBody("builds", params, null, ResourceContext.CONTENT_TYPE_JSON, Build.class); + return ctx.postWithBody( + "builds", + params, + null, + ResourceContext.CONTENT_TYPE_JSON, + Build.class, + ctx.http.timeoutWithHold(Timeout.MEDIUM, params.getLong("waitForFinish")), + // A build that timed out may still have been created; retrying would start a second one. + true); } /** @@ -163,10 +179,12 @@ public CompletableFuture build(String versionNumber, ActorBuildOptions op */ public CompletableFuture defaultBuild(Long waitForFinish) { QueryParams params = new QueryParams(); - // Clamp like the getWithWait twins so a large wait paired with a short client timeout cannot - // abort every attempt and burn the retry budget (the API caps server-side waiting at 60s). - params.addLong("waitForFinish", ctx.clampServerWait(waitForFinish)); - return ctx.getResourceRequired("builds/default", params, Build.class) + params.addLong("waitForFinish", waitForFinish); + return ctx.getResourceRequired( + "builds/default", + params, + Build.class, + ctx.http.timeoutWithHold(Timeout.SHORT, waitForFinish)) .thenApply(build -> new BuildClient(http, baseUrl, build.getId())); } diff --git a/src/main/java/com/apify/client/actor/ActorCollectionClient.java b/src/main/java/com/apify/client/actor/ActorCollectionClient.java index 9a829c2..75dc2ce 100644 --- a/src/main/java/com/apify/client/actor/ActorCollectionClient.java +++ b/src/main/java/com/apify/client/actor/ActorCollectionClient.java @@ -1,5 +1,6 @@ package com.apify.client.actor; +import com.apify.client.http.Timeout; import com.apify.client.internal.AbstractCollectionClient; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; @@ -19,6 +20,6 @@ public ActorCollectionClient(HttpClientCore http, String baseUrl) { /** Creates a new Actor. {@code actor} is any JSON-serializable Actor definition. */ public CompletableFuture create(Object actor) { - return ctx.createResource(new QueryParams(), actor, Actor.class); + return ctx.createResource(new QueryParams(), actor, Actor.class, Timeout.MEDIUM); } } diff --git a/src/main/java/com/apify/client/actor/ActorDefaultRunOptions.java b/src/main/java/com/apify/client/actor/ActorDefaultRunOptions.java index 4ba7cfd..8047863 100644 --- a/src/main/java/com/apify/client/actor/ActorDefaultRunOptions.java +++ b/src/main/java/com/apify/client/actor/ActorDefaultRunOptions.java @@ -8,6 +8,8 @@ public final class ActorDefaultRunOptions extends ApifyResource { private Long timeoutSecs; private Long memoryMbytes; private Boolean restartOnError; + private Double maxTotalChargeUsd; + private Boolean isMaxTotalChargeUsdSetByUser; /** The tag or number of the build run by default. */ public String getBuild() { @@ -28,4 +30,14 @@ public Long getMemoryMbytes() { public Boolean getRestartOnError() { return restartOnError; } + + /** The default maximum total charge in USD for a pay-per-event run. */ + public Double getMaxTotalChargeUsd() { + return maxTotalChargeUsd; + } + + /** Whether {@link #getMaxTotalChargeUsd()} was set by the user rather than defaulted. */ + public Boolean getIsMaxTotalChargeUsdSetByUser() { + return isMaxTotalChargeUsdSetByUser; + } } diff --git a/src/main/java/com/apify/client/actor/ActorEnvVar.java b/src/main/java/com/apify/client/actor/ActorEnvVar.java index d118105..db758bb 100644 --- a/src/main/java/com/apify/client/actor/ActorEnvVar.java +++ b/src/main/java/com/apify/client/actor/ActorEnvVar.java @@ -7,6 +7,7 @@ public final class ActorEnvVar extends ApifyResource { private String name; private String value; private Boolean isSecret; + private String valueHash; public ActorEnvVar() {} @@ -47,4 +48,9 @@ public ActorEnvVar setIsSecret(Boolean isSecret) { this.isSecret = isSecret; return this; } + + /** A hash of the variable value, returned for secret variables whose value is hidden. */ + public String getValueHash() { + return valueHash; + } } diff --git a/src/main/java/com/apify/client/actor/ActorEnvVarCollectionClient.java b/src/main/java/com/apify/client/actor/ActorEnvVarCollectionClient.java index 009bda3..cfe26d2 100644 --- a/src/main/java/com/apify/client/actor/ActorEnvVarCollectionClient.java +++ b/src/main/java/com/apify/client/actor/ActorEnvVarCollectionClient.java @@ -1,6 +1,7 @@ package com.apify.client.actor; import com.apify.client.PaginationList; +import com.apify.client.http.Timeout; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.ListPublisher; import com.apify.client.internal.QueryParams; @@ -21,7 +22,7 @@ public final class ActorEnvVarCollectionClient { /** Lists the version's environment variables. */ public CompletableFuture> list() { - return ctx.listResource("", new QueryParams(), ActorEnvVar.class); + return ctx.listResource("", new QueryParams(), ActorEnvVar.class, Timeout.SHORT); } /** diff --git a/src/main/java/com/apify/client/actor/ActorStandby.java b/src/main/java/com/apify/client/actor/ActorStandby.java index 688b733..d620b51 100644 --- a/src/main/java/com/apify/client/actor/ActorStandby.java +++ b/src/main/java/com/apify/client/actor/ActorStandby.java @@ -12,6 +12,9 @@ public final class ActorStandby extends ApifyResource { private Long maxRequestsPerActorRun; private Long memoryMbytes; private Boolean shouldPassActorInput; + private String tenancy; + private Boolean isConsoleAuthEnabled; + private Boolean isTokenlessEnabled; /** * Whether standby mode is enabled. Only present on {@link Actor#getActorStandby()} (the API @@ -57,4 +60,22 @@ public Long getMemoryMbytes() { public Boolean getShouldPassActorInput() { return shouldPassActorInput; } + + /** + * Who the Server Actor serves: {@code SINGLE_TENANT} (a private instance per caller) or a {@code + * MULTI_TENANT_*} pool. + */ + public String getTenancy() { + return tenancy; + } + + /** Whether Console authentication is enabled for the standby URL. */ + public Boolean getIsConsoleAuthEnabled() { + return isConsoleAuthEnabled; + } + + /** Whether requests to the standby URL may omit the API token. */ + public Boolean getIsTokenlessEnabled() { + return isTokenlessEnabled; + } } diff --git a/src/main/java/com/apify/client/actor/ActorVersion.java b/src/main/java/com/apify/client/actor/ActorVersion.java index e114b4e..d3560c8 100644 --- a/src/main/java/com/apify/client/actor/ActorVersion.java +++ b/src/main/java/com/apify/client/actor/ActorVersion.java @@ -6,6 +6,11 @@ public final class ActorVersion extends ApifyResource { private String versionNumber; private String sourceType; + private String tarballUrl; + private String gitHubGistUrl; + private Integer githubWebhookId; + private Integer gitlabWebhookId; + private String bitbucketWebhookId; /** The version identifier (e.g. {@code "0.1"}). */ public String getVersionNumber() { @@ -16,4 +21,29 @@ public String getVersionNumber() { public String getSourceType() { return sourceType; } + + /** The tarball URL the source is fetched from, for {@code TARBALL} versions. */ + public String getTarballUrl() { + return tarballUrl; + } + + /** The GitHub Gist URL the source is fetched from, for {@code GITHUB_GIST} versions. */ + public String getGitHubGistUrl() { + return gitHubGistUrl; + } + + /** The ID of the GitHub webhook that triggers builds on push, if configured. */ + public Integer getGithubWebhookId() { + return githubWebhookId; + } + + /** The ID of the GitLab webhook that triggers builds on push, if configured. */ + public Integer getGitlabWebhookId() { + return gitlabWebhookId; + } + + /** The ID of the Bitbucket webhook that triggers builds on push, if configured. */ + public String getBitbucketWebhookId() { + return bitbucketWebhookId; + } } diff --git a/src/main/java/com/apify/client/actor/ActorVersionCollectionClient.java b/src/main/java/com/apify/client/actor/ActorVersionCollectionClient.java index 4b18c98..af0c20a 100644 --- a/src/main/java/com/apify/client/actor/ActorVersionCollectionClient.java +++ b/src/main/java/com/apify/client/actor/ActorVersionCollectionClient.java @@ -2,6 +2,7 @@ import com.apify.client.ListOptions; import com.apify.client.PaginationList; +import com.apify.client.http.Timeout; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.ListPublisher; import com.apify.client.internal.QueryParams; @@ -27,7 +28,7 @@ public final class ActorVersionCollectionClient { * case; use {@link #iterate(ListOptions)}'s {@code limit} to cap the number returned client-side. */ public CompletableFuture> list(ListOptions options) { - return ctx.listResource("", new QueryParams(), ActorVersion.class); + return ctx.listResource("", new QueryParams(), ActorVersion.class, Timeout.SHORT); } /** diff --git a/src/main/java/com/apify/client/build/Build.java b/src/main/java/com/apify/client/build/Build.java index 38b159f..0b9fd7f 100644 --- a/src/main/java/com/apify/client/build/Build.java +++ b/src/main/java/com/apify/client/build/Build.java @@ -22,6 +22,11 @@ public final class Build extends ApifyResource { private BuildUsage usage; private BuildUsage usageUsd; private Double usageTotalUsd; + private String changelog; + private Integer exitCode; + private String gitCommitId; + private String gitBranchName; + private String createdByOrganizationMemberUserId; /** The unique build ID. */ public String getId() { @@ -121,4 +126,29 @@ public Double getUsageTotalUsd() { public boolean isTerminal() { return Statuses.isTerminal(status); } + + /** The changelog of the Actor version built. */ + public String getChangelog() { + return changelog; + } + + /** The exit code of the build process. */ + public Integer getExitCode() { + return exitCode; + } + + /** The Git commit the build was created from, for Git-sourced versions. */ + public String getGitCommitId() { + return gitCommitId; + } + + /** The Git branch the build was created from, for Git-sourced versions. */ + public String getGitBranchName() { + return gitBranchName; + } + + /** The organization member who started the build, when started on behalf of an organization. */ + public String getCreatedByOrganizationMemberUserId() { + return createdByOrganizationMemberUserId; + } } diff --git a/src/main/java/com/apify/client/build/BuildClient.java b/src/main/java/com/apify/client/build/BuildClient.java index 5a22197..844f7eb 100644 --- a/src/main/java/com/apify/client/build/BuildClient.java +++ b/src/main/java/com/apify/client/build/BuildClient.java @@ -1,5 +1,6 @@ package com.apify.client.build; +import com.apify.client.http.Timeout; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; @@ -27,20 +28,20 @@ public CompletableFuture> get() { /** * Fetches the build, optionally asking the API to wait up to {@code waitForFinishSecs} seconds - * for the build to finish before responding. The value is clamped so the server always responds - * before the client's per-request timeout; the API itself caps server-side waiting at 60 seconds. - * Pass {@code null} for an immediate fetch. + * for the build to finish before responding; the API itself caps server-side waiting at 60 + * seconds, and the request timeout is extended by that hold. Pass {@code null} for an immediate + * fetch. */ public CompletableFuture> getWithWait(Long waitForFinishSecs) { QueryParams params = new QueryParams(); - // Clamp to the client's per-request timeout so a short custom timeout doesn't abort the call. - params.addLong("waitForFinish", ctx.clampServerWait(waitForFinishSecs)); - return ctx.getResource("", params, Build.class); + params.addLong("waitForFinish", waitForFinishSecs); + return ctx.getResource( + "", params, Build.class, ctx.http.timeoutWithHold(Timeout.SHORT, waitForFinishSecs)); } /** Aborts the build and returns its updated state. */ public CompletableFuture abort() { - return ctx.postWithBody("abort", new QueryParams(), null, "", Build.class); + return ctx.postWithBody("abort", new QueryParams(), null, "", Build.class, Timeout.SHORT); } /** Deletes the build. */ @@ -61,7 +62,7 @@ public CompletableFuture waitForFinish(Long waitSecs) { * result is the raw OpenAPI document. */ public CompletableFuture> getOpenApiDefinition() { - return ctx.getRaw("openapi.json", new QueryParams()) + return ctx.getRaw("openapi.json", new QueryParams(), Timeout.MEDIUM) .thenApply( resp -> resp == null diff --git a/src/main/java/com/apify/client/build/BuildOptions.java b/src/main/java/com/apify/client/build/BuildOptions.java index 8ebd87b..d4b8544 100644 --- a/src/main/java/com/apify/client/build/BuildOptions.java +++ b/src/main/java/com/apify/client/build/BuildOptions.java @@ -8,6 +8,8 @@ public final class BuildOptions extends ApifyResource { private Boolean betaPackages; private Long memoryMbytes; private Long diskMbytes; + private String tag; + private Boolean debug; /** Whether the build reused a cached image layer. */ public Boolean getUseCache() { @@ -28,4 +30,14 @@ public Long getMemoryMbytes() { public Long getDiskMbytes() { return diskMbytes; } + + /** The build tag requested for the build. */ + public String getTag() { + return tag; + } + + /** Whether the build was run in debug mode. */ + public Boolean getDebug() { + return debug; + } } diff --git a/src/main/java/com/apify/client/dataset/Dataset.java b/src/main/java/com/apify/client/dataset/Dataset.java index 06c941d..8dc385c 100644 --- a/src/main/java/com/apify/client/dataset/Dataset.java +++ b/src/main/java/com/apify/client/dataset/Dataset.java @@ -1,7 +1,9 @@ package com.apify.client.dataset; import com.apify.client.ApifyResource; +import com.apify.client.internal.NormalizedUrlDeserializer; import java.time.Instant; +import tools.jackson.databind.annotation.JsonDeserialize; /** A dataset stores structured results from Actor runs. */ public final class Dataset extends ApifyResource { @@ -11,6 +13,13 @@ public final class Dataset extends ApifyResource { private Instant createdAt; private Instant modifiedAt; private long itemCount; + private String title; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) + private String consoleUrl; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) + private String itemsPublicUrl; /** The unique dataset ID. */ public String getId() { @@ -41,4 +50,22 @@ public Instant getModifiedAt() { public long getItemCount() { return itemCount; } + + /** The human-readable title of the dataset. */ + public String getTitle() { + return title; + } + + /** The URL of the dataset in the Apify Console (normalized). */ + public String getConsoleUrl() { + return consoleUrl; + } + + /** + * A public URL of the dataset items (signed when the dataset is not publicly accessible; + * normalized). + */ + public String getItemsPublicUrl() { + return itemsPublicUrl; + } } diff --git a/src/main/java/com/apify/client/dataset/DatasetClient.java b/src/main/java/com/apify/client/dataset/DatasetClient.java index 3e35d6c..9146602 100644 --- a/src/main/java/com/apify/client/dataset/DatasetClient.java +++ b/src/main/java/com/apify/client/dataset/DatasetClient.java @@ -2,6 +2,7 @@ import com.apify.client.PaginationList; import com.apify.client.http.ApiResponse; +import com.apify.client.http.Timeout; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.AsyncPaginatedPublisher; import com.apify.client.internal.Extras; @@ -11,6 +12,7 @@ import com.apify.client.internal.ResourceContext; import com.apify.client.internal.Signatures; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -175,7 +177,7 @@ public Flow.Publisher iterateItems( private CompletableFuture> fetchItemsPage( QueryParams params, Boolean desc, Class itemClass) { String url = ctx.mergedParams(params).applyToUrl(ctx.subUrl("items")); - return http.call("GET", url, null, "", http.baseRequestTimeout()) + return http.call("GET", url, null, "", Timeout.LONG) .thenApply( resp -> { JavaType listType = Json.parametric(List.class, Json.type(itemClass)); @@ -204,7 +206,7 @@ private CompletableFuture> fetchItemsPage( public CompletableFuture downloadItems( DownloadItemsFormat format, DatasetDownloadOptions options) { String url = itemsUrl(format, options); - return http.call("GET", url, null, "", http.baseRequestTimeout()).thenApply(ApiResponse::body); + return http.call("GET", url, null, "", Timeout.LONG).thenApply(ApiResponse::body); } /** @@ -232,22 +234,71 @@ private String itemsUrl(DownloadItemsFormat format, DatasetDownloadOptions optio } /** - * Pushes one or more items to the dataset. {@code items} must serialize to a JSON object or an - * array of objects. + * Pushes one or more items to the dataset. The API accepts only a JSON object or an array of + * objects, so {@code items} must be one of: + * + *
    + *
  • a single item that serializes to a JSON object (a {@link java.util.Map}, a POJO, a {@link + * JsonNode} object, ...); + *
  • an array or {@link java.util.Collection} of such items; + *
  • a {@link String} that already is the JSON serialization of either; it is sent as it is. + *
+ * + * Anything else (a plain string, a number, an array of strings, ...) is rejected with an {@link + * IllegalArgumentException} before a request is sent, since the API would reject it anyway. A + * push that times out is not retried, so the items cannot be stored twice. */ public CompletableFuture pushItems(Object items) { + byte[] body = items instanceof String json ? serializedItems(json) : serializedItems(items); // Route through mergedParams like every sibling method, so a context seeded with pinned filters // (e.g. actor(id).lastRun(...).dataset()) targets the same run's dataset on write as on read. String url = ctx.mergedParams(new QueryParams()).applyToUrl(ctx.subUrl("items")); return http.call( - "POST", - url, - Json.toBytes(items), - ResourceContext.CONTENT_TYPE_JSON_CHARSET, - http.baseRequestTimeout()) + "POST", url, body, ResourceContext.CONTENT_TYPE_JSON_CHARSET, Timeout.MEDIUM, true) .thenApply(resp -> null); } + /** Validates a pre-serialized {@code pushItems} payload and returns its bytes unchanged. */ + private static byte[] serializedItems(String json) { + byte[] bytes = json.getBytes(StandardCharsets.UTF_8); + JsonNode tree; + try { + tree = Json.parse(bytes, JsonNode.class); + } catch (RuntimeException e) { + throw new IllegalArgumentException( + "pushItems: a String must be the JSON serialization of an item or an array of items", e); + } + requireItemShape(tree); + return bytes; + } + + /** Serializes a {@code pushItems} payload after checking that it has an accepted shape. */ + private static byte[] serializedItems(Object items) { + JsonNode tree = items == null ? null : Json.toTree(items); + requireItemShape(tree); + return Json.toBytes(tree); + } + + private static void requireItemShape(JsonNode tree) { + boolean valid = tree != null && (tree.isObject() || isArrayOfObjects(tree)); + if (!valid) { + throw new IllegalArgumentException( + "pushItems: items must be a JSON object or an array of JSON objects"); + } + } + + private static boolean isArrayOfObjects(JsonNode tree) { + if (!tree.isArray()) { + return false; + } + for (JsonNode element : tree) { + if (!element.isObject()) { + return false; + } + } + return true; + } + /** * Returns statistical information about the dataset. * @@ -256,7 +307,7 @@ public CompletableFuture pushItems(Object items) { * dataset is gone", so a missing dataset is reported as a failure here too. */ public CompletableFuture getStatistics() { - return ctx.getResourceRequired("statistics", new QueryParams(), JsonNode.class); + return ctx.getResourceRequired("statistics", new QueryParams(), JsonNode.class, Timeout.SHORT); } /** diff --git a/src/main/java/com/apify/client/dataset/DatasetListItemsOptions.java b/src/main/java/com/apify/client/dataset/DatasetListItemsOptions.java index 6960086..f837e04 100644 --- a/src/main/java/com/apify/client/dataset/DatasetListItemsOptions.java +++ b/src/main/java/com/apify/client/dataset/DatasetListItemsOptions.java @@ -121,6 +121,47 @@ public DatasetListItemsOptions signature(String signature) { return this; } + /** Returns an independent copy of these options. */ + public DatasetListItemsOptions copy() { + DatasetListItemsOptions c = new DatasetListItemsOptions(); + c.offset = offset; + c.limit = limit; + c.desc = desc; + c.fields = fields; + c.outputFields = outputFields; + c.omit = omit; + c.skipEmpty = skipEmpty; + c.skipHidden = skipHidden; + c.clean = clean; + c.unwind = unwind; + c.flatten = flatten; + c.view = view; + c.simplified = simplified; + c.skipFailedPages = skipFailedPages; + c.signature = signature; + return c; + } + + /** + * Whether {@code clean}, {@code skipEmpty} or {@code unwind} is set: the server-side filters that + * can leave a page of existing rows without a single returned item. + */ + public boolean dropsRows() { + return Boolean.TRUE.equals(clean) + || Boolean.TRUE.equals(skipEmpty) + || (unwind != null && !unwind.isEmpty()); + } + + /** The first row to read, or 0 when not set. */ + public long offsetValueOrZero() { + return offset != null && offset > 0 ? offset : 0; + } + + /** The maximum number of rows to read, or {@code null} when not set. */ + public Long limitValueOrNull() { + return limit; + } + Boolean descValue() { return desc; } diff --git a/src/main/java/com/apify/client/http/BrotliHttpCompressor.java b/src/main/java/com/apify/client/http/BrotliHttpCompressor.java new file mode 100644 index 0000000..95942f5 --- /dev/null +++ b/src/main/java/com/apify/client/http/BrotliHttpCompressor.java @@ -0,0 +1,73 @@ +package com.apify.client.http; + +import com.aayushatharva.brotli4j.Brotli4jLoader; +import com.aayushatharva.brotli4j.encoder.Encoder; +import java.io.IOException; + +/** + * An {@link HttpCompressor} that brotli-encodes request bodies ({@code Content-Encoding: br}). + * + *

It needs a platform-specific native library bundled with {@code brotli4j}; {@link + * #isAvailable()} tells whether one loaded. The client uses this compressor by default where it is + * available and gzip elsewhere. + */ +public final class BrotliHttpCompressor implements HttpCompressor { + + private static final int MAX_QUALITY = 11; + + private static final int DEFAULT_QUALITY = 6; + + private final int quality; + + /** + * Whether the brotli native library loaded for the running platform. Brotli needs a native codec + * that is not bundled for every OS and CPU; the check never throws. + */ + public static boolean isAvailable() { + try { + Brotli4jLoader.ensureAvailability(); + return true; + } catch (Throwable t) { + // Catch Throwable: native loading raises UnsatisfiedLinkError/NoClassDefFoundError, not only + // Exception, and "unavailable" is the answer in every such case. + return false; + } + } + + /** + * A compressor with the default quality, 6. + * + * @throws IllegalStateException if the brotli native library is not available on this platform + */ + public BrotliHttpCompressor() { + this(DEFAULT_QUALITY); + } + + /** + * A compressor with the given quality. + * + * @param quality the compression quality from 0 (fastest) to 11 (smallest output) + * @throws IllegalArgumentException if {@code quality} is outside 0-11 + * @throws IllegalStateException if the brotli native library is not available on this platform + */ + public BrotliHttpCompressor(int quality) { + if (quality < 0 || quality > MAX_QUALITY) { + throw new IllegalArgumentException("brotli quality must be between 0 and 11, was " + quality); + } + if (!isAvailable()) { + throw new IllegalStateException( + "the brotli native library is not available on this platform; use GzipHttpCompressor"); + } + this.quality = quality; + } + + @Override + public String contentEncoding() { + return "br"; + } + + @Override + public byte[] compress(byte[] data) throws IOException { + return Encoder.compress(data, new Encoder.Parameters().setQuality(quality)); + } +} diff --git a/src/main/java/com/apify/client/http/GzipHttpCompressor.java b/src/main/java/com/apify/client/http/GzipHttpCompressor.java new file mode 100644 index 0000000..8c9d1c6 --- /dev/null +++ b/src/main/java/com/apify/client/http/GzipHttpCompressor.java @@ -0,0 +1,53 @@ +package com.apify.client.http; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPOutputStream; + +/** An {@link HttpCompressor} that gzip-encodes request bodies ({@code Content-Encoding: gzip}). */ +public final class GzipHttpCompressor implements HttpCompressor { + + /** The highest compression level, and the default. */ + private static final int MAX_LEVEL = 9; + + private static final int MIN_LEVEL = 1; + + private final int level; + + /** A compressor with the default level, 9. */ + public GzipHttpCompressor() { + this(MAX_LEVEL); + } + + /** + * A compressor with the given level. + * + * @param level the compression level from 1 (fastest) to 9 (smallest output) + * @throws IllegalArgumentException if {@code level} is outside 1-9 + */ + public GzipHttpCompressor(int level) { + if (level < MIN_LEVEL || level > MAX_LEVEL) { + throw new IllegalArgumentException("gzip level must be between 1 and 9, was " + level); + } + this.level = level; + } + + @Override + public String contentEncoding() { + return "gzip"; + } + + @Override + public byte[] compress(byte[] data) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(data.length); + try (GZIPOutputStream gzip = + new GZIPOutputStream(out) { + { + def.setLevel(level); + } + }) { + gzip.write(data); + } + return out.toByteArray(); + } +} diff --git a/src/main/java/com/apify/client/http/HttpCompressor.java b/src/main/java/com/apify/client/http/HttpCompressor.java new file mode 100644 index 0000000..6b4a3fe --- /dev/null +++ b/src/main/java/com/apify/client/http/HttpCompressor.java @@ -0,0 +1,28 @@ +package com.apify.client.http; + +import java.io.IOException; + +/** + * Compresses request bodies before they are sent. The client compresses a body only when it is at + * least 1 KiB, has a compressible content type and carries no {@code Content-Encoding} header of + * its own; the configured compressor then encodes it and its {@link #contentEncoding()} is sent as + * the {@code Content-Encoding} header. Streams are never compressed. + * + *

Use {@link BrotliHttpCompressor} (the default, where its native library is available) or + * {@link GzipHttpCompressor}, or implement this interface for another coding. Configure it with + * {@link com.apify.client.ApifyClientBuilder#compression(HttpCompressor)}. Implementations must be + * safe for concurrent use. + */ +public interface HttpCompressor { + + /** The {@code Content-Encoding} token that describes the output of {@link #compress}. */ + String contentEncoding(); + + /** + * Compresses {@code data}. A failure fails the request: the client does not fall back to another + * coding. + * + * @throws IOException if the data cannot be compressed + */ + byte[] compress(byte[] data) throws IOException; +} diff --git a/src/main/java/com/apify/client/http/RetryConfig.java b/src/main/java/com/apify/client/http/RetryConfig.java index 4472593..01a8ce6 100644 --- a/src/main/java/com/apify/client/http/RetryConfig.java +++ b/src/main/java/com/apify/client/http/RetryConfig.java @@ -4,6 +4,16 @@ /** Retry/timeout policy for the orchestrating HTTP client. Internal to the client. */ public final class RetryConfig { + + /** Default duration of the {@link Timeout#SHORT} tier. */ + public static final Duration DEFAULT_TIMEOUT_SHORT = Duration.ofSeconds(5); + + /** Default duration of the {@link Timeout#MEDIUM} tier. */ + public static final Duration DEFAULT_TIMEOUT_MEDIUM = Duration.ofSeconds(30); + + /** Default duration of the {@link Timeout#LONG} tier. */ + public static final Duration DEFAULT_TIMEOUT_LONG = Duration.ofSeconds(360); + /** * Maximum number of retries (the request is attempted up to {@code maxRetries + 1} times). Public * (not just within this package) so the internal HTTP client, which lives in a separate, @@ -17,21 +27,52 @@ public final class RetryConfig { /** Upper bound on the (exponentially growing) inter-retry delay. */ public final Duration maxDelayBetweenRetries; + /** Duration of the {@link Timeout#SHORT} tier. */ + public final Duration timeoutShort; + + /** Duration of the {@link Timeout#MEDIUM} tier. */ + public final Duration timeoutMedium; + + /** Duration of the {@link Timeout#LONG} tier. */ + public final Duration timeoutLong; + /** - * Maximum per-attempt request (socket) timeout, and the default base timeout for a call. Each - * attempt's timeout may grow from a smaller per-call base but is capped at this value. This - * bounds a single attempt, not the cumulative time across all retries. + * Cap on the timeout of a single request attempt. The first attempt gets the requested timeout, + * each retry doubles it, and none exceeds this value. This bounds a single attempt, not the + * cumulative time across all retries. */ public final Duration timeout; + /** A policy with the default tier durations and the given cap on a single attempt. */ public RetryConfig( int maxRetries, Duration minDelayBetweenRetries, Duration maxDelayBetweenRetries, Duration timeout) { + this( + maxRetries, + minDelayBetweenRetries, + maxDelayBetweenRetries, + DEFAULT_TIMEOUT_SHORT, + DEFAULT_TIMEOUT_MEDIUM, + DEFAULT_TIMEOUT_LONG, + timeout); + } + + public RetryConfig( + int maxRetries, + Duration minDelayBetweenRetries, + Duration maxDelayBetweenRetries, + Duration timeoutShort, + Duration timeoutMedium, + Duration timeoutLong, + Duration timeoutMax) { this.maxRetries = maxRetries; this.minDelayBetweenRetries = minDelayBetweenRetries; this.maxDelayBetweenRetries = maxDelayBetweenRetries; - this.timeout = timeout; + this.timeoutShort = timeoutShort; + this.timeoutMedium = timeoutMedium; + this.timeoutLong = timeoutLong; + this.timeout = timeoutMax; } } diff --git a/src/main/java/com/apify/client/http/Timeout.java b/src/main/java/com/apify/client/http/Timeout.java new file mode 100644 index 0000000..4ad80b0 --- /dev/null +++ b/src/main/java/com/apify/client/http/Timeout.java @@ -0,0 +1,110 @@ +package com.apify.client.http; + +import java.time.Duration; +import java.util.Optional; + +/** + * The timeout of a single API request attempt: one of the preconfigured {@linkplain Tier tiers}, an + * exact duration, or {@link #NO_TIMEOUT}. + * + *

Every client method picks the tier that fits the expected duration of its request: {@link + * #SHORT} for simple metadata reads and writes, {@link #MEDIUM} for listing, batch and trigger + * operations, {@link #LONG} for downloads, uploads and streaming. The duration of each tier is set + * on the {@link com.apify.client.ApifyClientBuilder}. Pass a {@code Timeout} to {@link + * com.apify.client.ApifyClient#withTimeout(Timeout)} to replace the tier of every request a client + * sends. + * + *

A tier or an explicit duration above the builder's {@code timeoutMax} is capped at it, and the + * client logs a warning when that happens. The timeout applies to each attempt separately and + * doubles with every retry, up to the same cap. With {@link #NO_TIMEOUT} the client never aborts + * the request, so a connection that stalls is not retried either. + */ +public final class Timeout { + + /** The preconfigured timeout tiers. */ + public enum Tier { + /** Simple metadata reads and writes (default 5 seconds). */ + SHORT, + /** Listing, batch and trigger operations (default 30 seconds). */ + MEDIUM, + /** Downloads, uploads and streaming (default 360 seconds). */ + LONG + } + + /** The {@link Tier#SHORT} tier. */ + public static final Timeout SHORT = new Timeout(Tier.SHORT, null, false); + + /** The {@link Tier#MEDIUM} tier. */ + public static final Timeout MEDIUM = new Timeout(Tier.MEDIUM, null, false); + + /** The {@link Tier#LONG} tier. */ + public static final Timeout LONG = new Timeout(Tier.LONG, null, false); + + /** Lets the request run for as long as it takes. */ + public static final Timeout NO_TIMEOUT = new Timeout(null, null, true); + + private final Tier tier; + private final Duration duration; + private final boolean noTimeout; + + private Timeout(Tier tier, Duration duration, boolean noTimeout) { + this.tier = tier; + this.duration = duration; + this.noTimeout = noTimeout; + } + + /** + * An exact per-attempt timeout. + * + * @throws IllegalArgumentException if {@code duration} is null, zero or negative; use {@link + * #NO_TIMEOUT} to run without a timeout + */ + public static Timeout of(Duration duration) { + if (duration == null || duration.isZero() || duration.isNegative()) { + throw new IllegalArgumentException( + "timeout must be strictly positive (use Timeout.NO_TIMEOUT to run without one)"); + } + return new Timeout(null, duration, false); + } + + /** An exact per-attempt timeout in seconds; see {@link #of(Duration)}. */ + public static Timeout ofSeconds(long seconds) { + return of(Duration.ofSeconds(seconds)); + } + + /** The tier this timeout names, or empty for an exact duration and for {@link #NO_TIMEOUT}. */ + public Optional tier() { + return Optional.ofNullable(tier); + } + + /** The exact duration, or empty for a tier and for {@link #NO_TIMEOUT}. */ + public Optional duration() { + return Optional.ofNullable(duration); + } + + /** Whether this is {@link #NO_TIMEOUT}. */ + public boolean isNoTimeout() { + return noTimeout; + } + + @Override + public boolean equals(Object o) { + return o instanceof Timeout other + && tier == other.tier + && noTimeout == other.noTimeout + && java.util.Objects.equals(duration, other.duration); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(tier, duration, noTimeout); + } + + @Override + public String toString() { + if (noTimeout) { + return "Timeout.NO_TIMEOUT"; + } + return tier != null ? "Timeout." + tier : "Timeout.of(" + duration + ")"; + } +} diff --git a/src/main/java/com/apify/client/internal/HttpClientCore.java b/src/main/java/com/apify/client/internal/HttpClientCore.java index 4eaea99..0e457af 100644 --- a/src/main/java/com/apify/client/internal/HttpClientCore.java +++ b/src/main/java/com/apify/client/internal/HttpClientCore.java @@ -1,20 +1,21 @@ package com.apify.client.internal; -import com.aayushatharva.brotli4j.Brotli4jLoader; -import com.aayushatharva.brotli4j.encoder.Encoder; import com.apify.client.http.ApiResponse; import com.apify.client.http.ApifyApiException; import com.apify.client.http.ApifyTransportException; +import com.apify.client.http.BrotliHttpCompressor; import com.apify.client.http.ConflictError; import com.apify.client.http.ForbiddenError; +import com.apify.client.http.GzipHttpCompressor; +import com.apify.client.http.HttpCompressor; import com.apify.client.http.HttpTransport; import com.apify.client.http.InvalidRequestError; import com.apify.client.http.NotFoundError; import com.apify.client.http.RateLimitError; import com.apify.client.http.RetryConfig; import com.apify.client.http.ServerError; +import com.apify.client.http.Timeout; import com.apify.client.http.UnauthorizedError; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -29,9 +30,11 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadLocalRandom; -import java.util.zip.GZIPOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The orchestrating HTTP client shared by every resource client. It owns the transport, the @@ -64,84 +67,162 @@ public final class HttpClientCore { */ private static final int MIN_COMPRESS_BYTES = 1024; - /** {@code Content-Encoding} value for brotli-compressed bodies (matches the reference client). */ - private static final String ENCODING_BROTLI = "br"; - - /** {@code Content-Encoding} value for gzip-compressed bodies (the fallback coding). */ - private static final String ENCODING_GZIP = "gzip"; - /** Prefix required before the token value in the {@code Authorization} header. */ private static final String AUTH_BEARER_PREFIX = "Bearer "; - /** - * Whether a brotli native codec loaded for the running platform. Resolved once at class load: - * brotli4j needs a platform-specific native library, so on platforms without one (or if loading - * fails) the client falls back to gzip. Mirrors the reference JS client, which prefers brotli and - * falls back to gzip. - */ - private static final boolean BROTLI_AVAILABLE = detectBrotli(); + /** The API holds a {@code waitForFinish} request for at most this many seconds. */ + private static final long MAX_HOLD_SECS = 60; - private static boolean detectBrotli() { - try { - Brotli4jLoader.ensureAvailability(); - return true; - } catch (Throwable t) { - // Native codec unavailable (no bundled binary for this OS/arch, or a link error). Catch - // Throwable because native loading can raise UnsatisfiedLinkError/NoClassDefFoundError, not - // just Exception. The client stays fully functional using gzip. - return false; - } - } - - /** - * Reports whether the brotli path is active on this platform. Public so tests outside this - * package (which lives in the non-exported {@code com.apify.client.internal} package) can assert - * on it. - */ - public static boolean brotliAvailable() { - return BROTLI_AVAILABLE; - } + private static final Logger LOG = LoggerFactory.getLogger(HttpClientCore.class); private final HttpTransport transport; private final String token; private final String userAgent; private final RetryConfig retry; + private final HttpCompressor compressor; + private final Map defaultHeaders; + + /** When set, replaces the timeout every request would otherwise get. */ + private final Timeout timeoutOverride; + + /** Requested timeouts already warned about, shared by every copy made by {@link #withTimeout}. */ + private final Set warnedCappedTimeouts; + /** + * @param compressor compresses large request bodies; {@code null} picks brotli where its native + * library is available and gzip elsewhere + * @param defaultHeaders headers sent with every request; they replace the built-in {@code + * User-Agent} and {@code Authorization} headers of the same name and are replaced by a header + * the request sets itself + */ public HttpClientCore( - HttpTransport transport, String token, String userAgent, RetryConfig retry) { + HttpTransport transport, + String token, + String userAgent, + RetryConfig retry, + HttpCompressor compressor, + Map defaultHeaders) { + this( + transport, + token, + userAgent, + retry, + compressor != null ? compressor : defaultCompressor(), + defaultHeaders == null ? Map.of() : Map.copyOf(defaultHeaders), + null, + ConcurrentHashMap.newKeySet()); + } + + private static HttpCompressor defaultCompressor() { + return BrotliHttpCompressor.isAvailable() + ? new BrotliHttpCompressor() + : new GzipHttpCompressor(); + } + + private HttpClientCore( + HttpTransport transport, + String token, + String userAgent, + RetryConfig retry, + HttpCompressor compressor, + Map defaultHeaders, + Timeout timeoutOverride, + Set warnedCappedTimeouts) { this.transport = transport; this.token = token; this.userAgent = userAgent; this.retry = retry; + this.compressor = compressor; + this.defaultHeaders = defaultHeaders; + this.timeoutOverride = timeoutOverride; + this.warnedCappedTimeouts = warnedCappedTimeouts; } public String userAgent() { return userAgent; } - /** The configured maximum per-attempt request timeout, in whole seconds. */ - public long requestTimeoutSeconds() { - return retry.timeout.getSeconds(); + /** + * A copy of this client that sends every request with {@code timeout} instead of the tier the + * calling method would pick. The transport, token and retry policy are shared. + */ + public HttpClientCore withTimeout(Timeout timeout) { + if (timeout == null) { + throw new IllegalArgumentException("timeout must not be null"); + } + return new HttpClientCore( + transport, + token, + userAgent, + retry, + compressor, + defaultHeaders, + timeout, + warnedCappedTimeouts); + } + + /** Whether {@link #withTimeout} replaced the per-method timeouts of this client. */ + public boolean hasTimeoutOverride() { + return timeoutOverride != null; } /** - * The default per-attempt request timeout used as the base for standard API calls. It is the - * client's configured timeout (single source of truth), so a caller-configured timeout applies to - * the first attempt too. Individual calls may pass a smaller base (e.g. key-value-store uploads), - * which then grows back up toward this value on retries. + * Resolves a timeout to the duration of its first attempt: a tier to its configured duration, an + * exact timeout to itself, and {@link Timeout#NO_TIMEOUT} to {@code null} (no timeout). A value + * above the configured maximum is capped at it, which logs a warning once per value, since the + * request does not get the whole time it asked for. The override set by {@link #withTimeout} + * takes precedence over {@code requested}. */ - public Duration baseRequestTimeout() { - return retry.timeout; + public Duration resolveTimeout(Timeout requested) { + Timeout effective = timeoutOverride != null ? timeoutOverride : requested; + if (effective.isNoTimeout()) { + return null; + } + Duration duration = + effective + .duration() + .orElseGet( + () -> + switch (effective.tier().orElseThrow()) { + case SHORT -> retry.timeoutShort; + case MEDIUM -> retry.timeoutMedium; + case LONG -> retry.timeoutLong; + }); + if (duration.compareTo(retry.timeout) > 0) { + if (warnedCappedTimeouts.add(duration)) { + LOG.warn( + "The requested timeout of {}s exceeds timeoutMax ({}s) and is capped at it. Raise" + + " timeoutMax on the client builder to allow longer request timeouts.", + duration.toMillis() / 1000.0, + retry.timeout.toMillis() / 1000.0); + } + return retry.timeout; + } + return duration; + } + + /** + * The timeout for a request that asks the API to hold its response for {@code holdSecs} with + * {@code waitForFinish}: the hold plus the round trip {@code tier} allows, so the client does not + * abort a request the API is still holding (which, for a start or build, would retry a call that + * creates a resource). An override set by {@link #withTimeout} is used as given. + */ + public Timeout timeoutWithHold(Timeout tier, Long holdSecs) { + if (timeoutOverride != null || tier.isNoTimeout() || holdSecs == null || holdSecs <= 0) { + return tier; + } + Duration base = resolveTimeout(tier); + return Timeout.of(base.plusSeconds(Math.min(holdSecs, MAX_HOLD_SECS))); } /** Sends a request with auth, User-Agent and the retry policy applied. */ public CompletableFuture call( - String method, String url, byte[] body, String contentType, Duration baseTimeout) { - return call(method, url, body, contentType, null, baseTimeout, false); + String method, String url, byte[] body, String contentType, Timeout timeout) { + return call(method, url, body, contentType, null, timeout, false); } /** - * Like {@link #call(String, String, byte[], String, Duration)} but, when {@code + * Like {@link #call(String, String, byte[], String, Timeout)} but, when {@code * doNotRetryTimeouts} is {@code true}, a transport timeout is treated as a terminal failure * rather than being retried (other transport/network errors and retryable statuses are still * retried). @@ -151,13 +232,13 @@ public CompletableFuture call( String url, byte[] body, String contentType, - Duration baseTimeout, + Timeout timeout, boolean doNotRetryTimeouts) { - return call(method, url, body, contentType, null, baseTimeout, doNotRetryTimeouts); + return call(method, url, body, contentType, null, timeout, doNotRetryTimeouts); } /** - * Like {@link #call(String, String, byte[], String, Duration)} but additionally sets the given + * Like {@link #call(String, String, byte[], String, Timeout)} but additionally sets the given * extra headers on every attempt. */ public CompletableFuture call( @@ -166,8 +247,8 @@ public CompletableFuture call( byte[] body, String contentType, Map extraHeaders, - Duration baseTimeout) { - return call(method, url, body, contentType, extraHeaders, baseTimeout, false); + Timeout timeout) { + return call(method, url, body, contentType, extraHeaders, timeout, false); } /** The canonical overload every other {@code call} convenience overload delegates to. */ @@ -177,17 +258,23 @@ public CompletableFuture call( byte[] body, String contentType, Map extraHeaders, - Duration baseTimeout, + Timeout timeout, boolean doNotRetryTimeouts) { + Duration baseTimeout = resolveTimeout(timeout); + // Compress the body once, up front, so every retry reuses the same encoded payload. byte[] requestBody = body; Map headers = extraHeaders; if (shouldCompress(requestBody, contentType, extraHeaders)) { - Compressed compressed = compress(requestBody, BROTLI_AVAILABLE); - requestBody = compressed.body(); + try { + requestBody = compressor.compress(requestBody); + } catch (IOException | RuntimeException e) { + // No fallback to another coding: the configured compressor is the only one that runs. + return failedFuture(new ApifyTransportException(e)); + } headers = new LinkedHashMap<>(extraHeaders == null ? Map.of() : extraHeaders); - headers.put(HttpHeaders.CONTENT_ENCODING, compressed.encoding()); + headers.put(HttpHeaders.CONTENT_ENCODING, compressor.contentEncoding()); } String path = extractPath(url); @@ -309,8 +396,10 @@ HttpRequest buildRequest( body != null ? HttpRequest.BodyPublishers.ofByteArray(body) : HttpRequest.BodyPublishers.noBody(); - HttpRequest.Builder b = - HttpRequest.newBuilder(URI.create(url)).method(method, publisher).timeout(timeout); + HttpRequest.Builder b = HttpRequest.newBuilder(URI.create(url)).method(method, publisher); + if (timeout != null) { + b.timeout(timeout); + } b.header(HttpHeaders.USER_AGENT, userAgent); if (token != null && !token.isEmpty()) { b.header(HttpHeaders.AUTHORIZATION, AUTH_BEARER_PREFIX + token); @@ -318,8 +407,11 @@ HttpRequest buildRequest( if (contentType != null && !contentType.isEmpty()) { b.header(HttpHeaders.CONTENT_TYPE, contentType); } + // setHeader replaces what is already set, so a default header wins over the built-in ones and + // a header of the request itself wins over both. + defaultHeaders.forEach(b::setHeader); if (extraHeaders != null) { - extraHeaders.forEach(b::header); + extraHeaders.forEach(b::setHeader); } return b.build(); } @@ -346,6 +438,9 @@ private CompletableFuture doAttempt( * across all retries. */ private Duration attemptTimeout(Duration base, int attempt) { + if (base == null) { + return null; // Timeout.NO_TIMEOUT + } Duration scaled = base; for (int i = 1; i < attempt; i++) { scaled = scaled.multipliedBy(2); @@ -468,70 +563,6 @@ public static boolean isCompressibleContentType(String contentType) { return true; } - /** - * A compressed request body together with the {@code Content-Encoding} token that describes it. - * - * @param body the compressed request body - * @param encoding the {@code Content-Encoding} token describing {@code body} - */ - public record Compressed(byte[] body, String encoding) {} - - /** - * Compresses a request body, preferring brotli and falling back to gzip, matching the reference - * JS client. When {@code preferBrotli} is {@code true} the body is brotli-encoded ({@code - * Content-Encoding: br}); otherwise it is gzip-encoded ({@code Content-Encoding: gzip}). Callers - * pass {@link #BROTLI_AVAILABLE}; making the coding an explicit parameter keeps this a pure - * function of its inputs rather than of hidden static state. Public so {@code CompressionTest} - * (outside this non-exported package) can exercise it directly. - * - *

If the brotli path itself fails despite the native codec having loaded, this falls back to - * gzip rather than failing the whole request - matching the reference JS client, which since apify-client-js#990 keys the - * fallback on compression actually failing rather than only on an upfront availability check. - * Catches {@code Throwable}, not just {@code RuntimeException}: a native codec that loaded - * successfully at startup can still fail a specific call with an {@code Error} (e.g. a partial or - * mismatched native library surfacing {@link UnsatisfiedLinkError} only once a method is actually - * invoked), the same reason {@link #detectBrotli()} itself catches {@code Throwable} rather than - * {@code Exception}. - */ - public static Compressed compress(byte[] data, boolean preferBrotli) { - if (preferBrotli) { - try { - return new Compressed(brotli(data), ENCODING_BROTLI); - } catch (Throwable t) { - return new Compressed(gzip(data), ENCODING_GZIP); - } - } - return new Compressed(gzip(data), ENCODING_GZIP); - } - - /** Brotli-compresses a request body using the loaded native codec. */ - private static byte[] brotli(byte[] data) { - try { - return Encoder.compress(data); - } catch (IOException e) { - // Encoding an in-memory byte[] performs no real I/O, so this is unreachable in practice. - throw new ApifyTransportException(e); - } - } - - /** - * Gzip-compresses a request body. Used as the fallback coding when no brotli native codec is - * available for the running platform; the JDK always ships a gzip codec ({@link - * GZIPOutputStream}). - */ - private static byte[] gzip(byte[] data) { - ByteArrayOutputStream out = new ByteArrayOutputStream(data.length); - try (GZIPOutputStream gz = new GZIPOutputStream(out)) { - gz.write(data); - } catch (IOException e) { - // Compressing an in-memory byte[] cannot perform real I/O, so this is unreachable in - // practice. - throw new ApifyTransportException(e); - } - return out.toByteArray(); - } - private static boolean isStatusRetryable(int status) { return status == RATE_LIMIT_EXCEEDED || status >= MIN_SERVER_ERROR; } @@ -655,7 +686,7 @@ public static String extractPath(String url) { * matching {@link HttpTransport}'s contract. */ public CompletableFuture> streamAsync(String url) { - HttpRequest request = buildRequest("GET", url, null, null, null, retry.timeout); + HttpRequest request = buildRequest("GET", url, null, null, null, resolveTimeout(Timeout.LONG)); CompletableFuture> result = new CompletableFuture<>(); transport .sendStreamingAsync(request) diff --git a/src/main/java/com/apify/client/internal/Json.java b/src/main/java/com/apify/client/internal/Json.java index f1d1096..bcf6155 100644 --- a/src/main/java/com/apify/client/internal/Json.java +++ b/src/main/java/com/apify/client/internal/Json.java @@ -68,6 +68,31 @@ public static byte[] toBytes(Object value) { return MAPPER.writeValueAsBytes(value); } + /** + * The request body for a caller-supplied payload (an Actor or task input): a {@code byte[]} and a + * {@link String} are already serialized and sent as they are (a string is UTF-8 encoded), so an + * input read from a file, or the exact bytes of a non-JSON content type, are not wrapped in a + * second layer of JSON. Anything else is serialized to JSON. A {@code null} payload sends no + * body. + */ + public static byte[] toRequestBody(Object payload) { + if (payload == null) { + return null; + } + if (payload instanceof byte[] bytes) { + return bytes; + } + if (payload instanceof String text) { + return text.getBytes(java.nio.charset.StandardCharsets.UTF_8); + } + return toBytes(payload); + } + + /** Converts a value to its JSON tree, as {@link #toBytes} would serialize it. */ + public static JsonNode toTree(Object value) { + return MAPPER.valueToTree(value); + } + /** Parses JSON bytes into the given type. */ public static T parse(byte[] body, JavaType type) { return MAPPER.readValue(body, type); diff --git a/src/main/java/com/apify/client/internal/NormalizedUrlDeserializer.java b/src/main/java/com/apify/client/internal/NormalizedUrlDeserializer.java new file mode 100644 index 0000000..a2659a6 --- /dev/null +++ b/src/main/java/com/apify/client/internal/NormalizedUrlDeserializer.java @@ -0,0 +1,22 @@ +package com.apify.client.internal; + +import tools.jackson.core.JsonParser; +import tools.jackson.databind.DeserializationContext; +import tools.jackson.databind.deser.std.StdDeserializer; + +/** + * Deserializes a {@code format: uri} string field of an API response into its {@linkplain + * UrlNormalizer#normalize normalized} form. Internal to the client; models reference it with + * {@code @JsonDeserialize}. + */ +public final class NormalizedUrlDeserializer extends StdDeserializer { + + public NormalizedUrlDeserializer() { + super(String.class); + } + + @Override + public String deserialize(JsonParser p, DeserializationContext ctxt) { + return UrlNormalizer.normalize(p.getValueAsString()); + } +} diff --git a/src/main/java/com/apify/client/internal/QueryParams.java b/src/main/java/com/apify/client/internal/QueryParams.java index a0ca627..5b0681e 100644 --- a/src/main/java/com/apify/client/internal/QueryParams.java +++ b/src/main/java/com/apify/client/internal/QueryParams.java @@ -64,6 +64,16 @@ public QueryParams addRaw(String key, String value) { return this; } + /** The value of the first {@code key} parameter parsed as an integer, or {@code null}. */ + public Long getLong(String key) { + for (Pair p : pairs) { + if (p.key().equals(key)) { + return Long.valueOf(p.value()); + } + } + return null; + } + boolean isEmpty() { return pairs.isEmpty(); } diff --git a/src/main/java/com/apify/client/internal/ResourceContext.java b/src/main/java/com/apify/client/internal/ResourceContext.java index c66c86e..035ec13 100644 --- a/src/main/java/com/apify/client/internal/ResourceContext.java +++ b/src/main/java/com/apify/client/internal/ResourceContext.java @@ -4,6 +4,7 @@ import com.apify.client.http.ApiResponse; import com.apify.client.http.ApifyApiException; import com.apify.client.http.ApifyTransportException; +import com.apify.client.http.Timeout; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.time.Duration; @@ -35,13 +36,6 @@ public final class ResourceContext { /** Server-side waitForFinish chunk size (the API caps server waiting at 60 seconds). */ private static final long WAIT_REQUEST_SECS = 60; - /** - * Safety margin subtracted from the configured per-request timeout when choosing the server-side - * {@code waitForFinish} value, so the server responds before the client's socket timeout fires - * (otherwise a short client timeout would abort every healthy poll and burn all retries). - */ - private static final long WAIT_TIMEOUT_MARGIN_SECS = 5; - /** * Finite upper bound used when the caller asks to wait "indefinitely" ({@code waitSecs == null}). * The API will not accept "Infinity", and an unbounded loop can spin forever on a transient 404; @@ -140,7 +134,7 @@ public static ResourceContext nestedCollection( public static ResourceContext single( HttpClientCore http, String baseUrl, String resourcePath, String id) { return new ResourceContext( - http, baseUrl + "/" + resourcePath + "/" + toSafeId(id), baseUrl, true); + http, baseUrl + "/" + resourcePath + "/" + encodePathSegment(toSafeId(id)), baseUrl, true); } /** A copy of this context with the origin used to build public URLs overridden. */ @@ -190,12 +184,17 @@ public ResourceContext seedParams(QueryParams inherited) { public CompletableFuture> getResource( String subPath, QueryParams params, JavaType dataType) { + return getResource(subPath, params, dataType, Timeout.SHORT); + } + + public CompletableFuture> getResource( + String subPath, QueryParams params, JavaType dataType, Timeout timeout) { // ofNullable, not of: an HTTP 200 with body {"data": null} unwraps to null, which is a valid // "no resource" answer rather than a programming error — never surface it as a raw NPE. // Explicit witness: getResourceRequired is the *receiver* of .handle below (not an argument // to another generic call), so unlike a direct `return`, its type parameter cannot be inferred // from the surrounding target type and must be pinned here. - return this.getResourceRequired(subPath, params, dataType) + return this.getResourceRequired(subPath, params, dataType, timeout) .handle( (value, error) -> { if (error == null) { @@ -214,10 +213,20 @@ public CompletableFuture> getResource( return getResource(subPath, params, Json.type(dataClass)); } + public CompletableFuture> getResource( + String subPath, QueryParams params, Class dataClass, Timeout timeout) { + return getResource(subPath, params, Json.type(dataClass), timeout); + } + public CompletableFuture getResourceRequired( String subPath, QueryParams params, JavaType dataType) { + return getResourceRequired(subPath, params, dataType, Timeout.SHORT); + } + + public CompletableFuture getResourceRequired( + String subPath, QueryParams params, JavaType dataType, Timeout timeout) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - return http.call("GET", u, null, "", http.baseRequestTimeout()) + return http.call("GET", u, null, "", timeout) .thenApply(resp -> Json.parseData(resp.body(), dataType)); } @@ -226,6 +235,11 @@ public CompletableFuture getResourceRequired( return getResourceRequired(subPath, params, Json.type(dataClass)); } + public CompletableFuture getResourceRequired( + String subPath, QueryParams params, Class dataClass, Timeout timeout) { + return getResourceRequired(subPath, params, Json.type(dataClass), timeout); + } + /** * Like {@link #getResource}, but a 404 is only swallowed to {@link Optional#empty()} when this * context {@link #ownId addresses its resource by an explicit id}. Otherwise (a nested path with @@ -236,10 +250,16 @@ public CompletableFuture getResourceRequired( */ public CompletableFuture> getResourceUnlessAmbiguous( String subPath, QueryParams params, JavaType dataType) { + return getResourceUnlessAmbiguous(subPath, params, dataType, Timeout.SHORT); + } + + public CompletableFuture> getResourceUnlessAmbiguous( + String subPath, QueryParams params, JavaType dataType, Timeout timeout) { if (ownId) { - return getResource(subPath, params, dataType); + return getResource(subPath, params, dataType, timeout); } - return this.getResourceRequired(subPath, params, dataType).thenApply(Optional::ofNullable); + return this.getResourceRequired(subPath, params, dataType, timeout) + .thenApply(Optional::ofNullable); } public CompletableFuture> getResourceUnlessAmbiguous( @@ -248,15 +268,20 @@ public CompletableFuture> getResourceUnlessAmbiguous( } public CompletableFuture updateResource(String subPath, Object body, Class dataClass) { + return updateResource(subPath, body, dataClass, Timeout.SHORT); + } + + public CompletableFuture updateResource( + String subPath, Object body, Class dataClass, Timeout timeout) { String u = mergedParams(new QueryParams()).applyToUrl(subUrl(subPath)); - return http.call("PUT", u, Json.toBytes(body), CONTENT_TYPE_JSON, http.baseRequestTimeout()) + return http.call("PUT", u, Json.toBytes(body), CONTENT_TYPE_JSON, timeout) .thenApply(resp -> Json.parseData(resp.body(), dataClass)); } /** Performs a DELETE; a not-found is treated as a successful no-op. */ public CompletableFuture deleteResource(String subPath) { String u = mergedParams(new QueryParams()).applyToUrl(subUrl(subPath)); - return http.call("DELETE", u, null, "", http.baseRequestTimeout()) + return http.call("DELETE", u, null, "", Timeout.SHORT) .handle( (resp, error) -> { if (error == null) { @@ -280,13 +305,18 @@ public CompletableFuture deleteResourceUnlessAmbiguous(String subPath) { return deleteResource(subPath); } String u = mergedParams(new QueryParams()).applyToUrl(subUrl(subPath)); - return http.call("DELETE", u, null, "", http.baseRequestTimeout()).thenApply(resp -> null); + return http.call("DELETE", u, null, "", Timeout.SHORT).thenApply(resp -> null); } public CompletableFuture> listResource( String subPath, QueryParams params, Class itemClass) { + return listResource(subPath, params, itemClass, Timeout.MEDIUM); + } + + public CompletableFuture> listResource( + String subPath, QueryParams params, Class itemClass, Timeout timeout) { JavaType listType = Json.parametric(PaginationList.class, Json.type(itemClass)); - return getResourceRequired(subPath, params, listType); + return getResourceRequired(subPath, params, listType, timeout); } /** @@ -320,8 +350,13 @@ public Flow.Publisher iterateResource( public CompletableFuture createResource( QueryParams params, Object body, Class dataClass) { + return createResource(params, body, dataClass, Timeout.SHORT); + } + + public CompletableFuture createResource( + QueryParams params, Object body, Class dataClass, Timeout timeout) { String u = mergedParams(params).applyToUrl(subUrl("")); - return http.call("POST", u, Json.toBytes(body), CONTENT_TYPE_JSON, http.baseRequestTimeout()) + return http.call("POST", u, Json.toBytes(body), CONTENT_TYPE_JSON, timeout) .thenApply(resp -> Json.parseData(resp.body(), dataClass)); } @@ -354,21 +389,59 @@ public CompletableFuture getOrCreateNamed(String name, Object body, Class } String u = params.applyToUrl(subUrl("")); byte[] bodyBytes = body == null ? null : Json.toBytes(body); - return http.call( - "POST", u, bodyBytes, body == null ? "" : CONTENT_TYPE_JSON, http.baseRequestTimeout()) + return http.call("POST", u, bodyBytes, body == null ? "" : CONTENT_TYPE_JSON, Timeout.SHORT) .thenApply(resp -> Json.parseData(resp.body(), dataClass)); } /** POST with a raw body (optional) and content type, unwrapping the data envelope. */ public CompletableFuture postWithBody( - String subPath, QueryParams params, byte[] body, String contentType, Class dataClass) { - return postWithBody(subPath, params, body, contentType, Json.type(dataClass)); + String subPath, + QueryParams params, + byte[] body, + String contentType, + Class dataClass, + Timeout timeout) { + return postWithBody(subPath, params, body, contentType, Json.type(dataClass), timeout, false); + } + + /** As {@link #postWithBody(String, QueryParams, byte[], String, Class, Timeout)}, see above. */ + public CompletableFuture postWithBody( + String subPath, + QueryParams params, + byte[] body, + String contentType, + Class dataClass, + Timeout timeout, + boolean doNotRetryTimeouts) { + return postWithBody( + subPath, params, body, contentType, Json.type(dataClass), timeout, doNotRetryTimeouts); } public CompletableFuture postWithBody( - String subPath, QueryParams params, byte[] body, String contentType, JavaType dataType) { + String subPath, + QueryParams params, + byte[] body, + String contentType, + JavaType dataType, + Timeout timeout) { + return postWithBody(subPath, params, body, contentType, dataType, timeout, false); + } + + /** + * As {@link #postWithBody(String, QueryParams, byte[], String, JavaType, Timeout)}, with control + * over whether a transport timeout is retried. A request that creates a billed resource (a run, a + * build) must not be retried on a timeout, since the API may already have created it. + */ + public CompletableFuture postWithBody( + String subPath, + QueryParams params, + byte[] body, + String contentType, + JavaType dataType, + Timeout timeout, + boolean doNotRetryTimeouts) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - return http.call("POST", u, body, contentType, http.baseRequestTimeout()) + return http.call("POST", u, body, contentType, timeout, doNotRetryTimeouts) .thenApply(resp -> Json.parseData(resp.body(), dataType)); } @@ -378,17 +451,27 @@ public CompletableFuture postWithBody( * plain object rather than the standard data envelope. */ public CompletableFuture postWithBodyNoEnvelope( - String subPath, QueryParams params, byte[] body, String contentType, Class dataClass) { + String subPath, + QueryParams params, + byte[] body, + String contentType, + Class dataClass, + Timeout timeout) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - return http.call("POST", u, body, contentType, http.baseRequestTimeout()) + return http.call("POST", u, body, contentType, timeout) .thenApply(resp -> Json.parse(resp.body(), dataClass)); } /** PUT with a raw body (optional) and content type, unwrapping the data envelope. */ public CompletableFuture putWithBody( - String subPath, QueryParams params, byte[] body, String contentType, Class dataClass) { + String subPath, + QueryParams params, + byte[] body, + String contentType, + Class dataClass, + Timeout timeout) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - return http.call("PUT", u, body, contentType, http.baseRequestTimeout()) + return http.call("PUT", u, body, contentType, timeout) .thenApply(resp -> Json.parseData(resp.body(), dataClass)); } @@ -398,26 +481,32 @@ public CompletableFuture putWithBody( * response is a plain object rather than the standard data envelope. */ public CompletableFuture putWithBodyNoEnvelope( - String subPath, QueryParams params, byte[] body, String contentType, Class dataClass) { + String subPath, + QueryParams params, + byte[] body, + String contentType, + Class dataClass, + Timeout timeout) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - return http.call("PUT", u, body, contentType, http.baseRequestTimeout()) + return http.call("PUT", u, body, contentType, timeout) .thenApply(resp -> Json.parse(resp.body(), dataClass)); } /** DELETE with a JSON body (used for batch request deletion), unwrapping the data envelope. */ public CompletableFuture deleteWithBody( - String subPath, QueryParams params, Object body, Class dataClass) { + String subPath, QueryParams params, Object body, Class dataClass, Timeout timeout) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - return http.call("DELETE", u, Json.toBytes(body), CONTENT_TYPE_JSON, http.baseRequestTimeout()) + return http.call("DELETE", u, Json.toBytes(body), CONTENT_TYPE_JSON, timeout) .thenApply(resp -> Json.parseData(resp.body(), dataClass)); } /** * GET returning the raw response (no data envelope). Completes with {@code null} on not-found. */ - public CompletableFuture getRaw(String subPath, QueryParams params) { + public CompletableFuture getRaw( + String subPath, QueryParams params, Timeout timeout) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - return http.call("GET", u, null, "", http.baseRequestTimeout()) + return http.call("GET", u, null, "", timeout) .handle( (resp, error) -> { if (error == null) { @@ -434,23 +523,26 @@ public CompletableFuture getRaw(String subPath, QueryParams params) /** * As {@link #getRaw}, but a 404 is not swallowed - it propagates as an {@link ApifyApiException}. */ - public CompletableFuture getRawRequired(String subPath, QueryParams params) { + public CompletableFuture getRawRequired( + String subPath, QueryParams params, Timeout timeout) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - return http.call("GET", u, null, "", http.baseRequestTimeout()); + return http.call("GET", u, null, "", timeout); } /** * Like {@link #getRaw}, but - mirroring {@link #getResourceUnlessAmbiguous} - a 404 is only * swallowed to {@code null} when this context addresses its resource by an explicit id. */ - public CompletableFuture getRawUnlessAmbiguous(String subPath, QueryParams params) { - return ownId ? getRaw(subPath, params) : getRawRequired(subPath, params); + public CompletableFuture getRawUnlessAmbiguous( + String subPath, QueryParams params, Timeout timeout) { + return ownId ? getRaw(subPath, params, timeout) : getRawRequired(subPath, params, timeout); } /** HEAD request; completes with whether the resource exists. */ - public CompletableFuture headExists(String subPath, QueryParams params) { + public CompletableFuture headExists( + String subPath, QueryParams params, Timeout timeout) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - return http.call("HEAD", u, null, "", http.baseRequestTimeout()) + return http.call("HEAD", u, null, "", timeout) .handle( (resp, error) -> { if (error == null) { @@ -467,7 +559,7 @@ public CompletableFuture headExists(String subPath, QueryParams params) /** PUT with raw bytes and a content type (used for key-value-store record uploads). */ public CompletableFuture putRaw( String subPath, QueryParams params, byte[] body, String contentType) { - return putRaw(subPath, params, body, contentType, http.baseRequestTimeout(), false); + return putRaw(subPath, params, body, contentType, Timeout.SHORT, false); } /** @@ -481,16 +573,16 @@ public CompletableFuture putRaw( QueryParams params, byte[] body, String contentType, - Duration timeout, + Timeout timeout, boolean doNotRetryTimeouts) { return putRaw(subPath, params, body, contentType, null, timeout, doNotRetryTimeouts); } /** - * As {@link #putRaw(String, QueryParams, byte[], String, Duration, boolean)}, additionally - * setting {@code extraHeaders} on the request (used to forward a caller-supplied {@code - * Content-Encoding} on a key-value-store record upload - see {@link HttpHeaders#CONTENT_ENCODING} - * - which also stops {@link HttpClientCore} from compressing the already-encoded body again). + * As {@link #putRaw(String, QueryParams, byte[], String, Timeout, boolean)}, additionally setting + * {@code extraHeaders} on the request (used to forward a caller-supplied {@code Content-Encoding} + * on a key-value-store record upload - see {@link HttpHeaders#CONTENT_ENCODING} - which also + * stops {@link HttpClientCore} from compressing the already-encoded body again). */ public CompletableFuture putRaw( String subPath, @@ -498,38 +590,13 @@ public CompletableFuture putRaw( byte[] body, String contentType, Map extraHeaders, - Duration timeout, + Timeout timeout, boolean doNotRetryTimeouts) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); return http.call("PUT", u, body, contentType, extraHeaders, timeout, doNotRetryTimeouts) .thenApply(resp -> null); } - /** - * The largest server-side {@code waitForFinish} value that is safe to send: below the configured - * per-request timeout by a safety margin (or the API's 60s cap when no finite timeout is set), so - * the server always responds before the client's socket timeout fires. - */ - private long serverWaitCapSecs() { - long configuredTimeoutSecs = http.requestTimeoutSeconds(); - return configuredTimeoutSecs > 0 - ? Math.max(0, configuredTimeoutSecs - WAIT_TIMEOUT_MARGIN_SECS) - : WAIT_REQUEST_SECS; - } - - /** - * Clamps a caller-supplied server-side {@code waitForFinish} value (seconds) to {@link - * #serverWaitCapSecs()}, so a synchronous get/wait can never ask the server to hold the - * connection longer than the client's own per-request timeout. Returns {@code null} for a {@code - * null} input (no server-side wait requested). - */ - public Long clampServerWait(Long waitForFinishSecs) { - if (waitForFinishSecs == null) { - return null; - } - return Math.min(Math.max(0, waitForFinishSecs), serverWaitCapSecs()); - } - /** * Polls a GET endpoint with {@code waitForFinish} until the resource reaches a terminal state or * the wait budget elapses. {@code waitSecs == null} means "wait indefinitely", implemented as a @@ -557,9 +624,7 @@ public CompletableFuture waitForFinish( : MAX_WAIT_FOR_FINISH_SECS; long budgetMillis = effectiveWaitSecs * 1000L; long start = System.currentTimeMillis(); - long serverWaitCap = serverWaitCapSecs(); - return pollOnce( - dataType, isTerminal, resourceName, start, budgetMillis, serverWaitCap, null, null); + return pollOnce(dataType, isTerminal, resourceName, start, budgetMillis, null, null); } /** @@ -594,18 +659,16 @@ private CompletableFuture pollOnce( String resourceName, long start, long budgetMillis, - long serverWaitCap, T lastSeen, Long absentSinceMillis) { long elapsed = System.currentTimeMillis() - start; long remainingSecs = (budgetMillis - elapsed) / 1000L; - long requestSecs = - Math.min(Math.min(Math.max(remainingSecs, 0), WAIT_REQUEST_SECS), serverWaitCap); + long requestSecs = Math.min(Math.max(remainingSecs, 0), WAIT_REQUEST_SECS); QueryParams params = new QueryParams(); params.addLong("waitForFinish", requestSecs); - return this.getResourceRequired("", params, dataType) + return this.getResourceRequired("", params, dataType, Timeout.NO_TIMEOUT) .handle( (value, error) -> { if (error == null) { @@ -653,7 +716,6 @@ private CompletableFuture pollOnce( resourceName, start, budgetMillis, - serverWaitCap, seen, absentSince)); }); @@ -715,7 +777,15 @@ public static String encodePathSegment(String input) { throw new IllegalArgumentException( "a URL path segment must not be empty or a dot segment (\".\" or \"..\"), was: " + input); } - return URLEncoder.encode(input, StandardCharsets.UTF_8).replace("+", "%20"); + // URLEncoder targets form encoding: it writes a space as "+" and escapes "~" and the + // characters of encodeURIComponent's unreserved set, none of which a path segment needs. + return URLEncoder.encode(input, StandardCharsets.UTF_8) + .replace("+", "%20") + .replace("%7E", "~") + .replace("%21", "!") + .replace("%27", "'") + .replace("%28", "(") + .replace("%29", ")"); } /** Extracts the origin ({@code scheme://host[:port]}) from a URL, dropping any path. */ diff --git a/src/main/java/com/apify/client/internal/RunStartSupport.java b/src/main/java/com/apify/client/internal/RunStartSupport.java index c02ce02..ba8df49 100644 --- a/src/main/java/com/apify/client/internal/RunStartSupport.java +++ b/src/main/java/com/apify/client/internal/RunStartSupport.java @@ -3,6 +3,7 @@ import com.apify.client.ApifyClient; import com.apify.client.actor.ActorStartOptions; import com.apify.client.http.ApifyApiException; +import com.apify.client.http.Timeout; import com.apify.client.log.StreamedLog; import com.apify.client.log.StreamedLogOptions; import com.apify.client.run.ActorRun; @@ -57,10 +58,10 @@ private RunStartSupport() {} /** * Starts a run on {@code ctx}'s {@code runs} sub-resource and completes with the created run as - * soon as it exists (no waiting). {@code input} is any JSON-serializable value, or {@code null} - * for no input. {@code applyParams} applies every start-option query parameter (build, memory, - * timeout, ...) the caller's options type configures; {@code contentType} is the input body's - * content type. + * soon as it exists (no waiting). {@code input} is any JSON-serializable value, a {@code byte[]} + * or {@link String} sent as it is, or {@code null} for no input. {@code applyParams} applies + * every start-option query parameter (build, memory, timeout, ...) the caller's options type + * configures; {@code contentType} is the input body's content type. * * @param waitForResourcesSecs mirrors {@link ActorStartOptions#waitForResources(Long)}: {@code * null} does not retry a start rejected for lack of resources (the default); {@link @@ -73,18 +74,38 @@ public static CompletableFuture start( Consumer applyParams, String contentType, Long waitForResourcesSecs) { + return start(ctx, input, applyParams, contentType, waitForResourcesSecs, null); + } + + /** + * As {@link #start(ResourceContext, Object, Consumer, String, Long)}, with an explicit request + * timeout; {@code null} picks the {@link Timeout#MEDIUM} tier extended by the start's {@code + * waitForFinish} hold, so the client does not abort a request the API is still holding. + */ + private static CompletableFuture start( + ResourceContext ctx, + Object input, + Consumer applyParams, + String contentType, + Long waitForResourcesSecs, + Timeout timeout) { QueryParams params = new QueryParams(); applyParams.accept(params); - byte[] body = input == null ? null : Json.toBytes(input); + byte[] body = Json.toRequestBody(input); + Timeout requestTimeout = + timeout != null + ? timeout + : ctx.http.timeoutWithHold(Timeout.MEDIUM, params.getLong("waitForFinish")); if (waitForResourcesSecs == null) { - return ctx.postWithBody("runs", params, body, contentType, ActorRun.class); + return postRun(ctx, params, body, contentType, requestTimeout); } long deadlineMillis = waitForResourcesSecs == ActorStartOptions.WAIT_FOR_RESOURCES_INDEFINITELY ? Long.MAX_VALUE : System.currentTimeMillis() + Math.min(Math.max(0, waitForResourcesSecs), MAX_WAIT_FOR_RESOURCES_SECS) * 1000L; - return startRetryingForResources(ctx, params, body, contentType, deadlineMillis); + return startRetryingForResources( + ctx, params, body, contentType, requestTimeout, deadlineMillis); } /** @@ -98,8 +119,9 @@ private static CompletableFuture startRetryingForResources( QueryParams params, byte[] body, String contentType, + Timeout timeout, long deadlineMillis) { - return ctx.postWithBody("runs", params, body, contentType, ActorRun.class) + return postRun(ctx, params, body, contentType, timeout) .handle( (run, error) -> { if (error == null) { @@ -125,11 +147,20 @@ private static CompletableFuture startRetryingForResources( .thenCompose( unused -> startRetryingForResources( - ctx, params, body, contentType, deadlineMillis)); + ctx, params, body, contentType, timeout, deadlineMillis)); }) .thenCompose(future -> future); } + /** + * Posts the run-start request. A start that timed out is not retried: the API may already have + * created the run, and a retry would start a second, billed one. + */ + private static CompletableFuture postRun( + ResourceContext ctx, QueryParams params, byte[] body, String contentType, Timeout timeout) { + return ctx.postWithBody("runs", params, body, contentType, ActorRun.class, timeout, true); + } + /** * Starts a run and waits (non-blocking, polling) for it to finish, without log streaming. * @@ -144,7 +175,7 @@ public static CompletableFuture call( String contentType, Long waitSecs, Long waitForResourcesSecs) { - return start(ctx, input, applyParams, contentType, waitForResourcesSecs) + return start(ctx, input, applyParams, contentType, waitForResourcesSecs, Timeout.NO_TIMEOUT) .thenCompose(run -> root.run(run.getId()).waitForFinish(waitSecs)); } @@ -173,7 +204,7 @@ public static CompletableFuture callWithLogStreaming( StreamedLogOptions logOptions, Long waitSecs, Long waitForResourcesSecs) { - return start(ctx, input, applyParams, contentType, waitForResourcesSecs) + return start(ctx, input, applyParams, contentType, waitForResourcesSecs, Timeout.NO_TIMEOUT) .thenCompose( run -> { RunClient runClient = root.run(run.getId()); diff --git a/src/main/java/com/apify/client/internal/UrlNormalizer.java b/src/main/java/com/apify/client/internal/UrlNormalizer.java new file mode 100644 index 0000000..4671337 --- /dev/null +++ b/src/main/java/com/apify/client/internal/UrlNormalizer.java @@ -0,0 +1,162 @@ +package com.apify.client.internal; + +import java.net.IDN; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Normalizes absolute URLs the way the WHATWG URL serializer does, so every Apify client returns + * the same string for the same {@code format: uri} field of the API: lowercase scheme and host, + * punycode host, default port dropped, an empty path serialized as {@code /}, dot segments + * resolved, and unsafe characters percent-encoded. Internal to the client. + */ +public final class UrlNormalizer { + + /** + * {@code scheme://authority[path][?query][#fragment]}, the only shape the spec's URL fields have. + */ + private static final Pattern URL = + Pattern.compile( + "^([A-Za-z][A-Za-z0-9+.-]*)://([^/?#]*)([^?#]*)(\\?[^#]*)?(#.*)?$", Pattern.DOTALL); + + /** Schemes with a default port and a mandatory (possibly empty) path. */ + private static final Map SPECIAL_SCHEMES = + Map.of("http", 80, "https", 443, "ws", 80, "wss", 443, "ftp", 21); + + private UrlNormalizer() {} + + /** + * Returns the normalized form of {@code value}. A {@code null} value, and a value that is not an + * absolute {@code scheme://host...} URL, is returned unchanged: the client never fails to read a + * response because of one odd URL field. + */ + public static String normalize(String value) { + if (value == null) { + return null; + } + Matcher m = URL.matcher(value.strip()); + if (!m.matches()) { + return value; + } + String scheme = m.group(1).toLowerCase(Locale.ROOT); + Integer defaultPort = SPECIAL_SCHEMES.get(scheme); + boolean special = defaultPort != null; + + String authority = m.group(2); + String userInfo = ""; + int at = authority.lastIndexOf('@'); + if (at >= 0) { + userInfo = encode(authority.substring(0, at), USERINFO_UNSAFE) + "@"; + authority = authority.substring(at + 1); + } + String host = authority; + String port = ""; + int colon = authority.lastIndexOf(':'); + if (colon >= 0 && authority.indexOf(']') < colon) { + host = authority.substring(0, colon); + port = authority.substring(colon + 1); + } + if (special && host.isEmpty()) { + return value; + } + host = normalizeHost(host); + if (host == null || !port.matches("\\d*")) { + return value; + } + if (!port.isEmpty() && defaultPort != null && Integer.parseInt(port) == defaultPort) { + port = ""; + } + if (!port.isEmpty()) { + port = ":" + Integer.parseInt(port); + } + + String rawPath = special ? m.group(3).replace('\\', '/') : m.group(3); + String path = + special ? resolveDotSegments(encode(rawPath, PATH_UNSAFE)) : encode(rawPath, PATH_UNSAFE); + if (special && path.isEmpty()) { + path = "/"; + } + String query = + m.group(4) == null ? "" : encode(m.group(4), special ? SPECIAL_QUERY_UNSAFE : QUERY_UNSAFE); + String fragment = m.group(5) == null ? "" : encode(m.group(5), FRAGMENT_UNSAFE); + return scheme + "://" + userInfo + host + port + path + query + fragment; + } + + /** + * Lowercases the host and converts a non-ASCII one to punycode; {@code null} if it is invalid. + */ + private static String normalizeHost(String host) { + if (host.startsWith("[")) { + return host.toLowerCase(Locale.ROOT); + } + try { + return IDN.toASCII(host, IDN.ALLOW_UNASSIGNED).toLowerCase(Locale.ROOT); + } catch (IllegalArgumentException e) { + return null; + } + } + + /** Resolves {@code .} and {@code ..} segments of an already-encoded path. */ + private static String resolveDotSegments(String path) { + if (path.isEmpty()) { + return path; + } + List out = new ArrayList<>(); + String[] segments = path.substring(1).split("/", -1); + for (int i = 0; i < segments.length; i++) { + String seg = segments[i]; + boolean last = i == segments.length - 1; + String lower = seg.toLowerCase(Locale.ROOT); + boolean dotDot = + lower.equals("..") + || lower.equals(".%2e") + || lower.equals("%2e.") + || lower.equals("%2e%2e"); + boolean dot = lower.equals(".") || lower.equals("%2e"); + if (dotDot) { + if (!out.isEmpty()) { + out.remove(out.size() - 1); + } + if (last) { + out.add(""); + } + } else if (dot) { + if (last) { + out.add(""); + } + } else { + out.add(seg); + } + } + return "/" + String.join("/", out); + } + + private static final String USERINFO_UNSAFE = " \"#<>?`{}/:;=@[\\]^|"; + private static final String PATH_UNSAFE = " \"#<>?`{}"; + private static final String QUERY_UNSAFE = " \"#<>"; + private static final String SPECIAL_QUERY_UNSAFE = " \"#<>'"; + private static final String FRAGMENT_UNSAFE = " \"<>`"; + + /** + * Percent-encodes the ASCII characters in {@code unsafe}, control characters and every non-ASCII + * character (as UTF-8). An existing {@code %XX} escape is kept as it is. + */ + private static String encode(String s, String unsafe) { + StringBuilder sb = new StringBuilder(s.length()); + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + for (byte b : bytes) { + int c = b & 0xFF; + if (c < 0x20 || c >= 0x7F || unsafe.indexOf(c) >= 0) { + sb.append('%').append(String.format("%02X", c)); + } else { + sb.append((char) c); + } + } + return sb.toString(); + } +} diff --git a/src/main/java/com/apify/client/keyvalue/KeyValueStore.java b/src/main/java/com/apify/client/keyvalue/KeyValueStore.java index e3202ea..73555b9 100644 --- a/src/main/java/com/apify/client/keyvalue/KeyValueStore.java +++ b/src/main/java/com/apify/client/keyvalue/KeyValueStore.java @@ -1,7 +1,9 @@ package com.apify.client.keyvalue; import com.apify.client.ApifyResource; +import com.apify.client.internal.NormalizedUrlDeserializer; import java.time.Instant; +import tools.jackson.databind.annotation.JsonDeserialize; /** A key-value store holds arbitrary data records. */ public final class KeyValueStore extends ApifyResource { @@ -10,6 +12,16 @@ public final class KeyValueStore extends ApifyResource { private String userId; private Instant createdAt; private Instant modifiedAt; + private String title; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) + private String consoleUrl; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) + private String keysPublicUrl; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) + private String recordsPublicUrl; /** The unique store ID. */ public String getId() { @@ -35,4 +47,29 @@ public Instant getCreatedAt() { public Instant getModifiedAt() { return modifiedAt; } + + /** The human-readable title of the store. */ + public String getTitle() { + return title; + } + + /** The URL of the store in the Apify Console (normalized). */ + public String getConsoleUrl() { + return consoleUrl; + } + + /** + * A public URL of the store keys (signed when the store is not publicly accessible; normalized). + */ + public String getKeysPublicUrl() { + return keysPublicUrl; + } + + /** + * A public base URL of the store records (signed when the store is not publicly accessible; + * normalized). + */ + public String getRecordsPublicUrl() { + return recordsPublicUrl; + } } diff --git a/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java b/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java index 4b7e3fe..d222e3b 100644 --- a/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java +++ b/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java @@ -1,5 +1,6 @@ package com.apify.client.keyvalue; +import com.apify.client.http.Timeout; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.Extras; import com.apify.client.internal.HttpClientCore; @@ -8,7 +9,6 @@ import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; import com.apify.client.internal.Signatures; -import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Optional; @@ -65,7 +65,7 @@ public CompletableFuture> get() { /** Updates the store metadata (e.g. name) and returns the updated object. */ public CompletableFuture update(Object newFields) { - return ctx.updateResource("", newFields, KeyValueStore.class); + return ctx.updateResource("", newFields, KeyValueStore.class, Timeout.LONG); } /** @@ -80,7 +80,7 @@ public CompletableFuture delete() { public CompletableFuture listKeys(ListKeysOptions options) { QueryParams params = new QueryParams(); options.apply(params); - return ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class); + return ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class, Timeout.MEDIUM); } /** @@ -228,7 +228,7 @@ private void fetchPage() { params.addLong("limit", pageLimit); params.addString("exclusiveStartKey", cursor); params.extend(filters); - ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class) + ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class, Timeout.MEDIUM) .whenComplete( (page, error) -> { if (cancelled.get()) { @@ -275,7 +275,8 @@ private void applyPage(KeyValueStoreKeysPage page) { /** Reports whether a record with the given key exists. */ public CompletableFuture recordExists(String key) { - return ctx.headExists("records/" + ResourceContext.encodePathSegment(key), new QueryParams()); + return ctx.headExists( + "records/" + ResourceContext.encodePathSegment(key), new QueryParams(), Timeout.LONG); } /** @@ -291,7 +292,7 @@ public CompletableFuture> getRecord( String key, GetRecordOptions options) { QueryParams params = new QueryParams(); options.apply(params); - return ctx.getRaw("records/" + ResourceContext.encodePathSegment(key), params) + return ctx.getRaw("records/" + ResourceContext.encodePathSegment(key), params, Timeout.LONG) .thenApply( resp -> { if (resp == null) { @@ -326,10 +327,7 @@ public CompletableFuture setRecord( } Map extraHeaders = contentEncoding == null ? null : Map.of(HttpHeaders.CONTENT_ENCODING, contentEncoding); - Duration timeout = - options.timeoutSecsValue() != null - ? Duration.ofSeconds(options.timeoutSecsValue()) - : ctx.http.baseRequestTimeout(); + Timeout timeout = options.timeoutValue() != null ? options.timeoutValue() : Timeout.LONG; return ctx.putRaw( "records/" + ResourceContext.encodePathSegment(key), new QueryParams(), diff --git a/src/main/java/com/apify/client/keyvalue/KeyValueStoreKey.java b/src/main/java/com/apify/client/keyvalue/KeyValueStoreKey.java index 6b57461..38a691d 100644 --- a/src/main/java/com/apify/client/keyvalue/KeyValueStoreKey.java +++ b/src/main/java/com/apify/client/keyvalue/KeyValueStoreKey.java @@ -1,12 +1,17 @@ package com.apify.client.keyvalue; import com.apify.client.ApifyResource; +import com.apify.client.internal.NormalizedUrlDeserializer; +import tools.jackson.databind.annotation.JsonDeserialize; /** A single key listed from a key-value store. */ public final class KeyValueStoreKey extends ApifyResource { private String key; private long size; + @JsonDeserialize(using = NormalizedUrlDeserializer.class) + private String recordPublicUrl; + /** The record key. */ public String getKey() { return key; @@ -16,4 +21,9 @@ public String getKey() { public long getSize() { return size; } + + /** A public URL of the record (signed when the store is not publicly accessible; normalized). */ + public String getRecordPublicUrl() { + return recordPublicUrl; + } } diff --git a/src/main/java/com/apify/client/keyvalue/SetRecordOptions.java b/src/main/java/com/apify/client/keyvalue/SetRecordOptions.java index 24891cd..3017815 100644 --- a/src/main/java/com/apify/client/keyvalue/SetRecordOptions.java +++ b/src/main/java/com/apify/client/keyvalue/SetRecordOptions.java @@ -1,22 +1,34 @@ package com.apify.client.keyvalue; +import com.apify.client.http.Timeout; + /** * Write options for {@link KeyValueStoreClient#setRecord(String, byte[], String, * SetRecordOptions)}, mirroring the reference client's {@code timeoutSecs}/{@code * doNotRetryTimeouts}/{@code contentEncoding}. */ public final class SetRecordOptions { - private Long timeoutSecs; + private Timeout timeout; private Boolean doNotRetryTimeouts; private String contentEncoding; /** - * Per-request timeout for the upload, in seconds. Use it to shorten the wait for this upload; - * defaults to (and is capped at) the client's configured overall request timeout, so a value - * larger than that timeout has no effect. + * Per-request timeout for the upload, in seconds. Shorthand for {@link #timeout(Timeout) + * timeout(Timeout.ofSeconds(timeoutSecs))}; a {@code null} or non-positive value leaves the + * default. */ public SetRecordOptions timeoutSecs(Long timeoutSecs) { - this.timeoutSecs = timeoutSecs; + this.timeout = timeoutSecs == null || timeoutSecs <= 0 ? null : Timeout.ofSeconds(timeoutSecs); + return this; + } + + /** + * Timeout of the upload request: a tier, an exact duration or {@link Timeout#NO_TIMEOUT}. + * Defaults to {@link Timeout#LONG}. A value above the client's {@code timeoutMax} is capped at + * it. + */ + public SetRecordOptions timeout(Timeout timeout) { + this.timeout = timeout; return this; } @@ -46,8 +58,8 @@ public SetRecordOptions contentEncoding(String contentEncoding) { return this; } - Long timeoutSecsValue() { - return timeoutSecs; + Timeout timeoutValue() { + return timeout; } boolean doNotRetryTimeoutsValue() { diff --git a/src/main/java/com/apify/client/log/LogClient.java b/src/main/java/com/apify/client/log/LogClient.java index 7d5edae..e8e0def 100644 --- a/src/main/java/com/apify/client/log/LogClient.java +++ b/src/main/java/com/apify/client/log/LogClient.java @@ -1,5 +1,6 @@ package com.apify.client.log; +import com.apify.client.http.Timeout; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.QueryParams; @@ -49,7 +50,7 @@ public CompletableFuture> get() { public CompletableFuture> get(LogOptions options) { QueryParams params = new QueryParams(); options.apply(params); - return ctx.getRawUnlessAmbiguous("", params) + return ctx.getRawUnlessAmbiguous("", params, Timeout.LONG) .thenApply( resp -> resp == null @@ -71,7 +72,7 @@ public CompletableFuture> getAsBytes() { public CompletableFuture> getAsBytes(LogOptions options) { QueryParams params = new QueryParams(); options.apply(params); - return ctx.getRawUnlessAmbiguous("", params) + return ctx.getRawUnlessAmbiguous("", params, Timeout.LONG) .thenApply(resp -> resp == null ? Optional.empty() : Optional.of(resp.body())); } diff --git a/src/main/java/com/apify/client/requestqueue/RequestQueue.java b/src/main/java/com/apify/client/requestqueue/RequestQueue.java index d52c1eb..c380ab0 100644 --- a/src/main/java/com/apify/client/requestqueue/RequestQueue.java +++ b/src/main/java/com/apify/client/requestqueue/RequestQueue.java @@ -1,7 +1,9 @@ package com.apify.client.requestqueue; import com.apify.client.ApifyResource; +import com.apify.client.internal.NormalizedUrlDeserializer; import java.time.Instant; +import tools.jackson.databind.annotation.JsonDeserialize; /** A request queue stores URLs to be crawled. */ public final class RequestQueue extends ApifyResource { @@ -11,6 +13,10 @@ public final class RequestQueue extends ApifyResource { private Instant createdAt; private Instant modifiedAt; private long totalRequestCount; + private String title; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) + private String consoleUrl; /** The unique queue ID. */ public String getId() { @@ -41,4 +47,14 @@ public Instant getModifiedAt() { public long getTotalRequestCount() { return totalRequestCount; } + + /** The human-readable title of the queue. */ + public String getTitle() { + return title; + } + + /** The URL of the queue in the Apify Console (normalized). */ + public String getConsoleUrl() { + return consoleUrl; + } } diff --git a/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java b/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java index 2363e01..0593932 100644 --- a/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java +++ b/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java @@ -2,12 +2,14 @@ import com.apify.client.http.ApifyApiException; import com.apify.client.http.ApifyTransportException; +import com.apify.client.http.Timeout; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.Async; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; +import java.io.ByteArrayOutputStream; import java.time.Duration; import java.util.ArrayList; import java.util.HashSet; @@ -126,7 +128,7 @@ public CompletableFuture listHead(Long limit) { QueryParams params = new QueryParams(); params.addLong("limit", limit); withClientKey(params); - return ctx.getResourceRequired("head", params, RequestQueueHead.class); + return ctx.getResourceRequired("head", params, RequestQueueHead.class, Timeout.SHORT); } /** Adds a request to the queue. If {@code forefront} is true, it is added to the front. */ @@ -140,7 +142,8 @@ public CompletableFuture addRequest( params, Json.toBytes(request), ResourceContext.CONTENT_TYPE_JSON, - RequestQueueOperationInfo.class); + RequestQueueOperationInfo.class, + Timeout.SHORT); } /** Fetches a request by ID, or empty if it does not exist. */ @@ -148,7 +151,8 @@ public CompletableFuture> getRequest(String id) { return ctx.getResource( "requests/" + ResourceContext.encodePathSegment(id), new QueryParams(), - RequestQueueRequest.class); + RequestQueueRequest.class, + Timeout.SHORT); } /** @@ -165,7 +169,8 @@ public CompletableFuture updateRequest( params, Json.toBytes(request), ResourceContext.CONTENT_TYPE_JSON, - RequestQueueOperationInfo.class); + RequestQueueOperationInfo.class, + Timeout.MEDIUM); } /** Deletes a request by ID. */ @@ -174,7 +179,7 @@ public CompletableFuture deleteRequest(String id) { String url = ctx.mergedParams(params) .applyToUrl(ctx.subUrl("requests/" + ResourceContext.encodePathSegment(id))); - return http.call("DELETE", url, null, "", http.baseRequestTimeout()) + return http.call("DELETE", url, null, "", Timeout.SHORT) .handle( (resp, error) -> { if (error == null) { @@ -199,7 +204,8 @@ public CompletableFuture listAndLockHead(long lockSecs, QueryParams params = new QueryParams(); params.addLong("lockSecs", lockSecs).addLong("limit", limit); withClientKey(params); - return ctx.postWithBody("head/lock", params, null, "", LockedRequestQueueHead.class); + return ctx.postWithBody( + "head/lock", params, null, "", LockedRequestQueueHead.class, Timeout.MEDIUM); } /** @@ -245,11 +251,18 @@ public CompletableFuture batchAddRequests( MAX_PAYLOAD_SIZE_BYTES - (long) Math.ceil(MAX_PAYLOAD_SIZE_BYTES * PAYLOAD_SAFETY_BUFFER_PERCENT); - List> chunks = new ArrayList<>(); - for (int start = 0; start < requests.size(); ) { - int end = Math.min(start + MAX_REQUESTS_PER_BATCH, requests.size()); - List countSlice = requests.subList(start, end); - List chunk = sliceByByteLength(countSlice, payloadSizeLimitBytes, start); + // Serialize every request exactly once: the byte lengths decide the chunk boundaries and the + // same bytes are joined into each chunk's request body (and into the body of a retry). + List entries = new ArrayList<>(requests.size()); + for (RequestQueueRequest request : requests) { + entries.add(new SerializedRequest(request, Json.toBytes(request))); + } + + List> chunks = new ArrayList<>(); + for (int start = 0; start < entries.size(); ) { + int end = Math.min(start + MAX_REQUESTS_PER_BATCH, entries.size()); + List countSlice = entries.subList(start, end); + List chunk = sliceByByteLength(countSlice, payloadSizeLimitBytes, start); chunks.add(new ArrayList<>(chunk)); start += chunk.size(); } @@ -286,7 +299,7 @@ public CompletableFuture batchAddRequests( * worker is a chain of async continuations, not a blocked thread. */ private CompletableFuture runBatchAddWorker( - List> chunks, + List> chunks, AtomicInteger nextChunkIndex, BatchAddResult[] resultsByChunk, boolean forefront, @@ -313,17 +326,17 @@ private CompletableFuture runBatchAddWorker( * @throws IllegalArgumentException if any individual request's own JSON encoding already exceeds * {@code maxByteLength} (no chunk size could ever fit it) */ - private static List sliceByByteLength( - List requests, long maxByteLength, int startIndex) { - if (jsonByteLength(requests) < maxByteLength) { + private static List sliceByByteLength( + List requests, long maxByteLength, int startIndex) { + if (jsonArrayByteLength(requests) < maxByteLength) { return requests; } - List sliced = new ArrayList<>(); + List sliced = new ArrayList<>(); long byteLength = EMPTY_JSON_ARRAY_BYTES; for (int i = 0; i < requests.size(); i++) { - RequestQueueRequest request = requests.get(i); - long itemBytes = jsonByteLength(request); + SerializedRequest request = requests.get(i); + long itemBytes = request.json().length; if (itemBytes > maxByteLength) { throw new IllegalArgumentException( "RequestQueueClient.batchAddRequests: the request at index " @@ -334,8 +347,8 @@ private static List sliceByByteLength( } // Every element but the first is preceded by a comma in the serialized array (`[a,b,c]` has // count-1 commas); account for it here so this incremental estimate agrees exactly with - // jsonByteLength(requests)'s full-array measurement above, rather than relying on being - // slightly under it. + // jsonArrayByteLength's full-array measurement above, rather than relying on being slightly + // under it. long commaBytes = sliced.isEmpty() ? 0 : 1; if (byteLength + commaBytes + itemBytes >= maxByteLength) { break; @@ -352,9 +365,30 @@ private static List sliceByByteLength( return sliced; } - /** The byte length of {@code value}'s JSON encoding, as sent on the wire. */ - private static long jsonByteLength(Object value) { - return Json.toBytes(value).length; + /** A request together with its JSON encoding, so it is serialized only once. */ + private record SerializedRequest(RequestQueueRequest request, byte[] json) {} + + /** The byte length of the JSON array that joins the given requests' encodings. */ + private static long jsonArrayByteLength(List requests) { + long length = EMPTY_JSON_ARRAY_BYTES; + for (SerializedRequest request : requests) { + length += request.json().length; + } + return requests.isEmpty() ? length : length + requests.size() - 1; + } + + /** The JSON array of the given requests' encodings, as sent in a batch request body. */ + private static byte[] jsonArray(List requests) { + ByteArrayOutputStream out = new ByteArrayOutputStream((int) jsonArrayByteLength(requests)); + out.write('['); + for (int i = 0; i < requests.size(); i++) { + if (i > 0) { + out.write(','); + } + out.writeBytes(requests.get(i).json()); + } + out.write(']'); + return out.toByteArray(); } /** @@ -368,7 +402,7 @@ private static long jsonByteLength(Object value) { * ApifyApiException} and {@link com.apify.client.http.ApifyTransportException}. */ private CompletableFuture batchAddChunkWithRetries( - List chunk, boolean forefront, BatchAddRequestsOptions options) { + List chunk, boolean forefront, BatchAddRequestsOptions options) { int maxRetries = options.maxUnprocessedRequestsRetriesValue(); long minDelayMillis = options.minDelayBetweenUnprocessedRequestsRetriesMillisValue(); return batchAddAttempt(chunk, chunk, List.of(), forefront, 0, maxRetries, minDelayMillis); @@ -376,8 +410,8 @@ private CompletableFuture batchAddChunkWithRetries( /** One attempt of the retry loop backing {@link #batchAddChunkWithRetries}. */ private CompletableFuture batchAddAttempt( - List chunk, - List toSend, + List chunk, + List toSend, List processedSoFar, boolean forefront, int attempt, @@ -404,7 +438,7 @@ private CompletableFuture batchAddAttempt( if (response != null) { processed.addAll(response.getProcessedRequests()); } - List stillRemaining = requestsNotYetProcessed(chunk, processed); + List stillRemaining = requestsNotYetProcessed(chunk, processed); // Any client-level failure (response == null, i.e. the batchAddChunk call above // failed) stops retrying this chunk immediately; whatever has not been confirmed // processed is reported as unprocessed below, never thrown - matching the reference @@ -429,27 +463,30 @@ private CompletableFuture batchAddAttempt( } private static BatchAddResult finish( - List chunk, List processed) { + List chunk, List processed) { BatchAddResult result = new BatchAddResult(); result.setProcessedRequests(processed); // Unprocessed = everything sent minus everything acknowledged processed. Computing it here // (instead of trusting the last response) stays correct even if the API returns fewer entries. - result.setUnprocessedRequests(requestsNotYetProcessed(chunk, processed)); + result.setUnprocessedRequests( + requestsNotYetProcessed(chunk, processed).stream() + .map(SerializedRequest::request) + .toList()); return result; } - private static List requestsNotYetProcessed( - List chunk, List processed) { + private static List requestsNotYetProcessed( + List chunk, List processed) { Set processedKeys = new HashSet<>(); for (RequestQueueOperationInfo info : processed) { if (info.getUniqueKey() != null) { processedKeys.add(info.getUniqueKey()); } } - List remaining = new ArrayList<>(); - for (RequestQueueRequest request : chunk) { - if (!processedKeys.contains(request.getUniqueKey())) { - remaining.add(request); + List remaining = new ArrayList<>(); + for (SerializedRequest entry : chunk) { + if (!processedKeys.contains(entry.request().getUniqueKey())) { + remaining.add(entry); } } return remaining; @@ -470,16 +507,17 @@ private static Duration backoffDelay(int attempt, long minDelayMillis) { } private CompletableFuture batchAddChunk( - List requests, boolean forefront) { + List requests, boolean forefront) { QueryParams params = new QueryParams(); params.addBool("forefront", forefront); withClientKey(params); return ctx.postWithBody( "requests/batch", params, - Json.toBytes(requests), + jsonArray(requests), ResourceContext.CONTENT_TYPE_JSON, - Json.type(BatchAddResult.class)); + Json.type(BatchAddResult.class), + Timeout.MEDIUM); } /** @@ -488,7 +526,8 @@ private CompletableFuture batchAddChunk( */ public CompletableFuture batchDeleteRequests(Object requests) { QueryParams params = withClientKey(new QueryParams()); - return ctx.deleteWithBody("requests/batch", params, requests, BatchDeleteResult.class); + return ctx.deleteWithBody( + "requests/batch", params, requests, BatchDeleteResult.class, Timeout.SHORT); } /** Lists the queue's requests with pagination. */ @@ -497,7 +536,7 @@ public CompletableFuture listRequests(ListRequestsOptions options) QueryParams params = new QueryParams(); options.apply(params); withClientKey(params); - return ctx.getResourceRequired("requests", params, RequestsList.class); + return ctx.getResourceRequired("requests", params, RequestsList.class, Timeout.MEDIUM); } /** @@ -514,7 +553,8 @@ public CompletableFuture prolongRequestLock( params, null, "", - RequestLockInfo.class); + RequestLockInfo.class, + Timeout.MEDIUM); } /** @@ -528,7 +568,7 @@ public CompletableFuture deleteRequestLock(String id, boolean forefront) { String url = ctx.mergedParams(params) .applyToUrl(ctx.subUrl("requests/" + ResourceContext.encodePathSegment(id) + "/lock")); - return http.call("DELETE", url, null, "", http.baseRequestTimeout()) + return http.call("DELETE", url, null, "", Timeout.SHORT) .handle( (resp, error) -> { if (error == null) { @@ -546,7 +586,8 @@ public CompletableFuture deleteRequestLock(String id, boolean forefront) { /** Releases all locks the client holds on this queue's requests. */ public CompletableFuture unlockRequests() { QueryParams params = withClientKey(new QueryParams()); - return ctx.postWithBody("requests/unlock", params, null, "", UnlockRequestsResult.class); + return ctx.postWithBody( + "requests/unlock", params, null, "", UnlockRequestsResult.class, Timeout.LONG); } /** @@ -705,7 +746,7 @@ private void fetchPage() { } params.addCsv("filter", filter); withClientKey(params); - ctx.getResourceRequired("requests", params, RequestsList.class) + ctx.getResourceRequired("requests", params, RequestsList.class, Timeout.MEDIUM) .whenComplete( (page, error) -> { if (cancelled.get()) { diff --git a/src/main/java/com/apify/client/run/ActorRun.java b/src/main/java/com/apify/client/run/ActorRun.java index 059ddd7..194de9c 100644 --- a/src/main/java/com/apify/client/run/ActorRun.java +++ b/src/main/java/com/apify/client/run/ActorRun.java @@ -1,11 +1,13 @@ package com.apify.client.run; import com.apify.client.ApifyResource; +import com.apify.client.internal.NormalizedUrlDeserializer; import com.apify.client.internal.Statuses; import java.time.Instant; import java.util.Collections; import java.util.Map; import tools.jackson.databind.JsonNode; +import tools.jackson.databind.annotation.JsonDeserialize; /** A single execution of an Actor. */ public final class ActorRun extends ApifyResource { @@ -21,7 +23,10 @@ public final class ActorRun extends ApifyResource { private String defaultDatasetId; private String defaultKeyValueStoreId; private String defaultRequestQueueId; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) private String containerUrl; + private String generalAccess; private Map chargedEventCounts; private JsonNode pricingInfo; @@ -36,6 +41,18 @@ public final class ActorRun extends ApifyResource { private Boolean isContainerServerReady; private String gitBranchName; private JsonNode storageIds; + private ActorRunStandby standby; + private Map accountedChargedEventCounts; + private Map eventUsage; + private Double externalChargedTotalUsd; + private JsonNode integrationTracking; + private String createdByOrganizationMemberUserId; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) + private String consoleUrl; + + @JsonDeserialize(contentUsing = NormalizedUrlDeserializer.class) + private Map output; /** The unique run ID. */ public String getId() { @@ -199,4 +216,49 @@ public JsonNode getStorageIds() { public boolean isTerminal() { return Statuses.isTerminal(status); } + + /** + * The Server Actor deployment that served this run; {@code null} for runs not started by a Server + * Actor request. + */ + public ActorRunStandby getStandby() { + return standby; + } + + /** The charged event counts already accounted for in billing. */ + public Map getAccountedChargedEventCounts() { + return accountedChargedEventCounts == null + ? null + : Collections.unmodifiableMap(accountedChargedEventCounts); + } + + /** Usage of each pay-per-event event, keyed by event name. */ + public Map getEventUsage() { + return eventUsage == null ? null : Collections.unmodifiableMap(eventUsage); + } + + /** The total USD charged for the run outside the platform billing. */ + public Double getExternalChargedTotalUsd() { + return externalChargedTotalUsd; + } + + /** Tracking of the third-party integration that started the run, if any. */ + public JsonNode getIntegrationTracking() { + return integrationTracking; + } + + /** The organization member who started the run, when started on behalf of an organization. */ + public String getCreatedByOrganizationMemberUserId() { + return createdByOrganizationMemberUserId; + } + + /** The URL of the run in the Apify Console. */ + public String getConsoleUrl() { + return consoleUrl; + } + + /** URLs of the run output, keyed by output name. */ + public Map getOutput() { + return output == null ? null : Collections.unmodifiableMap(output); + } } diff --git a/src/main/java/com/apify/client/run/ActorRunMeta.java b/src/main/java/com/apify/client/run/ActorRunMeta.java index be5181f..e49b721 100644 --- a/src/main/java/com/apify/client/run/ActorRunMeta.java +++ b/src/main/java/com/apify/client/run/ActorRunMeta.java @@ -10,6 +10,10 @@ public final class ActorRunMeta extends ApifyResource { private String userAgent; private String scheduleId; private Instant scheduledAt; + private String actorRunId; + private String webhookDispatchId; + private String scheduledAct2Id; + private String scheduledActorTaskId; /** What triggered the run (e.g. {@code "WEB"}, {@code "API"}, {@code "SCHEDULER"}). */ public String getOrigin() { @@ -35,4 +39,24 @@ public String getScheduleId() { public Instant getScheduledAt() { return scheduledAt; } + + /** The ID of the Actor run that started this run, if it was started by another run. */ + public String getActorRunId() { + return actorRunId; + } + + /** The ID of the webhook dispatch that started this run, if any. */ + public String getWebhookDispatchId() { + return webhookDispatchId; + } + + /** The ID of the scheduled Actor that started this run, if any. */ + public String getScheduledAct2Id() { + return scheduledAct2Id; + } + + /** The ID of the scheduled Actor task that started this run, if any. */ + public String getScheduledActorTaskId() { + return scheduledActorTaskId; + } } diff --git a/src/main/java/com/apify/client/run/ActorRunOptions.java b/src/main/java/com/apify/client/run/ActorRunOptions.java index 1a9b504..6e7de39 100644 --- a/src/main/java/com/apify/client/run/ActorRunOptions.java +++ b/src/main/java/com/apify/client/run/ActorRunOptions.java @@ -15,6 +15,7 @@ public final class ActorRunOptions extends ApifyResource { private Long maxItems; private Double maxTotalChargeUsd; private Boolean restartOnError; + private Boolean isMaxTotalChargeUsdSetByUser; /** The tag or number of the build that was run. */ public String getBuild() { @@ -50,4 +51,9 @@ public Double getMaxTotalChargeUsd() { public Boolean getRestartOnError() { return restartOnError; } + + /** Whether {@link #getMaxTotalChargeUsd()} was set by the user rather than defaulted. */ + public Boolean getIsMaxTotalChargeUsdSetByUser() { + return isMaxTotalChargeUsdSetByUser; + } } diff --git a/src/main/java/com/apify/client/run/ActorRunStandby.java b/src/main/java/com/apify/client/run/ActorRunStandby.java new file mode 100644 index 0000000..594baed --- /dev/null +++ b/src/main/java/com/apify/client/run/ActorRunStandby.java @@ -0,0 +1,18 @@ +package com.apify.client.run; + +import com.apify.client.ApifyResource; + +/** The Server Actor (Actor Standby) deployment that served an {@link ActorRun}. */ +public final class ActorRunStandby extends ApifyResource { + private String deployment; + + /** + * Which pool the run serves: {@code SINGLE_TENANT} is a private instance in the caller's account, + * while the {@code MULTI_TENANT_FREE_USERS}, {@code MULTI_TENANT_PAYING_USERS} and {@code + * MULTI_TENANT_OWNER_PAID} pools run in the Actor owner's account and serve many callers (charges + * in them require a {@link RunChargeOptions#requestId(String) request ID}). + */ + public String getDeployment() { + return deployment; + } +} diff --git a/src/main/java/com/apify/client/run/RunClient.java b/src/main/java/com/apify/client/run/RunClient.java index 2236931..36b589f 100644 --- a/src/main/java/com/apify/client/run/RunClient.java +++ b/src/main/java/com/apify/client/run/RunClient.java @@ -2,6 +2,9 @@ import com.apify.client.ApifyClient; import com.apify.client.dataset.DatasetClient; +import com.apify.client.dataset.DatasetListItemsOptions; +import com.apify.client.http.Timeout; +import com.apify.client.internal.AsyncPaginatedPublisher; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; import com.apify.client.internal.QueryParams; @@ -15,10 +18,12 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import tools.jackson.databind.JsonNode; /** * A client for a specific Actor run. @@ -81,15 +86,15 @@ public CompletableFuture> get() { /** * Fetches the run, optionally asking the API to wait up to {@code waitForFinishSecs} seconds for - * the run to reach a terminal state before responding. The value is clamped so the server always - * responds before the client's per-request timeout; the API itself caps server-side waiting at 60 - * seconds. Pass {@code null} for an immediate fetch. + * the run to reach a terminal state before responding; the API itself caps server-side waiting at + * 60 seconds, and the request timeout is extended by that hold. Pass {@code null} for an + * immediate fetch. */ public CompletableFuture> getWithWait(Long waitForFinishSecs) { QueryParams params = new QueryParams(); - // Clamp to the client's per-request timeout so a short custom timeout doesn't abort the call. - params.addLong("waitForFinish", ctx.clampServerWait(waitForFinishSecs)); - return ctx.getResource("", params, ActorRun.class); + params.addLong("waitForFinish", waitForFinishSecs); + return ctx.getResource( + "", params, ActorRun.class, ctx.http.timeoutWithHold(Timeout.SHORT, waitForFinishSecs)); } /** Updates the run with the given fields and returns the updated object. */ @@ -110,14 +115,15 @@ public CompletableFuture delete() { public CompletableFuture abort(Boolean gracefully) { QueryParams params = new QueryParams(); params.addBool("gracefully", gracefully); - return ctx.postWithBody("abort", params, null, "", ActorRun.class); + return ctx.postWithBody("abort", params, null, "", ActorRun.class, Timeout.MEDIUM); } /** * Transforms the run into a run of another Actor with a new input. {@code targetActorId} is the * Actor to metamorph into ({@code username/actor-name} or the plain Actor id; normalized to the * URL-safe {@code username~actor-name} form before sending, matching the reference client's - * {@code _toSafeId}); {@code input} is the new input ({@code null} for none). + * {@code _toSafeId}); {@code input} is the new input: a JSON-serializable value, a {@code byte[]} + * or {@link String} that is sent as it is, or {@code null} for none. */ public CompletableFuture metamorph( String targetActorId, Object input, MetamorphOptions options) { @@ -132,14 +138,14 @@ public CompletableFuture metamorph( if (options.buildValue() != null && !options.buildValue().isEmpty()) { params.addString("build", options.buildValue()); } - byte[] body = input == null ? null : Json.toBytes(input); + byte[] body = Json.toRequestBody(input); return ctx.postWithBody( - "metamorph", params, body, options.contentTypeOrDefault(), ActorRun.class); + "metamorph", params, body, options.contentTypeOrDefault(), ActorRun.class, Timeout.MEDIUM); } /** Reboots the run (restarts its container while keeping the same run). */ public CompletableFuture reboot() { - return ctx.postWithBody("reboot", new QueryParams(), null, "", ActorRun.class); + return ctx.postWithBody("reboot", new QueryParams(), null, "", ActorRun.class, Timeout.MEDIUM); } /** @@ -152,7 +158,7 @@ public CompletableFuture resurrect(RunResurrectOptions options) { } QueryParams params = new QueryParams(); options.apply(params); - return ctx.postWithBody("resurrect", params, null, "", ActorRun.class); + return ctx.postWithBody("resurrect", params, null, "", ActorRun.class, Timeout.MEDIUM); } /** @@ -190,7 +196,7 @@ public CompletableFuture charge(RunChargeOptions options) { Json.toBytes(body), ResourceContext.CONTENT_TYPE_JSON, headers, - ctx.http.baseRequestTimeout()) + Timeout.SHORT) .thenApply(resp -> null); } @@ -225,6 +231,61 @@ public DatasetClient dataset() { return DatasetClient.nested(ctx.http, ctx.subUrl(""), "dataset", ctx.baseParams); } + /** Default number of dataset rows requested per API call by {@link #iterateDatasetItems}. */ + private static final long DEFAULT_ITERATE_CHUNK_SIZE = 1000; + + /** Default number of seconds {@link #iterateDatasetItems} waits for the run between polls. */ + private static final long DEFAULT_ITERATE_POLL_INTERVAL_SECS = 5; + + /** + * Iterates over the items of the run's default dataset while the run is still producing them, + * decoding each into a generic {@link JsonNode}. See {@link #iterateDatasetItems( + * DatasetListItemsOptions, Long, Long, Class)}. + */ + public Flow.Publisher iterateDatasetItems(DatasetListItemsOptions options) { + return iterateDatasetItems(options, null, null, JsonNode.class); + } + + /** + * Iterates over the items of the run's default dataset while the run is still producing them, + * decoding each item into {@code itemClass}: a lazy, backpressure-aware publisher that ends once + * the run has finished and every item has been read. + * + *

While the run has not finished, each poll yields the rows below the dataset's {@code + * itemCount} and then waits up to {@code pollIntervalSecs} for the run to finish, so the last + * rows are read as soon as it does. Once the run is terminal, the rows past the (lagging) {@code + * itemCount} are read a page at a time until none are left. On a client from {@code lastRun()}, + * the iteration sticks to the run that its first request resolves to. + * + * @param options the items options: {@code offset} (rows to skip) and {@code limit} (the maximum + * number of dataset rows to scan: fewer items are yielded when filters drop some, + * more when {@code unwind} splits a row into several), and the item filters ({@code clean}, + * {@code fields}, {@code omit}, {@code unwind}, {@code skipEmpty}, {@code skipHidden}, ...). + * {@code desc} is ignored, since rows are read in the order they were pushed. May be {@code + * null}. + * @param chunkSize the maximum number of dataset rows requested per API call; {@code null} means + * 1000 + * @param pollIntervalSecs how long to wait for the run to finish between polls, in seconds; + * {@code null} means 5 + * @param itemClass the type each item is decoded into + */ + public Flow.Publisher iterateDatasetItems( + DatasetListItemsOptions options, Long chunkSize, Long pollIntervalSecs, Class itemClass) { + DatasetListItemsOptions opts = options != null ? options.copy() : new DatasetListItemsOptions(); + long pageSize = chunkSize != null && chunkSize > 0 ? chunkSize : DEFAULT_ITERATE_CHUNK_SIZE; + long poll = + pollIntervalSecs != null && pollIntervalSecs >= 0 + ? pollIntervalSecs + : DEFAULT_ITERATE_POLL_INTERVAL_SECS; + RunDatasetItems items = new RunDatasetItems<>(this, opts, pageSize, poll, itemClass); + return new AsyncPaginatedPublisher<>(null, null, null, (offset, limit) -> items.next()); + } + + /** The client for {@code run}: this one, or a direct client for its id if this is a last-run. */ + RunClient pinned(ActorRun run) { + return run.getId() == null || run.getId().equals(id) ? this : root.run(run.getId()); + } + /** A client for this run's default key-value store. */ public KeyValueStoreClient keyValueStore() { return KeyValueStoreClient.nested(ctx.http, ctx.subUrl(""), "key-value-store", ctx.baseParams); diff --git a/src/main/java/com/apify/client/run/RunDatasetItems.java b/src/main/java/com/apify/client/run/RunDatasetItems.java new file mode 100644 index 0000000..2836431 --- /dev/null +++ b/src/main/java/com/apify/client/run/RunDatasetItems.java @@ -0,0 +1,179 @@ +package com.apify.client.run; + +import com.apify.client.PaginationList; +import com.apify.client.dataset.Dataset; +import com.apify.client.dataset.DatasetClient; +import com.apify.client.dataset.DatasetListItemsOptions; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +/** + * The paging state machine behind {@link RunClient#iterateDatasetItems}: it reads a run's default + * dataset while the run is still producing items, one page per {@link #next()} call. + * + *

While the run has not finished, each round reads the rows below the dataset's {@code + * itemCount} and then waits up to the poll interval for the run to finish, so the last rows are + * read as soon as it does. Each page is requested with a {@code limit} that ends at {@code + * itemCount}, so it covers exactly the rows it asks for, whatever the filters or {@code unwind} do + * to the items. {@code itemCount} lags a few seconds behind the pushed items, so once the run is + * terminal the rows past it are read a page at a time until none are left. + * + *

Not thread-safe: the publisher that drives it fetches one page at a time. + */ +final class RunDatasetItems { + + private final RunClient runClient; + private final DatasetListItemsOptions options; + private final Class itemClass; + private final long pageSize; + private final long pollIntervalSecs; + + /** The first row this iteration must not read, or {@code null} for no limit. */ + private final Long end; + + private enum Phase { + START, + POLL, + TAIL, + DONE + } + + private Phase phase = Phase.START; + private long position; + private long itemCount; + private boolean finished; + private DatasetClient dataset; + private RunClient resolvedRun; + + RunDatasetItems( + RunClient runClient, + DatasetListItemsOptions options, + long pageSize, + long pollIntervalSecs, + Class itemClass) { + this.runClient = runClient; + this.options = options; + this.itemClass = itemClass; + this.pageSize = pageSize; + this.pollIntervalSecs = pollIntervalSecs; + this.position = options.offsetValueOrZero(); + Long limit = options.limitValueOrNull(); + this.end = limit != null && limit > 0 ? position + limit : null; + } + + /** + * Reads the next page. Completes with a page whose scanned count is non-zero while there may be + * more rows, and with an empty page that scanned nothing once iteration is over. + */ + CompletableFuture> next() { + switch (phase) { + case START: + return runClient.get().thenCompose(run -> start(run)).thenCompose(unused -> next()); + case POLL: + return poll(); + case TAIL: + return tail(); + default: + return CompletableFuture.completedFuture(endOfData()); + } + } + + private CompletableFuture start(Optional run) { + // A last-run client resolves runs/last on each request, so a newer run would swap the dataset + // mid-iteration; pin the run its first request resolves to. + resolvedRun = run.isPresent() ? runClient.pinned(run.get()) : runClient; + dataset = resolvedRun.dataset(); + finished = run.isEmpty() || run.get().isTerminal(); + phase = Phase.POLL; + return refreshItemCount(); + } + + private CompletableFuture refreshItemCount() { + return dataset + .get() + .thenAccept( + ds -> { + long count = ds.map(Dataset::getItemCount).orElse(0L); + itemCount = end != null ? Math.min(count, end) : count; + }); + } + + private CompletableFuture> poll() { + if (position < itemCount) { + long limit = Math.min(pageSize, itemCount - position); + return readPage(limit); + } + if (end != null && position >= end) { + phase = Phase.DONE; + return CompletableFuture.completedFuture(endOfData()); + } + if (finished) { + phase = Phase.TAIL; + return tail(); + } + return resolvedRun + .waitForFinish(pollIntervalSecs) + .thenCompose( + run -> { + finished = run.isTerminal(); + return refreshItemCount(); + }) + .thenCompose(unused -> next()); + } + + private CompletableFuture> tail() { + long limit = end != null ? Math.min(pageSize, end - position) : pageSize; + return page(position, limit) + .thenCompose( + page -> { + Long scanned = page.getScannedCount(); + boolean empty = page.getItems().isEmpty() && (scanned == null || scanned == 0); + if (!empty) { + return CompletableFuture.completedFuture(advance(page, limit)); + } + if (!options.dropsRows()) { + return finish(); + } + // A page that clean, skipEmpty or unwind emptied past a lagging itemCount reports no + // scanned rows either, so a plain read of the next row tells whether any are left. + return dataset + .listItems(options.copy().desc(null).offset(position).limit(1L), itemClass) + .thenCompose( + probe -> + probe.getItems().isEmpty() + ? finish() + : CompletableFuture.completedFuture(advance(page, limit))); + }); + } + + private CompletableFuture> finish() { + phase = Phase.DONE; + return CompletableFuture.completedFuture(endOfData()); + } + + private CompletableFuture> readPage(long limit) { + return page(position, limit).thenApply(page -> advance(page, limit)); + } + + private CompletableFuture> page(long offset, long limit) { + return dataset.listItems(options.copy().desc(null).offset(offset).limit(limit), itemClass); + } + + /** Moves past the {@code limit} rows a page covered and marks it as having scanned them. */ + private PaginationList advance(PaginationList page, long limit) { + position += limit; + if (end != null && position >= end && phase == Phase.TAIL) { + phase = Phase.DONE; + } + page.setScannedCount(Math.max(limit, 1)); + return page; + } + + private PaginationList endOfData() { + PaginationList page = new PaginationList<>(); + page.setItems(List.of()); + page.setScannedCount(0L); + return page; + } +} diff --git a/src/main/java/com/apify/client/run/RunListOptions.java b/src/main/java/com/apify/client/run/RunListOptions.java index f77dee4..e97ee85 100644 --- a/src/main/java/com/apify/client/run/RunListOptions.java +++ b/src/main/java/com/apify/client/run/RunListOptions.java @@ -8,12 +8,14 @@ * Run-specific filters for {@link RunCollectionClient#list(ListOptions, RunListOptions)}. The * {@code startedAfter}/{@code startedBefore} filters are honoured by the top-level ({@code * /v2/actor-runs}) and Actor-scoped ({@code /v2/actors/{actorId}/runs}) run collections; the - * task-scoped run collection accepts only {@code status}. + * task-scoped run collection accepts only {@code status}. {@code deployment} is honoured by the + * top-level and Actor-scoped collections. */ public final class RunListOptions { private List status; private String startedAfter; private String startedBefore; + private List deployment; /** * Filter by one or more run statuses (e.g. {@code "SUCCEEDED"}, {@code "RUNNING"}). Sent as a @@ -36,8 +38,21 @@ public RunListOptions startedBefore(String startedBefore) { return this; } + /** + * Filter Actor Standby runs by the deployment pool that serves them: {@code SINGLE_TENANT}, + * {@code MULTI_TENANT_FREE_USERS}, {@code MULTI_TENANT_PAYING_USERS} or {@code + * MULTI_TENANT_OWNER_PAID}. Sent as a comma-separated list. The API requires the {@link + * #status(List) status} filter to be set too, with only non-terminal statuses ({@code READY}, + * {@code RUNNING}, {@code TIMING-OUT} or {@code ABORTING}). + */ + public RunListOptions deployment(List deployment) { + this.deployment = deployment == null ? null : List.copyOf(deployment); + return this; + } + void apply(QueryParams q) { q.addCsv("status", status) + .addCsv("deployment", deployment) .addString("startedAfter", startedAfter) .addString("startedBefore", startedBefore); } diff --git a/src/main/java/com/apify/client/schedule/Schedule.java b/src/main/java/com/apify/client/schedule/Schedule.java index 28ddb92..e8f1e00 100644 --- a/src/main/java/com/apify/client/schedule/Schedule.java +++ b/src/main/java/com/apify/client/schedule/Schedule.java @@ -23,6 +23,8 @@ public final class Schedule extends ApifyResource { private Instant lastRunAt; private List actions = List.of(); private ScheduleNotifications notifications; + private JsonNode uiState; + private JsonNode integrationTracking; /** The unique schedule ID. */ public String getId() { @@ -107,4 +109,17 @@ public List getActions() { public ScheduleNotifications getNotifications() { return notifications; } + + /** + * The state of the Console schedule editor; {@code null} once an API update of the cron + * expression or timezone has dropped it. + */ + public JsonNode getUiState() { + return uiState; + } + + /** Tracking of the third-party integration that created the schedule, if any. */ + public JsonNode getIntegrationTracking() { + return integrationTracking; + } } diff --git a/src/main/java/com/apify/client/schedule/ScheduleClient.java b/src/main/java/com/apify/client/schedule/ScheduleClient.java index 8c1d06b..86b7654 100644 --- a/src/main/java/com/apify/client/schedule/ScheduleClient.java +++ b/src/main/java/com/apify/client/schedule/ScheduleClient.java @@ -1,5 +1,6 @@ package com.apify.client.schedule; +import com.apify.client.http.Timeout; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; @@ -42,6 +43,6 @@ public CompletableFuture delete() { */ public CompletableFuture> getLog() { JavaType listType = Json.parametric(List.class, Json.type(ScheduleInvoked.class)); - return ctx.getResourceRequired("log", new QueryParams(), listType); + return ctx.getResourceRequired("log", new QueryParams(), listType, Timeout.MEDIUM); } } diff --git a/src/main/java/com/apify/client/store/ActorStoreListItem.java b/src/main/java/com/apify/client/store/ActorStoreListItem.java index cd2fde8..ed25137 100644 --- a/src/main/java/com/apify/client/store/ActorStoreListItem.java +++ b/src/main/java/com/apify/client/store/ActorStoreListItem.java @@ -2,6 +2,8 @@ import com.apify.client.ApifyResource; import com.apify.client.actor.ActorStats; +import com.apify.client.internal.NormalizedUrlDeserializer; +import tools.jackson.databind.annotation.JsonDeserialize; /** An Actor as listed in the Apify Store. */ public final class ActorStoreListItem extends ApifyResource { @@ -13,8 +15,13 @@ public final class ActorStoreListItem extends ApifyResource { private ActorStats stats; private PricingInfo currentPricingInfo; private String pictureUrl; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) private String userPictureUrl; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) private String url; + private String readmeSummary; /** The unique Actor ID. */ diff --git a/src/main/java/com/apify/client/store/StoreListOptions.java b/src/main/java/com/apify/client/store/StoreListOptions.java index df086b3..69ce18e 100644 --- a/src/main/java/com/apify/client/store/StoreListOptions.java +++ b/src/main/java/com/apify/client/store/StoreListOptions.java @@ -15,6 +15,7 @@ public final class StoreListOptions implements ListOptionsLike { private Boolean includeUnrunnableActors; private Boolean allowsAgenticUsers; private String responseFormat; + private Boolean includeInputSchema; /** Number of Actors to skip. */ public StoreListOptions offset(Long offset) { @@ -83,6 +84,16 @@ public StoreListOptions responseFormat(String responseFormat) { return this; } + /** + * Include a compact {@code inputSchema} (property types and required properties only) of each + * Actor's default build in the results, available through {@code ActorStoreListItem#getExtra()}. + * With this flag the API defaults {@code limit} to, and caps it at, {@code 10}. + */ + public StoreListOptions includeInputSchema(Boolean includeInputSchema) { + this.includeInputSchema = includeInputSchema; + return this; + } + @Override public Long limitValue() { return limit; @@ -111,6 +122,7 @@ public void applyFilters(QueryParams q) { .addString("pricingModel", pricingModel) .addBool("includeUnrunnableActors", includeUnrunnableActors) .addBool("allowsAgenticUsers", allowsAgenticUsers) - .addString("responseFormat", responseFormat); + .addString("responseFormat", responseFormat) + .addBool("includeInputSchema", includeInputSchema); } } diff --git a/src/main/java/com/apify/client/task/Task.java b/src/main/java/com/apify/client/task/Task.java index d60d939..303003c 100644 --- a/src/main/java/com/apify/client/task/Task.java +++ b/src/main/java/com/apify/client/task/Task.java @@ -2,8 +2,10 @@ import com.apify.client.ApifyResource; import com.apify.client.actor.ActorStandby; +import com.apify.client.internal.NormalizedUrlDeserializer; import java.time.Instant; import tools.jackson.databind.JsonNode; +import tools.jackson.databind.annotation.JsonDeserialize; /** A pre-configured Actor run (an Actor task). */ public final class Task extends ApifyResource { @@ -22,6 +24,9 @@ public final class Task extends ApifyResource { private Boolean isPublic; private TaskPublicConfig publicConfig; + @JsonDeserialize(using = NormalizedUrlDeserializer.class) + private String standbyUrl; + /** The unique task ID. */ public String getId() { return id; @@ -100,4 +105,9 @@ public Boolean getIsPublic() { public TaskPublicConfig getPublicConfig() { return publicConfig; } + + /** The Actor Standby URL of the task, if standby is enabled (normalized). */ + public String getStandbyUrl() { + return standbyUrl; + } } diff --git a/src/main/java/com/apify/client/task/TaskClient.java b/src/main/java/com/apify/client/task/TaskClient.java index 2b74c4f..cc5e6f6 100644 --- a/src/main/java/com/apify/client/task/TaskClient.java +++ b/src/main/java/com/apify/client/task/TaskClient.java @@ -1,6 +1,7 @@ package com.apify.client.task; import com.apify.client.ApifyClient; +import com.apify.client.http.Timeout; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; @@ -138,7 +139,7 @@ public CompletableFuture call(Object input, TaskCallOptions options, L * is gone", so a missing task is reported as a failure here too. */ public CompletableFuture getInput() { - return ctx.getRawRequired("input", new QueryParams()) + return ctx.getRawRequired("input", new QueryParams(), Timeout.SHORT) .thenApply(resp -> Json.parse(resp.body(), JsonNode.class)); } @@ -149,7 +150,8 @@ public CompletableFuture updateInput(Object input) { new QueryParams(), Json.toBytes(input), ResourceContext.CONTENT_TYPE_JSON, - JsonNode.class); + JsonNode.class, + Timeout.SHORT); } /** diff --git a/src/main/java/com/apify/client/task/TaskCollectionClient.java b/src/main/java/com/apify/client/task/TaskCollectionClient.java index 298920b..7b6bab0 100644 --- a/src/main/java/com/apify/client/task/TaskCollectionClient.java +++ b/src/main/java/com/apify/client/task/TaskCollectionClient.java @@ -1,6 +1,7 @@ package com.apify.client.task; import com.apify.client.ListOptions; +import com.apify.client.http.Timeout; import com.apify.client.internal.AbstractCollectionClient; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; @@ -20,6 +21,6 @@ public TaskCollectionClient(HttpClientCore http, String baseUrl) { /** Creates a new task. {@code task} is any JSON-serializable task definition. */ public CompletableFuture create(Object task) { - return ctx.createResource(new QueryParams(), task, Task.class); + return ctx.createResource(new QueryParams(), task, Task.class, Timeout.MEDIUM); } } diff --git a/src/main/java/com/apify/client/task/TaskOptions.java b/src/main/java/com/apify/client/task/TaskOptions.java index 238067b..244731d 100644 --- a/src/main/java/com/apify/client/task/TaskOptions.java +++ b/src/main/java/com/apify/client/task/TaskOptions.java @@ -8,6 +8,8 @@ public final class TaskOptions extends ApifyResource { private Long timeoutSecs; private Long memoryMbytes; private Boolean restartOnError; + private Double maxTotalChargeUsd; + private Boolean isMaxTotalChargeUsdSetByUser; /** The tag or number of the build the task runs by default. */ public String getBuild() { @@ -28,4 +30,14 @@ public Long getMemoryMbytes() { public Boolean getRestartOnError() { return restartOnError; } + + /** The maximum total charge in USD for a pay-per-event run of the task. */ + public Double getMaxTotalChargeUsd() { + return maxTotalChargeUsd; + } + + /** Whether {@link #getMaxTotalChargeUsd()} was set by the user rather than defaulted. */ + public Boolean getIsMaxTotalChargeUsdSetByUser() { + return isMaxTotalChargeUsdSetByUser; + } } diff --git a/src/main/java/com/apify/client/task/TaskStats.java b/src/main/java/com/apify/client/task/TaskStats.java index 571d898..36a5745 100644 --- a/src/main/java/com/apify/client/task/TaskStats.java +++ b/src/main/java/com/apify/client/task/TaskStats.java @@ -1,13 +1,20 @@ package com.apify.client.task; import com.apify.client.ApifyResource; +import java.time.Instant; /** Usage statistics for a {@link Task}. */ public final class TaskStats extends ApifyResource { private long totalRuns; + private Instant lastRunStartedAt; /** The total number of runs started from this task. */ public long getTotalRuns() { return totalRuns; } + + /** When the most recent run of the task started. */ + public Instant getLastRunStartedAt() { + return lastRunStartedAt; + } } diff --git a/src/main/java/com/apify/client/user/UserClient.java b/src/main/java/com/apify/client/user/UserClient.java index b414c11..cc14214 100644 --- a/src/main/java/com/apify/client/user/UserClient.java +++ b/src/main/java/com/apify/client/user/UserClient.java @@ -12,9 +12,9 @@ /** * A client for accessing user data ({@code /v2/users/{userId}} or {@code /v2/users/me}). * - *

For the current user ({@code me}), it also exposes account usage and limits. Those endpoints - * only exist for {@code me} and throw {@link IllegalStateException} if called on another user's - * client. + *

It also exposes the user's monthly usage and account limits ({@code + * /v2/users/{userId}/usage/monthly} and {@code /v2/users/{userId}/limits}), for {@code me} or for a + * user id the API token may access. */ public final class UserClient { @@ -22,11 +22,9 @@ public final class UserClient { public static final String ME = "me"; private final ResourceContext ctx; - private final boolean isMe; public UserClient(HttpClientCore http, String baseUrl, String id) { this.ctx = ResourceContext.single(http, baseUrl, ApiPaths.USERS, id); - this.isMe = ME.equals(id); } /** @@ -39,19 +37,18 @@ public CompletableFuture> get() { } /** - * Fetches the current account's monthly usage for the current month. Only available for {@code - * me}. Completes with the raw JSON usage report. + * Fetches the user's monthly usage for the current month. Completes with the raw JSON usage + * report. */ public CompletableFuture monthlyUsage() { return monthlyUsage(""); } /** - * Fetches the current account's monthly usage for the month containing the given date (formatted - * as {@code YYYY-MM-DD}). An empty date reports the current month. Only available for {@code me}. + * Fetches the user's monthly usage for the month containing the given date (formatted as {@code + * YYYY-MM-DD}). An empty date reports the current month. */ public CompletableFuture monthlyUsage(String date) { - requireMe(); QueryParams params = new QueryParams(); if (date != null && !date.isEmpty()) { params.addString("date", date); @@ -59,26 +56,17 @@ public CompletableFuture monthlyUsage(String date) { return ctx.getResourceRequired("usage/monthly", params, JsonNode.class); } - /** Fetches the current account's resource limits. Only available for {@code me}. */ + /** Fetches the user's account and usage limits. */ public CompletableFuture limits() { - requireMe(); return ctx.getResourceRequired("limits", new QueryParams(), JsonNode.class); } - /** Updates the current account's resource limits. Only available for {@code me}. */ + /** Updates the user's account and usage limits. */ public CompletableFuture updateLimits(Object newLimits) { - requireMe(); // Route through the shared PUT primitive for consistency with every other resource client's // update-style call; the response body (the updated limits) is intentionally not parsed, // matching the reference client's void-returning updateLimits. return ctx.putRaw( "limits", new QueryParams(), Json.toBytes(newLimits), ResourceContext.CONTENT_TYPE_JSON); } - - private void requireMe() { - if (!isMe) { - throw new IllegalStateException( - "this operation is only available for the current user (use me())"); - } - } } diff --git a/src/main/java/com/apify/client/user/UserProfile.java b/src/main/java/com/apify/client/user/UserProfile.java index 857eb0d..909c896 100644 --- a/src/main/java/com/apify/client/user/UserProfile.java +++ b/src/main/java/com/apify/client/user/UserProfile.java @@ -1,15 +1,23 @@ package com.apify.client.user; import com.apify.client.ApifyResource; +import com.apify.client.internal.NormalizedUrlDeserializer; +import tools.jackson.databind.annotation.JsonDeserialize; /** The public profile of a {@link User}. */ public final class UserProfile extends ApifyResource { private String bio; private String readme; private String name; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) private String pictureUrl; + private String githubUsername; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) private String websiteUrl; + private String twitterUsername; /** A short biography, if set. */ diff --git a/src/main/java/com/apify/client/webhook/Webhook.java b/src/main/java/com/apify/client/webhook/Webhook.java index bddf348..5e82441 100644 --- a/src/main/java/com/apify/client/webhook/Webhook.java +++ b/src/main/java/com/apify/client/webhook/Webhook.java @@ -1,10 +1,12 @@ package com.apify.client.webhook; import com.apify.client.ApifyResource; +import com.apify.client.internal.NormalizedUrlDeserializer; import java.time.Instant; import java.util.Collections; import java.util.List; import tools.jackson.databind.JsonNode; +import tools.jackson.databind.annotation.JsonDeserialize; /** A webhook notifies an external service when specific events occur. */ public final class Webhook extends ApifyResource { @@ -18,12 +20,23 @@ public final class Webhook extends ApifyResource { private JsonNode condition; private boolean ignoreSslErrors; private boolean doNotRetry; + + @JsonDeserialize(using = NormalizedUrlDeserializer.class) private String requestUrl; + private String payloadTemplate; private String headersTemplate; private String description; private WebhookLastDispatch lastDispatch; private WebhookStats stats; + private Boolean isApifyIntegration; + private Boolean isEnabled; + private String actionType; + private String title; + private JsonNode slackMessage; + private JsonNode googleMail; + private JsonNode googleDrive; + private JsonNode githubIssue; /** The unique webhook ID. */ public String getId() { @@ -120,4 +133,46 @@ public WebhookLastDispatch getLastDispatch() { public WebhookStats getStats() { return stats; } + + /** Whether the webhook was created by an Apify integration (e.g. Slack or Google Drive). */ + public Boolean getIsApifyIntegration() { + return isApifyIntegration; + } + + /** Whether the webhook is enabled. */ + public Boolean getIsEnabled() { + return isEnabled; + } + + /** + * The kind of action the webhook performs (e.g. {@code "HTTP_REQUEST"} or an integration action). + */ + public String getActionType() { + return actionType; + } + + /** The human-readable title of the webhook. */ + public String getTitle() { + return title; + } + + /** Slack message configuration, present for Slack integration webhooks. */ + public JsonNode getSlackMessage() { + return slackMessage; + } + + /** Google Mail configuration, present for Google Mail integration webhooks. */ + public JsonNode getGoogleMail() { + return googleMail; + } + + /** Google Drive configuration, present for Google Drive integration webhooks. */ + public JsonNode getGoogleDrive() { + return googleDrive; + } + + /** GitHub issue configuration, present for GitHub integration webhooks. */ + public JsonNode getGithubIssue() { + return githubIssue; + } } diff --git a/src/main/java/com/apify/client/webhook/WebhookClient.java b/src/main/java/com/apify/client/webhook/WebhookClient.java index 20f02c8..189c1fc 100644 --- a/src/main/java/com/apify/client/webhook/WebhookClient.java +++ b/src/main/java/com/apify/client/webhook/WebhookClient.java @@ -1,5 +1,6 @@ package com.apify.client.webhook; +import com.apify.client.http.Timeout; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.QueryParams; @@ -34,7 +35,8 @@ public CompletableFuture delete() { /** Dispatches the webhook immediately and completes with the resulting dispatch. */ public CompletableFuture test() { - return ctx.postWithBody("test", new QueryParams(), null, "", WebhookDispatch.class); + return ctx.postWithBody( + "test", new QueryParams(), null, "", WebhookDispatch.class, Timeout.MEDIUM); } /** A client for this webhook's dispatch collection. */ diff --git a/src/main/java/com/apify/client/webhook/WebhookDispatchWebhookInfo.java b/src/main/java/com/apify/client/webhook/WebhookDispatchWebhookInfo.java index 750ab6b..35de6e1 100644 --- a/src/main/java/com/apify/client/webhook/WebhookDispatchWebhookInfo.java +++ b/src/main/java/com/apify/client/webhook/WebhookDispatchWebhookInfo.java @@ -1,13 +1,17 @@ package com.apify.client.webhook; import com.apify.client.ApifyResource; +import com.apify.client.internal.NormalizedUrlDeserializer; +import tools.jackson.databind.annotation.JsonDeserialize; /** * A summary of the {@link Webhook} that produced a {@link WebhookDispatch} (only the subset of * fields the API includes alongside a dispatch, not the full webhook object). */ public final class WebhookDispatchWebhookInfo extends ApifyResource { + @JsonDeserialize(using = NormalizedUrlDeserializer.class) private String requestUrl; + private boolean isAdHoc; /** diff --git a/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java b/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java index 62b5771..a87dfa3 100644 --- a/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java +++ b/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java @@ -421,41 +421,7 @@ void getWithWaitForwardsWaitForFinish() { } @Test - void getWithWaitClampsServerWaitToConfiguredTimeout() { - // With a 10s per-request timeout, a caller asking for waitForFinish=60 must be clamped below - // the timeout (10 - 5s margin = 5) so the get can't abort itself on the socket timeout. - MockTransport backend = - MockTransport.ofConstant(200, "{\"data\":{\"id\":\"r1\",\"status\":\"RUNNING\"}}"); - ApifyClient client = - ApifyClient.builder() - .token("t") - .httpTransport(backend) - .maxRetries(0) - .timeout(Duration.ofSeconds(10)) - .build(); - client.run("r1").getWithWait(60L).join(); - assertTrue(backend.lastUrl.contains("waitForFinish=5"), backend.lastUrl); - assertFalse(backend.lastUrl.contains("waitForFinish=60"), backend.lastUrl); - } - - @Test - void waitForFinishClampsServerWaitToConfiguredTimeout() { - MockTransport backend = - MockTransport.ofConstant(200, "{\"data\":{\"id\":\"r1\",\"status\":\"SUCCEEDED\"}}"); - ApifyClient client = - ApifyClient.builder() - .token("t") - .httpTransport(backend) - .maxRetries(0) - .minDelayBetweenRetries(Duration.ofMillis(1)) - .timeout(Duration.ofSeconds(20)) - .build(); - client.run("r1").waitForFinish(120L).join(); - // 20s timeout - 5s margin = 15s server wait cap (below the 60s API cap and the 120s budget). - assertTrue(backend.lastUrl.contains("waitForFinish=15"), backend.lastUrl); - } - - @Test + @SuppressWarnings("deprecation") // the deprecated timeout(Duration) alias is what is under test void builderRejectsZeroOrNegativeTimeout() { // Unlike the delay params (where zero legitimately means "no delay"), a zero/negative timeout // must be rejected at build time: it would otherwise build a client whose first request fails diff --git a/src/test/java/com/apify/client/ClientInputTest.java b/src/test/java/com/apify/client/ClientInputTest.java new file mode 100644 index 0000000..73f2acc --- /dev/null +++ b/src/test/java/com/apify/client/ClientInputTest.java @@ -0,0 +1,277 @@ +package com.apify.client; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.apify.client.actor.ActorStartOptions; +import com.apify.client.run.ActorRun; +import com.apify.client.run.RunListOptions; +import com.apify.client.store.StoreListOptions; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Offline tests for what the client sends: pushed dataset items, Actor inputs, default headers, + * path-safe resource ids, and the query parameters added with the OpenAPI spec updates. + */ +class ClientInputTest { + + private static final String RUN_JSON = "{\"data\":{\"id\":\"r1\",\"status\":\"READY\"}}"; + + private static ApifyClient client(MockTransport backend) { + return ApifyClient.builder().token("t").httpTransport(backend).maxRetries(0).build(); + } + + // ---- pushItems accepts only what the API accepts -------------------------------------------- + + @Test + void pushItemsSendsAnObjectOrAnArrayOfObjects() { + MockTransport backend = MockTransport.ofConstant(201, "{}"); + client(backend).dataset("d").pushItems(Map.of("a", 1)).join(); + assertEquals("{\"a\":1}", backend.lastBody); + + client(backend).dataset("d").pushItems(List.of(Map.of("a", 1), Map.of("b", 2))).join(); + assertEquals("[{\"a\":1},{\"b\":2}]", backend.lastBody); + } + + @Test + void pushItemsSendsAPreSerializedStringAsItIs() { + MockTransport backend = MockTransport.ofConstant(201, "{}"); + client(backend).dataset("d").pushItems("[{\"a\": 1}, {\"b\": 2}]").join(); + assertEquals("[{\"a\": 1}, {\"b\": 2}]", backend.lastBody, "not wrapped in a JSON string"); + } + + @Test + void pushItemsRejectsShapesTheApiWouldReject() { + MockTransport backend = MockTransport.ofConstant(201, "{}"); + var dataset = client(backend).dataset("d"); + assertThrows(IllegalArgumentException.class, () -> dataset.pushItems(null)); + assertThrows(IllegalArgumentException.class, () -> dataset.pushItems(42)); + assertThrows(IllegalArgumentException.class, () -> dataset.pushItems(List.of("a", "b"))); + assertThrows(IllegalArgumentException.class, () -> dataset.pushItems(new String[] {"a"})); + assertThrows(IllegalArgumentException.class, () -> dataset.pushItems("just text")); + assertThrows(IllegalArgumentException.class, () -> dataset.pushItems("[1, 2]")); + assertThrows( + IllegalArgumentException.class, () -> dataset.pushItems(List.of(Map.of("a", 1), "b"))); + assertEquals(0, backend.calls, "nothing is sent for an invalid payload"); + } + + // ---- Actor input ----------------------------------------------------------------------------- + + @Test + void aByteArrayInputIsSentAsRawBytes() { + MockTransport backend = MockTransport.ofConstant(201, RUN_JSON); + byte[] input = "{\"x\": 1}".getBytes(StandardCharsets.UTF_8); + client(backend).actor("a").start(input, new ActorStartOptions()).join(); + assertArrayEquals(input, backend.lastBodyBytes); + } + + @Test + void aStringInputIsSentAsItIsWithTheGivenContentType() { + MockTransport backend = MockTransport.ofConstant(201, RUN_JSON); + client(backend) + .actor("a") + .start("a,b\n1,2", new ActorStartOptions().contentType("text/csv")) + .join(); + assertEquals("a,b\n1,2", backend.lastBody); + assertEquals("text/csv", backend.lastHeaders.firstValue("Content-Type").orElse(null)); + } + + @Test + void anObjectInputIsSerializedToJson() { + MockTransport backend = MockTransport.ofConstant(201, RUN_JSON); + client(backend).actor("a").start(Map.of("k", "v"), new ActorStartOptions()).join(); + assertEquals("{\"k\":\"v\"}", backend.lastBody); + } + + // ---- default headers ------------------------------------------------------------------------- + + @Test + void defaultHeadersAreSentWithEveryRequestAndWinOverTheBuiltInOnes() { + MockTransport backend = MockTransport.ofConstant(200, RUN_JSON); + ApifyClient client = + ApifyClient.builder() + .token("t") + .httpTransport(backend) + .maxRetries(0) + .header("X-Integration", "my-platform") + .header("user-agent", "custom-agent/1") + .build(); + client.run("r1").get().join(); + assertEquals("my-platform", backend.lastHeaders.firstValue("X-Integration").orElse(null)); + assertEquals( + List.of("custom-agent/1"), + backend.lastHeaders.allValues("User-Agent"), + "replaced, not added"); + assertEquals("Bearer t", backend.lastHeaders.firstValue("Authorization").orElse(null)); + } + + @Test + void aHeaderSetAgainReplacesTheEarlierOneCaseInsensitively() { + MockTransport backend = MockTransport.ofConstant(200, RUN_JSON); + ApifyClient.builder() + .token("t") + .httpTransport(backend) + .header("X-A", "1") + .header("x-a", "2") + .build() + .run("r1") + .get() + .join(); + assertEquals(List.of("2"), backend.lastHeaders.allValues("X-A")); + } + + @Test + void builderRejectsABlankHeaderNameAndANullValue() { + assertThrows(IllegalArgumentException.class, () -> ApifyClient.builder().header(" ", "v")); + assertThrows(IllegalArgumentException.class, () -> ApifyClient.builder().header("X", null)); + } + + // ---- path-safe resource ids ------------------------------------------------------------------ + + @Test + void resourceIdsAreEncodedIntoASinglePathSegment() { + MockTransport backend = MockTransport.ofConstant(200, RUN_JSON); + ApifyClient client = client(backend); + client.run("r1?x=1#frag").get().join(); + assertTrue(backend.lastUrl.contains("/actor-runs/r1%3Fx%3D1%23frag"), backend.lastUrl); + + client.run("../../users/me").get().join(); + assertTrue(backend.lastUrl.contains("/actor-runs/..~..~users~me"), backend.lastUrl); + + client.actor("apify/hello-world").get().join(); + assertTrue(backend.lastUrl.contains("/actors/apify~hello-world"), backend.lastUrl); + } + + @Test + void anEmptyOrDotResourceIdIsRejectedInsteadOfAddressingTheCollection() { + ApifyClient client = client(MockTransport.ofConstant(200, RUN_JSON)); + assertThrows(IllegalArgumentException.class, () -> client.run("")); + assertThrows(IllegalArgumentException.class, () -> client.run("..")); + assertThrows(IllegalArgumentException.class, () -> client.dataset(".")); + } + + // ---- users by id ----------------------------------------------------------------------------- + + @Test + void limitsAndMonthlyUsageAreAvailableForAnyUserId() { + MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{}}"); + client(backend).user("someone").limits().join(); + assertTrue(backend.lastUrl.endsWith("/users/someone/limits"), backend.lastUrl); + + client(backend).user("someone").monthlyUsage("2026-10-01").join(); + assertTrue( + backend.lastUrl.contains("/users/someone/usage/monthly?date=2026-10-01"), backend.lastUrl); + + client(backend).user("someone").updateLimits(Map.of("maxMonthlyUsageUsd", 5)).join(); + assertEquals("PUT", backend.lastMethod); + assertTrue(backend.lastUrl.endsWith("/users/someone/limits"), backend.lastUrl); + } + + // ---- query parameters added with the spec ---------------------------------------------------- + + @Test + void runListsFilterByStandbyDeployment() { + MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"items\":[],\"total\":0}}"); + RunListOptions filter = + new RunListOptions() + .status(List.of("RUNNING")) + .deployment(List.of("SINGLE_TENANT", "MULTI_TENANT_FREE_USERS")); + client(backend).runs().list(new ListOptions(), filter).join(); + assertTrue( + backend.lastUrl.contains("deployment=SINGLE_TENANT%2CMULTI_TENANT_FREE_USERS"), + backend.lastUrl); + client(backend).actor("a").runs().list(new ListOptions(), filter).join(); + assertTrue(backend.lastUrl.contains("deployment="), backend.lastUrl); + } + + @Test + void storeListCanIncludeTheInputSchema() { + MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"items\":[],\"total\":0}}"); + client(backend).store().list(new StoreListOptions().includeInputSchema(true)).join(); + assertTrue(backend.lastUrl.contains("includeInputSchema=1"), backend.lastUrl); + } + + // ---- new response fields --------------------------------------------------------------------- + + @Test + void runExposesTheServerActorDeploymentAndTheNewFields() { + MockTransport backend = + MockTransport.ofConstant( + 200, + "{\"data\":{\"id\":\"r1\",\"standby\":{\"deployment\":\"MULTI_TENANT_PAYING_USERS\"}," + + "\"consoleUrl\":\"https://console.apify.com/actors/runs/r1\"," + + "\"externalChargedTotalUsd\":0.5," + + "\"eventUsage\":{\"x\":{\"eventTotalUsd\":1.0,\"eventTitle\":\"X\"}}," + + "\"meta\":{\"origin\":\"API\",\"actorRunId\":\"parent\"}," + + "\"options\":{\"isMaxTotalChargeUsdSetByUser\":true}}}"); + ActorRun run = client(backend).run("r1").get().join().orElseThrow(); + assertEquals("MULTI_TENANT_PAYING_USERS", run.getStandby().getDeployment()); + assertEquals(0.5, run.getExternalChargedTotalUsd()); + assertEquals("parent", run.getMeta().getActorRunId()); + assertTrue(run.getOptions().getIsMaxTotalChargeUsdSetByUser()); + assertEquals(1.0, run.getEventUsage().get("x").get("eventTotalUsd").asDouble()); + } + + @Test + void aRunWithoutStandbyHasNone() { + MockTransport backend = + MockTransport.ofConstant(200, "{\"data\":{\"id\":\"r1\",\"standby\":null}}"); + assertEquals(null, client(backend).run("r1").get().join().orElseThrow().getStandby()); + } + + @Test + void scheduleToleratesANullUiState() { + MockTransport backend = + MockTransport.ofConstant( + 200, "{\"data\":{\"id\":\"s1\",\"cronExpression\":\"* * * * *\",\"uiState\":null}}"); + var schedule = client(backend).schedule("s1").get().join().orElseThrow(); + assertEquals(null, schedule.getUiState()); + + backend = + MockTransport.ofConstant( + 200, "{\"data\":{\"id\":\"s1\",\"uiState\":{\"mode\":\"advanced\"}}}"); + schedule = client(backend).schedule("s1").get().join().orElseThrow(); + assertEquals("advanced", schedule.getUiState().get("mode").asText()); + } + + @Test + void aPendingRequestKeepsANullHandledAt() { + MockTransport backend = + MockTransport.ofConstant( + 200, + "{\"data\":{\"id\":\"q1\",\"uniqueKey\":\"u\",\"url\":\"https://x.test\",\"handledAt\":null}}"); + var request = client(backend).requestQueue("q").getRequest("q1").join().orElseThrow(); + assertEquals(null, request.getHandledAt()); + assertFalse(backend.lastUrl.isEmpty()); + } + + @Test + void webhookIntegrationFieldsAreTyped() { + MockTransport backend = + MockTransport.ofConstant( + 200, + "{\"data\":{\"id\":\"w1\",\"isApifyIntegration\":true,\"isEnabled\":true," + + "\"actionType\":\"SLACK_MESSAGE\",\"title\":\"T\"," + + "\"slackMessage\":{\"channelId\":\"C1\"}}}"); + var webhook = client(backend).webhook("w1").get().join().orElseThrow(); + assertTrue(webhook.getIsApifyIntegration()); + assertEquals("SLACK_MESSAGE", webhook.getActionType()); + assertEquals("C1", webhook.getSlackMessage().get("channelId").asText()); + } + + @Test + void clientIsBuildableWithTheLegacyAndNewTimeoutSettersTogether() { + ApifyClient.builder() + .token("t") + .timeoutShort(Duration.ofSeconds(1)) + .timeoutMax(Duration.ofSeconds(30)) + .build(); + } +} diff --git a/src/test/java/com/apify/client/CompressionTest.java b/src/test/java/com/apify/client/CompressionTest.java index 4392dd2..4fd1336 100644 --- a/src/test/java/com/apify/client/CompressionTest.java +++ b/src/test/java/com/apify/client/CompressionTest.java @@ -3,12 +3,16 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; import com.aayushatharva.brotli4j.Brotli4jLoader; import com.aayushatharva.brotli4j.decoder.Decoder; import com.aayushatharva.brotli4j.decoder.DirectDecompress; +import com.apify.client.http.BrotliHttpCompressor; +import com.apify.client.http.GzipHttpCompressor; +import com.apify.client.http.HttpCompressor; import com.apify.client.internal.HttpClientCore; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -22,15 +26,9 @@ /** * Offline tests for request-body compression. Mirrors the reference JS client: bodies of at least * 1024 bytes are compressed and announced via {@code Content-Encoding}, smaller bodies are sent - * verbatim. The client prefers brotli ({@code br}) and falls back to gzip; both codings are - * exercised here — the brotli/gzip decision (a pure function) directly, and the live client path - * via a real request. - * - *

Not exercised here: {@code HttpClientCore.compress}'s per-call fallback from a - * failing brotli encode to gzip (as opposed to the upfront {@code BROTLI_AVAILABLE} check, - * which the brotli-path tests below do exercise). Accepted gap: brotli4j's {@code Encoder.compress} - * has no portable way to be forced to throw once the native codec has loaded, and the fallback only - * chooses between two codings already covered above. + * verbatim. The client uses brotli ({@code br}) by default where its native codec is available and + * gzip elsewhere; the compressor is configurable, and its failure fails the request. Both + * compressors are exercised directly, and the live client path via a real request. */ class CompressionTest { @@ -82,28 +80,111 @@ private static boolean nativeBrotliExpected() { return linux && commonArch && !musl; } - // --- Pure encoding decision: both paths are directly reachable and verified round-trip. --- + // --- The compressors themselves: both codings are reachable and verified round-trip. --- @Test - void brotliPathEncodesAsBrAndRoundTrips() throws IOException { - assumeTrue(HttpClientCore.brotliAvailable(), "no brotli native codec on this platform"); + void brotliCompressorEncodesAsBrAndRoundTrips() throws IOException { + assumeTrue(BrotliHttpCompressor.isAvailable(), "no brotli native codec on this platform"); byte[] payload = payload(4096, (byte) 'a'); - HttpClientCore.Compressed c = HttpClientCore.compress(payload, true); + HttpCompressor compressor = new BrotliHttpCompressor(); + byte[] compressed = compressor.compress(payload); - assertEquals("br", c.encoding(), "brotli path must announce Content-Encoding: br"); - assertTrue(c.body().length < payload.length, "brotli-compressed body should be smaller"); - assertArrayEquals( - payload, unbrotli(c.body()), "server must recover the original body from brotli"); + assertEquals("br", compressor.contentEncoding()); + assertTrue(compressed.length < payload.length, "brotli-compressed body should be smaller"); + assertArrayEquals(payload, unbrotli(compressed)); } @Test - void gzipPathEncodesAsGzipAndRoundTrips() throws IOException { + void gzipCompressorEncodesAsGzipAndRoundTrips() throws IOException { byte[] payload = payload(4096, (byte) 'a'); - HttpClientCore.Compressed c = HttpClientCore.compress(payload, false); + HttpCompressor compressor = new GzipHttpCompressor(); + byte[] compressed = compressor.compress(payload); - assertEquals("gzip", c.encoding(), "gzip fallback must announce Content-Encoding: gzip"); - assertTrue(c.body().length < payload.length, "gzip-compressed body should be smaller"); - assertArrayEquals(payload, gunzip(c.body()), "server must recover the original body from gzip"); + assertEquals("gzip", compressor.contentEncoding()); + assertTrue(compressed.length < payload.length, "gzip-compressed body should be smaller"); + assertArrayEquals(payload, gunzip(compressed)); + } + + @Test + void gzipLevelTradesSizeForSpeed() throws IOException { + byte[] payload = new byte[64 * 1024]; + new java.util.Random(7).nextBytes(payload); + for (int i = 0; i < payload.length; i += 3) { + payload[i] = 'x'; // partly compressible, so the levels differ + } + byte[] fastest = new GzipHttpCompressor(1).compress(payload); + byte[] smallest = new GzipHttpCompressor(9).compress(payload); + + assertArrayEquals(payload, gunzip(fastest)); + assertArrayEquals(payload, gunzip(smallest)); + assertTrue(smallest.length <= fastest.length, "level 9 must not be larger than level 1"); + } + + @Test + void compressorsRejectAnOutOfRangeQualityAtConstruction() { + assertThrows(IllegalArgumentException.class, () -> new GzipHttpCompressor(0)); + assertThrows(IllegalArgumentException.class, () -> new GzipHttpCompressor(10)); + assertThrows(IllegalArgumentException.class, () -> new BrotliHttpCompressor(-1)); + assertThrows(IllegalArgumentException.class, () -> new BrotliHttpCompressor(12)); + } + + // --- The configured compressor is the one the client runs, with no silent fallback. --- + + @Test + void configuredGzipCompressorIsUsedForLargeBodies() throws IOException { + MockTransport backend = MockTransport.ofConstant(201, ""); + byte[] payload = payload(4096, (byte) 'a'); + + ApifyClient.builder() + .token("t") + .httpTransport(backend) + .maxRetries(0) + .compression(new GzipHttpCompressor(3)) + .build() + .keyValueStore(STORE_ID) + .setRecord(RECORD_KEY, payload, CONTENT_TYPE) + .join(); + + assertEquals("gzip", backend.lastHeaders.firstValue("Content-Encoding").orElse(null)); + assertArrayEquals(payload, gunzip(backend.lastBodyBytes)); + } + + @Test + void customCompressorIsUsedAndItsFailureFailsTheRequest() { + HttpCompressor failing = + new HttpCompressor() { + @Override + public String contentEncoding() { + return "x-custom"; + } + + @Override + public byte[] compress(byte[] data) throws IOException { + throw new IOException("boom"); + } + }; + MockTransport backend = MockTransport.ofConstant(201, ""); + ApifyClient client = + ApifyClient.builder() + .token("t") + .httpTransport(backend) + .maxRetries(0) + .compression(failing) + .build(); + + var future = + client + .keyValueStore(STORE_ID) + .setRecord(RECORD_KEY, payload(4096, (byte) 'a'), CONTENT_TYPE); + + var error = assertThrows(java.util.concurrent.CompletionException.class, future::join); + assertTrue(error.getCause() instanceof com.apify.client.http.ApifyTransportException); + assertEquals(0, backend.calls, "a body that could not be compressed must not be sent"); + } + + @Test + void builderRejectsNullCompression() { + assertThrows(IllegalArgumentException.class, () -> ApifyClient.builder().compression(null)); } // --- Brotli native codec must load where a native is bundled (CI/runtime), so brotli runs. --- @@ -112,7 +193,7 @@ void gzipPathEncodesAsGzipAndRoundTrips() throws IOException { void brotliNativeCodecLoadsWhereBundled() { assumeTrue(nativeBrotliExpected(), "no brotli native bundled for this platform; gzip fallback"); assertTrue( - HttpClientCore.brotliAvailable(), + BrotliHttpCompressor.isAvailable(), "brotli native codec must load on glibc x86_64/aarch64 so the preferred brotli path runs"); } @@ -125,7 +206,7 @@ void largeBodyIsCompressedWithPreferredEncoding() throws IOException { client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, CONTENT_TYPE).join(); - boolean brotli = HttpClientCore.brotliAvailable(); + boolean brotli = BrotliHttpCompressor.isAvailable(); String expected = brotli ? "br" : "gzip"; assertEquals( expected, @@ -201,7 +282,7 @@ void rawFormatUnderCompressedPrefixIsStillCompressed() throws IOException { // image/bmp sits under the "image/" prefix but is raw, so it should still be compressed. client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, "image/bmp").join(); - boolean brotli = HttpClientCore.brotliAvailable(); + boolean brotli = BrotliHttpCompressor.isAvailable(); assertEquals( brotli ? "br" : "gzip", backend.lastHeaders.firstValue("Content-Encoding").orElse(null)); byte[] recovered = brotli ? unbrotli(backend.lastBodyBytes) : gunzip(backend.lastBodyBytes); @@ -215,7 +296,7 @@ void bodyExactlyAtThresholdIsCompressed() throws IOException { client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, CONTENT_TYPE).join(); - boolean brotli = HttpClientCore.brotliAvailable(); + boolean brotli = BrotliHttpCompressor.isAvailable(); assertEquals( brotli ? "br" : "gzip", backend.lastHeaders.firstValue("Content-Encoding").orElse(null), diff --git a/src/test/java/com/apify/client/MockTransport.java b/src/test/java/com/apify/client/MockTransport.java index e013f8a..7841209 100644 --- a/src/test/java/com/apify/client/MockTransport.java +++ b/src/test/java/com/apify/client/MockTransport.java @@ -57,6 +57,15 @@ static final class Scripted { final List bodies = new ArrayList<>(); final List urls = new ArrayList<>(); + /** The per-attempt timeout of the last request, empty when the request had none. */ + Optional lastTimeout = Optional.empty(); + + /** The per-attempt timeout of every request, in order. */ + final List> timeouts = new ArrayList<>(); + + /** When set, answers each request itself instead of serving the scripted queue. */ + private java.util.function.Function router; + /** One scripted response for {@link #sendStreamingAsync}: a status + ready-made body stream. */ private static final class StreamEntry { final int status; @@ -82,6 +91,13 @@ private static final class StreamEntry { this.responses = responses; } + /** A transport that answers every request with {@code router}'s response for it. */ + static MockTransport ofRouter(java.util.function.Function router) { + MockTransport transport = new MockTransport(List.of(new Scripted(500, "no route", null))); + transport.router = router; + return transport; + } + static MockTransport ofConstant(int status, String body) { return new MockTransport(List.of(new Scripted(status, body, null))); } @@ -114,6 +130,8 @@ public synchronized CompletableFuture> sendAsync(HttpReques lastUrl = request.uri().toString(); urls.add(lastUrl); lastMethod = request.method(); + lastTimeout = request.timeout(); + timeouts.add(lastTimeout); lastBodyBytes = readBody(request); lastBody = lastBodyBytes == null ? null : new String(lastBodyBytes, StandardCharsets.UTF_8); if (lastBody != null) { @@ -122,7 +140,7 @@ public synchronized CompletableFuture> sendAsync(HttpReques if (idx >= responses.size()) { idx = responses.size() - 1; // repeat the last entry } - Scripted r = responses.get(idx); + Scripted r = router != null ? router.apply(request) : responses.get(idx); if (r.error != null) { CompletableFuture> failed = new CompletableFuture<>(); failed.completeExceptionally(r.error); diff --git a/src/test/java/com/apify/client/RunDatasetItemsTest.java b/src/test/java/com/apify/client/RunDatasetItemsTest.java new file mode 100644 index 0000000..a546ce7 --- /dev/null +++ b/src/test/java/com/apify/client/RunDatasetItemsTest.java @@ -0,0 +1,170 @@ +package com.apify.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.apify.client.dataset.DatasetListItemsOptions; +import java.net.URI; +import java.net.http.HttpRequest; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import org.junit.jupiter.api.Test; +import tools.jackson.databind.JsonNode; + +/** + * {@code RunClient.iterateDatasetItems}: iterating a run's dataset while the run is still producing + * items, against a scripted run and dataset. + */ +class RunDatasetItemsTest { + + /** A fake run + dataset: {@code itemCount} grows by one batch per run poll. */ + private static final class FakeRun { + final List rows = new ArrayList<>(); + final List requests = new ArrayList<>(); + int itemCountReported; + String status = "RUNNING"; + final AtomicInteger runGets = new AtomicInteger(); + // Rows made visible per poll of the run, then the run finishes with `lagging` rows not yet + // counted. + final int rowsPerPoll; + final int totalRows; + final int laggingAtFinish; + + FakeRun(int totalRows, int rowsPerPoll, int laggingAtFinish) { + this.totalRows = totalRows; + this.rowsPerPoll = rowsPerPoll; + this.laggingAtFinish = laggingAtFinish; + for (int i = 0; i < totalRows; i++) { + rows.add("{\"n\":" + i + "}"); + } + } + + MockTransport.Scripted answer(HttpRequest request) { + URI uri = request.uri(); + String path = uri.getPath(); + String query = uri.getQuery() == null ? "" : uri.getQuery(); + requests.add(path + (query.isEmpty() ? "" : "?" + query)); + if (path.endsWith("/actor-runs/r1")) { + int poll = runGets.incrementAndGet(); + // The first get resolves the run; each later poll reveals another batch, then finishes. + int visible = Math.min(poll * rowsPerPoll, totalRows); + boolean done = visible >= totalRows; + itemCountReported = done ? totalRows - laggingAtFinish : visible; + status = done ? "SUCCEEDED" : "RUNNING"; + return MockTransport.ok(200, "{\"data\":{\"id\":\"r1\",\"status\":\"" + status + "\"}}"); + } + if (path.endsWith("/actor-runs/r1/dataset")) { + return MockTransport.ok( + 200, "{\"data\":{\"id\":\"d1\",\"itemCount\":" + itemCountReported + "}}"); + } + if (path.endsWith("/actor-runs/r1/dataset/items")) { + Map params = new java.util.HashMap<>(); + for (String pair : query.split("&")) { + String[] kv = pair.split("=", 2); + params.put(kv[0], kv.length > 1 ? kv[1] : ""); + } + int offset = Integer.parseInt(params.getOrDefault("offset", "0")); + int limit = Integer.parseInt(params.getOrDefault("limit", "1000")); + int end = Math.min(totalRows, offset + limit); + List page = offset >= end ? List.of() : rows.subList(offset, end); + return MockTransport.ok( + 200, + "[" + String.join(",", page) + "]", + Map.of("X-Apify-Pagination-Count", String.valueOf(page.size()))); + } + return MockTransport.ok(404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"x\"}}"); + } + } + + private static ApifyClient client(Function router) { + return ApifyClient.builder() + .token("t") + .httpTransport(MockTransport.ofRouter(router)) + .maxRetries(0) + .minDelayBetweenRetries(Duration.ofMillis(1)) + .build(); + } + + private static List ns(List items) { + return items.stream().map(n -> n.get("n").asInt()).toList(); + } + + private static List range(int fromInclusive, int toExclusive) { + List out = new ArrayList<>(); + for (int i = fromInclusive; i < toExclusive; i++) { + out.add(i); + } + return out; + } + + @Test + void readsRowsWhileTheRunProducesThemAndTheTailAfterItFinishes() { + // 7 rows, 3 become visible per poll; when the run finishes the counter still lags by 2 rows. + FakeRun fake = new FakeRun(7, 3, 2); + List items = + Publishers.collect( + client(fake::answer) + .run("r1") + .iterateDatasetItems(new DatasetListItemsOptions(), 2L, 0L, JsonNode.class)) + .join(); + assertEquals(range(0, 7), ns(items), "every row exactly once, in order"); + } + + @Test + void aFinishedRunIsReadInPagesOfTheChunkSize() { + FakeRun fake = new FakeRun(5, 10, 0); + List items = + Publishers.collect( + client(fake::answer) + .run("r1") + .iterateDatasetItems(new DatasetListItemsOptions(), 2L, 0L, JsonNode.class)) + .join(); + assertEquals(range(0, 5), ns(items)); + long itemRequests = fake.requests.stream().filter(r -> r.contains("/dataset/items")).count(); + assertTrue(itemRequests >= 3, "5 rows in pages of 2 need at least 3 item requests"); + assertTrue( + fake.requests.stream() + .filter(r -> r.contains("/dataset/items")) + .anyMatch(r -> r.contains("limit=2")), + fake.requests.toString()); + } + + @Test + void offsetAndLimitBoundTheRowsThatAreRead() { + FakeRun fake = new FakeRun(20, 20, 0); + List items = + Publishers.collect( + client(fake::answer) + .run("r1") + .iterateDatasetItems( + new DatasetListItemsOptions().offset(4L).limit(6L), 4L, 0L, JsonNode.class)) + .join(); + assertEquals(range(4, 10), ns(items)); + } + + @Test + void anEmptyDatasetEndsTheIterationOnceTheRunHasFinished() { + FakeRun fake = new FakeRun(0, 1, 0); + List items = + Publishers.collect(client(fake::answer).run("r1").iterateDatasetItems(null)).join(); + assertTrue(items.isEmpty()); + } + + @Test + void orderIsNeverReversedByDesc() { + FakeRun fake = new FakeRun(4, 4, 0); + List items = + Publishers.collect( + client(fake::answer) + .run("r1") + .iterateDatasetItems(new DatasetListItemsOptions().desc(true))) + .join(); + assertEquals(range(0, 4), ns(items)); + assertTrue( + fake.requests.stream().noneMatch(r -> r.contains("desc=")), fake.requests.toString()); + } +} diff --git a/src/test/java/com/apify/client/TimeoutTierTest.java b/src/test/java/com/apify/client/TimeoutTierTest.java new file mode 100644 index 0000000..f895e69 --- /dev/null +++ b/src/test/java/com/apify/client/TimeoutTierTest.java @@ -0,0 +1,269 @@ +package com.apify.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.apify.client.actor.ActorBuildOptions; +import com.apify.client.actor.ActorStartOptions; +import com.apify.client.http.ApifyTransportException; +import com.apify.client.http.Timeout; +import com.apify.client.keyvalue.SetRecordOptions; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletionException; +import org.junit.jupiter.api.Test; + +/** + * Offline tests for the timeout tiers: which tier each kind of method picks, how the builder + * configures the tiers and the cap, how {@link ApifyClient#withTimeout(Timeout)} overrides them, + * and how a retry grows the timeout of an attempt. + */ +class TimeoutTierTest { + + private static final String RUN_JSON = "{\"data\":{\"id\":\"r1\",\"status\":\"RUNNING\"}}"; + + private static ApifyClientBuilder builder(MockTransport backend) { + return ApifyClient.builder() + .token("t") + .httpTransport(backend) + .maxRetries(0) + .minDelayBetweenRetries(Duration.ofMillis(1)); + } + + private static ApifyClient client(MockTransport backend) { + return builder(backend).build(); + } + + private static Optional secs(long seconds) { + return Optional.of(Duration.ofSeconds(seconds)); + } + + @Test + void metadataReadUsesTheShortTier() { + MockTransport backend = MockTransport.ofConstant(200, RUN_JSON); + client(backend).run("r1").get().join(); + assertEquals(secs(5), backend.lastTimeout); + } + + @Test + void listingUsesTheMediumTier() { + MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"items\":[],\"total\":0}}"); + client(backend).actors().list(new com.apify.client.actor.ActorListOptions()).join(); + assertEquals(secs(30), backend.lastTimeout); + } + + @Test + void downloadsAndRecordReadsUseTheLongTier() { + MockTransport backend = MockTransport.ofConstant(200, "[]"); + client(backend) + .dataset("d1") + .listItems(new com.apify.client.dataset.DatasetListItemsOptions()) + .join(); + assertEquals(secs(360), backend.lastTimeout); + + backend = MockTransport.ofConstant(200, "abc"); + client(backend).keyValueStore("s1").getRecord("k").join(); + assertEquals(secs(360), backend.lastTimeout); + } + + @Test + void keyValueStoreUpdateAndRequestQueueUnlockFollowTheReferenceTiers() { + MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"id\":\"s1\"}}"); + client(backend).keyValueStore("s1").update(java.util.Map.of("name", "x")).join(); + assertEquals(secs(360), backend.lastTimeout); + + backend = MockTransport.ofConstant(200, "{\"data\":{\"unlockedCount\":0}}"); + client(backend).requestQueue("q1").unlockRequests().join(); + assertEquals(secs(360), backend.lastTimeout); + } + + @Test + void startIsMediumExtendedByTheWaitForFinishHold() { + MockTransport backend = MockTransport.ofConstant(201, RUN_JSON); + client(backend).actor("a1").start(null, new ActorStartOptions()).join(); + assertEquals(secs(30), backend.lastTimeout, "no hold: the plain medium tier"); + + client(backend).actor("a1").start(null, new ActorStartOptions().waitForFinish(20L)).join(); + assertEquals(secs(50), backend.lastTimeout, "medium tier plus the 20s hold"); + + client(backend).actor("a1").start(null, new ActorStartOptions().waitForFinish(500L)).join(); + assertEquals(secs(90), backend.lastTimeout, "the hold is capped at the API's 60s"); + } + + @Test + void getWithWaitIsShortExtendedByTheHold() { + MockTransport backend = MockTransport.ofConstant(200, RUN_JSON); + client(backend).run("r1").getWithWait(30L).join(); + assertTrue(backend.lastUrl.contains("waitForFinish=30"), backend.lastUrl); + assertEquals(secs(35), backend.lastTimeout); + } + + @Test + void waitForFinishPollsHaveNoTimeoutAndHoldForTheRemainingBudget() { + MockTransport backend = + MockTransport.ofConstant(200, "{\"data\":{\"id\":\"r1\",\"status\":\"SUCCEEDED\"}}"); + client(backend).run("r1").waitForFinish(120L).join(); + assertEquals(Optional.empty(), backend.lastTimeout, "polls run without a timeout"); + assertTrue(backend.lastUrl.contains("waitForFinish=60"), backend.lastUrl); + } + + @Test + void callSendsTheStartWithoutATimeout() { + MockTransport backend = + MockTransport.ofRouter( + request -> + request.method().equals("POST") + ? MockTransport.ok(201, RUN_JSON) + : MockTransport.ok(200, "{\"data\":{\"id\":\"r1\",\"status\":\"SUCCEEDED\"}}")); + client(backend).actor("a1").call(null, new ActorStartOptions(), 10L).join(); + assertEquals(Optional.empty(), backend.timeouts.get(0), "the start inside call()"); + } + + @Test + void builderConfiguresEveryTier() { + MockTransport backend = MockTransport.ofConstant(200, RUN_JSON); + ApifyClient client = + builder(backend) + .timeoutShort(Duration.ofSeconds(2)) + .timeoutMedium(Duration.ofSeconds(7)) + .timeoutLong(Duration.ofSeconds(90)) + .build(); + client.run("r1").get().join(); + assertEquals(secs(2), backend.lastTimeout); + client.runs().list(new com.apify.client.ListOptions(), null).join(); + assertEquals(secs(7), backend.lastTimeout); + client.keyValueStore("s").getRecord("k").join(); + assertEquals(secs(90), backend.lastTimeout); + } + + @Test + void aTierAboveTimeoutMaxIsCapped() { + MockTransport backend = MockTransport.ofConstant(200, RUN_JSON); + ApifyClient client = + builder(backend) + .timeoutShort(Duration.ofMinutes(10)) + .timeoutMax(Duration.ofMinutes(1)) + .build(); + client.run("r1").get().join(); + assertEquals(secs(60), backend.lastTimeout); + } + + @Test + void withTimeoutReplacesTheTierOfEveryRequest() { + MockTransport backend = MockTransport.ofConstant(200, RUN_JSON); + ApifyClient client = client(backend); + + client.withTimeout(Timeout.ofSeconds(120)).run("r1").get().join(); + assertEquals(secs(120), backend.lastTimeout); + + client.withTimeout(Timeout.LONG).run("r1").get().join(); + assertEquals(secs(360), backend.lastTimeout); + + client.withTimeout(Timeout.NO_TIMEOUT).run("r1").get().join(); + assertEquals(Optional.empty(), backend.lastTimeout); + + client.run("r1").get().join(); + assertEquals(secs(5), backend.lastTimeout, "the original client keeps its own tiers"); + } + + @Test + void withTimeoutIsUsedAsGivenForAWaitForFinishHold() { + MockTransport backend = MockTransport.ofConstant(200, RUN_JSON); + client(backend).withTimeout(Timeout.ofSeconds(10)).run("r1").getWithWait(30L).join(); + assertEquals(secs(10), backend.lastTimeout); + } + + @Test + void aRetryDoublesTheTimeoutUpToTheCap() { + MockTransport backend = + new MockTransport( + List.of( + MockTransport.ok(500, "{}"), + MockTransport.ok(500, "{}"), + MockTransport.ok(500, "{}"), + MockTransport.ok(200, RUN_JSON))); + ApifyClient client = builder(backend).maxRetries(5).timeoutMax(Duration.ofSeconds(15)).build(); + client.run("r1").get().join(); + assertEquals(List.of(secs(5), secs(10), secs(15), secs(15)), backend.timeouts); + } + + @Test + void aTimedOutStartIsNotRetriedSoNoSecondRunIsStarted() { + MockTransport backend = + new MockTransport(List.of(MockTransport.timeoutError(), MockTransport.ok(201, RUN_JSON))); + ApifyClient client = builder(backend).maxRetries(3).build(); + + var future = client.actor("a1").start(null, new ActorStartOptions()); + var error = assertThrows(CompletionException.class, future::join); + assertTrue(error.getCause() instanceof ApifyTransportException); + assertEquals(1, backend.calls, "a start that timed out must not be sent again"); + } + + @Test + void aTimedOutBuildTriggerIsNotRetried() { + MockTransport backend = + new MockTransport(List.of(MockTransport.timeoutError(), MockTransport.ok(201, RUN_JSON))); + ApifyClient client = builder(backend).maxRetries(3).build(); + + var future = client.actor("a1").build("0.1", new ActorBuildOptions()); + assertThrows(CompletionException.class, future::join); + assertEquals(1, backend.calls); + } + + @Test + void setRecordTakesATierAnExactDurationOrNoTimeout() { + MockTransport backend = MockTransport.ofConstant(201, ""); + ApifyClient client = client(backend); + byte[] value = {1}; + + client.keyValueStore("s").setRecord("k", value, "text/plain").join(); + assertEquals(secs(360), backend.lastTimeout, "defaults to the long tier"); + + client + .keyValueStore("s") + .setRecord("k", value, "text/plain", new SetRecordOptions().timeoutSecs(12L)) + .join(); + assertEquals(secs(12), backend.lastTimeout); + + client + .keyValueStore("s") + .setRecord("k", value, "text/plain", new SetRecordOptions().timeout(Timeout.SHORT)) + .join(); + assertEquals(secs(5), backend.lastTimeout); + + client + .keyValueStore("s") + .setRecord("k", value, "text/plain", new SetRecordOptions().timeout(Timeout.NO_TIMEOUT)) + .join(); + assertEquals(Optional.empty(), backend.lastTimeout); + } + + @Test + void timeoutsAreValidated() { + assertThrows(IllegalArgumentException.class, () -> Timeout.of(Duration.ZERO)); + assertThrows(IllegalArgumentException.class, () -> Timeout.ofSeconds(-1)); + assertThrows(IllegalArgumentException.class, () -> Timeout.of(null)); + assertThrows( + IllegalArgumentException.class, () -> ApifyClient.builder().timeoutShort(Duration.ZERO)); + assertThrows(IllegalArgumentException.class, () -> ApifyClient.builder().timeoutMedium(null)); + assertThrows( + IllegalArgumentException.class, + () -> ApifyClient.builder().timeoutLong(Duration.ofSeconds(-1))); + assertThrows( + IllegalArgumentException.class, () -> ApifyClient.builder().timeoutMax(Duration.ZERO)); + assertThrows(IllegalArgumentException.class, () -> ApifyClient.create("t").withTimeout(null)); + } + + @Test + void timeoutValuesDescribeThemselves() { + assertEquals(Timeout.ofSeconds(3), Timeout.of(Duration.ofSeconds(3))); + assertFalse(Timeout.SHORT.equals(Timeout.MEDIUM)); + assertEquals(Optional.of(Timeout.Tier.LONG), Timeout.LONG.tier()); + assertTrue(Timeout.NO_TIMEOUT.isNoTimeout()); + assertEquals(Optional.of(Duration.ofSeconds(3)), Timeout.ofSeconds(3).duration()); + assertEquals("Timeout.NO_TIMEOUT", Timeout.NO_TIMEOUT.toString()); + } +} diff --git a/src/test/java/com/apify/client/UrlNormalizationTest.java b/src/test/java/com/apify/client/UrlNormalizationTest.java new file mode 100644 index 0000000..fd48c9a --- /dev/null +++ b/src/test/java/com/apify/client/UrlNormalizationTest.java @@ -0,0 +1,114 @@ +package com.apify.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.apify.client.dataset.Dataset; +import com.apify.client.internal.Json; +import com.apify.client.internal.UrlNormalizer; +import com.apify.client.run.ActorRun; +import com.apify.client.webhook.Webhook; +import org.junit.jupiter.api.Test; + +/** + * The spec's {@code format: uri} fields come back in the WHATWG URL serialization, so every Apify + * client returns the same string for the same field. + */ +class UrlNormalizationTest { + + @Test + void emptyPathBecomesASlashAndTheHostIsLowercased() { + assertEquals("https://example.com/", UrlNormalizer.normalize("https://EXAMPLE.com")); + assertEquals("http://example.com/a", UrlNormalizer.normalize("HTTP://Example.COM/a")); + } + + @Test + void defaultPortsAreDroppedAndOthersKept() { + assertEquals("https://example.com/", UrlNormalizer.normalize("https://example.com:443")); + assertEquals("http://example.com/", UrlNormalizer.normalize("http://example.com:80/")); + assertEquals( + "https://example.com:8443/x", UrlNormalizer.normalize("https://example.com:8443/x")); + assertEquals("http://example.com:443/", UrlNormalizer.normalize("http://example.com:443/")); + } + + @Test + void nonAsciiHostsArePunycoded() { + assertEquals( + "https://xn--bcher-kva.example/", UrlNormalizer.normalize("https://bücher.example")); + } + + @Test + void unsafeCharactersArePercentEncodedAndExistingEscapesKept() { + assertEquals( + "https://example.com/a%20b?q=%22x%22#f", + UrlNormalizer.normalize("https://example.com/a b?q=\"x\"#f")); + assertEquals("https://example.com/%C3%A9", UrlNormalizer.normalize("https://example.com/é")); + assertEquals("https://example.com/a%2Fb", UrlNormalizer.normalize("https://example.com/a%2Fb")); + } + + @Test + void dotSegmentsAreResolved() { + assertEquals("https://example.com/b", UrlNormalizer.normalize("https://example.com/a/../b")); + assertEquals("https://example.com/a/", UrlNormalizer.normalize("https://example.com/a/./")); + assertEquals("https://example.com/", UrlNormalizer.normalize("https://example.com/..")); + } + + @Test + void queryAndFragmentAreKept() { + assertEquals( + "https://example.com/?a=1&b=2#top", + UrlNormalizer.normalize("https://example.com?a=1&b=2#top")); + } + + @Test + void aNormalizedUrlStaysTheSame() { + String url = "https://console.apify.com/actors/runs/abc?tab=output"; + assertEquals(url, UrlNormalizer.normalize(url)); + } + + @Test + void valuesThatAreNotAbsoluteUrlsAreLeftAlone() { + assertNull(UrlNormalizer.normalize(null)); + assertEquals("not a url", UrlNormalizer.normalize("not a url")); + assertEquals("/relative/path", UrlNormalizer.normalize("/relative/path")); + assertEquals("https:///nohost", UrlNormalizer.normalize("https:///nohost")); + } + + @Test + void responseModelsNormalizeTheirUrlFields() { + String run = + "{\"id\":\"r1\",\"consoleUrl\":\"https://CONSOLE.apify.com\"," + + "\"containerUrl\":\"https://abc.runs.apify.net:443\"," + + "\"output\":{\"dataset\":\"https://api.apify.com:443/v2/datasets/d\"}}"; + ActorRun parsed = + Json.parse(run.getBytes(java.nio.charset.StandardCharsets.UTF_8), ActorRun.class); + assertEquals("https://console.apify.com/", parsed.getConsoleUrl()); + assertEquals("https://abc.runs.apify.net/", parsed.getContainerUrl()); + assertEquals("https://api.apify.com/v2/datasets/d", parsed.getOutput().get("dataset")); + + Dataset dataset = + Json.parse( + "{\"id\":\"d\",\"itemsPublicUrl\":\"https://api.apify.com\",\"title\":\"T\"}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8), + Dataset.class); + assertEquals("https://api.apify.com/", dataset.getItemsPublicUrl()); + assertEquals("T", dataset.getTitle()); + + Webhook webhook = + Json.parse( + "{\"id\":\"w\",\"requestUrl\":\"https://Example.com/hook\"}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8), + Webhook.class); + assertEquals("https://example.com/hook", webhook.getRequestUrl()); + } + + @Test + void aNullUrlFieldStaysNull() { + ActorRun parsed = + Json.parse( + "{\"id\":\"r1\",\"containerUrl\":null}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8), + ActorRun.class); + assertNull(parsed.getContainerUrl()); + } +} diff --git a/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java b/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java index 0f9a872..ce4a351 100644 --- a/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java @@ -24,6 +24,7 @@ import java.util.Optional; import java.util.concurrent.CopyOnWriteArrayList; import org.junit.jupiter.api.Test; +import tools.jackson.databind.JsonNode; class ActorRunIntegrationTest extends IntegrationBase { @@ -46,6 +47,54 @@ void listRuns() { assertTrue(page.getTotal() >= 0); } + @Test + void listRunsFilteredByStandbyDeployment() { + ApifyClient client = requireClient(); + // The deployment filter needs a status filter with only non-terminal statuses; whether any run + // matches depends on the account, so the API accepting the combination is what is asserted. + var filter = + new RunListOptions() + .status(List.of("READY", "RUNNING")) + .deployment(List.of("SINGLE_TENANT", "MULTI_TENANT_FREE_USERS")); + var page = client.runs().list(new ListOptions().limit(5L), filter).join(); + assertTrue(page.getTotal() >= 0); + var actorPage = + client.actor("apify/hello-world").runs().list(new ListOptions().limit(5L), filter).join(); + assertTrue(actorPage.getTotal() >= 0); + } + + @Test + void iterateRunDatasetItemsOfAFinishedRun() { + ApifyClient client = requireClient(); + ActorRun run = + client + .actor("apify/hello-world") + .call(null, new ActorStartOptions(), TEST_ACTOR_WAIT_SECS) + .join(); + assertEquals("SUCCEEDED", run.getStatus()); + List viaRun = + Publishers.collect(client.run(run.getId()).iterateDatasetItems(null)).join(); + List viaDataset = + Publishers.collect( + client + .dataset(run.getDefaultDatasetId()) + .iterateItems(new DatasetListItemsOptions())) + .join(); + assertEquals(viaDataset, viaRun, "the run iterator yields the run's dataset items"); + } + + @Test + void clientWithATimeoutOverrideStillWorks() { + ApifyClient client = requireClient(); + var page = + client + .withTimeout(com.apify.client.http.Timeout.ofSeconds(60)) + .runs() + .list(new ListOptions().limit(1L), new RunListOptions()) + .join(); + assertTrue(page.getTotal() >= 0); + } + @Test void runActorAndReadOutputs() { ApifyClient client = requireClient(); @@ -200,6 +249,8 @@ void runAbortUpdateResurrectDelete() { // Clean up the resurrected run so it doesn't linger on the shared test account. RunClient resurrectedClient = client.run(resurrected.getId()); resurrectedClient.abort(false).join(); + // An aborting run cannot be deleted yet (cannot-remove-running-run), so let it finish first. + resurrectedClient.waitForFinish(60L).join(); resurrectedClient.delete().join(); } diff --git a/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java b/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java index 5a83cf4..054710c 100644 --- a/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java @@ -39,6 +39,26 @@ void getDataset() { } } + @Test + void pushPreSerializedItemsAndReadNormalizedUrls() { + ApifyClient client = requireClient(); + Dataset ds = client.datasets().getOrCreate(uniqueName("ds-json")).join(); + try { + DatasetClient dataset = client.dataset(ds.getId()); + // A String is the JSON serialization of the items, sent as it is. + dataset.pushItems("[{\"n\": 1}, {\"n\": 2}]").join(); + var items = dataset.listItems(new DatasetListItemsOptions()).join(); + assertEquals(2, items.getItems().size()); + + var fetched = dataset.get().join().orElseThrow(); + assertTrue( + fetched.getConsoleUrl() != null && fetched.getConsoleUrl().startsWith("https://"), + "consoleUrl is typed and a normalized absolute URL: " + fetched.getConsoleUrl()); + } finally { + client.dataset(ds.getId()).delete().join(); + } + } + @Test void datasetCrudFlow() throws java.io.IOException { ApifyClient client = requireClient(); diff --git a/src/test/java/com/apify/client/integration/IntegrationBase.java b/src/test/java/com/apify/client/integration/IntegrationBase.java index f2c50af..ab0d127 100644 --- a/src/test/java/com/apify/client/integration/IntegrationBase.java +++ b/src/test/java/com/apify/client/integration/IntegrationBase.java @@ -55,16 +55,16 @@ public abstract class IntegrationBase { /** * {@link #pollUntil} attempt count bounding how long a CRUD-flow test waits for a just-created - * resource to surface in its own top-level collection {@code list()} (a write and the LIST - * endpoint's index can converge asynchronously - the same class of eventual-consistency race as - * {@link #STREAM_CATCH_UP_ATTEMPTS} and {@code ActorRunIntegrationTest#RUN_LIST_FIND_ATTEMPTS}, - * just with a shorter budget since these are exclusively-owned resources, not entries in a - * heavily-shared public collection). + * resource to surface in its own top-level collection {@code list()}. The list endpoints read + * from database secondaries with up to 90 s of staleness, so under load a fresh resource can be + * missing from the listing for that long. {@link #LIST_FIND_ATTEMPTS} attempts, {@link + * #LIST_FIND_BACKOFF_MILLIS} apart, cover that window; the wait still ends as soon as the + * resource shows up. */ - static final int LIST_FIND_ATTEMPTS = 5; + static final int LIST_FIND_ATTEMPTS = 30; /** Poll interval used while waiting out {@link #LIST_FIND_ATTEMPTS}. */ - static final long LIST_FIND_BACKOFF_MILLIS = 500L; + static final long LIST_FIND_BACKOFF_MILLIS = 3_000L; private static final SecureRandom RANDOM = new SecureRandom(); diff --git a/src/test/java/com/apify/client/integration/StoreIntegrationTest.java b/src/test/java/com/apify/client/integration/StoreIntegrationTest.java index 9b2f7d2..df562b2 100644 --- a/src/test/java/com/apify/client/integration/StoreIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/StoreIntegrationTest.java @@ -19,6 +19,18 @@ void listStore() { assertTrue(page.getItems().size() <= 5); } + @Test + void listStoreWithInputSchema() { + ApifyClient client = requireClient(); + // With includeInputSchema the API defaults limit to (and caps it at) 10. + var page = + client.store().list(new StoreListOptions().includeInputSchema(true).limit(3L)).join(); + assertTrue(page.getItems().size() <= 3); + assertTrue( + page.getItems().stream().allMatch(item -> item.getExtra().containsKey("inputSchema")), + "every listed Actor carries its compact inputSchema"); + } + @Test void iterateStore() { ApifyClient client = requireClient(); diff --git a/src/test/java/com/apify/client/integration/UserIntegrationTest.java b/src/test/java/com/apify/client/integration/UserIntegrationTest.java index 5f2993f..3ebf0e1 100644 --- a/src/test/java/com/apify/client/integration/UserIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/UserIntegrationTest.java @@ -49,6 +49,17 @@ void getLimits() { assertTrue(limits != null && !limits.isEmpty()); } + @Test + void getLimitsAndMonthlyUsageByUserId() { + ApifyClient client = requireClient(); + var me = client.me().get().join().orElseThrow(); + // users/{userId}/limits and users/{userId}/usage/monthly address the own account by its id. + JsonNode limits = client.user(me.getId()).limits().join(); + assertTrue(limits != null && !limits.isEmpty()); + JsonNode usage = client.user(me.getId()).monthlyUsage().join(); + assertTrue(usage != null && !usage.isEmpty()); + } + // Note: `me().updateLimits(...)` is intentionally NOT exercised by a live integration test. It // mutates account-global state that cannot be isolated between the concurrent cross-language test // runs the test-requirements mandate (unlike the per-run resources every other test creates), so From 096ccb3661d3bd34cc9ea4eeefa7b287cbe8678f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 16:37:37 +0000 Subject: [PATCH 2/6] fix: address review of the v3 port (cancellation, hold clamping, URL normalization, tail probe) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAKY9wkdABFznxvsCXt --- CHANGELOG.md | 12 +- README.md | 9 +- docs/README.md | 2 +- docs/actors.md | 2 +- .../java/com/apify/client/Publishers.java | 10 +- .../com/apify/client/actor/ActorClient.java | 21 +-- .../java/com/apify/client/build/Build.java | 7 +- .../com/apify/client/build/BuildClient.java | 6 +- .../dataset/DatasetListItemsOptions.java | 68 +++++--- .../client/http/DefaultHttpTransport.java | 7 + .../java/com/apify/client/internal/Async.java | 18 ++- .../internal/AsyncPaginatedPublisher.java | 38 +++-- .../client/internal/CancellableFuture.java | 151 ++++++++++++++++++ .../internal/DatasetItemsOptionsAccess.java | 56 +++++++ .../apify/client/internal/HttpClientCore.java | 99 ++++++++---- .../apify/client/internal/ListPublisher.java | 1 + .../apify/client/internal/QueryParams.java | 6 + .../client/internal/ResourceContext.java | 33 +--- .../client/internal/RunStartSupport.java | 26 ++- .../apify/client/internal/UrlNormalizer.java | 46 ++++-- .../client/keyvalue/KeyValueStoreClient.java | 41 +++-- .../requestqueue/RequestQueueClient.java | 44 ++--- .../java/com/apify/client/run/RunClient.java | 9 +- .../com/apify/client/run/RunDatasetItems.java | 19 ++- .../com/apify/client/CancellationTest.java | 151 ++++++++++++++++++ .../com/apify/client/CompressionTest.java | 28 ++++ .../java/com/apify/client/MockTransport.java | 26 +++ .../com/apify/client/RunDatasetItemsTest.java | 33 +++- .../com/apify/client/TimeoutTierTest.java | 50 +++--- .../apify/client/UrlNormalizationTest.java | 24 +++ 30 files changed, 819 insertions(+), 224 deletions(-) create mode 100644 src/main/java/com/apify/client/internal/CancellableFuture.java create mode 100644 src/main/java/com/apify/client/internal/DatasetItemsOptionsAccess.java create mode 100644 src/test/java/com/apify/client/CancellationTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index a12f2ff..6f0c37b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,14 +44,18 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). only one that runs, and its failure fails the request. Brotli stays the default (quality 6) where its native library loads, gzip elsewhere. - `batchAddRequests` serializes each request once instead of twice. -- A `waitForFinish` hold is no longer clamped below the request timeout; the timeout of the request - is extended by the hold instead. +- A `waitForFinish` hold extends the timeout of its request (the tier's plus the hold) instead of + being clamped below it; a hold is cut down only to what `timeoutMax` (or a `withTimeout` override) + leaves, so the API still answers before the client gives up. +- `Build.getChangelog()` is deprecated, as the specification deprecates `Build.changelog`. - The integration tests wait up to 90 s for a just-created resource to appear in a listing. ### Fixed -- A timed-out Actor or task start, build trigger or `pushItems` request is no longer retried, so it - cannot start a second billed run or store items twice. +- A `pushItems` request that times out is no longer retried, so the items cannot be stored twice. +- Cancelling a future the client returned (`CompletableFuture.cancel`) now aborts the request in + flight and stops the retries, `waitForFinish`/`call` polling and iteration pages that would follow; + cancelling a derived stage (`thenApply`, `thenCompose`, ...) cancels the call it comes from. - `pushItems` rejects what the API rejects (anything but an object or an array of objects) before a request is sent, and sends a pre-serialized `String` as it is. - An Actor or task input given as `byte[]` or `String` is sent as it is instead of being wrapped in diff --git a/README.md b/README.md index 5ab1b29..b3eb7b8 100644 --- a/README.md +++ b/README.md @@ -206,8 +206,11 @@ List items = An override is used as given, so for a `waitForFinish` request it has to cover the hold too. `KeyValueStoreClient.setRecord(...)` additionally takes the timeout in its `SetRecordOptions`. -A request that creates a billed resource (an Actor or task run, a build) or pushes dataset items is -not retried when it times out, since the API may already have processed it. +A `pushItems` request that times out is not retried, since the API may already have stored the items. + +A hold is cut down to what the request timeout can cover: with a small `timeoutMax` (or an override +from `withTimeout`), `waitForFinish` is sent as a smaller value so the API answers before the client +gives up on the request. ### Request compression and default headers @@ -381,7 +384,7 @@ constants: - `Version.CLIENT_VERSION` — the semantic version of this client (`0.12.0`). - `Version.API_SPEC_VERSION` — the version of the [Apify OpenAPI specification](https://docs.apify.com/api/openapi.json) (its `info.version` field) that this client's endpoints, parameters and models were last generated - and checked against (`v2-2026-10-26T112451Z`). It is a snapshot, not a live compatibility + and checked against (`v2-2026-10-29T100357Z`). It is a snapshot, not a live compatibility guarantee: the client keeps working against newer, backward-compatible spec revisions, but a feature added to the API after this snapshot has no corresponding method here yet. diff --git a/docs/README.md b/docs/README.md index f9a8294..422a603 100644 --- a/docs/README.md +++ b/docs/README.md @@ -44,7 +44,7 @@ option types, so import each resource you use from its own package: | `com.apify.client.http` | `ApifyClientException`, `ApifyApiException`, `ApifyTransportException`, `HttpTransport`, `DefaultHttpTransport`, `HttpTimeoutException`, `Timeout`, `HttpCompressor`, `BrotliHttpCompressor`, `GzipHttpCompressor` | | `com.apify.client.actor` | `Actor`, `ActorClient`, `ActorCollectionClient`, `ActorListOptions`, `ActorStartOptions`, `ActorCallOptions`, `ActorBuildOptions`, `ActorStandby`, `ActorVersion`, `SourceCodeFile`, `ActorVersionClient`, `ActorVersionCollectionClient`, `ActorEnvVar`, `ActorEnvVarClient`, `ActorEnvVarCollectionClient`, `ValidateInputOptions`, `ActorStats`, `ActorDefaultRunOptions` | | `com.apify.client.build` | `Build`, `BuildClient`, `BuildCollectionClient`, `BuildActVersion`, `ActorDefinition`, `ActorDefinitionMeta`, `ActorDefinitionStorages`, `BuildMeta`, `BuildStats`, `BuildOptions`, `BuildUsage` | -| `com.apify.client.run` | `ActorRun`, `ActorRunStats`, `ActorRunOptions`, `ActorRunMeta`, `ActorRunUsage`, `RunClient`, `RunCollectionClient`, `RunListOptions`, `LastRunOptions`, `MetamorphOptions`, `RunChargeOptions`, `RunResurrectOptions`, `SetStatusMessageOptions` | +| `com.apify.client.run` | `ActorRun`, `ActorRunStats`, `ActorRunOptions`, `ActorRunMeta`, `ActorRunStandby`, `ActorRunUsage`, `RunClient`, `RunCollectionClient`, `RunListOptions`, `LastRunOptions`, `MetamorphOptions`, `RunChargeOptions`, `RunResurrectOptions`, `SetStatusMessageOptions` | | `com.apify.client.dataset` | `Dataset`, `DatasetClient`, `DatasetCollectionClient`, `DatasetListItemsOptions`, `DatasetDownloadOptions`, `DownloadItemsFormat` | | `com.apify.client.keyvalue` | `KeyValueStore`, `KeyValueStoreClient`, `KeyValueStoreCollectionClient`, `KeyValueStoreRecord`, `KeyValueStoreKeysPage`, `KeyValueStoreKey`, `GetRecordOptions`, `SetRecordOptions`, `ListKeysOptions` | | `com.apify.client.requestqueue` | `RequestQueue`, `RequestQueueClient`, `RequestQueueCollectionClient`, `RequestQueueRequest`, `RequestQueueHead`, `LockedRequestQueueHead`, `RequestQueueOperationInfo`, `RequestLockInfo`, `UnlockRequestsResult`, `RequestsList`, `BatchAddResult`, `BatchDeleteResult`, `DeletedRequestInfo`, `ListRequestsOptions`, `BatchAddRequestsOptions` | diff --git a/docs/actors.md b/docs/actors.md index 4655b21..f4e5a70 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -46,7 +46,7 @@ Actor created = client.actors().create(Map.of( | `get()` | Fetch the Actor. Completes with `Optional`. | | `update(Object)` | Update the Actor with the given fields. Completes with `Actor`. | | `delete()` | Delete the Actor. | -| `start(Object input, ActorStartOptions)` | Start a run, returning immediately. Completes with `ActorRun`. The `input` is any JSON-serializable value, or a `byte[]`/`String` that is sent as it is (set the matching `ActorStartOptions.contentType(...)`), or `null`. A start that times out is not retried, so no second run is started. | +| `start(Object input, ActorStartOptions)` | Start a run, returning immediately. Completes with `ActorRun`. The `input` is any JSON-serializable value, or a `byte[]`/`String` that is sent as it is (set the matching `ActorStartOptions.contentType(...)`), or `null`. | | `call(Object input, ActorStartOptions, Long waitSecs)` | Start a run and poll until it finishes (`null` waits indefinitely); does **not** stream the run's log. Completes with `ActorRun`. | | `call(Object input, ActorCallOptions, Long waitSecs)` | As above, additionally streaming the run's log for the duration of the wait by default (matching the reference client's `call` defaulting `options.log` to `'default'`). Use `ActorCallOptions.disableLogStreaming()` to opt out, or `logOptions(StreamedLogOptions)` for a custom destination. | | `validateInput(Object input)` / `validateInput(Object input, ValidateInputOptions)` | Validate an input against the Actor's input schema. Completes with `boolean`. `ValidateInputOptions` fields (optional): `build` (`String`), `contentType` (`String`). | diff --git a/src/main/java/com/apify/client/Publishers.java b/src/main/java/com/apify/client/Publishers.java index cf10cfa..fcb1926 100644 --- a/src/main/java/com/apify/client/Publishers.java +++ b/src/main/java/com/apify/client/Publishers.java @@ -1,5 +1,6 @@ package com.apify.client; +import com.apify.client.internal.CancellableFuture; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; @@ -30,13 +31,20 @@ private Publishers() {} * publisher signals completion, or exceptionally with whatever error the publisher signalled. */ public static CompletableFuture> collect(Flow.Publisher publisher) { - CompletableFuture> result = new CompletableFuture<>(); + CompletableFuture> result = new CancellableFuture<>(); ConcurrentLinkedQueue items = new ConcurrentLinkedQueue<>(); AtomicBoolean done = new AtomicBoolean(); publisher.subscribe( new Flow.Subscriber<>() { @Override public void onSubscribe(Flow.Subscription subscription) { + // Cancelling the returned future cancels the subscription (and so the page in flight). + result.whenComplete( + (list, error) -> { + if (result.isCancelled()) { + subscription.cancel(); + } + }); subscription.request(Long.MAX_VALUE); } diff --git a/src/main/java/com/apify/client/actor/ActorClient.java b/src/main/java/com/apify/client/actor/ActorClient.java index dfd327e..36666dc 100644 --- a/src/main/java/com/apify/client/actor/ActorClient.java +++ b/src/main/java/com/apify/client/actor/ActorClient.java @@ -161,15 +161,11 @@ public CompletableFuture build(String versionNumber, ActorBuildOptions op QueryParams params = new QueryParams(); params.addString("version", versionNumber); options.apply(params); + HttpClientCore.WaitHold hold = + ctx.http.waitHold(Timeout.MEDIUM, params.getLong("waitForFinish")); + params.setLong("waitForFinish", hold.waitForFinishSecs()); return ctx.postWithBody( - "builds", - params, - null, - ResourceContext.CONTENT_TYPE_JSON, - Build.class, - ctx.http.timeoutWithHold(Timeout.MEDIUM, params.getLong("waitForFinish")), - // A build that timed out may still have been created; retrying would start a second one. - true); + "builds", params, null, ResourceContext.CONTENT_TYPE_JSON, Build.class, hold.timeout()); } /** @@ -179,12 +175,9 @@ public CompletableFuture build(String versionNumber, ActorBuildOptions op */ public CompletableFuture defaultBuild(Long waitForFinish) { QueryParams params = new QueryParams(); - params.addLong("waitForFinish", waitForFinish); - return ctx.getResourceRequired( - "builds/default", - params, - Build.class, - ctx.http.timeoutWithHold(Timeout.SHORT, waitForFinish)) + HttpClientCore.WaitHold hold = ctx.http.waitHold(Timeout.SHORT, waitForFinish); + params.addLong("waitForFinish", hold.waitForFinishSecs()); + return ctx.getResourceRequired("builds/default", params, Build.class, hold.timeout()) .thenApply(build -> new BuildClient(http, baseUrl, build.getId())); } diff --git a/src/main/java/com/apify/client/build/Build.java b/src/main/java/com/apify/client/build/Build.java index 0b9fd7f..8a60e80 100644 --- a/src/main/java/com/apify/client/build/Build.java +++ b/src/main/java/com/apify/client/build/Build.java @@ -127,7 +127,12 @@ public boolean isTerminal() { return Statuses.isTerminal(status); } - /** The changelog of the Actor version built. */ + /** + * The changelog of the Actor version built. + * + * @deprecated The OpenAPI specification marks {@code Build.changelog} as deprecated. + */ + @Deprecated public String getChangelog() { return changelog; } diff --git a/src/main/java/com/apify/client/build/BuildClient.java b/src/main/java/com/apify/client/build/BuildClient.java index 844f7eb..4004234 100644 --- a/src/main/java/com/apify/client/build/BuildClient.java +++ b/src/main/java/com/apify/client/build/BuildClient.java @@ -34,9 +34,9 @@ public CompletableFuture> get() { */ public CompletableFuture> getWithWait(Long waitForFinishSecs) { QueryParams params = new QueryParams(); - params.addLong("waitForFinish", waitForFinishSecs); - return ctx.getResource( - "", params, Build.class, ctx.http.timeoutWithHold(Timeout.SHORT, waitForFinishSecs)); + HttpClientCore.WaitHold hold = ctx.http.waitHold(Timeout.SHORT, waitForFinishSecs); + params.addLong("waitForFinish", hold.waitForFinishSecs()); + return ctx.getResource("", params, Build.class, hold.timeout()); } /** Aborts the build and returns its updated state. */ diff --git a/src/main/java/com/apify/client/dataset/DatasetListItemsOptions.java b/src/main/java/com/apify/client/dataset/DatasetListItemsOptions.java index f837e04..0e6399a 100644 --- a/src/main/java/com/apify/client/dataset/DatasetListItemsOptions.java +++ b/src/main/java/com/apify/client/dataset/DatasetListItemsOptions.java @@ -1,5 +1,6 @@ package com.apify.client.dataset; +import com.apify.client.internal.DatasetItemsOptionsAccess; import com.apify.client.internal.QueryParams; import java.util.List; @@ -121,8 +122,53 @@ public DatasetListItemsOptions signature(String signature) { return this; } + static { + DatasetItemsOptionsAccess.register( + new DatasetItemsOptionsAccess() { + @Override + public long offset(DatasetListItemsOptions options) { + return options.offset != null && options.offset > 0 ? options.offset : 0; + } + + @Override + public Long limit(DatasetListItemsOptions options) { + return options.limit; + } + + @Override + public boolean dropsRows(DatasetListItemsOptions options) { + return Boolean.TRUE.equals(options.clean) + || Boolean.TRUE.equals(options.skipEmpty) + || (options.unwind != null && !options.unwind.isEmpty()); + } + + @Override + public DatasetListItemsOptions snapshot(DatasetListItemsOptions options) { + DatasetListItemsOptions c = + options != null ? options.copy() : new DatasetListItemsOptions(); + c.desc = null; + return c; + } + + @Override + public DatasetListItemsOptions page( + DatasetListItemsOptions options, long offset, long limit) { + DatasetListItemsOptions c = options.copy(); + c.desc = null; + c.offset = offset; + c.limit = limit; + return c; + } + + @Override + public DatasetListItemsOptions plainPage(long offset, long limit) { + return new DatasetListItemsOptions().offset(offset).limit(limit); + } + }); + } + /** Returns an independent copy of these options. */ - public DatasetListItemsOptions copy() { + private DatasetListItemsOptions copy() { DatasetListItemsOptions c = new DatasetListItemsOptions(); c.offset = offset; c.limit = limit; @@ -142,26 +188,6 @@ public DatasetListItemsOptions copy() { return c; } - /** - * Whether {@code clean}, {@code skipEmpty} or {@code unwind} is set: the server-side filters that - * can leave a page of existing rows without a single returned item. - */ - public boolean dropsRows() { - return Boolean.TRUE.equals(clean) - || Boolean.TRUE.equals(skipEmpty) - || (unwind != null && !unwind.isEmpty()); - } - - /** The first row to read, or 0 when not set. */ - public long offsetValueOrZero() { - return offset != null && offset > 0 ? offset : 0; - } - - /** The maximum number of rows to read, or {@code null} when not set. */ - public Long limitValueOrNull() { - return limit; - } - Boolean descValue() { return desc; } diff --git a/src/main/java/com/apify/client/http/DefaultHttpTransport.java b/src/main/java/com/apify/client/http/DefaultHttpTransport.java index 325da87..027ef37 100644 --- a/src/main/java/com/apify/client/http/DefaultHttpTransport.java +++ b/src/main/java/com/apify/client/http/DefaultHttpTransport.java @@ -69,6 +69,13 @@ public CompletableFuture> sendStreamingAsync(HttpReque */ private static CompletableFuture unwrapTimeout(CompletableFuture future) { CompletableFuture result = new CompletableFuture<>(); + // Cancelling the returned future aborts the exchange the JDK client has in flight. + result.whenComplete( + (value, error) -> { + if (result.isCancelled()) { + future.cancel(true); + } + }); future.whenComplete( (value, error) -> { if (error == null) { diff --git a/src/main/java/com/apify/client/internal/Async.java b/src/main/java/com/apify/client/internal/Async.java index e773c8e..4c0aea8 100644 --- a/src/main/java/com/apify/client/internal/Async.java +++ b/src/main/java/com/apify/client/internal/Async.java @@ -4,6 +4,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; @@ -32,11 +33,22 @@ public final class Async { private Async() {} - /** Returns a future that completes after {@code delay} has elapsed (never negative/blocking). */ + /** + * Returns a future that completes after {@code delay} has elapsed (never negative/blocking). + * Cancelling it cancels the timer. + */ public static CompletableFuture delay(Duration delay) { - CompletableFuture future = new CompletableFuture<>(); + CancellableFuture future = new CancellableFuture<>(); long millis = Math.max(0, delay.toMillis()); - SCHEDULER.schedule(() -> future.complete(null), millis, TimeUnit.MILLISECONDS); + ScheduledFuture timer = + SCHEDULER.schedule(() -> future.complete(null), millis, TimeUnit.MILLISECONDS); + // A cancelled delay releases its timer instead of firing into a future nobody waits for. + future.whenComplete( + (unused, error) -> { + if (future.isCancelled()) { + timer.cancel(false); + } + }); return future; } } diff --git a/src/main/java/com/apify/client/internal/AsyncPaginatedPublisher.java b/src/main/java/com/apify/client/internal/AsyncPaginatedPublisher.java index 155cfbf..b3fbf54 100644 --- a/src/main/java/com/apify/client/internal/AsyncPaginatedPublisher.java +++ b/src/main/java/com/apify/client/internal/AsyncPaginatedPublisher.java @@ -112,6 +112,7 @@ private final class Session implements Flow.Subscription { private long yielded; private boolean exhausted; private boolean fetchInFlight; + private volatile CompletableFuture inFlight; Session(Flow.Subscriber subscriber) { this.subscriber = subscriber; @@ -141,6 +142,11 @@ public void request(long n) { @Override public void cancel() { cancelled.set(true); + // Abort the page being fetched, with its request and retries. + CompletableFuture fetch = inFlight; + if (fetch != null) { + fetch.cancel(true); + } } /** Enters the drain loop unless another drain (of this same session) is already running. */ @@ -203,22 +209,22 @@ private void fetchNextPage() { fetchInFlight = true; Long capRemaining = totalLimit != null ? totalLimit - yielded : null; Long pageLimit = minForLimit(capRemaining, chunkSize); - fetcher - .fetch(offset, pageLimit) - .whenComplete( - (page, error) -> { - fetchInFlight = false; - if (cancelled.get()) { - draining.set(false); - return; - } - if (error != null) { - terminate(() -> subscriber.onError(HttpClientCore.unwrapCompletion(error))); - return; - } - applyPage(page); - drainLoop(); - }); + CompletableFuture> fetch = fetcher.fetch(offset, pageLimit); + inFlight = fetch; + fetch.whenComplete( + (page, error) -> { + fetchInFlight = false; + if (cancelled.get()) { + draining.set(false); + return; + } + if (error != null) { + terminate(() -> subscriber.onError(HttpClientCore.unwrapCompletion(error))); + return; + } + applyPage(page); + drainLoop(); + }); } private void applyPage(PaginationList page) { diff --git a/src/main/java/com/apify/client/internal/CancellableFuture.java b/src/main/java/com/apify/client/internal/CancellableFuture.java new file mode 100644 index 0000000..fe93c3f --- /dev/null +++ b/src/main/java/com/apify/client/internal/CancellableFuture.java @@ -0,0 +1,151 @@ +package com.apify.client.internal; + +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.function.Function; + +/** + * A {@link CompletableFuture} whose cancellation reaches the work behind it. Cancelling a plain + * {@code CompletableFuture} completes only that future: the request it waits for, the retry or poll + * chain that produced it and the timers that drive it keep running. Every future the client returns + * is a {@code CancellableFuture}, and so is every stage derived from it with {@code thenApply}, + * {@code thenCompose}, {@code handle}, ...: cancelling the future a caller holds cancels each + * upstream stage (and the future {@code thenCompose} is waiting on) down to the HTTP request in + * flight, and stops the retries, polls and pages that would follow. Internal to the client. + */ +public final class CancellableFuture extends CompletableFuture { + + /** The futures this one depends on, cancelled together with it. */ + private final ConcurrentLinkedQueue> upstream = + new ConcurrentLinkedQueue<>(); + + public CancellableFuture() {} + + /** + * A future that completes as {@code source} does and, when cancelled, cancels {@code source}. + * Wraps the future a transport returns, so cancelling a call aborts the request in flight. + */ + public static CancellableFuture wrap(CompletionStage source) { + CancellableFuture result = new CancellableFuture<>(); + CompletableFuture sourceFuture = source.toCompletableFuture(); + result.link(sourceFuture); + sourceFuture.whenComplete( + (value, error) -> { + if (error == null) { + result.complete(value); + } else { + result.completeExceptionally(error); + } + }); + return result; + } + + /** An already completed future. */ + public static CancellableFuture completed(T value) { + CancellableFuture future = new CancellableFuture<>(); + future.complete(value); + return future; + } + + /** An already failed future. */ + public static CancellableFuture failed(Throwable error) { + CancellableFuture future = new CancellableFuture<>(); + future.completeExceptionally(error); + return future; + } + + /** A future that completes once all of {@code futures} have, and cancels them when cancelled. */ + public static CancellableFuture allOf(List> futures) { + CancellableFuture result = new CancellableFuture<>(); + futures.forEach(result::link); + CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)) + .whenComplete( + (unused, error) -> { + if (error == null) { + result.complete(null); + } else { + result.completeExceptionally(error); + } + }); + return result; + } + + /** Makes {@code other} be cancelled whenever this future is (now, if it already is). */ + public void link(CompletableFuture other) { + upstream.add(other); + if (isCancelled()) { + other.cancel(true); + } + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + boolean cancelled = super.cancel(mayInterruptIfRunning); + if (cancelled) { + for (CompletableFuture source : upstream) { + source.cancel(mayInterruptIfRunning); + } + } + return cancelled; + } + + /** Every stage derived from this one cancels this one when it is cancelled. */ + @Override + public CompletableFuture newIncompleteFuture() { + CancellableFuture dependent = new CancellableFuture<>(); + dependent.upstream.add(this); + return dependent; + } + + /** + * As {@link CompletableFuture#thenCompose}, additionally linking the future {@code fn} returns, + * so cancelling the composed stage cancels the work it was waiting on, and skipping {@code fn} + * altogether once the composed stage is cancelled (so a cancelled call sends no further request). + */ + @Override + public CompletableFuture thenCompose( + Function> fn) { + Composition composition = new Composition<>(); + CompletableFuture composed = + super.thenCompose( + value -> { + CancellableFuture dependent = composition.dependent; + if (dependent != null && dependent.isCancelled()) { + return failed(new CancellationException()); + } + CompletionStage inner = fn.apply(value); + CompletableFuture innerFuture = inner.toCompletableFuture(); + synchronized (composition) { + dependent = composition.dependent; + if (dependent == null) { + composition.inner = innerFuture; + } + } + if (dependent != null) { + dependent.link(innerFuture); + } + return inner; + }); + CancellableFuture dependent = (CancellableFuture) composed; + CompletableFuture early; + synchronized (composition) { + composition.dependent = dependent; + early = composition.inner; + } + if (early != null) { + dependent.link(early); + } + return composed; + } + + /** + * Hands the composed stage and the future {@code fn} returned to each other, whichever is first. + */ + private static final class Composition { + volatile CancellableFuture dependent; + CompletableFuture inner; + } +} diff --git a/src/main/java/com/apify/client/internal/DatasetItemsOptionsAccess.java b/src/main/java/com/apify/client/internal/DatasetItemsOptionsAccess.java new file mode 100644 index 0000000..6139505 --- /dev/null +++ b/src/main/java/com/apify/client/internal/DatasetItemsOptionsAccess.java @@ -0,0 +1,56 @@ +package com.apify.client.internal; + +import com.apify.client.dataset.DatasetListItemsOptions; + +/** + * Cross-package access to the item options of {@link DatasetListItemsOptions}, whose accessors are + * package-private because only the client itself needs them. {@code DatasetListItemsOptions} + * registers its implementation when it is loaded, so code in another package (the run client's + * dataset iteration) can read and derive options without those accessors being public API. Internal + * to the client. + */ +public interface DatasetItemsOptionsAccess { + + /** The registered implementation; the options class is loaded before it is looked up. */ + static DatasetItemsOptionsAccess get() { + // Loading the options class runs its static initializer, which registers the implementation. + new DatasetListItemsOptions(); + return Holder.instance; + } + + /** Registers the implementation; called once by {@link DatasetListItemsOptions}. */ + static void register(DatasetItemsOptionsAccess access) { + Holder.instance = access; + } + + /** The first row {@code options} asks for, or 0. */ + long offset(DatasetListItemsOptions options); + + /** The maximum number of rows {@code options} asks for, or {@code null}. */ + Long limit(DatasetListItemsOptions options); + + /** + * Whether {@code clean}, {@code skipEmpty} or {@code unwind} is set: the server-side filters that + * can leave a page of existing rows without a single returned item. + */ + boolean dropsRows(DatasetListItemsOptions options); + + /** + * An independent copy of {@code options} (the defaults for {@code null}) for reading rows in the + * order they were pushed, so a later change of the caller's options cannot reach the iteration. + */ + DatasetListItemsOptions snapshot(DatasetListItemsOptions options); + + /** A copy of {@code options} for the given page of rows, in the order they were pushed. */ + DatasetListItemsOptions page(DatasetListItemsOptions options, long offset, long limit); + + /** Options for the given page of rows with none of the item filters of the caller. */ + DatasetListItemsOptions plainPage(long offset, long limit); + + /** Holds the registered implementation. */ + final class Holder { + private static volatile DatasetItemsOptionsAccess instance; + + private Holder() {} + } +} diff --git a/src/main/java/com/apify/client/internal/HttpClientCore.java b/src/main/java/com/apify/client/internal/HttpClientCore.java index 0e457af..00854c8 100644 --- a/src/main/java/com/apify/client/internal/HttpClientCore.java +++ b/src/main/java/com/apify/client/internal/HttpClientCore.java @@ -28,6 +28,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; @@ -73,6 +74,14 @@ public final class HttpClientCore { /** The API holds a {@code waitForFinish} request for at most this many seconds. */ private static final long MAX_HOLD_SECS = 60; + /** + * Seconds of a caller-chosen request timeout kept free of a {@code waitForFinish} hold, so the + * API answers before the client's own timeout fires. + */ + private static final long HOLD_MARGIN_SECS = 5; + + private static final double MILLIS_PER_SECOND = 1000.0; + private static final Logger LOG = LoggerFactory.getLogger(HttpClientCore.class); private final HttpTransport transport; @@ -161,11 +170,6 @@ public HttpClientCore withTimeout(Timeout timeout) { warnedCappedTimeouts); } - /** Whether {@link #withTimeout} replaced the per-method timeouts of this client. */ - public boolean hasTimeoutOverride() { - return timeoutOverride != null; - } - /** * Resolves a timeout to the duration of its first attempt: a tier to its configured duration, an * exact timeout to itself, and {@link Timeout#NO_TIMEOUT} to {@code null} (no timeout). A value @@ -193,8 +197,8 @@ public Duration resolveTimeout(Timeout requested) { LOG.warn( "The requested timeout of {}s exceeds timeoutMax ({}s) and is capped at it. Raise" + " timeoutMax on the client builder to allow longer request timeouts.", - duration.toMillis() / 1000.0, - retry.timeout.toMillis() / 1000.0); + duration.toMillis() / MILLIS_PER_SECOND, + retry.timeout.toMillis() / MILLIS_PER_SECOND); } return retry.timeout; } @@ -202,19 +206,42 @@ public Duration resolveTimeout(Timeout requested) { } /** - * The timeout for a request that asks the API to hold its response for {@code holdSecs} with - * {@code waitForFinish}: the hold plus the round trip {@code tier} allows, so the client does not - * abort a request the API is still holding (which, for a start or build, would retry a call that - * creates a resource). An override set by {@link #withTimeout} is used as given. + * The {@code waitForFinish} value to send and the timeout to send it with, for a request that + * asks the API to hold its response for up to {@code requestedHold} seconds. + * + *

The hold is at most {@link #MAX_HOLD_SECS} and is cut down to what the request timeout can + * cover, so the client never aborts a request the API is still holding (which, for a start or a + * build, would retry a call that creates a resource). Without an override the timeout is the + * tier's plus the hold, and the hold is limited by what {@code timeoutMax} leaves above the tier. + * An override set by {@link #withTimeout} is kept as given, and the hold is limited to it less a + * safety margin. */ - public Timeout timeoutWithHold(Timeout tier, Long holdSecs) { - if (timeoutOverride != null || tier.isNoTimeout() || holdSecs == null || holdSecs <= 0) { - return tier; + public WaitHold waitHold(Timeout tier, Long requestedHold) { + if (requestedHold == null) { + return new WaitHold(null, tier); } + long hold = Math.min(Math.max(requestedHold, 0), MAX_HOLD_SECS); Duration base = resolveTimeout(tier); - return Timeout.of(base.plusSeconds(Math.min(holdSecs, MAX_HOLD_SECS))); + if (base == null) { + return new WaitHold(hold, tier); + } + if (timeoutOverride != null) { + long coverable = Math.max(0, base.toSeconds() - HOLD_MARGIN_SECS); + return new WaitHold(Math.min(hold, coverable), tier); + } + hold = Math.min(hold, Math.max(0, retry.timeout.toSeconds() - base.toSeconds())); + return new WaitHold(hold, hold > 0 ? Timeout.of(base.plusSeconds(hold)) : tier); } + /** + * The outcome of {@link #waitHold}. + * + * @param waitForFinishSecs the {@code waitForFinish} query value to send, or {@code null} for + * none + * @param timeout the timeout to send the request with + */ + public record WaitHold(Long waitForFinishSecs, Timeout timeout) {} + /** Sends a request with auth, User-Agent and the retry policy applied. */ public CompletableFuture call( String method, String url, byte[] body, String contentType, Timeout timeout) { @@ -269,8 +296,9 @@ public CompletableFuture call( if (shouldCompress(requestBody, contentType, extraHeaders)) { try { requestBody = compressor.compress(requestBody); - } catch (IOException | RuntimeException e) { - // No fallback to another coding: the configured compressor is the only one that runs. + } catch (Exception | LinkageError e) { + // No fallback to another coding: the configured compressor is the only one that runs. A + // LinkageError is what a native codec that fails to load raises. return failedFuture(new ApifyTransportException(e)); } headers = new LinkedHashMap<>(extraHeaders == null ? Map.of() : extraHeaders); @@ -371,6 +399,10 @@ private CompletableFuture attempt( null, false, isStatusRetryable(resp.statusCode()), apiError); } Throwable cause = unwrapCompletion(err); + if (cause instanceof CancellationException cancellation) { + // The caller cancelled the call: end it, never retry. + return new AttemptOutcome(null, false, false, cancellation); + } ApifyTransportException transportError = cause instanceof ApifyTransportException apifyTransportException ? apifyTransportException @@ -424,8 +456,7 @@ private CompletableFuture doAttempt( Map extraHeaders, Duration timeout) { HttpRequest request = buildRequest(method, url, body, contentType, extraHeaders, timeout); - return transport - .sendAsync(request) + return CancellableFuture.wrap(transport.sendAsync(request)) .thenApply(resp -> new ApiResponse(resp.statusCode(), resp.headers(), resp.body())); } @@ -531,8 +562,8 @@ private static boolean shouldCompress( * ending in {@code +json}/{@code +xml} (e.g. {@code image/svg+xml}). A missing content type is * assumed compressible. Matches the reference JS client's {@code isCompressibleContentType}. * - *

Public (like {@link #compress} and {@link #brotliAvailable}) so {@code CompressionTest}, - * outside this non-exported package, can exercise the classification directly. + *

Public so {@code CompressionTest}, outside this non-exported package, can exercise the + * classification directly. */ public static boolean isCompressibleContentType(String contentType) { if (contentType == null || contentType.isEmpty()) { @@ -687,17 +718,21 @@ public static String extractPath(String url) { */ public CompletableFuture> streamAsync(String url) { HttpRequest request = buildRequest("GET", url, null, null, null, resolveTimeout(Timeout.LONG)); - CompletableFuture> result = new CompletableFuture<>(); - transport - .sendStreamingAsync(request) - .whenComplete( - (resp, err) -> { - if (err == null) { - result.complete(resp); - } else { - result.completeExceptionally(new ApifyTransportException(unwrapCompletion(err))); - } - }); + CancellableFuture> result = new CancellableFuture<>(); + CompletableFuture> exchange = transport.sendStreamingAsync(request); + // Cancelling the returned future aborts the exchange. + result.link(exchange); + exchange.whenComplete( + (resp, err) -> { + Throwable cause = err == null ? null : unwrapCompletion(err); + if (cause == null) { + result.complete(resp); + } else if (cause instanceof CancellationException) { + result.cancel(true); + } else { + result.completeExceptionally(new ApifyTransportException(cause)); + } + }); return result; } diff --git a/src/main/java/com/apify/client/internal/ListPublisher.java b/src/main/java/com/apify/client/internal/ListPublisher.java index 05f38a3..7aa6188 100644 --- a/src/main/java/com/apify/client/internal/ListPublisher.java +++ b/src/main/java/com/apify/client/internal/ListPublisher.java @@ -79,6 +79,7 @@ public void request(long n) { @Override public void cancel() { cancelled.set(true); + source.cancel(true); } private void drain() { diff --git a/src/main/java/com/apify/client/internal/QueryParams.java b/src/main/java/com/apify/client/internal/QueryParams.java index 5b0681e..eca5c53 100644 --- a/src/main/java/com/apify/client/internal/QueryParams.java +++ b/src/main/java/com/apify/client/internal/QueryParams.java @@ -34,6 +34,12 @@ public QueryParams addLong(String key, Long value) { return this; } + /** Sets an integer parameter, replacing any earlier value; a {@code null} value removes it. */ + public QueryParams setLong(String key, Long value) { + pairs.removeIf(pair -> pair.key().equals(key)); + return addLong(key, value); + } + /** Adds a floating-point parameter if the value is non-null. */ public QueryParams addDouble(String key, Double value) { if (value != null) { diff --git a/src/main/java/com/apify/client/internal/ResourceContext.java b/src/main/java/com/apify/client/internal/ResourceContext.java index 035ec13..6dd000e 100644 --- a/src/main/java/com/apify/client/internal/ResourceContext.java +++ b/src/main/java/com/apify/client/internal/ResourceContext.java @@ -401,20 +401,7 @@ public CompletableFuture postWithBody( String contentType, Class dataClass, Timeout timeout) { - return postWithBody(subPath, params, body, contentType, Json.type(dataClass), timeout, false); - } - - /** As {@link #postWithBody(String, QueryParams, byte[], String, Class, Timeout)}, see above. */ - public CompletableFuture postWithBody( - String subPath, - QueryParams params, - byte[] body, - String contentType, - Class dataClass, - Timeout timeout, - boolean doNotRetryTimeouts) { - return postWithBody( - subPath, params, body, contentType, Json.type(dataClass), timeout, doNotRetryTimeouts); + return postWithBody(subPath, params, body, contentType, Json.type(dataClass), timeout); } public CompletableFuture postWithBody( @@ -424,24 +411,8 @@ public CompletableFuture postWithBody( String contentType, JavaType dataType, Timeout timeout) { - return postWithBody(subPath, params, body, contentType, dataType, timeout, false); - } - - /** - * As {@link #postWithBody(String, QueryParams, byte[], String, JavaType, Timeout)}, with control - * over whether a transport timeout is retried. A request that creates a billed resource (a run, a - * build) must not be retried on a timeout, since the API may already have created it. - */ - public CompletableFuture postWithBody( - String subPath, - QueryParams params, - byte[] body, - String contentType, - JavaType dataType, - Timeout timeout, - boolean doNotRetryTimeouts) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - return http.call("POST", u, body, contentType, timeout, doNotRetryTimeouts) + return http.call("POST", u, body, contentType, timeout) .thenApply(resp -> Json.parseData(resp.body(), dataType)); } diff --git a/src/main/java/com/apify/client/internal/RunStartSupport.java b/src/main/java/com/apify/client/internal/RunStartSupport.java index ba8df49..8a7b0c6 100644 --- a/src/main/java/com/apify/client/internal/RunStartSupport.java +++ b/src/main/java/com/apify/client/internal/RunStartSupport.java @@ -92,12 +92,15 @@ private static CompletableFuture start( QueryParams params = new QueryParams(); applyParams.accept(params); byte[] body = Json.toRequestBody(input); - Timeout requestTimeout = - timeout != null - ? timeout - : ctx.http.timeoutWithHold(Timeout.MEDIUM, params.getLong("waitForFinish")); + Timeout requestTimeout = timeout; + if (requestTimeout == null) { + HttpClientCore.WaitHold hold = + ctx.http.waitHold(Timeout.MEDIUM, params.getLong("waitForFinish")); + params.setLong("waitForFinish", hold.waitForFinishSecs()); + requestTimeout = hold.timeout(); + } if (waitForResourcesSecs == null) { - return postRun(ctx, params, body, contentType, requestTimeout); + return ctx.postWithBody("runs", params, body, contentType, ActorRun.class, requestTimeout); } long deadlineMillis = waitForResourcesSecs == ActorStartOptions.WAIT_FOR_RESOURCES_INDEFINITELY @@ -121,7 +124,7 @@ private static CompletableFuture startRetryingForResources( String contentType, Timeout timeout, long deadlineMillis) { - return postRun(ctx, params, body, contentType, timeout) + return ctx.postWithBody("runs", params, body, contentType, ActorRun.class, timeout) .handle( (run, error) -> { if (error == null) { @@ -152,15 +155,6 @@ private static CompletableFuture startRetryingForResources( .thenCompose(future -> future); } - /** - * Posts the run-start request. A start that timed out is not retried: the API may already have - * created the run, and a retry would start a second, billed one. - */ - private static CompletableFuture postRun( - ResourceContext ctx, QueryParams params, byte[] body, String contentType, Timeout timeout) { - return ctx.postWithBody("runs", params, body, contentType, ActorRun.class, timeout, true); - } - /** * Starts a run and waits (non-blocking, polling) for it to finish, without log streaming. * @@ -211,7 +205,7 @@ public static CompletableFuture callWithLogStreaming( CompletableFuture streamedLogFuture = logStreamingEnabled ? startStreamedLogQuietly(runClient, logOptions) - : CompletableFuture.completedFuture(null); + : CancellableFuture.completed(null); return streamedLogFuture.thenCompose( streamedLog -> runClient diff --git a/src/main/java/com/apify/client/internal/UrlNormalizer.java b/src/main/java/com/apify/client/internal/UrlNormalizer.java index 4671337..b0be071 100644 --- a/src/main/java/com/apify/client/internal/UrlNormalizer.java +++ b/src/main/java/com/apify/client/internal/UrlNormalizer.java @@ -22,7 +22,12 @@ public final class UrlNormalizer { */ private static final Pattern URL = Pattern.compile( - "^([A-Za-z][A-Za-z0-9+.-]*)://([^/?#]*)([^?#]*)(\\?[^#]*)?(#.*)?$", Pattern.DOTALL); + "^(?[A-Za-z][A-Za-z0-9+.-]*)://(?[^/?#]*)(?[^?#]*)" + + "(?\\?[^#]*)?(?#.*)?$", + Pattern.DOTALL); + + /** The highest TCP port; a larger one makes the URL invalid. */ + private static final int MAX_PORT = 65535; /** Schemes with a default port and a mandatory (possibly empty) path. */ private static final Map SPECIAL_SCHEMES = @@ -43,15 +48,15 @@ public static String normalize(String value) { if (!m.matches()) { return value; } - String scheme = m.group(1).toLowerCase(Locale.ROOT); + String scheme = m.group("scheme").toLowerCase(Locale.ROOT); Integer defaultPort = SPECIAL_SCHEMES.get(scheme); boolean special = defaultPort != null; - String authority = m.group(2); + String authority = m.group("authority"); String userInfo = ""; int at = authority.lastIndexOf('@'); if (at >= 0) { - userInfo = encode(authority.substring(0, at), USERINFO_UNSAFE) + "@"; + userInfo = encodeUserInfo(authority.substring(0, at)) + "@"; authority = authority.substring(at + 1); } String host = authority; @@ -65,28 +70,49 @@ public static String normalize(String value) { return value; } host = normalizeHost(host); - if (host == null || !port.matches("\\d*")) { + // At most five digits: anything longer is above MAX_PORT, and keeps parseInt from overflowing. + if (host == null || !port.matches("\\d{0,5}") || (!port.isEmpty() && toPort(port) > MAX_PORT)) { return value; } - if (!port.isEmpty() && defaultPort != null && Integer.parseInt(port) == defaultPort) { + if (!port.isEmpty() && defaultPort != null && toPort(port) == defaultPort) { port = ""; } if (!port.isEmpty()) { - port = ":" + Integer.parseInt(port); + port = ":" + toPort(port); } - String rawPath = special ? m.group(3).replace('\\', '/') : m.group(3); + String rawPath = special ? m.group("path").replace('\\', '/') : m.group("path"); String path = special ? resolveDotSegments(encode(rawPath, PATH_UNSAFE)) : encode(rawPath, PATH_UNSAFE); if (special && path.isEmpty()) { path = "/"; } + String rawQuery = m.group("query"); String query = - m.group(4) == null ? "" : encode(m.group(4), special ? SPECIAL_QUERY_UNSAFE : QUERY_UNSAFE); - String fragment = m.group(5) == null ? "" : encode(m.group(5), FRAGMENT_UNSAFE); + rawQuery == null ? "" : encode(rawQuery, special ? SPECIAL_QUERY_UNSAFE : QUERY_UNSAFE); + String rawFragment = m.group("fragment"); + String fragment = rawFragment == null ? "" : encode(rawFragment, FRAGMENT_UNSAFE); return scheme + "://" + userInfo + host + port + path + query + fragment; } + private static int toPort(String digits) { + return Integer.parseInt(digits); + } + + /** + * Encodes the {@code username[:password]} part of the authority: each of the two is encoded on + * its own, so the separator between them stays a colon. + */ + private static String encodeUserInfo(String userInfo) { + int colon = userInfo.indexOf(':'); + if (colon < 0) { + return encode(userInfo, USERINFO_UNSAFE); + } + return encode(userInfo.substring(0, colon), USERINFO_UNSAFE) + + ":" + + encode(userInfo.substring(colon + 1), USERINFO_UNSAFE); + } + /** * Lowercases the host and converts a non-ASCII one to punycode; {@code null} if it is invalid. */ diff --git a/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java b/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java index d222e3b..b65c1b9 100644 --- a/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java +++ b/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java @@ -150,6 +150,7 @@ private final class Session implements Flow.Subscription { private final AtomicLong requested = new AtomicLong(); private final AtomicBoolean draining = new AtomicBoolean(); private final AtomicBoolean cancelled = new AtomicBoolean(); + private volatile CompletableFuture inFlight; private final AtomicBoolean terminated = new AtomicBoolean(); // Only touched from within the draining-guarded section; see AsyncPaginatedPublisher.Session. @@ -182,6 +183,10 @@ public void request(long n) { @Override public void cancel() { cancelled.set(true); + CompletableFuture fetch = inFlight; + if (fetch != null) { + fetch.cancel(true); + } } private void drain() { @@ -228,23 +233,25 @@ private void fetchPage() { params.addLong("limit", pageLimit); params.addString("exclusiveStartKey", cursor); params.extend(filters); - ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class, Timeout.MEDIUM) - .whenComplete( - (page, error) -> { - if (cancelled.get()) { - draining.set(false); - return; - } - if (error != null) { - if (terminated.compareAndSet(false, true)) { - subscriber.onError(HttpClientCore.unwrapCompletion(error)); - } - draining.set(false); - return; - } - applyPage(page); - drainLoop(); - }); + CompletableFuture fetch = + ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class, Timeout.MEDIUM); + inFlight = fetch; + fetch.whenComplete( + (page, error) -> { + if (cancelled.get()) { + draining.set(false); + return; + } + if (error != null) { + if (terminated.compareAndSet(false, true)) { + subscriber.onError(HttpClientCore.unwrapCompletion(error)); + } + draining.set(false); + return; + } + applyPage(page); + drainLoop(); + }); } private void applyPage(KeyValueStoreKeysPage page) { diff --git a/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java b/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java index 0593932..b87200b 100644 --- a/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java +++ b/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java @@ -5,6 +5,7 @@ import com.apify.client.http.Timeout; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.Async; +import com.apify.client.internal.CancellableFuture; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; import com.apify.client.internal.QueryParams; @@ -280,7 +281,7 @@ public CompletableFuture batchAddRequests( for (int w = 0; w < workerCount; w++) { workers.add(runBatchAddWorker(chunks, nextChunkIndex, resultsByChunk, forefront, options)); } - return CompletableFuture.allOf(workers.toArray(CompletableFuture[]::new)) + return CancellableFuture.allOf(workers) .thenApply( unused -> { BatchAddResult merged = new BatchAddResult(); @@ -660,6 +661,7 @@ private final class Session implements Flow.Subscription { private final AtomicLong requested = new AtomicLong(); private final AtomicBoolean draining = new AtomicBoolean(); private final AtomicBoolean cancelled = new AtomicBoolean(); + private volatile CompletableFuture inFlight; private final AtomicBoolean terminated = new AtomicBoolean(); // Only touched from within the draining-guarded section; see AsyncPaginatedPublisher.Session. @@ -691,6 +693,10 @@ public void request(long n) { @Override public void cancel() { cancelled.set(true); + CompletableFuture fetch = inFlight; + if (fetch != null) { + fetch.cancel(true); + } } private void drain() { @@ -746,23 +752,25 @@ private void fetchPage() { } params.addCsv("filter", filter); withClientKey(params); - ctx.getResourceRequired("requests", params, RequestsList.class, Timeout.MEDIUM) - .whenComplete( - (page, error) -> { - if (cancelled.get()) { - draining.set(false); - return; - } - if (error != null) { - if (terminated.compareAndSet(false, true)) { - subscriber.onError(HttpClientCore.unwrapCompletion(error)); - } - draining.set(false); - return; - } - applyPage(page, capRemaining); - drainLoop(); - }); + CompletableFuture fetch = + ctx.getResourceRequired("requests", params, RequestsList.class, Timeout.MEDIUM); + inFlight = fetch; + fetch.whenComplete( + (page, error) -> { + if (cancelled.get()) { + draining.set(false); + return; + } + if (error != null) { + if (terminated.compareAndSet(false, true)) { + subscriber.onError(HttpClientCore.unwrapCompletion(error)); + } + draining.set(false); + return; + } + applyPage(page, capRemaining); + drainLoop(); + }); } private void applyPage(RequestsList page, Long capRemaining) { diff --git a/src/main/java/com/apify/client/run/RunClient.java b/src/main/java/com/apify/client/run/RunClient.java index 36b589f..0fc5b65 100644 --- a/src/main/java/com/apify/client/run/RunClient.java +++ b/src/main/java/com/apify/client/run/RunClient.java @@ -92,9 +92,9 @@ public CompletableFuture> get() { */ public CompletableFuture> getWithWait(Long waitForFinishSecs) { QueryParams params = new QueryParams(); - params.addLong("waitForFinish", waitForFinishSecs); - return ctx.getResource( - "", params, ActorRun.class, ctx.http.timeoutWithHold(Timeout.SHORT, waitForFinishSecs)); + HttpClientCore.WaitHold hold = ctx.http.waitHold(Timeout.SHORT, waitForFinishSecs); + params.addLong("waitForFinish", hold.waitForFinishSecs()); + return ctx.getResource("", params, ActorRun.class, hold.timeout()); } /** Updates the run with the given fields and returns the updated object. */ @@ -271,13 +271,12 @@ public Flow.Publisher iterateDatasetItems(DatasetListItemsOptions opti */ public Flow.Publisher iterateDatasetItems( DatasetListItemsOptions options, Long chunkSize, Long pollIntervalSecs, Class itemClass) { - DatasetListItemsOptions opts = options != null ? options.copy() : new DatasetListItemsOptions(); long pageSize = chunkSize != null && chunkSize > 0 ? chunkSize : DEFAULT_ITERATE_CHUNK_SIZE; long poll = pollIntervalSecs != null && pollIntervalSecs >= 0 ? pollIntervalSecs : DEFAULT_ITERATE_POLL_INTERVAL_SECS; - RunDatasetItems items = new RunDatasetItems<>(this, opts, pageSize, poll, itemClass); + RunDatasetItems items = new RunDatasetItems<>(this, options, pageSize, poll, itemClass); return new AsyncPaginatedPublisher<>(null, null, null, (offset, limit) -> items.next()); } diff --git a/src/main/java/com/apify/client/run/RunDatasetItems.java b/src/main/java/com/apify/client/run/RunDatasetItems.java index 2836431..aae86c4 100644 --- a/src/main/java/com/apify/client/run/RunDatasetItems.java +++ b/src/main/java/com/apify/client/run/RunDatasetItems.java @@ -4,9 +4,11 @@ import com.apify.client.dataset.Dataset; import com.apify.client.dataset.DatasetClient; import com.apify.client.dataset.DatasetListItemsOptions; +import com.apify.client.internal.DatasetItemsOptionsAccess; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import tools.jackson.databind.JsonNode; /** * The paging state machine behind {@link RunClient#iterateDatasetItems}: it reads a run's default @@ -26,6 +28,7 @@ final class RunDatasetItems { private final RunClient runClient; private final DatasetListItemsOptions options; private final Class itemClass; + private final DatasetItemsOptionsAccess access; private final long pageSize; private final long pollIntervalSecs; @@ -53,12 +56,13 @@ private enum Phase { long pollIntervalSecs, Class itemClass) { this.runClient = runClient; - this.options = options; + this.access = DatasetItemsOptionsAccess.get(); + this.options = access.snapshot(options); this.itemClass = itemClass; this.pageSize = pageSize; this.pollIntervalSecs = pollIntervalSecs; - this.position = options.offsetValueOrZero(); - Long limit = options.limitValueOrNull(); + this.position = access.offset(this.options); + Long limit = access.limit(this.options); this.end = limit != null && limit > 0 ? position + limit : null; } @@ -132,13 +136,14 @@ private CompletableFuture> tail() { if (!empty) { return CompletableFuture.completedFuture(advance(page, limit)); } - if (!options.dropsRows()) { + if (!access.dropsRows(options)) { return finish(); } // A page that clean, skipEmpty or unwind emptied past a lagging itemCount reports no - // scanned rows either, so a plain read of the next row tells whether any are left. + // scanned rows either, so a read of the next row without those filters (they would + // empty it again) tells whether any rows are left. return dataset - .listItems(options.copy().desc(null).offset(position).limit(1L), itemClass) + .listItems(access.plainPage(position, 1L), JsonNode.class) .thenCompose( probe -> probe.getItems().isEmpty() @@ -157,7 +162,7 @@ private CompletableFuture> readPage(long limit) { } private CompletableFuture> page(long offset, long limit) { - return dataset.listItems(options.copy().desc(null).offset(offset).limit(limit), itemClass); + return dataset.listItems(access.page(options, offset, limit), itemClass); } /** Moves past the {@code limit} rows a page covered and marks it as having scanned them. */ diff --git a/src/test/java/com/apify/client/CancellationTest.java b/src/test/java/com/apify/client/CancellationTest.java new file mode 100644 index 0000000..791ac35 --- /dev/null +++ b/src/test/java/com/apify/client/CancellationTest.java @@ -0,0 +1,151 @@ +package com.apify.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.apify.client.actor.ActorListOptions; +import com.apify.client.actor.ActorStartOptions; +import com.apify.client.requestqueue.RequestQueueRequest; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.function.BooleanSupplier; +import org.junit.jupiter.api.Test; + +/** + * Cancelling a future the client returned cancels the work behind it: the request in flight, the + * retries, the polling and the pages that would follow. + */ +class CancellationTest { + + private static final String RUN_JSON = "{\"data\":{\"id\":\"r1\",\"status\":\"RUNNING\"}}"; + + private static ApifyClient client(MockTransport backend, int maxRetries) { + return ApifyClient.builder() + .token("t") + .httpTransport(backend) + .maxRetries(maxRetries) + .minDelayBetweenRetries(Duration.ofMillis(300)) + .build(); + } + + private static void await(BooleanSupplier condition) throws InterruptedException { + for (int i = 0; i < 200 && !condition.getAsBoolean(); i++) { + Thread.sleep(10); + } + assertTrue(condition.getAsBoolean(), "condition not reached in time"); + } + + @Test + void cancellingARequestCancelsTheTransportExchange() throws InterruptedException { + MockTransport backend = MockTransport.ofHanging(request -> true); + CompletableFuture future = client(backend, 0).run("r1").get(); + await(() -> backend.calls == 1); + + assertTrue(future.cancel(true)); + + assertTrue(future.isCancelled()); + assertTrue(backend.allHungCancelled(), "the request in flight is aborted"); + assertThrows(CancellationException.class, future::join); + } + + @Test + void cancellingADerivedStageCancelsTheCallItComesFrom() throws InterruptedException { + MockTransport backend = MockTransport.ofHanging(request -> true); + CompletableFuture derived = + client(backend, 0) + .run("r1") + .get() + .thenApply(run -> "x") + .thenCompose(CompletableFuture::completedFuture); + await(() -> backend.calls == 1); + + derived.cancel(true); + + assertTrue(backend.allHungCancelled()); + } + + @Test + void cancellingDuringTheBackoffStopsTheRetries() throws InterruptedException { + MockTransport backend = MockTransport.ofConstant(503, "{}"); + CompletableFuture future = client(backend, 5).run("r1").get(); + await(() -> backend.calls == 1); + + future.cancel(true); + Thread.sleep(900); // longer than the first backoff delay (300-600 ms) + + assertEquals(1, backend.calls, "a cancelled call is not retried"); + } + + @Test + void cancellingAWaitForFinishStopsThePolling() throws InterruptedException { + MockTransport backend = MockTransport.ofHanging(request -> false); + // The run never finishes: every poll answers RUNNING at once. + CompletableFuture future = client(backend, 0).run("r1").waitForFinish(600L); + await(() -> backend.calls >= 2); + + future.cancel(true); + int callsAtCancel = backend.calls; + Thread.sleep(600); + + assertTrue(backend.calls <= callsAtCancel + 1, "polling stops: " + backend.calls); + assertTrue(future.isCancelled()); + } + + @Test + void cancellingAHangingPollAbortsItsRequest() throws InterruptedException { + MockTransport backend = MockTransport.ofHanging(request -> true); + CompletableFuture future = client(backend, 0).run("r1").waitForFinish(600L); + await(() -> backend.calls == 1); + + future.cancel(true); + + assertTrue(backend.allHungCancelled(), "a stalled poll can be escaped by cancelling"); + Thread.sleep(300); + assertEquals(1, backend.calls); + } + + @Test + void cancellingCallStopsItsPolling() throws InterruptedException { + MockTransport backend = MockTransport.ofHanging(request -> request.method().equals("GET")); + CompletableFuture future = + client(backend, 0).actor("a1").call(null, new ActorStartOptions(), null); + await(() -> backend.hung.size() == 1); + + future.cancel(true); + + assertTrue(backend.allHungCancelled()); + } + + @Test + void cancellingACollectedIterationCancelsThePageInFlight() throws InterruptedException { + MockTransport backend = MockTransport.ofHanging(request -> true); + CompletableFuture future = + Publishers.collect(client(backend, 0).actors().iterate(new ActorListOptions(), 10L)); + await(() -> backend.calls == 1); + + future.cancel(true); + + assertTrue(backend.allHungCancelled()); + } + + @Test + void cancellingABatchAddCancelsTheChunksInFlight() throws InterruptedException { + MockTransport backend = MockTransport.ofHanging(request -> true); + List requests = + List.of( + new RequestQueueRequest().setUrl("https://a.test").setUniqueKey("a"), + new RequestQueueRequest().setUrl("https://b.test").setUniqueKey("b")); + CompletableFuture future = + client(backend, 0).requestQueue("q1").batchAddRequests(requests, false); + await(() -> backend.calls == 1); + + future.cancel(true); + + assertTrue(backend.allHungCancelled()); + assertFalse(future.isDone() && !future.isCancelled()); + } +} diff --git a/src/test/java/com/apify/client/CompressionTest.java b/src/test/java/com/apify/client/CompressionTest.java index 4fd1336..1e3d030 100644 --- a/src/test/java/com/apify/client/CompressionTest.java +++ b/src/test/java/com/apify/client/CompressionTest.java @@ -182,6 +182,34 @@ public byte[] compress(byte[] data) throws IOException { assertEquals(0, backend.calls, "a body that could not be compressed must not be sent"); } + @Test + void aCompressorThatThrowsAnErrorFailsTheFutureInsteadOfThrowing() { + HttpCompressor broken = + new HttpCompressor() { + @Override + public String contentEncoding() { + return "x-broken"; + } + + @Override + public byte[] compress(byte[] data) { + throw new UnsatisfiedLinkError("native codec missing"); + } + }; + MockTransport backend = MockTransport.ofConstant(201, ""); + ApifyClient client = + ApifyClient.builder().token("t").httpTransport(backend).compression(broken).build(); + + var future = + client + .keyValueStore(STORE_ID) + .setRecord(RECORD_KEY, payload(4096, (byte) 'a'), CONTENT_TYPE); + + var error = assertThrows(java.util.concurrent.CompletionException.class, future::join); + assertTrue(error.getCause() instanceof com.apify.client.http.ApifyTransportException); + assertEquals(0, backend.calls); + } + @Test void builderRejectsNullCompression() { assertThrows(IllegalArgumentException.class, () -> ApifyClient.builder().compression(null)); diff --git a/src/test/java/com/apify/client/MockTransport.java b/src/test/java/com/apify/client/MockTransport.java index 7841209..792ee60 100644 --- a/src/test/java/com/apify/client/MockTransport.java +++ b/src/test/java/com/apify/client/MockTransport.java @@ -63,6 +63,15 @@ static final class Scripted { /** The per-attempt timeout of every request, in order. */ final List> timeouts = new ArrayList<>(); + /** + * Requests this predicate accepts are never answered; their futures are kept in {@link #hung}. + */ + private java.util.function.Predicate hangWhen = request -> false; + + /** The futures of the requests that were left unanswered. */ + final List>> hung = + new java.util.concurrent.CopyOnWriteArrayList<>(); + /** When set, answers each request itself instead of serving the scripted queue. */ private java.util.function.Function router; @@ -98,6 +107,18 @@ static MockTransport ofRouter(java.util.function.Function return transport; } + /** A transport that never answers the requests {@code hangWhen} accepts. */ + static MockTransport ofHanging(java.util.function.Predicate hangWhen) { + MockTransport transport = ofConstant(200, "{\"data\":{\"id\":\"r1\",\"status\":\"RUNNING\"}}"); + transport.hangWhen = hangWhen; + return transport; + } + + /** Whether every request left unanswered has been cancelled by the client. */ + boolean allHungCancelled() { + return !hung.isEmpty() && hung.stream().allMatch(CompletableFuture::isCancelled); + } + static MockTransport ofConstant(int status, String body) { return new MockTransport(List.of(new Scripted(status, body, null))); } @@ -137,6 +158,11 @@ public synchronized CompletableFuture> sendAsync(HttpReques if (lastBody != null) { bodies.add(lastBody); } + if (hangWhen.test(request)) { + CompletableFuture> never = new CompletableFuture<>(); + hung.add(never); + return never; + } if (idx >= responses.size()) { idx = responses.size() - 1; // repeat the last entry } diff --git a/src/test/java/com/apify/client/RunDatasetItemsTest.java b/src/test/java/com/apify/client/RunDatasetItemsTest.java index a546ce7..3960b16 100644 --- a/src/test/java/com/apify/client/RunDatasetItemsTest.java +++ b/src/test/java/com/apify/client/RunDatasetItemsTest.java @@ -43,6 +43,14 @@ private static final class FakeRun { } } + /** Replaces the first {@code count} rows with empty items. */ + FakeRun withEmptyHead(int count) { + for (int i = 0; i < count; i++) { + rows.set(i, "{}"); + } + return this; + } + MockTransport.Scripted answer(HttpRequest request) { URI uri = request.uri(); String path = uri.getPath(); @@ -70,11 +78,16 @@ MockTransport.Scripted answer(HttpRequest request) { int offset = Integer.parseInt(params.getOrDefault("offset", "0")); int limit = Integer.parseInt(params.getOrDefault("limit", "1000")); int end = Math.min(totalRows, offset + limit); - List page = offset >= end ? List.of() : rows.subList(offset, end); + List scanned = offset >= end ? List.of() : rows.subList(offset, end); + // skipEmpty drops the empty rows from what is returned, but they were still scanned. + List page = + "1".equals(params.get("skipEmpty")) + ? scanned.stream().filter(row -> !row.equals("{}")).toList() + : scanned; return MockTransport.ok( 200, "[" + String.join(",", page) + "]", - Map.of("X-Apify-Pagination-Count", String.valueOf(page.size()))); + Map.of("X-Apify-Pagination-Count", String.valueOf(scanned.size()))); } return MockTransport.ok(404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"x\"}}"); } @@ -167,4 +180,20 @@ void orderIsNeverReversedByDesc() { assertTrue( fake.requests.stream().noneMatch(r -> r.contains("desc=")), fake.requests.toString()); } + + @Test + void theTailKeepsReadingPastRowsAFilterEmptiedAndLaggingCounts() { + // Two empty rows, then two with data; the run has finished but the dataset's itemCount still + // reads 0, so every row is read by the tail. skipEmpty empties the first page entirely, which + // must not end the iteration: the next row is probed without the filter. + FakeRun fake = new FakeRun(4, 4, 4).withEmptyHead(2); + List items = + Publishers.collect( + client(fake::answer) + .run("r1") + .iterateDatasetItems( + new DatasetListItemsOptions().skipEmpty(true), 2L, 0L, JsonNode.class)) + .join(); + assertEquals(List.of(2, 3), ns(items)); + } } diff --git a/src/test/java/com/apify/client/TimeoutTierTest.java b/src/test/java/com/apify/client/TimeoutTierTest.java index f895e69..cd63290 100644 --- a/src/test/java/com/apify/client/TimeoutTierTest.java +++ b/src/test/java/com/apify/client/TimeoutTierTest.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.apify.client.actor.ActorBuildOptions; import com.apify.client.actor.ActorStartOptions; import com.apify.client.http.ApifyTransportException; import com.apify.client.http.Timeout; @@ -170,10 +169,36 @@ void withTimeoutReplacesTheTierOfEveryRequest() { } @Test - void withTimeoutIsUsedAsGivenForAWaitForFinishHold() { + void withTimeoutIsKeptAndTheHoldIsCutToWhatItCovers() { MockTransport backend = MockTransport.ofConstant(200, RUN_JSON); client(backend).withTimeout(Timeout.ofSeconds(10)).run("r1").getWithWait(30L).join(); - assertEquals(secs(10), backend.lastTimeout); + assertEquals(secs(10), backend.lastTimeout, "the override is used as given"); + assertTrue( + backend.lastUrl.contains("waitForFinish=5"), "10s less the 5s margin: " + backend.lastUrl); + } + + @Test + void aHoldIsCutToWhatTimeoutMaxLeavesAboveTheTier() { + MockTransport backend = MockTransport.ofConstant(200, RUN_JSON); + ApifyClient client = builder(backend).timeoutMax(Duration.ofSeconds(12)).build(); + + client.run("r1").getWithWait(30L).join(); + assertTrue(backend.lastUrl.contains("waitForFinish=7"), "5s tier, 12s cap: " + backend.lastUrl); + assertEquals(secs(12), backend.lastTimeout, "the request is not given up before the hold ends"); + + client.actor("a1").start(null, new ActorStartOptions().waitForFinish(60L)).join(); + assertTrue( + backend.lastUrl.contains("waitForFinish=0"), + "the 30s medium tier is capped at 12s, so no hold fits: " + backend.lastUrl); + } + + @Test + void aTimedOutStartIsRetriedLikeAnyOtherRequest() { + MockTransport backend = + new MockTransport(List.of(MockTransport.timeoutError(), MockTransport.ok(201, RUN_JSON))); + ApifyClient client = builder(backend).maxRetries(3).build(); + client.actor("a1").start(null, new ActorStartOptions()).join(); + assertEquals(2, backend.calls); } @Test @@ -191,26 +216,15 @@ void aRetryDoublesTheTimeoutUpToTheCap() { } @Test - void aTimedOutStartIsNotRetriedSoNoSecondRunIsStarted() { + void aTimedOutPushIsNotRetriedSoItemsAreNotStoredTwice() { MockTransport backend = - new MockTransport(List.of(MockTransport.timeoutError(), MockTransport.ok(201, RUN_JSON))); + new MockTransport(List.of(MockTransport.timeoutError(), MockTransport.ok(201, "{}"))); ApifyClient client = builder(backend).maxRetries(3).build(); - var future = client.actor("a1").start(null, new ActorStartOptions()); + var future = client.dataset("d1").pushItems(java.util.Map.of("a", 1)); var error = assertThrows(CompletionException.class, future::join); assertTrue(error.getCause() instanceof ApifyTransportException); - assertEquals(1, backend.calls, "a start that timed out must not be sent again"); - } - - @Test - void aTimedOutBuildTriggerIsNotRetried() { - MockTransport backend = - new MockTransport(List.of(MockTransport.timeoutError(), MockTransport.ok(201, RUN_JSON))); - ApifyClient client = builder(backend).maxRetries(3).build(); - - var future = client.actor("a1").build("0.1", new ActorBuildOptions()); - assertThrows(CompletionException.class, future::join); - assertEquals(1, backend.calls); + assertEquals(1, backend.calls, "a push that timed out must not be sent again"); } @Test diff --git a/src/test/java/com/apify/client/UrlNormalizationTest.java b/src/test/java/com/apify/client/UrlNormalizationTest.java index fd48c9a..d20aba2 100644 --- a/src/test/java/com/apify/client/UrlNormalizationTest.java +++ b/src/test/java/com/apify/client/UrlNormalizationTest.java @@ -66,6 +66,30 @@ void aNormalizedUrlStaysTheSame() { assertEquals(url, UrlNormalizer.normalize(url)); } + @Test + void userInfoKeepsItsSeparatorAndEncodesEachPart() { + assertEquals( + "https://user:pass@example.com/hook", + UrlNormalizer.normalize("https://user:pass@example.com/hook")); + assertEquals( + "https://us%20er:p%40ss@example.com/", + UrlNormalizer.normalize("https://us er:p@ss@Example.com")); + assertEquals( + "https://token@example.com/", UrlNormalizer.normalize("https://token@example.com")); + } + + @Test + void anInvalidPortLeavesTheValueAlone() { + assertEquals( + "https://example.com:99999999999/", + UrlNormalizer.normalize("https://example.com:99999999999/")); + assertEquals( + "https://example.com:65536/", UrlNormalizer.normalize("https://example.com:65536/")); + assertEquals( + "https://example.com:65535/", UrlNormalizer.normalize("https://example.com:65535")); + assertEquals("https://example.com:abc/", UrlNormalizer.normalize("https://example.com:abc/")); + } + @Test void valuesThatAreNotAbsoluteUrlsAreLeftAlone() { assertNull(UrlNormalizer.normalize(null)); From 565ba1453a3b138e64782e74d93bc186428ef097 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 16:53:59 +0000 Subject: [PATCH 3/6] fix: close the log of a cancelled call, release completed stages, hold polls under withTimeout, document cancellation Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAKY9wkdABFznxvsCXt --- CHANGELOG.md | 1 + README.md | 28 ++++- docs/README.md | 8 ++ .../java/com/apify/client/ApifyClient.java | 5 +- .../client/internal/CancellableFuture.java | 23 +++- .../client/internal/ResourceContext.java | 7 +- .../client/internal/RunStartSupport.java | 52 ++++++--- .../apify/client/internal/UrlNormalizer.java | 40 ++++--- .../com/apify/client/CancellationTest.java | 110 ++++++++++++++++++ .../com/apify/client/ClientInputTest.java | 4 +- .../apify/client/UrlNormalizationTest.java | 15 +++ 11 files changed, 240 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f0c37b..04f40fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Cancelling a future the client returned (`CompletableFuture.cancel`) now aborts the request in flight and stops the retries, `waitForFinish`/`call` polling and iteration pages that would follow; cancelling a derived stage (`thenApply`, `thenCompose`, ...) cancels the call it comes from. + Cancelling `call(...)` also stops its log redirection. See the Cancellation section of the README. - `pushItems` rejects what the API rejects (anything but an object or an array of objects) before a request is sent, and sends a pre-serialized `String` as it is. - An Actor or task input given as `byte[]` or `String` is sent as it is instead of being wrapped in diff --git a/README.md b/README.md index b3eb7b8..990352f 100644 --- a/README.md +++ b/README.md @@ -203,14 +203,34 @@ List items = .getItems(); ``` -An override is used as given, so for a `waitForFinish` request it has to cover the hold too. `KeyValueStoreClient.setRecord(...)` additionally takes the timeout in its `SetRecordOptions`. A `pushItems` request that times out is not retried, since the API may already have stored the items. -A hold is cut down to what the request timeout can cover: with a small `timeoutMax` (or an override -from `withTimeout`), `waitForFinish` is sent as a smaller value so the API answers before the client -gives up on the request. +A `waitForFinish` hold is cut down to what the request timeout can cover, so the API answers before +the client gives up on the request: with a small `timeoutMax`, the hold is what `timeoutMax` leaves +above the tier, and under a `withTimeout` override (which is kept as given) it is the override less +5 seconds. The same applies to every poll of `waitForFinish(...)` and `call(...)`. + +### Cancellation + +Every method returns a `CompletableFuture` (or a `Flow.Publisher`), and cancelling it cancels the +work behind it: `future.cancel(true)` aborts the request in flight and stops the retries, the +`waitForFinish`/`call` polling and the pages of an iteration that would follow; cancelling the +`Flow.Subscription` of a publisher does the same for the page being fetched. This is the way out of a +request that has no timeout (the polls of `waitForFinish`/`call` and `Timeout.NO_TIMEOUT`): + +```java +CompletableFuture call = client.actor("my-actor").call(null, new ActorStartOptions(), null); +// ... later, when the result is no longer needed: +call.cancel(true); // stops the polling and the log redirection; the run itself keeps running +``` + +A cancelled future completes with a `CancellationException`. Cancelling a stage derived from a +client future (`future.thenApply(...)`, `.thenCompose(...)`) cancels the call it comes from, and so +fails the other stages derived from the same future with a `CancellationException` too; derive one +stage per call, or cancel the future the client returned. Cancelling does not undo what the API +already did: a started run keeps running (abort it with `RunClient.abort(...)`). ### Request compression and default headers diff --git a/docs/README.md b/docs/README.md index 422a603..3138800 100644 --- a/docs/README.md +++ b/docs/README.md @@ -73,6 +73,14 @@ Raw-JSON return values use Jackson's `tools.jackson.databind.JsonNode`. Jackson `tools.jackson.*` package, not the older `com.fasterxml.jackson.databind.*` from Jackson 2) is a transitive dependency of this client, so it is already on your classpath. +## Cancellation + +Cancelling a future a client method returned (`future.cancel(true)`), or the `Flow.Subscription` of +an iteration, aborts the request in flight and stops the retries, the `waitForFinish`/`call` polling +and the pages that would follow; see [Cancellation](../README.md#cancellation) for the details and +the caveat about derived stages. A cancelled `call(...)` also stops redirecting the run's log, but +the run itself keeps running. + ## Raw JSON values A few methods return data whose shape is not modelled by this client and is instead exposed as a diff --git a/src/main/java/com/apify/client/ApifyClient.java b/src/main/java/com/apify/client/ApifyClient.java index d0260df..206f847 100644 --- a/src/main/java/com/apify/client/ApifyClient.java +++ b/src/main/java/com/apify/client/ApifyClient.java @@ -111,8 +111,9 @@ public static ApifyClientBuilder builder() { * } * * A timeout above the builder's {@code timeoutMax} is capped at it. For a method that asks the - * API to hold the response with {@code waitForFinish}, the given timeout is used as it is, so it - * has to cover the hold too. + * API to hold the response with {@code waitForFinish} (and for the polling of {@code + * waitForFinish}/{@code call}), the given timeout is kept and the hold is cut to what it covers, + * the timeout less 5 seconds, so the API answers before the client gives up on the request. */ public ApifyClient withTimeout(Timeout timeout) { return new ApifyClient(http.withTimeout(timeout), baseUrl, publicBaseUrl); diff --git a/src/main/java/com/apify/client/internal/CancellableFuture.java b/src/main/java/com/apify/client/internal/CancellableFuture.java index fe93c3f..ea3938e 100644 --- a/src/main/java/com/apify/client/internal/CancellableFuture.java +++ b/src/main/java/com/apify/client/internal/CancellableFuture.java @@ -1,5 +1,6 @@ package com.apify.client.internal; +import java.lang.ref.WeakReference; import java.util.List; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; @@ -18,8 +19,14 @@ */ public final class CancellableFuture extends CompletableFuture { - /** The futures this one depends on, cancelled together with it. */ - private final ConcurrentLinkedQueue> upstream = + /** + * The futures this one depends on, cancelled together with it. Held weakly: a future that is + * still pending is kept alive by whatever will complete it (the request in flight, the timer, the + * stage it feeds), and holding a completed one strongly would keep its result (a response body, + * say) reachable from every later stage for as long as the call lives, which for a long {@code + * waitForFinish} poll loop is unbounded. + */ + private final ConcurrentLinkedQueue>> upstream = new ConcurrentLinkedQueue<>(); public CancellableFuture() {} @@ -75,7 +82,7 @@ public static CancellableFuture allOf(List> /** Makes {@code other} be cancelled whenever this future is (now, if it already is). */ public void link(CompletableFuture other) { - upstream.add(other); + upstream.add(new WeakReference<>(other)); if (isCancelled()) { other.cancel(true); } @@ -85,8 +92,11 @@ public void link(CompletableFuture other) { public boolean cancel(boolean mayInterruptIfRunning) { boolean cancelled = super.cancel(mayInterruptIfRunning); if (cancelled) { - for (CompletableFuture source : upstream) { - source.cancel(mayInterruptIfRunning); + for (WeakReference> reference : upstream) { + CompletableFuture source = reference.get(); + if (source != null) { + source.cancel(mayInterruptIfRunning); + } } } return cancelled; @@ -96,7 +106,7 @@ public boolean cancel(boolean mayInterruptIfRunning) { @Override public CompletableFuture newIncompleteFuture() { CancellableFuture dependent = new CancellableFuture<>(); - dependent.upstream.add(this); + dependent.upstream.add(new WeakReference<>(this)); return dependent; } @@ -134,6 +144,7 @@ public CompletableFuture thenCompose( synchronized (composition) { composition.dependent = dependent; early = composition.inner; + composition.inner = null; } if (early != null) { dependent.link(early); diff --git a/src/main/java/com/apify/client/internal/ResourceContext.java b/src/main/java/com/apify/client/internal/ResourceContext.java index 6dd000e..5fc05ef 100644 --- a/src/main/java/com/apify/client/internal/ResourceContext.java +++ b/src/main/java/com/apify/client/internal/ResourceContext.java @@ -636,10 +636,13 @@ private CompletableFuture pollOnce( long remainingSecs = (budgetMillis - elapsed) / 1000L; long requestSecs = Math.min(Math.max(remainingSecs, 0), WAIT_REQUEST_SECS); + // The polls have no timeout of their own; under a withTimeout override the hold is cut to what + // the override covers, so the API answers before the client gives up on a poll. + HttpClientCore.WaitHold hold = http.waitHold(Timeout.NO_TIMEOUT, requestSecs); QueryParams params = new QueryParams(); - params.addLong("waitForFinish", requestSecs); + params.addLong("waitForFinish", hold.waitForFinishSecs()); - return this.getResourceRequired("", params, dataType, Timeout.NO_TIMEOUT) + return this.getResourceRequired("", params, dataType, hold.timeout()) .handle( (value, error) -> { if (error == null) { diff --git a/src/main/java/com/apify/client/internal/RunStartSupport.java b/src/main/java/com/apify/client/internal/RunStartSupport.java index 8a7b0c6..ca3ffa8 100644 --- a/src/main/java/com/apify/client/internal/RunStartSupport.java +++ b/src/main/java/com/apify/client/internal/RunStartSupport.java @@ -11,6 +11,8 @@ import java.time.Duration; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -198,25 +200,37 @@ public static CompletableFuture callWithLogStreaming( StreamedLogOptions logOptions, Long waitSecs, Long waitForResourcesSecs) { - return start(ctx, input, applyParams, contentType, waitForResourcesSecs, Timeout.NO_TIMEOUT) - .thenCompose( - run -> { - RunClient runClient = root.run(run.getId()); - CompletableFuture streamedLogFuture = - logStreamingEnabled - ? startStreamedLogQuietly(runClient, logOptions) - : CancellableFuture.completed(null); - return streamedLogFuture.thenCompose( - streamedLog -> - runClient - .waitForFinish(waitSecs) - .whenComplete( - (result, error) -> { - if (streamedLog != null) { - streamedLog.close(); - } - })); - }); + AtomicReference opened = new AtomicReference<>(); + AtomicBoolean finished = new AtomicBoolean(); + CompletableFuture result = + start(ctx, input, applyParams, contentType, waitForResourcesSecs, Timeout.NO_TIMEOUT) + .thenCompose( + run -> { + RunClient runClient = root.run(run.getId()); + CompletableFuture streamedLogFuture = + logStreamingEnabled + ? startStreamedLogQuietly(runClient, logOptions) + : CancellableFuture.completed(null); + return streamedLogFuture.thenCompose( + streamedLog -> { + opened.set(streamedLog); + if (finished.get() && streamedLog != null) { + streamedLog.close(); // the call ended while the log was being opened + } + return runClient.waitForFinish(waitSecs); + }); + }); + // Closed from a callback of the returned future itself, which runs whether the call finishes, + // fails or is cancelled (a stage that is cancelled never runs its own callbacks). + result.whenComplete( + (run, error) -> { + finished.set(true); + StreamedLog streamedLog = opened.get(); + if (streamedLog != null) { + streamedLog.close(); + } + }); + return result; } /** diff --git a/src/main/java/com/apify/client/internal/UrlNormalizer.java b/src/main/java/com/apify/client/internal/UrlNormalizer.java index b0be071..ab9477b 100644 --- a/src/main/java/com/apify/client/internal/UrlNormalizer.java +++ b/src/main/java/com/apify/client/internal/UrlNormalizer.java @@ -56,7 +56,8 @@ public static String normalize(String value) { String userInfo = ""; int at = authority.lastIndexOf('@'); if (at >= 0) { - userInfo = encodeUserInfo(authority.substring(0, at)) + "@"; + String encoded = encodeUserInfo(authority.substring(0, at)); + userInfo = encoded.isEmpty() ? "" : encoded + "@"; authority = authority.substring(at + 1); } String host = authority; @@ -70,15 +71,24 @@ public static String normalize(String value) { return value; } host = normalizeHost(host); - // At most five digits: anything longer is above MAX_PORT, and keeps parseInt from overflowing. - if (host == null || !port.matches("\\d{0,5}") || (!port.isEmpty() && toPort(port) > MAX_PORT)) { + if (host == null || !port.matches("\\d*")) { return value; } - if (!port.isEmpty() && defaultPort != null && toPort(port) == defaultPort) { - port = ""; + // Leading zeros do not count; a port with more digits than MAX_PORT has is above it, which also + // keeps parseInt from overflowing. + String digits = port.replaceFirst("^0+(?=\\d)", ""); + if (digits.length() > String.valueOf(MAX_PORT).length()) { + return value; } - if (!port.isEmpty()) { - port = ":" + toPort(port); + port = ""; + if (!digits.isEmpty()) { + int number = Integer.parseInt(digits); + if (number > MAX_PORT) { + return value; + } + if (defaultPort == null || number != defaultPort) { + port = ":" + number; + } } String rawPath = special ? m.group("path").replace('\\', '/') : m.group("path"); @@ -95,22 +105,16 @@ public static String normalize(String value) { return scheme + "://" + userInfo + host + port + path + query + fragment; } - private static int toPort(String digits) { - return Integer.parseInt(digits); - } - /** * Encodes the {@code username[:password]} part of the authority: each of the two is encoded on - * its own, so the separator between them stays a colon. + * its own, so the separator between them stays a colon, and an empty one is dropped. */ private static String encodeUserInfo(String userInfo) { int colon = userInfo.indexOf(':'); - if (colon < 0) { - return encode(userInfo, USERINFO_UNSAFE); - } - return encode(userInfo.substring(0, colon), USERINFO_UNSAFE) - + ":" - + encode(userInfo.substring(colon + 1), USERINFO_UNSAFE); + String username = encode(colon < 0 ? userInfo : userInfo.substring(0, colon), USERINFO_UNSAFE); + String password = colon < 0 ? "" : encode(userInfo.substring(colon + 1), USERINFO_UNSAFE); + // An empty username or password is dropped, along with the separator when the password is. + return password.isEmpty() ? username : username + ":" + password; } /** diff --git a/src/test/java/com/apify/client/CancellationTest.java b/src/test/java/com/apify/client/CancellationTest.java index 791ac35..bc3ece3 100644 --- a/src/test/java/com/apify/client/CancellationTest.java +++ b/src/test/java/com/apify/client/CancellationTest.java @@ -148,4 +148,114 @@ void cancellingABatchAddCancelsTheChunksInFlight() throws InterruptedException { assertTrue(backend.allHungCancelled()); assertFalse(future.isDone() && !future.isCancelled()); } + + @Test + void cancellingCallStopsTheLogRedirection() throws Exception { + MockTransport backend = + MockTransport.ofHanging( + request -> + request.uri().getQuery() != null + && request.uri().getQuery().contains("waitForFinish")); + java.util.concurrent.atomic.AtomicBoolean streamClosed = + new java.util.concurrent.atomic.AtomicBoolean(); + backend.scriptStream( + 200, + new java.io.InputStream() { + private final byte[] data = + "2999-01-01T00:00:00.000Z hello\n".getBytes(java.nio.charset.StandardCharsets.UTF_8); + private final java.util.concurrent.CountDownLatch closed = + new java.util.concurrent.CountDownLatch(1); + private int pos; + + // Serves one log line, then blocks like a live log with nothing more to say. + @Override + public synchronized int read(byte[] b, int off, int len) throws java.io.IOException { + if (pos < data.length) { + int n = Math.min(len, data.length - pos); + System.arraycopy(data, pos, b, off, n); + pos += n; + return n; + } + try { + closed.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + throw new java.io.IOException("stream closed"); + } + + @Override + public int read() throws java.io.IOException { + byte[] one = new byte[1]; + int n = read(one, 0, 1); + return n == -1 ? -1 : one[0] & 0xff; + } + + @Override + public void close() { + streamClosed.set(true); + closed.countDown(); + } + }); + CompletableFuture future = + client(backend, 0) + .actor("a1") + .call(null, new com.apify.client.actor.ActorCallOptions(), null); + // The start, the run fetch for the log prefix, the poll (hung) and the log stream itself. + await(() -> backend.hung.size() == 1 && backend.calls >= 4); + + future.cancel(true); + await(streamClosed::get); + assertTrue(backend.allHungCancelled()); + } + + @Test + void pollsUnderAnOverrideAreHeldOnlyAsLongAsTheOverrideCovers() { + MockTransport backend = + MockTransport.ofConstant(200, "{\"data\":{\"id\":\"r1\",\"status\":\"SUCCEEDED\"}}"); + client(backend, 0) + .withTimeout(com.apify.client.http.Timeout.ofSeconds(10)) + .run("r1") + .waitForFinish(120L) + .join(); + assertTrue(backend.lastUrl.contains("waitForFinish=5"), backend.lastUrl); + assertEquals(java.util.Optional.of(Duration.ofSeconds(10)), backend.lastTimeout); + } + + @Test + void aLongPollLoopDoesNotKeepEarlierResponsesReachable() throws InterruptedException { + // Each poll answers with a 1 MiB body; the loop is still pending after several polls. The + // future the caller holds must not keep the earlier polls' responses (or parsed runs) alive. + java.util.List> bodies = new java.util.ArrayList<>(); + String padding = "x".repeat(1 << 20); + MockTransport backend = + MockTransport.ofRouter( + request -> { + MockTransport.Scripted response = + MockTransport.ok( + 200, + "{\"data\":{\"id\":\"r1\",\"status\":\"RUNNING\",\"statusMessage\":\"" + + padding + + "\"}}"); + bodies.add(new java.lang.ref.WeakReference<>(response.body)); + return response; + }); + CompletableFuture pending = client(backend, 0).run("r1").waitForFinish(600L); + await(() -> backend.calls >= 8); + + int collected = 0; + for (int attempt = 0; attempt < 20 && collected < 4; attempt++) { + System.gc(); + Thread.sleep(50); + collected = 0; + for (var reference : List.copyOf(bodies).subList(0, bodies.size() - 2)) { + if (reference.get() == null) { + collected++; + } + } + } + pending.cancel(true); + assertTrue( + collected >= 4, "earlier poll responses stay reachable: only " + collected + " freed"); + } } diff --git a/src/test/java/com/apify/client/ClientInputTest.java b/src/test/java/com/apify/client/ClientInputTest.java index 73f2acc..d3cf95f 100644 --- a/src/test/java/com/apify/client/ClientInputTest.java +++ b/src/test/java/com/apify/client/ClientInputTest.java @@ -238,7 +238,7 @@ void scheduleToleratesANullUiState() { MockTransport.ofConstant( 200, "{\"data\":{\"id\":\"s1\",\"uiState\":{\"mode\":\"advanced\"}}}"); schedule = client(backend).schedule("s1").get().join().orElseThrow(); - assertEquals("advanced", schedule.getUiState().get("mode").asText()); + assertEquals("advanced", schedule.getUiState().get("mode").asString()); } @Test @@ -263,7 +263,7 @@ void webhookIntegrationFieldsAreTyped() { var webhook = client(backend).webhook("w1").get().join().orElseThrow(); assertTrue(webhook.getIsApifyIntegration()); assertEquals("SLACK_MESSAGE", webhook.getActionType()); - assertEquals("C1", webhook.getSlackMessage().get("channelId").asText()); + assertEquals("C1", webhook.getSlackMessage().get("channelId").asString()); } @Test diff --git a/src/test/java/com/apify/client/UrlNormalizationTest.java b/src/test/java/com/apify/client/UrlNormalizationTest.java index d20aba2..7b05ab2 100644 --- a/src/test/java/com/apify/client/UrlNormalizationTest.java +++ b/src/test/java/com/apify/client/UrlNormalizationTest.java @@ -78,6 +78,21 @@ void userInfoKeepsItsSeparatorAndEncodesEachPart() { "https://token@example.com/", UrlNormalizer.normalize("https://token@example.com")); } + @Test + void emptyUserInfoAndPasswordAreDropped() { + assertEquals("https://user@h.test/", UrlNormalizer.normalize("https://user:@h.test/")); + assertEquals("https://h.test/", UrlNormalizer.normalize("https://@h.test/")); + assertEquals("https://h.test/", UrlNormalizer.normalize("https://:@h.test")); + } + + @Test + void leadingZerosOfAPortDoNotCount() { + assertEquals("https://h.test/x", UrlNormalizer.normalize("https://h.test:0000443/x")); + assertEquals("http://h.test/", UrlNormalizer.normalize("http://h.test:00080/")); + assertEquals("https://h.test:8080/", UrlNormalizer.normalize("https://h.test:008080/")); + assertEquals("https://h.test:0/", UrlNormalizer.normalize("https://h.test:000/")); + } + @Test void anInvalidPortLeavesTheValueAlone() { assertEquals( From 6aa6a8697f88a9435a299a9cdfa264fcaf6a458d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 17:35:02 +0000 Subject: [PATCH 4/6] fix: cancel call() with a custom log destination, close a log that is still opening, exempt polls from withTimeout Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAKY9wkdABFznxvsCXt --- CHANGELOG.md | 6 +- README.md | 9 +- .../java/com/apify/client/ApifyClient.java | 7 +- .../apify/client/internal/HttpClientCore.java | 12 +++ .../client/internal/ResourceContext.java | 15 ++-- .../client/internal/RunStartSupport.java | 41 +++++---- .../apify/client/internal/UrlNormalizer.java | 3 +- .../com/apify/client/log/StreamedLog.java | 64 ++++++++----- .../java/com/apify/client/run/RunClient.java | 5 +- .../com/apify/client/CancellationTest.java | 90 +++++++++++++++++-- .../java/com/apify/client/MockTransport.java | 26 ++++-- .../apify/client/UrlNormalizationTest.java | 8 ++ 12 files changed, 219 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04f40fe..2f8b131 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,7 +46,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `batchAddRequests` serializes each request once instead of twice. - A `waitForFinish` hold extends the timeout of its request (the tier's plus the hold) instead of being clamped below it; a hold is cut down only to what `timeoutMax` (or a `withTimeout` override) - leaves, so the API still answers before the client gives up. + leaves, so the API still answers before the client gives up. The polls of `waitForFinish`/`call` + have no timeout and ignore `withTimeout`. - `Build.getChangelog()` is deprecated, as the specification deprecates `Build.changelog`. - The integration tests wait up to 90 s for a just-created resource to appear in a listing. @@ -56,7 +57,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Cancelling a future the client returned (`CompletableFuture.cancel`) now aborts the request in flight and stops the retries, `waitForFinish`/`call` polling and iteration pages that would follow; cancelling a derived stage (`thenApply`, `thenCompose`, ...) cancels the call it comes from. - Cancelling `call(...)` also stops its log redirection. See the Cancellation section of the README. + Cancelling `call(...)` also stops its log redirection, and closing a `StreamedLog` while its + stream is still opening launches no reader thread (the `start()` future is cancelled). See the Cancellation section of the README. - `pushItems` rejects what the API rejects (anything but an object or an array of objects) before a request is sent, and sends a pre-serialized `String` as it is. - An Actor or task input given as `byte[]` or `String` is sent as it is instead of being wrapped in diff --git a/README.md b/README.md index 990352f..2931d77 100644 --- a/README.md +++ b/README.md @@ -209,8 +209,10 @@ A `pushItems` request that times out is not retried, since the API may already h A `waitForFinish` hold is cut down to what the request timeout can cover, so the API answers before the client gives up on the request: with a small `timeoutMax`, the hold is what `timeoutMax` leaves -above the tier, and under a `withTimeout` override (which is kept as given) it is the override less -5 seconds. The same applies to every poll of `waitForFinish(...)` and `call(...)`. +above the tier, and under a `withTimeout` override (which is capped at `timeoutMax` like any timeout) +it is the override less 5 seconds. The polls of `waitForFinish(...)` and `call(...)` are not affected +by `withTimeout`: they have no timeout and always hold for up to a minute, so cancel the returned +future (see below) to stop one that never answers. ### Cancellation @@ -221,7 +223,8 @@ work behind it: `future.cancel(true)` aborts the request in flight and stops the request that has no timeout (the polls of `waitForFinish`/`call` and `Timeout.NO_TIMEOUT`): ```java -CompletableFuture call = client.actor("my-actor").call(null, new ActorStartOptions(), null); +CompletableFuture call = + client.actor("my-actor").call(null, new ActorCallOptions(), null); // redirects the run's log // ... later, when the result is no longer needed: call.cancel(true); // stops the polling and the log redirection; the run itself keeps running ``` diff --git a/src/main/java/com/apify/client/ApifyClient.java b/src/main/java/com/apify/client/ApifyClient.java index 206f847..2ee3ab6 100644 --- a/src/main/java/com/apify/client/ApifyClient.java +++ b/src/main/java/com/apify/client/ApifyClient.java @@ -111,9 +111,10 @@ public static ApifyClientBuilder builder() { * } * * A timeout above the builder's {@code timeoutMax} is capped at it. For a method that asks the - * API to hold the response with {@code waitForFinish} (and for the polling of {@code - * waitForFinish}/{@code call}), the given timeout is kept and the hold is cut to what it covers, - * the timeout less 5 seconds, so the API answers before the client gives up on the request. + * API to hold the response with {@code waitForFinish}, the given timeout is kept and the hold is + * cut to what it covers, the timeout less 5 seconds, so the API answers before the client gives + * up on the request. The polling of {@code waitForFinish}/{@code call} is not affected: it has no + * timeout, and cancelling the returned future is the way to stop one that never answers. */ public ApifyClient withTimeout(Timeout timeout) { return new ApifyClient(http.withTimeout(timeout), baseUrl, publicBaseUrl); diff --git a/src/main/java/com/apify/client/internal/HttpClientCore.java b/src/main/java/com/apify/client/internal/HttpClientCore.java index 00854c8..2b3554c 100644 --- a/src/main/java/com/apify/client/internal/HttpClientCore.java +++ b/src/main/java/com/apify/client/internal/HttpClientCore.java @@ -170,6 +170,18 @@ public HttpClientCore withTimeout(Timeout timeout) { warnedCappedTimeouts); } + /** + * This client without the override set by {@link #withTimeout}: every request gets the timeout of + * its own method again. + */ + public HttpClientCore withoutTimeoutOverride() { + if (timeoutOverride == null) { + return this; + } + return new HttpClientCore( + transport, token, userAgent, retry, compressor, defaultHeaders, null, warnedCappedTimeouts); + } + /** * Resolves a timeout to the duration of its first attempt: a tier to its configured duration, an * exact timeout to itself, and {@link Timeout#NO_TIMEOUT} to {@code null} (no timeout). A value diff --git a/src/main/java/com/apify/client/internal/ResourceContext.java b/src/main/java/com/apify/client/internal/ResourceContext.java index 5fc05ef..d1014be 100644 --- a/src/main/java/com/apify/client/internal/ResourceContext.java +++ b/src/main/java/com/apify/client/internal/ResourceContext.java @@ -595,7 +595,13 @@ public CompletableFuture waitForFinish( : MAX_WAIT_FOR_FINISH_SECS; long budgetMillis = effectiveWaitSecs * 1000L; long start = System.currentTimeMillis(); - return pollOnce(dataType, isTerminal, resourceName, start, budgetMillis, null, null); + // The polls have no timeout, and a withTimeout override does not apply to them: it would cut + // every hold short (a 5 s override would poll four times a second), and cancelling the returned + // future is the way out of a poll that never answers. + ResourceContext polling = + new ResourceContext( + http.withoutTimeoutOverride(), url, baseParams, apiOrigin, publicOrigin, ownId); + return polling.pollOnce(dataType, isTerminal, resourceName, start, budgetMillis, null, null); } /** @@ -636,13 +642,10 @@ private CompletableFuture pollOnce( long remainingSecs = (budgetMillis - elapsed) / 1000L; long requestSecs = Math.min(Math.max(remainingSecs, 0), WAIT_REQUEST_SECS); - // The polls have no timeout of their own; under a withTimeout override the hold is cut to what - // the override covers, so the API answers before the client gives up on a poll. - HttpClientCore.WaitHold hold = http.waitHold(Timeout.NO_TIMEOUT, requestSecs); QueryParams params = new QueryParams(); - params.addLong("waitForFinish", hold.waitForFinishSecs()); + params.addLong("waitForFinish", requestSecs); - return this.getResourceRequired("", params, dataType, hold.timeout()) + return this.getResourceRequired("", params, dataType, Timeout.NO_TIMEOUT) .handle( (value, error) -> { if (error == null) { diff --git a/src/main/java/com/apify/client/internal/RunStartSupport.java b/src/main/java/com/apify/client/internal/RunStartSupport.java index ca3ffa8..fd5681e 100644 --- a/src/main/java/com/apify/client/internal/RunStartSupport.java +++ b/src/main/java/com/apify/client/internal/RunStartSupport.java @@ -207,18 +207,11 @@ public static CompletableFuture callWithLogStreaming( .thenCompose( run -> { RunClient runClient = root.run(run.getId()); - CompletableFuture streamedLogFuture = + CompletableFuture logStarted = logStreamingEnabled - ? startStreamedLogQuietly(runClient, logOptions) + ? startStreamedLogQuietly(runClient, logOptions, opened, finished) : CancellableFuture.completed(null); - return streamedLogFuture.thenCompose( - streamedLog -> { - opened.set(streamedLog); - if (finished.get() && streamedLog != null) { - streamedLog.close(); // the call ended while the log was being opened - } - return runClient.waitForFinish(waitSecs); - }); + return logStarted.thenCompose(unused -> runClient.waitForFinish(waitSecs)); }); // Closed from a callback of the returned future itself, which runs whether the call finishes, // fails or is cancelled (a stage that is cancelled never runs its own callbacks). @@ -235,17 +228,29 @@ public static CompletableFuture callWithLogStreaming( /** * Starts {@code call}'s default log streaming, swallowing (rather than propagating) any failure - * to open the live log stream, so a transient log-endpoint issue cannot abort the run itself. - * Completes with the started {@link StreamedLog}, or {@code null} if it could not be started. + * to open the live log stream, so a transient log-endpoint issue cannot abort the run itself. The + * {@link StreamedLog} is handed to {@code opened} as soon as it exists, before it is started, so + * a call that ends while the stream is still opening can close it; one that has already ended + * ({@code finished}) closes it right away. */ - private static CompletableFuture startStreamedLogQuietly( - RunClient runClient, StreamedLogOptions logOptions) { + private static CompletableFuture startStreamedLogQuietly( + RunClient runClient, + StreamedLogOptions logOptions, + AtomicReference opened, + AtomicBoolean finished) { CompletableFuture created = - logOptions != null ? runClient.getStreamedLog(logOptions) : runClient.getStreamedLog(); + CancellableFuture.wrap( + logOptions != null ? runClient.getStreamedLog(logOptions) : runClient.getStreamedLog()); return created .thenCompose( - streamedLog -> - streamedLog.start().handle((unused, error) -> error == null ? streamedLog : null)) - .exceptionally(error -> null); + streamedLog -> { + opened.set(streamedLog); + if (finished.get()) { + streamedLog.close(); + return CancellableFuture.completed(null); + } + return streamedLog.start(); + }) + .handle((unused, error) -> (Void) null); } } diff --git a/src/main/java/com/apify/client/internal/UrlNormalizer.java b/src/main/java/com/apify/client/internal/UrlNormalizer.java index ab9477b..c571a15 100644 --- a/src/main/java/com/apify/client/internal/UrlNormalizer.java +++ b/src/main/java/com/apify/client/internal/UrlNormalizer.java @@ -67,7 +67,8 @@ public static String normalize(String value) { host = authority.substring(0, colon); port = authority.substring(colon + 1); } - if (special && host.isEmpty()) { + // An empty host is valid only for a non-special scheme without credentials or a port. + if (host.isEmpty() && (special || at >= 0 || !port.isEmpty())) { return value; } host = normalizeHost(host); diff --git a/src/main/java/com/apify/client/log/StreamedLog.java b/src/main/java/com/apify/client/log/StreamedLog.java index c2497d5..2289ca6 100644 --- a/src/main/java/com/apify/client/log/StreamedLog.java +++ b/src/main/java/com/apify/client/log/StreamedLog.java @@ -2,6 +2,7 @@ import com.apify.client.http.ApifyApiException; import com.apify.client.internal.Async; +import com.apify.client.internal.CancellableFuture; import com.apify.client.internal.HttpClientCore; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -103,6 +104,12 @@ public final class StreamedLog implements AutoCloseable { */ private boolean starting; + /** + * Set by {@link #close()} while a {@link #start()} is still opening its stream: the stream is + * then closed as soon as it opens, and no reader thread is launched. + */ + private boolean closeRequested; + public StreamedLog(LogClient logClient, Consumer destination, boolean fromStart) { this.logClient = logClient; this.destination = destination; @@ -132,26 +139,40 @@ public synchronized CompletableFuture start() { starting = true; stopLogging = false; forwardingFailed = false; - return logClient.stream(new LogOptions().raw(true)) - .handle( - (stream, error) -> { - synchronized (StreamedLog.this) { - starting = false; - if (error != null) { - Throwable cause = HttpClientCore.unwrapCompletion(error); - if (cause instanceof RuntimeException runtimeException) { - throw runtimeException; - } - throw new IllegalStateException("failed to open log stream", cause); - } - activeStream = stream; - Thread thread = new Thread(this::streamLog, "apify-streamed-log"); - thread.setDaemon(true); - streamingThread = thread; - thread.start(); - } - return null; - }); + CancellableFuture started = new CancellableFuture<>(); + CompletableFuture opening = logClient.stream(new LogOptions().raw(true)); + // Cancelling the returned future cancels the stream-open request. The callback below hangs off + // the open request itself, so it runs however that ends and always resets the opening state. + started.link(opening); + opening.whenComplete( + (stream, error) -> { + synchronized (StreamedLog.this) { + starting = false; + if (error != null) { + closeRequested = false; + Throwable cause = HttpClientCore.unwrapCompletion(error); + started.completeExceptionally( + cause instanceof RuntimeException + ? cause + : new IllegalStateException("failed to open log stream", cause)); + return; + } + if (closeRequested || started.isCancelled()) { + // Closed (or cancelled) while the stream was opening: do not launch a reader for it. + closeRequested = false; + closeQuietly(stream); + started.cancel(true); + return; + } + activeStream = stream; + Thread thread = new Thread(this::streamLog, "apify-streamed-log"); + thread.setDaemon(true); + streamingThread = thread; + thread.start(); + started.complete(null); + } + }); + return started; } /** @@ -182,6 +203,9 @@ public synchronized void stop() { */ @Override public synchronized void close() { + if (starting) { + closeRequested = true; + } if (streamingThread != null) { stopStreaming(); } diff --git a/src/main/java/com/apify/client/run/RunClient.java b/src/main/java/com/apify/client/run/RunClient.java index 0fc5b65..2dd5570 100644 --- a/src/main/java/com/apify/client/run/RunClient.java +++ b/src/main/java/com/apify/client/run/RunClient.java @@ -5,6 +5,7 @@ import com.apify.client.dataset.DatasetListItemsOptions; import com.apify.client.http.Timeout; import com.apify.client.internal.AsyncPaginatedPublisher; +import com.apify.client.internal.CancellableFuture; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; import com.apify.client.internal.QueryParams; @@ -333,11 +334,11 @@ public CompletableFuture getStreamedLog() { public CompletableFuture getStreamedLog(StreamedLogOptions options) { Consumer destination = options.destination(); if (destination != null) { - return CompletableFuture.completedFuture( + return CancellableFuture.completed( new StreamedLog(log(), destination, options.fromStartValue())); } if (options.prefixValue() != null) { - return CompletableFuture.completedFuture( + return CancellableFuture.completed( new StreamedLog( log(), defaultLogDestination(options.prefixValue()), options.fromStartValue())); } diff --git a/src/test/java/com/apify/client/CancellationTest.java b/src/test/java/com/apify/client/CancellationTest.java index bc3ece3..3ce5534 100644 --- a/src/test/java/com/apify/client/CancellationTest.java +++ b/src/test/java/com/apify/client/CancellationTest.java @@ -210,23 +210,30 @@ public void close() { } @Test - void pollsUnderAnOverrideAreHeldOnlyAsLongAsTheOverrideCovers() { + void pollsIgnoreAWithTimeoutOverrideAndKeepHoldingTheFullMinute() { MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"id\":\"r1\",\"status\":\"SUCCEEDED\"}}"); client(backend, 0) - .withTimeout(com.apify.client.http.Timeout.ofSeconds(10)) + .withTimeout(com.apify.client.http.Timeout.ofSeconds(5)) .run("r1") .waitForFinish(120L) .join(); - assertTrue(backend.lastUrl.contains("waitForFinish=5"), backend.lastUrl); - assertEquals(java.util.Optional.of(Duration.ofSeconds(10)), backend.lastTimeout); + assertTrue(backend.lastUrl.contains("waitForFinish=60"), backend.lastUrl); + assertEquals(java.util.Optional.empty(), backend.lastTimeout, "a poll has no timeout"); } @Test void aLongPollLoopDoesNotKeepEarlierResponsesReachable() throws InterruptedException { // Each poll answers with a 1 MiB body; the loop is still pending after several polls. The // future the caller holds must not keep the earlier polls' responses (or parsed runs) alive. - java.util.List> bodies = new java.util.ArrayList<>(); + // The test needs the JVM to honour System.gc(); skip it where it does not + // (-XX:+DisableExplicitGC). + java.lang.ref.WeakReference control = new java.lang.ref.WeakReference<>(new Object()); + System.gc(); + org.junit.jupiter.api.Assumptions.assumeTrue(control.get() == null, "System.gc() is a no-op"); + + java.util.List> bodies = + new java.util.concurrent.CopyOnWriteArrayList<>(); String padding = "x".repeat(1 << 20); MockTransport backend = MockTransport.ofRouter( @@ -258,4 +265,77 @@ void aLongPollLoopDoesNotKeepEarlierResponsesReachable() throws InterruptedExcep assertTrue( collected >= 4, "earlier poll responses stay reachable: only " + collected + " freed"); } + + @Test + void cancellingCallWithACustomLogDestinationStillStopsThePolling() throws InterruptedException { + MockTransport backend = + MockTransport.ofHanging( + request -> + request.uri().getQuery() != null + && request.uri().getQuery().contains("waitForFinish")); + backend.scriptStream(200, "2999-01-01T00:00:00.000Z hello\n"); + var options = + new com.apify.client.actor.ActorCallOptions() + .logOptions(new com.apify.client.log.StreamedLogOptions().toLog(line -> {})); + CompletableFuture future = client(backend, 0).actor("a1").call(null, options, null); + await(() -> backend.hung.size() == 1); + + future.cancel(true); + + assertTrue(backend.allHungCancelled(), "the poll is aborted with a custom log destination"); + } + + @Test + void aCancelWhileTheLogStreamIsOpeningLeavesNoReaderThread() throws InterruptedException { + MockTransport backend = + MockTransport.ofHanging( + request -> + request.uri().getQuery() != null + && request.uri().getQuery().contains("waitForFinish")); + backend.scriptStream(200, "2999-01-01T00:00:00.000Z hello\n"); + CompletableFuture gate = new CompletableFuture<>(); + backend.gateStreams(gate); // the log stream does not open until the gate is released + long readersBefore = readerThreads(); + CompletableFuture future = + client(backend, 0) + .actor("a1") + .call(null, new com.apify.client.actor.ActorCallOptions(), null); + await(() -> backend.calls >= 3); // start, run fetch for the prefix, and the stream open request + + future.cancel(true); + gate.complete(null); // the stream opens after the call has been cancelled + Thread.sleep(300); + + assertEquals(readersBefore, readerThreads(), "no log reader thread is left behind"); + } + + @Test + void closingAStreamedLogWhileItsStreamOpensLaunchesNoReader() throws Exception { + MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"id\":\"r1\"}}"); + backend.scriptStream(200, "2999-01-01T00:00:00.000Z hello\n"); + CompletableFuture gate = new CompletableFuture<>(); + backend.gateStreams(gate); + long readersBefore = readerThreads(); + var log = + client(backend, 0) + .run("r1") + .getStreamedLog(new com.apify.client.log.StreamedLogOptions().toLog(line -> {})) + .join(); + + CompletableFuture started = log.start(); + log.close(); + gate.complete(null); + Thread.sleep(300); + + assertEquals(readersBefore, readerThreads()); + assertTrue(started.isCancelled(), "the start that was closed is cancelled"); + log.start().join(); // a closed-while-opening log can be started again + log.close(); + } + + private static long readerThreads() { + return Thread.getAllStackTraces().keySet().stream() + .filter(thread -> thread.getName().equals("apify-streamed-log") && thread.isAlive()) + .count(); + } } diff --git a/src/test/java/com/apify/client/MockTransport.java b/src/test/java/com/apify/client/MockTransport.java index 792ee60..fed677f 100644 --- a/src/test/java/com/apify/client/MockTransport.java +++ b/src/test/java/com/apify/client/MockTransport.java @@ -72,6 +72,14 @@ static final class Scripted { final List>> hung = new java.util.concurrent.CopyOnWriteArrayList<>(); + /** When set, streaming responses are delivered only once this gate completes. */ + private volatile CompletableFuture streamGate; + + /** Holds back every streaming response until {@code gate} completes. */ + void gateStreams(CompletableFuture gate) { + this.streamGate = gate; + } + /** When set, answers each request itself instead of serving the scripted queue. */ private java.util.function.Function router; @@ -202,16 +210,20 @@ public synchronized CompletableFuture> sendStreamingAs calls++; lastUrl = request.uri().toString(); lastMethod = request.method(); + HttpResponse response; if (streamQueue.isEmpty()) { Scripted r = responses.get(0); - return CompletableFuture.completedFuture( - new FakeStreamResponse( - request.uri(), r.status, new java.io.ByteArrayInputStream(r.body))); + response = + new FakeStreamResponse(request.uri(), r.status, new java.io.ByteArrayInputStream(r.body)); + } else { + int idx = Math.min(streamCallIndex++, streamQueue.size() - 1); + StreamEntry entry = streamQueue.get(idx); + response = new FakeStreamResponse(request.uri(), entry.status, entry.body); } - int idx = Math.min(streamCallIndex++, streamQueue.size() - 1); - StreamEntry entry = streamQueue.get(idx); - return CompletableFuture.completedFuture( - new FakeStreamResponse(request.uri(), entry.status, entry.body)); + CompletableFuture gate = streamGate; + return gate == null + ? CompletableFuture.completedFuture(response) + : gate.thenApply(unused -> response); } private static byte[] readBody(HttpRequest request) { diff --git a/src/test/java/com/apify/client/UrlNormalizationTest.java b/src/test/java/com/apify/client/UrlNormalizationTest.java index 7b05ab2..b77fa98 100644 --- a/src/test/java/com/apify/client/UrlNormalizationTest.java +++ b/src/test/java/com/apify/client/UrlNormalizationTest.java @@ -85,6 +85,14 @@ void emptyUserInfoAndPasswordAreDropped() { assertEquals("https://h.test/", UrlNormalizer.normalize("https://:@h.test")); } + @Test + void anEmptyHostWithCredentialsOrAPortIsInvalid() { + assertEquals("foo://@/", UrlNormalizer.normalize("foo://@/")); + assertEquals("foo://user@/", UrlNormalizer.normalize("foo://user@/")); + assertEquals("foo://:80/", UrlNormalizer.normalize("foo://:80/")); + assertEquals("foo:///path", UrlNormalizer.normalize("foo:///path")); + } + @Test void leadingZerosOfAPortDoNotCount() { assertEquals("https://h.test/x", UrlNormalizer.normalize("https://h.test:0000443/x")); From 130c3f4785ec22a01ee1d1772d1364bc97a7788d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 17:46:29 +0000 Subject: [PATCH 5/6] fix: complete StreamedLog.start() outside its lock, close a log started just after the call ended Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAKY9wkdABFznxvsCXt --- CHANGELOG.md | 3 +- .../client/internal/RunStartSupport.java | 11 +++-- .../apify/client/internal/UrlNormalizer.java | 7 +-- .../com/apify/client/log/StreamedLog.java | 46 ++++++++++++------- .../com/apify/client/CancellationTest.java | 37 ++++++++++++++- .../apify/client/UrlNormalizationTest.java | 1 + 6 files changed, 79 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f8b131..bb0fba9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). flight and stops the retries, `waitForFinish`/`call` polling and iteration pages that would follow; cancelling a derived stage (`thenApply`, `thenCompose`, ...) cancels the call it comes from. Cancelling `call(...)` also stops its log redirection, and closing a `StreamedLog` while its - stream is still opening launches no reader thread (the `start()` future is cancelled). See the Cancellation section of the README. + stream is still opening launches no reader thread (the `start()` future is cancelled). See the + Cancellation section of the README. - `pushItems` rejects what the API rejects (anything but an object or an array of objects) before a request is sent, and sends a pre-serialized `String` as it is. - An Actor or task input given as `byte[]` or `String` is sent as it is instead of being wrapped in diff --git a/src/main/java/com/apify/client/internal/RunStartSupport.java b/src/main/java/com/apify/client/internal/RunStartSupport.java index fd5681e..8f526e8 100644 --- a/src/main/java/com/apify/client/internal/RunStartSupport.java +++ b/src/main/java/com/apify/client/internal/RunStartSupport.java @@ -239,8 +239,7 @@ private static CompletableFuture startStreamedLogQuietly( AtomicReference opened, AtomicBoolean finished) { CompletableFuture created = - CancellableFuture.wrap( - logOptions != null ? runClient.getStreamedLog(logOptions) : runClient.getStreamedLog()); + logOptions != null ? runClient.getStreamedLog(logOptions) : runClient.getStreamedLog(); return created .thenCompose( streamedLog -> { @@ -249,7 +248,13 @@ private static CompletableFuture startStreamedLogQuietly( streamedLog.close(); return CancellableFuture.completed(null); } - return streamedLog.start(); + CompletableFuture started = streamedLog.start(); + if (finished.get()) { + // The call ended between the check above and the start: close() while the stream + // is opening makes it launch no reader. + streamedLog.close(); + } + return started; }) .handle((unused, error) -> (Void) null); } diff --git a/src/main/java/com/apify/client/internal/UrlNormalizer.java b/src/main/java/com/apify/client/internal/UrlNormalizer.java index c571a15..83c8933 100644 --- a/src/main/java/com/apify/client/internal/UrlNormalizer.java +++ b/src/main/java/com/apify/client/internal/UrlNormalizer.java @@ -63,12 +63,13 @@ public static String normalize(String value) { String host = authority; String port = ""; int colon = authority.lastIndexOf(':'); - if (colon >= 0 && authority.indexOf(']') < colon) { + boolean hasPortSeparator = colon >= 0 && authority.indexOf(']') < colon; + if (hasPortSeparator) { host = authority.substring(0, colon); port = authority.substring(colon + 1); } - // An empty host is valid only for a non-special scheme without credentials or a port. - if (host.isEmpty() && (special || at >= 0 || !port.isEmpty())) { + // An empty host is valid only for a non-special scheme without credentials or a port separator. + if (host.isEmpty() && (special || at >= 0 || hasPortSeparator)) { return value; } host = normalizeHost(host); diff --git a/src/main/java/com/apify/client/log/StreamedLog.java b/src/main/java/com/apify/client/log/StreamedLog.java index 2289ca6..1d8c4df 100644 --- a/src/main/java/com/apify/client/log/StreamedLog.java +++ b/src/main/java/com/apify/client/log/StreamedLog.java @@ -126,6 +126,9 @@ public StreamedLog(LogClient logClient, Consumer destination, boolean fr * leaving the reader blocked on a live read that never returns and {@code join()} hanging * forever. * + *

Cancelling the returned future cancels the stream-open request; {@link #close()} while the + * stream is still opening cancels it too. + * * @throws IllegalStateException if redirection is already running (or a previous {@link #start()} * call's stream-open is still in flight) * @implNote the returned future completes exceptionally with an {@link ApifyApiException} if the @@ -146,31 +149,38 @@ public synchronized CompletableFuture start() { started.link(opening); opening.whenComplete( (stream, error) -> { + // The outcome is decided under the lock but delivered after it is released: the stages + // chained on the returned future run on this thread, and one of them may wait for a + // destination consumer that itself calls close() (which takes the lock). + Runnable outcome; synchronized (StreamedLog.this) { starting = false; if (error != null) { closeRequested = false; Throwable cause = HttpClientCore.unwrapCompletion(error); - started.completeExceptionally( + Throwable failure = cause instanceof RuntimeException ? cause - : new IllegalStateException("failed to open log stream", cause)); - return; - } - if (closeRequested || started.isCancelled()) { + : new IllegalStateException("failed to open log stream", cause); + outcome = () -> started.completeExceptionally(failure); + } else if (closeRequested || started.isCancelled()) { // Closed (or cancelled) while the stream was opening: do not launch a reader for it. closeRequested = false; - closeQuietly(stream); - started.cancel(true); - return; + outcome = + () -> { + closeQuietly(stream); + started.cancel(true); + }; + } else { + activeStream = stream; + Thread thread = new Thread(this::streamLog, "apify-streamed-log"); + thread.setDaemon(true); + streamingThread = thread; + thread.start(); + outcome = () -> started.complete(null); } - activeStream = stream; - Thread thread = new Thread(this::streamLog, "apify-streamed-log"); - thread.setDaemon(true); - streamingThread = thread; - thread.start(); - started.complete(null); } + outcome.run(); }); return started; } @@ -197,9 +207,11 @@ public synchronized void stop() { } /** - * Stops redirection if it is still running; a no-op otherwise. Fully idempotent and safe under a - * concurrent {@link #stop()} or a double {@code close()}: the running-check and the stop happen - * atomically under the monitor, so unlike {@link #stop()} this never throws. + * Stops redirection if it is still running; a no-op otherwise. If {@link #start()} is still + * opening the log stream, the stream is closed as soon as it opens, no reader is launched and the + * future {@code start()} returned is cancelled. Fully idempotent and safe under a concurrent + * {@link #stop()} or a double {@code close()}: the running-check and the stop happen atomically + * under the monitor, so unlike {@link #stop()} this never throws. */ @Override public synchronized void close() { diff --git a/src/test/java/com/apify/client/CancellationTest.java b/src/test/java/com/apify/client/CancellationTest.java index 3ce5534..f9a56fe 100644 --- a/src/test/java/com/apify/client/CancellationTest.java +++ b/src/test/java/com/apify/client/CancellationTest.java @@ -286,7 +286,7 @@ void cancellingCallWithACustomLogDestinationStillStopsThePolling() throws Interr } @Test - void aCancelWhileTheLogStreamIsOpeningLeavesNoReaderThread() throws InterruptedException { + void aCancelBeforeTheLogStreamOpensLeavesNoReaderThread() throws InterruptedException { MockTransport backend = MockTransport.ofHanging( request -> @@ -303,7 +303,7 @@ void aCancelWhileTheLogStreamIsOpeningLeavesNoReaderThread() throws InterruptedE await(() -> backend.calls >= 3); // start, run fetch for the prefix, and the stream open request future.cancel(true); - gate.complete(null); // the stream opens after the call has been cancelled + gate.complete(null); // the held-back response arrives for an exchange that was cancelled Thread.sleep(300); assertEquals(readersBefore, readerThreads(), "no log reader thread is left behind"); @@ -333,6 +333,39 @@ void closingAStreamedLogWhileItsStreamOpensLaunchesNoReader() throws Exception { log.close(); } + @Test + void aDestinationThatClosesTheLogCannotDeadlockTheStageChainedOnStart() throws Exception { + MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"id\":\"r1\"}}"); + backend.scriptStream(200, "2999-01-01T00:00:00.000Z hello\n"); + CompletableFuture gate = new CompletableFuture<>(); + backend.gateStreams(gate); + CompletableFuture sawLine = new CompletableFuture<>(); + var holder = + new java.util.concurrent.atomic.AtomicReference(); + holder.set( + client(backend, 0) + .run("r1") + .getStreamedLog( + new com.apify.client.log.StreamedLogOptions() + .toLog( + line -> { + holder.get().close(); // allowed from the destination: takes the lock + sawLine.complete(null); + })) + .join()); + + // A stage chained on start() waits for the destination, which needs the StreamedLog lock: the + // future must therefore be completed after the lock is released. + CompletableFuture chained = + holder + .get() + .start() + .thenRun(() -> sawLine.orTimeout(5, java.util.concurrent.TimeUnit.SECONDS).join()); + gate.complete(null); + + chained.get(10, java.util.concurrent.TimeUnit.SECONDS); + } + private static long readerThreads() { return Thread.getAllStackTraces().keySet().stream() .filter(thread -> thread.getName().equals("apify-streamed-log") && thread.isAlive()) diff --git a/src/test/java/com/apify/client/UrlNormalizationTest.java b/src/test/java/com/apify/client/UrlNormalizationTest.java index b77fa98..ac388ef 100644 --- a/src/test/java/com/apify/client/UrlNormalizationTest.java +++ b/src/test/java/com/apify/client/UrlNormalizationTest.java @@ -90,6 +90,7 @@ void anEmptyHostWithCredentialsOrAPortIsInvalid() { assertEquals("foo://@/", UrlNormalizer.normalize("foo://@/")); assertEquals("foo://user@/", UrlNormalizer.normalize("foo://user@/")); assertEquals("foo://:80/", UrlNormalizer.normalize("foo://:80/")); + assertEquals("foo://:/", UrlNormalizer.normalize("foo://:/")); assertEquals("foo:///path", UrlNormalizer.normalize("foo:///path")); } From 005190a595433c6b39b127da8ebb9d604514c074 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 17:51:46 +0000 Subject: [PATCH 6/6] docs: reword the StreamedLog.start() javadoc on close() during the open Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ocuAKY9wkdABFznxvsCXt --- src/main/java/com/apify/client/log/StreamedLog.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/apify/client/log/StreamedLog.java b/src/main/java/com/apify/client/log/StreamedLog.java index 1d8c4df..ce39d38 100644 --- a/src/main/java/com/apify/client/log/StreamedLog.java +++ b/src/main/java/com/apify/client/log/StreamedLog.java @@ -126,8 +126,9 @@ public StreamedLog(LogClient logClient, Consumer destination, boolean fr * leaving the reader blocked on a live read that never returns and {@code join()} hanging * forever. * - *

Cancelling the returned future cancels the stream-open request; {@link #close()} while the - * stream is still opening cancels it too. + *

Cancelling the returned future cancels the stream-open request. {@link #close()} while the + * stream is still opening closes the stream once it opens, launches no reader and cancels the + * returned future. * * @throws IllegalStateException if redirection is already running (or a previous {@link #start()} * call's stream-open is still in flight)