diff --git a/docs/examples/accept_user_input.mdx b/docs/examples/accept_user_input.mdx deleted file mode 100644 index ddd872f0a848..000000000000 --- a/docs/examples/accept_user_input.mdx +++ /dev/null @@ -1,23 +0,0 @@ ---- -id: accept-user-input -title: Accept user input ---- - -import CodeBlock from '@theme/CodeBlock'; -import AcceptInputSource from '!!raw-loader!./accept_user_input.ts'; - -This example accepts and logs user input: - - - {AcceptInputSource} - - -To provide the actor with input, create a `INPUT.json` file inside the "default" key-value store: - -```bash -{PROJECT_FOLDER}/storage/key_value_stores/default/INPUT.json -``` - -Anything in this file will be available to the actor when it runs. - -To learn about other ways to provide an actor with input, refer to the [Apify Platform Documentation](https://apify.com/docs/actor#run). diff --git a/docs/examples/accept_user_input.ts b/docs/examples/accept_user_input.ts deleted file mode 100644 index 25d4b48ddb59..000000000000 --- a/docs/examples/accept_user_input.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { KeyValueStore } from 'crawlee'; - -const input = await KeyValueStore.getInput(); -console.log(input); diff --git a/docs/examples/crawl_single_url.mdx b/docs/examples/crawl_single_url.mdx index d15e1d8f5c56..99415225c397 100644 --- a/docs/examples/crawl_single_url.mdx +++ b/docs/examples/crawl_single_url.mdx @@ -14,4 +14,4 @@ to grab the HTML of a web page. {CrawlSource} -If you don't want to hard-code the URL into the script, refer to the [Accept User Input](./accept-user-input) example. +If you don't want to hard-code the URL into the script, read it from a command-line argument or an environment variable, or, on the Apify platform, from the Actor input with [`Actor.getInput()`](https://docs.apify.com/sdk/js/reference/class/Actor#getInput). diff --git a/docs/guides/result_storage.mdx b/docs/guides/result_storage.mdx index e0890fa7a61c..9e21c17eb3ab 100644 --- a/docs/guides/result_storage.mdx +++ b/docs/guides/result_storage.mdx @@ -14,7 +14,7 @@ By default, Crawlee storage is managed by the `KeyValueStore` class. In order to simplify access to the default key-value store, Crawlee also provides `KeyValueStore.getValue()` and `KeyValueStore.setValue()` functions. @@ -29,7 +29,7 @@ The data is stored in the directory specified by the `CRAWLEE_STORAGE_DIR` envir `{STORE_ID}` is the name or the ID of the key-value store. The default key-value store has ID `default`. The `{KEY}` is the key of the record, used as the filename verbatim — no extension is derived from the content type, which is recorded in the `{KEY}.__metadata__.json` sidecar instead. -A file you drop into the directory by hand is adopted as a record when the store is opened, keyed by its filename (see the [v4 upgrading guide](../upgrading/upgrading-to-v4#out-of-band-key-value-files-eg-a-hand-placed-inputjson) for the run-input exception). Files added while the crawler is running are not picked up — write those with `setValue()`. +A file you drop into the directory by hand is adopted as a record when the store is opened, keyed by its filename (see the [v4 upgrading guide](../upgrading/upgrading-to-v4#out-of-band-key-value-files-eg-a-hand-placed-inputjson)). Files added while the crawler is running are not picked up — write those with `setValue()`. ::: @@ -38,12 +38,6 @@ The following code demonstrates basic operations of key-value stores: ```javascript import { KeyValueStore } from 'crawlee'; -// Get the INPUT from the default key-value store -const input = await KeyValueStore.getInput(); - -// Write the OUTPUT to the default key-value store -await KeyValueStore.setValue('OUTPUT', { myResult: 123 }); - // Open a named key-value store const store = await KeyValueStore.open('some-name'); @@ -61,7 +55,7 @@ const value = await store.getValue('some-key'); await store.setValue('some-key', null); ``` -To see a real-world example of how to get the input from the key-value store, see the [Screenshots](../examples/capture-screenshot) example. +To see a real-world example of using the key-value store, see the [Screenshots](../examples/capture-screenshot) example. ## Dataset @@ -113,7 +107,7 @@ import { purgeDefaultStorages } from 'crawlee'; await purgeDefaultStorages(); ``` -Calling this function will clean up the run-scoped results storage directories - the default ones and any alias-keyed one - except the `INPUT` key in the default key-value store directory. This is a shortcut for running (optional) `purge` method on the `StorageBackend` interface, in other words it will call the `purge` method of the underlying storage implementation we are currently using. In addition, this method will make sure the storage is purged only once for a given execution context, so it is safe to call it multiple times. +Calling this function will clean up the run-scoped results storage directories - the default ones and any alias-keyed one. This is a shortcut for running (optional) `purge` method on the `StorageBackend` interface, in other words it will call the `purge` method of the underlying storage implementation we are currently using. In addition, this method will make sure the storage is purged only once for a given execution context, so it is safe to call it multiple times. ## Transactional storage diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md index 04f28552054b..d34b847d9c93 100644 --- a/docs/public-api/crawlee-core.api.md +++ b/docs/public-api/crawlee-core.api.md @@ -154,7 +154,6 @@ export const crawleeConfigFields: { persistStateIntervalMillis: ConfigField>>; internalTimeoutMillis: ConfigField>>; systemInfoIntervalMillis: ConfigField>>; - inputKey: ConfigField>; headless: ConfigField>>; xvfb: ConfigField>>; chromeExecutablePath: ConfigField>; diff --git a/docs/public-api/crawlee-fs-storage.api.md b/docs/public-api/crawlee-fs-storage.api.md index 3d11b20b8d58..721080885c86 100644 --- a/docs/public-api/crawlee-fs-storage.api.md +++ b/docs/public-api/crawlee-fs-storage.api.md @@ -4,9 +4,12 @@ ```ts +import { AdoptionCandidate } from '@crawlee/fs-storage-native'; import type { CrawleeLogger } from '@crawlee/types'; import type * as storage from '@crawlee/types'; +export { AdoptionCandidate } + // @public export class FileSystemStorageBackend implements storage.StorageBackend { constructor(options: FileSystemStorageOptions); @@ -19,6 +22,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend { // (undocumented) readonly datasetsDirectory: string; getStorageBackendCacheKey(): string; + protected keyValueStoreAdoptionCandidates(_options: KeyValueStoreHookOptions): AdoptionCandidate[]; // (undocumented) readonly keyValueStoresDirectory: string; // (undocumented) @@ -26,6 +30,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend { // (undocumented) readonly logger?: CrawleeLogger; purge(): Promise; + protected purgeKeyValueStore(store: PurgeableKeyValueStoreBackend, _options: KeyValueStoreHookOptions): Promise; // (undocumented) readonly requestQueueAccess: 'single' | 'shared'; // (undocumented) @@ -37,12 +42,21 @@ export class FileSystemStorageBackend implements storage.StorageBackend { // @public (undocumented) export interface FileSystemStorageOptions { - inputKey?: string; localDataDirectory: string; logger?: CrawleeLogger; requestQueueAccess?: 'single' | 'shared'; } +// @public +export interface KeyValueStoreHookOptions { + isDefaultStore: boolean; +} + +// @public +export interface PurgeableKeyValueStoreBackend extends storage.KeyValueStoreBackend { + purgeExcept(keys: string[]): Promise; +} + // (No @packageDocumentation comment for this package) ``` diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md index be633dc3675d..c16b7ef579af 100644 --- a/docs/upgrading/upgrading_v4.md +++ b/docs/upgrading/upgrading_v4.md @@ -214,6 +214,14 @@ Configuration instances are immutable — attempting to assign a property throws Previously, environment variables always won. Now `new Configuration({ headless: false })` works even when `CRAWLEE_HEADLESS=true` is set. +### `KeyValueStore.getInput()` and `Configuration.inputKey` moved to the Apify SDK + +Reading the run input is an Apify platform concern, so Crawlee no longer has any notion of it: + +- **`KeyValueStore.getInput()` is removed.** Use `Actor.getInput()` from `apify`, which also handles the platform-assigned input key, encrypted secrets and input schema defaults. +- **`Configuration.inputKey` and the `CRAWLEE_INPUT_KEY` environment variable are removed.** The Apify SDK's `Configuration` defines `inputKey` itself. +- **The default key-value store is purged in full**, `INPUT` included. Sparing a key, and claiming a bare `` / `.json` file as the record `` (see [Out-of-band key-value files](#out-of-band-key-value-files-eg-a-hand-placed-inputjson)), is left to subclasses of `FileSystemStorageBackend` through its two protected hooks, `keyValueStoreAdoptionCandidates` and `purgeKeyValueStore`. The Apify SDK's `ApifyFileSystemStorageBackend` does this for its run input; plain Crawlee treats a hand-placed `INPUT.json` as any other file. + ### Service management moved from `Configuration` to `ServiceLocator` The service management functionality has been extracted from `Configuration` into a new `ServiceLocator` class. @@ -1879,10 +1887,10 @@ Keys are literal. `aaa` and `aaa.json` are two distinct keys, and `FileSystemSto What v4 does instead is *adopt* value files that turn up in a store directory without the `.__metadata__.json` sidecar that marks a record — the Apify CLI's input, a project template, a v3 store directory, a file you dropped in with an editor. Opening the store writes the missing sidecar (the value bytes are never touched), and from then on the file is an ordinary record: read by `getValue`, enumerated by `listKeys`, removed by `deleteValue`. Two rules decide the key: -- In the **default** store, the run-input keys (`INPUT` and the configured `inputKey`) claim a bare `INPUT` or `INPUT.json`. The key is `INPUT` while the file keeps its name, so `listKeys` reports `INPUT`, `getValue('INPUT.json')` is `undefined`, and `getPublicUrl('INPUT')` points at `INPUT.json`. If both files are present, opening the store fails instead of guessing which one is the input. +- A subclass of `FileSystemStorageBackend` may claim specific files under a key of its own by overriding `keyValueStoreAdoptionCandidates`. The Apify SDK's `ApifyFileSystemStorageBackend` claims a bare `INPUT` or `INPUT.json` in the **default** store (and the same for the configured `ACTOR_INPUT_KEY`) as the record `INPUT`: the file keeps its name, `listKeys` reports `INPUT`, `getValue('INPUT.json')` is `undefined`, and `getPublicUrl('INPUT')` points at `INPUT.json`. If both files are present, opening the store fails instead of guessing which one is the input. Plain Crawlee claims nothing, so there a bare `INPUT.json` is just a file named `INPUT.json`. - Every other sidecar-less file becomes a record **keyed by its filename**, in every store: a hand-placed `some-key.json` is the key `some-key.json`, and so is an `INPUT.json` in a store other than the default one. Dotfiles are skipped. -A `.json` file is adopted as `application/json; charset=utf-8` and anything else as `application/octet-stream`; there is no content sniffing. Adopted records are subject to the purge of the default store on start like any other record — only the run-input keys are spared. +A `.json` file is adopted as `application/json; charset=utf-8` and anything else as `application/octet-stream`; there is no content sniffing. Adopted records are subject to the purge of the default store on start like any other record, unless a subclass spares them through `purgeKeyValueStore` — the Apify SDK keeps its run input this way. Beyond the literal keys, three v3 behaviors are gone: @@ -2239,6 +2247,7 @@ The full list of removed exports and members, for ctrl-F purposes. Where a repla - `context.blockResources` and `context.cacheResponses` — no longer attached to the crawling context. The functionality is still available as deprecated functions, accessible both via the `puppeteerUtils` namespace (`puppeteerUtils.blockResources`, `puppeteerUtils.cacheResponses`) and as top-level exports from `@crawlee/puppeteer` (`import { blockResources, cacheResponses } from '@crawlee/puppeteer'`). Unlike the old context helpers, these take an explicit `page` argument — e.g. `await blockResources(page)`. Both are `@deprecated` and will be removed in a future release, so migrate away from them. - `context.closeCookieModals`, `playwrightUtils.closeCookieModals` and `puppeteerUtils.closeCookieModals` — removed along with the optional `idcac-playwright` peer dependency (see [Crawling context no longer includes `closeCookieModals`](#crawling-context-no-longer-includes-closecookiemodals) and the [cookie modals guide](../guides/cookie-modals)) - `Configuration.systemInfoV2` / `CRAWLEE_SYSTEM_INFO_V2` environment variable — the v2 behavior is now the default (see [Available resource detection](#available-resource-detection)) +- `KeyValueStore.getInput()` and `Configuration.inputKey` / `CRAWLEE_INPUT_KEY` — reading the run input moved to the Apify SDK (see [`KeyValueStore.getInput()` and `Configuration.inputKey` moved to the Apify SDK](#keyvaluestoregetinput-and-configurationinputkey-moved-to-the-apify-sdk)) - `Configuration.defaultDatasetId` / `defaultKeyValueStoreId` / `defaultRequestQueueId` and their `CRAWLEE_DEFAULT_*_ID` environment variables — the default storage is addressed by a reserved alias, not by a configurable ID. Open a storage by name if you need a specific one. - `checkAndSerialize` and `chunkBySize` functions (from `@crawlee/core`) — value (de)serialization now lives in the `KeyValueStore` frontend; use `serializeValue` / `parseValue` (see [`maybeStringify` is removed](#maybestringify-is-removed)) - `BASIC_CRAWLER_TIMEOUT_BUFFER_SECS` constant (from `@crawlee/basic`) — was an internal timeout buffer, no longer exported diff --git a/packages/core/src/configuration.ts b/packages/core/src/configuration.ts index 7ac3832ec223..f225b6bebb24 100644 --- a/packages/core/src/configuration.ts +++ b/packages/core/src/configuration.ts @@ -68,8 +68,6 @@ export const crawleeConfigFields = { internalTimeoutMillis: field(coerceNumber.optional(), 'CRAWLEE_INTERNAL_TIMEOUT'), /** @default 1_000 */ systemInfoIntervalMillis: field(coerceNumber.default(1_000)), - /** @default 'INPUT' */ - inputKey: field(z.string().default('INPUT'), 'CRAWLEE_INPUT_KEY'), /** @default true */ headless: field(coerceBoolean.default(true), 'CRAWLEE_HEADLESS'), /** @default false */ @@ -160,7 +158,6 @@ export interface Configuration extends ResolvedConfigValues {} * * Key | Environment Variable | Default Value * ---|---|--- - * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'` * `xvfb` | `CRAWLEE_XVFB` | `false` * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | - * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | - diff --git a/packages/core/src/memory-storage/memory-storage.ts b/packages/core/src/memory-storage/memory-storage.ts index d946ddfece57..f459894ccc29 100644 --- a/packages/core/src/memory-storage/memory-storage.ts +++ b/packages/core/src/memory-storage/memory-storage.ts @@ -192,9 +192,6 @@ export class MemoryStorageBackend implements storage.StorageBackend { // marks them as run-scoped. `'default'` is the exception — it collapses onto the default storage. const isRunScoped = (store: { name?: string }) => store.name === undefined || store.name === 'default'; - const isDefault = (store: { name?: string; cacheKey: string }) => - store.name === 'default' || store.cacheKey === 'default'; - const purgeRunScoped = async ( cache: T[], purgeStore: (store: T) => Promise, @@ -203,10 +200,7 @@ export class MemoryStorageBackend implements storage.StorageBackend { }; await Promise.all([ - // Only the default store holds the run input, so it is the only one that keeps `INPUT`. - purgeRunScoped(this.#keyValueStoreBackendCache, async (store) => - isDefault(store) ? store.purgeExceptInput() : store.purge(), - ), + purgeRunScoped(this.#keyValueStoreBackendCache, async (store) => store.purge()), purgeRunScoped(this.#datasetBackendCache, async (store) => store.purge()), purgeRunScoped(this.#requestQueueBackendCache, async (store) => store.purge()), ]); diff --git a/packages/core/src/memory-storage/resource-clients/key-value-store.ts b/packages/core/src/memory-storage/resource-clients/key-value-store.ts index 72c26b43c0c1..b56c12b1336b 100644 --- a/packages/core/src/memory-storage/resource-clients/key-value-store.ts +++ b/packages/core/src/memory-storage/resource-clients/key-value-store.ts @@ -27,12 +27,6 @@ const inputRecordSchema = z.object({ contentType: z.string().min(1).optional(), }); -/** - * Key under which a run's input is stored in the default key-value store. Matches Crawlee's default - * `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge. - */ -const KEY_VALUE_STORE_INPUT_KEY = 'INPUT'; - export interface KeyValueStoreBackendOptions { name?: string; id?: string; @@ -90,22 +84,6 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValue this.updateTimestamps(true); } - /** - * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the - * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its - * extension variants) when purging the default store. The in-memory key has no extension, so we - * preserve the bare `INPUT` key only. - */ - async purgeExceptInput(): Promise { - for (const key of this.#keyValueEntries.keys()) { - if (key !== KEY_VALUE_STORE_INPUT_KEY) { - this.#keyValueEntries.delete(key); - } - } - - this.updateTimestamps(true); - } - async listKeys(options: storage.KeyValueStoreListKeysOptions = {}): Promise { const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions); diff --git a/packages/core/src/service_locator.ts b/packages/core/src/service_locator.ts index d8eba146dd32..4b24373668cf 100644 --- a/packages/core/src/service_locator.ts +++ b/packages/core/src/service_locator.ts @@ -248,7 +248,6 @@ export class ServiceLocator implements ServiceLocatorInterface { this.#storageBackend = configuration.persistStorage ? new FileSystemStorageBackend({ localDataDirectory: configuration.storageDir, - inputKey: configuration.inputKey, logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }), }) : new MemoryStorageBackend({ diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts index f4f228ffa960..42be0abad495 100644 --- a/packages/core/src/storages/key_value_store.ts +++ b/packages/core/src/storages/key_value_store.ts @@ -64,8 +64,6 @@ const openOptionsSchema = z.strictObject({ * To access the default key-value store directly, you can use the * {@apilink KeyValueStore.getValue} and {@apilink KeyValueStore.setValue} convenience functions. * - * To access the input, you can also use the {@apilink KeyValueStore.getInput} convenience function. - * * `KeyValueStore` stores its data on a local disk. * * If the `CRAWLEE_STORAGE_DIR` environment variable is set, the data is stored in @@ -80,7 +78,7 @@ const openOptionsSchema = z.strictObject({ * * ```javascript * // Get crawler input from the default key-value store. - * const input = await KeyValueStore.getInput(); + * const input = await KeyValueStore.getValue('INPUT'); * // Get some value from the default key-value store. * const otherValue = await KeyValueStore.getValue('my-key'); * @@ -996,31 +994,6 @@ export class KeyValueStore { const store = await this.open(); return store.setValue(key, value, options); } - - /** - * Gets the crawler input value from the default {@apilink KeyValueStore} associated with the current crawler run. - * - * The input is read from the default {@apilink KeyValueStore} under the configured input key - * (`CRAWLEE_INPUT_KEY`, default `INPUT`). - * - * Note that the `getInput()` function does not cache the value read from the key-value store. - * If you need to use the input multiple times in your crawler, - * it is far more efficient to read it once and store it locally. - * - * For more information, see {@apilink KeyValueStore.open} - * and {@apilink KeyValueStore.getValue}. - * - * @returns - * Returns a promise that resolves to an object, string - * or [`Buffer`](https://nodejs.org/api/buffer.html), depending - * on the MIME content type of the record, or `null` - * if the record is missing. - * @ignore - */ - static async getInput(): Promise { - const store = await this.open(); - return store.getValue(store.configuration.inputKey); - } } /** Normalizes a codec-serialized value into the `Buffer | ArrayBuffer` shape raw record reads promise. */ diff --git a/packages/core/src/storages/utils.ts b/packages/core/src/storages/utils.ts index 0abc1affe9b0..7a77fc84d33b 100644 --- a/packages/core/src/storages/utils.ts +++ b/packages/core/src/storages/utils.ts @@ -23,7 +23,7 @@ interface PurgeDefaultStorageOptions { /** * Cleans up the local storage folder (defaults to `./storage`) created when running code locally. * Purging empties the storages that belong to a single run — the default one and every alias-keyed one — - * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched. + * Named storages persist across runs and are not touched. * * Purging of storages is happening automatically when we run our crawler (or when we open some storage * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration} @@ -38,7 +38,7 @@ export async function purgeDefaultStorages(options?: PurgeDefaultStorageOptions) /** * Cleans up the local storage folder (defaults to `./storage`) created when running code locally. * Purging empties the storages that belong to a single run — the default one and every alias-keyed one — - * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched. + * Named storages persist across runs and are not touched. * * Purging of storages is happening automatically when we run our crawler (or when we open some storage * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration} diff --git a/packages/core/test/core/configuration.test.ts b/packages/core/test/core/configuration.test.ts index fb2cce96d050..330be8948dc8 100644 --- a/packages/core/test/core/configuration.test.ts +++ b/packages/core/test/core/configuration.test.ts @@ -35,7 +35,7 @@ describe('Configuration', () => { describe('defaults', () => { it('returns schema defaults when nothing is set', () => { const config = new Configuration(); - expect(config.inputKey).toBe('INPUT'); + expect(config.chromeExecutablePath).toBeUndefined(); expect(config.headless).toBe(true); expect(config.xvfb).toBe(false); expect(config.disableBrowserSandbox).toBe(false); @@ -82,9 +82,9 @@ describe('Configuration', () => { }); it('constructor options override env vars for string fields', () => { - setEnv('CRAWLEE_INPUT_KEY', 'from-env'); - const config = new Configuration({ inputKey: 'from-constructor' }); - expect(config.inputKey).toBe('from-constructor'); + setEnv('CRAWLEE_CHROME_EXECUTABLE_PATH', 'from-env'); + const config = new Configuration({ chromeExecutablePath: 'from-constructor' }); + expect(config.chromeExecutablePath).toBe('from-constructor'); }); it('constructor options override env vars for number fields', () => { @@ -148,9 +148,9 @@ describe('Configuration', () => { setEnv('CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', ''); expect(new Configuration().persistStateIntervalMillis).toBe(60_000); - // Empty string env var falls through to default ('INPUT'), not coerced to '' - setEnv('CRAWLEE_INPUT_KEY', ''); - expect(new Configuration().inputKey).toBe('INPUT'); + // Empty string env var falls through to default ('./storage'), not coerced to '' + setEnv('CRAWLEE_STORAGE_DIR', ''); + expect(new Configuration().storageDir).toBe('./storage'); // Optional fields with no default stay undefined setEnv('CRAWLEE_MEMORY_MBYTES', ''); @@ -185,11 +185,11 @@ describe('Configuration', () => { it('exposes resolved values as instance properties', () => { const config = new Configuration({ headless: false, - inputKey: 'my-input-key', + chromeExecutablePath: '/usr/bin/chromium', persistStateIntervalMillis: 5_000, }); expect(config.headless).toBe(false); - expect(config.inputKey).toBe('my-input-key'); + expect(config.chromeExecutablePath).toBe('/usr/bin/chromium'); expect(config.persistStateIntervalMillis).toBe(5_000); }); }); @@ -246,19 +246,19 @@ describe('Configuration', () => { }); it('loads values from crawlee.json', () => { - writeFileSync(crawleeJsonPath, JSON.stringify({ inputKey: 'from-file' })); + writeFileSync(crawleeJsonPath, JSON.stringify({ chromeExecutablePath: 'from-file' })); fileCreated = true; const config = new Configuration(); - expect(config.inputKey).toBe('from-file'); + expect(config.chromeExecutablePath).toBe('from-file'); }); it('constructor options override crawlee.json', () => { - writeFileSync(crawleeJsonPath, JSON.stringify({ inputKey: 'from-file' })); + writeFileSync(crawleeJsonPath, JSON.stringify({ chromeExecutablePath: 'from-file' })); fileCreated = true; - const config = new Configuration({ inputKey: 'from-constructor' }); - expect(config.inputKey).toBe('from-constructor'); + const config = new Configuration({ chromeExecutablePath: 'from-constructor' }); + expect(config.chromeExecutablePath).toBe('from-constructor'); }); it('env vars override crawlee.json', () => { @@ -283,7 +283,7 @@ describe('Configuration', () => { it('handles missing crawlee.json gracefully', () => { // No file created — should fall through to defaults const config = new Configuration(); - expect(config.inputKey).toBe('INPUT'); + expect(config.storageDir).toBe('./storage'); }); it('handles malformed crawlee.json gracefully', () => { @@ -291,7 +291,7 @@ describe('Configuration', () => { fileCreated = true; const config = new Configuration(); - expect(config.inputKey).toBe('INPUT'); + expect(config.storageDir).toBe('./storage'); }); }); diff --git a/packages/core/test/memory-storage/key-value-store/purge.test.ts b/packages/core/test/memory-storage/key-value-store/purge.test.ts deleted file mode 100644 index 9fa058fcc502..000000000000 --- a/packages/core/test/memory-storage/key-value-store/purge.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { MemoryStorageBackend } from '@crawlee/core'; -import type { KeyValueStoreBackend } from '@crawlee/types'; - -describe('MemoryStorageBackend.purge preserves the default key-value store input', () => { - test('purging keeps INPUT in the default store but removes everything else', async () => { - const storage = new MemoryStorageBackend(); - const store: KeyValueStoreBackend = await storage.createKeyValueStoreBackend({ name: 'default' }); - - await store.setValue({ - key: 'INPUT', - value: JSON.stringify({ hello: 'world' }), - contentType: 'application/json; charset=utf-8', - }); - await store.setValue({ - key: 'some-other-key', - value: JSON.stringify({ foo: 'bar' }), - contentType: 'application/json; charset=utf-8', - }); - - await storage.purge(); - - // INPUT must survive the purge (parity with FileSystemStorageBackend)... - const input = await store.getValue('INPUT'); - expect(input?.value.toString()).toBe(JSON.stringify({ hello: 'world' })); - - // ...while every other record is removed. - expect(await store.getValue('some-other-key')).toBeUndefined(); - const { items: keys } = await store.listKeys(); - expect(keys.map((item) => item.key)).toEqual(['INPUT']); - }); - - test('purging a non-default store removes INPUT as well', async () => { - const storage = new MemoryStorageBackend(); - const store: KeyValueStoreBackend = await storage.createKeyValueStoreBackend({ name: 'not-default' }); - - await store.setValue({ - key: 'INPUT', - value: JSON.stringify({ hello: 'world' }), - contentType: 'application/json; charset=utf-8', - }); - - // `purge` on the storage backend only touches default storages, so a named store keeps its data. - await storage.purge(); - expect((await store.getValue('INPUT'))?.value.toString()).toBe(JSON.stringify({ hello: 'world' })); - - // Purging the store directly clears everything, including INPUT. - await store.purge(); - expect(await store.getValue('INPUT')).toBeUndefined(); - }); -}); diff --git a/packages/fs-storage/src/file-system-storage.ts b/packages/fs-storage/src/file-system-storage.ts index 8fffc2269d2a..40ba21d30250 100644 --- a/packages/fs-storage/src/file-system-storage.ts +++ b/packages/fs-storage/src/file-system-storage.ts @@ -14,7 +14,6 @@ import { RequestQueueBackend } from './resource-clients/request-queue.js'; const fileSystemStorageOptionsSchema = z.object({ localDataDirectory: z.string(), requestQueueAccess: z.enum(['single', 'shared']).default('single'), - inputKey: z.string().min(1).default('INPUT'), logger: schemas.logger.optional(), }); @@ -34,9 +33,6 @@ const DEFAULT_STORAGE_ALIAS = '__default__'; /** The directory the default storage lives in, one level below `datasets` / `key_value_stores` / etc. */ const DEFAULT_STORAGE_DIRECTORY = 'default'; -/** The conventional run-input key, always treated as one alongside the configured `inputKey`. */ -const DEFAULT_INPUT_KEY = 'INPUT'; - /** * Content types declared for adopted files. The native client infers nothing from extensions, and * neither do we beyond this: a `.json` file is JSON, anything else is bytes for the @@ -45,6 +41,18 @@ const DEFAULT_INPUT_KEY = 'INPUT'; const ADOPTED_JSON_CONTENT_TYPE = 'application/json; charset=utf-8'; const ADOPTED_BINARY_CONTENT_TYPE = 'application/octet-stream'; +/** What the key-value store hooks of {@link FileSystemStorageBackend} learn about the store at hand. */ +export interface KeyValueStoreHookOptions { + /** Whether the store is the run's default key-value store. */ + isDefaultStore: boolean; +} + +/** A file-system key-value store backend as seen by {@link FileSystemStorageBackend.purgeKeyValueStore}. */ +export interface PurgeableKeyValueStoreBackend extends storage.KeyValueStoreBackend { + /** Remove every record from the store except the given keys. */ + purgeExcept(keys: string[]): Promise; +} + export interface FileSystemStorageOptions { /** * Path to directory where the data will be saved. @@ -72,19 +80,6 @@ export interface FileSystemStorageOptions { * @default 'single' */ requestQueueAccess?: 'single' | 'shared'; - - /** - * The key the run input is read from — Crawlee's `inputKey` (`CRAWLEE_INPUT_KEY`). - * - * Like the conventional `INPUT`, this key may arrive in the default key-value store as a bare value - * file with no metadata sidecar (e.g. the Apify CLI writes the effective input to `__CLI_INPUT.json` - * and points the run at that key). Such a file is adopted into a record under this key when the - * store is opened, and the key is preserved when the default store is purged — exactly like - * `INPUT`, which is always treated as an input key regardless of this setting. - * - * @default 'INPUT' - */ - inputKey?: string; } /** @@ -102,22 +97,18 @@ export class FileSystemStorageBackend implements storage.StorageBackend { readonly requestQueuesDirectory: string; readonly logger?: CrawleeLogger; readonly requestQueueAccess: 'single' | 'shared'; - /** `INPUT` plus the configured `inputKey`, deduplicated. */ - readonly #inputKeys: string[]; - readonly #keyValueStoreBackendCache: KeyValueStoreBackend[] = []; readonly #datasetBackendCache: DatasetBackend[] = []; readonly #requestQueueBackendCache: RequestQueueBackend[] = []; constructor(options: FileSystemStorageOptions) { - const { logger, requestQueueAccess, inputKey, localDataDirectory } = parseArgument( + const { logger, requestQueueAccess, localDataDirectory } = parseArgument( options, fileSystemStorageOptionsSchema, ); this.logger = logger; this.requestQueueAccess = requestQueueAccess; - this.#inputKeys = [...new Set([DEFAULT_INPUT_KEY, inputKey])]; this.localDataDirectory = localDataDirectory; this.datasetsDirectory = resolve(this.localDataDirectory, 'datasets'); @@ -202,14 +193,13 @@ export class FileSystemStorageBackend implements storage.StorageBackend { this.localDataDirectory, // useTestClock — always real wall-clock outside of native tests. undefined, - this.#adoptionCandidates(cacheKey === DEFAULT_STORAGE_DIRECTORY), + this.keyValueStoreAdoptionCandidates({ isDefaultStore: cacheKey === DEFAULT_STORAGE_DIRECTORY }), ); const newStore = await KeyValueStoreBackend.create({ name: alias ? undefined : (name ?? cacheKey), cacheKey, nativeBackend, logger: this.logger, - inputKeys: this.#inputKeys, }); this.#keyValueStoreBackendCache.push(newStore); @@ -254,28 +244,19 @@ export class FileSystemStorageBackend implements storage.StorageBackend { /** * What the native `open` may turn into records: value files sitting in the store directory with no * metadata sidecar, written out-of-band by a CLI, a project template, a v3 Crawlee or a text editor. + * Each becomes a record keyed by its filename, with the content type derived from the extension alone. * - * Each run-input key claims a bare `` or `.json` first — that is the layout the Apify CLI - * and the templates produce, and the key must end up being `INPUT` rather than `INPUT.json`. Only - * the default store holds the run input, so elsewhere an `INPUT.json` is just a file named - * `INPUT.json`, adopted by the trailing sweep like every other one. + * Subclasses may claim specific files under a key of their own by prepending candidates — the Apify + * SDK adopts a bare `INPUT` or `INPUT.json` in the default store as the record `INPUT` this way. * * Once adopted, the file is an ordinary record: readable, listed, deletable, and — in a run-scoped - * store — purged on start unless its key is a run-input key. + * store — purged on start unless {@link purgeKeyValueStore} spares it. Files adoption skips (dotfiles) + * are purged all the same: the purge sweeps the directory and keeps only the store metadata and the + * spared keys. + * */ - #adoptionCandidates(isDefaultStore: boolean): AdoptionCandidate[] { - const inputCandidates: AdoptionCandidate[] = isDefaultStore - ? this.#inputKeys.map((key) => ({ - key, - files: [ - { filename: key, contentType: ADOPTED_BINARY_CONTENT_TYPE }, - { filename: `${key}.json`, contentType: ADOPTED_JSON_CONTENT_TYPE }, - ], - })) - : []; - + protected keyValueStoreAdoptionCandidates(_options: KeyValueStoreHookOptions): AdoptionCandidate[] { return [ - ...inputCandidates, { files: [ { filename: '*.json', contentType: ADOPTED_JSON_CONTENT_TYPE }, @@ -285,6 +266,17 @@ export class FileSystemStorageBackend implements storage.StorageBackend { ]; } + /** + * Empties one run-scoped key-value store during {@link purge}. Subclasses may spare some keys of the + * default store with {@link PurgeableKeyValueStoreBackend.purgeExcept} — the Apify SDK keeps its run input. + */ + protected async purgeKeyValueStore( + store: PurgeableKeyValueStoreBackend, + _options: KeyValueStoreHookOptions, + ): Promise { + await store.purge(); + } + async storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise { let backends: (KeyValueStoreBackend | DatasetBackend | RequestQueueBackend)[]; let baseDir: string; @@ -383,8 +375,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend { this.#purgeRunScopedStorages( this.keyValueStoresDirectory, async (alias) => this.createKeyValueStoreBackend({ alias }) as Promise, - // Only the default store holds the run input, so it is the only one that keeps `INPUT`. - async (store, isDefault) => (isDefault ? store.purgeExceptInput() : store.purge()), + async (store, isDefault) => this.purgeKeyValueStore(store, { isDefaultStore: isDefault }), ), this.#purgeRunScopedStorages( this.datasetsDirectory, diff --git a/packages/fs-storage/src/index.ts b/packages/fs-storage/src/index.ts index 9219ced98964..72ad14b27dff 100644 --- a/packages/fs-storage/src/index.ts +++ b/packages/fs-storage/src/index.ts @@ -1 +1,2 @@ export * from './file-system-storage.js'; +export type { AdoptionCandidate } from '@crawlee/fs-storage-native'; diff --git a/packages/fs-storage/src/resource-clients/key-value-store.ts b/packages/fs-storage/src/resource-clients/key-value-store.ts index adbe2b87caee..9659f43fc253 100644 --- a/packages/fs-storage/src/resource-clients/key-value-store.ts +++ b/packages/fs-storage/src/resource-clients/key-value-store.ts @@ -35,11 +35,6 @@ export interface KeyValueStoreBackendOptions { cacheKey: string; nativeBackend: NativeFileSystemKeyValueStoreBackend; logger?: CrawleeLogger; - /** - * The run-input keys of this store, deduplicated — `INPUT` plus `FileSystemStorageOptions.inputKey`. - * All that is left of the run-input special case: the keys {@link purgeExceptInput} spares. - */ - inputKeys: string[]; } /** @@ -56,15 +51,11 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV readonly #nativeBackend: NativeFileSystemKeyValueStoreBackend; - /** See {@link KeyValueStoreBackendOptions.inputKeys}. */ - readonly #inputKeys: string[]; - constructor(options: KeyValueStoreBackendOptions) { super(); this.name = options.name; this.cacheKey = options.cacheKey; this.#nativeBackend = options.nativeBackend; - this.#inputKeys = options.inputKeys; } get keyValueStoreDirectory(): string { @@ -90,12 +81,11 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV } /** - * Remove every record from the store except the run input. Used by - * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run - * while preserving the run's input. + * Remove every record from the store except the given keys. For {@link FileSystemStorageBackend} + * subclasses that keep some of the default store's records across the purge on start. */ - async purgeExceptInput(): Promise { - await this.#nativeBackend.purge(this.#inputKeys); + async purgeExcept(keys: string[]): Promise { + await this.#nativeBackend.purge(keys); } async listKeys(options: storage.KeyValueStoreListKeysOptions = {}): Promise { diff --git a/packages/fs-storage/test/default-storage-layout.test.ts b/packages/fs-storage/test/default-storage-layout.test.ts index 46dd50f4c423..05a04dfa6dfa 100644 --- a/packages/fs-storage/test/default-storage-layout.test.ts +++ b/packages/fs-storage/test/default-storage-layout.test.ts @@ -24,32 +24,20 @@ describe('the default storage on disk', () => { expect(await readdir(storage.requestQueuesDirectory)).toEqual(['default']); }); - // The documented way to supply input to a local run: drop a file into the default key-value store - // directory by hand. It only works if that directory is the one the default store actually opens. - test('reads an INPUT.json placed in the default key-value store directory by hand', async () => { + // A hand-placed file only turns up as a record if the `default` directory is the one the default + // store actually opens. + test('adopts a file placed in the default key-value store directory by hand', async () => { const storage = new FileSystemStorageBackend({ localDataDirectory: tmpLocation }); await mkdir(resolve(storage.keyValueStoresDirectory, 'default'), { recursive: true }); await writeFile( - resolve(storage.keyValueStoresDirectory, 'default', 'INPUT.json'), + resolve(storage.keyValueStoresDirectory, 'default', 'hand-placed.json'), JSON.stringify({ hello: 'world' }), ); const defaultStore = await storage.createKeyValueStoreBackend(); - expect((await defaultStore.getValue('INPUT'))?.value.toString()).toBe(JSON.stringify({ hello: 'world' })); - }); - - test('keeps a hand-placed INPUT.json across a purge', async () => { - const storage = new FileSystemStorageBackend({ localDataDirectory: tmpLocation }); - await mkdir(resolve(storage.keyValueStoresDirectory, 'default'), { recursive: true }); - await writeFile( - resolve(storage.keyValueStoresDirectory, 'default', 'INPUT.json'), + expect((await defaultStore.getValue('hand-placed.json'))?.value.toString()).toBe( JSON.stringify({ hello: 'world' }), ); - - await storage.purge(); - - const defaultStore = await storage.createKeyValueStoreBackend(); - expect((await defaultStore.getValue('INPUT'))?.value.toString()).toBe(JSON.stringify({ hello: 'world' })); }); }); diff --git a/packages/fs-storage/test/key-value-store/adoption.test.ts b/packages/fs-storage/test/key-value-store/adoption.test.ts index a7996774fb33..61ee5de0638a 100644 --- a/packages/fs-storage/test/key-value-store/adoption.test.ts +++ b/packages/fs-storage/test/key-value-store/adoption.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto'; import { mkdir, readdir, rm, writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; +import type { AdoptionCandidate, KeyValueStoreHookOptions, PurgeableKeyValueStoreBackend } from '@crawlee/fs-storage'; import { FileSystemStorageBackend } from '@crawlee/fs-storage'; import type { KeyValueStoreRecord } from '@crawlee/types'; @@ -11,20 +12,21 @@ import type { KeyValueStoreRecord } from '@crawlee/types'; // candidates when it opens a store, so the native client writes the missing sidecar once and // everything afterwards — reads, listings, deletes, purge — deals in ordinary records. // -// The run-input keys claim a bare `` or `.json` in the default store; every other file is -// adopted under its own filename as the key. Content types come from the extension alone: `.json` is -// JSON, anything else is bytes for the `KeyValueStore` frontend to make sense of. +// Every file is adopted under its own filename as the key. Content types come from the extension +// alone: `.json` is JSON, anything else is bytes for the `KeyValueStore` frontend to make sense of. +// Claiming a file under a different key, and sparing it on purge, is left to subclasses (the Apify +// SDK does both for its run input). const payload = JSON.stringify({ hello: 'from disk' }); /** A fresh backend over `directory`, seeded with sidecar-less files in one key-value store. */ -async function seedStore( +async function seedStore( directory: string, store: string, files: Record, - options: { inputKey?: string } = {}, -): Promise { - const storage = new FileSystemStorageBackend({ localDataDirectory: directory, ...options }); + Backend: new (options: { localDataDirectory: string }) => T = FileSystemStorageBackend as never, +): Promise { + const storage = new Backend({ localDataDirectory: directory }); const storeDirectory = resolve(storage.keyValueStoresDirectory, store); await mkdir(storeDirectory, { recursive: true }); for (const [file, content] of Object.entries(files)) { @@ -33,115 +35,7 @@ async function seedStore( return storage; } -describe('a sidecar-less run-input file in the default store', () => { - const tmpLocation = resolve(import.meta.dirname, './tmp/adoption-input'); - - afterEach(async () => { - await rm(tmpLocation, { force: true, recursive: true }); - }); - - test('is a record under the input key, not under its filename', async () => { - const storage = await seedStore(tmpLocation, 'default', { 'INPUT.json': payload }); - const store = await storage.createKeyValueStoreBackend(); - - expect(await store.getValue('INPUT')).toStrictEqual({ - key: 'INPUT', - value: Buffer.from(payload), - contentType: 'application/json; charset=utf-8', - }); - expect((await store.listKeys()).items.map((item) => item.key)).toEqual(['INPUT']); - expect(await store.recordExists('INPUT')).toBe(true); - expect(await store.getPublicUrl('INPUT')).toMatch(/\/INPUT\.json$/); - - // The extension is the file's, not the key's. - expect(await store.getValue('INPUT.json')).toBeUndefined(); - expect(await store.recordExists('INPUT.json')).toBe(false); - }); - - test('is deleted by deleteValue instead of resurrecting the key', async () => { - const storage = await seedStore(tmpLocation, 'default', { 'INPUT.json': payload }); - const store = await storage.createKeyValueStoreBackend(); - - await store.deleteValue('INPUT'); - - expect(await store.getValue('INPUT')).toBeUndefined(); - expect(await readdir(resolve(storage.keyValueStoresDirectory, 'default'))).not.toContain('INPUT.json'); - }); - - test('reads as bytes when it has no extension', async () => { - const storage = await seedStore(tmpLocation, 'default', { INPUT: payload }); - const store = await storage.createKeyValueStoreBackend(); - - // No sniffing: the extension is the only thing that makes a file JSON, so an extensionless - // one is bytes. Turning those into a parsed input is the caller's job. - expect(await store.getValue('INPUT')).toStrictEqual({ - key: 'INPUT', - value: Buffer.from(payload), - contentType: 'application/octet-stream', - }); - }); - - test('is adopted verbatim even when it is malformed JSON', async () => { - const storage = await seedStore(tmpLocation, 'default', { 'INPUT.json': '{' }); - const store = await storage.createKeyValueStoreBackend(); - - // The backend is a byte transport — parsing, and any error from it, belongs to the frontend. - expect(await store.getValue('INPUT')).toStrictEqual({ - key: 'INPUT', - value: Buffer.from('{'), - contentType: 'application/json; charset=utf-8', - }); - }); - - test('fails the open when both candidate files are present', async () => { - const storage = await seedStore(tmpLocation, 'default', { INPUT: 'bytes', 'INPUT.json': payload }); - - // Picking one would silently ignore the other, and there is no way to guess which one the - // user means. - await expect(storage.createKeyValueStoreBackend()).rejects.toThrow(/Multiple candidate files for key 'INPUT'/); - }); - - test('is left alone when the input key already has a record', async () => { - const storage = await seedStore(tmpLocation, 'default', {}); - const store = await storage.createKeyValueStoreBackend(); - await store.setValue({ key: 'INPUT', value: 'tracked', contentType: 'text/plain; charset=utf-8' }); - await writeFile(resolve(storage.keyValueStoresDirectory, 'default', 'INPUT.json'), payload); - - // Reopening must not rebind the key: a stray file is not allowed to take over a record the - // run wrote itself, and adopting it under its own filename would make `INPUT.json` a second - // key for what the user thinks is the input. - const reopened = await new FileSystemStorageBackend({ - localDataDirectory: tmpLocation, - }).createKeyValueStoreBackend(); - - expect((await reopened.getValue('INPUT'))?.value.toString()).toBe('tracked'); - expect((await reopened.listKeys()).items.map((item) => item.key)).toEqual(['INPUT']); - }); - - test('is adopted under the configured input key', async () => { - const inputKey = '__CLI_INPUT'; - const storage = await seedStore(tmpLocation, 'default', { [`${inputKey}.json`]: payload }, { inputKey }); - const store = await storage.createKeyValueStoreBackend(); - - expect(await store.getValue(inputKey)).toStrictEqual({ - key: inputKey, - value: Buffer.from(payload), - contentType: 'application/json; charset=utf-8', - }); - expect((await store.listKeys()).items.map((item) => item.key)).toEqual([inputKey]); - }); - - test('is adopted under its filename when a different input key is configured', async () => { - const storage = await seedStore(tmpLocation, 'default', { '__CLI_INPUT.json': payload }); - const store = await storage.createKeyValueStoreBackend(); - - // Not this run's input, so it is an ordinary file: a record named after itself. - expect(await store.getValue('__CLI_INPUT')).toBeUndefined(); - expect((await store.getValue('__CLI_INPUT.json'))?.value.toString()).toBe(payload); - }); -}); - -describe('sidecar-less files outside the run input', () => { +describe('sidecar-less files', () => { const tmpLocation = resolve(import.meta.dirname, './tmp/adoption-sweep'); afterEach(async () => { @@ -167,21 +61,39 @@ describe('sidecar-less files outside the run input', () => { expect(await store.getValue('.hidden')).toBeUndefined(); }); - test('include an INPUT.json in a store that is not the default one', async () => { - const storage = await seedStore(tmpLocation, 'named-store', { 'INPUT.json': payload }); - const store = await storage.createKeyValueStoreBackend({ name: 'named-store' }); + test('include an INPUT.json in the default store, keyed by its filename like any other', async () => { + const storage = await seedStore(tmpLocation, 'default', { 'INPUT.json': payload }); + const store = await storage.createKeyValueStoreBackend(); - // Only the default store holds the run input, so here `INPUT.json` is just a file that - // happens to be called that. + // Crawlee has no notion of a run input, so the file is a record named after itself. expect(await store.getValue('INPUT')).toBeUndefined(); expect((await store.getValue('INPUT.json'))?.value.toString()).toBe(payload); }); - test('are adopted in the default store too', async () => { - const storage = await seedStore(tmpLocation, 'default', { 'leftover.json': payload }); + test('are adopted verbatim even when they are malformed JSON', async () => { + const storage = await seedStore(tmpLocation, 'default', { 'broken.json': '{' }); const store = await storage.createKeyValueStoreBackend(); - expect((await store.getValue('leftover.json'))?.value.toString()).toBe(payload); + // The backend is a byte transport — parsing, and any error from it, belongs to the frontend. + expect(await store.getValue('broken.json')).toStrictEqual({ + key: 'broken.json', + value: Buffer.from('{'), + contentType: 'application/json; charset=utf-8', + }); + }); + + test('are purged from a run-scoped store on start, adopted or not', async () => { + const storage = await seedStore(tmpLocation, 'default', { + 'INPUT.json': payload, + 'leftover.json': '{}', + '.hidden': 'never adopted', + }); + + await storage.purge(); + + // The purge sweeps the directory, so the dotfile adoption skipped goes too; only the store + // metadata is left. + expect(await readdir(resolve(storage.keyValueStoresDirectory, 'default'))).toEqual(['__metadata__.json']); }); }); @@ -223,24 +135,79 @@ describe('a hand-written store directory', () => { }); }); -describe('purging a store with adopted records', () => { - const tmpLocation = resolve(import.meta.dirname, './tmp/adoption-purge'); +// What the Apify SDK does for its run input: claim a bare `INPUT` / `INPUT.json` in the default store +// as the record `INPUT`, and keep that record when the store is purged on start. +class InputAwareBackend extends FileSystemStorageBackend { + protected override keyValueStoreAdoptionCandidates(options: KeyValueStoreHookOptions): AdoptionCandidate[] { + const inputCandidate: AdoptionCandidate = { + key: 'INPUT', + files: [ + { filename: 'INPUT', contentType: 'application/octet-stream' }, + { filename: 'INPUT.json', contentType: 'application/json; charset=utf-8' }, + ], + }; + + return [...(options.isDefaultStore ? [inputCandidate] : []), ...super.keyValueStoreAdoptionCandidates(options)]; + } + + protected override async purgeKeyValueStore( + store: PurgeableKeyValueStoreBackend, + options: KeyValueStoreHookOptions, + ): Promise { + await (options.isDefaultStore ? store.purgeExcept(['INPUT']) : store.purge()); + } +} + +describe('a subclass claiming keys through the hooks', () => { + const tmpLocation = resolve(import.meta.dirname, './tmp/adoption-hooks'); afterEach(async () => { await rm(tmpLocation, { force: true, recursive: true }); }); - test('keeps the run input and drops everything else', async () => { - const inputKey = '__CLI_INPUT'; + test('adopts the file under the claimed key, not under its filename', async () => { + const storage = await seedStore(tmpLocation, 'default', { 'INPUT.json': payload }, InputAwareBackend); + const store = await storage.createKeyValueStoreBackend(); + + expect(await store.getValue('INPUT')).toStrictEqual({ + key: 'INPUT', + value: Buffer.from(payload), + contentType: 'application/json; charset=utf-8', + }); + expect((await store.listKeys()).items.map((item) => item.key)).toEqual(['INPUT']); + expect(await store.getPublicUrl('INPUT')).toMatch(/\/INPUT\.json$/); + + // The extension is the file's, not the key's. + expect(await store.getValue('INPUT.json')).toBeUndefined(); + }); + + test('fails the open when both candidate files are present', async () => { + const storage = await seedStore( + tmpLocation, + 'default', + { INPUT: 'bytes', 'INPUT.json': payload }, + InputAwareBackend, + ); + + // Picking one would silently ignore the other, and there is no way to guess which one the + // user means. + await expect(storage.createKeyValueStoreBackend()).rejects.toThrow(/Multiple candidate files for key 'INPUT'/); + }); + + test('only receives isDefaultStore for the default store', async () => { + const storage = await seedStore(tmpLocation, 'named-store', { 'INPUT.json': payload }, InputAwareBackend); + const store = await storage.createKeyValueStoreBackend({ name: 'named-store' }); + + expect(await store.getValue('INPUT')).toBeUndefined(); + expect((await store.getValue('INPUT.json'))?.value.toString()).toBe(payload); + }); + + test('spares the claimed key on purge and drops everything else', async () => { const storage = await seedStore( tmpLocation, 'default', - { - 'INPUT.json': payload, - [`${inputKey}.json`]: JSON.stringify({ hello: 'from the cli' }), - 'leftover.json': JSON.stringify({ leftover: true }), - }, - { inputKey }, + { 'INPUT.json': payload, 'leftover.json': '{}' }, + InputAwareBackend, ); // Purge-on-start opens the store, so adoption runs first and the input survives as a record — @@ -248,13 +215,13 @@ describe('purging a store with adopted records', () => { await storage.purge(); const store = await storage.createKeyValueStoreBackend(); - expect((await store.listKeys()).items.map((item) => item.key)).toEqual(['INPUT', inputKey]); + expect((await store.listKeys()).items.map((item) => item.key)).toEqual(['INPUT']); expect((await store.getValue('INPUT'))?.value.toString()).toBe(payload); expect(await readdir(resolve(storage.keyValueStoresDirectory, 'default'))).not.toContain('leftover.json'); }); - test('drops the input of a non-default store', async () => { - const storage = await seedStore(tmpLocation, 'other', { 'INPUT.json': payload }); + test('purges a non-default store in full', async () => { + const storage = await seedStore(tmpLocation, 'other', { 'INPUT.json': payload }, InputAwareBackend); await storage.createKeyValueStoreBackend({ alias: 'other' }); await storage.purge(); diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts index 49c337fa59f5..e098b7329729 100644 --- a/packages/types/src/storages.ts +++ b/packages/types/src/storages.ts @@ -478,7 +478,7 @@ export interface StorageBackend { /** * Empty the run-scoped storages — the default one and every alias-keyed one, including any left - * behind by a previous run. Named storages persist across runs, as does the default store's `INPUT`. + * behind by a previous run. Named storages persist across runs. */ purge?(): Promise; teardown?(): Promise; diff --git a/test/core/storages/storage_purge.test.ts b/test/core/storages/storage_purge.test.ts index 991d4e5a2caf..1dbe6f70ebaa 100644 --- a/test/core/storages/storage_purge.test.ts +++ b/test/core/storages/storage_purge.test.ts @@ -1,6 +1,7 @@ import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; +import type { KeyValueStoreHookOptions, PurgeableKeyValueStoreBackend } from '@crawlee/fs-storage'; import { FileSystemStorageBackend } from '@crawlee/fs-storage'; import { MemoryStorageBackend, RequestQueue, purgeDefaultStorages, serviceLocator } from '@crawlee/core'; import type { KeyValueStoreBackend, RequestQueueBackend, StorageBackend } from '@crawlee/types'; @@ -68,9 +69,8 @@ describe.each([ await expect(namedDataset.getData()).resolves.toMatchObject({ items: [{ from: 'named' }] }); }); - // The run input lives in the default key-value store, so that one store keeps its `INPUT` key. - // An alias-keyed store is just another run-scoped storage — nothing there is the run input. - test('keeps INPUT in the default key-value store but not in an alias-keyed one', async () => { + // Crawlee has no notion of a run input: nothing in the default key-value store is special. + test('empties the default key-value store like any other run-scoped one', async () => { const backend = createBackend(); const defaultStore = await backend.createKeyValueStoreBackend(); @@ -81,11 +81,37 @@ describe.each([ await backend.purge!(); - expect(await readInput(defaultStore)).toBe(input.value); + expect(await readInput(defaultStore)).toBeUndefined(); expect(await readInput(aliasStore)).toBeUndefined(); }); }); +// The Apify SDK spares its run input through the `purgeKeyValueStore` hook, in the default store only. +// An alias-keyed store is just another run-scoped storage — nothing there is the run input. +test('a FileSystemStorageBackend subclass can spare keys of the default key-value store on purge', async () => { + class InputAwareBackend extends FileSystemStorageBackend { + protected override async purgeKeyValueStore( + store: PurgeableKeyValueStoreBackend, + options: KeyValueStoreHookOptions, + ): Promise { + await (options.isDefaultStore ? store.purgeExcept(['INPUT']) : store.purge()); + } + } + + const backend = new InputAwareBackend({ localDataDirectory: temporaryDirectory() }); + + const defaultStore = await backend.createKeyValueStoreBackend(); + const aliasStore = await backend.createKeyValueStoreBackend({ alias: 'run-scoped' }); + + await defaultStore.setValue(input); + await aliasStore.setValue(input); + + await backend.purge(); + + expect(await readInput(defaultStore)).toBe(input.value); + expect(await readInput(aliasStore)).toBeUndefined(); +}); + // The file system backend has to find leftovers on disk, since a fresh process starts with an empty // backend cache and knows nothing about the storages the previous run opened. describe('FileSystemStorageBackend.purge over a pre-existing storage directory', () => { diff --git a/test/e2e/input-json5/actor/main.js b/test/e2e/input-json5/actor/main.js index b554cf7c47dc..a485ec371a1b 100644 --- a/test/e2e/input-json5/actor/main.js +++ b/test/e2e/input-json5/actor/main.js @@ -1,4 +1,4 @@ -import { Actor, Dataset, KeyValueStore, log } from 'apify'; +import { Actor, Dataset, log } from 'apify'; const mainOptions = { exit: Actor.isAtHome(), @@ -9,7 +9,7 @@ const mainOptions = { }; await Actor.main(async () => { - const a = await KeyValueStore.getInput(); + const a = await Actor.getInput(); log.info('val', a); diff --git a/test/e2e/request-queue-with-concurrency/actor/main.js b/test/e2e/request-queue-with-concurrency/actor/main.js index 023d0e8a98d4..336e7e0cf765 100644 --- a/test/e2e/request-queue-with-concurrency/actor/main.js +++ b/test/e2e/request-queue-with-concurrency/actor/main.js @@ -17,7 +17,7 @@ const mainOptions = { }; await Actor.main(async () => { - const input = await Actor.getInputOrThrow(); + const input = await Actor.getInput(); log.info('Starting the crawler', input);