From 392dcead139a7b92bdd094d2264dc330f7cb0555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Fri, 13 Mar 2026 18:36:47 +0100 Subject: [PATCH 01/34] feat: redesign Configuration class for v4 Refactor the SDK Configuration class to match the new crawlee core Configuration redesign: - Subclass core Configuration using `protected static override fields` - Direct property access (`config.token`) instead of `config.get('token')` - Immutable: values set via constructor, no `set()` method - Priority: constructor options > env vars > schema defaults - isAtHome conditional defaults moved into field definitions - Use serviceLocator instead of config.useStorageClient/getEventManager - Import z, coerceNumber, coerceBoolean from @crawlee/core (no direct zod dep) - Update all .get()/.set() call sites in actor.ts, charging.ts, etc. - Update tests to use property access Depends on crawlee PR: apify/crawlee#3474 Co-Authored-By: Claude Opus 4.6 --- packages/apify/src/actor.ts | 71 ++-- packages/apify/src/charging.ts | 13 +- packages/apify/src/configuration.ts | 395 ++++++++++--------- packages/apify/src/key_value_store.ts | 4 +- packages/apify/src/platform_event_manager.ts | 2 +- packages/apify/src/proxy_configuration.ts | 13 +- test/apify/actor.test.ts | 17 +- test/apify/events.test.ts | 2 +- 8 files changed, 268 insertions(+), 249 deletions(-) diff --git a/packages/apify/src/actor.ts b/packages/apify/src/actor.ts index 627cacaefb..c972ba8599 100644 --- a/packages/apify/src/actor.ts +++ b/packages/apify/src/actor.ts @@ -9,11 +9,11 @@ import type { UseStateOptions, } from '@crawlee/core'; import { - Configuration as CoreConfiguration, Dataset, EventType, purgeDefaultStorages, RequestQueue, + serviceLocator, StorageManager, } from '@crawlee/core'; import type { @@ -490,19 +490,19 @@ export class Actor { printOutdatedSdkWarning(); // reset global config instance to respect APIFY_ prefixed env vars - CoreConfiguration.globalConfig = Configuration.getGlobalConfig(); + serviceLocator.setConfiguration(Configuration.getGlobalConfig()); if (this.isAtHome()) { - this.config.set('availableMemoryRatio', 1); - this.config.set('disableBrowserSandbox', true); // for browser launcher, adds `--no-sandbox` to args - this.config.useStorageClient(this.apifyClient); - this.config.useEventManager(this.eventManager); + // availableMemoryRatio and disableBrowserSandbox are now set via + // conditional defaults in the Configuration constructor (isAtHome check) + serviceLocator.setStorageClient(this.apifyClient); + serviceLocator.setEventManager(this.eventManager); } else if (options.storage) { - this.config.useStorageClient(options.storage); + serviceLocator.setStorageClient(options.storage); } // Init the event manager the config uses - await this.config.getEventManager().init(); + await serviceLocator.getEventManager().init(); log.debug(`Events initialized`); await purgeDefaultStorages({ @@ -534,8 +534,8 @@ export class Actor { options.exit ??= true; options.exitCode ??= EXIT_CODES.SUCCESS; options.timeoutSecs ??= 30; - const client = this.config.getStorageClient(); - const events = this.config.getEventManager(); + const client = serviceLocator.getStorageClient(); + const events = serviceLocator.getEventManager(); // Close the event manager and emit the final PERSIST_STATE event await events.close(); @@ -601,14 +601,14 @@ export class Actor { * @ignore */ on(event: EventTypeName, listener: (...args: any[]) => any): void { - this.config.getEventManager().on(event, listener); + serviceLocator.getEventManager().on(event, listener); } /** * @ignore */ off(event: EventTypeName, listener?: (...args: any[]) => any): void { - this.config.getEventManager().off(event, listener); + serviceLocator.getEventManager().off(event, listener); } /** @@ -776,12 +776,10 @@ export class Actor { } const { - customAfterSleepMillis = this.config.get( - 'metamorphAfterSleepMillis', - ), + customAfterSleepMillis = this.config.metamorphAfterSleepMillis, ...metamorphOpts } = options; - const runId = this.config.get('actorRunId')!; + const runId = this.config.actorRunId!; await this.apifyClient .run(runId) .metamorph(targetActorId, input, metamorphOpts); @@ -815,27 +813,24 @@ export class Actor { this.isRebooting = true; // Waiting for all the listeners to finish, as `.reboot()` kills the container. + const eventManager = serviceLocator.getEventManager(); await Promise.all([ // `persistState` for individual RequestLists, RequestQueue... instances to be persisted - ...this.config - .getEventManager() + ...eventManager .listeners(EventType.PERSIST_STATE) - .map(async (x) => x()), + .map(async (x: (...args: any[]) => any) => x()), // `migrating` to pause Apify crawlers - ...this.config - .getEventManager() + ...eventManager .listeners(EventType.MIGRATING) - .map(async (x) => x()), + .map(async (x: (...args: any[]) => any) => x()), ]); - const runId = this.config.get('actorRunId')!; + const runId = this.config.actorRunId!; await this.apifyClient.run(runId).reboot(); // Wait some time for container to be stopped. const { - customAfterSleepMillis = this.config.get( - 'metamorphAfterSleepMillis', - ), + customAfterSleepMillis = this.config.metamorphAfterSleepMillis, } = options; await sleep(customAfterSleepMillis); } @@ -873,7 +868,7 @@ export class Actor { return undefined; } - const runId = this.config.get('actorRunId')!; + const runId = this.config.actorRunId!; if (!runId) { throw new Error( `Environment variable ${ACTOR_ENV_VARS.RUN_ID} is not set!`, @@ -924,7 +919,7 @@ export class Actor { break; } - const client = this.config.getStorageClient(); + const client = serviceLocator.getStorageClient(); // just to be sure, this should be fast await addTimeoutToPromise( @@ -937,7 +932,7 @@ export class Actor { 'Setting status message timed out after 1s', ).catch((e) => log.warning(e.message)); - const runId = this.config.get('actorRunId')!; + const runId = this.config.actorRunId!; if (runId) { // just to be sure, this should be fast @@ -1213,13 +1208,9 @@ export class Actor { async getInput(): Promise { this._ensureActorInit('getInput'); - const inputSecretsPrivateKeyFile = this.config.get( - 'inputSecretsPrivateKeyFile', - ); - const inputSecretsPrivateKeyPassphrase = this.config.get( - 'inputSecretsPrivateKeyPassphrase', - ); - const input = await this.getValue(this.config.get('inputKey')); + const {inputSecretsPrivateKeyFile} = this.config; + const {inputSecretsPrivateKeyPassphrase} = this.config; + const input = await this.getValue(this.config.inputKey); if ( ow.isValid(input, ow.object.nonEmpty) && inputSecretsPrivateKeyFile && @@ -1476,18 +1467,14 @@ export class Actor { * @ignore */ newClient(options: ApifyClientOptions = {}): ApifyClient { - const { storageDir, ...storageClientOptions } = this.config.get( - 'storageClientOptions', - ) as Dictionary; const { apifyVersion, crawleeVersion } = getSystemInfo(); return new ApifyClient({ - baseUrl: this.config.get('apiBaseUrl'), - token: this.config.get('token'), + baseUrl: this.config.apiBaseUrl, + token: this.config.token, userAgentSuffix: [ `SDK/${apifyVersion}`, `Crawlee/${crawleeVersion}`, ], - ...storageClientOptions, ...options, // allow overriding the instance configuration }); } diff --git a/packages/apify/src/charging.ts b/packages/apify/src/charging.ts index 5ac7777846..e6100f5652 100644 --- a/packages/apify/src/charging.ts +++ b/packages/apify/src/charging.ts @@ -87,12 +87,11 @@ export class ChargingManager { private apifyClient: ApifyClient; constructor(configuration: Configuration, apifyClient: ApifyClient) { - this.maxTotalChargeUsd = - configuration.get('maxTotalChargeUsd') || Infinity; // convert `0` to `Infinity` in case the value is an empty string - this.isAtHome = configuration.get('isAtHome'); - this.actorRunId = configuration.get('actorRunId'); - this.purgeChargingLogDataset = configuration.get('purgeOnStart'); - this.useChargingLogDataset = configuration.get('useChargingLogDataset'); + this.maxTotalChargeUsd = configuration.maxTotalChargeUsd || Infinity; // convert `0` to `Infinity` in case the value is an empty string + this.isAtHome = !!configuration.isAtHome; + this.actorRunId = configuration.actorRunId; + this.purgeChargingLogDataset = configuration.purgeOnStart; + this.useChargingLogDataset = configuration.useChargingLogDataset; if (this.useChargingLogDataset && this.isAtHome) { throw new Error( @@ -100,7 +99,7 @@ export class ChargingManager { ); } - if (configuration.get('testPayPerEvent')) { + if (configuration.testPayPerEvent) { if (this.isAtHome) { throw new Error( 'Using the ACTOR_TEST_PAY_PER_EVENT environment variable is only supported in a local development environment', diff --git a/packages/apify/src/configuration.ts b/packages/apify/src/configuration.ts index b8dfacd42c..932da5b94e 100644 --- a/packages/apify/src/configuration.ts +++ b/packages/apify/src/configuration.ts @@ -1,7 +1,16 @@ -import type { ConfigurationOptions as CoreConfigurationOptions } from '@crawlee/core'; -import { Configuration as CoreConfiguration } from '@crawlee/core'; +/* eslint-disable no-use-before-define */ +import { AsyncLocalStorage } from 'node:async_hooks'; + +import type { ConfigField, FieldsInput, FieldsOutput } from '@crawlee/core'; +import { + coerceBoolean, + coerceNumber, + Configuration as CoreConfiguration, + crawleeConfigFields, + field, + z, +} from '@crawlee/core'; -import type { META_ORIGINS } from '@apify/consts'; import { ACTOR_ENV_VARS, APIFY_ENV_VARS, @@ -9,37 +18,189 @@ import { LOCAL_APIFY_ENV_VARS, } from '@apify/consts'; -export interface ConfigurationOptions extends CoreConfigurationOptions { - metamorphAfterSleepMillis?: number; - actorEventsWsUrl?: string; - token?: string; - actorId?: string; - actorRunId?: string; - actorTaskId?: string; - apiBaseUrl?: string; - // apiBaseUrl is the internal API URL, accessible only within the platform(private network), - // while apiPublicBaseUrl is the public API URL, available externally(through internet). - apiPublicBaseUrl?: string; - containerPort?: number; - containerUrl?: string; - proxyHostname?: string; - proxyPassword?: string; - proxyPort?: number; - proxyStatusUrl?: string; - /** - * @deprecated use `containerPort` instead - */ - standbyPort?: number; - standbyUrl?: string; - isAtHome?: boolean; - userId?: string; - inputSecretsPrivateKeyPassphrase?: string; - inputSecretsPrivateKeyFile?: string; - maxTotalChargeUsd?: number; - metaOrigin?: (typeof META_ORIGINS)[keyof typeof META_ORIGINS]; - testPayPerEvent?: boolean; - useChargingLogDataset?: boolean; -} +// --- isAtHome check (simple env var presence) --- +const isAtHome = !!process.env[APIFY_ENV_VARS.IS_AT_HOME]; + +// --- Apify config field definitions --- + +export const apifyConfigFields = { + // Inherit all crawlee fields, overriding env vars where the SDK supports ACTOR_/APIFY_ aliases + ...crawleeConfigFields, + + // Override crawlee fields with ACTOR_/APIFY_ env var aliases + defaultDatasetId: field( + z + .string() + .default(LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_DATASET_ID]), + [ + ACTOR_ENV_VARS.DEFAULT_DATASET_ID, + 'APIFY_DEFAULT_DATASET_ID', + 'CRAWLEE_DEFAULT_DATASET_ID', + ], + ), + defaultKeyValueStoreId: field( + z + .string() + .default( + LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID], + ), + [ + ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, + 'APIFY_DEFAULT_KEY_VALUE_STORE_ID', + 'CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID', + ], + ), + defaultRequestQueueId: field( + z + .string() + .default( + LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID], + ), + [ + ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, + 'APIFY_DEFAULT_REQUEST_QUEUE_ID', + 'CRAWLEE_DEFAULT_REQUEST_QUEUE_ID', + ], + ), + inputKey: field(z.string().default('INPUT'), [ + ACTOR_ENV_VARS.INPUT_KEY, + 'APIFY_INPUT_KEY', + 'CRAWLEE_INPUT_KEY', + ]), + memoryMbytes: field(coerceNumber.optional(), [ + ACTOR_ENV_VARS.MEMORY_MBYTES, + 'APIFY_MEMORY_MBYTES', + 'CRAWLEE_MEMORY_MBYTES', + ]), + availableMemoryRatio: field(coerceNumber.default(isAtHome ? 1 : 0.25), [ + 'CRAWLEE_AVAILABLE_MEMORY_RATIO', + 'APIFY_AVAILABLE_MEMORY_RATIO', + ]), + disableBrowserSandbox: field( + isAtHome ? coerceBoolean.default(true) : coerceBoolean.optional(), + ['CRAWLEE_DISABLE_BROWSER_SANDBOX', 'APIFY_DISABLE_BROWSER_SANDBOX'], + ), + persistStateIntervalMillis: field(coerceNumber.default(60_000), [ + 'CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', + 'APIFY_PERSIST_STATE_INTERVAL_MILLIS', + 'APIFY_TEST_PERSIST_INTERVAL_MILLIS', + ]), + headless: field(coerceBoolean.default(true), [ + 'CRAWLEE_HEADLESS', + 'APIFY_HEADLESS', + ]), + xvfb: field(coerceBoolean.default(false), ['CRAWLEE_XVFB', 'APIFY_XVFB']), + chromeExecutablePath: field(z.string().optional(), [ + 'CRAWLEE_CHROME_EXECUTABLE_PATH', + 'APIFY_CHROME_EXECUTABLE_PATH', + ]), + defaultBrowserPath: field(z.string().optional(), [ + 'CRAWLEE_DEFAULT_BROWSER_PATH', + 'APIFY_DEFAULT_BROWSER_PATH', + ]), + purgeOnStart: field(coerceBoolean.default(true), [ + 'CRAWLEE_PURGE_ON_START', + 'APIFY_PURGE_ON_START', + ]), + + // Apify-specific fields + metamorphAfterSleepMillis: field( + coerceNumber.default(300_000), + 'APIFY_METAMORPH_AFTER_SLEEP_MILLIS', + ), + actorEventsWsUrl: field(z.string().optional(), [ + ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL, + 'APIFY_ACTOR_EVENTS_WS_URL', + ]), + token: field(z.string().optional(), 'APIFY_TOKEN'), + actorId: field(z.string().optional(), [ + ACTOR_ENV_VARS.ID, + 'APIFY_ACTOR_ID', + ]), + actorRunId: field(z.string().optional(), [ + ACTOR_ENV_VARS.RUN_ID, + 'APIFY_ACTOR_RUN_ID', + ]), + actorTaskId: field(z.string().optional(), [ + ACTOR_ENV_VARS.TASK_ID, + 'APIFY_ACTOR_TASK_ID', + ]), + apiBaseUrl: field( + z.string().default('https://api.apify.com'), + 'APIFY_API_BASE_URL', + ), + apiPublicBaseUrl: field( + z.string().default('https://api.apify.com'), + 'APIFY_API_PUBLIC_BASE_URL', + ), + containerPort: field( + coerceNumber.default( + +LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.WEB_SERVER_PORT], + ), + [ACTOR_ENV_VARS.WEB_SERVER_PORT, 'APIFY_CONTAINER_PORT'], + ), + containerUrl: field( + z.string().default(LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.WEB_SERVER_URL]), + [ACTOR_ENV_VARS.WEB_SERVER_URL, 'APIFY_CONTAINER_URL'], + ), + proxyHostname: field( + z.string().default(LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_HOSTNAME]), + 'APIFY_PROXY_HOSTNAME', + ), + proxyPassword: field(z.string().optional(), 'APIFY_PROXY_PASSWORD'), + proxyPort: field( + coerceNumber.default(+LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_PORT]), + 'APIFY_PROXY_PORT', + ), + proxyStatusUrl: field( + z.string().default('http://proxy.apify.com'), + 'APIFY_PROXY_STATUS_URL', + ), + /** @deprecated use `containerPort` instead */ + standbyPort: field( + coerceNumber.default( + +LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.STANDBY_PORT], + ), + ACTOR_ENV_VARS.STANDBY_PORT, + ), + standbyUrl: field(z.string().optional(), ACTOR_ENV_VARS.STANDBY_URL), + isAtHome: field(coerceBoolean.optional(), 'APIFY_IS_AT_HOME'), + userId: field(z.string().optional(), 'APIFY_USER_ID'), + inputSecretsPrivateKeyPassphrase: field( + z.string().optional(), + 'APIFY_INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE', + ), + inputSecretsPrivateKeyFile: field( + z.string().optional(), + 'APIFY_INPUT_SECRETS_PRIVATE_KEY_FILE', + ), + maxTotalChargeUsd: field( + coerceNumber.optional(), + ACTOR_ENV_VARS.MAX_TOTAL_CHARGE_USD, + ), + metaOrigin: field(z.string().optional(), 'APIFY_META_ORIGIN'), + testPayPerEvent: field( + coerceBoolean.default(false), + 'ACTOR_TEST_PAY_PER_EVENT', + ), + useChargingLogDataset: field( + coerceBoolean.default(false), + 'ACTOR_USE_CHARGING_LOG_DATASET', + ), +}; + +// --- Type utilities --- + +export type ApifyConfigurationInput = FieldsInput; +export type ApifyResolvedConfigValues = FieldsOutput; + +/** @deprecated Use {@link ApifyConfigurationInput} instead. */ +export type ConfigurationOptions = ApifyConfigurationInput; + +// --- Configuration class --- + +// eslint-disable-next-line @typescript-eslint/no-empty-object-type, @typescript-eslint/no-unsafe-declaration-merging +export interface Configuration extends ApifyResolvedConfigValues {} /** * `Configuration` is a value object holding the SDK configuration. We can use it in two ways: @@ -48,38 +209,34 @@ export interface ConfigurationOptions extends CoreConfigurationOptions { * * ```javascript * import { Actor } from 'apify'; - * import { BasicCrawler } from 'crawlee'; * * const sdk = new Actor({ token: '123' }); - * console.log(sdk.config.get('token')); // '123' - * - * const crawler = new BasicCrawler({ - * // ... crawler options - * }, sdk.config); + * console.log(sdk.config.token); // '123' * ``` * * 2. To get the global configuration (singleton instance). It will respect the environment variables. * * ```javascript - * import { BasicCrawler, Configuration } from 'crawlee'; + * import { Configuration } from 'apify'; * - * // Get the global configuration * const config = Configuration.getGlobalConfig(); - * // Set the 'persistStateIntervalMillis' option - * // of global configuration to 30 seconds - * config.set('persistStateIntervalMillis', 30_000); - * - * // No need to pass the configuration to the crawler, - * // as it's using the global configuration by default - * const crawler = new BasicCrawler(); + * console.log(config.headless); + * console.log(config.persistStateIntervalMillis); * ``` * + * Configuration is immutable — values are set via the constructor and cannot be changed afterwards. + * The priority order for resolving values is (highest to lowest): + * + * ```text + * constructor options > environment variables > crawlee.json > schema defaults + * ``` + * * ## Supported Configuration Options * * Key | Environment Variable | Default Value * ---|---|--- * `memoryMbytes` | `ACTOR_MEMORY_MBYTES` | - - * `headless` | `APIFY_HEADLESS` | - + * `headless` | `APIFY_HEADLESS` | `true` * `persistStateIntervalMillis` | `APIFY_PERSIST_STATE_INTERVAL_MILLIS` | `60e3` * `token` | `APIFY_TOKEN` | - * `isAtHome` | `APIFY_IS_AT_HOME` | - @@ -112,127 +269,16 @@ export interface ConfigurationOptions extends CoreConfigurationOptions { * `chromeExecutablePath` | `APIFY_CHROME_EXECUTABLE_PATH` | - * `defaultBrowserPath` | `APIFY_DEFAULT_BROWSER_PATH` | - */ +// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging export class Configuration extends CoreConfiguration { - /** @inheritDoc */ - // eslint-disable-next-line no-use-before-define -- Self-reference - static override globalConfig?: Configuration; - - // maps environment variables to config keys (e.g. `APIFY_MEMORY_MBYTES` to `memoryMbytes`) - protected static override ENV_MAP = { - // regular crawlee env vars are also supported - ...CoreConfiguration.ENV_MAP, - - // support crawlee env vars prefixed with `APIFY_` too - APIFY_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio', - APIFY_PURGE_ON_START: 'purgeOnStart', - APIFY_MEMORY_MBYTES: 'memoryMbytes', - APIFY_DEFAULT_DATASET_ID: 'defaultDatasetId', - APIFY_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId', - APIFY_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId', - APIFY_INPUT_KEY: 'inputKey', - APIFY_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis', - APIFY_HEADLESS: 'headless', - APIFY_XVFB: 'xvfb', - APIFY_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath', - APIFY_DEFAULT_BROWSER_PATH: 'defaultBrowserPath', - APIFY_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox', - - // as well as apify specific ones - APIFY_TOKEN: 'token', - APIFY_METAMORPH_AFTER_SLEEP_MILLIS: 'metamorphAfterSleepMillis', - APIFY_TEST_PERSIST_INTERVAL_MILLIS: 'persistStateIntervalMillis', // for BC, seems to be unused - APIFY_ACTOR_EVENTS_WS_URL: 'actorEventsWsUrl', - APIFY_ACTOR_ID: 'actorId', - APIFY_API_BASE_URL: 'apiBaseUrl', - APIFY_API_PUBLIC_BASE_URL: 'apiPublicBaseUrl', - APIFY_IS_AT_HOME: 'isAtHome', - APIFY_ACTOR_RUN_ID: 'actorRunId', - APIFY_ACTOR_TASK_ID: 'actorTaskId', - APIFY_CONTAINER_PORT: 'containerPort', - APIFY_CONTAINER_URL: 'containerUrl', - APIFY_USER_ID: 'userId', - APIFY_PROXY_HOSTNAME: 'proxyHostname', - APIFY_PROXY_PASSWORD: 'proxyPassword', - APIFY_PROXY_STATUS_URL: 'proxyStatusUrl', - APIFY_PROXY_PORT: 'proxyPort', - APIFY_INPUT_SECRETS_PRIVATE_KEY_FILE: 'inputSecretsPrivateKeyFile', - APIFY_INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE: - 'inputSecretsPrivateKeyPassphrase', - APIFY_META_ORIGIN: 'metaOrigin', - - // Actor env vars - ACTOR_DEFAULT_DATASET_ID: 'defaultDatasetId', - ACTOR_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId', - ACTOR_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId', - ACTOR_EVENTS_WEBSOCKET_URL: 'actorEventsWsUrl', - ACTOR_ID: 'actorId', - ACTOR_INPUT_KEY: 'inputKey', - ACTOR_MEMORY_MBYTES: 'memoryMbytes', - ACTOR_RUN_ID: 'actorRunId', - ACTOR_STANDBY_PORT: 'standbyPort', - ACTOR_STANDBY_URL: 'standbyUrl', - ACTOR_TASK_ID: 'actorTaskId', - ACTOR_WEB_SERVER_PORT: 'containerPort', - ACTOR_WEB_SERVER_URL: 'containerUrl', - ACTOR_MAX_TOTAL_CHARGE_USD: 'maxTotalChargeUsd', - ACTOR_TEST_PAY_PER_EVENT: 'testPayPerEvent', - ACTOR_USE_CHARGING_LOG_DATASET: 'useChargingLogDataset', - }; + /** @internal */ + static storage = new AsyncLocalStorage(); - protected static override INTEGER_VARS = [ - ...CoreConfiguration.INTEGER_VARS, - 'proxyPort', - 'containerPort', - 'metamorphAfterSleepMillis', - 'maxTotalChargeUsd', - ]; + /** @internal */ + static globalConfig?: Configuration; - protected static override BOOLEAN_VARS = [ - ...CoreConfiguration.BOOLEAN_VARS, - 'isAtHome', - 'testPayPerEvent', - 'useChargingLogDataset', - ]; - - protected static override DEFAULTS = { - ...CoreConfiguration.DEFAULTS, - defaultKeyValueStoreId: - LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID], - defaultDatasetId: - LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_DATASET_ID], - defaultRequestQueueId: - LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID], - inputKey: 'INPUT', - apiBaseUrl: 'https://api.apify.com', - apiPublicBaseUrl: 'https://api.apify.com', - proxyStatusUrl: 'http://proxy.apify.com', - proxyHostname: LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_HOSTNAME], - proxyPort: +LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_PORT], - containerPort: +LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.WEB_SERVER_PORT], - containerUrl: LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.WEB_SERVER_URL], - standbyPort: +LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.STANDBY_PORT], - metamorphAfterSleepMillis: 300e3, - persistStateIntervalMillis: 60e3, // This value is mentioned in jsdoc in `events.js`, if you update it here, update it there too. - testPayPerEvent: false, - useChargingLogDataset: false, - }; - - /** - * @inheritDoc - */ - override get< - T extends keyof ConfigurationOptions, - U extends ConfigurationOptions[T], - >(key: T, defaultValue?: U): U { - return super.get(key as keyof CoreConfigurationOptions, defaultValue); - } - - /** - * @inheritDoc - */ - override set(key: keyof ConfigurationOptions, value?: any) { - super.set(key as keyof CoreConfigurationOptions, value); - } + protected static override fields: Record = + apifyConfigFields; /** * @inheritDoc @@ -250,18 +296,7 @@ export class Configuration extends CoreConfiguration { * Resets global configuration instance. The default instance holds configuration based on env vars, * if we want to change them, we need to first reset the global state. Used mainly for testing purposes. */ - static override resetGlobalState(): void { + static resetGlobalState(): void { delete this.globalConfig; } } - -// monkey patch the core class so it respects the new options too -CoreConfiguration.getGlobalConfig = Configuration.getGlobalConfig; -// @ts-expect-error protected property -CoreConfiguration.ENV_MAP = Configuration.ENV_MAP; -// @ts-expect-error protected property -CoreConfiguration.INTEGER_VARS = Configuration.INTEGER_VARS; -// @ts-expect-error protected property -CoreConfiguration.BOOLEAN_VARS = Configuration.BOOLEAN_VARS; -// @ts-expect-error protected property -CoreConfiguration.DEFAULTS = Configuration.DEFAULTS; diff --git a/packages/apify/src/key_value_store.ts b/packages/apify/src/key_value_store.ts index 89f2138d2c..a26a12e8f1 100644 --- a/packages/apify/src/key_value_store.ts +++ b/packages/apify/src/key_value_store.ts @@ -18,12 +18,12 @@ export class KeyValueStore extends CoreKeyValueStore { */ override getPublicUrl(key: string): string { const config = this.config as Configuration; - if (!config.get('isAtHome') && getPublicUrl) { + if (!config.isAtHome && getPublicUrl) { return getPublicUrl.call(this, key); } const publicUrl = new URL( - `${config.get('apiPublicBaseUrl')}/v2/key-value-stores/${this.id}/records/${key}`, + `${config.apiPublicBaseUrl}/v2/key-value-stores/${this.id}/records/${key}`, ); if (this.storageObject?.urlSigningSecretKey) { diff --git a/packages/apify/src/platform_event_manager.ts b/packages/apify/src/platform_event_manager.ts index a71c5ee68e..4385eebd60 100644 --- a/packages/apify/src/platform_event_manager.ts +++ b/packages/apify/src/platform_event_manager.ts @@ -62,7 +62,7 @@ export class PlatformEventManager extends EventManager { } await super.init(); - const eventsWsUrl = this.config.get('actorEventsWsUrl'); + const eventsWsUrl = (this.config as Configuration).actorEventsWsUrl; // Locally there is no web socket to connect, so just print a log message. if (!eventsWsUrl) { diff --git a/packages/apify/src/proxy_configuration.ts b/packages/apify/src/proxy_configuration.ts index 569ea84ae3..0d452ab645 100644 --- a/packages/apify/src/proxy_configuration.ts +++ b/packages/apify/src/proxy_configuration.ts @@ -205,7 +205,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { apifyProxyGroups = [], countryCode, apifyProxyCountry, - password = config.get('proxyPassword'), + password = config.proxyPassword, tieredProxyConfig, tieredProxyUrls, } = options; @@ -221,8 +221,8 @@ export class ProxyConfiguration extends CoreProxyConfiguration { const groupsToUse = groups.length ? groups : apifyProxyGroups; const countryCodeToUse = countryCode || apifyProxyCountry; - const hostname = config.get('proxyHostname'); - const port = config.get('proxyPort'); + const hostname = config.proxyHostname; + const port = config.proxyPort; // Validation if ( @@ -438,7 +438,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { */ // TODO: Make this private protected async _setPasswordIfToken(): Promise { - const token = this.config.get('token'); + const {token} = (this.config as Configuration); if (!token) return; try { @@ -500,10 +500,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { } | undefined > { - const proxyStatusUrl = this.config.get( - 'proxyStatusUrl', - 'http://proxy.apify.com', - ); + const {proxyStatusUrl} = (this.config as Configuration); const requestOpts = { url: `${proxyStatusUrl}/?format=json`, proxyUrl: await this.newUrl(), diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index 47565929ff..2687b1a476 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -705,13 +705,14 @@ describe('Actor', () => { expect(getValueSpy).toBeCalledWith(KEY_VALUE_STORE_KEYS.INPUT); expect(val1).toBe(123); - // Uses value from config - sdk.config.set('inputKey', 'some-value'); - const val2 = await sdk.getInput(); - expect(getValueSpy).toBeCalledTimes(2); - expect(getValueSpy).toBeCalledWith('some-value'); + // Uses value from config - create a new Actor with custom inputKey + const sdk2 = new Actor({ inputKey: 'some-value' }); + const getValueSpy2 = vitest.spyOn(sdk2 as any, 'getValue'); + getValueSpy2.mockImplementation(async () => 123); + const val2 = await sdk2.getInput(); + expect(getValueSpy2).toBeCalledTimes(1); + expect(getValueSpy2).toBeCalledWith('some-value'); expect(val2).toBe(123); - sdk.config.set('inputKey', undefined); // restore defaults }); test('setValue()', async () => { @@ -1285,14 +1286,14 @@ describe('Actor', () => { test('should work', async () => { await Actor.init(); process.env.ACTOR_MAX_TOTAL_CHARGE_USD = ''; - expect(Actor.config.get('maxTotalChargeUsd')).toBe(0); + expect(Actor.config.maxTotalChargeUsd).toBe(0); expect(Actor.getChargingManager().getMaxTotalChargeUsd()).toBe( Infinity, ); // the value in charging manager is cached, so we cant test that here process.env.ACTOR_MAX_TOTAL_CHARGE_USD = '123'; - expect(Actor.config.get('maxTotalChargeUsd')).toBe(123); + expect(Actor.config.maxTotalChargeUsd).toBe(123); await Actor.exit({ exit: false }); }); }); diff --git a/test/apify/events.test.ts b/test/apify/events.test.ts index cb7804fbb7..d6ec40d3ec 100644 --- a/test/apify/events.test.ts +++ b/test/apify/events.test.ts @@ -130,7 +130,7 @@ describe('events', () => { test('should send persist state events in regular interval', async () => { const eventsReceived = []; - const interval = config.get('persistStateIntervalMillis')!; + const interval = config.persistStateIntervalMillis; events.on(EventType.PERSIST_STATE, (data) => eventsReceived.push(data)); await events.init(); From 18a23d940601c963bc7a6ef3759941b15aa88c91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Wed, 25 Mar 2026 08:58:06 +0100 Subject: [PATCH 02/34] fix: update configuration integration for crawlee Configuration redesign - Import `z` from `zod` directly (no longer re-exported from crawlee core) - Define `coerceNumber` locally (no longer exported from crawlee core) - Add constructor override to accept `ApifyConfigurationInput` - Import `ConfigurationOptions` from SDK configuration instead of core - Fix test that mutated env vars after init (immutable config) Depends on: apify/crawlee#3080 Co-Authored-By: Claude Opus 4.6 (1M context) --- package-lock.json | 4 ++-- packages/apify/package.json | 3 ++- packages/apify/src/actor.ts | 6 +++--- packages/apify/src/configuration.ts | 12 ++++++++++-- test/apify/actor.test.ts | 13 +++++++------ 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1b05834ad6..1f4a8a966c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20046,7 +20046,6 @@ "version": "3.25.28", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.28.tgz", "integrity": "sha512-/nt/67WYKnr5by3YS7LroZJbtcCBurDKKPBPWWzaxvVCGuG/NOsiKkrjoOhI8mJ+SQUXEbUzeB3S+6XDUEEj7Q==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" @@ -20195,7 +20194,8 @@ "ow": "^2.0.0", "semver": "^7.7.2", "tslib": "^2.8.1", - "ws": "^8.18.2" + "ws": "^8.18.2", + "zod": "^3.24.0 || ^4.0.0" }, "engines": { "node": ">=22.0.0" diff --git a/packages/apify/package.json b/packages/apify/package.json index f3ddcf0be1..ebb8146b6d 100644 --- a/packages/apify/package.json +++ b/packages/apify/package.json @@ -62,6 +62,7 @@ "ow": "^2.0.0", "semver": "^7.7.2", "tslib": "^2.8.1", - "ws": "^8.18.2" + "ws": "^8.18.2", + "zod": "^3.24.0 || ^4.0.0" } } diff --git a/packages/apify/src/actor.ts b/packages/apify/src/actor.ts index c972ba8599..86fd1b9dce 100644 --- a/packages/apify/src/actor.ts +++ b/packages/apify/src/actor.ts @@ -1,7 +1,6 @@ import { createPrivateKey } from 'node:crypto'; import type { - ConfigurationOptions, EventManager, EventTypeName, IStorage, @@ -46,6 +45,7 @@ import { addTimeoutToPromise } from '@apify/timeout'; import type { ChargeOptions, ChargeResult } from './charging.js'; import { ChargingManager } from './charging.js'; +import type { ConfigurationOptions } from './configuration.js'; import { Configuration } from './configuration.js'; import { KeyValueStore } from './key_value_store.js'; import { PlatformEventManager } from './platform_event_manager.js'; @@ -1208,8 +1208,8 @@ export class Actor { async getInput(): Promise { this._ensureActorInit('getInput'); - const {inputSecretsPrivateKeyFile} = this.config; - const {inputSecretsPrivateKeyPassphrase} = this.config; + const { inputSecretsPrivateKeyFile } = this.config; + const { inputSecretsPrivateKeyPassphrase } = this.config; const input = await this.getValue(this.config.inputKey); if ( ow.isValid(input, ow.object.nonEmpty) && diff --git a/packages/apify/src/configuration.ts b/packages/apify/src/configuration.ts index 932da5b94e..4228f5d93b 100644 --- a/packages/apify/src/configuration.ts +++ b/packages/apify/src/configuration.ts @@ -4,12 +4,11 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import type { ConfigField, FieldsInput, FieldsOutput } from '@crawlee/core'; import { coerceBoolean, - coerceNumber, Configuration as CoreConfiguration, crawleeConfigFields, field, - z, } from '@crawlee/core'; +import { z } from 'zod'; import { ACTOR_ENV_VARS, @@ -18,6 +17,11 @@ import { LOCAL_APIFY_ENV_VARS, } from '@apify/consts'; +const coerceNumber = z.preprocess((val) => { + if (typeof val === 'string') return Number(val); + return val; +}, z.number()); + // --- isAtHome check (simple env var presence) --- const isAtHome = !!process.env[APIFY_ENV_VARS.IS_AT_HOME]; @@ -280,6 +284,10 @@ export class Configuration extends CoreConfiguration { protected static override fields: Record = apifyConfigFields; + constructor(options: ApifyConfigurationInput = {}) { + super(options as any); + } + /** * @inheritDoc */ diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index 2687b1a476..f1b88fc281 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -1283,18 +1283,19 @@ describe('Actor', () => { }); describe('Actor.config and PPE', () => { - test('should work', async () => { - await Actor.init(); + test('empty string maxTotalChargeUsd coerces to 0, charging manager treats as Infinity', async () => { process.env.ACTOR_MAX_TOTAL_CHARGE_USD = ''; + await Actor.init(); expect(Actor.config.maxTotalChargeUsd).toBe(0); expect(Actor.getChargingManager().getMaxTotalChargeUsd()).toBe( Infinity, ); - - // the value in charging manager is cached, so we cant test that here - process.env.ACTOR_MAX_TOTAL_CHARGE_USD = '123'; - expect(Actor.config.maxTotalChargeUsd).toBe(123); await Actor.exit({ exit: false }); }); + + test('numeric maxTotalChargeUsd is correctly resolved from constructor options', () => { + const sdk = new Actor({ maxTotalChargeUsd: 123 }); + expect(sdk.config.maxTotalChargeUsd).toBe(123); + }); }); }); From 4f7f6f9fcbcb445921d3c46c8cb7589c73393358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Mon, 20 Apr 2026 16:23:32 +0200 Subject: [PATCH 03/34] fix: preserve storageClientOptions pass-through in Actor.newClient Restore the destructuring of `storageDir` and spread of remaining `storageClientOptions` into the `ApifyClient` constructor so that arbitrary client options configured via `storageClientOptions` continue to reach the client. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/apify/src/actor.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/apify/src/actor.ts b/packages/apify/src/actor.ts index 86fd1b9dce..f75a3a64fd 100644 --- a/packages/apify/src/actor.ts +++ b/packages/apify/src/actor.ts @@ -1467,6 +1467,9 @@ export class Actor { * @ignore */ newClient(options: ApifyClientOptions = {}): ApifyClient { + const { storageDir, ...storageClientOptions } = (this.config.get( + 'storageClientOptions', + ) ?? {}) as Dictionary; const { apifyVersion, crawleeVersion } = getSystemInfo(); return new ApifyClient({ baseUrl: this.config.apiBaseUrl, @@ -1475,6 +1478,7 @@ export class Actor { `SDK/${apifyVersion}`, `Crawlee/${crawleeVersion}`, ], + ...storageClientOptions, ...options, // allow overriding the instance configuration }); } From b6c34ea472b324cfc49f74dc7479a19a1b2841a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 17:47:39 +0200 Subject: [PATCH 04/34] chore: bump crawlee to ^4.0.0-beta.49 and finish config-redesign integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reuse `coerceNumber` from `@crawlee/core` instead of defining a local copy; otherwise `FieldsOutput` produces a structurally distinct (but equivalent) `availableMemoryRatio` type that breaks declaration-merging with crawlee's `Configuration`. - Drop the dead `storageClientOptions`/`storageDir` destructuring in `Actor.newClient()` — neither key exists in the redesigned Configuration; `options` already covers the override path. The remaining build errors (proxy/storage/event drift) are unrelated to the config redesign and tracked in separate follow-up PRs against the v4 branch. --- package-lock.json | 1837 ++++++++++++++++----------- package.json | 8 +- packages/apify/package.json | 6 +- packages/apify/src/actor.ts | 4 - packages/apify/src/configuration.ts | 6 +- 5 files changed, 1125 insertions(+), 736 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1f4a8a966c..8923e74741 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,9 +16,9 @@ "@apify/input_secrets": "^1.1.72", "@apify/tsconfig": "^0.1.1", "@commitlint/config-conventional": "^19.8.1", - "@crawlee/core": "^4.0.0-beta.0", - "@crawlee/types": "^4.0.0-beta.0", - "@crawlee/utils": "^4.0.0-beta.0", + "@crawlee/core": "^4.0.0-beta.49", + "@crawlee/types": "^4.0.0-beta.49", + "@crawlee/utils": "^4.0.0-beta.49", "@playwright/browser-chromium": "^1.52.0", "@types/content-type": "^1.1.8", "@types/fs-extra": "^11.0.4", @@ -27,7 +27,7 @@ "@types/tough-cookie": "^4.0.5", "@types/ws": "^8.18.1", "commitlint": "^19.8.1", - "crawlee": "^4.0.0-beta.0", + "crawlee": "^4.0.0-beta.49", "eslint": "^9.27.0", "eslint-config-prettier": "^10.1.5", "fs-extra": "^11.3.0", @@ -194,6 +194,16 @@ "node": ">=6.9.0" } }, + "node_modules/@borewit/text-codec": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@borewit/text-codec/-/text-codec-0.2.2.tgz", + "integrity": "sha512-DDaRehssg1aNrH4+2hnj1B7vnUGEjU6OIlyRdkMd0aUdIUvKXrJfXsy8LVtXAy7DRvYVluWbMspsRhz2lcW0mQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/@commitlint/cli": { "version": "19.8.1", "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-19.8.1.tgz", @@ -464,19 +474,106 @@ "node": ">=v18" } }, + "node_modules/@crawlee/cheerio": { + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-4.0.0-beta.49.tgz", + "integrity": "sha512-WYFTBQE+7Rzg/cb0OxeOx7yF/IgZTHZLsT0HQEA1yV/0YWuxUFFydJeUDYFhkP3xyzutuOR81sPYkI5xAf/qrQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@crawlee/http": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", + "@crawlee/utils": "4.0.0-beta.49", + "cheerio": "^1.0.0", + "htmlparser2": "^10.0.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@crawlee/cheerio/node_modules/cheerio": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^7.19.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=20.18.1" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/@crawlee/cheerio/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/@crawlee/cheerio/node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "dev": true, + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/@crawlee/cheerio/node_modules/undici": { + "version": "7.25.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", + "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/@crawlee/cli": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-4.0.0-beta.4.tgz", - "integrity": "sha512-IgjQTitc09MM9FIiMAzX5Xx9/6AwevvBVqbyoRMsnJqA1/sPF1Xr7I7I8ImsneZTOjEfEBEq4FycNaOUaUD2PQ==", + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-4.0.0-beta.49.tgz", + "integrity": "sha512-QhEqBe274Q0ogdiLD/3QtToFQwLisYT+5RHOsJW7EkGEsQ9kJ/h5KD4IRi9CsDStB9IEww5vFv0tJRcwwWqpzQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@crawlee/templates": "4.0.0-beta.4", + "@crawlee/templates": "4.0.0-beta.49", "@inquirer/prompts": "^7.5.0", "ansi-colors": "^4.1.3", "fs-extra": "^11.3.0", "tslib": "^2.8.1", - "yargs": "^17.7.2" + "yargs": "^18.0.0" }, "bin": { "crawlee": "index.js" @@ -485,10 +582,138 @@ "node": ">=22.0.0" } }, + "node_modules/@crawlee/cli/node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@crawlee/cli/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@crawlee/cli/node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@crawlee/cli/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@crawlee/cli/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@crawlee/cli/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@crawlee/cli/node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@crawlee/cli/node_modules/yargs": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz", + "integrity": "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^7.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/@crawlee/cli/node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, "node_modules/@crawlee/core": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-4.0.0-beta.4.tgz", - "integrity": "sha512-gagThRFMLQB0N/Uw8ad+XpRo8xgNEYgHXCSU5H5MVvK6vS4nxm0s2CI7LBj0v1o92rsoM5We284FPlZTe8Qn0w==", + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-4.0.0-beta.49.tgz", + "integrity": "sha512-k5FBU/R+gwrL/FWu6fFSciU4aM+nYsUJy4YOAk3Cur9Yom8YF8JRbGGeAaFEvDDjJZxWTR5s0GyYnzHeidx9Gw==", "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.41.0", @@ -497,22 +722,21 @@ "@apify/pseudo_url": "^2.0.59", "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/memory-storage": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "@crawlee/utils": "4.0.0-beta.4", + "@crawlee/memory-storage": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", + "@crawlee/utils": "4.0.0-beta.49", "@sapphire/async-queue": "^1.5.5", "@vladfrangu/async_event_emitter": "^2.4.6", "csv-stringify": "^6.5.2", - "fs-extra": "^11.3.0", - "got-scraping": "^4.1.1", "json5": "^2.2.3", "minimatch": "^10.0.1", "ow": "^2.0.0", "stream-json": "^1.9.1", "tldts": "^7.0.6", - "tough-cookie": "^5.1.2", + "tough-cookie": "^6.0.0", "tslib": "^2.8.1", - "type-fest": "^4.41.0" + "type-fest": "^4.41.0", + "zod": "^3.24.0 || ^4.0.0" }, "engines": { "node": ">=22.0.0" @@ -613,41 +837,99 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/linkedom": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-4.0.0-beta.4.tgz", - "integrity": "sha512-eOGSsQs2NylAvMG+OiQwe+CiBHGR5f+OtcMo0+vSKh+jno8IlnoSCSvkLQ8pYUo6lWtXe4m2VDfV06Yc9jTrSw==", + "node_modules/@crawlee/core/node_modules/tough-cookie": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz", + "integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@crawlee/got-scraping-client": { + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/got-scraping-client/-/got-scraping-client-4.0.0-beta.49.tgz", + "integrity": "sha512-L1IWFC/kBQx9TNg/F7SVMjlnG92T95I1ZEL76x5dyGP9hBWBjaZHL8vsVSnoHDYh7ADWsbKjHyyD1CsMUjVimw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@crawlee/http-client": "4.0.0-beta.49", + "got-scraping": "^4.2.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@crawlee/http": { + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-4.0.0-beta.49.tgz", + "integrity": "sha512-q6vJ29s1TLynGBKPoHx4AbIMyl1KFw3oRPPWLakOj4wqK72W8KqG/0BHP6LPtgwBBgBF4TKhxEnp7xMT29f0KQ==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/http": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "linkedom": "^0.18.10", + "@crawlee/basic": "4.0.0-beta.49", + "@crawlee/core": "4.0.0-beta.49", + "@crawlee/http-client": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", + "@crawlee/utils": "4.0.0-beta.49", + "@types/content-type": "^1.1.8", + "cheerio": "^1.0.0", + "content-type": "^1.0.5", + "iconv-lite": "^0.7.2", + "mime-types": "^3.0.1", "ow": "^2.0.0", - "tslib": "^2.8.1" + "tslib": "^2.8.1", + "type-fest": "^4.41.0" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@crawlee/http-client": { + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/http-client/-/http-client-4.0.0-beta.49.tgz", + "integrity": "sha512-UMK5FLyFmAZHFNaaMhITwKQgEPdJZ5GJYvGCDHig4H5Hmc83aiBWIgGNqmQL+t/1YXkWVA0NhrBAgonk3m64aw==", + "license": "Apache-2.0", + "dependencies": { + "@crawlee/types": "4.0.0-beta.49", + "tough-cookie": "^6.0.0" }, "engines": { "node": ">=22.0.0" } }, - "node_modules/@crawlee/linkedom/node_modules/@crawlee/basic": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.4.tgz", - "integrity": "sha512-GnHrt8eylp/9MSLy5tXHrELwTA3Prm8YGVkzyO/8mfZZJqvz7L16uIAvO+ZWhg1x31MoHJKEQLlolaO6CDj2nA==", + "node_modules/@crawlee/http-client/node_modules/tough-cookie": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz", + "integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@crawlee/http/node_modules/@crawlee/basic": { + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.49.tgz", + "integrity": "sha512-6rRchZXH1KyLvkcZpTPeuRcbEDdl/l0XOsVTK5lHxkQvj63LrE+YwlmZxEA3LViiva9soNUll7VoFfWhEL77iw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@apify/log": "^2.5.18", "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "@crawlee/utils": "4.0.0-beta.4", + "@crawlee/core": "4.0.0-beta.49", + "@crawlee/got-scraping-client": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", + "@crawlee/utils": "4.0.0-beta.49", "csv-stringify": "^6.5.2", "fs-extra": "^11.3.0", - "got-scraping": "^4.1.1", "ow": "^2.0.0", "tldts": "^7.0.6", "tslib": "^2.8.1", @@ -657,33 +939,218 @@ "node": ">=22.0.0" } }, - "node_modules/@crawlee/linkedom/node_modules/@crawlee/http": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-4.0.0-beta.4.tgz", - "integrity": "sha512-G0MWHpYcSUO73b+ZST5HT7XtiiYkONaMZ+cgKm3TMbtOh0zuaOx8SdcQTNjeMnJdxmSpbmLbQl2ihynXGxhQdA==", + "node_modules/@crawlee/http/node_modules/@sindresorhus/is": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-6.3.1.tgz", + "integrity": "sha512-FX4MfcifwJyFOI2lPoX7PQxCqx8BG1HCho7WdiXwpEQx1Ycij0JxkfYtGK7yqNScrZGSlt6RE6sw8QYoH7eKnQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@crawlee/http/node_modules/callsites": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-4.2.0.tgz", + "integrity": "sha512-kfzR4zzQtAE9PC7CzZsjl3aBNbXWuXiSeOCdLcPpBfGW8YuCqQHcRPFDbr/BPVmd3EEPVpuFzLyuT/cUhPr4OQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@crawlee/http/node_modules/cheerio": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^7.19.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=20.18.1" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/@crawlee/http/node_modules/dot-prop": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-8.0.2.tgz", + "integrity": "sha512-xaBe6ZT4DHPkg0k4Ytbvn5xoxgpG0jOS1dYxSOwAHPuNLjP3/OzN0gH55SrLqpx8cBfSaVt91lXYkApjb+nYdQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^3.8.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@crawlee/http/node_modules/dot-prop/node_modules/type-fest": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.13.1.tgz", + "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@crawlee/http/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/@crawlee/http/node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "dev": true, + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/@crawlee/http/node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@crawlee/http/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@crawlee/http/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@crawlee/http/node_modules/ow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ow/-/ow-2.0.0.tgz", + "integrity": "sha512-ESUigmGrdhUZ2nQSFNkeKSl6ZRPupXzprMs3yF9DYlNVpJ8XAjM/fI9RUZxA7PI1K9HQDCCvBo1jr/GEIo9joQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^6.3.0", + "callsites": "^4.1.0", + "dot-prop": "^8.0.2", + "environment": "^1.0.0", + "fast-equals": "^5.0.1", + "is-identifier": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@crawlee/http/node_modules/undici": { + "version": "7.25.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", + "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/@crawlee/jsdom": { + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-4.0.0-beta.49.tgz", + "integrity": "sha512-RpO6fJfB8GMxokqfNTanU/zTSZJCFCw0fu3aTfyTG5njk/IENySRyJcIFWz+uUNuzF3trJVW4XEJBq4PBzvUjA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@apify/timeout": "^0.3.2", - "@apify/utilities": "^2.15.5", - "@crawlee/basic": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "@crawlee/utils": "4.0.0-beta.4", - "@types/content-type": "^1.1.8", + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.7.10", + "@crawlee/http": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", + "@crawlee/utils": "4.0.0-beta.49", + "@types/jsdom": "^21.1.7", "cheerio": "^1.0.0", - "content-type": "^1.0.5", - "got-scraping": "^4.1.1", - "iconv-lite": "^0.6.3", - "mime-types": "^3.0.1", + "jsdom": "^26.1.0", "ow": "^2.0.0", - "tslib": "^2.8.1", - "type-fest": "^4.41.0" + "tslib": "^2.8.1" }, "engines": { "node": ">=22.0.0" } }, - "node_modules/@crawlee/linkedom/node_modules/@sindresorhus/is": { + "node_modules/@crawlee/jsdom/node_modules/@sindresorhus/is": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-6.3.1.tgz", "integrity": "sha512-FX4MfcifwJyFOI2lPoX7PQxCqx8BG1HCho7WdiXwpEQx1Ycij0JxkfYtGK7yqNScrZGSlt6RE6sw8QYoH7eKnQ==", @@ -696,7 +1163,7 @@ "url": "https://github.com/sindresorhus/is?sponsor=1" } }, - "node_modules/@crawlee/linkedom/node_modules/callsites": { + "node_modules/@crawlee/jsdom/node_modules/callsites": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-4.2.0.tgz", "integrity": "sha512-kfzR4zzQtAE9PC7CzZsjl3aBNbXWuXiSeOCdLcPpBfGW8YuCqQHcRPFDbr/BPVmd3EEPVpuFzLyuT/cUhPr4OQ==", @@ -709,33 +1176,33 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/linkedom/node_modules/cheerio": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", - "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", + "node_modules/@crawlee/jsdom/node_modules/cheerio": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", "dev": true, "license": "MIT", "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", - "domutils": "^3.1.0", - "encoding-sniffer": "^0.2.0", - "htmlparser2": "^9.1.0", - "parse5": "^7.1.2", - "parse5-htmlparser2-tree-adapter": "^7.0.0", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", - "undici": "^6.19.5", + "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" }, "engines": { - "node": ">=18.17" + "node": ">=20.18.1" }, "funding": { "url": "https://github.com/cheeriojs/cheerio?sponsor=1" } }, - "node_modules/@crawlee/linkedom/node_modules/dot-prop": { + "node_modules/@crawlee/jsdom/node_modules/dot-prop": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-8.0.2.tgz", "integrity": "sha512-xaBe6ZT4DHPkg0k4Ytbvn5xoxgpG0jOS1dYxSOwAHPuNLjP3/OzN0gH55SrLqpx8cBfSaVt91lXYkApjb+nYdQ==", @@ -745,66 +1212,149 @@ "type-fest": "^3.8.0" }, "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@crawlee/jsdom/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/@crawlee/jsdom/node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "dev": true, + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/@crawlee/jsdom/node_modules/ow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ow/-/ow-2.0.0.tgz", + "integrity": "sha512-ESUigmGrdhUZ2nQSFNkeKSl6ZRPupXzprMs3yF9DYlNVpJ8XAjM/fI9RUZxA7PI1K9HQDCCvBo1jr/GEIo9joQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^6.3.0", + "callsites": "^4.1.0", + "dot-prop": "^8.0.2", + "environment": "^1.0.0", + "fast-equals": "^5.0.1", + "is-identifier": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@crawlee/jsdom/node_modules/type-fest": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.13.1.tgz", + "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@crawlee/jsdom/node_modules/undici": { + "version": "7.25.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", + "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/@crawlee/linkedom": { + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-4.0.0-beta.49.tgz", + "integrity": "sha512-fOBVgl4sVcrKM/3DF96SfaMQjJQuEoMvtci67X1huPrPtjkrREqGrDufCtSttkcSkGu/gQCxMN/yv+oyzBAujQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@apify/timeout": "^0.3.2", + "@apify/utilities": "^2.15.5", + "@crawlee/http": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", + "@crawlee/utils": "4.0.0-beta.49", + "linkedom": "^0.18.10", + "ow": "^2.0.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" } }, - "node_modules/@crawlee/linkedom/node_modules/dot-prop/node_modules/type-fest": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.13.1.tgz", - "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==", + "node_modules/@crawlee/linkedom/node_modules/@sindresorhus/is": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-6.3.1.tgz", + "integrity": "sha512-FX4MfcifwJyFOI2lPoX7PQxCqx8BG1HCho7WdiXwpEQx1Ycij0JxkfYtGK7yqNScrZGSlt6RE6sw8QYoH7eKnQ==", "dev": true, - "license": "(MIT OR CC0-1.0)", + "license": "MIT", "engines": { - "node": ">=14.16" + "node": ">=16" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@crawlee/linkedom/node_modules/htmlparser2": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", - "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", - "dev": true, - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.1.0", - "entities": "^4.5.0" + "url": "https://github.com/sindresorhus/is?sponsor=1" } }, - "node_modules/@crawlee/linkedom/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "node_modules/@crawlee/linkedom/node_modules/callsites": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-4.2.0.tgz", + "integrity": "sha512-kfzR4zzQtAE9PC7CzZsjl3aBNbXWuXiSeOCdLcPpBfGW8YuCqQHcRPFDbr/BPVmd3EEPVpuFzLyuT/cUhPr4OQ==", "dev": true, "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/linkedom/node_modules/mime-types": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.1.tgz", - "integrity": "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==", + "node_modules/@crawlee/linkedom/node_modules/dot-prop": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-8.0.2.tgz", + "integrity": "sha512-xaBe6ZT4DHPkg0k4Ytbvn5xoxgpG0jOS1dYxSOwAHPuNLjP3/OzN0gH55SrLqpx8cBfSaVt91lXYkApjb+nYdQ==", "dev": true, "license": "MIT", "dependencies": { - "mime-db": "^1.54.0" + "type-fest": "^3.8.0" }, "engines": { - "node": ">= 0.6" + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/@crawlee/linkedom/node_modules/ow": { @@ -828,20 +1378,33 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@crawlee/linkedom/node_modules/type-fest": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.13.1.tgz", + "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@crawlee/memory-storage": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-4.0.0-beta.4.tgz", - "integrity": "sha512-40HxqveRwyeiwpnHw1L8cwo6BP+/UUBqehOzsODGmOZaNTFU6Nm+8hRXlbYbeqzyWwxXOn15bAf2STHsZct/gw==", + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-4.0.0-beta.49.tgz", + "integrity": "sha512-rLniynMhfrJoXn7rFHfj4NpsVvdRn3PIJi6YFL3G7s4JAiqqfhYOKshW+f6XdldEeb1z0YyGpgOknLo6VupktA==", "license": "Apache-2.0", "dependencies": { - "@apify/log": "^2.5.18", - "@crawlee/types": "4.0.0-beta.4", + "@crawlee/types": "4.0.0-beta.49", "@sapphire/async-queue": "^1.5.5", "@sapphire/shapeshift": "^4.0.0", "content-type": "^1.0.5", "fs-extra": "^11.3.0", "json5": "^2.2.3", "mime-types": "^3.0.1", + "p-limit": "^6.2.0", "proper-lockfile": "^4.1.2", "tslib": "^2.8.1" }, @@ -859,106 +1422,99 @@ } }, "node_modules/@crawlee/memory-storage/node_modules/mime-types": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.1.tgz", - "integrity": "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "license": "MIT", "dependencies": { "mime-db": "^1.54.0" }, "engines": { - "node": ">= 0.6" - } - }, - "node_modules/@crawlee/templates": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-4.0.0-beta.4.tgz", - "integrity": "sha512-tpkotGuxSxeMlzmKnJvPfb+2RnUTn32Yl8OkO8hMGu+dPSU81clHlCXoOyfIsWHCTlCWjxggITAMOYEL/hwYKw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "ansi-colors": "^4.1.3", - "inquirer": "^12.6.0", - "tslib": "^2.8.1", - "yargonaut": "^1.1.4", - "yargs": "^17.7.2" + "node": ">=18" }, - "engines": { - "node": ">=22.0.0" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/@crawlee/templates/node_modules/inquirer": { - "version": "12.6.1", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-12.6.1.tgz", - "integrity": "sha512-MGFnzHVS3l3oM3cy+LWkyR7UUtVEn3D5U41CZbEY34szToWoJAvaVtCTz1mxsEzZFk/HXWyCArn0HDgloTXMDw==", - "dev": true, + "node_modules/@crawlee/memory-storage/node_modules/p-limit": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.2.0.tgz", + "integrity": "sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==", "license": "MIT", "dependencies": { - "@inquirer/core": "^10.1.11", - "@inquirer/prompts": "^7.5.1", - "@inquirer/type": "^3.0.6", - "ansi-escapes": "^4.3.2", - "mute-stream": "^2.0.0", - "run-async": "^3.0.0", - "rxjs": "^7.8.2" + "yocto-queue": "^1.1.1" }, "engines": { "node": ">=18" }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/templates/node_modules/mute-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", - "integrity": "sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==", - "dev": true, - "license": "ISC", + "node_modules/@crawlee/memory-storage/node_modules/yocto-queue": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", + "license": "MIT", "engines": { - "node": "^18.17.0 || >=20.5.0" + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/templates/node_modules/run-async": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-3.0.0.tgz", - "integrity": "sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==", + "node_modules/@crawlee/templates": { + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-4.0.0-beta.49.tgz", + "integrity": "sha512-5/cBYgNuMXnVemxUDqagg0lgZURlc28NVuY1vfON+CG0C6Dg/GLwMlTLOmcroQeDJF7bWxVxDrsHomTk0Mm30A==", "dev": true, - "license": "MIT", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.8.1" + }, "engines": { - "node": ">=0.12.0" + "node": ">=22.0.0" } }, "node_modules/@crawlee/types": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-4.0.0-beta.4.tgz", - "integrity": "sha512-DSJRUfTa5NO4xt/ayWD8U/XkHusUry4XTaTcYdEJa9tVYwuf2SyF5xRwd6NdcEiJeeVHn5MM7GU6oknvugfezw==", + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-4.0.0-beta.49.tgz", + "integrity": "sha512-p33njES5N539Zgco1aroUZgmWeXp+i6MTPUk7R7wLEdV/k+aw2Y1cjw8wGQpKH9/ZZIpsvqx56U26bC2w1nmcQ==", "license": "Apache-2.0", "dependencies": { + "tough-cookie": "^6.0.0", "tslib": "^2.8.1" }, "engines": { "node": ">=22.0.0" } }, + "node_modules/@crawlee/types/node_modules/tough-cookie": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz", + "integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/@crawlee/utils": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-4.0.0-beta.4.tgz", - "integrity": "sha512-oZOORB+gIkEeeExEd47Cmbgkrh4pEqo+vklFY3gk55iFFq7X6W6hu4k2rvQP/FlIMhWMc3ETFJFID08OS6p23g==", + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-4.0.0-beta.49.tgz", + "integrity": "sha512-JmTTVsJBFJNPUZsdH35Yde/jRFJoZtloVtEYLGZoqo79rT9RmRi+SDr22XFYJ7bb/LEXY+NOWBovdMge+fnpHw==", "license": "Apache-2.0", "dependencies": { - "@apify/log": "^2.5.18", "@apify/ps-tree": "^1.2.0", - "@crawlee/types": "4.0.0-beta.4", + "@crawlee/http-client": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", "@types/sax": "^1.2.7", "cheerio": "^1.0.0", - "file-type": "^20.5.0", - "got-scraping": "^4.1.1", + "domhandler": "^5.0.3", + "file-type": "^21.0.0", "ow": "^2.0.0", "robots-parser": "^3.0.1", "sax": "^1.4.1", @@ -1963,18 +2519,28 @@ "node": ">=6.9.0" } }, + "node_modules/@inquirer/ansi": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-1.0.2.tgz", + "integrity": "sha512-S8qNSZiYzFd0wAcyG5AXCvUHC5Sr7xpZ9wZ2py9XR88jUz8wooStVx5M6dRzczbBWjic9NP7+rY0Xi7qqK/aMQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@inquirer/checkbox": { - "version": "4.1.6", - "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.1.6.tgz", - "integrity": "sha512-62u896rWCtKKE43soodq5e/QcRsA22I+7/4Ov7LESWnKRO6BVo2A1DFLDmXL9e28TB0CfHc3YtkbPm7iwajqkg==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.3.2.tgz", + "integrity": "sha512-VXukHf0RR1doGe6Sm4F0Em7SWYLTHSsbGfJdS9Ja2bX5/D5uwVOEjr07cncLROdBvmnvCATYEWlHqYmXv2IlQA==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/core": "^10.1.11", - "@inquirer/figures": "^1.0.11", - "@inquirer/type": "^3.0.6", - "ansi-escapes": "^4.3.2", - "yoctocolors-cjs": "^2.1.2" + "@inquirer/ansi": "^1.0.2", + "@inquirer/core": "^10.3.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" }, "engines": { "node": ">=18" @@ -1989,14 +2555,14 @@ } }, "node_modules/@inquirer/confirm": { - "version": "5.1.10", - "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.10.tgz", - "integrity": "sha512-FxbQ9giWxUWKUk2O5XZ6PduVnH2CZ/fmMKMBkH71MHJvWr7WL5AHKevhzF1L5uYWB2P548o1RzVxrNd3dpmk6g==", + "version": "5.1.21", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.21.tgz", + "integrity": "sha512-KR8edRkIsUayMXV+o3Gv+q4jlhENF9nMYUZs9PA2HzrXeHI8M5uDag70U7RJn9yyiMZSbtF5/UexBtAVtZGSbQ==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/core": "^10.1.11", - "@inquirer/type": "^3.0.6" + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" }, "engines": { "node": ">=18" @@ -2011,20 +2577,20 @@ } }, "node_modules/@inquirer/core": { - "version": "10.1.11", - "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.11.tgz", - "integrity": "sha512-BXwI/MCqdtAhzNQlBEFE7CEflhPkl/BqvAuV/aK6lW3DClIfYVDWPP/kXuXHtBWC7/EEbNqd/1BGq2BGBBnuxw==", + "version": "10.3.2", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.3.2.tgz", + "integrity": "sha512-43RTuEbfP8MbKzedNqBrlhhNKVwoK//vUFNW3Q3vZ88BLcrs4kYpGg+B2mm5p2K/HfygoCxuKwJJiv8PbGmE0A==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/figures": "^1.0.11", - "@inquirer/type": "^3.0.6", - "ansi-escapes": "^4.3.2", + "@inquirer/ansi": "^1.0.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", "cli-width": "^4.1.0", "mute-stream": "^2.0.0", "signal-exit": "^4.1.0", "wrap-ansi": "^6.2.0", - "yoctocolors-cjs": "^2.1.2" + "yoctocolors-cjs": "^2.1.3" }, "engines": { "node": ">=18" @@ -2072,15 +2638,15 @@ } }, "node_modules/@inquirer/editor": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.11.tgz", - "integrity": "sha512-YoZr0lBnnLFPpfPSNsQ8IZyKxU47zPyVi9NLjCWtna52//M/xuL0PGPAxHxxYhdOhnvY2oBafoM+BI5w/JK7jw==", + "version": "4.2.23", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.23.tgz", + "integrity": "sha512-aLSROkEwirotxZ1pBaP8tugXRFCxW94gwrQLxXfrZsKkfjOYC1aRvAZuhpJOb5cu4IBTJdsCigUlf2iCOu4ZDQ==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/core": "^10.1.11", - "@inquirer/type": "^3.0.6", - "external-editor": "^3.1.0" + "@inquirer/core": "^10.3.2", + "@inquirer/external-editor": "^1.0.3", + "@inquirer/type": "^3.0.10" }, "engines": { "node": ">=18" @@ -2095,15 +2661,37 @@ } }, "node_modules/@inquirer/expand": { - "version": "4.0.13", - "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.13.tgz", - "integrity": "sha512-HgYNWuZLHX6q5y4hqKhwyytqAghmx35xikOGY3TcgNiElqXGPas24+UzNPOwGUZa5Dn32y25xJqVeUcGlTv+QQ==", + "version": "4.0.23", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.23.tgz", + "integrity": "sha512-nRzdOyFYnpeYTTR2qFwEVmIWypzdAx/sIkCMeTNTcflFOovfqUk+HcFhQQVBftAh9gmGrpFj6QcGEqrDMDOiew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/external-editor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", + "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/core": "^10.1.11", - "@inquirer/type": "^3.0.6", - "yoctocolors-cjs": "^2.1.2" + "chardet": "^2.1.1", + "iconv-lite": "^0.7.0" }, "engines": { "node": ">=18" @@ -2117,10 +2705,34 @@ } } }, + "node_modules/@inquirer/external-editor/node_modules/chardet": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", + "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@inquirer/external-editor/node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/@inquirer/figures": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.12.tgz", - "integrity": "sha512-MJttijd8rMFcKJC8NYmprWr6hD3r9Gd9qUC0XwPNwoEPWSMVJwA2MlXxF+nhZZNMY+HXsWa+o7KY2emWYIn0jQ==", + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.15.tgz", + "integrity": "sha512-t2IEY+unGHOzAaVM5Xx6DEWKeXlDDcNPeDyUpsRc6CUhBfU3VQOEl+Vssh7VNp1dR8MdUJBWhuObjXCsVpjN5g==", "dev": true, "license": "MIT", "engines": { @@ -2128,14 +2740,14 @@ } }, "node_modules/@inquirer/input": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.1.10.tgz", - "integrity": "sha512-kV3BVne3wJ+j6reYQUZi/UN9NZGZLxgc/tfyjeK3mrx1QI7RXPxGp21IUTv+iVHcbP4ytZALF8vCHoxyNSC6qg==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.3.1.tgz", + "integrity": "sha512-kN0pAM4yPrLjJ1XJBjDxyfDduXOuQHrBB8aLDMueuwUGn+vNpF7Gq7TvyVxx8u4SHlFFj4trmj+a2cbpG4Jn1g==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/core": "^10.1.11", - "@inquirer/type": "^3.0.6" + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" }, "engines": { "node": ">=18" @@ -2150,14 +2762,14 @@ } }, "node_modules/@inquirer/number": { - "version": "3.0.13", - "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.13.tgz", - "integrity": "sha512-IrLezcg/GWKS8zpKDvnJ/YTflNJdG0qSFlUM/zNFsdi4UKW/CO+gaJpbMgQ20Q58vNKDJbEzC6IebdkprwL6ew==", + "version": "3.0.23", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.23.tgz", + "integrity": "sha512-5Smv0OK7K0KUzUfYUXDXQc9jrf8OHo4ktlEayFlelCjwMXz0299Y8OrI+lj7i4gCBY15UObk76q0QtxjzFcFcg==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/core": "^10.1.11", - "@inquirer/type": "^3.0.6" + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" }, "engines": { "node": ">=18" @@ -2172,15 +2784,15 @@ } }, "node_modules/@inquirer/password": { - "version": "4.0.13", - "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.13.tgz", - "integrity": "sha512-NN0S/SmdhakqOTJhDwOpeBEEr8VdcYsjmZHDb0rblSh2FcbXQOr+2IApP7JG4WE3sxIdKytDn4ed3XYwtHxmJQ==", + "version": "4.0.23", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.23.tgz", + "integrity": "sha512-zREJHjhT5vJBMZX/IUbyI9zVtVfOLiTO66MrF/3GFZYZ7T4YILW5MSkEYHceSii/KtRk+4i3RE7E1CUXA2jHcA==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/core": "^10.1.11", - "@inquirer/type": "^3.0.6", - "ansi-escapes": "^4.3.2" + "@inquirer/ansi": "^1.0.2", + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" }, "engines": { "node": ">=18" @@ -2195,22 +2807,22 @@ } }, "node_modules/@inquirer/prompts": { - "version": "7.5.1", - "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.5.1.tgz", - "integrity": "sha512-5AOrZPf2/GxZ+SDRZ5WFplCA2TAQgK3OYrXCYmJL5NaTu4ECcoWFlfUZuw7Es++6Njv7iu/8vpYJhuzxUH76Vg==", + "version": "7.10.1", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.10.1.tgz", + "integrity": "sha512-Dx/y9bCQcXLI5ooQ5KyvA4FTgeo2jYj/7plWfV5Ak5wDPKQZgudKez2ixyfz7tKXzcJciTxqLeK7R9HItwiByg==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/checkbox": "^4.1.6", - "@inquirer/confirm": "^5.1.10", - "@inquirer/editor": "^4.2.11", - "@inquirer/expand": "^4.0.13", - "@inquirer/input": "^4.1.10", - "@inquirer/number": "^3.0.13", - "@inquirer/password": "^4.0.13", - "@inquirer/rawlist": "^4.1.1", - "@inquirer/search": "^3.0.13", - "@inquirer/select": "^4.2.1" + "@inquirer/checkbox": "^4.3.2", + "@inquirer/confirm": "^5.1.21", + "@inquirer/editor": "^4.2.23", + "@inquirer/expand": "^4.0.23", + "@inquirer/input": "^4.3.1", + "@inquirer/number": "^3.0.23", + "@inquirer/password": "^4.0.23", + "@inquirer/rawlist": "^4.1.11", + "@inquirer/search": "^3.2.2", + "@inquirer/select": "^4.4.2" }, "engines": { "node": ">=18" @@ -2225,15 +2837,15 @@ } }, "node_modules/@inquirer/rawlist": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.1.1.tgz", - "integrity": "sha512-VBUC0jPN2oaOq8+krwpo/mf3n/UryDUkKog3zi+oIi8/e5hykvdntgHUB9nhDM78RubiyR1ldIOfm5ue+2DeaQ==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.1.11.tgz", + "integrity": "sha512-+LLQB8XGr3I5LZN/GuAHo+GpDJegQwuPARLChlMICNdwW7OwV2izlCSCxN6cqpL0sMXmbKbFcItJgdQq5EBXTw==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/core": "^10.1.11", - "@inquirer/type": "^3.0.6", - "yoctocolors-cjs": "^2.1.2" + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" }, "engines": { "node": ">=18" @@ -2248,16 +2860,16 @@ } }, "node_modules/@inquirer/search": { - "version": "3.0.13", - "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.0.13.tgz", - "integrity": "sha512-9g89d2c5Izok/Gw/U7KPC3f9kfe5rA1AJ24xxNZG0st+vWekSk7tB9oE+dJv5JXd0ZSijomvW0KPMoBd8qbN4g==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.2.2.tgz", + "integrity": "sha512-p2bvRfENXCZdWF/U2BXvnSI9h+tuA8iNqtUKb9UWbmLYCRQxd8WkvwWvYn+3NgYaNwdUkHytJMGG4MMLucI1kA==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/core": "^10.1.11", - "@inquirer/figures": "^1.0.11", - "@inquirer/type": "^3.0.6", - "yoctocolors-cjs": "^2.1.2" + "@inquirer/core": "^10.3.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" }, "engines": { "node": ">=18" @@ -2272,17 +2884,17 @@ } }, "node_modules/@inquirer/select": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.2.1.tgz", - "integrity": "sha512-gt1Kd5XZm+/ddemcT3m23IP8aD8rC9drRckWoP/1f7OL46Yy2FGi8DSmNjEjQKtPl6SV96Kmjbl6p713KXJ/Jg==", + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.4.2.tgz", + "integrity": "sha512-l4xMuJo55MAe+N7Qr4rX90vypFwCajSakx59qe/tMaC1aEHWLyw68wF4o0A4SLAY4E0nd+Vt+EyskeDIqu1M6w==", "dev": true, "license": "MIT", "dependencies": { - "@inquirer/core": "^10.1.11", - "@inquirer/figures": "^1.0.11", - "@inquirer/type": "^3.0.6", - "ansi-escapes": "^4.3.2", - "yoctocolors-cjs": "^2.1.2" + "@inquirer/ansi": "^1.0.2", + "@inquirer/core": "^10.3.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" }, "engines": { "node": ">=18" @@ -2297,9 +2909,9 @@ } }, "node_modules/@inquirer/type": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.6.tgz", - "integrity": "sha512-/mKVCtVpyBu3IDarv0G+59KC4stsD5mDsGpYh+GKs1NZT88Jh52+cuoA1AtLk2Q0r/quNl+1cSUyLRHBFeD0XA==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.10.tgz", + "integrity": "sha512-BvziSRxfz5Ov8ch0z/n3oijRSEcEsHnhggm4xFZe93DHcUCTlutlq9Ox4SVENAfcRD22UQq7T/atg9Wr3k09eA==", "dev": true, "license": "MIT", "engines": { @@ -4185,14 +4797,13 @@ } }, "node_modules/@tokenizer/inflate": { - "version": "0.2.7", - "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.7.tgz", - "integrity": "sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==", + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.4.1.tgz", + "integrity": "sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA==", "license": "MIT", "dependencies": { - "debug": "^4.4.0", - "fflate": "^0.8.2", - "token-types": "^6.0.0" + "debug": "^4.4.3", + "token-types": "^6.1.1" }, "engines": { "node": ">=18" @@ -4203,9 +4814,9 @@ } }, "node_modules/@tokenizer/inflate/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -7407,24 +8018,24 @@ } }, "node_modules/crawlee": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-4.0.0-beta.4.tgz", - "integrity": "sha512-ez2meEG/8QNZEDGcZebV0r4VjQgDWSkihbMVLipEeOFyWr8mois/z+wvgxUiMYQI1nwCzDXYdY0OEvlHXWJwHg==", + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-4.0.0-beta.49.tgz", + "integrity": "sha512-/+m7LJ0m4nLmdA+zzb0ZL21DjNGUjveoPs+ckbiaPpE0LT8bJBalBbzhRcmUCZHWuY5L6ncNJcS2HTGBe20VaA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@crawlee/basic": "4.0.0-beta.4", - "@crawlee/browser": "4.0.0-beta.4", - "@crawlee/browser-pool": "4.0.0-beta.4", - "@crawlee/cheerio": "4.0.0-beta.4", - "@crawlee/cli": "4.0.0-beta.4", - "@crawlee/core": "4.0.0-beta.4", - "@crawlee/http": "4.0.0-beta.4", - "@crawlee/jsdom": "4.0.0-beta.4", - "@crawlee/linkedom": "4.0.0-beta.4", - "@crawlee/playwright": "4.0.0-beta.4", - "@crawlee/puppeteer": "4.0.0-beta.4", - "@crawlee/utils": "4.0.0-beta.4", + "@crawlee/basic": "4.0.0-beta.49", + "@crawlee/browser": "4.0.0-beta.49", + "@crawlee/browser-pool": "4.0.0-beta.49", + "@crawlee/cheerio": "4.0.0-beta.49", + "@crawlee/cli": "4.0.0-beta.49", + "@crawlee/core": "4.0.0-beta.49", + "@crawlee/http": "4.0.0-beta.49", + "@crawlee/jsdom": "4.0.0-beta.49", + "@crawlee/linkedom": "4.0.0-beta.49", + "@crawlee/playwright": "4.0.0-beta.49", + "@crawlee/puppeteer": "4.0.0-beta.49", + "@crawlee/utils": "4.0.0-beta.49", "import-local": "^3.2.0", "tslib": "^2.8.1" }, @@ -7435,10 +8046,14 @@ "node": ">=22.0.0" }, "peerDependencies": { + "idcac-playwright": "*", "playwright": "*", "puppeteer": "*" }, "peerDependenciesMeta": { + "idcac-playwright": { + "optional": true + }, "playwright": { "optional": true }, @@ -7448,21 +8063,20 @@ } }, "node_modules/crawlee/node_modules/@crawlee/basic": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.4.tgz", - "integrity": "sha512-GnHrt8eylp/9MSLy5tXHrELwTA3Prm8YGVkzyO/8mfZZJqvz7L16uIAvO+ZWhg1x31MoHJKEQLlolaO6CDj2nA==", + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.49.tgz", + "integrity": "sha512-6rRchZXH1KyLvkcZpTPeuRcbEDdl/l0XOsVTK5lHxkQvj63LrE+YwlmZxEA3LViiva9soNUll7VoFfWhEL77iw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@apify/log": "^2.5.18", "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "@crawlee/utils": "4.0.0-beta.4", + "@crawlee/core": "4.0.0-beta.49", + "@crawlee/got-scraping-client": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", + "@crawlee/utils": "4.0.0-beta.49", "csv-stringify": "^6.5.2", "fs-extra": "^11.3.0", - "got-scraping": "^4.1.1", "ow": "^2.0.0", "tldts": "^7.0.6", "tslib": "^2.8.1", @@ -7473,17 +8087,17 @@ } }, "node_modules/crawlee/node_modules/@crawlee/browser": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-4.0.0-beta.4.tgz", - "integrity": "sha512-yyvzSVNn7O5Q63BHWdqtgkd3hDMn3DxSv/Vr3fiHXRUL4CiAeHZrzSSl/VgzzTMIuv7GXSVWvLkdXGGYZX2peA==", + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-4.0.0-beta.49.tgz", + "integrity": "sha512-RBzbtscHGi7qY7bn3KqsfV2flFt0XlKp7OOdGx8VsSSzdVhIb+YXFOrcdMR6DGEyvxtMSZxNZ4U8qaKM/YwOUg==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", - "@crawlee/basic": "4.0.0-beta.4", - "@crawlee/browser-pool": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "@crawlee/utils": "4.0.0-beta.4", + "@crawlee/basic": "4.0.0-beta.49", + "@crawlee/browser-pool": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", + "@crawlee/utils": "4.0.0-beta.49", "ow": "^2.0.0", "tslib": "^2.8.1", "type-fest": "^4.41.0" @@ -7505,128 +8119,61 @@ } }, "node_modules/crawlee/node_modules/@crawlee/browser-pool": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-4.0.0-beta.4.tgz", - "integrity": "sha512-OtzcPk50a2fTTQVR3euwySuXa6tEmNjsZ3mIwb20yMNlZxXdnFvrNMt/8hrJe6CV9ccWQI0GuRp8WwltR634fg==", + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-4.0.0-beta.49.tgz", + "integrity": "sha512-oroowDwagffg1Ybbn3y3Eey2i2p+rhdu9f2dBtDvH4EF+4KPpfW7Z5FG9knLTx+3OcaRp9QumZUuyFANH+wN1w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@apify/log": "^2.5.18", "@apify/timeout": "^0.3.2", - "@crawlee/core": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "fingerprint-generator": "^2.1.66", - "fingerprint-injector": "^2.1.66", + "@crawlee/core": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", + "fingerprint-generator": "^2.1.68", + "fingerprint-injector": "^2.1.68", "lodash.merge": "^4.6.2", "nanoid": "^5.1.5", "ow": "^2.0.0", "p-limit": "^6.2.0", "proxy-chain": "^2.5.8", "quick-lru": "^7.0.1", - "tiny-typed-emitter": "^2.1.0", - "tslib": "^2.8.1" - }, - "engines": { - "node": ">=22.0.0" - }, - "peerDependencies": { - "playwright": "*", - "puppeteer": "*" - }, - "peerDependenciesMeta": { - "playwright": { - "optional": true - }, - "puppeteer": { - "optional": true - } - } - }, - "node_modules/crawlee/node_modules/@crawlee/cheerio": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-4.0.0-beta.4.tgz", - "integrity": "sha512-ZQsJBiAZlR05lLm6AsNuwat+qhqaA9UYiAGCY8DtwAUK3EGM0gAGB0WHd7e6tzD3W0yFMQGgG04XIitYVttVyQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@crawlee/http": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "@crawlee/utils": "4.0.0-beta.4", - "cheerio": "^1.0.0", - "htmlparser2": "^10.0.0", - "tslib": "^2.8.1" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/crawlee/node_modules/@crawlee/http": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-4.0.0-beta.4.tgz", - "integrity": "sha512-G0MWHpYcSUO73b+ZST5HT7XtiiYkONaMZ+cgKm3TMbtOh0zuaOx8SdcQTNjeMnJdxmSpbmLbQl2ihynXGxhQdA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@apify/timeout": "^0.3.2", - "@apify/utilities": "^2.15.5", - "@crawlee/basic": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "@crawlee/utils": "4.0.0-beta.4", - "@types/content-type": "^1.1.8", - "cheerio": "^1.0.0", - "content-type": "^1.0.5", - "got-scraping": "^4.1.1", - "iconv-lite": "^0.6.3", - "mime-types": "^3.0.1", - "ow": "^2.0.0", - "tslib": "^2.8.1", - "type-fest": "^4.41.0" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/crawlee/node_modules/@crawlee/jsdom": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-4.0.0-beta.4.tgz", - "integrity": "sha512-nBrVYyz2kJCSF/s8yWpWf90L7cr7cpByJacnpNpQTwp8+2QSRrJmU60uddFeQEhC2nWq0WnWsuMA1Mx7cR25qw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@apify/timeout": "^0.3.2", - "@apify/utilities": "^2.15.5", - "@crawlee/http": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "@crawlee/utils": "4.0.0-beta.4", - "@types/jsdom": "^21.1.7", - "cheerio": "^1.0.0", - "jsdom": "^26.1.0", - "ow": "^2.0.0", + "tiny-typed-emitter": "^2.1.0", "tslib": "^2.8.1" }, "engines": { "node": ">=22.0.0" + }, + "peerDependencies": { + "playwright": "*", + "puppeteer": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + } } }, "node_modules/crawlee/node_modules/@crawlee/playwright": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-4.0.0-beta.4.tgz", - "integrity": "sha512-mAynANJHAoxPbOnxkyDnLIsS+Ujj40q2fApJMWgdz95Ti0Kg/HJBacjeRV+cGZpA/brB8FV7J4/ZGu26laip1g==", + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-4.0.0-beta.49.tgz", + "integrity": "sha512-ozU/eUAgnW0AhyWYUe4Kvipqfl8JyJ2ZZoB5jqWG7PG0zmgEx9CQ5s+aMopl0zmZu31dW7nnZk9HQZa71ZYcTg==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.3", - "@apify/log": "^2.5.18", "@apify/timeout": "^0.3.2", - "@crawlee/browser": "4.0.0-beta.4", - "@crawlee/browser-pool": "4.0.0-beta.4", - "@crawlee/core": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "@crawlee/utils": "4.0.0-beta.4", + "@crawlee/basic": "4.0.0-beta.49", + "@crawlee/browser": "4.0.0-beta.49", + "@crawlee/browser-pool": "4.0.0-beta.49", + "@crawlee/cheerio": "4.0.0-beta.49", + "@crawlee/core": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", + "@crawlee/utils": "4.0.0-beta.49", "cheerio": "^1.0.0", "idcac-playwright": "^0.1.3", "jquery": "^3.7.1", - "lodash.isequal": "^4.5.0", "ml-logistic-regression": "^2.0.0", "ml-matrix": "^6.12.1", "ow": "^2.0.0", @@ -7637,30 +8184,41 @@ "node": ">=22.0.0" }, "peerDependencies": { + "idcac-playwright": "^0.2.0", "playwright": "*" }, "peerDependenciesMeta": { + "idcac-playwright": { + "optional": true + }, "playwright": { "optional": true } } }, + "node_modules/crawlee/node_modules/@crawlee/playwright/node_modules/idcac-playwright": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/idcac-playwright/-/idcac-playwright-0.1.3.tgz", + "integrity": "sha512-VVYQ4sv6OrUJKVzYaIP1hq0qAHd1O22HW5LnL1Wf6zkrLStQ/QEg4iJ0rllIOEpd+Rmm+635AJD59A+Vw+2PgQ==", + "dev": true, + "license": "ISC" + }, "node_modules/crawlee/node_modules/@crawlee/puppeteer": { - "version": "4.0.0-beta.4", - "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-4.0.0-beta.4.tgz", - "integrity": "sha512-LiXfvOz3RXmtzVte/vk4PkKEdBzoXrmQmTOeHHtbD7z9AYMaHGI1wimEuu5Z2wYNEQ0n0yi7fkXiiajxsC54sg==", + "version": "4.0.0-beta.49", + "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-4.0.0-beta.49.tgz", + "integrity": "sha512-fpMaaXqSNr86FTWVvkWvNXlKRApRtURo37U9AHVLtwQTyBbmgFeETnfjHauAlEj29mFaRgk4lyKXEcDmWU/Psw==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.3", - "@apify/log": "^2.5.18", - "@crawlee/browser": "4.0.0-beta.4", - "@crawlee/browser-pool": "4.0.0-beta.4", - "@crawlee/types": "4.0.0-beta.4", - "@crawlee/utils": "4.0.0-beta.4", + "@crawlee/browser": "4.0.0-beta.49", + "@crawlee/browser-pool": "4.0.0-beta.49", + "@crawlee/core": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.49", + "@crawlee/utils": "4.0.0-beta.49", "cheerio": "^1.0.0", "devtools-protocol": "*", - "idcac-playwright": "^0.1.3", + "idcac-playwright": "^0.2.0", "jquery": "^3.7.1", "ow": "^2.0.0", "tslib": "^2.8.1" @@ -7669,9 +8227,13 @@ "node": ">=22.0.0" }, "peerDependencies": { + "idcac-playwright": "^0.2.0", "puppeteer": "*" }, "peerDependenciesMeta": { + "idcac-playwright": { + "optional": true + }, "puppeteer": { "optional": true } @@ -7704,51 +8266,31 @@ } }, "node_modules/crawlee/node_modules/cheerio": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", - "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", "dev": true, "license": "MIT", "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", - "domutils": "^3.1.0", - "encoding-sniffer": "^0.2.0", - "htmlparser2": "^9.1.0", - "parse5": "^7.1.2", - "parse5-htmlparser2-tree-adapter": "^7.0.0", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", - "undici": "^6.19.5", + "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" }, "engines": { - "node": ">=18.17" + "node": ">=20.18.1" }, "funding": { "url": "https://github.com/cheeriojs/cheerio?sponsor=1" } }, - "node_modules/crawlee/node_modules/cheerio/node_modules/htmlparser2": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", - "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", - "dev": true, - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.1.0", - "entities": "^4.5.0" - } - }, "node_modules/crawlee/node_modules/dot-prop": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-8.0.2.tgz", @@ -7778,10 +8320,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/crawlee/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/crawlee/node_modules/htmlparser2": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz", - "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", "dev": true, "funding": [ "https://github.com/fb55/htmlparser2?sponsor=1", @@ -7794,50 +8349,14 @@ "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", - "domutils": "^3.2.1", - "entities": "^6.0.0" - } - }, - "node_modules/crawlee/node_modules/htmlparser2/node_modules/entities": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.0.tgz", - "integrity": "sha512-aKstq2TDOndCn4diEyp9Uq/Flu2i1GlLkc6XIDQSDMuaFE3OPW5OphLCyQ5SpSJZTb4reN+kTcYru5yIfXoRPw==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/crawlee/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/crawlee/node_modules/mime-types": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.1.tgz", - "integrity": "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">= 0.6" + "domutils": "^3.2.2", + "entities": "^7.0.1" } }, "node_modules/crawlee/node_modules/nanoid": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz", - "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==", + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.9.tgz", + "integrity": "sha512-ZUvP7KeBLe3OZ1ypw6dI/TzYJuvHP77IM4Ry73waSQTLn8/g8rpdjfyVAh7t1/+FjBtG4lCP42MEbDxOsRpBMw==", "dev": true, "funding": [ { @@ -7891,9 +8410,9 @@ } }, "node_modules/crawlee/node_modules/quick-lru": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.0.1.tgz", - "integrity": "sha512-kLjThirJMkWKutUKbZ8ViqFc09tDQhlbQo2MNuVeLWbRauqYP96Sm6nzlQ24F0HFjUNZ4i9+AgldJ9H6DZXi7g==", + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.3.0.tgz", + "integrity": "sha512-k9lSsjl36EJdK7I06v7APZCbyGT2vMTsYSRX1Q2nbYmnkBqgUhRkAuzH08Ciotteu/PLJmIF2+tti7o3C/ts2g==", "dev": true, "license": "MIT", "engines": { @@ -7903,10 +8422,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/crawlee/node_modules/undici": { + "version": "7.25.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", + "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/crawlee/node_modules/yocto-queue": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.1.tgz", - "integrity": "sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", "dev": true, "license": "MIT", "engines": { @@ -8540,9 +9069,9 @@ } }, "node_modules/encoding-sniffer": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.0.tgz", - "integrity": "sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==", + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", + "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", "license": "MIT", "dependencies": { "iconv-lite": "^0.6.3", @@ -9571,25 +10100,6 @@ "pend": "~1.2.0" } }, - "node_modules/fflate": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", - "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", - "license": "MIT" - }, - "node_modules/figlet": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/figlet/-/figlet-1.8.1.tgz", - "integrity": "sha512-kEC3Sme+YvA8Hkibv0NR1oClGcWia0VB2fC1SlMy027cwe795Xx40Xiv/nw/iFAwQLupymWh+uhAAErn/7hwPg==", - "dev": true, - "license": "MIT", - "bin": { - "figlet": "bin/index.js" - }, - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -9630,18 +10140,18 @@ } }, "node_modules/file-type": { - "version": "20.5.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-20.5.0.tgz", - "integrity": "sha512-BfHZtG/l9iMm4Ecianu7P8HRD2tBHLtjXinm4X62XBOYzi7CYA7jyqfJzOvXHqzVrVPYqBo2/GvbARMaaJkKVg==", + "version": "21.3.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-21.3.4.tgz", + "integrity": "sha512-Ievi/yy8DS3ygGvT47PjSfdFoX+2isQueoYP1cntFW1JLYAuS4GD7NUPGg4zv2iZfV52uDyk5w5Z0TdpRS6Q1g==", "license": "MIT", "dependencies": { - "@tokenizer/inflate": "^0.2.6", - "strtok3": "^10.2.0", - "token-types": "^6.0.0", + "@tokenizer/inflate": "^0.4.1", + "strtok3": "^10.3.4", + "token-types": "^6.1.1", "uint8array-extras": "^1.4.0" }, "engines": { - "node": ">=18" + "node": ">=20" }, "funding": { "url": "https://github.com/sindresorhus/file-type?sponsor=1" @@ -9712,14 +10222,14 @@ } }, "node_modules/fingerprint-generator": { - "version": "2.1.66", - "resolved": "https://registry.npmjs.org/fingerprint-generator/-/fingerprint-generator-2.1.66.tgz", - "integrity": "sha512-2CvoY+OPcCOWkoIMQim80uNH+ED1+2rM9QXIcSih7ovBMLOmyr3Sp9IOtfccd05QlGDzulU2M9Oav8jOgTlCBA==", + "version": "2.1.82", + "resolved": "https://registry.npmjs.org/fingerprint-generator/-/fingerprint-generator-2.1.82.tgz", + "integrity": "sha512-5Z/yCKW324pMyMarpIKe/QPdkrFWKNJv3ktdU+fXHri80+HAwNE6QhMvEvsMkK9Q8DeCXZlpPHV77UBa1nFb4A==", "dev": true, "license": "Apache-2.0", "dependencies": { - "generative-bayesian-network": "^2.1.66", - "header-generator": "^2.1.66", + "generative-bayesian-network": "^2.1.82", + "header-generator": "^2.1.82", "tslib": "^2.4.0" }, "engines": { @@ -9727,13 +10237,13 @@ } }, "node_modules/fingerprint-injector": { - "version": "2.1.66", - "resolved": "https://registry.npmjs.org/fingerprint-injector/-/fingerprint-injector-2.1.66.tgz", - "integrity": "sha512-h5llsoG0xoDeEo2czjzvo1niEU0xgCMwhs5/jtAxiBf7IiP/wW1Is3DJMEB+4V4PwIvqNQqLlnD07X23D7tErA==", + "version": "2.1.82", + "resolved": "https://registry.npmjs.org/fingerprint-injector/-/fingerprint-injector-2.1.82.tgz", + "integrity": "sha512-FN7W1wbhHk2PBCF6wpBEcFnmOdGUItZnbpVBtYVcQ1/iGM0skNUDqJyH1YOjmpQiqEl2Rhh7qWNXYsivjsT+tg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "fingerprint-generator": "^2.1.66", + "fingerprint-generator": "^2.1.82", "tslib": "^2.4.0" }, "engines": { @@ -10029,9 +10539,9 @@ } }, "node_modules/generative-bayesian-network": { - "version": "2.1.66", - "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.1.66.tgz", - "integrity": "sha512-gbBsyaaEJj/LHp3473TQrMDdcKiRzI8Sn2CbcG/lwONZkp0n9/ChC1mjzcbZQtHHCuqjn+JouSSbzLeepUMbuw==", + "version": "2.1.82", + "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.1.82.tgz", + "integrity": "sha512-DH4NrmQheoMaJErdVv2IzaqkbOYSDQZmiZTV6UPDJYRDK2EyPpIQ88XRcYdPeFrUjS1N0Jj25H3HUywoJ1dbow==", "license": "Apache-2.0", "dependencies": { "adm-zip": "^0.5.9", @@ -10857,9 +11367,9 @@ } }, "node_modules/got-scraping": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/got-scraping/-/got-scraping-4.1.1.tgz", - "integrity": "sha512-MbT+NMMU4VgvOg2tFIPOSIrMfH986fm0LJ17RxBLKlyTs3gh3xIMETpe+zdPaXY7tH1j6YYeqtfG0TnVMx6V2g==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/got-scraping/-/got-scraping-4.2.1.tgz", + "integrity": "sha512-rhOlO1L4H4Cm31smHJqPtAaXOUrhSKsiTrbZSHKFQW1E/mkTDopnHHpRnXJpqzE0faj+zPsVQnyifIqO+K+cLQ==", "license": "Apache-2.0", "dependencies": { "got": "^14.2.1", @@ -11001,29 +11511,6 @@ "node": ">=6" } }, - "node_modules/has-ansi": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", - "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^2.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/has-ansi/node_modules/ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/has-bigints": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.1.0.tgz", @@ -11123,13 +11610,13 @@ } }, "node_modules/header-generator": { - "version": "2.1.66", - "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.1.66.tgz", - "integrity": "sha512-g0jd79o0CyzyK0Jega4pAG1eJhykhPNfBLpOnUINtX2YkToVvRSBZ+B2wtmIjqwKHXK8DNWxylKuXnZmLs1yMQ==", + "version": "2.1.82", + "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.1.82.tgz", + "integrity": "sha512-4NjPB0+bAKjPoponSmTOkK58IEF2W22sOJA5O48k/MxbCZgOm+jrU4WVR53Z2I6xFgIPkVrQmKtt1LAbWtfqXw==", "license": "Apache-2.0", "dependencies": { "browserslist": "^4.21.1", - "generative-bayesian-network": "^2.1.66", + "generative-bayesian-network": "^2.1.82", "ow": "^0.28.1", "tslib": "^2.4.0" }, @@ -11301,11 +11788,11 @@ } }, "node_modules/idcac-playwright": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/idcac-playwright/-/idcac-playwright-0.1.3.tgz", - "integrity": "sha512-VVYQ4sv6OrUJKVzYaIP1hq0qAHd1O22HW5LnL1Wf6zkrLStQ/QEg4iJ0rllIOEpd+Rmm+635AJD59A+Vw+2PgQ==", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/idcac-playwright/-/idcac-playwright-0.2.0.tgz", + "integrity": "sha512-qJH7vQgq3TKnhea/3Z3jlEJL7NC9vK9BkLClAzQHVRepBtq1fWfSI4fSuMKcPq7nDUTTlIEIS+vU+GRwwR1BXw==", "dev": true, - "license": "ISC" + "license": "GPL-3.0-only" }, "node_modules/identifier-regex": { "version": "1.0.0", @@ -11608,9 +12095,9 @@ } }, "node_modules/is-any-array": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-2.0.1.tgz", - "integrity": "sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-3.0.0.tgz", + "integrity": "sha512-o4h+tylWykC4BD1vaejp6gDxoM13bwW8FGuNs4yIKpj8xbBJcRxJx8vZpq0dCr7ZDEfeKjmsi/euolKhX6f/ww==", "dev": true, "license": "MIT" }, @@ -12964,9 +13451,9 @@ } }, "node_modules/linkedom": { - "version": "0.18.10", - "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.10.tgz", - "integrity": "sha512-ESCqVAtme2GI3zZnlVRidiydByV6WmPlmKeFzFVQslADiAO2Wi+H6xL/5kr/pUOESjEoVb2Eb3cYFJ/TQhQOWA==", + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.12.tgz", + "integrity": "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==", "dev": true, "license": "ISC", "dependencies": { @@ -12975,12 +13462,23 @@ "html-escaper": "^3.0.3", "htmlparser2": "^10.0.0", "uhyphen": "^0.2.0" + }, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "canvas": ">= 2" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } } }, "node_modules/linkedom/node_modules/entities": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.0.tgz", - "integrity": "sha512-aKstq2TDOndCn4diEyp9Uq/Flu2i1GlLkc6XIDQSDMuaFE3OPW5OphLCyQ5SpSJZTb4reN+kTcYru5yIfXoRPw==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", "dev": true, "license": "BSD-2-Clause", "engines": { @@ -12991,9 +13489,9 @@ } }, "node_modules/linkedom/node_modules/htmlparser2": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz", - "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", "dev": true, "funding": [ "https://github.com/fb55/htmlparser2?sponsor=1", @@ -13006,8 +13504,8 @@ "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", - "domutils": "^3.2.1", - "entities": "^6.0.0" + "domutils": "^3.2.2", + "entities": "^7.0.1" } }, "node_modules/lint-staged": { @@ -13991,35 +14489,35 @@ } }, "node_modules/ml-array-max": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/ml-array-max/-/ml-array-max-1.2.4.tgz", - "integrity": "sha512-BlEeg80jI0tW6WaPyGxf5Sa4sqvcyY6lbSn5Vcv44lp1I2GR6AWojfUvLnGTNsIXrZ8uqWmo8VcG1WpkI2ONMQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ml-array-max/-/ml-array-max-2.0.0.tgz", + "integrity": "sha512-QQZ4kENwpWmyNb98UXRDFXrmtIXuXtt1+bSbda/2KA85+F+rrJP8hZk6QOkCQXM2Th9mUDYdq/PNByPdT9ID4A==", "dev": true, "license": "MIT", "dependencies": { - "is-any-array": "^2.0.0" + "is-any-array": "^3.0.0" } }, "node_modules/ml-array-min": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/ml-array-min/-/ml-array-min-1.2.3.tgz", - "integrity": "sha512-VcZ5f3VZ1iihtrGvgfh/q0XlMobG6GQ8FsNyQXD3T+IlstDv85g8kfV0xUG1QPRO/t21aukaJowDzMTc7j5V6Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ml-array-min/-/ml-array-min-2.0.0.tgz", + "integrity": "sha512-GRj6Ky6sW9vGL6yIjgsHmXZ9YgrdmcQ8nCxPqEGeKc6dkfYg1XDYxGFxADUjNuZyoCd5PUscWAS4N+cFaX6hFg==", "dev": true, "license": "MIT", "dependencies": { - "is-any-array": "^2.0.0" + "is-any-array": "^3.0.0" } }, "node_modules/ml-array-rescale": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/ml-array-rescale/-/ml-array-rescale-1.3.7.tgz", - "integrity": "sha512-48NGChTouvEo9KBctDfHC3udWnQKNKEWN0ziELvY3KG25GR5cA8K8wNVzracsqSW1QEkAXjTNx+ycgAv06/1mQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ml-array-rescale/-/ml-array-rescale-2.0.0.tgz", + "integrity": "sha512-2GGtKfSno94/kIloWGvpp/U5Q5vLvLrza+SAaGsLeo6Xj4mEbA6Gqx+oTfZFkxnd1grT2X007HfJNs3T5BsiVg==", "dev": true, "license": "MIT", "dependencies": { - "is-any-array": "^2.0.0", - "ml-array-max": "^1.2.4", - "ml-array-min": "^1.2.3" + "is-any-array": "^3.0.0", + "ml-array-max": "^2.0.0", + "ml-array-min": "^2.0.0" } }, "node_modules/ml-logistic-regression": { @@ -14033,14 +14531,14 @@ } }, "node_modules/ml-matrix": { - "version": "6.12.1", - "resolved": "https://registry.npmjs.org/ml-matrix/-/ml-matrix-6.12.1.tgz", - "integrity": "sha512-TJ+8eOFdp+INvzR4zAuwBQJznDUfktMtOB6g/hUcGh3rcyjxbz4Te57Pgri8Q9bhSQ7Zys4IYOGhFdnlgeB6Lw==", + "version": "6.12.2", + "resolved": "https://registry.npmjs.org/ml-matrix/-/ml-matrix-6.12.2.tgz", + "integrity": "sha512-GC+BnW+pBh8Auap8goAxY0senAmF0IEoc3HNVSfnfbvGw0buuDIYb9kAKMS1l+GiwJ1rfK2bzJ8IHhwjzATSFA==", "dev": true, "license": "MIT", "dependencies": { - "is-any-array": "^2.0.1", - "ml-array-rescale": "^1.3.7" + "is-any-array": "^3.0.0", + "ml-array-rescale": "^2.0.0" } }, "node_modules/modify-values": { @@ -15307,15 +15805,6 @@ "node": ">=6" } }, - "node_modules/parent-require": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/parent-require/-/parent-require-1.0.0.tgz", - "integrity": "sha512-2MXDNZC4aXdkkap+rBBMv0lUsfJqvX5/2FiYYnfCnorZt3Pk06/IOR5KeaoghgS2w07MLWgjbsnyaq6PdHn2LQ==", - "dev": true, - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/parse-conflict-json": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/parse-conflict-json/-/parse-conflict-json-3.0.1.tgz", @@ -15519,19 +16008,6 @@ "through": "~2.3" } }, - "node_modules/peek-readable": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-7.0.0.tgz", - "integrity": "sha512-nri2TO5JE3/mRryik9LlHFT53cgHfRK0Lt0BAZQXku/AW3E6XLt2GaY8siWi7dvW/m1z0ecn+J+bpDa9ZN3IsQ==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -15965,9 +16441,9 @@ } }, "node_modules/proxy-chain": { - "version": "2.5.8", - "resolved": "https://registry.npmjs.org/proxy-chain/-/proxy-chain-2.5.8.tgz", - "integrity": "sha512-TqKOYRD/1Gga/JhiwmdYHJoj0zMJkKGofQ9bHQuSm+vexczatt81fkUHTVMyci+2mWczXiTNv1Eom+2v3Da5og==", + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/proxy-chain/-/proxy-chain-2.7.1.tgz", + "integrity": "sha512-LtXu0miohJYrHWJxv8wA6EoGreRcX1hxKb7qlE1pMFH+BXE7bqMvpyhzR/JvR6M5SzYKzyHFpvfmYJrZeMtwAg==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -17611,13 +18087,12 @@ } }, "node_modules/strtok3": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.2.2.tgz", - "integrity": "sha512-Xt18+h4s7Z8xyZ0tmBoRmzxcop97R4BAh+dXouUDCYn+Em+1P3qpkUfI5ueWLT8ynC5hZ+q4iPEmGG1urvQGBg==", + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.5.tgz", + "integrity": "sha512-ki4hZQfh5rX0QDLLkOCj+h+CVNkqmp/CMf8v8kZpkNVK6jGQooMytqzLZYUVYIZcFZ6yDB70EfD8POcFXiF5oA==", "license": "MIT", "dependencies": { - "@tokenizer/token": "^0.3.0", - "peek-readable": "^7.0.0" + "@tokenizer/token": "^0.3.0" }, "engines": { "node": ">=18" @@ -18020,11 +18495,12 @@ } }, "node_modules/token-types": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.0.0.tgz", - "integrity": "sha512-lbDrTLVsHhOMljPscd0yitpozq7Ga2M5Cvez5AjGg8GASBjtt6iERCAJ93yommPmz62fb45oFIXHEZ3u9bfJEA==", + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.1.2.tgz", + "integrity": "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==", "license": "MIT", "dependencies": { + "@borewit/text-codec": "^0.2.1", "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" }, @@ -18040,6 +18516,7 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "tldts": "^6.1.32" @@ -18052,6 +18529,7 @@ "version": "6.1.86", "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", + "dev": true, "license": "MIT", "dependencies": { "tldts-core": "^6.1.86" @@ -18064,6 +18542,7 @@ "version": "6.1.86", "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", + "dev": true, "license": "MIT" }, "node_modules/tr46": { @@ -18521,9 +19000,9 @@ "license": "ISC" }, "node_modules/uint8array-extras": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.4.0.tgz", - "integrity": "sha512-ZPtzy0hu4cZjv3z5NW9gfKnNLjoz4y6uv4HlelAjDK7sY/xOkKZv9xK/WQpcsBB3jEybChz9DPC2U/+cusjJVQ==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.5.0.tgz", + "integrity": "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==", "license": "MIT", "engines": { "node": ">=18" @@ -19894,88 +20373,6 @@ "node": ">= 14.6" } }, - "node_modules/yargonaut": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/yargonaut/-/yargonaut-1.1.4.tgz", - "integrity": "sha512-rHgFmbgXAAzl+1nngqOcwEljqHGG9uUZoPjsdZEs1w5JW9RXYzrSvH/u70C1JE5qFi0qjsdhnUX/dJRpWqitSA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "chalk": "^1.1.1", - "figlet": "^1.1.1", - "parent-require": "^1.0.0" - } - }, - "node_modules/yargonaut/node_modules/ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/yargonaut/node_modules/ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/yargonaut/node_modules/chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/yargonaut/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/yargonaut/node_modules/strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^2.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/yargonaut/node_modules/supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", @@ -20030,9 +20427,9 @@ } }, "node_modules/yoctocolors-cjs": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.2.tgz", - "integrity": "sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.3.tgz", + "integrity": "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw==", "dev": true, "license": "MIT", "engines": { @@ -20185,9 +20582,9 @@ "@apify/log": "^2.5.18", "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "^4.0.0-beta.0", - "@crawlee/types": "^4.0.0-beta.0", - "@crawlee/utils": "^4.0.0-beta.0", + "@crawlee/core": "^4.0.0-beta.49", + "@crawlee/types": "^4.0.0-beta.49", + "@crawlee/utils": "^4.0.0-beta.49", "apify-client": "^2.12.4", "fs-extra": "^11.3.0", "got-scraping": "^4.1.1", diff --git a/package.json b/package.json index 22528a5dce..9fed883e8f 100644 --- a/package.json +++ b/package.json @@ -78,10 +78,10 @@ "@types/tough-cookie": "^4.0.5", "@types/ws": "^8.18.1", "commitlint": "^19.8.1", - "crawlee": "^4.0.0-beta.0", - "@crawlee/core": "^4.0.0-beta.0", - "@crawlee/types": "^4.0.0-beta.0", - "@crawlee/utils": "^4.0.0-beta.0", + "crawlee": "^4.0.0-beta.49", + "@crawlee/core": "^4.0.0-beta.49", + "@crawlee/types": "^4.0.0-beta.49", + "@crawlee/utils": "^4.0.0-beta.49", "eslint": "^9.27.0", "eslint-config-prettier": "^10.1.5", "fs-extra": "^11.3.0", diff --git a/packages/apify/package.json b/packages/apify/package.json index ebb8146b6d..7cea7e856d 100644 --- a/packages/apify/package.json +++ b/packages/apify/package.json @@ -53,9 +53,9 @@ "@apify/log": "^2.5.18", "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "^4.0.0-beta.0", - "@crawlee/types": "^4.0.0-beta.0", - "@crawlee/utils": "^4.0.0-beta.0", + "@crawlee/core": "^4.0.0-beta.49", + "@crawlee/types": "^4.0.0-beta.49", + "@crawlee/utils": "^4.0.0-beta.49", "apify-client": "^2.12.4", "fs-extra": "^11.3.0", "got-scraping": "^4.1.1", diff --git a/packages/apify/src/actor.ts b/packages/apify/src/actor.ts index f75a3a64fd..86fd1b9dce 100644 --- a/packages/apify/src/actor.ts +++ b/packages/apify/src/actor.ts @@ -1467,9 +1467,6 @@ export class Actor { * @ignore */ newClient(options: ApifyClientOptions = {}): ApifyClient { - const { storageDir, ...storageClientOptions } = (this.config.get( - 'storageClientOptions', - ) ?? {}) as Dictionary; const { apifyVersion, crawleeVersion } = getSystemInfo(); return new ApifyClient({ baseUrl: this.config.apiBaseUrl, @@ -1478,7 +1475,6 @@ export class Actor { `SDK/${apifyVersion}`, `Crawlee/${crawleeVersion}`, ], - ...storageClientOptions, ...options, // allow overriding the instance configuration }); } diff --git a/packages/apify/src/configuration.ts b/packages/apify/src/configuration.ts index 4228f5d93b..d50aec13ec 100644 --- a/packages/apify/src/configuration.ts +++ b/packages/apify/src/configuration.ts @@ -4,6 +4,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import type { ConfigField, FieldsInput, FieldsOutput } from '@crawlee/core'; import { coerceBoolean, + coerceNumber, Configuration as CoreConfiguration, crawleeConfigFields, field, @@ -17,11 +18,6 @@ import { LOCAL_APIFY_ENV_VARS, } from '@apify/consts'; -const coerceNumber = z.preprocess((val) => { - if (typeof val === 'string') return Number(val); - return val; -}, z.number()); - // --- isAtHome check (simple env var presence) --- const isAtHome = !!process.env[APIFY_ENV_VARS.IS_AT_HOME]; From a99be15ebd559dad21cae70ac96f224ef488f74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 17:50:36 +0200 Subject: [PATCH 05/34] fix(actor): adapt PlatformEventManager to crawlee v4 EventManager API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crawlee v4's `EventManager` constructor now requires `EventManagerOptions` (just `persistStateIntervalMillis`), and the base class no longer carries a `config` field — the previous `override readonly config` pattern is no longer valid. - Drop the `override` and store `config` as own readonly property. - Forward `persistStateIntervalMillis` to `super()`. - Add a `fromConfig()` factory mirroring `LocalEventManager.fromConfig()` so the SDK plays nicely with the new ServiceLocator-driven init path. Stacked on #583 (config redesign); rebases onto v4 once that lands. --- packages/apify/src/platform_event_manager.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/apify/src/platform_event_manager.ts b/packages/apify/src/platform_event_manager.ts index 4385eebd60..fa15013c85 100644 --- a/packages/apify/src/platform_event_manager.ts +++ b/packages/apify/src/platform_event_manager.ts @@ -1,4 +1,4 @@ -import { EventManager, EventType } from '@crawlee/core'; +import { EventManager, EventType, serviceLocator } from '@crawlee/core'; import { WebSocket } from 'ws'; import { ACTOR_ENV_VARS, ACTOR_EVENT_NAMES } from '@apify/consts'; @@ -48,8 +48,19 @@ export class PlatformEventManager extends EventManager { /** Websocket connection to Actor events. */ private eventsWs?: WebSocket; - constructor(override readonly config = Configuration.getGlobalConfig()) { - super(); + constructor(readonly config: Configuration = Configuration.getGlobalConfig() as Configuration) { + super({ persistStateIntervalMillis: config.persistStateIntervalMillis }); + } + + /** + * Creates a `PlatformEventManager` from a (resolved) Configuration, mirroring + * `LocalEventManager.fromConfig()` from crawlee. Falls back to the global + * configuration if none is provided. + */ + static fromConfig(config?: Configuration): PlatformEventManager { + return new PlatformEventManager( + config ?? (serviceLocator.getConfiguration() as Configuration), + ); } /** @@ -62,7 +73,7 @@ export class PlatformEventManager extends EventManager { } await super.init(); - const eventsWsUrl = (this.config as Configuration).actorEventsWsUrl; + const eventsWsUrl = this.config.actorEventsWsUrl; // Locally there is no web socket to connect, so just print a log message. if (!eventsWsUrl) { From ccf96c5bd9fb758f2e39725ba084ea7646735076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 17:56:43 +0200 Subject: [PATCH 06/34] fix: adapt SDK storage layer to crawlee v4 StorageClient interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crawlee v4 reshaped its `StorageClient` interface (async factory methods that accept `id` *or* `name`), removed the cached `storageObject` from `KeyValueStore`, and made `getPublicUrl` async. The existing SDK code targeted the v3 shape and no longer compiles. Changes: - New `ApifyStorageClient` adapter wraps `apify-client`'s legacy `dataset()/keyValueStore()/requestQueue()` accessors and exposes the `createDatasetClient/createKeyValueStoreClient/createRequestQueueClient` factories crawlee now expects. Names are resolved to IDs via the collection `getOrCreate(name)` calls. apify-client's resource clients don't yet implement v4-only members like `getMetadata` / `getRecordPublicUrl`; the adapter casts through with a TODO comment so the structural alignment can land separately upstream. - `Actor.init` and `_openStorage` now wrap `this.apifyClient` in `ApifyStorageClient` before handing it to crawlee. - `KeyValueStore.getPublicUrl` is now async; the per-store `urlSigningSecretKey` is fetched on demand via the (private) `client.getMetadata()` instead of the removed `storageObject` cache. URL-signing behaviour for platform-mode reads is preserved. - `Actor.openRequestQueue` reads `totalRequestCount` via the new `client.getMetadata()` (the old `client.get()` was dropped). - `StorageManager.openStorage` is now `(class, id?, client?)` — removed the trailing `this.config` argument. Stacked on #583 (config redesign); rebases onto v4 once that lands. --- packages/apify/src/actor.ts | 18 +++--- packages/apify/src/apify_storage_client.ts | 72 ++++++++++++++++++++++ packages/apify/src/key_value_store.ts | 38 ++++++++---- 3 files changed, 106 insertions(+), 22 deletions(-) create mode 100644 packages/apify/src/apify_storage_client.ts diff --git a/packages/apify/src/actor.ts b/packages/apify/src/actor.ts index 86fd1b9dce..2905b377bb 100644 --- a/packages/apify/src/actor.ts +++ b/packages/apify/src/actor.ts @@ -47,6 +47,7 @@ import type { ChargeOptions, ChargeResult } from './charging.js'; import { ChargingManager } from './charging.js'; import type { ConfigurationOptions } from './configuration.js'; import { Configuration } from './configuration.js'; +import { ApifyStorageClient } from './apify_storage_client.js'; import { KeyValueStore } from './key_value_store.js'; import { PlatformEventManager } from './platform_event_manager.js'; import type { ProxyConfigurationOptions } from './proxy_configuration.js'; @@ -495,7 +496,9 @@ export class Actor { if (this.isAtHome()) { // availableMemoryRatio and disableBrowserSandbox are now set via // conditional defaults in the Configuration constructor (isAtHome check) - serviceLocator.setStorageClient(this.apifyClient); + serviceLocator.setStorageClient( + new ApifyStorageClient(this.apifyClient), + ); serviceLocator.setEventManager(this.eventManager); } else if (options.storage) { serviceLocator.setStorageClient(options.storage); @@ -1310,7 +1313,7 @@ export class Actor { // eslint-disable-next-line dot-notation queue['initialCount'] = - (await queue.client.get())?.totalRequestCount ?? 0; + (await queue.client.getMetadata())?.totalRequestCount ?? 0; return queue; } @@ -2232,13 +2235,10 @@ export class Actor { id?: string, options: OpenStorageOptions = {}, ) { - const client = options.forceCloud ? this.apifyClient : undefined; - return StorageManager.openStorage( - storageClass, - id, - client, - this.config, - ); + const client = options.forceCloud + ? new ApifyStorageClient(this.apifyClient) + : undefined; + return StorageManager.openStorage(storageClass, id, client); } private _ensureActorInit(methodCalled: string) { diff --git a/packages/apify/src/apify_storage_client.ts b/packages/apify/src/apify_storage_client.ts new file mode 100644 index 0000000000..4926452fc4 --- /dev/null +++ b/packages/apify/src/apify_storage_client.ts @@ -0,0 +1,72 @@ +import type { + CreateDatasetClientOptions, + CreateKeyValueStoreClientOptions, + CreateRequestQueueClientOptions, + DatasetClient, + KeyValueStoreClient, + RequestQueueClient, + StorageClient, +} from '@crawlee/types'; +import type { ApifyClient } from 'apify-client'; + +/** + * Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`, + * `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's + * `StorageClient` interface (async factory methods accepting either an `id` + * or a `name`). + * + * When only a `name` is provided, we resolve it to a concrete ID via the + * collection client's `getOrCreate(name)` — matching the behaviour the SDK + * relied on in v3 when storages were opened by name. + */ +export class ApifyStorageClient implements StorageClient { + constructor(private readonly client: ApifyClient) {} + + async createDatasetClient( + options?: CreateDatasetClientOptions, + ): Promise { + const id = + options?.id ?? + (options?.name + ? (await this.client.datasets().getOrCreate(options.name)).id + : undefined); + // apify-client's resource clients overlap with `@crawlee/types`' shapes + // but don't yet implement the v4-added members (`getMetadata`, + // `getRecordPublicUrl`). Cast through for now; a follow-up should + // bring apify-client into structural alignment. + return this.client.dataset(id ?? '') as unknown as DatasetClient; + } + + async createKeyValueStoreClient( + options?: CreateKeyValueStoreClientOptions, + ): Promise { + const id = + options?.id ?? + (options?.name + ? ( + await this.client + .keyValueStores() + .getOrCreate(options.name) + ).id + : undefined); + return this.client.keyValueStore(id ?? '') as unknown as KeyValueStoreClient; + } + + async createRequestQueueClient( + options?: CreateRequestQueueClientOptions, + ): Promise { + const id = + options?.id ?? + (options?.name + ? ( + await this.client + .requestQueues() + .getOrCreate(options.name) + ).id + : undefined); + return this.client.requestQueue( + id ?? '', + options?.clientKey ? { clientKey: options.clientKey } : undefined, + ) as unknown as RequestQueueClient; + } +} diff --git a/packages/apify/src/key_value_store.ts b/packages/apify/src/key_value_store.ts index a26a12e8f1..13ea1fd0a7 100644 --- a/packages/apify/src/key_value_store.ts +++ b/packages/apify/src/key_value_store.ts @@ -1,12 +1,18 @@ import type { StorageManagerOptions } from '@crawlee/core'; import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core'; +import type { KeyValueStoreInfo } from '@crawlee/types'; import { createHmacSignature } from '@apify/utilities'; import type { Configuration } from './configuration.js'; -// @ts-ignore newer crawlee versions already declare this method in core -const { getPublicUrl } = CoreKeyValueStore.prototype; +// crawlee v4 dropped the `storageObject` cache from `KeyValueStore`, so the +// per-store `urlSigningSecretKey` (which is part of the platform's metadata +// response but not declared on `@crawlee/types`' `KeyValueStoreInfo`) has to +// be fetched on demand and accessed through a structural-typed augmentation. +type ApifyKeyValueStoreInfo = KeyValueStoreInfo & { + urlSigningSecretKey?: string; +}; /** * @inheritDoc @@ -15,24 +21,33 @@ export class KeyValueStore extends CoreKeyValueStore { /** * Returns a URL for the given key that may be used to publicly * access the value in the remote key-value store. + * + * On the Apify platform the URL is signed with the store's + * `urlSigningSecretKey` so that anyone with the URL can read the record + * without authentication. Locally we delegate to crawlee's default + * implementation (which produces a `file://` URL or returns `undefined`). */ - override getPublicUrl(key: string): string { + override async getPublicUrl(key: string): Promise { const config = this.config as Configuration; - if (!config.isAtHome && getPublicUrl) { - return getPublicUrl.call(this, key); + if (!config.isAtHome) { + return super.getPublicUrl(key); } const publicUrl = new URL( `${config.apiPublicBaseUrl}/v2/key-value-stores/${this.id}/records/${key}`, ); - if (this.storageObject?.urlSigningSecretKey) { + // `client` is `private` on `CoreKeyValueStore`; bypass the visibility + // check to fetch the per-store secret. There is no public crawlee API + // surface for this yet — track upstream exposure as a follow-up. + const metadata = (await ( + this as unknown as { client: { getMetadata(): Promise } } + ).client.getMetadata()) as ApifyKeyValueStoreInfo; + + if (metadata?.urlSigningSecretKey) { publicUrl.searchParams.append( 'signature', - createHmacSignature( - this.storageObject.urlSigningSecretKey as string, - key, - ), + createHmacSignature(metadata.urlSigningSecretKey, key), ); } @@ -49,6 +64,3 @@ export class KeyValueStore extends CoreKeyValueStore { return super.open(storeIdOrName, options) as unknown as KeyValueStore; } } - -// @ts-ignore newer crawlee versions already declare this method in core -CoreKeyValueStore.prototype.getPublicUrl = KeyValueStore.prototype.getPublicUrl; From 93d2cae779d895f7f42f1dfb170044dca6cb897f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 18:02:40 +0200 Subject: [PATCH 07/34] fix: adapt SDK ProxyConfiguration to crawlee v4 API Crawlee v4 reshaped `ProxyConfiguration`: - `newProxyInfo` and `newUrl` now take a single `TieredProxyOptions` argument; the previous `(sessionId, options)` pair is gone. - The protected `_handleCustomUrl(sessionId)` helper was removed; the `_callNewUrlFunction` and `_handleTieredUrl` helpers now take options only. - `ProxyInfo` (in `@crawlee/types`) no longer carries `sessionId`. Changes: - `newProxyInfo` and `newUrl` accept `string | number | TieredProxyOptions | undefined` so existing SDK callers that pass a raw `sessionId` keep working, while the override remains compatible with crawlee's v4 signature. A small `parseSessionIdOrOptions` helper discriminates and pulls `sessionId` from `options.request` when no explicit one is given. - Inlined custom-URL session stickiness via a new private `getSessionIndex(sessionId)` (replacing the removed `_handleCustomUrl`), keyed on `usedProxyUrls` like the base class. - Re-declared `sessionId?: string` on the SDK's `ProxyInfo` interface so users can still read `proxyInfo.sessionId` (v3 carried it on the base type). - Re-imported `ProxyInfo` from `@crawlee/types` (no longer re-exported from `@crawlee/core`). - Tightened a `proxyUrls.some(url => url.includes(...))` access for the new `(string | null)[]` array shape. Stacked on #583 (config redesign); rebases onto v4 once that lands. --- packages/apify/src/proxy_configuration.ts | 98 +++++++++++++++++------ 1 file changed, 75 insertions(+), 23 deletions(-) diff --git a/packages/apify/src/proxy_configuration.ts b/packages/apify/src/proxy_configuration.ts index 0d452ab645..3aa527d807 100644 --- a/packages/apify/src/proxy_configuration.ts +++ b/packages/apify/src/proxy_configuration.ts @@ -1,14 +1,10 @@ -import type { - ProxyConfigurationOptions as CoreProxyConfigurationOptions, - ProxyInfo as CoreProxyInfo, -} from '@crawlee/core'; +import type { ProxyConfigurationOptions as CoreProxyConfigurationOptions } from '@crawlee/core'; import { ProxyConfiguration as CoreProxyConfiguration } from '@crawlee/core'; +import type { ProxyInfo as CoreProxyInfo } from '@crawlee/types'; import { gotScraping } from 'got-scraping'; import ow from 'ow'; import { APIFY_ENV_VARS, APIFY_PROXY_VALUE_REGEX } from '@apify/consts'; -import { cryptoRandomObjectId } from '@apify/utilities'; - import { Actor } from './actor.js'; import { Configuration } from './configuration.js'; @@ -18,6 +14,22 @@ const CHECK_ACCESS_REQUEST_TIMEOUT_MILLIS = 4_000; const CHECK_ACCESS_MAX_ATTEMPTS = 2; const COUNTRY_CODE_REGEX = /^[A-Z]{2}$/; +type CoreProxyOptions = Parameters[0]; + +/** + * Bridges the SDK's legacy `(sessionId)` calling style with crawlee v4's + * `(options)` shape — pulls `sessionId` from a `Request` carried in `options` + * when no explicit `sessionId` is given. + */ +function parseSessionIdOrOptions( + arg: string | number | CoreProxyOptions | undefined, +): { sessionId: string | undefined; options: CoreProxyOptions } { + if (typeof arg === 'string' || typeof arg === 'number') { + return { sessionId: String(arg), options: undefined }; + } + return { sessionId: arg?.request?.sessionId, options: arg }; +} + export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions { /** @@ -100,6 +112,13 @@ export interface ProxyConfigurationOptions * ``` */ export interface ProxyInfo extends CoreProxyInfo { + /** + * The Apify Proxy session identifier the URL was minted for, if any. + * v3 carried this on the base `ProxyInfo`; v4 dropped it, so the SDK + * re-declares it here for users that read `proxyInfo.sessionId`. + */ + sessionId?: string; + /** * An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy). * If not provided, the proxy will select the groups automatically. @@ -241,7 +260,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { this.port = port; this.usesApifyProxy = !this.proxyUrls && !this.newUrlFunction; - if (proxyUrls && proxyUrls.some((url) => url.includes('apify.com'))) { + if (proxyUrls && proxyUrls.some((url) => url?.includes('apify.com'))) { this.log.warning( 'Some Apify proxy features may work incorrectly. Please consider setting up Apify properties instead of `proxyUrls`.\n' + 'See https://sdk.apify.com/docs/guides/proxy-management#apify-proxy-configuration', @@ -304,10 +323,18 @@ export class ProxyConfiguration extends CoreProxyConfiguration { * @return Represents information about used proxy and its configuration. */ override async newProxyInfo( - sessionId?: string | number, - options?: Parameters[1], + sessionIdOrOptions?: + | string + | number + | Parameters[0], ): Promise { - if (typeof sessionId === 'number') sessionId = `${sessionId}`; + // crawlee v4 dropped the `(sessionId, options)` overload — `newProxyInfo` + // now takes a single `TieredProxyOptions` argument and pulls `sessionId` + // from `options.request`. Keep the SDK's legacy "pass sessionId directly" + // shape working by discriminating at runtime. + const { sessionId, options } = parseSessionIdOrOptions( + sessionIdOrOptions, + ); ow( sessionId, ow.optional.string @@ -315,15 +342,15 @@ export class ProxyConfiguration extends CoreProxyConfiguration { .matches(APIFY_PROXY_VALUE_REGEX), ); - const proxyInfo = await super.newProxyInfo(sessionId, options); - if (!proxyInfo) return proxyInfo; + const url = await this.newUrl(sessionIdOrOptions); + if (!url) return undefined; const { groups, countryCode, password, port, hostname } = ( - this.usesApifyProxy ? this : new URL(proxyInfo.url) + this.usesApifyProxy ? this : new URL(url) ) as ProxyConfiguration; return { - ...proxyInfo, + url, sessionId, groups, countryCode, @@ -333,6 +360,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { : decodeURIComponent(password!), hostname, port: port!, + proxyTier: options?.proxyTier, }; } @@ -350,10 +378,14 @@ export class ProxyConfiguration extends CoreProxyConfiguration { * For example, `http://bob:password123@proxy.example.com:8000` */ override async newUrl( - sessionId?: string | number, - options?: Parameters[1], + sessionIdOrOptions?: + | string + | number + | Parameters[0], ): Promise { - if (typeof sessionId === 'number') sessionId = `${sessionId}`; + const { sessionId, options } = parseSessionIdOrOptions( + sessionIdOrOptions, + ); ow( sessionId, ow.optional.string @@ -362,27 +394,47 @@ export class ProxyConfiguration extends CoreProxyConfiguration { ); if (this.newUrlFunction) { return ( - (await this._callNewUrlFunction(sessionId, { + (await this._callNewUrlFunction({ request: options?.request, })) ?? undefined ); } if (this.proxyUrls) { - return this._handleCustomUrl(sessionId); + // `_handleCustomUrl` was removed from `CoreProxyConfiguration` in + // v4; inline the rotation logic to preserve session-stickiness. + const index = + sessionId !== undefined + ? this.getSessionIndex(sessionId) + : (this.nextCustomUrlIndex += 1) % this.proxyUrls.length; + return this.proxyUrls[index] ?? undefined; } if (this.tieredProxyUrls) { return ( - this._handleTieredUrl( - sessionId ?? cryptoRandomObjectId(6), - options, - ).proxyUrl ?? undefined + this._handleTieredUrl(options ?? {}).proxyUrl ?? undefined ); } return this.composeDefaultUrl(sessionId); } + /** + * Stable per-session index into `proxyUrls`, replacing the removed + * `_handleCustomUrl(sessionId)` from crawlee v3. + */ + private getSessionIndex(sessionId: string): number { + if (!this.usedProxyUrls.has(sessionId)) { + this.usedProxyUrls.set( + sessionId, + this.proxyUrls![ + this.usedProxyUrls.size % this.proxyUrls!.length + ], + ); + } + return this.proxyUrls!.indexOf(this.usedProxyUrls.get(sessionId)!); + } + + protected _generateTieredProxyUrls( tieredProxyConfig: NonNullable< ProxyConfigurationOptions['tieredProxyConfig'] From 9d11039556a3de40ae924b03a2d0eb55accd2338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 18:18:30 +0200 Subject: [PATCH 08/34] test(actor): reset service locator and Actor singleton between newClient cases Crawlee v4's Configuration resolves env vars eagerly at construction, so the existing 'Actor.newClient() reads environment variables correctly' test reads stale values once a prior test or import-time side effect has already created the singleton. Reset both before each case. --- test/apify/utils.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/apify/utils.test.ts b/test/apify/utils.test.ts index acff5f2324..325d400cc4 100644 --- a/test/apify/utils.test.ts +++ b/test/apify/utils.test.ts @@ -1,6 +1,7 @@ import type { IncomingMessage } from 'node:http'; import type { Request } from '@crawlee/core'; +import { serviceLocator } from '@crawlee/core'; import { createRequestDebugInfo } from '@crawlee/utils'; import { Actor } from 'apify'; import semver from 'semver'; @@ -21,6 +22,15 @@ describe('Actor.isAtHome()', () => { }); describe('Actor.newClient()', () => { + // crawlee v4's `Configuration` resolves env vars eagerly at construction, + // so tests that mutate env vars after the global config / Actor singleton + // exists would otherwise read stale values. Reset both so each test + // observes the env it just wrote. + beforeEach(() => { + serviceLocator.reset(); + (Actor as unknown as { _instance?: Actor })._instance = undefined; + }); + test('reads environment variables correctly', () => { process.env[APIFY_ENV_VARS.API_BASE_URL] = 'http://www.example.com:1234/path'; From c1b48155aec91555fa65fa10dd224b0af466f535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 18:18:39 +0200 Subject: [PATCH 09/34] test(events): use serviceLocator.setEventManager and reset between cases `Configuration.useEventManager()` was removed in crawlee v4. Install the platform event manager via the global service locator instead, and reset between tests so each case can register a fresh manager without hitting `ServiceConflictError`. --- test/apify/events.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/apify/events.test.ts b/test/apify/events.test.ts index d6ec40d3ec..7cd290b232 100644 --- a/test/apify/events.test.ts +++ b/test/apify/events.test.ts @@ -1,4 +1,4 @@ -import { EventType } from '@crawlee/core'; +import { EventType, serviceLocator } from '@crawlee/core'; import type { Dictionary } from '@crawlee/utils'; import { sleep } from '@crawlee/utils'; import { Actor, Configuration, PlatformEventManager } from 'apify'; @@ -14,7 +14,9 @@ describe('events', () => { beforeEach(() => { wss = new WebSocketServer({ port: 9099 }); events = new PlatformEventManager(config); - config.useEventManager(events); + // crawlee v4 removed `Configuration.useEventManager()`; the event + // manager is now installed on the global service locator. + serviceLocator.setEventManager(events); vitest.useFakeTimers(); process.env[ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL] = @@ -25,6 +27,9 @@ describe('events', () => { vitest.useRealTimers(); delete process.env[ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL]; delete process.env[APIFY_ENV_VARS.TOKEN]; + // Reset the service locator so the next test can register a + // fresh event manager without hitting `ServiceConflictError`. + serviceLocator.reset(); await new Promise((resolve) => { wss.close(resolve); }); From 90f0f8765904dd0a82105915b9452e7dd951f0fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 18:18:49 +0200 Subject: [PATCH 10/34] test: migrate MemoryStorageEmulator to crawlee v4 service locator Replace the removed `StorageManager.clearCache()` and `Configuration.useStorageClient()` with `serviceLocator.reset()` plus `serviceLocator.setStorageClient()`. --- test/MemoryStorageEmulator.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/MemoryStorageEmulator.ts b/test/MemoryStorageEmulator.ts index c5d4511236..c0bc20bafc 100644 --- a/test/MemoryStorageEmulator.ts +++ b/test/MemoryStorageEmulator.ts @@ -1,9 +1,8 @@ import { rm } from 'node:fs/promises'; import { resolve } from 'node:path'; -import { StorageManager } from '@crawlee/core'; +import { serviceLocator } from '@crawlee/core'; import { MemoryStorage } from '@crawlee/memory-storage'; -import { Configuration } from 'apify'; import { ensureDir } from 'fs-extra'; import log from '@apify/log'; @@ -20,7 +19,10 @@ export class MemoryStorageEmulator { protected localStorageDirectories: string[] = []; async init(dirName = cryptoRandomObjectId(10)) { - StorageManager.clearCache(); + // crawlee v4 dropped `StorageManager.clearCache()` and + // `Configuration.useStorageClient()`; reset the service locator + // and re-register the in-memory client instead. + serviceLocator.reset(); const localStorageDir = resolve(LOCAL_EMULATION_DIR, dirName); this.localStorageDirectories.push(localStorageDir); await ensureDir(localStorageDir); @@ -28,7 +30,7 @@ export class MemoryStorageEmulator { const storage = new MemoryStorage({ localDataDirectory: localStorageDir, }); - Configuration.getGlobalConfig().useStorageClient(storage); + serviceLocator.setStorageClient(storage); log.debug( `Initialized emulated memory storage in folder ${localStorageDir}`, ); @@ -40,7 +42,7 @@ export class MemoryStorageEmulator { }); await Promise.all(promises); - StorageManager.clearCache(); + serviceLocator.reset(); } static toString() { From 075907cfadc58ef062bf002b999ad5e106b97cbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 18:30:01 +0200 Subject: [PATCH 11/34] test(events): seed env vars before resolving Configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crawlee v4's `Configuration` is eager — `actorEventsWsUrl` is read once at construction, so a global config that pre-existed the `beforeEach` would never see the websocket URL we set, and `events.init()` would silently never connect. Move the env-var setup above `Configuration.getGlobalConfig()` and reset the SDK's static singleton so each test rebuilds a fresh config. --- test/apify/events.test.ts | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/test/apify/events.test.ts b/test/apify/events.test.ts index 7cd290b232..798ced8841 100644 --- a/test/apify/events.test.ts +++ b/test/apify/events.test.ts @@ -8,28 +8,34 @@ import { ACTOR_ENV_VARS, APIFY_ENV_VARS } from '@apify/consts'; describe('events', () => { let wss: WebSocketServer = null!; - const config = Configuration.getGlobalConfig(); let events: PlatformEventManager = null!; beforeEach(() => { + // Set env vars BEFORE creating the Configuration — crawlee v4 resolves + // env-var-backed fields eagerly at construction, so a global config + // built earlier in the run wouldn't see `actorEventsWsUrl` and + // `events.init()` would silently never open the websocket. + process.env[ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL] = + 'ws://localhost:9099/someRunId'; + process.env[APIFY_ENV_VARS.TOKEN] = 'dummy'; + wss = new WebSocketServer({ port: 9099 }); + // Reset both the service locator and the SDK's static singleton so + // the new Configuration picks up the env vars we just set. + serviceLocator.reset(); + (Configuration as unknown as { globalConfig?: Configuration }).globalConfig = undefined; + const config = Configuration.getGlobalConfig(); events = new PlatformEventManager(config); - // crawlee v4 removed `Configuration.useEventManager()`; the event - // manager is now installed on the global service locator. serviceLocator.setEventManager(events); vitest.useFakeTimers(); - process.env[ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL] = - 'ws://localhost:9099/someRunId'; - process.env[APIFY_ENV_VARS.TOKEN] = 'dummy'; }); afterEach(async () => { vitest.useRealTimers(); delete process.env[ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL]; delete process.env[APIFY_ENV_VARS.TOKEN]; - // Reset the service locator so the next test can register a - // fresh event manager without hitting `ServiceConflictError`. serviceLocator.reset(); + (Configuration as unknown as { globalConfig?: Configuration }).globalConfig = undefined; await new Promise((resolve) => { wss.close(resolve); }); @@ -135,7 +141,7 @@ describe('events', () => { test('should send persist state events in regular interval', async () => { const eventsReceived = []; - const interval = config.persistStateIntervalMillis; + const interval = events.config.persistStateIntervalMillis; events.on(EventType.PERSIST_STATE, (data) => eventsReceived.push(data)); await events.init(); From be6811fc1ad739ac73c484df74b28957292f9894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 18:30:13 +0200 Subject: [PATCH 12/34] test: also reset SDK Configuration.globalConfig between cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK's `Configuration` keeps its own static singleton separate from crawlee's serviceLocator. Resetting only the locator wasn't enough — `Configuration.getGlobalConfig()` still handed back the stale cached SDK config (which was built before the test set `APIFY_TOKEN`). --- test/apify/utils.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/apify/utils.test.ts b/test/apify/utils.test.ts index 325d400cc4..897f23d0c7 100644 --- a/test/apify/utils.test.ts +++ b/test/apify/utils.test.ts @@ -3,7 +3,7 @@ import type { IncomingMessage } from 'node:http'; import type { Request } from '@crawlee/core'; import { serviceLocator } from '@crawlee/core'; import { createRequestDebugInfo } from '@crawlee/utils'; -import { Actor } from 'apify'; +import { Actor, Configuration } from 'apify'; import semver from 'semver'; import { APIFY_ENV_VARS } from '@apify/consts'; @@ -28,6 +28,9 @@ describe('Actor.newClient()', () => { // observes the env it just wrote. beforeEach(() => { serviceLocator.reset(); + // The SDK's `Configuration` keeps its own static singleton (separate + // from crawlee's serviceLocator) so tests need to clear that cache too. + (Configuration as unknown as { globalConfig?: Configuration }).globalConfig = undefined; (Actor as unknown as { _instance?: Actor })._instance = undefined; }); From c890b8d46bc87f73dfe16cf93e562ca683e0b7f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 18:34:49 +0200 Subject: [PATCH 13/34] chore: fix import sort in actor.ts after ApifyStorageClient addition --- packages/apify/src/actor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apify/src/actor.ts b/packages/apify/src/actor.ts index 2905b377bb..640a94a93a 100644 --- a/packages/apify/src/actor.ts +++ b/packages/apify/src/actor.ts @@ -43,11 +43,11 @@ import { decryptInputSecrets } from '@apify/input_secrets'; import log from '@apify/log'; import { addTimeoutToPromise } from '@apify/timeout'; +import { ApifyStorageClient } from './apify_storage_client.js'; import type { ChargeOptions, ChargeResult } from './charging.js'; import { ChargingManager } from './charging.js'; import type { ConfigurationOptions } from './configuration.js'; import { Configuration } from './configuration.js'; -import { ApifyStorageClient } from './apify_storage_client.js'; import { KeyValueStore } from './key_value_store.js'; import { PlatformEventManager } from './platform_event_manager.js'; import type { ProxyConfigurationOptions } from './proxy_configuration.js'; From a4325b926fd672fe3b81e9fff88b9deb7d23184e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 18:34:59 +0200 Subject: [PATCH 14/34] chore: fix import sort in proxy_configuration.ts --- packages/apify/src/proxy_configuration.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/apify/src/proxy_configuration.ts b/packages/apify/src/proxy_configuration.ts index 3aa527d807..b1347876d6 100644 --- a/packages/apify/src/proxy_configuration.ts +++ b/packages/apify/src/proxy_configuration.ts @@ -5,6 +5,7 @@ import { gotScraping } from 'got-scraping'; import ow from 'ow'; import { APIFY_ENV_VARS, APIFY_PROXY_VALUE_REGEX } from '@apify/consts'; + import { Actor } from './actor.js'; import { Configuration } from './configuration.js'; From 58ce5364c8a40146752f865161bbaab8b8eb868a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 18:35:21 +0200 Subject: [PATCH 15/34] chore: silence no-underscore-dangle for Actor._instance reset --- test/apify/utils.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/apify/utils.test.ts b/test/apify/utils.test.ts index 897f23d0c7..7b6491a3e2 100644 --- a/test/apify/utils.test.ts +++ b/test/apify/utils.test.ts @@ -31,6 +31,7 @@ describe('Actor.newClient()', () => { // The SDK's `Configuration` keeps its own static singleton (separate // from crawlee's serviceLocator) so tests need to clear that cache too. (Configuration as unknown as { globalConfig?: Configuration }).globalConfig = undefined; + // eslint-disable-next-line no-underscore-dangle -- `_instance` is the upstream Actor singleton field (Actor as unknown as { _instance?: Actor })._instance = undefined; }); From 5a34b349d8175c0c9c47db80ecaf961398c94c19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:11:55 +0200 Subject: [PATCH 16/34] chore: prettier --- test/apify/utils.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/apify/utils.test.ts b/test/apify/utils.test.ts index 7b6491a3e2..b7c051badd 100644 --- a/test/apify/utils.test.ts +++ b/test/apify/utils.test.ts @@ -30,7 +30,9 @@ describe('Actor.newClient()', () => { serviceLocator.reset(); // The SDK's `Configuration` keeps its own static singleton (separate // from crawlee's serviceLocator) so tests need to clear that cache too. - (Configuration as unknown as { globalConfig?: Configuration }).globalConfig = undefined; + ( + Configuration as unknown as { globalConfig?: Configuration } + ).globalConfig = undefined; // eslint-disable-next-line no-underscore-dangle -- `_instance` is the upstream Actor singleton field (Actor as unknown as { _instance?: Actor })._instance = undefined; }); From c01e89d8224ecf0214c7fb6206b4942034d73cf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:11:58 +0200 Subject: [PATCH 17/34] chore: prettier --- packages/apify/src/platform_event_manager.ts | 8 ++++++-- test/apify/events.test.ts | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/apify/src/platform_event_manager.ts b/packages/apify/src/platform_event_manager.ts index fa15013c85..0f9a7f3167 100644 --- a/packages/apify/src/platform_event_manager.ts +++ b/packages/apify/src/platform_event_manager.ts @@ -48,8 +48,12 @@ export class PlatformEventManager extends EventManager { /** Websocket connection to Actor events. */ private eventsWs?: WebSocket; - constructor(readonly config: Configuration = Configuration.getGlobalConfig() as Configuration) { - super({ persistStateIntervalMillis: config.persistStateIntervalMillis }); + constructor( + readonly config: Configuration = Configuration.getGlobalConfig() as Configuration, + ) { + super({ + persistStateIntervalMillis: config.persistStateIntervalMillis, + }); } /** diff --git a/test/apify/events.test.ts b/test/apify/events.test.ts index 798ced8841..a98ed0d500 100644 --- a/test/apify/events.test.ts +++ b/test/apify/events.test.ts @@ -23,7 +23,9 @@ describe('events', () => { // Reset both the service locator and the SDK's static singleton so // the new Configuration picks up the env vars we just set. serviceLocator.reset(); - (Configuration as unknown as { globalConfig?: Configuration }).globalConfig = undefined; + ( + Configuration as unknown as { globalConfig?: Configuration } + ).globalConfig = undefined; const config = Configuration.getGlobalConfig(); events = new PlatformEventManager(config); serviceLocator.setEventManager(events); @@ -35,7 +37,9 @@ describe('events', () => { delete process.env[ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL]; delete process.env[APIFY_ENV_VARS.TOKEN]; serviceLocator.reset(); - (Configuration as unknown as { globalConfig?: Configuration }).globalConfig = undefined; + ( + Configuration as unknown as { globalConfig?: Configuration } + ).globalConfig = undefined; await new Promise((resolve) => { wss.close(resolve); }); From 56f0c3e1bc8cd3efbc5f86b3626c9e92cb48a8e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:12:01 +0200 Subject: [PATCH 18/34] chore: prettier --- packages/apify/src/apify_storage_client.ts | 18 +++++++----------- packages/apify/src/key_value_store.ts | 4 +++- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/apify/src/apify_storage_client.ts b/packages/apify/src/apify_storage_client.ts index 4926452fc4..6301411010 100644 --- a/packages/apify/src/apify_storage_client.ts +++ b/packages/apify/src/apify_storage_client.ts @@ -43,13 +43,12 @@ export class ApifyStorageClient implements StorageClient { const id = options?.id ?? (options?.name - ? ( - await this.client - .keyValueStores() - .getOrCreate(options.name) - ).id + ? (await this.client.keyValueStores().getOrCreate(options.name)) + .id : undefined); - return this.client.keyValueStore(id ?? '') as unknown as KeyValueStoreClient; + return this.client.keyValueStore( + id ?? '', + ) as unknown as KeyValueStoreClient; } async createRequestQueueClient( @@ -58,11 +57,8 @@ export class ApifyStorageClient implements StorageClient { const id = options?.id ?? (options?.name - ? ( - await this.client - .requestQueues() - .getOrCreate(options.name) - ).id + ? (await this.client.requestQueues().getOrCreate(options.name)) + .id : undefined); return this.client.requestQueue( id ?? '', diff --git a/packages/apify/src/key_value_store.ts b/packages/apify/src/key_value_store.ts index 13ea1fd0a7..356180bdcd 100644 --- a/packages/apify/src/key_value_store.ts +++ b/packages/apify/src/key_value_store.ts @@ -41,7 +41,9 @@ export class KeyValueStore extends CoreKeyValueStore { // check to fetch the per-store secret. There is no public crawlee API // surface for this yet — track upstream exposure as a follow-up. const metadata = (await ( - this as unknown as { client: { getMetadata(): Promise } } + this as unknown as { + client: { getMetadata(): Promise }; + } ).client.getMetadata()) as ApifyKeyValueStoreInfo; if (metadata?.urlSigningSecretKey) { From 31344feef780347718aea9b109d4d3005fb4f194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:12:04 +0200 Subject: [PATCH 19/34] chore: prettier --- packages/apify/src/proxy_configuration.ts | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/packages/apify/src/proxy_configuration.ts b/packages/apify/src/proxy_configuration.ts index b1347876d6..7ba3a65818 100644 --- a/packages/apify/src/proxy_configuration.ts +++ b/packages/apify/src/proxy_configuration.ts @@ -333,9 +333,8 @@ export class ProxyConfiguration extends CoreProxyConfiguration { // now takes a single `TieredProxyOptions` argument and pulls `sessionId` // from `options.request`. Keep the SDK's legacy "pass sessionId directly" // shape working by discriminating at runtime. - const { sessionId, options } = parseSessionIdOrOptions( - sessionIdOrOptions, - ); + const { sessionId, options } = + parseSessionIdOrOptions(sessionIdOrOptions); ow( sessionId, ow.optional.string @@ -384,9 +383,8 @@ export class ProxyConfiguration extends CoreProxyConfiguration { | number | Parameters[0], ): Promise { - const { sessionId, options } = parseSessionIdOrOptions( - sessionIdOrOptions, - ); + const { sessionId, options } = + parseSessionIdOrOptions(sessionIdOrOptions); ow( sessionId, ow.optional.string @@ -411,9 +409,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { } if (this.tieredProxyUrls) { - return ( - this._handleTieredUrl(options ?? {}).proxyUrl ?? undefined - ); + return this._handleTieredUrl(options ?? {}).proxyUrl ?? undefined; } return this.composeDefaultUrl(sessionId); @@ -435,7 +431,6 @@ export class ProxyConfiguration extends CoreProxyConfiguration { return this.proxyUrls!.indexOf(this.usedProxyUrls.get(sessionId)!); } - protected _generateTieredProxyUrls( tieredProxyConfig: NonNullable< ProxyConfigurationOptions['tieredProxyConfig'] @@ -491,7 +486,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { */ // TODO: Make this private protected async _setPasswordIfToken(): Promise { - const {token} = (this.config as Configuration); + const { token } = this.config as Configuration; if (!token) return; try { @@ -553,7 +548,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { } | undefined > { - const {proxyStatusUrl} = (this.config as Configuration); + const { proxyStatusUrl } = this.config as Configuration; const requestOpts = { url: `${proxyStatusUrl}/?format=json`, proxyUrl: await this.newUrl(), From fda43178d7087511fa32cf83fcbe95fed438debe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:14:00 +0200 Subject: [PATCH 20/34] test: also reset SDK Configuration.globalConfig and Actor singleton on emulator init/destroy --- test/MemoryStorageEmulator.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test/MemoryStorageEmulator.ts b/test/MemoryStorageEmulator.ts index c0bc20bafc..cab9eab144 100644 --- a/test/MemoryStorageEmulator.ts +++ b/test/MemoryStorageEmulator.ts @@ -3,11 +3,25 @@ import { resolve } from 'node:path'; import { serviceLocator } from '@crawlee/core'; import { MemoryStorage } from '@crawlee/memory-storage'; +import { Actor, Configuration } from 'apify'; import { ensureDir } from 'fs-extra'; import log from '@apify/log'; import { cryptoRandomObjectId } from '@apify/utilities'; +function resetGlobalState() { + serviceLocator.reset(); + // The SDK's `Configuration` keeps its own static singleton (separate + // from crawlee's serviceLocator), and `Actor` caches a default + // instance with the resolved config. Both must be cleared so each + // test starts with a fresh config that reads the env vars it just set. + ( + Configuration as unknown as { globalConfig?: Configuration } + ).globalConfig = undefined; + // eslint-disable-next-line no-underscore-dangle -- `_instance` is the upstream Actor singleton field + (Actor as unknown as { _instance?: Actor })._instance = undefined; +} + const LOCAL_EMULATION_DIR = resolve( __dirname, '..', @@ -22,7 +36,7 @@ export class MemoryStorageEmulator { // crawlee v4 dropped `StorageManager.clearCache()` and // `Configuration.useStorageClient()`; reset the service locator // and re-register the in-memory client instead. - serviceLocator.reset(); + resetGlobalState(); const localStorageDir = resolve(LOCAL_EMULATION_DIR, dirName); this.localStorageDirectories.push(localStorageDir); await ensureDir(localStorageDir); @@ -42,7 +56,7 @@ export class MemoryStorageEmulator { }); await Promise.all(promises); - serviceLocator.reset(); + resetGlobalState(); } static toString() { From 99e5683c6c2e3e98f02ce505d66d9fb152db1b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:17:21 +0200 Subject: [PATCH 21/34] test: align actor.test.ts mocks/expectations with v4 StorageClient adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `openRequestQueue should open storage`: mock client uses `getMetadata()` (the v3 `get()` was dropped on RequestQueueClient). - Both Storage API tests assert that StorageManager.openStorage is called with an ApifyStorageClient (matched structurally) instead of the raw ApifyClient — the SDK now wraps it for crawlee v4. --- test/apify/actor.test.ts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index f1b88fc281..9fd1cd49ff 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -758,13 +758,26 @@ describe('Actor', () => { 'openStorage', ); + // crawlee v4's `RequestQueueClient` exposes metadata via + // `getMetadata()` (the v3 `get()` was dropped). const mockRQ = { - client: { get: () => ({ totalRequestCount: 10 }) }, + client: { + getMetadata: async () => ({ totalRequestCount: 10 }), + }, }; openStorageSpy.mockImplementationOnce(async () => mockRQ); const queue = await sdk.openRequestQueue(queueId, options); - expect(openStorageSpy).toBeCalledWith(queueId, sdk.apifyClient); + // The SDK now wraps `apifyClient` in an `ApifyStorageClient` + // adapter to satisfy crawlee v4's `StorageClient` interface. + expect(openStorageSpy).toBeCalledWith( + queueId, + expect.objectContaining({ + createDatasetClient: expect.any(Function), + createKeyValueStoreClient: expect.any(Function), + createRequestQueueClient: expect.any(Function), + }), + ); expect(openStorageSpy).toBeCalledTimes(1); // @ts-expect-error private prop @@ -783,7 +796,11 @@ describe('Actor', () => { expect(mockOpenStorage).toBeCalledTimes(1); expect(mockOpenStorage).toBeCalledWith( datasetName, - sdk.apifyClient, + expect.objectContaining({ + createDatasetClient: expect.any(Function), + createKeyValueStoreClient: expect.any(Function), + createRequestQueueClient: expect.any(Function), + }), ); }); }); From 4fd8e3e010f122ed1df5100e84a9e71958499418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:17:24 +0200 Subject: [PATCH 22/34] test(reboot): use serviceLocator.getEventManager (v4 replacement) --- test/apify/actor.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index f1b88fc281..6acc52c7ab 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -1,6 +1,11 @@ import { createPublicKey } from 'node:crypto'; -import { Configuration, EventType, StorageManager } from '@crawlee/core'; +import { + Configuration, + EventType, + serviceLocator, + StorageManager, +} from '@crawlee/core'; import { sleep } from '@crawlee/utils'; import type { ApifyEnv } from 'apify'; import { Actor, Dataset, KeyValueStore, ProxyConfiguration } from 'apify'; @@ -1089,7 +1094,9 @@ describe('Actor', () => { const migratingSpy = vitest.fn(persistResource(50)); const persistStateSpy = vitest.fn(persistResource(50)); - const events = Configuration.getEventManager(); + // crawlee v4 removed `Configuration.getEventManager()`; the + // event manager now lives on the global service locator. + const events = serviceLocator.getEventManager(); events.on(EventType.PERSIST_STATE, persistStateSpy); events.on(EventType.MIGRATING, migratingSpy); From b42b603f6c20305fcc21045cc7fc49b1869c3212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:17:26 +0200 Subject: [PATCH 23/34] test(actor): adapt to crawlee v4 eager Configuration resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reword "empty string maxTotalChargeUsd" assertion: under Option A the empty env var is now treated as unset, so `config.maxTotalChargeUsd` is `undefined` (charging manager still defaults to Infinity). - Actor.getInput tests now build a fresh Actor *after* setting the env vars they exercise — eager config resolution means a single module-scoped TestingActor would carry stale values. --- test/apify/actor.test.ts | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index f1b88fc281..293fafb3cb 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -1182,9 +1182,14 @@ describe('Actor', () => { }); describe('Actor.getInput', () => { - const TestingActor = new Actor(); + // crawlee v4's Configuration resolves env vars eagerly at + // construction, so anything that needs to read env-var-driven + // config (like `inputKey` or `inputSecretsPrivateKey*`) must + // construct a fresh Actor *after* the env var is set. + const buildActor = () => new Actor(); test('should work', async () => { + const TestingActor = buildActor(); await expect(TestingActor.getInput()).resolves.toBeNull(); await expect(TestingActor.getInputOrThrow()).rejects.toThrowError( 'Input does not exist', @@ -1197,19 +1202,21 @@ describe('Actor', () => { await TestingActor.getInput(); - // Uses value from env var. + // Uses value from env var — needs a fresh Actor to pick it up. process.env[ACTOR_ENV_VARS.INPUT_KEY] = 'some-value'; - mockGetValue.mockImplementation(async (key) => + const ActorWithInputKey = buildActor(); + const mockGetValue2 = vitest.spyOn(ActorWithInputKey, 'getValue'); + mockGetValue2.mockImplementation(async (key) => expect(key).toBe('some-value'), ); - await TestingActor.getInput(); + await ActorWithInputKey.getInput(); delete process.env[ACTOR_ENV_VARS.INPUT_KEY]; mockGetValue.mockRestore(); + mockGetValue2.mockRestore(); }); test('should work with input secrets', async () => { - const mockGetValue = vitest.spyOn(TestingActor, 'getValue'); const originalInput = { secret: 'foo', nonSecret: 'bar' }; const likeInputSchema = { properties: { secret: { type: 'string', isSecret: true } }, @@ -1224,12 +1231,15 @@ describe('Actor', () => { expect(encryptedInput.secret.startsWith('ENCRYPTED_')).toBe(true); expect(encryptedInput.nonSecret).toBe(originalInput.nonSecret); - mockGetValue.mockImplementation(async (key) => encryptedInput); - + // Set the secrets env vars *before* constructing the Actor so + // the resolved config picks them up. process.env[APIFY_ENV_VARS.INPUT_SECRETS_PRIVATE_KEY_FILE] = testingPrivateKeyFile; process.env[APIFY_ENV_VARS.INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE] = testingPrivateKeyPassphrase; + const TestingActor = buildActor(); + const mockGetValue = vitest.spyOn(TestingActor, 'getValue'); + mockGetValue.mockImplementation(async (key) => encryptedInput); const input = await TestingActor.getInput(); expect(input).toStrictEqual(originalInput); @@ -1283,10 +1293,15 @@ describe('Actor', () => { }); describe('Actor.config and PPE', () => { - test('empty string maxTotalChargeUsd coerces to 0, charging manager treats as Infinity', async () => { + test('empty string maxTotalChargeUsd is treated as unset, charging manager defaults to Infinity', async () => { + // crawlee v4 (Option A) treats empty-string env vars as unset + // rather than coercing them to `0` / `false` / `''`. The + // resolved config falls through to the schema default + // (undefined) and the charging manager interprets that as + // "no limit". process.env.ACTOR_MAX_TOTAL_CHARGE_USD = ''; await Actor.init(); - expect(Actor.config.maxTotalChargeUsd).toBe(0); + expect(Actor.config.maxTotalChargeUsd).toBeUndefined(); expect(Actor.getChargingManager().getMaxTotalChargeUsd()).toBe( Infinity, ); From 2d90549480bcf20fc2273187b7d8d725cd595405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:18:34 +0200 Subject: [PATCH 24/34] test(getInput): reset cached singletons before each fresh Actor build --- test/apify/actor.test.ts | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index 293fafb3cb..4503128728 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -1,6 +1,11 @@ import { createPublicKey } from 'node:crypto'; -import { Configuration, EventType, StorageManager } from '@crawlee/core'; +import { + Configuration, + EventType, + serviceLocator, + StorageManager, +} from '@crawlee/core'; import { sleep } from '@crawlee/utils'; import type { ApifyEnv } from 'apify'; import { Actor, Dataset, KeyValueStore, ProxyConfiguration } from 'apify'; @@ -1183,10 +1188,18 @@ describe('Actor', () => { describe('Actor.getInput', () => { // crawlee v4's Configuration resolves env vars eagerly at - // construction, so anything that needs to read env-var-driven - // config (like `inputKey` or `inputSecretsPrivateKey*`) must - // construct a fresh Actor *after* the env var is set. - const buildActor = () => new Actor(); + // construction, and both the SDK's static `Configuration.globalConfig` + // and `Actor._instance` are cached singletons. Build a fresh Actor + // for every env-var change so the resolved config picks them up. + const buildActor = () => { + serviceLocator.reset(); + ( + Configuration as unknown as { globalConfig?: Configuration } + ).globalConfig = undefined; + // eslint-disable-next-line no-underscore-dangle -- `_instance` is the upstream Actor singleton field + (Actor as unknown as { _instance?: Actor })._instance = undefined; + return new Actor(); + }; test('should work', async () => { const TestingActor = buildActor(); From ecacdc6965887ec80bf1e917f6b2035c2537e826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:22:16 +0200 Subject: [PATCH 25/34] fix(proxy): preserve v3 rotation/username/validation semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Custom URL rotation: post-increment the round-robin index so the first sessionless call returns proxyUrls[0] (was off-by-one). - Surface `username` on the returned ProxyInfo by parsing it out of the resolved URL — v3 carried it via `super.newProxyInfo`. - parseSessionIdOrOptions now rejects non-plain objects (e.g. Date, Array) so `newUrl(new Date())` throws as users expect. test: `newUrl({})` is no longer 'invalid' — empty TieredProxyOptions is a legal v4 call shape; documented the carve-out. --- packages/apify/src/proxy_configuration.ts | 31 ++++++++++++++++++++--- test/apify/proxy_configuration.test.ts | 6 +++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/packages/apify/src/proxy_configuration.ts b/packages/apify/src/proxy_configuration.ts index 7ba3a65818..c988b4be71 100644 --- a/packages/apify/src/proxy_configuration.ts +++ b/packages/apify/src/proxy_configuration.ts @@ -20,15 +20,29 @@ type CoreProxyOptions = Parameters[0]; /** * Bridges the SDK's legacy `(sessionId)` calling style with crawlee v4's * `(options)` shape — pulls `sessionId` from a `Request` carried in `options` - * when no explicit `sessionId` is given. + * when no explicit `sessionId` is given. Rejects values that are neither a + * sessionId nor a plain options object (e.g. `Date`, arrays). */ function parseSessionIdOrOptions( arg: string | number | CoreProxyOptions | undefined, ): { sessionId: string | undefined; options: CoreProxyOptions } { + if (arg === undefined) { + return { sessionId: undefined, options: undefined }; + } if (typeof arg === 'string' || typeof arg === 'number') { return { sessionId: String(arg), options: undefined }; } - return { sessionId: arg?.request?.sessionId, options: arg }; + if ( + typeof arg !== 'object' || + arg === null || + Array.isArray(arg) || + Object.getPrototypeOf(arg) !== Object.prototype + ) { + throw new TypeError( + 'Expected sessionId (string/number) or a TieredProxyOptions object', + ); + } + return { sessionId: arg.request?.sessionId, options: arg }; } export interface ProxyConfigurationOptions @@ -349,6 +363,13 @@ export class ProxyConfiguration extends CoreProxyConfiguration { this.usesApifyProxy ? this : new URL(url) ) as ProxyConfiguration; + // Extract `username` from the resolved URL — crawlee v3 carried it + // on `ProxyInfo` and tests rely on it (e.g. for Apify Proxy session + // formatting). v4's `super.newProxyInfo` would surface this, but we + // bypass `super` here so the SDK can keep its legacy `sessionId` + // calling convention. + const username = new URL(url).username || undefined; + return { url, sessionId, @@ -360,6 +381,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { : decodeURIComponent(password!), hostname, port: port!, + username, proxyTier: options?.proxyTier, }; } @@ -401,10 +423,13 @@ export class ProxyConfiguration extends CoreProxyConfiguration { if (this.proxyUrls) { // `_handleCustomUrl` was removed from `CoreProxyConfiguration` in // v4; inline the rotation logic to preserve session-stickiness. + // Round-robin index for sessionless calls (post-increment so the + // first call returns proxyUrls[0]); per-session sticky mapping + // when a sessionId is provided. const index = sessionId !== undefined ? this.getSessionIndex(sessionId) - : (this.nextCustomUrlIndex += 1) % this.proxyUrls.length; + : this.nextCustomUrlIndex++ % this.proxyUrls.length; return this.proxyUrls[index] ?? undefined; } diff --git a/test/apify/proxy_configuration.test.ts b/test/apify/proxy_configuration.test.ts index 8c61a63177..16fc946810 100644 --- a/test/apify/proxy_configuration.test.ts +++ b/test/apify/proxy_configuration.test.ts @@ -175,8 +175,10 @@ describe('ProxyConfiguration', () => { proxyConfiguration.newUrl('a-b'), ).rejects.toThrow(), expect(proxyConfiguration.newUrl('a$b')).rejects.toThrow(), - // @ts-expect-error invalid input - expect(proxyConfiguration.newUrl({})).rejects.toThrow(), + // crawlee v4 made `newUrl` accept `TieredProxyOptions`, so + // an empty object is a valid (sessionless) call now. We only + // reject inputs that are neither a sessionId nor a plain + // options object. // @ts-expect-error invalid input expect(proxyConfiguration.newUrl(new Date())).rejects.toThrow(), expect( From 41e668926bd80335a8ec2661958fefdb743f30e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:24:47 +0200 Subject: [PATCH 26/34] test(getInput): also overwrite actor.config explicitly for env-var refresh --- test/apify/actor.test.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index 4503128728..ac28de741b 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -1188,9 +1188,12 @@ describe('Actor', () => { describe('Actor.getInput', () => { // crawlee v4's Configuration resolves env vars eagerly at - // construction, and both the SDK's static `Configuration.globalConfig` - // and `Actor._instance` are cached singletons. Build a fresh Actor - // for every env-var change so the resolved config picks them up. + // construction. The SDK's static `Configuration.globalConfig` and + // `Configuration.storage` (AsyncLocalStorage) plus `Actor._instance` + // can all hold onto a stale config across tests, so for each + // env-var change we both reset those singletons *and* explicitly + // overwrite `actor.config` with a fresh `new Configuration()` so + // env vars set in the test body are guaranteed to be observed. const buildActor = () => { serviceLocator.reset(); ( @@ -1198,7 +1201,10 @@ describe('Actor', () => { ).globalConfig = undefined; // eslint-disable-next-line no-underscore-dangle -- `_instance` is the upstream Actor singleton field (Actor as unknown as { _instance?: Actor })._instance = undefined; - return new Actor(); + const actor = new Actor(); + (actor as unknown as { config: Configuration }).config = + new Configuration(); + return actor; }; test('should work', async () => { From ae1d9118048096a4c39022cab6942d2bf597d089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:28:48 +0200 Subject: [PATCH 27/34] fix(proxy): support legacy (sessionId, options) two-arg form; trim ProxyInfo shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - newUrl/newProxyInfo accept an optional second `legacyOptions` argument so existing callers that pass `(sessionId, {request})` keep working under the v4 shape too. - Returned ProxyInfo omits Apify-only fields (groups, countryCode) when not using Apify Proxy and only includes `proxyTier` when defined — matches v3's strict-deep-equal expectations. --- packages/apify/src/proxy_configuration.ts | 46 +++++++++++++++-------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/packages/apify/src/proxy_configuration.ts b/packages/apify/src/proxy_configuration.ts index c988b4be71..27e6fd4e6d 100644 --- a/packages/apify/src/proxy_configuration.ts +++ b/packages/apify/src/proxy_configuration.ts @@ -18,19 +18,21 @@ const COUNTRY_CODE_REGEX = /^[A-Z]{2}$/; type CoreProxyOptions = Parameters[0]; /** - * Bridges the SDK's legacy `(sessionId)` calling style with crawlee v4's - * `(options)` shape — pulls `sessionId` from a `Request` carried in `options` - * when no explicit `sessionId` is given. Rejects values that are neither a - * sessionId nor a plain options object (e.g. `Date`, arrays). + * Bridges the SDK's legacy `(sessionId, options?)` calling style with + * crawlee v4's `(options)` shape — pulls `sessionId` from a `Request` + * carried in `options` when no explicit `sessionId` is given. Rejects + * values that are neither a sessionId nor a plain options object + * (e.g. `Date`, arrays). */ function parseSessionIdOrOptions( arg: string | number | CoreProxyOptions | undefined, + legacyOptions?: CoreProxyOptions, ): { sessionId: string | undefined; options: CoreProxyOptions } { if (arg === undefined) { - return { sessionId: undefined, options: undefined }; + return { sessionId: undefined, options: legacyOptions }; } if (typeof arg === 'string' || typeof arg === 'number') { - return { sessionId: String(arg), options: undefined }; + return { sessionId: String(arg), options: legacyOptions }; } if ( typeof arg !== 'object' || @@ -342,13 +344,16 @@ export class ProxyConfiguration extends CoreProxyConfiguration { | string | number | Parameters[0], + legacyOptions?: Parameters[0], ): Promise { // crawlee v4 dropped the `(sessionId, options)` overload — `newProxyInfo` // now takes a single `TieredProxyOptions` argument and pulls `sessionId` // from `options.request`. Keep the SDK's legacy "pass sessionId directly" // shape working by discriminating at runtime. - const { sessionId, options } = - parseSessionIdOrOptions(sessionIdOrOptions); + const { sessionId, options } = parseSessionIdOrOptions( + sessionIdOrOptions, + legacyOptions, + ); ow( sessionId, ow.optional.string @@ -356,7 +361,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration { .matches(APIFY_PROXY_VALUE_REGEX), ); - const url = await this.newUrl(sessionIdOrOptions); + const url = await this.newUrl(sessionIdOrOptions, legacyOptions); if (!url) return undefined; const { groups, countryCode, password, port, hostname } = ( @@ -370,11 +375,12 @@ export class ProxyConfiguration extends CoreProxyConfiguration { // calling convention. const username = new URL(url).username || undefined; - return { + // Build the result lazily: omit Apify-only fields when the SDK is + // wrapping a custom `proxyUrls` rotation (matches v3 shape, which + // tests rely on with strict deep-equal). + const result: Partial = { url, sessionId, - groups, - countryCode, // this.password is not encoded, but the password from the URL will be, we need to normalize password: this.usesApifyProxy ? (password ?? '') @@ -382,8 +388,15 @@ export class ProxyConfiguration extends CoreProxyConfiguration { hostname, port: port!, username, - proxyTier: options?.proxyTier, }; + if (this.usesApifyProxy) { + result.groups = groups; + if (countryCode !== undefined) result.countryCode = countryCode; + } + if (options?.proxyTier !== undefined) { + result.proxyTier = options.proxyTier; + } + return result as ProxyInfo; } /** @@ -404,9 +417,12 @@ export class ProxyConfiguration extends CoreProxyConfiguration { | string | number | Parameters[0], + legacyOptions?: Parameters[0], ): Promise { - const { sessionId, options } = - parseSessionIdOrOptions(sessionIdOrOptions); + const { sessionId, options } = parseSessionIdOrOptions( + sessionIdOrOptions, + legacyOptions, + ); ow( sessionId, ow.optional.string From 327eb314eb686d9f762ae25d1e946bd0f0d20b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:30:52 +0200 Subject: [PATCH 28/34] fix(proxy): decode username; reset cached singletons in createProxyConfiguration tests - ProxyInfo.username is now the decoded form (`user@name` rather than `user%40name`), matching v3 behaviour and the test expectations. - Added a beforeEach to the `Actor.createProxyConfiguration()` describe that resets serviceLocator + Configuration.globalConfig + Actor._instance so each test sees the env vars it sets. --- packages/apify/src/proxy_configuration.ts | 8 ++++++-- test/apify/proxy_configuration.test.ts | 21 +++++++++++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/packages/apify/src/proxy_configuration.ts b/packages/apify/src/proxy_configuration.ts index 27e6fd4e6d..fee9e753dc 100644 --- a/packages/apify/src/proxy_configuration.ts +++ b/packages/apify/src/proxy_configuration.ts @@ -372,8 +372,12 @@ export class ProxyConfiguration extends CoreProxyConfiguration { // on `ProxyInfo` and tests rely on it (e.g. for Apify Proxy session // formatting). v4's `super.newProxyInfo` would surface this, but we // bypass `super` here so the SDK can keep its legacy `sessionId` - // calling convention. - const username = new URL(url).username || undefined; + // calling convention. Decode the URL-encoded username so callers + // see the human-readable form (matches v3 behaviour). + const rawUsername = new URL(url).username; + const username = rawUsername + ? decodeURIComponent(rawUsername) + : undefined; // Build the result lazily: omit Apify-only fields when the SDK is // wrapping a custom `proxyUrls` rotation (matches v3 shape, which diff --git a/test/apify/proxy_configuration.test.ts b/test/apify/proxy_configuration.test.ts index 16fc946810..ecee1ca7a5 100644 --- a/test/apify/proxy_configuration.test.ts +++ b/test/apify/proxy_configuration.test.ts @@ -1,10 +1,23 @@ -import { Actor, ProxyConfiguration } from 'apify'; +import { Actor, Configuration, ProxyConfiguration } from 'apify'; import { UserClient } from 'apify-client'; -import { type Dictionary, Request, sleep } from 'crawlee'; +import { type Dictionary, Request, serviceLocator, sleep } from 'crawlee'; import { gotScraping } from 'got-scraping'; import { APIFY_ENV_VARS, LOCAL_APIFY_ENV_VARS } from '@apify/consts'; +// crawlee v4's Configuration resolves env vars eagerly at construction, +// and the SDK keeps `Configuration.globalConfig` plus `Actor._instance` as +// cached singletons. Tests in this file mutate proxy-related env vars at +// runtime, so we have to clear those caches before each test. +function resetGlobalState() { + serviceLocator.reset(); + ( + Configuration as unknown as { globalConfig?: Configuration } + ).globalConfig = undefined; + // eslint-disable-next-line no-underscore-dangle -- `_instance` is the upstream Actor singleton field + (Actor as unknown as { _instance?: Actor })._instance = undefined; +} + const groups = ['GROUP1', 'GROUP2']; const hostname = LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_HOSTNAME]; const port = Number(LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_PORT]); @@ -559,6 +572,10 @@ describe('ProxyConfiguration', () => { describe('Actor.createProxyConfiguration()', () => { const userData = { proxy: { password } }; + beforeEach(() => { + resetGlobalState(); + }); + test('should work with all options', async () => { const status = { connected: true }; const proxyUrl = proxyUrlNoSession; From 09fdf11e56b30631ecb84f311d267ae96a722f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:33:32 +0200 Subject: [PATCH 29/34] test(actor): import SDK Configuration in test file (not crawlee's) Crawlee's Configuration uses crawleeConfigFields and only knows about `CRAWLEE_INPUT_KEY`. The SDK extension adds `ACTOR_INPUT_KEY` / `APIFY_INPUT_KEY` env-var aliases, which the test relies on. Importing Configuration from 'apify' makes `new Configuration()` inside buildActor() resolve those env vars correctly. --- test/apify/actor.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index ac28de741b..14565498db 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -1,14 +1,15 @@ import { createPublicKey } from 'node:crypto'; -import { - Configuration, - EventType, - serviceLocator, - StorageManager, -} from '@crawlee/core'; +import { EventType, serviceLocator, StorageManager } from '@crawlee/core'; import { sleep } from '@crawlee/utils'; import type { ApifyEnv } from 'apify'; -import { Actor, Dataset, KeyValueStore, ProxyConfiguration } from 'apify'; +import { + Actor, + Configuration, + Dataset, + KeyValueStore, + ProxyConfiguration, +} from 'apify'; import type { WebhookUpdateData } from 'apify-client'; import { ActorClient, ApifyClient, RunClient, TaskClient } from 'apify-client'; From 5b6598accc55816ea8db8467ca2ba9f38f7a6a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:45:07 +0200 Subject: [PATCH 30/34] =?UTF-8?q?chore:=20add=20cheerio=20as=20devDep=20?= =?UTF-8?q?=E2=80=94=20workaround=20missing=20@crawlee/linkedom=20dep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `@crawlee/linkedom@4.0.0-beta.49`'s `linkedom-crawler.js` imports `cheerio` without declaring it as a dependency. Locally this works when a parent directory has cheerio installed; CI's fresh install fails. Adding it directly here keeps tests green until the upstream package fixes the missing dep declaration. --- package-lock.json | 418 +++++++--------------------------------------- package.json | 7 +- 2 files changed, 65 insertions(+), 360 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8923e74741..4f9107adf9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "@types/semver": "^7.7.0", "@types/tough-cookie": "^4.0.5", "@types/ws": "^8.18.1", + "cheerio": "^1.2.0", "commitlint": "^19.8.1", "crawlee": "^4.0.0-beta.49", "eslint": "^9.27.0", @@ -492,75 +493,6 @@ "node": ">=22.0.0" } }, - "node_modules/@crawlee/cheerio/node_modules/cheerio": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", - "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cheerio-select": "^2.1.0", - "dom-serializer": "^2.0.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "encoding-sniffer": "^0.2.1", - "htmlparser2": "^10.1.0", - "parse5": "^7.3.0", - "parse5-htmlparser2-tree-adapter": "^7.1.0", - "parse5-parser-stream": "^7.1.2", - "undici": "^7.19.0", - "whatwg-mimetype": "^4.0.0" - }, - "engines": { - "node": ">=20.18.1" - }, - "funding": { - "url": "https://github.com/cheeriojs/cheerio?sponsor=1" - } - }, - "node_modules/@crawlee/cheerio/node_modules/entities": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", - "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/@crawlee/cheerio/node_modules/htmlparser2": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", - "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", - "dev": true, - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "entities": "^7.0.1" - } - }, - "node_modules/@crawlee/cheerio/node_modules/undici": { - "version": "7.25.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", - "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, "node_modules/@crawlee/cli": { "version": "4.0.0-beta.49", "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-4.0.0-beta.49.tgz", @@ -965,32 +897,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/http/node_modules/cheerio": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", - "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cheerio-select": "^2.1.0", - "dom-serializer": "^2.0.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "encoding-sniffer": "^0.2.1", - "htmlparser2": "^10.1.0", - "parse5": "^7.3.0", - "parse5-htmlparser2-tree-adapter": "^7.1.0", - "parse5-parser-stream": "^7.1.2", - "undici": "^7.19.0", - "whatwg-mimetype": "^4.0.0" - }, - "engines": { - "node": ">=20.18.1" - }, - "funding": { - "url": "https://github.com/cheeriojs/cheerio?sponsor=1" - } - }, "node_modules/@crawlee/http/node_modules/dot-prop": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-8.0.2.tgz", @@ -1020,39 +926,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/http/node_modules/entities": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", - "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/@crawlee/http/node_modules/htmlparser2": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", - "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", - "dev": true, - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "entities": "^7.0.1" - } - }, "node_modules/@crawlee/http/node_modules/iconv-lite": { "version": "0.7.2", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", @@ -1118,16 +991,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/http/node_modules/undici": { - "version": "7.25.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", - "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, "node_modules/@crawlee/jsdom": { "version": "4.0.0-beta.49", "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-4.0.0-beta.49.tgz", @@ -1176,32 +1039,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/jsdom/node_modules/cheerio": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", - "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cheerio-select": "^2.1.0", - "dom-serializer": "^2.0.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "encoding-sniffer": "^0.2.1", - "htmlparser2": "^10.1.0", - "parse5": "^7.3.0", - "parse5-htmlparser2-tree-adapter": "^7.1.0", - "parse5-parser-stream": "^7.1.2", - "undici": "^7.19.0", - "whatwg-mimetype": "^4.0.0" - }, - "engines": { - "node": ">=20.18.1" - }, - "funding": { - "url": "https://github.com/cheeriojs/cheerio?sponsor=1" - } - }, "node_modules/@crawlee/jsdom/node_modules/dot-prop": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-8.0.2.tgz", @@ -1218,39 +1055,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/jsdom/node_modules/entities": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", - "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/@crawlee/jsdom/node_modules/htmlparser2": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", - "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", - "dev": true, - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "entities": "^7.0.1" - } - }, "node_modules/@crawlee/jsdom/node_modules/ow": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ow/-/ow-2.0.0.tgz", @@ -1285,16 +1089,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/jsdom/node_modules/undici": { - "version": "7.25.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", - "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, "node_modules/@crawlee/linkedom": { "version": "4.0.0-beta.49", "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-4.0.0-beta.49.tgz", @@ -1549,31 +1343,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/utils/node_modules/cheerio": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", - "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", - "license": "MIT", - "dependencies": { - "cheerio-select": "^2.1.0", - "dom-serializer": "^2.0.0", - "domhandler": "^5.0.3", - "domutils": "^3.1.0", - "encoding-sniffer": "^0.2.0", - "htmlparser2": "^9.1.0", - "parse5": "^7.1.2", - "parse5-htmlparser2-tree-adapter": "^7.0.0", - "parse5-parser-stream": "^7.1.2", - "undici": "^6.19.5", - "whatwg-mimetype": "^4.0.0" - }, - "engines": { - "node": ">=18.17" - }, - "funding": { - "url": "https://github.com/cheeriojs/cheerio?sponsor=1" - } - }, "node_modules/@crawlee/utils/node_modules/dot-prop": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-8.0.2.tgz", @@ -1589,25 +1358,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@crawlee/utils/node_modules/htmlparser2": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", - "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.1.0", - "entities": "^4.5.0" - } - }, "node_modules/@crawlee/utils/node_modules/ow": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ow/-/ow-2.0.0.tgz", @@ -6470,6 +6220,31 @@ "node": ">= 16" } }, + "node_modules/cheerio": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^7.19.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=20.18.1" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, "node_modules/cheerio-select": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", @@ -8265,32 +8040,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/crawlee/node_modules/cheerio": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", - "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cheerio-select": "^2.1.0", - "dom-serializer": "^2.0.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "encoding-sniffer": "^0.2.1", - "htmlparser2": "^10.1.0", - "parse5": "^7.3.0", - "parse5-htmlparser2-tree-adapter": "^7.1.0", - "parse5-parser-stream": "^7.1.2", - "undici": "^7.19.0", - "whatwg-mimetype": "^4.0.0" - }, - "engines": { - "node": ">=20.18.1" - }, - "funding": { - "url": "https://github.com/cheeriojs/cheerio?sponsor=1" - } - }, "node_modules/crawlee/node_modules/dot-prop": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-8.0.2.tgz", @@ -8320,39 +8069,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/crawlee/node_modules/entities": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", - "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/crawlee/node_modules/htmlparser2": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", - "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", - "dev": true, - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "entities": "^7.0.1" - } - }, "node_modules/crawlee/node_modules/nanoid": { "version": "5.1.9", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.9.tgz", @@ -8422,16 +8138,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/crawlee/node_modules/undici": { - "version": "7.25.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", - "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, "node_modules/crawlee/node_modules/yocto-queue": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", @@ -11657,6 +11363,37 @@ "dev": true, "license": "MIT" }, + "node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/http-cache-semantics": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", @@ -13475,39 +13212,6 @@ } } }, - "node_modules/linkedom/node_modules/entities": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", - "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/linkedom/node_modules/htmlparser2": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", - "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", - "dev": true, - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "entities": "^7.0.1" - } - }, "node_modules/lint-staged": { "version": "16.0.0", "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-16.0.0.tgz", @@ -19031,12 +18735,12 @@ } }, "node_modules/undici": { - "version": "6.21.3", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.3.tgz", - "integrity": "sha512-gBLkYIlEnSp8pFbT64yFgGE6UIB9tAkhukC23PmMDCe5Nd+cRqKxSjw5y54MK2AZMgZfJWMaNE4nYUHgi1XEOw==", + "version": "7.25.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", + "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", "license": "MIT", "engines": { - "node": ">=18.17" + "node": ">=20.18.1" } }, "node_modules/undici-types": { diff --git a/package.json b/package.json index 9fed883e8f..cf480a6f5d 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,9 @@ "@apify/input_secrets": "^1.1.72", "@apify/tsconfig": "^0.1.1", "@commitlint/config-conventional": "^19.8.1", + "@crawlee/core": "^4.0.0-beta.49", + "@crawlee/types": "^4.0.0-beta.49", + "@crawlee/utils": "^4.0.0-beta.49", "@playwright/browser-chromium": "^1.52.0", "@types/content-type": "^1.1.8", "@types/fs-extra": "^11.0.4", @@ -77,11 +80,9 @@ "@types/semver": "^7.7.0", "@types/tough-cookie": "^4.0.5", "@types/ws": "^8.18.1", + "cheerio": "^1.2.0", "commitlint": "^19.8.1", "crawlee": "^4.0.0-beta.49", - "@crawlee/core": "^4.0.0-beta.49", - "@crawlee/types": "^4.0.0-beta.49", - "@crawlee/utils": "^4.0.0-beta.49", "eslint": "^9.27.0", "eslint-config-prettier": "^10.1.5", "fs-extra": "^11.3.0", From 32634c26b5213048ec8dbe39a6753f9d3312c298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 19:50:44 +0200 Subject: [PATCH 31/34] test: clear Configuration AsyncLocalStorage between tests (Node 22 fix) Actor.init() calls Configuration.storage.enterWith(this.config), which sticks the resolved config onto the current async context and persists across tests on Node 22 (but not Node 24+). The cached value short- circuits Configuration.getGlobalConfig() so subsequent tests never see the env vars they just set. Reset the AsyncLocalStorage value alongside the other singletons in the test emulator so addWebhook (and friends) see ACTOR_RUN_ID etc. --- test/MemoryStorageEmulator.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/MemoryStorageEmulator.ts b/test/MemoryStorageEmulator.ts index cab9eab144..7851667ea7 100644 --- a/test/MemoryStorageEmulator.ts +++ b/test/MemoryStorageEmulator.ts @@ -20,6 +20,16 @@ function resetGlobalState() { ).globalConfig = undefined; // eslint-disable-next-line no-underscore-dangle -- `_instance` is the upstream Actor singleton field (Actor as unknown as { _instance?: Actor })._instance = undefined; + // `Actor.init()` calls `Configuration.storage.enterWith(this.config)`, + // which sticks the resolved config onto the current async context and + // persists across tests on some Node versions (observed on Node 22 + // but not Node 24+). Clear the AsyncLocalStorage value so the next + // `Configuration.getGlobalConfig()` falls through to a fresh build. + if (Configuration.storage?.getStore()) { + ( + Configuration.storage as unknown as { enterWith(v: unknown): void } + ).enterWith(undefined); + } } const LOCAL_EMULATION_DIR = resolve( From 2711b958128ff552d1d292f6d17c61cf15e533af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 20:14:55 +0200 Subject: [PATCH 32/34] chore: bump crawlee to ^4.0.0-beta.51 and drop cheerio workaround `@crawlee/linkedom@4.0.0-beta.51` now declares cheerio as a direct dependency (apify/crawlee#3620), so the SDK no longer has to ship its own cheerio devDep to mask the missing declaration. --- package-lock.json | 252 ++++++++++++++++++------------------ package.json | 9 +- packages/apify/package.json | 6 +- 3 files changed, 133 insertions(+), 134 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4f9107adf9..42a6d9ef0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,9 +16,9 @@ "@apify/input_secrets": "^1.1.72", "@apify/tsconfig": "^0.1.1", "@commitlint/config-conventional": "^19.8.1", - "@crawlee/core": "^4.0.0-beta.49", - "@crawlee/types": "^4.0.0-beta.49", - "@crawlee/utils": "^4.0.0-beta.49", + "@crawlee/core": "^4.0.0-beta.51", + "@crawlee/types": "^4.0.0-beta.51", + "@crawlee/utils": "^4.0.0-beta.51", "@playwright/browser-chromium": "^1.52.0", "@types/content-type": "^1.1.8", "@types/fs-extra": "^11.0.4", @@ -26,9 +26,8 @@ "@types/semver": "^7.7.0", "@types/tough-cookie": "^4.0.5", "@types/ws": "^8.18.1", - "cheerio": "^1.2.0", "commitlint": "^19.8.1", - "crawlee": "^4.0.0-beta.49", + "crawlee": "^4.0.0-beta.51", "eslint": "^9.27.0", "eslint-config-prettier": "^10.1.5", "fs-extra": "^11.3.0", @@ -476,15 +475,15 @@ } }, "node_modules/@crawlee/cheerio": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-4.0.0-beta.49.tgz", - "integrity": "sha512-WYFTBQE+7Rzg/cb0OxeOx7yF/IgZTHZLsT0HQEA1yV/0YWuxUFFydJeUDYFhkP3xyzutuOR81sPYkI5xAf/qrQ==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-4.0.0-beta.51.tgz", + "integrity": "sha512-PrJ4QHdSp57R3cL6+vGCWXrl9avyhjZGKJf+Z1pOpnpc/U5eZPmcdhVvOJEou1Tgm4gYDflMDtrYuGfBsAVB6w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@crawlee/http": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/http": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "cheerio": "^1.0.0", "htmlparser2": "^10.0.0", "tslib": "^2.8.1" @@ -494,13 +493,13 @@ } }, "node_modules/@crawlee/cli": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-4.0.0-beta.49.tgz", - "integrity": "sha512-QhEqBe274Q0ogdiLD/3QtToFQwLisYT+5RHOsJW7EkGEsQ9kJ/h5KD4IRi9CsDStB9IEww5vFv0tJRcwwWqpzQ==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-4.0.0-beta.51.tgz", + "integrity": "sha512-AZtps02lgeQyX0zQ5uiWbcWU8FhXp7XsIzYtbzMg4zR9KQgzhevtpj8rxuBBlzCNgPMqJgYX86LPXdGxr9sz+w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@crawlee/templates": "4.0.0-beta.49", + "@crawlee/templates": "4.0.0-beta.51", "@inquirer/prompts": "^7.5.0", "ansi-colors": "^4.1.3", "fs-extra": "^11.3.0", @@ -643,9 +642,9 @@ } }, "node_modules/@crawlee/core": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-4.0.0-beta.49.tgz", - "integrity": "sha512-k5FBU/R+gwrL/FWu6fFSciU4aM+nYsUJy4YOAk3Cur9Yom8YF8JRbGGeAaFEvDDjJZxWTR5s0GyYnzHeidx9Gw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-4.0.0-beta.51.tgz", + "integrity": "sha512-zCmpN5kiNfZjfX1//JgvYrMXxv7toCF47aZxf/uyCbFqpEAQoatrBSsqZeNLPnj/90acG5UUfqF9wVM9Ie3Zug==", "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.41.0", @@ -654,9 +653,9 @@ "@apify/pseudo_url": "^2.0.59", "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/memory-storage": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/memory-storage": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "@sapphire/async-queue": "^1.5.5", "@vladfrangu/async_event_emitter": "^2.4.6", "csv-stringify": "^6.5.2", @@ -782,13 +781,13 @@ } }, "node_modules/@crawlee/got-scraping-client": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/got-scraping-client/-/got-scraping-client-4.0.0-beta.49.tgz", - "integrity": "sha512-L1IWFC/kBQx9TNg/F7SVMjlnG92T95I1ZEL76x5dyGP9hBWBjaZHL8vsVSnoHDYh7ADWsbKjHyyD1CsMUjVimw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/got-scraping-client/-/got-scraping-client-4.0.0-beta.51.tgz", + "integrity": "sha512-IKT7BrlIU/rtFi+b6wnvN47uos34sxVGlX4vZQ8OnOZKHTfNLg1cWBqz/BZhLfgxr5B7ZnmkHWzCBr22kM3zDg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@crawlee/http-client": "4.0.0-beta.49", + "@crawlee/http-client": "4.0.0-beta.51", "got-scraping": "^4.2.1" }, "engines": { @@ -796,19 +795,19 @@ } }, "node_modules/@crawlee/http": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-4.0.0-beta.49.tgz", - "integrity": "sha512-q6vJ29s1TLynGBKPoHx4AbIMyl1KFw3oRPPWLakOj4wqK72W8KqG/0BHP6LPtgwBBgBF4TKhxEnp7xMT29f0KQ==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-4.0.0-beta.51.tgz", + "integrity": "sha512-FUaGGZt/PbHuyDgNCna7uqOcq6J5rlus5Sy+LDU/TCTJTcCfuyYkRyEZ4XAba6QoFhSh8ykq8U2v7XQgdm3W5A==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/basic": "4.0.0-beta.49", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/http-client": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/basic": "4.0.0-beta.51", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/http-client": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "@types/content-type": "^1.1.8", "cheerio": "^1.0.0", "content-type": "^1.0.5", @@ -823,12 +822,12 @@ } }, "node_modules/@crawlee/http-client": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/http-client/-/http-client-4.0.0-beta.49.tgz", - "integrity": "sha512-UMK5FLyFmAZHFNaaMhITwKQgEPdJZ5GJYvGCDHig4H5Hmc83aiBWIgGNqmQL+t/1YXkWVA0NhrBAgonk3m64aw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/http-client/-/http-client-4.0.0-beta.51.tgz", + "integrity": "sha512-/NcX3ESNXGRVYGBgkKCE035OidlOgsFYAd5zBSWTep/zsLefUISq2HEPTZYVpAvrgIVyaVjbNSilk1NpVjJLxw==", "license": "Apache-2.0", "dependencies": { - "@crawlee/types": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.51", "tough-cookie": "^6.0.0" }, "engines": { @@ -848,18 +847,18 @@ } }, "node_modules/@crawlee/http/node_modules/@crawlee/basic": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.49.tgz", - "integrity": "sha512-6rRchZXH1KyLvkcZpTPeuRcbEDdl/l0XOsVTK5lHxkQvj63LrE+YwlmZxEA3LViiva9soNUll7VoFfWhEL77iw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.51.tgz", + "integrity": "sha512-t4YhLgqWiy4N17YzeLugUIiwQDtXALqTl3GrKSZFnZSuHHux6nlQwbSX8x5FCtzW0q/On9ZGt/GTR+WMTka97g==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/got-scraping-client": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/got-scraping-client": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "csv-stringify": "^6.5.2", "fs-extra": "^11.3.0", "ow": "^2.0.0", @@ -992,17 +991,17 @@ } }, "node_modules/@crawlee/jsdom": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-4.0.0-beta.49.tgz", - "integrity": "sha512-RpO6fJfB8GMxokqfNTanU/zTSZJCFCw0fu3aTfyTG5njk/IENySRyJcIFWz+uUNuzF3trJVW4XEJBq4PBzvUjA==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-4.0.0-beta.51.tgz", + "integrity": "sha512-x2w8NkQfu14LO4yvVGTi4kNWDn3cgJq+XNSqS3UNAOraxQUPTSJpEicR+hwjt21Fb/tQRKMLRhdTgnqACWRoFw==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/http": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/http": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "@types/jsdom": "^21.1.7", "cheerio": "^1.0.0", "jsdom": "^26.1.0", @@ -1090,17 +1089,18 @@ } }, "node_modules/@crawlee/linkedom": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-4.0.0-beta.49.tgz", - "integrity": "sha512-fOBVgl4sVcrKM/3DF96SfaMQjJQuEoMvtci67X1huPrPtjkrREqGrDufCtSttkcSkGu/gQCxMN/yv+oyzBAujQ==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-4.0.0-beta.51.tgz", + "integrity": "sha512-OY2ku6ZO2M26ViuYygBGDntxIVNd4zbuaglPdbgyLekX2lw5e6tbqWSLZ7pOF1FTOllCpXVRetRRArWcOBMc7A==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/http": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/http": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", + "cheerio": "^1.0.0", "linkedom": "^0.18.10", "ow": "^2.0.0", "tslib": "^2.8.1" @@ -1186,12 +1186,12 @@ } }, "node_modules/@crawlee/memory-storage": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-4.0.0-beta.49.tgz", - "integrity": "sha512-rLniynMhfrJoXn7rFHfj4NpsVvdRn3PIJi6YFL3G7s4JAiqqfhYOKshW+f6XdldEeb1z0YyGpgOknLo6VupktA==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-4.0.0-beta.51.tgz", + "integrity": "sha512-SiaQjdZaA7S7BdK5NeCK8LR0GeCNJq4x6qlv7fcQ+MqebLHKNEcmUcuwYXf9u4IoYweNfO+v9N+RkmfjyP4C1A==", "license": "Apache-2.0", "dependencies": { - "@crawlee/types": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.51", "@sapphire/async-queue": "^1.5.5", "@sapphire/shapeshift": "^4.0.0", "content-type": "^1.0.5", @@ -1259,9 +1259,9 @@ } }, "node_modules/@crawlee/templates": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-4.0.0-beta.49.tgz", - "integrity": "sha512-5/cBYgNuMXnVemxUDqagg0lgZURlc28NVuY1vfON+CG0C6Dg/GLwMlTLOmcroQeDJF7bWxVxDrsHomTk0Mm30A==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-4.0.0-beta.51.tgz", + "integrity": "sha512-YDGdBpv2ztbdY1EbVOV3lyJWjGeYGhBprA3WD58sLX+eIx7vUiZUZXdznt3AMf1FnGvfbCsbpFoVPa72xydtYw==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -1272,9 +1272,9 @@ } }, "node_modules/@crawlee/types": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-4.0.0-beta.49.tgz", - "integrity": "sha512-p33njES5N539Zgco1aroUZgmWeXp+i6MTPUk7R7wLEdV/k+aw2Y1cjw8wGQpKH9/ZZIpsvqx56U26bC2w1nmcQ==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-4.0.0-beta.51.tgz", + "integrity": "sha512-U/m8K41Fe8q5rvLqFvjclNTvRPvRsrphatEBssV1J8q/Rjhn71SnVVB0qVKYusNMLjLnaup3tlBY/01ahJDDSw==", "license": "Apache-2.0", "dependencies": { "tough-cookie": "^6.0.0", @@ -1297,14 +1297,14 @@ } }, "node_modules/@crawlee/utils": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-4.0.0-beta.49.tgz", - "integrity": "sha512-JmTTVsJBFJNPUZsdH35Yde/jRFJoZtloVtEYLGZoqo79rT9RmRi+SDr22XFYJ7bb/LEXY+NOWBovdMge+fnpHw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-4.0.0-beta.51.tgz", + "integrity": "sha512-DHvn2fo/6pIHTiHAi5tenkSWWi2KTm6sicnAIY38H98ZOiys4Yxh22wvfRN4cE8wm8pbUucQ8IYIzKSxbNLd+Q==", "license": "Apache-2.0", "dependencies": { "@apify/ps-tree": "^1.2.0", - "@crawlee/http-client": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", + "@crawlee/http-client": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", "@types/sax": "^1.2.7", "cheerio": "^1.0.0", "domhandler": "^5.0.3", @@ -7793,24 +7793,24 @@ } }, "node_modules/crawlee": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-4.0.0-beta.49.tgz", - "integrity": "sha512-/+m7LJ0m4nLmdA+zzb0ZL21DjNGUjveoPs+ckbiaPpE0LT8bJBalBbzhRcmUCZHWuY5L6ncNJcS2HTGBe20VaA==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-4.0.0-beta.51.tgz", + "integrity": "sha512-i/w8O7D7mdmUr8hdKEWSSHEIGqk4+nrknoWtEJiwnyOQAyzg7G98QsHUBeGmAR+j40EDPJui3wzrh7B6tUNkMA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@crawlee/basic": "4.0.0-beta.49", - "@crawlee/browser": "4.0.0-beta.49", - "@crawlee/browser-pool": "4.0.0-beta.49", - "@crawlee/cheerio": "4.0.0-beta.49", - "@crawlee/cli": "4.0.0-beta.49", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/http": "4.0.0-beta.49", - "@crawlee/jsdom": "4.0.0-beta.49", - "@crawlee/linkedom": "4.0.0-beta.49", - "@crawlee/playwright": "4.0.0-beta.49", - "@crawlee/puppeteer": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/basic": "4.0.0-beta.51", + "@crawlee/browser": "4.0.0-beta.51", + "@crawlee/browser-pool": "4.0.0-beta.51", + "@crawlee/cheerio": "4.0.0-beta.51", + "@crawlee/cli": "4.0.0-beta.51", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/http": "4.0.0-beta.51", + "@crawlee/jsdom": "4.0.0-beta.51", + "@crawlee/linkedom": "4.0.0-beta.51", + "@crawlee/playwright": "4.0.0-beta.51", + "@crawlee/puppeteer": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "import-local": "^3.2.0", "tslib": "^2.8.1" }, @@ -7838,18 +7838,18 @@ } }, "node_modules/crawlee/node_modules/@crawlee/basic": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.49.tgz", - "integrity": "sha512-6rRchZXH1KyLvkcZpTPeuRcbEDdl/l0XOsVTK5lHxkQvj63LrE+YwlmZxEA3LViiva9soNUll7VoFfWhEL77iw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.51.tgz", + "integrity": "sha512-t4YhLgqWiy4N17YzeLugUIiwQDtXALqTl3GrKSZFnZSuHHux6nlQwbSX8x5FCtzW0q/On9ZGt/GTR+WMTka97g==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/got-scraping-client": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/got-scraping-client": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "csv-stringify": "^6.5.2", "fs-extra": "^11.3.0", "ow": "^2.0.0", @@ -7862,17 +7862,17 @@ } }, "node_modules/crawlee/node_modules/@crawlee/browser": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-4.0.0-beta.49.tgz", - "integrity": "sha512-RBzbtscHGi7qY7bn3KqsfV2flFt0XlKp7OOdGx8VsSSzdVhIb+YXFOrcdMR6DGEyvxtMSZxNZ4U8qaKM/YwOUg==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-4.0.0-beta.51.tgz", + "integrity": "sha512-Z0VoAetDT5HL8vtQYiZkOVrFFpoeoq/6Yvc6hK6Sp6tQ5Snb/+uVXcTL9LTQ0a/9G0S3yzbgvuYaHpR2AInK1Q==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", - "@crawlee/basic": "4.0.0-beta.49", - "@crawlee/browser-pool": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/basic": "4.0.0-beta.51", + "@crawlee/browser-pool": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "ow": "^2.0.0", "tslib": "^2.8.1", "type-fest": "^4.41.0" @@ -7894,15 +7894,15 @@ } }, "node_modules/crawlee/node_modules/@crawlee/browser-pool": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-4.0.0-beta.49.tgz", - "integrity": "sha512-oroowDwagffg1Ybbn3y3Eey2i2p+rhdu9f2dBtDvH4EF+4KPpfW7Z5FG9knLTx+3OcaRp9QumZUuyFANH+wN1w==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-4.0.0-beta.51.tgz", + "integrity": "sha512-2YSgDy8QFrTpzYYiLSctcWyu9MaQzaV0fDrcRMO/a1+WeiwPc5cOwpRjtJYXVRjN4ty1x3/pUbY+mNaGjUDUng==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", "fingerprint-generator": "^2.1.68", "fingerprint-injector": "^2.1.68", "lodash.merge": "^4.6.2", @@ -7931,21 +7931,21 @@ } }, "node_modules/crawlee/node_modules/@crawlee/playwright": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-4.0.0-beta.49.tgz", - "integrity": "sha512-ozU/eUAgnW0AhyWYUe4Kvipqfl8JyJ2ZZoB5jqWG7PG0zmgEx9CQ5s+aMopl0zmZu31dW7nnZk9HQZa71ZYcTg==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-4.0.0-beta.51.tgz", + "integrity": "sha512-kJFB2VEeD8xp42oQuf1yReADuhuqZnHxLU5H6IneJ0BhG4JHy5/Za2QT7LtkwBfATZSbIyxiBlgk8vevU25t5A==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.3", "@apify/timeout": "^0.3.2", - "@crawlee/basic": "4.0.0-beta.49", - "@crawlee/browser": "4.0.0-beta.49", - "@crawlee/browser-pool": "4.0.0-beta.49", - "@crawlee/cheerio": "4.0.0-beta.49", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/basic": "4.0.0-beta.51", + "@crawlee/browser": "4.0.0-beta.51", + "@crawlee/browser-pool": "4.0.0-beta.51", + "@crawlee/cheerio": "4.0.0-beta.51", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "cheerio": "^1.0.0", "idcac-playwright": "^0.1.3", "jquery": "^3.7.1", @@ -7979,18 +7979,18 @@ "license": "ISC" }, "node_modules/crawlee/node_modules/@crawlee/puppeteer": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-4.0.0-beta.49.tgz", - "integrity": "sha512-fpMaaXqSNr86FTWVvkWvNXlKRApRtURo37U9AHVLtwQTyBbmgFeETnfjHauAlEj29mFaRgk4lyKXEcDmWU/Psw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-4.0.0-beta.51.tgz", + "integrity": "sha512-IrPUaR9Y4AeVyCnfZSAWHVvFxAAhVlCJDuEtcL7YIhAuBrvTRW9aGqx6vAhzTsrEwpvuk1OX2mL1lE8s/xbUiw==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.3", - "@crawlee/browser": "4.0.0-beta.49", - "@crawlee/browser-pool": "4.0.0-beta.49", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/browser": "4.0.0-beta.51", + "@crawlee/browser-pool": "4.0.0-beta.51", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "cheerio": "^1.0.0", "devtools-protocol": "*", "idcac-playwright": "^0.2.0", @@ -20286,9 +20286,9 @@ "@apify/log": "^2.5.18", "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "^4.0.0-beta.49", - "@crawlee/types": "^4.0.0-beta.49", - "@crawlee/utils": "^4.0.0-beta.49", + "@crawlee/core": "^4.0.0-beta.51", + "@crawlee/types": "^4.0.0-beta.51", + "@crawlee/utils": "^4.0.0-beta.51", "apify-client": "^2.12.4", "fs-extra": "^11.3.0", "got-scraping": "^4.1.1", diff --git a/package.json b/package.json index cf480a6f5d..7d7ba48891 100644 --- a/package.json +++ b/package.json @@ -70,9 +70,9 @@ "@apify/input_secrets": "^1.1.72", "@apify/tsconfig": "^0.1.1", "@commitlint/config-conventional": "^19.8.1", - "@crawlee/core": "^4.0.0-beta.49", - "@crawlee/types": "^4.0.0-beta.49", - "@crawlee/utils": "^4.0.0-beta.49", + "@crawlee/core": "^4.0.0-beta.51", + "@crawlee/types": "^4.0.0-beta.51", + "@crawlee/utils": "^4.0.0-beta.51", "@playwright/browser-chromium": "^1.52.0", "@types/content-type": "^1.1.8", "@types/fs-extra": "^11.0.4", @@ -80,9 +80,8 @@ "@types/semver": "^7.7.0", "@types/tough-cookie": "^4.0.5", "@types/ws": "^8.18.1", - "cheerio": "^1.2.0", "commitlint": "^19.8.1", - "crawlee": "^4.0.0-beta.49", + "crawlee": "^4.0.0-beta.51", "eslint": "^9.27.0", "eslint-config-prettier": "^10.1.5", "fs-extra": "^11.3.0", diff --git a/packages/apify/package.json b/packages/apify/package.json index 7cea7e856d..9173c9696b 100644 --- a/packages/apify/package.json +++ b/packages/apify/package.json @@ -53,9 +53,9 @@ "@apify/log": "^2.5.18", "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "^4.0.0-beta.49", - "@crawlee/types": "^4.0.0-beta.49", - "@crawlee/utils": "^4.0.0-beta.49", + "@crawlee/core": "^4.0.0-beta.51", + "@crawlee/types": "^4.0.0-beta.51", + "@crawlee/utils": "^4.0.0-beta.51", "apify-client": "^2.12.4", "fs-extra": "^11.3.0", "got-scraping": "^4.1.1", From 40d296c2bae910e81c10ecfc6ca61ed0bc6ba4c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 20:22:12 +0200 Subject: [PATCH 33/34] fix(proxy): drop tieredProxyUrls/tieredProxyConfig support crawlee v4 (apify/crawlee#3599, beta.51) removed `tieredProxyUrls`, `tieredProxyConfig`, `_handleTieredUrl`, and `proxyTier` from `ProxyConfiguration` / `ProxyInfo`. The SDK's wrapper used to thread those through to the base class; with the upstream API gone, that plumbing has to go too. - Remove the `tieredProxyConfig` field from the SDK's `ProxyConfigurationOptions`. - Drop the constructor branch that forwarded `tieredProxyUrls` / `tieredProxyConfig` to the base class and the now-unreachable `_generateTieredProxyUrls` helper. - Drop the `tieredProxyUrls` short-circuit and `proxyTier` field from `newUrl` / `newProxyInfo`. - Drop the corresponding test groups in `proxy_configuration.test.ts`. --- package-lock.json | 320 +++++++++++++--------- package.json | 8 +- packages/apify/package.json | 6 +- packages/apify/src/proxy_configuration.ts | 52 +--- test/apify/proxy_configuration.test.ts | 139 +--------- 5 files changed, 214 insertions(+), 311 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8923e74741..8c26ac07c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,9 +16,9 @@ "@apify/input_secrets": "^1.1.72", "@apify/tsconfig": "^0.1.1", "@commitlint/config-conventional": "^19.8.1", - "@crawlee/core": "^4.0.0-beta.49", - "@crawlee/types": "^4.0.0-beta.49", - "@crawlee/utils": "^4.0.0-beta.49", + "@crawlee/core": "^4.0.0-beta.51", + "@crawlee/types": "^4.0.0-beta.51", + "@crawlee/utils": "^4.0.0-beta.51", "@playwright/browser-chromium": "^1.52.0", "@types/content-type": "^1.1.8", "@types/fs-extra": "^11.0.4", @@ -27,7 +27,7 @@ "@types/tough-cookie": "^4.0.5", "@types/ws": "^8.18.1", "commitlint": "^19.8.1", - "crawlee": "^4.0.0-beta.49", + "crawlee": "^4.0.0-beta.51", "eslint": "^9.27.0", "eslint-config-prettier": "^10.1.5", "fs-extra": "^11.3.0", @@ -475,15 +475,15 @@ } }, "node_modules/@crawlee/cheerio": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-4.0.0-beta.49.tgz", - "integrity": "sha512-WYFTBQE+7Rzg/cb0OxeOx7yF/IgZTHZLsT0HQEA1yV/0YWuxUFFydJeUDYFhkP3xyzutuOR81sPYkI5xAf/qrQ==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/cheerio/-/cheerio-4.0.0-beta.51.tgz", + "integrity": "sha512-PrJ4QHdSp57R3cL6+vGCWXrl9avyhjZGKJf+Z1pOpnpc/U5eZPmcdhVvOJEou1Tgm4gYDflMDtrYuGfBsAVB6w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@crawlee/http": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/http": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "cheerio": "^1.0.0", "htmlparser2": "^10.0.0", "tslib": "^2.8.1" @@ -562,13 +562,13 @@ } }, "node_modules/@crawlee/cli": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-4.0.0-beta.49.tgz", - "integrity": "sha512-QhEqBe274Q0ogdiLD/3QtToFQwLisYT+5RHOsJW7EkGEsQ9kJ/h5KD4IRi9CsDStB9IEww5vFv0tJRcwwWqpzQ==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-4.0.0-beta.51.tgz", + "integrity": "sha512-AZtps02lgeQyX0zQ5uiWbcWU8FhXp7XsIzYtbzMg4zR9KQgzhevtpj8rxuBBlzCNgPMqJgYX86LPXdGxr9sz+w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@crawlee/templates": "4.0.0-beta.49", + "@crawlee/templates": "4.0.0-beta.51", "@inquirer/prompts": "^7.5.0", "ansi-colors": "^4.1.3", "fs-extra": "^11.3.0", @@ -711,9 +711,9 @@ } }, "node_modules/@crawlee/core": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-4.0.0-beta.49.tgz", - "integrity": "sha512-k5FBU/R+gwrL/FWu6fFSciU4aM+nYsUJy4YOAk3Cur9Yom8YF8JRbGGeAaFEvDDjJZxWTR5s0GyYnzHeidx9Gw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-4.0.0-beta.51.tgz", + "integrity": "sha512-zCmpN5kiNfZjfX1//JgvYrMXxv7toCF47aZxf/uyCbFqpEAQoatrBSsqZeNLPnj/90acG5UUfqF9wVM9Ie3Zug==", "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.41.0", @@ -722,9 +722,9 @@ "@apify/pseudo_url": "^2.0.59", "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/memory-storage": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/memory-storage": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "@sapphire/async-queue": "^1.5.5", "@vladfrangu/async_event_emitter": "^2.4.6", "csv-stringify": "^6.5.2", @@ -850,13 +850,13 @@ } }, "node_modules/@crawlee/got-scraping-client": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/got-scraping-client/-/got-scraping-client-4.0.0-beta.49.tgz", - "integrity": "sha512-L1IWFC/kBQx9TNg/F7SVMjlnG92T95I1ZEL76x5dyGP9hBWBjaZHL8vsVSnoHDYh7ADWsbKjHyyD1CsMUjVimw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/got-scraping-client/-/got-scraping-client-4.0.0-beta.51.tgz", + "integrity": "sha512-IKT7BrlIU/rtFi+b6wnvN47uos34sxVGlX4vZQ8OnOZKHTfNLg1cWBqz/BZhLfgxr5B7ZnmkHWzCBr22kM3zDg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@crawlee/http-client": "4.0.0-beta.49", + "@crawlee/http-client": "4.0.0-beta.51", "got-scraping": "^4.2.1" }, "engines": { @@ -864,19 +864,19 @@ } }, "node_modules/@crawlee/http": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-4.0.0-beta.49.tgz", - "integrity": "sha512-q6vJ29s1TLynGBKPoHx4AbIMyl1KFw3oRPPWLakOj4wqK72W8KqG/0BHP6LPtgwBBgBF4TKhxEnp7xMT29f0KQ==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/http/-/http-4.0.0-beta.51.tgz", + "integrity": "sha512-FUaGGZt/PbHuyDgNCna7uqOcq6J5rlus5Sy+LDU/TCTJTcCfuyYkRyEZ4XAba6QoFhSh8ykq8U2v7XQgdm3W5A==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/basic": "4.0.0-beta.49", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/http-client": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/basic": "4.0.0-beta.51", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/http-client": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "@types/content-type": "^1.1.8", "cheerio": "^1.0.0", "content-type": "^1.0.5", @@ -891,12 +891,12 @@ } }, "node_modules/@crawlee/http-client": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/http-client/-/http-client-4.0.0-beta.49.tgz", - "integrity": "sha512-UMK5FLyFmAZHFNaaMhITwKQgEPdJZ5GJYvGCDHig4H5Hmc83aiBWIgGNqmQL+t/1YXkWVA0NhrBAgonk3m64aw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/http-client/-/http-client-4.0.0-beta.51.tgz", + "integrity": "sha512-/NcX3ESNXGRVYGBgkKCE035OidlOgsFYAd5zBSWTep/zsLefUISq2HEPTZYVpAvrgIVyaVjbNSilk1NpVjJLxw==", "license": "Apache-2.0", "dependencies": { - "@crawlee/types": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.51", "tough-cookie": "^6.0.0" }, "engines": { @@ -916,18 +916,18 @@ } }, "node_modules/@crawlee/http/node_modules/@crawlee/basic": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.49.tgz", - "integrity": "sha512-6rRchZXH1KyLvkcZpTPeuRcbEDdl/l0XOsVTK5lHxkQvj63LrE+YwlmZxEA3LViiva9soNUll7VoFfWhEL77iw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.51.tgz", + "integrity": "sha512-t4YhLgqWiy4N17YzeLugUIiwQDtXALqTl3GrKSZFnZSuHHux6nlQwbSX8x5FCtzW0q/On9ZGt/GTR+WMTka97g==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/got-scraping-client": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/got-scraping-client": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "csv-stringify": "^6.5.2", "fs-extra": "^11.3.0", "ow": "^2.0.0", @@ -1129,17 +1129,17 @@ } }, "node_modules/@crawlee/jsdom": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-4.0.0-beta.49.tgz", - "integrity": "sha512-RpO6fJfB8GMxokqfNTanU/zTSZJCFCw0fu3aTfyTG5njk/IENySRyJcIFWz+uUNuzF3trJVW4XEJBq4PBzvUjA==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-4.0.0-beta.51.tgz", + "integrity": "sha512-x2w8NkQfu14LO4yvVGTi4kNWDn3cgJq+XNSqS3UNAOraxQUPTSJpEicR+hwjt21Fb/tQRKMLRhdTgnqACWRoFw==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.0", "@apify/utilities": "^2.7.10", - "@crawlee/http": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/http": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "@types/jsdom": "^21.1.7", "cheerio": "^1.0.0", "jsdom": "^26.1.0", @@ -1296,17 +1296,18 @@ } }, "node_modules/@crawlee/linkedom": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-4.0.0-beta.49.tgz", - "integrity": "sha512-fOBVgl4sVcrKM/3DF96SfaMQjJQuEoMvtci67X1huPrPtjkrREqGrDufCtSttkcSkGu/gQCxMN/yv+oyzBAujQ==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/linkedom/-/linkedom-4.0.0-beta.51.tgz", + "integrity": "sha512-OY2ku6ZO2M26ViuYygBGDntxIVNd4zbuaglPdbgyLekX2lw5e6tbqWSLZ7pOF1FTOllCpXVRetRRArWcOBMc7A==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/http": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/http": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", + "cheerio": "^1.0.0", "linkedom": "^0.18.10", "ow": "^2.0.0", "tslib": "^2.8.1" @@ -1341,6 +1342,32 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@crawlee/linkedom/node_modules/cheerio": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^7.19.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=20.18.1" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, "node_modules/@crawlee/linkedom/node_modules/dot-prop": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-8.0.2.tgz", @@ -1357,6 +1384,39 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@crawlee/linkedom/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/@crawlee/linkedom/node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "dev": true, + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, "node_modules/@crawlee/linkedom/node_modules/ow": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ow/-/ow-2.0.0.tgz", @@ -1391,13 +1451,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@crawlee/linkedom/node_modules/undici": { + "version": "7.25.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", + "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/@crawlee/memory-storage": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-4.0.0-beta.49.tgz", - "integrity": "sha512-rLniynMhfrJoXn7rFHfj4NpsVvdRn3PIJi6YFL3G7s4JAiqqfhYOKshW+f6XdldEeb1z0YyGpgOknLo6VupktA==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/memory-storage/-/memory-storage-4.0.0-beta.51.tgz", + "integrity": "sha512-SiaQjdZaA7S7BdK5NeCK8LR0GeCNJq4x6qlv7fcQ+MqebLHKNEcmUcuwYXf9u4IoYweNfO+v9N+RkmfjyP4C1A==", "license": "Apache-2.0", "dependencies": { - "@crawlee/types": "4.0.0-beta.49", + "@crawlee/types": "4.0.0-beta.51", "@sapphire/async-queue": "^1.5.5", "@sapphire/shapeshift": "^4.0.0", "content-type": "^1.0.5", @@ -1465,9 +1535,9 @@ } }, "node_modules/@crawlee/templates": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-4.0.0-beta.49.tgz", - "integrity": "sha512-5/cBYgNuMXnVemxUDqagg0lgZURlc28NVuY1vfON+CG0C6Dg/GLwMlTLOmcroQeDJF7bWxVxDrsHomTk0Mm30A==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-4.0.0-beta.51.tgz", + "integrity": "sha512-YDGdBpv2ztbdY1EbVOV3lyJWjGeYGhBprA3WD58sLX+eIx7vUiZUZXdznt3AMf1FnGvfbCsbpFoVPa72xydtYw==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -1478,9 +1548,9 @@ } }, "node_modules/@crawlee/types": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-4.0.0-beta.49.tgz", - "integrity": "sha512-p33njES5N539Zgco1aroUZgmWeXp+i6MTPUk7R7wLEdV/k+aw2Y1cjw8wGQpKH9/ZZIpsvqx56U26bC2w1nmcQ==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/types/-/types-4.0.0-beta.51.tgz", + "integrity": "sha512-U/m8K41Fe8q5rvLqFvjclNTvRPvRsrphatEBssV1J8q/Rjhn71SnVVB0qVKYusNMLjLnaup3tlBY/01ahJDDSw==", "license": "Apache-2.0", "dependencies": { "tough-cookie": "^6.0.0", @@ -1503,14 +1573,14 @@ } }, "node_modules/@crawlee/utils": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-4.0.0-beta.49.tgz", - "integrity": "sha512-JmTTVsJBFJNPUZsdH35Yde/jRFJoZtloVtEYLGZoqo79rT9RmRi+SDr22XFYJ7bb/LEXY+NOWBovdMge+fnpHw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/utils/-/utils-4.0.0-beta.51.tgz", + "integrity": "sha512-DHvn2fo/6pIHTiHAi5tenkSWWi2KTm6sicnAIY38H98ZOiys4Yxh22wvfRN4cE8wm8pbUucQ8IYIzKSxbNLd+Q==", "license": "Apache-2.0", "dependencies": { "@apify/ps-tree": "^1.2.0", - "@crawlee/http-client": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", + "@crawlee/http-client": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", "@types/sax": "^1.2.7", "cheerio": "^1.0.0", "domhandler": "^5.0.3", @@ -8018,24 +8088,24 @@ } }, "node_modules/crawlee": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-4.0.0-beta.49.tgz", - "integrity": "sha512-/+m7LJ0m4nLmdA+zzb0ZL21DjNGUjveoPs+ckbiaPpE0LT8bJBalBbzhRcmUCZHWuY5L6ncNJcS2HTGBe20VaA==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-4.0.0-beta.51.tgz", + "integrity": "sha512-i/w8O7D7mdmUr8hdKEWSSHEIGqk4+nrknoWtEJiwnyOQAyzg7G98QsHUBeGmAR+j40EDPJui3wzrh7B6tUNkMA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@crawlee/basic": "4.0.0-beta.49", - "@crawlee/browser": "4.0.0-beta.49", - "@crawlee/browser-pool": "4.0.0-beta.49", - "@crawlee/cheerio": "4.0.0-beta.49", - "@crawlee/cli": "4.0.0-beta.49", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/http": "4.0.0-beta.49", - "@crawlee/jsdom": "4.0.0-beta.49", - "@crawlee/linkedom": "4.0.0-beta.49", - "@crawlee/playwright": "4.0.0-beta.49", - "@crawlee/puppeteer": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/basic": "4.0.0-beta.51", + "@crawlee/browser": "4.0.0-beta.51", + "@crawlee/browser-pool": "4.0.0-beta.51", + "@crawlee/cheerio": "4.0.0-beta.51", + "@crawlee/cli": "4.0.0-beta.51", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/http": "4.0.0-beta.51", + "@crawlee/jsdom": "4.0.0-beta.51", + "@crawlee/linkedom": "4.0.0-beta.51", + "@crawlee/playwright": "4.0.0-beta.51", + "@crawlee/puppeteer": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "import-local": "^3.2.0", "tslib": "^2.8.1" }, @@ -8063,18 +8133,18 @@ } }, "node_modules/crawlee/node_modules/@crawlee/basic": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.49.tgz", - "integrity": "sha512-6rRchZXH1KyLvkcZpTPeuRcbEDdl/l0XOsVTK5lHxkQvj63LrE+YwlmZxEA3LViiva9soNUll7VoFfWhEL77iw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/basic/-/basic-4.0.0-beta.51.tgz", + "integrity": "sha512-t4YhLgqWiy4N17YzeLugUIiwQDtXALqTl3GrKSZFnZSuHHux6nlQwbSX8x5FCtzW0q/On9ZGt/GTR+WMTka97g==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/got-scraping-client": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/got-scraping-client": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "csv-stringify": "^6.5.2", "fs-extra": "^11.3.0", "ow": "^2.0.0", @@ -8087,17 +8157,17 @@ } }, "node_modules/crawlee/node_modules/@crawlee/browser": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-4.0.0-beta.49.tgz", - "integrity": "sha512-RBzbtscHGi7qY7bn3KqsfV2flFt0XlKp7OOdGx8VsSSzdVhIb+YXFOrcdMR6DGEyvxtMSZxNZ4U8qaKM/YwOUg==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/browser/-/browser-4.0.0-beta.51.tgz", + "integrity": "sha512-Z0VoAetDT5HL8vtQYiZkOVrFFpoeoq/6Yvc6hK6Sp6tQ5Snb/+uVXcTL9LTQ0a/9G0S3yzbgvuYaHpR2AInK1Q==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", - "@crawlee/basic": "4.0.0-beta.49", - "@crawlee/browser-pool": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/basic": "4.0.0-beta.51", + "@crawlee/browser-pool": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "ow": "^2.0.0", "tslib": "^2.8.1", "type-fest": "^4.41.0" @@ -8119,15 +8189,15 @@ } }, "node_modules/crawlee/node_modules/@crawlee/browser-pool": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-4.0.0-beta.49.tgz", - "integrity": "sha512-oroowDwagffg1Ybbn3y3Eey2i2p+rhdu9f2dBtDvH4EF+4KPpfW7Z5FG9knLTx+3OcaRp9QumZUuyFANH+wN1w==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/browser-pool/-/browser-pool-4.0.0-beta.51.tgz", + "integrity": "sha512-2YSgDy8QFrTpzYYiLSctcWyu9MaQzaV0fDrcRMO/a1+WeiwPc5cOwpRjtJYXVRjN4ty1x3/pUbY+mNaGjUDUng==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/timeout": "^0.3.2", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", "fingerprint-generator": "^2.1.68", "fingerprint-injector": "^2.1.68", "lodash.merge": "^4.6.2", @@ -8156,21 +8226,21 @@ } }, "node_modules/crawlee/node_modules/@crawlee/playwright": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-4.0.0-beta.49.tgz", - "integrity": "sha512-ozU/eUAgnW0AhyWYUe4Kvipqfl8JyJ2ZZoB5jqWG7PG0zmgEx9CQ5s+aMopl0zmZu31dW7nnZk9HQZa71ZYcTg==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/playwright/-/playwright-4.0.0-beta.51.tgz", + "integrity": "sha512-kJFB2VEeD8xp42oQuf1yReADuhuqZnHxLU5H6IneJ0BhG4JHy5/Za2QT7LtkwBfATZSbIyxiBlgk8vevU25t5A==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.3", "@apify/timeout": "^0.3.2", - "@crawlee/basic": "4.0.0-beta.49", - "@crawlee/browser": "4.0.0-beta.49", - "@crawlee/browser-pool": "4.0.0-beta.49", - "@crawlee/cheerio": "4.0.0-beta.49", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/basic": "4.0.0-beta.51", + "@crawlee/browser": "4.0.0-beta.51", + "@crawlee/browser-pool": "4.0.0-beta.51", + "@crawlee/cheerio": "4.0.0-beta.51", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "cheerio": "^1.0.0", "idcac-playwright": "^0.1.3", "jquery": "^3.7.1", @@ -8204,18 +8274,18 @@ "license": "ISC" }, "node_modules/crawlee/node_modules/@crawlee/puppeteer": { - "version": "4.0.0-beta.49", - "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-4.0.0-beta.49.tgz", - "integrity": "sha512-fpMaaXqSNr86FTWVvkWvNXlKRApRtURo37U9AHVLtwQTyBbmgFeETnfjHauAlEj29mFaRgk4lyKXEcDmWU/Psw==", + "version": "4.0.0-beta.51", + "resolved": "https://registry.npmjs.org/@crawlee/puppeteer/-/puppeteer-4.0.0-beta.51.tgz", + "integrity": "sha512-IrPUaR9Y4AeVyCnfZSAWHVvFxAAhVlCJDuEtcL7YIhAuBrvTRW9aGqx6vAhzTsrEwpvuk1OX2mL1lE8s/xbUiw==", "dev": true, "license": "Apache-2.0", "dependencies": { "@apify/datastructures": "^2.0.3", - "@crawlee/browser": "4.0.0-beta.49", - "@crawlee/browser-pool": "4.0.0-beta.49", - "@crawlee/core": "4.0.0-beta.49", - "@crawlee/types": "4.0.0-beta.49", - "@crawlee/utils": "4.0.0-beta.49", + "@crawlee/browser": "4.0.0-beta.51", + "@crawlee/browser-pool": "4.0.0-beta.51", + "@crawlee/core": "4.0.0-beta.51", + "@crawlee/types": "4.0.0-beta.51", + "@crawlee/utils": "4.0.0-beta.51", "cheerio": "^1.0.0", "devtools-protocol": "*", "idcac-playwright": "^0.2.0", @@ -20582,9 +20652,9 @@ "@apify/log": "^2.5.18", "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "^4.0.0-beta.49", - "@crawlee/types": "^4.0.0-beta.49", - "@crawlee/utils": "^4.0.0-beta.49", + "@crawlee/core": "^4.0.0-beta.51", + "@crawlee/types": "^4.0.0-beta.51", + "@crawlee/utils": "^4.0.0-beta.51", "apify-client": "^2.12.4", "fs-extra": "^11.3.0", "got-scraping": "^4.1.1", diff --git a/package.json b/package.json index 9fed883e8f..1b0bfac3d8 100644 --- a/package.json +++ b/package.json @@ -78,10 +78,10 @@ "@types/tough-cookie": "^4.0.5", "@types/ws": "^8.18.1", "commitlint": "^19.8.1", - "crawlee": "^4.0.0-beta.49", - "@crawlee/core": "^4.0.0-beta.49", - "@crawlee/types": "^4.0.0-beta.49", - "@crawlee/utils": "^4.0.0-beta.49", + "crawlee": "^4.0.0-beta.51", + "@crawlee/core": "^4.0.0-beta.51", + "@crawlee/types": "^4.0.0-beta.51", + "@crawlee/utils": "^4.0.0-beta.51", "eslint": "^9.27.0", "eslint-config-prettier": "^10.1.5", "fs-extra": "^11.3.0", diff --git a/packages/apify/package.json b/packages/apify/package.json index 7cea7e856d..9173c9696b 100644 --- a/packages/apify/package.json +++ b/packages/apify/package.json @@ -53,9 +53,9 @@ "@apify/log": "^2.5.18", "@apify/timeout": "^0.3.2", "@apify/utilities": "^2.15.5", - "@crawlee/core": "^4.0.0-beta.49", - "@crawlee/types": "^4.0.0-beta.49", - "@crawlee/utils": "^4.0.0-beta.49", + "@crawlee/core": "^4.0.0-beta.51", + "@crawlee/types": "^4.0.0-beta.51", + "@crawlee/utils": "^4.0.0-beta.51", "apify-client": "^2.12.4", "fs-extra": "^11.3.0", "got-scraping": "^4.1.1", diff --git a/packages/apify/src/proxy_configuration.ts b/packages/apify/src/proxy_configuration.ts index fee9e753dc..770ef3c474 100644 --- a/packages/apify/src/proxy_configuration.ts +++ b/packages/apify/src/proxy_configuration.ts @@ -85,15 +85,6 @@ export interface ProxyConfigurationOptions * configurate the proxy by UI input schema. You should use the `countryCode` option in your crawler code. */ apifyProxyCountry?: string; - - /** - * Multiple different ProxyConfigurationOptions stratified into tiers. Crawlee crawlers will switch between those tiers - * based on the blocked request statistics. - */ - tieredProxyConfig?: Omit< - ProxyConfigurationOptions, - keyof CoreProxyConfigurationOptions | 'tieredProxyConfig' - >[]; } /** @@ -229,10 +220,6 @@ export class ProxyConfiguration extends CoreProxyConfiguration { apifyProxyCountry: ow.optional.string.matches(COUNTRY_CODE_REGEX), password: ow.optional.string, - tieredProxyUrls: ow.optional.array.ofType( - ow.array.ofType(ow.string), - ), - tieredProxyConfig: ow.optional.array.ofType(ow.object), }), ); @@ -242,18 +229,11 @@ export class ProxyConfiguration extends CoreProxyConfiguration { countryCode, apifyProxyCountry, password = config.proxyPassword, - tieredProxyConfig, - tieredProxyUrls, } = options; - this.tieredProxyUrls ??= tieredProxyUrls; - - if (tieredProxyConfig) { - this.tieredProxyUrls = this._generateTieredProxyUrls( - tieredProxyConfig, - options, - ); - } + // crawlee v4 (>=beta.51) removed `tieredProxyUrls` / + // `tieredProxyConfig` (see apify/crawlee#3599) — the SDK no + // longer threads tiered config through to the base class. const groupsToUse = groups.length ? groups : apifyProxyGroups; const countryCodeToUse = countryCode || apifyProxyCountry; @@ -347,10 +327,10 @@ export class ProxyConfiguration extends CoreProxyConfiguration { legacyOptions?: Parameters[0], ): Promise { // crawlee v4 dropped the `(sessionId, options)` overload — `newProxyInfo` - // now takes a single `TieredProxyOptions` argument and pulls `sessionId` + // now takes a single `NewUrlOptions` argument and pulls `sessionId` // from `options.request`. Keep the SDK's legacy "pass sessionId directly" // shape working by discriminating at runtime. - const { sessionId, options } = parseSessionIdOrOptions( + const { sessionId } = parseSessionIdOrOptions( sessionIdOrOptions, legacyOptions, ); @@ -397,9 +377,6 @@ export class ProxyConfiguration extends CoreProxyConfiguration { result.groups = groups; if (countryCode !== undefined) result.countryCode = countryCode; } - if (options?.proxyTier !== undefined) { - result.proxyTier = options.proxyTier; - } return result as ProxyInfo; } @@ -453,10 +430,6 @@ export class ProxyConfiguration extends CoreProxyConfiguration { return this.proxyUrls[index] ?? undefined; } - if (this.tieredProxyUrls) { - return this._handleTieredUrl(options ?? {}).proxyUrl ?? undefined; - } - return this.composeDefaultUrl(sessionId); } @@ -476,21 +449,6 @@ export class ProxyConfiguration extends CoreProxyConfiguration { return this.proxyUrls!.indexOf(this.usedProxyUrls.get(sessionId)!); } - protected _generateTieredProxyUrls( - tieredProxyConfig: NonNullable< - ProxyConfigurationOptions['tieredProxyConfig'] - >, - globalOptions: ProxyConfigurationOptions, - ) { - return tieredProxyConfig.map((config) => [ - new ProxyConfiguration({ - ...globalOptions, - ...config, - tieredProxyConfig: undefined, - }).composeDefaultUrl(), - ]); - } - /** * Returns proxy username. */ diff --git a/test/apify/proxy_configuration.test.ts b/test/apify/proxy_configuration.test.ts index ecee1ca7a5..d42b0808cb 100644 --- a/test/apify/proxy_configuration.test.ts +++ b/test/apify/proxy_configuration.test.ts @@ -501,72 +501,10 @@ describe('ProxyConfiguration', () => { }); }); - describe('With tieredProxyUrls', () => { - test('proxy configuration accepts the tiered urls (Crawlee style)', async () => { - const proxyConfiguration = new ProxyConfiguration({ - tieredProxyUrls: [ - ['http://proxy.com:1111'], - ['http://proxy.com:2222'], - ['http://proxy.com:3333'], - ['http://proxy.com:4444'], - ], - }); - - // through newUrl() - expect( - await proxyConfiguration.newUrl('abc', { - request: new Request({ url: 'http://example.com' }) as any, - }), - ).toEqual('http://proxy.com:1111'); - - // through newProxyInfo() - expect( - (await proxyConfiguration.newProxyInfo('abc', { - request: new Request({ - url: 'http://example.com', - }) as any, - }))!.url, - ).toEqual('http://proxy.com:1111'); - }); - - test('shorthand tieredProxyConfig gets correctly expanded', async () => { - const proxyConfiguration = new ProxyConfiguration({ - password: 'password', - countryCode: 'DE', - tieredProxyConfig: [ - { - groups: ['GROUP1'], - countryCode: 'CZ', - }, - { - groups: ['GROUP2'], - countryCode: 'US', - }, - { - groups: ['GROUP3', 'GROUP4'], - }, - { - groups: ['GROUP3', 'GROUP4'], - countryCode: undefined, - }, - ], - }); - - // eslint-disable-next-line dot-notation - expect(proxyConfiguration['tieredProxyUrls']).toEqual([ - [ - 'http://groups-GROUP1,country-CZ:password@proxy.apify.com:8000', - ], - [ - 'http://groups-GROUP2,country-US:password@proxy.apify.com:8000', - ], - [ - 'http://groups-GROUP3+GROUP4,country-DE:password@proxy.apify.com:8000', - ], - ['http://groups-GROUP3+GROUP4:password@proxy.apify.com:8000'], - ]); - }); - }); + // `tieredProxyUrls` / `tieredProxyConfig` were removed from + // crawlee v4 (apify/crawlee#3599); the corresponding test groups + // were dropped here and in the `Actor.createProxyConfiguration()` + // describe below. }); describe('Actor.createProxyConfiguration()', () => { @@ -733,70 +671,7 @@ describe('Actor.createProxyConfiguration()', () => { gotScrapingSpy.mockRestore(); }); - describe('With tieredProxyUrls', () => { - test('proxy configuration accepts the tiered urls (Crawlee style)', async () => { - const proxyConfiguration = await Actor.createProxyConfiguration({ - tieredProxyUrls: [ - ['http://proxy.com:1111'], - ['http://proxy.com:2222'], - ['http://proxy.com:3333'], - ['http://proxy.com:4444'], - ], - }); - - // through newUrl() - expect( - await proxyConfiguration!.newUrl('abc', { - request: new Request({ url: 'http://example.com' }) as any, - }), - ).toEqual('http://proxy.com:1111'); - - // through newProxyInfo() - expect( - (await proxyConfiguration!.newProxyInfo('abc', { - request: new Request({ - url: 'http://example.com', - }) as any, - }))!.url, - ).toEqual('http://proxy.com:1111'); - }); - - test('shorthand tieredProxyConfig gets correctly expanded', async () => { - const proxyConfiguration = await Actor.createProxyConfiguration({ - password: 'password', - countryCode: 'DE', - tieredProxyConfig: [ - { - groups: ['GROUP1'], - countryCode: 'CZ', - }, - { - groups: ['GROUP2'], - countryCode: 'US', - }, - { - groups: ['GROUP3', 'GROUP4'], - }, - { - groups: ['GROUP3', 'GROUP4'], - countryCode: undefined, - }, - ], - }); - - // eslint-disable-next-line dot-notation - expect(proxyConfiguration!['tieredProxyUrls']).toEqual([ - [ - 'http://groups-GROUP1,country-CZ:password@proxy.apify.com:8000', - ], - [ - 'http://groups-GROUP2,country-US:password@proxy.apify.com:8000', - ], - [ - 'http://groups-GROUP3+GROUP4,country-DE:password@proxy.apify.com:8000', - ], - ['http://groups-GROUP3+GROUP4:password@proxy.apify.com:8000'], - ]); - }); - }); + // `tieredProxyUrls` / `tieredProxyConfig` were removed from + // crawlee v4 (apify/crawlee#3599); the corresponding test groups + // were dropped here too. }); From 6a8a6bc54d54374405a7015f40d69a84ad9c8be0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Thu, 30 Apr 2026 20:55:22 +0200 Subject: [PATCH 34/34] test: replace Configuration.storage with fresh AsyncLocalStorage on reset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Actor.init()` calls `Configuration.storage.enterWith(this.config)`, which sets the AsyncLocalStorage value on whichever async context the test runner happened to be on. `enterWith(undefined)` from a child async branch (vitest's beforeEach) doesn't unwind that — on Node 22 the test body re-enters a sibling context where the original `enterWith` is still in effect, so `getStore()` still returns the stale Configuration even after our reset. Swapping the entire `AsyncLocalStorage` instance for a fresh one guarantees `getStore()` returns `undefined` for every async branch that follows, fixing the addWebhook test failures on Node 22. --- test/MemoryStorageEmulator.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/test/MemoryStorageEmulator.ts b/test/MemoryStorageEmulator.ts index 7851667ea7..b061ea2473 100644 --- a/test/MemoryStorageEmulator.ts +++ b/test/MemoryStorageEmulator.ts @@ -1,3 +1,4 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; import { rm } from 'node:fs/promises'; import { resolve } from 'node:path'; @@ -21,15 +22,16 @@ function resetGlobalState() { // eslint-disable-next-line no-underscore-dangle -- `_instance` is the upstream Actor singleton field (Actor as unknown as { _instance?: Actor })._instance = undefined; // `Actor.init()` calls `Configuration.storage.enterWith(this.config)`, - // which sticks the resolved config onto the current async context and - // persists across tests on some Node versions (observed on Node 22 - // but not Node 24+). Clear the AsyncLocalStorage value so the next - // `Configuration.getGlobalConfig()` falls through to a fresh build. - if (Configuration.storage?.getStore()) { - ( - Configuration.storage as unknown as { enterWith(v: unknown): void } - ).enterWith(undefined); - } + // which sticks the resolved config onto the *outer* async context + // (vitest's test runner). `enterWith(undefined)` from a child context + // (this beforeEach) doesn't propagate back up, so on Node 22 the next + // test still sees the stale store. Replace the entire AsyncLocalStorage + // instance to guarantee `getStore()` returns `undefined` everywhere. + ( + Configuration as unknown as { + storage: AsyncLocalStorage; + } + ).storage = new AsyncLocalStorage(); } const LOCAL_EMULATION_DIR = resolve(