From 7598aaf41679abe7283d5db703f31e4ace0a8720 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Thu, 21 May 2026 12:30:56 +0530 Subject: [PATCH 01/11] feat: add support for asset scaning in import command --- .talismanrc | 18 ++---------------- .../src/import/module-importer.ts | 16 ++++++++++++++++ .../src/import/modules/assets.ts | 7 +++++++ .../src/types/import-config.ts | 1 + 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/.talismanrc b/.talismanrc index 16286bff7..1999c2dd4 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,18 +1,4 @@ fileignoreconfig: - - filename: pnpm-lock.yaml - checksum: 264c0294416c2aa2028c8283831aa7ed17d63e8c69553a7b2b9774336bc0811f - - filename: packages/contentstack-bootstrap/test/bootstrap.test.js - checksum: 37b502482fc32831c39091dd1755e8b0a9f6f8bac89e5eb9d397c6af1f213d83 - - filename: packages/contentstack-bootstrap/test/utils.test.js - checksum: e0ca2eb58ab1c3ac3b4d9c14a17bb8f4788678074dd65f6acc128a8a8f51997f - - filename: packages/contentstack-export-to-csv/test/unit/base-command.test.ts - checksum: 4c2befce053135453c1db31f21351bf3f797ff2f15723ca52e183ed6ed9e48e4 - - filename: packages/contentstack-export-to-csv/test/unit/utils/teams-export.functional.test.ts - checksum: 17cdae91c22a935309bf4514b6616f1aea91fd0166fede182e673ade43667a36 - - filename: packages/contentstack-export-to-csv/test/unit/utils/interactive.test.ts - checksum: 72c1e719e5c51a42debc817a5fe5bb1adee63b2d823df2e9ee36d0b970db7886 - - filename: packages/contentstack-export-to-csv/test/unit/utils/api-client.functional.test.ts - checksum: 6d6638919ef7260f642d32cf730e2654d159d2d3582714fb2d7a9c209b9c6eeb - - filename: packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.ts - checksum: 9621a9d013796e99a25e894404c4d7c6799bd33bde77852f09d5e32b4d1c1c49 + - filename: packages/contentstack-import/src/import/modules/assets.ts + checksum: 8d6d1cadf9178f79aeae508f157c019342e1899f9c3f8116bd25b4ec27e593be version: '1.0' diff --git a/packages/contentstack-import/src/import/module-importer.ts b/packages/contentstack-import/src/import/module-importer.ts index 930350a18..109c54160 100755 --- a/packages/contentstack-import/src/import/module-importer.ts +++ b/packages/contentstack-import/src/import/module-importer.ts @@ -26,6 +26,12 @@ class ModuleImporter { const stackDetails: Record = await this.stackAPIClient.fetch(); this.importConfig.stackName = stackDetails.name as string; this.importConfig.org_uid = stackDetails.org_uid as string; + + const assetScanningEnabled = await this.detectAssetScanning(this.importConfig.org_uid); + if (assetScanningEnabled) { + this.importConfig.assetScanningEnabled = true; + this.importConfig.skipAssetsPublish = true; + } } await this.resolveImportPath(); @@ -194,6 +200,16 @@ class ModuleImporter { log.error(`Audit failed with following error. ${error}`, this.importConfig.context); } } + + private async detectAssetScanning(orgUid: string): Promise { + try { + const orgDetails = await this.managementAPIClient.organization(orgUid).fetch({ include_plan: true }); + const features: Array<{ uid: string; enabled?: boolean }> = orgDetails?.plan?.features || []; + return features.some((f) => (f.uid === 'assetsScan' || f.uid === 'amAssetsScan') && f.enabled === true); + } catch { + return false; + } + } } export default ModuleImporter; diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index c65345d91..c2ab64bc9 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -197,6 +197,13 @@ export default class ImportAssets extends BaseClass { this.completeProgress(true); log.success('Assets imported successfully!', this.importConfig.context); + + if (this.importConfig.assetScanningEnabled) { + log.info('ℹ Asset Scanning is enabled for this stack.', this.importConfig.context); + log.info(' Assets cannot be published immediately — scanning must complete first.', this.importConfig.context); + log.info(' Once scanning is done, publish your assets using:', this.importConfig.context); + log.info(' csdx cm:stacks:bulk-assets --data-dir ./content --stack-api-key ', this.importConfig.context); + } } catch (error) { this.completeProgress(false, error?.message || 'Asset import failed'); handleAndLogError(error, { ...this.importConfig.context }); diff --git a/packages/contentstack-import/src/types/import-config.ts b/packages/contentstack-import/src/types/import-config.ts index 1587a360e..60fa64389 100644 --- a/packages/contentstack-import/src/types/import-config.ts +++ b/packages/contentstack-import/src/types/import-config.ts @@ -14,6 +14,7 @@ export default interface ImportConfig extends DefaultConfig, ExternalConfig { authenticationMethod?: string; skipAssetsPublish?: boolean; skipEntriesPublish?: boolean; + assetScanningEnabled?: boolean; cliLogsPath: string; canCreatePrivateApp: boolean; contentDir: string; From 09f2db8da97dbb7d7977736e52b49ac6d10f6f36 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Thu, 21 May 2026 12:34:32 +0530 Subject: [PATCH 02/11] chore: update talisman file --- .talismanrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.talismanrc b/.talismanrc index 1999c2dd4..88cd645b3 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,4 @@ fileignoreconfig: - - filename: packages/contentstack-import/src/import/modules/assets.ts +- filename: packages/contentstack-import/src/import/modules/assets.ts checksum: 8d6d1cadf9178f79aeae508f157c019342e1899f9c3f8116bd25b4ec27e593be version: '1.0' From d94825f0f306dd9b071a20d33c575e918ffb16c0 Mon Sep 17 00:00:00 2001 From: Naman Dembla Date: Thu, 28 May 2026 14:29:35 +0530 Subject: [PATCH 03/11] chore: update log message --- packages/contentstack-import/src/import/modules/assets.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index c2ab64bc9..abf864637 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -199,7 +199,7 @@ export default class ImportAssets extends BaseClass { log.success('Assets imported successfully!', this.importConfig.context); if (this.importConfig.assetScanningEnabled) { - log.info('ℹ Asset Scanning is enabled for this stack.', this.importConfig.context); + log.info(' Asset Scanning is enabled for this stack.', this.importConfig.context); log.info(' Assets cannot be published immediately — scanning must complete first.', this.importConfig.context); log.info(' Once scanning is done, publish your assets using:', this.importConfig.context); log.info(' csdx cm:stacks:bulk-assets --data-dir ./content --stack-api-key ', this.importConfig.context); From be27c50f3f87fbdc6083977cc9ac8a699552a285 Mon Sep 17 00:00:00 2001 From: Naman Dembla Date: Thu, 28 May 2026 14:30:28 +0530 Subject: [PATCH 04/11] chore: update log message spacing --- packages/contentstack-import/src/import/modules/assets.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index abf864637..fbce6e743 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -199,7 +199,7 @@ export default class ImportAssets extends BaseClass { log.success('Assets imported successfully!', this.importConfig.context); if (this.importConfig.assetScanningEnabled) { - log.info(' Asset Scanning is enabled for this stack.', this.importConfig.context); + log.info(' Asset Scanning is enabled for this stack.', this.importConfig.context); log.info(' Assets cannot be published immediately — scanning must complete first.', this.importConfig.context); log.info(' Once scanning is done, publish your assets using:', this.importConfig.context); log.info(' csdx cm:stacks:bulk-assets --data-dir ./content --stack-api-key ', this.importConfig.context); From 661e8e198e3fe7f9b385be550c6ae223fda92726 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Fri, 29 May 2026 14:13:10 +0530 Subject: [PATCH 05/11] feat: feat: add asset scanning support --- .talismanrc | 4 + .../src/base-bulk-command.ts | 12 +- .../src/commands/cm/stacks/bulk-assets.ts | 284 +++++++++++++++++- .../src/interfaces/index.ts | 8 +- .../src/messages/index.ts | 19 ++ .../src/services/asset-service.ts | 49 ++- .../src/services/bulk-operation-service.ts | 60 +++- .../src/utils/batch-queue-handler.ts | 14 +- .../src/utils/config-builder.ts | 70 +++-- .../src/utils/data-dir-asset-fetcher.ts | 114 +++++++ .../src/utils/helpers.ts | 36 ++- .../src/utils/index.ts | 6 + .../src/utils/item-fetcher.ts | 1 + 13 files changed, 611 insertions(+), 66 deletions(-) create mode 100644 packages/contentstack-bulk-operations/src/utils/data-dir-asset-fetcher.ts diff --git a/.talismanrc b/.talismanrc index 3db7ed059..d07a9f629 100644 --- a/.talismanrc +++ b/.talismanrc @@ -5,4 +5,8 @@ fileignoreconfig: checksum: 580932f192dd3fdd8bb2c55b7a7a78f1694f646ef5c5041f86c75668778f7ecb - filename: packages/contentstack-bulk-operations/test/unit/utils/asset-uids-from-file.test.ts checksum: 8123f7a675a0275795b59b15d0f2d5f8f1e57ccbecf3f97249a0dc5a037b9203 +- filename: packages/contentstack-bulk-operations/src/utils/data-dir-asset-fetcher.ts + checksum: ddcf4601ac47be300eba0eb901e4d45d748c2c4eab676c56677cb9a802fe3db0 +- filename: packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts + checksum: c8767e79c1010cccf984d2124b2c6252b7cba1fb1b6a9f216fa7ddb5b195a935 version: '1.0' diff --git a/packages/contentstack-bulk-operations/src/base-bulk-command.ts b/packages/contentstack-bulk-operations/src/base-bulk-command.ts index e5f0bc62c..a6b0a987a 100644 --- a/packages/contentstack-bulk-operations/src/base-bulk-command.ts +++ b/packages/contentstack-bulk-operations/src/base-bulk-command.ts @@ -132,7 +132,7 @@ export abstract class BaseBulkCommand extends Command { protected rateLimiter!: AdaptiveRateLimiter; protected retryStrategy!: RetryStrategy; protected operationExecutor!: OperationExecutor; - private batchResults: Map = new Map(); + protected batchResults: Map = new Map(); protected parsedFlags: any; /** @@ -165,7 +165,7 @@ export abstract class BaseBulkCommand extends Command { } // Fill missing required flags via interactive prompts - flags = await fillMissingFlags(flags); + flags = await this.resolveFlagsInteractively(flags); this.parsedFlags = flags; await this.buildConfiguration(flags); @@ -265,6 +265,14 @@ export abstract class BaseBulkCommand extends Command { } } + + /** + * Resolve flags interactively — subclasses can override to skip prompts for specific modes. + */ + protected async resolveFlagsInteractively(flags: any): Promise { + return await fillMissingFlags(flags); + } + /** * Build operation configuration */ diff --git a/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts b/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts index 71122a91d..53bc50c1a 100644 --- a/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts +++ b/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts @@ -1,12 +1,17 @@ -import { flags, handleAndLogError, FlagInput } from '@contentstack/cli-utilities'; +import * as fs from 'fs'; +import * as path from 'path'; -import { ResourceType } from '../../../interfaces'; +import { flags, handleAndLogError, log } from '@contentstack/cli-utilities'; + +import { AssetPublishData, BulkOperationResult, OperationType, ResourceType } from '../../../interfaces'; import { BaseBulkCommand } from '../../../base-bulk-command'; -import { $t, messages, fetchAssets } from '../../../utils'; +import { $t, messages, fetchAssets, scanDataDirStats, BATCH_CONSTANTS, categorizeByScanStatus } from '../../../utils'; +import type { DataDirScanStats } from '../../../utils'; +import { AssetService } from '../../../services'; /** * Bulk operations command for assets - * Supports publish, unpublish, and cross publish operations + * Supports publish, unpublish, cross-publish, and data-dir publish operations */ export default class BulkAssets extends BaseBulkCommand { static description = messages.BULK_ASSETS_DESCRIPTION; @@ -32,25 +37,47 @@ export default class BulkAssets extends BaseBulkCommand { // Revert (unpublish) previously published assets using success log '<%= config.bin %> <%= command.id %> --revert ./bulk-operation -a myAlias', + + // Publish assets from exported content folder (e.g. after asset scanning clears) + '<%= config.bin %> <%= command.id %> --data-dir ./content --operation publish -k blt123', ]; - static flags: FlagInput = { + static flags = { ...BaseBulkCommand.baseFlags, 'folder-uid': flags.string({ description: messages.FOLDER_UID, }), + 'data-dir': flags.string({ + char: 'd', + description: messages.DATA_DIR_FLAG_DESC, + }), + 'dry-run': flags.boolean({ + description: messages.DRY_RUN_FLAG_DESC, + default: false, + }), }; protected resourceType: ResourceType = ResourceType.ASSET; + protected async resolveFlagsInteractively(flags: any): Promise { + if (flags['data-dir']) { + return flags; + } + return super.resolveFlagsInteractively(flags); + } + async run(): Promise { try { - // Handle cross-publish separately if source-env is specified if (this.bulkOperationConfig.sourceEnv) { await this.handleCrossPublish(this.parsedFlags); return; } + if (this.bulkOperationConfig.dataDir) { + await this.runDataDirFlow(); + return; + } + const assets = await this.fetchItems(); if (assets.length === 0) { @@ -58,18 +85,39 @@ export default class BulkAssets extends BaseBulkCommand { return; } - this.logger.info( - $t(messages.FOUND_ASSETS_TO_OPERATE, { count: assets.length, operation: this.parsedFlags.operation || '' }) - ); + const { clean, pending, quarantined, noStatus } = categorizeByScanStatus(assets); + const scanningEnabled = clean.length + pending.length + quarantined.length > 0; + const publishable = scanningEnabled ? clean : [...clean, ...noStatus]; + + if (scanningEnabled) { + // Log individual skipped assets + pending.forEach((a) => this.logger.warn($t(messages.SCAN_STATUS_SKIPPED_PENDING, { uid: a.uid }))); + quarantined.forEach((a) => this.logger.warn($t(messages.SCAN_STATUS_SKIPPED_QUARANTINED, { uid: a.uid }))); - // Confirm operation - const confirmed = await this.confirmOperation(assets); + this.printScanningDashboard({ + total: assets.length, + clean: clean.length, + pending: pending.length, + quarantined: quarantined.length, + }); + + if (publishable.length === 0) { + this.logger.warn($t(messages.NO_PUBLISHABLE_ASSETS)); + return; + } + } else { + log.info( + $t(messages.FOUND_ASSETS_TO_OPERATE, { count: assets.length, operation: this.parsedFlags.operation || '' }) + ); + } + + const confirmed = await this.confirmOperation(publishable); if (!confirmed) { this.logger.warn($t(messages.OPERATION_CANCELLED)); return; } - const result = await this.executeBulkOperation(assets); + const result = await this.executeBulkOperation(publishable); this.printOperationSummary(result); } catch (error) { handleAndLogError(error); @@ -78,6 +126,218 @@ export default class BulkAssets extends BaseBulkCommand { } } + private async runDataDirFlow(): Promise { + const { dataDir, dryRun } = this.bulkOperationConfig; + + // Capture original CLI locales/envs before pass 1 overwrites them on the config. + const cliLocales = [...(this.bulkOperationConfig.locales || [])]; + const cliEnvs = [...(this.bulkOperationConfig.environments || [])]; + + // Pass 1 — count-only scan: no AssetPublishData objects built, one chunk in memory at a time. + let stats: DataDirScanStats; + try { + stats = await scanDataDirStats(dataDir!, cliEnvs, cliLocales, this.logger); + } catch (err: any) { + this.logger.error($t(messages.DATA_DIR_READ_ERROR, { path: dataDir!, error: err.message || String(err) })); + return; + } + + this.bulkOperationConfig.environments = stats.environments; + this.bulkOperationConfig.locales = stats.locales; + + // Pass 1.5 — fetch scan status for all target UIDs (post-import UIDs on the destination stack). + const targetUids = Object.values(stats.assetUidMapper); + const assetService = new AssetService(this.managementStack, this.deliveryStack, this.logger); + const scanStatusMap = await assetService.fetchScanStatusByUIDs(targetUids); + + let cleanCount = 0; + let pendingCount = 0; + let quarantinedCount = 0; + for (const uid of targetUids) { + const status = scanStatusMap.get(uid); + if (status === 'pending') pendingCount++; + else if (status === 'quarantined') quarantinedCount++; + else cleanCount++; // clean or undefined (scanning disabled) — both are publishable + } + + this.printScanningDashboard({ + total: stats.eligible + stats.skipped + stats.unmapped, + localSkipped: stats.skipped, + unmapped: stats.unmapped, + clean: cleanCount, + pending: pendingCount, + quarantined: quarantinedCount, + }); + + if (cleanCount === 0) { + this.logger.warn($t(messages.NO_PUBLISHABLE_ASSETS)); + return; + } + + // new Array(n) has .length === n but allocates no elements — just for the count. + const confirmed = await this.confirmOperation(new Array(cleanCount)); + if (!confirmed) { + this.logger.warn($t(messages.OPERATION_CANCELLED)); + return; + } + + if (dryRun) { + log.info($t(messages.DATA_DIR_DRY_RUN)); + return; + } + + // Pass 2 — stream and publish: one chunk at a time, batches of ≤50 items enqueued directly. + // stats.assetUidMapper and stats.assetsIndex are reused from pass 1 — no second disk read. + const result = await this.streamAndPublish( + dataDir!, + cliLocales, + stats.totalItems, + stats.assetUidMapper, + stats.assetsIndex, + scanStatusMap + ); + this.printOperationSummary(result); + } + + /** + * Pass 2 of the data-dir flow. + * Reads chunk files one at a time, fills a working batch of ≤50 AssetPublishData items, + * and enqueues each batch directly into the queue manager without ever holding the full + * asset list in memory. Peak memory: one chunk file + one batch of ≤50 items. + * + * assetUidMapper and assetsIndex are passed in from pass 1 to avoid re-reading those files. + * scanStatusMap filters out non-clean assets before enqueueing. + */ + private async streamAndPublish( + dataDir: string, + cliLocales: string[], + totalItemCount: number, + assetUidMapper: Record, + assetsIndex: Record, + scanStatusMap: Map + ): Promise { + // Snapshot both arrays so in-flight mutations to bulkOperationConfig can't corrupt payloads. + const environments = [...this.bulkOperationConfig.environments!]; + const locales = [...this.bulkOperationConfig.locales!]; + const operation = this.bulkOperationConfig.operation as OperationType; + const startTime = Date.now(); + + // Warn early if the mapper is empty — all assets will be skipped and the user needs to know why. + if (Object.keys(assetUidMapper).length === 0) { + this.logger.warn( + 'Asset UID mapper is empty — all assets will be skipped. Ensure the import completed successfully.' + ); + } + + const useOverrideLocales = cliLocales.length > 0; + const BATCH_SIZE = BATCH_CONSTANTS.maxItems; + // totalItemCount comes from pass 1 using identical counting logic — used as upper bound for totalBatches. + // Scan status filtering may reduce the actual count; the invariant check below will log any mismatch. + const totalBatches = Math.ceil(totalItemCount / BATCH_SIZE); + + let workingBatch: AssetPublishData[] = []; + let batchNumber = 0; + let totalSubmitted = 0; + + this.batchResults.clear(); + + const flushBatch = (): void => { + if (workingBatch.length === 0) return; + batchNumber++; + this.queueManager.enqueue(ResourceType.ASSET, operation, { + items: [...workingBatch], + environments, + locales, + batchNumber, + totalBatches, + operation, + }); + totalSubmitted += workingBatch.length; + workingBatch = []; + }; + + for (const chunkFilename of Object.values(assetsIndex)) { + const chunkPath = path.join(dataDir, 'assets', chunkFilename); + const chunkData: Record = JSON.parse(fs.readFileSync(chunkPath, 'utf-8')); + + for (const asset of Object.values(chunkData)) { + if (!asset.publish_details || asset.publish_details.length === 0) continue; + const targetUid = assetUidMapper[asset.uid as string]; + if (!targetUid) continue; + + // Skip assets that did not pass scanning. + const scanStatus = scanStatusMap.get(targetUid); + if (scanStatus === 'quarantined') { + this.logger.warn($t(messages.SCAN_STATUS_SKIPPED_QUARANTINED, { uid: targetUid })); + continue; + } + if (scanStatus === 'pending') { + this.logger.warn($t(messages.SCAN_STATUS_SKIPPED_PENDING, { uid: targetUid })); + continue; + } + + const assetLocales: string[] = useOverrideLocales + ? cliLocales + : [...new Set(asset.publish_details.map((pd: any) => pd.locale as string))]; + + for (const locale of assetLocales) { + workingBatch.push({ type: 'asset', uid: targetUid, locale, version: asset._version }); + if (workingBatch.length >= BATCH_SIZE) { + flushBatch(); + } + } + } + // chunkData falls out of scope here — GC can reclaim it before the next chunk is read. + } + + flushBatch(); + + // Invariant: pass 1 and pass 2 use identical counting logic (excluding scan status filtering). + // If batchNumber < totalBatches, scan status filtering reduced the published count — expected. + if (batchNumber !== totalBatches) { + this.logger.debug( + `Batch count: predicted ${totalBatches}, actual ${batchNumber}. Difference is expected when assets are skipped due to scan status.` + ); + } + + await this.queueManager.waitForCompletion(); + + const duration = Date.now() - startTime; + const jobIds = [...this.batchResults.values()].map((r) => r.jobId).filter((id): id is string => !!id); + + return { success: 0, failed: 0, total: totalSubmitted, duration, jobIds }; + } + + private printScanningDashboard(opts: { + total: number; + clean: number; + pending: number; + quarantined: number; + localSkipped?: number; + unmapped?: number; + }): void { + const { total, clean, pending, quarantined, localSkipped, unmapped } = opts; + const SEP = '─'.repeat(42); + + log.info(''); + log.info(` ${messages.DATA_DIR_ASSET_SCANNING_HEADER}`); + log.info(' ' + SEP); + log.info(` ${messages.DATA_DIR_TOTAL.padEnd(38)} ${total}`); + if (localSkipped !== undefined) { + log.warn(` ${messages.DATA_DIR_NO_PUBLISH_DETAILS.padEnd(38)} ${localSkipped}`); + } + if (unmapped !== undefined) { + log.warn(` ${messages.DATA_DIR_UNMAPPED.padEnd(38)} ${unmapped}`); + } + log.info(' ' + SEP); + log.info(` ${messages.SCAN_STATUS_CLEAN.padEnd(38)} ${clean}`); + if (pending > 0) log.warn(` ${messages.SCAN_STATUS_PENDING.padEnd(38)} ${pending}`); + if (quarantined > 0) log.warn(` ${messages.SCAN_STATUS_QUARANTINED.padEnd(38)} ${quarantined}`); + log.info(' ' + SEP); + log.info(` ${messages.DATA_DIR_WILL_PUBLISH.padEnd(38)} ${clean}`); + log.info(''); + } + protected async fetchItems(): Promise { return await fetchAssets(this.bulkOperationConfig, this.managementStack, this.deliveryStack, this.logger); } diff --git a/packages/contentstack-bulk-operations/src/interfaces/index.ts b/packages/contentstack-bulk-operations/src/interfaces/index.ts index c315fba34..13a4f2959 100644 --- a/packages/contentstack-bulk-operations/src/interfaces/index.ts +++ b/packages/contentstack-bulk-operations/src/interfaces/index.ts @@ -58,6 +58,8 @@ export interface BulkOperationConfig { // Asset-specific options folderUid?: string; + dataDir?: string; + dryRun?: boolean; // Cross-publish sourceEnv?: string; @@ -135,6 +137,7 @@ export interface Asset { title?: string; _version?: number; publish_details?: PublishDetails[]; + _asset_scan_status?: 'pending' | 'clean' | 'quarantined'; [key: string]: any; } @@ -196,6 +199,8 @@ export interface CommandFlags { // Asset-specific flags 'folder-uid'?: string; + 'data-dir'?: string; + 'dry-run'?: boolean; /** AM bulk delete/move */ 'space-uid'?: string; @@ -254,6 +259,7 @@ export interface AssetPublishData { locale: string; version?: number; publish_details?: PublishDetails[]; + _asset_scan_status?: 'pending' | 'clean' | 'quarantined'; } /** One row for AM bulk-delete payload `{ uid, locale }[]`. */ @@ -444,4 +450,4 @@ export interface CrossPublishConfig { contentTypes?: string[]; resourceType: ResourceType; deliveryStack: DeliveryStack | null; // Pass delivery stack from command (optional) -} \ No newline at end of file +} diff --git a/packages/contentstack-bulk-operations/src/messages/index.ts b/packages/contentstack-bulk-operations/src/messages/index.ts index a285f734e..bc9633540 100644 --- a/packages/contentstack-bulk-operations/src/messages/index.ts +++ b/packages/contentstack-bulk-operations/src/messages/index.ts @@ -210,6 +210,25 @@ const bulkAssetsMsg = { CROSS_PUBLISHING: 'Cross-publishing from {sourceEnv} to {targetEnvs}', SYNCED_ASSETS: 'Synced {count} assets from {sourceEnv}', ASSETS_READY_FOR_CROSS_PUBLISH: '{count} assets ready for cross-publish', + + // Data-dir / scanning dashboard + DATA_DIR_ASSET_SCANNING_HEADER: 'Asset Scan Status', + DATA_DIR_TOTAL: 'Total assets found', + DATA_DIR_VALID: 'Clean (will publish)', + DATA_DIR_NO_PUBLISH_DETAILS: 'No publish details (skipped)', + DATA_DIR_UNMAPPED: 'Not imported / UID unmapped (skipped)', + DATA_DIR_WILL_PUBLISH: 'Will publish', + DATA_DIR_DRY_RUN: 'Dry run — no publish API calls will be made.', + DATA_DIR_FLAG_DESC: 'Path to exported content folder containing asset publish details.', + DRY_RUN_FLAG_DESC: 'Preview the publish plan without making any API calls.', + DATA_DIR_READ_ERROR: 'Failed to read data directory at {path}: {error}', + SCAN_STATUS_CLEAN: 'Clean (will publish)', + SCAN_STATUS_PENDING: 'Still scanning (skipped)', + SCAN_STATUS_QUARANTINED: 'Quarantined (skipped)', + SCAN_STATUS_SKIPPED_PENDING: 'Skipped (still scanning): {uid}', + SCAN_STATUS_SKIPPED_QUARANTINED: 'Skipped (quarantined): {uid}', + SCAN_STATUS_FETCHING: 'Checking asset scan status for {count} assets...', + NO_PUBLISHABLE_ASSETS: 'No publishable assets — all assets are either still scanning or quarantined.', }; /** diff --git a/packages/contentstack-bulk-operations/src/services/asset-service.ts b/packages/contentstack-bulk-operations/src/services/asset-service.ts index aba844fb1..903ce3730 100644 --- a/packages/contentstack-bulk-operations/src/services/asset-service.ts +++ b/packages/contentstack-bulk-operations/src/services/asset-service.ts @@ -37,7 +37,7 @@ export class AssetService { const batchUids = uids.slice(i, i + BATCH_CONSTANTS.assetFetchBatchSize); const batchPromises = batchUids.map(async (uid) => { try { - const asset = this.deliveryStack ? await this.deliveryStack.asset(uid).fetch() : undefined; + const asset = await this.deliveryStack?.asset(uid).fetch(); return asset; } catch (error: any) { // Asset might not exist or not be published to this environment @@ -125,7 +125,13 @@ export class AssetService { try { while (hasMore) { - const queryOptions: any = { skip, limit, include_count: true, include_publish_details: true }; + const queryOptions: any = { + skip, + limit, + include_count: true, + include_publish_details: true, + include_asset_scan_status: true, + }; // Add any filters from options if (options.query) { @@ -207,7 +213,14 @@ export class AssetService { while (hasMore) { const query = this.stack .asset() - .query({ skip, limit, include_count: true, include_publish_details: true, folder: folderUid }); + .query({ + skip, + limit, + include_count: true, + include_publish_details: true, + include_asset_scan_status: true, + folder: folderUid, + }); const response = await query.find(); const assets = response.items || []; @@ -273,4 +286,34 @@ export class AssetService { throw error; } } + + /** + * Fetch scan status for a specific list of asset UIDs. + * Batches requests at 100 UIDs per call to stay within API limits. + * Returns a Map — undefined means scanning is not enabled on the stack. + */ + async fetchScanStatusByUIDs(uids: string[]): Promise> { + const statusMap = new Map(); + if (uids.length === 0) return statusMap; + + this.logger.info($t(messages.SCAN_STATUS_FETCHING, { count: uids.length })); + + const BATCH = 100; + for (let i = 0; i < uids.length; i += BATCH) { + const batch = uids.slice(i, i + BATCH); + try { + const response = await this.stack + .asset() + .query({ uid: { $in: batch }, include_asset_scan_status: true, limit: BATCH }) + .find(); + for (const asset of response.items || []) { + statusMap.set(asset.uid, asset._asset_scan_status); + } + } catch (error: any) { + this.logger.warn(`Failed to fetch scan status for batch starting at index ${i}: ${error?.message}`); + } + } + + return statusMap; + } } diff --git a/packages/contentstack-bulk-operations/src/services/bulk-operation-service.ts b/packages/contentstack-bulk-operations/src/services/bulk-operation-service.ts index 705d1a90c..18c9618f5 100644 --- a/packages/contentstack-bulk-operations/src/services/bulk-operation-service.ts +++ b/packages/contentstack-bulk-operations/src/services/bulk-operation-service.ts @@ -47,13 +47,15 @@ export class BulkOperationService { async executeBulkPublish( items: Array, operation: OperationType, - resourceType: ResourceType + resourceType: ResourceType, + environments?: string[], + locales?: string[] ): Promise { this.logger.info($t(messages.SUBMITTING_BULK_JOB, { operation, count: items.length })); try { // Step 1: Submit bulk job - const jobId = await this.submitBulkJob(items, operation, resourceType); + const jobId = await this.submitBulkJob(items, operation, resourceType, environments, locales); this.logger.debug($t(messages.BULK_JOB_CREATED, { jobId })); // Return immediate result after job submission @@ -78,10 +80,12 @@ export class BulkOperationService { private async submitBulkJob( items: Array, operation: OperationType, - resourceType: ResourceType + resourceType: ResourceType, + environments?: string[], + locales?: string[] ): Promise { try { - const payload = this.prepareBulkPayload(items, operation, resourceType); + const payload = this.prepareBulkPayload(items, operation, resourceType, environments, locales); let response: any; switch (operation) { case OperationType.PUBLISH: @@ -203,16 +207,23 @@ export class BulkOperationService { private prepareBulkPayload( items: Array, operation: OperationType, - resourceType: ResourceType + resourceType: ResourceType, + environments?: string[], + locales?: string[] ): any { if (resourceType === ResourceType.ENTRY) { - return this.prepareEntryBulkPayload(items as EntryPublishData[], operation); + return this.prepareEntryBulkPayload(items as EntryPublishData[], operation, environments, locales); } else { - return this.prepareAssetBulkPayload(items as AssetPublishData[], operation); + return this.prepareAssetBulkPayload(items as AssetPublishData[], operation, environments, locales); } } - private prepareEntryBulkPayload(items: EntryPublishData[], operation: OperationType): any { + private prepareEntryBulkPayload( + items: EntryPublishData[], + operation: OperationType, + batchEnvironments?: string[], + batchLocales?: string[] + ): any { const entries = items.map((item) => { const entry: any = { uid: item.uid, @@ -233,8 +244,17 @@ export class BulkOperationService { return entry; }); - const environments = items[0]?.publish_details?.map((pd) => pd.environment) || []; - const locales = Array.from(new Set(items.map((item) => item.locale))); + const environments = batchEnvironments?.length + ? batchEnvironments + : items[0]?.publish_details?.map((pd) => pd.environment) || []; + const locales = batchLocales?.length ? batchLocales : Array.from(new Set(items.map((item) => item.locale))); + + if (!environments.length) { + throw new Error('No environments for bulk publish. Ensure entries have publish_details with environment data.'); + } + if (!locales.length) { + throw new Error('No locales for bulk publish. Ensure entries have a locale field.'); + } return { entries, @@ -244,14 +264,28 @@ export class BulkOperationService { }; } - private prepareAssetBulkPayload(items: AssetPublishData[], operation: OperationType): any { + private prepareAssetBulkPayload( + items: AssetPublishData[], + operation: OperationType, + batchEnvironments?: string[], + batchLocales?: string[] + ): any { const assets = items.map((item) => ({ uid: item.uid, version: item.version, })); - const environments = items[0]?.publish_details?.map((pd) => pd.environment) || []; - const locales = items[0]?.publish_details?.map((pd) => pd.locale) || []; + const environments = batchEnvironments?.length + ? batchEnvironments + : items[0]?.publish_details?.map((pd) => pd.environment) || []; + const locales = batchLocales?.length ? batchLocales : items[0]?.publish_details?.map((pd) => pd.locale) || []; + + if (!environments.length) { + throw new Error('No environments for bulk publish. Ensure assets have publish_details with environment data.'); + } + if (!locales.length) { + throw new Error('No locales for bulk publish. Ensure assets have publish_details with locale data.'); + } return { assets, diff --git a/packages/contentstack-bulk-operations/src/utils/batch-queue-handler.ts b/packages/contentstack-bulk-operations/src/utils/batch-queue-handler.ts index e32eadcd8..4e624e5da 100644 --- a/packages/contentstack-bulk-operations/src/utils/batch-queue-handler.ts +++ b/packages/contentstack-bulk-operations/src/utils/batch-queue-handler.ts @@ -21,7 +21,7 @@ export function setupBatchQueueListeners(config: BatchQueueConfig) { } logger.info( - `Processing batch ${batch.batchNumber ?? 0}/${batch.totalBatches ?? 0}: ` + + `Processing batch ${batch.batchNumber}/${batch.totalBatches}: ` + `${batch.items.length} items, ` + `${batch.locales.length} locales, ` + `${batch.environments.length} environments` @@ -29,7 +29,13 @@ export function setupBatchQueueListeners(config: BatchQueueConfig) { (async () => { try { - const result = await bulkService.executeBulkPublish(batch.items, batch.operation, resourceType); + const result = await bulkService.executeBulkPublish( + batch.items, + batch.operation, + resourceType, + batch.environments, + batch.locales + ); batchResults.set(item.id, result); queueManager.updateItemStatus(item.id, OperationStatus.SUCCESS); @@ -76,7 +82,7 @@ export function setupBatchQueueListeners(config: BatchQueueConfig) { if (!batch) return; handleAndLogError(error, { - batchNumber: `${batch.batchNumber ?? 0}/${batch.totalBatches ?? 0}`, + batchNumber: `${batch.batchNumber}/${batch.totalBatches}`, itemCount: batch.items.length, }); @@ -109,7 +115,7 @@ async function handleRetryOrFailure({ : retryStrategy.getDelay(item.retryCount); logger.warn( - `Batch ${batch.batchNumber ?? 0}/${batch.totalBatches ?? 0} failed with ${ + `Batch ${batch.batchNumber}/${batch.totalBatches} failed with ${ isRateLimit ? '429 Rate Limit' : getErrorCode(error) }, retrying in ${Math.ceil(delay / 1000)}s` ); diff --git a/packages/contentstack-bulk-operations/src/utils/config-builder.ts b/packages/contentstack-bulk-operations/src/utils/config-builder.ts index c53deb98c..e92e9fb7c 100644 --- a/packages/contentstack-bulk-operations/src/utils/config-builder.ts +++ b/packages/contentstack-bulk-operations/src/utils/config-builder.ts @@ -69,24 +69,28 @@ function validateConfig(config: BulkOperationConfig): string[] { errors.push(`Invalid operation type: ${config.operation}. Must be 'publish' or 'unpublish'`); } - // Environments validation - if ( - (operation === OperationType.PUBLISH || operation === OperationType.UNPUBLISH) && - (!config.environments || config.environments.length === 0) - ) { - errors.push('Environments are required for publish/unpublish operations'); - } - if (config.environments?.some((env) => !env || env.trim() === '')) { - errors.push('Environment list cannot contain empty values'); + // Environments validation — skipped when assets are read from a data directory + if (!config.dataDir) { + if ( + (operation === OperationType.PUBLISH || operation === OperationType.UNPUBLISH) && + (!config.environments || config.environments.length === 0) + ) { + errors.push('Environments are required for publish/unpublish operations'); + } + if (config.environments?.some((env) => !env || env.trim() === '')) { + errors.push('Environment list cannot contain empty values'); + } } - // Locales validation + // Locales validation — skipped when assets are read from a data directory const isNonLocalized = config.filter === FilterType.NON_LOCALIZED; - if (!isNonLocalized && (!config.locales || config.locales.length === 0)) { - errors.push('Locales are required'); - } - if (config.locales?.some((locale) => !locale || locale.trim() === '')) { - errors.push('Locale list cannot contain empty values'); + if (!config.dataDir) { + if (!isNonLocalized && (!config.locales || config.locales.length === 0)) { + errors.push('Locales are required'); + } + if (config.locales?.some((locale) => !locale || locale.trim() === '')) { + errors.push('Locale list cannot contain empty values'); + } } // Filter validation @@ -144,24 +148,28 @@ function validateCommandFlags(flags: CommandFlags): string[] { const operation = flags.operation as OperationType; - // Environment validation - if ( - (operation === OperationType.PUBLISH || operation === OperationType.UNPUBLISH) && - (!flags.environments || flags.environments.length === 0) - ) { - errors.push('Environments are required for publish/unpublish operations'); - } - if (flags.environments?.some((env) => !env || env.trim() === '')) { - errors.push('Environment list cannot contain empty values'); + // Environment validation — skipped when assets are read from a data directory + if (!flags['data-dir']) { + if ( + (operation === OperationType.PUBLISH || operation === OperationType.UNPUBLISH) && + (!flags.environments || flags.environments.length === 0) + ) { + errors.push('Environments are required for publish/unpublish operations'); + } + if (flags.environments?.some((env) => !env || env.trim() === '')) { + errors.push('Environment list cannot contain empty values'); + } } - // Locale validation + // Locale validation — skipped when assets are read from a data directory const isNonLocalized = flags.filter === FilterType.NON_LOCALIZED; - if (!isNonLocalized && (!flags.locales || flags.locales.length === 0)) { - errors.push('Locales are required'); - } - if (flags.locales?.some((locale) => !locale || locale.trim() === '')) { - errors.push('Locale list cannot contain empty values'); + if (!flags['data-dir']) { + if (!isNonLocalized && (!flags.locales || flags.locales.length === 0)) { + errors.push('Locales are required'); + } + if (flags.locales?.some((locale) => !locale || locale.trim() === '')) { + errors.push('Locale list cannot contain empty values'); + } } // Content types validation @@ -243,6 +251,8 @@ export function buildConfig(flags: CommandFlags): BulkOperationConfig { contentTypes: flags['content-types'] !== undefined ? expandFlagStringList(flags['content-types']) : undefined, includeVariants: flags['include-variants'], folderUid: flags['folder-uid'], + dataDir: flags['data-dir'], + dryRun: flags['dry-run'], sourceEnv: flags['source-env'], publishMode: (flags['publish-mode'] as PublishMode) || PublishMode.BULK, apiVersion: flags['api-version'] || '3', diff --git a/packages/contentstack-bulk-operations/src/utils/data-dir-asset-fetcher.ts b/packages/contentstack-bulk-operations/src/utils/data-dir-asset-fetcher.ts new file mode 100644 index 000000000..71ea47bd3 --- /dev/null +++ b/packages/contentstack-bulk-operations/src/utils/data-dir-asset-fetcher.ts @@ -0,0 +1,114 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +export interface DataDirScanStats { + /** Number of assets eligible for publish (have publish_details + mapped UID). */ + eligible: number; + /** Total AssetPublishData items that will be created (eligible × locale expansions). */ + totalItems: number; + skipped: number; + unmapped: number; + environments: string[]; + locales: string[]; + /** Reusable in pass 2 — already loaded during pass 1, avoids a second disk read. */ + assetUidMapper: Record; + /** Reusable in pass 2 — already loaded during pass 1, avoids a second disk read. */ + assetsIndex: Record; +} + +/** + * Pass 1: count-only scan of the data directory. + * Reads chunk files one at a time, counts eligible/skipped/unmapped, and + * discovers environments and locales — without building any AssetPublishData objects. + * Memory footprint: uid mapper + env map + one chunk at a time. + * + * Returns assetUidMapper and assetsIndex so pass 2 (streamAndPublish) can reuse them + * without re-reading the same files from disk. + */ +export async function scanDataDirStats( + dataDir: string, + overrideEnvs?: string[], + overrideLocales?: string[], + logger?: any +): Promise { + const assetsIndexPath = path.join(dataDir, 'assets', 'assets.json'); + const environmentsPath = path.join(dataDir, 'environments', 'environments.json'); + const assetUidMapperPath = path.join(dataDir, 'mapper', 'assets', 'uid-mapping.json'); + + if (!fs.existsSync(assetsIndexPath)) { + throw new Error( + `Asset index not found: ${assetsIndexPath}. Ensure --data-dir points to the import backup directory.` + ); + } + + let assetUidMapper: Record = {}; + if (fs.existsSync(assetUidMapperPath)) { + assetUidMapper = JSON.parse(fs.readFileSync(assetUidMapperPath, 'utf-8')); + } else { + logger?.warn( + `Asset UID mapper not found: ${assetUidMapperPath}. Ensure --data-dir points to the import backup directory.` + ); + } + + const environmentsMap: Record = {}; + if (fs.existsSync(environmentsPath)) { + const envData: Record = JSON.parse(fs.readFileSync(environmentsPath, 'utf-8')); + for (const [uid, env] of Object.entries(envData)) { + environmentsMap[uid] = (env as any).name || uid; + } + } else { + logger?.warn(`Environments file not found: ${environmentsPath}`); + } + + const assetsIndex: Record = JSON.parse(fs.readFileSync(assetsIndexPath, 'utf-8')); + + let eligible = 0; + let totalItems = 0; + let skipped = 0; + let unmapped = 0; + const allEnvs = new Set(); + const allLocales = new Set(); + + for (const chunkFilename of Object.values(assetsIndex)) { + const chunkPath = path.join(dataDir, 'assets', chunkFilename); + const chunkData: Record = JSON.parse(fs.readFileSync(chunkPath, 'utf-8')); + + for (const asset of Object.values(chunkData)) { + if (!asset.publish_details || asset.publish_details.length === 0) { + skipped++; + continue; + } + + const targetUid = assetUidMapper[asset.uid as string]; + if (!targetUid) { + unmapped++; + continue; + } + + eligible++; + + if (!overrideLocales?.length) { + for (const pd of asset.publish_details) { + if (pd.locale) allLocales.add(pd.locale as string); + } + } + if (!overrideEnvs?.length) { + for (const pd of asset.publish_details) { + const envName = environmentsMap[pd.environment] || pd.environment; + if (envName) allEnvs.add(envName as string); + } + } + + const localeCount = overrideLocales?.length + ? overrideLocales.length + : new Set(asset.publish_details.map((pd: any) => pd.locale as string)).size; + totalItems += localeCount; + } + // chunkData falls out of scope here — GC reclaims it + } + + const environments = overrideEnvs?.length ? overrideEnvs : [...allEnvs]; + const locales = overrideLocales?.length ? overrideLocales : [...allLocales]; + + return { eligible, totalItems, skipped, unmapped, environments, locales, assetUidMapper, assetsIndex }; +} diff --git a/packages/contentstack-bulk-operations/src/utils/helpers.ts b/packages/contentstack-bulk-operations/src/utils/helpers.ts index 7e5beb260..8b7666048 100644 --- a/packages/contentstack-bulk-operations/src/utils/helpers.ts +++ b/packages/contentstack-bulk-operations/src/utils/helpers.ts @@ -1,7 +1,7 @@ import chalk from 'chalk'; import { getLogPath } from '@contentstack/cli-utilities'; import { $t, messages } from './index'; -import { AssetPublishData, EntryPublishData, BulkOperationResult, BulkJobResult } from '../interfaces'; +import { AssetPublishData, EntryPublishData, BulkOperationResult, BulkJobResult, Asset } from '../interfaces'; export function chunkArray(array: T[], chunkSize: number): T[][] { const chunks: T[][] = []; @@ -122,3 +122,37 @@ export function logSummary(result: any): void { console.log(''); } + +/** + * Categorize assets by their _asset_scan_status field. + * Assets with no status field belong to stacks where scanning is disabled — treat as publishable. + */ +export function categorizeByScanStatus(assets: Asset[]): { + clean: Asset[]; + pending: Asset[]; + quarantined: Asset[]; + noStatus: Asset[]; +} { + const clean: Asset[] = []; + const pending: Asset[] = []; + const quarantined: Asset[] = []; + const noStatus: Asset[] = []; + + for (const asset of assets) { + switch (asset._asset_scan_status) { + case 'clean': + clean.push(asset); + break; + case 'pending': + pending.push(asset); + break; + case 'quarantined': + quarantined.push(asset); + break; + default: + noStatus.push(asset); + } + } + + return { clean, pending, quarantined, noStatus }; +} diff --git a/packages/contentstack-bulk-operations/src/utils/index.ts b/packages/contentstack-bulk-operations/src/utils/index.ts index 02c334e3e..66497598c 100644 --- a/packages/contentstack-bulk-operations/src/utils/index.ts +++ b/packages/contentstack-bulk-operations/src/utils/index.ts @@ -21,6 +21,7 @@ import { aggregateBatchResults, createOperationResult, logSummary, + categorizeByScanStatus, } from './helpers'; import { setupBatchQueueListeners } from './batch-queue-handler'; import { confirmOperation } from './operation-confirmation'; @@ -51,6 +52,8 @@ import { validateAndBuildBulkDeleteItems, LoadAssetUidsError, } from './asset-uids-from-file'; +import { scanDataDirStats } from './data-dir-asset-fetcher'; +import type { DataDirScanStats } from './data-dir-asset-fetcher'; import { compareFieldValues, compareNonLocalizedFields, @@ -90,6 +93,7 @@ export { fetchAssets, fetchEntries, logSummary, + categorizeByScanStatus, logOperationInfo, validateBatch, enqueueIndividualItems, @@ -117,4 +121,6 @@ export { loadBulkDeleteItemsFromFile, validateAndBuildBulkDeleteItems, LoadAssetUidsError, + scanDataDirStats, }; +export type { DataDirScanStats }; diff --git a/packages/contentstack-bulk-operations/src/utils/item-fetcher.ts b/packages/contentstack-bulk-operations/src/utils/item-fetcher.ts index 35076ade2..c794520ce 100644 --- a/packages/contentstack-bulk-operations/src/utils/item-fetcher.ts +++ b/packages/contentstack-bulk-operations/src/utils/item-fetcher.ts @@ -221,6 +221,7 @@ export async function fetchAssets( environment: env, locale, })), + _asset_scan_status: asset._asset_scan_status, }); } } From 13eae5915e36f1a0fa5ebf4d95f8957eca4d129c Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Fri, 29 May 2026 14:15:52 +0530 Subject: [PATCH 06/11] chore: add config in bulk package --- packages/contentstack-bulk-operations/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/contentstack-bulk-operations/package.json b/packages/contentstack-bulk-operations/package.json index 39b6c1599..8f787d1bf 100644 --- a/packages/contentstack-bulk-operations/package.json +++ b/packages/contentstack-bulk-operations/package.json @@ -112,6 +112,7 @@ }, "csdxConfig": { "shortCommandName": { + "cm:stacks:bulk-am-assets": "BOAM", "cm:stacks:bulk-assets": "BOA", "cm:stacks:bulk-entries": "BOE", "cm:stacks:bulk-taxonomies": "BOT" From 04726617b3658525272815cac5257268b125bbcf Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Fri, 5 Jun 2026 15:57:10 +0530 Subject: [PATCH 07/11] chore: fix failing test and lint issues --- .../src/base-bulk-command.ts | 3 +-- .../src/services/asset-service.ts | 18 ++++++++---------- .../services/bulk-operation-service.test.ts | 15 +++++---------- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/packages/contentstack-bulk-operations/src/base-bulk-command.ts b/packages/contentstack-bulk-operations/src/base-bulk-command.ts index 567961bce..2693105fe 100644 --- a/packages/contentstack-bulk-operations/src/base-bulk-command.ts +++ b/packages/contentstack-bulk-operations/src/base-bulk-command.ts @@ -164,7 +164,7 @@ export abstract class BaseBulkCommand extends Command { } // Fill missing required flags via interactive prompts - flags = await this.resolveFlagsInteractively(flags); + flags = await this.resolveFlagsInteractively(flags); this.parsedFlags = flags; await this.buildConfiguration(flags); @@ -264,7 +264,6 @@ export abstract class BaseBulkCommand extends Command { } } - /** * Resolve flags interactively — subclasses can override to skip prompts for specific modes. */ diff --git a/packages/contentstack-bulk-operations/src/services/asset-service.ts b/packages/contentstack-bulk-operations/src/services/asset-service.ts index 903ce3730..562e316af 100644 --- a/packages/contentstack-bulk-operations/src/services/asset-service.ts +++ b/packages/contentstack-bulk-operations/src/services/asset-service.ts @@ -211,16 +211,14 @@ export class AssetService { try { while (hasMore) { - const query = this.stack - .asset() - .query({ - skip, - limit, - include_count: true, - include_publish_details: true, - include_asset_scan_status: true, - folder: folderUid, - }); + const query = this.stack.asset().query({ + skip, + limit, + include_count: true, + include_publish_details: true, + include_asset_scan_status: true, + folder: folderUid, + }); const response = await query.find(); const assets = response.items || []; diff --git a/packages/contentstack-bulk-operations/test/unit/services/bulk-operation-service.test.ts b/packages/contentstack-bulk-operations/test/unit/services/bulk-operation-service.test.ts index 60e1d6881..b34a84e20 100644 --- a/packages/contentstack-bulk-operations/test/unit/services/bulk-operation-service.test.ts +++ b/packages/contentstack-bulk-operations/test/unit/services/bulk-operation-service.test.ts @@ -132,7 +132,7 @@ describe('BulkOperationService', () => { content_type: 'blog', locale: 'en-us', version: 1, - publish_details: [], + publish_details: [{ environment: 'production', locale: 'en-us', version: 1 }], }, ]; @@ -168,7 +168,7 @@ describe('BulkOperationService', () => { content_type: 'blog', locale: 'en-us', version: 1, - publish_details: [], + publish_details: [{ environment: 'production', locale: 'en-us', version: 1 }], }, ]; @@ -390,14 +390,9 @@ describe('BulkOperationService', () => { } as EntryPublishData, ]; - const payload = (bulkOperationService as any).prepareBulkPayload( - mockItems, - OperationType.PUBLISH, - ResourceType.ENTRY - ); - - expect(payload.entries).to.have.lengthOf(1); - expect(payload.environments).to.deep.equal([]); + expect(() => + (bulkOperationService as any).prepareBulkPayload(mockItems, OperationType.PUBLISH, ResourceType.ENTRY) + ).to.throw('No environments for bulk publish'); }); }); From 33bd17aceafb8c0e758ccea6d715d7e4fa5f78ec Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Mon, 8 Jun 2026 12:48:52 +0530 Subject: [PATCH 08/11] feat: enhance bulk asset operations with data-dir support and user prompts --- .../src/commands/cm/stacks/bulk-assets.ts | 4 +-- .../src/messages/index.ts | 5 ++++ .../src/utils/interactive.ts | 25 +++++++++++++++++-- .../src/commands/cm/stacks/import.ts | 5 ++++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts b/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts index 53bc50c1a..5c4770578 100644 --- a/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts +++ b/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts @@ -5,7 +5,7 @@ import { flags, handleAndLogError, log } from '@contentstack/cli-utilities'; import { AssetPublishData, BulkOperationResult, OperationType, ResourceType } from '../../../interfaces'; import { BaseBulkCommand } from '../../../base-bulk-command'; -import { $t, messages, fetchAssets, scanDataDirStats, BATCH_CONSTANTS, categorizeByScanStatus } from '../../../utils'; +import { $t, messages, fetchAssets, scanDataDirStats, BATCH_CONSTANTS, categorizeByScanStatus, fillMissingFlags } from '../../../utils'; import type { DataDirScanStats } from '../../../utils'; import { AssetService } from '../../../services'; @@ -63,7 +63,7 @@ export default class BulkAssets extends BaseBulkCommand { if (flags['data-dir']) { return flags; } - return super.resolveFlagsInteractively(flags); + return fillMissingFlags(flags, { promptDataDir: true }); } async run(): Promise { diff --git a/packages/contentstack-bulk-operations/src/messages/index.ts b/packages/contentstack-bulk-operations/src/messages/index.ts index ab7e6f7d9..c51fe6023 100644 --- a/packages/contentstack-bulk-operations/src/messages/index.ts +++ b/packages/contentstack-bulk-operations/src/messages/index.ts @@ -379,6 +379,11 @@ const interactiveMsg = { TAXONOMY_UNSUPPORTED_RETRY: 'Retry and revert are not supported for bulk-taxonomies.', TAXONOMY_UNSUPPORTED_CROSS_PUBLISH: 'Cross-publish is not supported for bulk-taxonomies.', + // Data-dir (backup folder) prompt + USE_DATA_DIR_PROMPT: 'Do you want to publish assets using publish details from an import backup folder (data-dir)?', + ENTER_DATA_DIR: 'Enter the path to the import backup folder (data-dir):', + DATA_DIR_REQUIRED: 'Backup folder path is required', + // Errors NO_DELIVERY_TOKENS_FOUND: 'No delivery token aliases found. Add one using: csdx auth:tokens:add -a --delivery-token --api-key --environment --type delivery', diff --git a/packages/contentstack-bulk-operations/src/utils/interactive.ts b/packages/contentstack-bulk-operations/src/utils/interactive.ts index 2aa68cdc1..a17485e9a 100644 --- a/packages/contentstack-bulk-operations/src/utils/interactive.ts +++ b/packages/contentstack-bulk-operations/src/utils/interactive.ts @@ -151,7 +151,7 @@ async function promptForSourceAlias(): Promise { /** * Fills in missing required flags by prompting the user */ -export async function fillMissingFlags(flags: any): Promise { +export async function fillMissingFlags(flags: any, options?: { promptDataDir?: boolean }): Promise { const updatedFlags = { ...flags }; // Skip interactive mode for retry/revert operations @@ -169,9 +169,10 @@ export async function fillMissingFlags(flags: any): Promise { // Check if non-localized filter is used const isNonLocalized = updatedFlags.filter === FilterType.NON_LOCALIZED; const needsLocales = !isNonLocalized && (!updatedFlags.locales || updatedFlags.locales.length === 0); + const needsDataDir = options?.promptDataDir && !updatedFlags['data-dir']; // Only show interactive mode header if we need to prompt - if (needsCredentials || needsOperation || needsEnvironments || needsLocales) { + if (needsCredentials || needsOperation || needsEnvironments || needsLocales || needsDataDir) { cliux.print(messages.INTERACTIVE_MODE_START, { color: 'cyan' }); didPrompt = true; } @@ -191,6 +192,26 @@ export async function fillMissingFlags(flags: any): Promise { updatedFlags.operation = await promptForOperation(); } + // 2.5. Data-dir (import backup folder) — alternative to environments/locales for asset publish + if (needsDataDir) { + const useDataDir = await cliux.inquire({ + type: 'confirm', + name: 'useDataDir', + message: messages.USE_DATA_DIR_PROMPT, + default: false, + }); + if (useDataDir) { + updatedFlags['data-dir'] = await cliux.inquire({ + type: 'input', + name: 'dataDir', + message: messages.ENTER_DATA_DIR, + validate: (v: string) => (!v?.trim() ? messages.DATA_DIR_REQUIRED : true), + }); + cliux.print(messages.INTERACTIVE_MODE_COMPLETE, { color: 'green' }); + return updatedFlags; + } + } + // 3. Check for cross-publish mode const isCrossPublish = updatedFlags['source-env'] || updatedFlags['source-alias']; diff --git a/packages/contentstack-import/src/commands/cm/stacks/import.ts b/packages/contentstack-import/src/commands/cm/stacks/import.ts index 1d56aca9a..b1aadbb72 100644 --- a/packages/contentstack-import/src/commands/cm/stacks/import.ts +++ b/packages/contentstack-import/src/commands/cm/stacks/import.ts @@ -159,6 +159,11 @@ export default class ImportCommand extends Command { backupDir = importConfig.backupDir; //Note: Final summary is now handled by summary manager CLIProgressManager.printGlobalSummary(); + if (importConfig.assetScanningEnabled) { + cliux.print('\nAsset Scanning is enabled — assets were not published.', { color: 'yellow' }); + cliux.print(' Once scanning completes, publish your assets using:', { color: 'yellow' }); + cliux.print(` csdx cm:stacks:bulk-assets --data-dir ${backupDir} --stack-api-key ${importConfig.apiKey}`, { color: 'cyan' }); + } this.logSuccessAndBackupMessages(backupDir, importConfig); // Clear progress module setting now that import is complete clearProgressModuleSetting(); From 4e091242f171e2e15bdca4a7b3c36ff2d644631f Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Mon, 8 Jun 2026 12:49:33 +0530 Subject: [PATCH 09/11] chore: update talisman --- .talismanrc | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/.talismanrc b/.talismanrc index ca3f83227..736fbb319 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,26 +1,6 @@ fileignoreconfig: - filename: pnpm-lock.yaml checksum: acb2fc21dd3481f162aedaccecbe45d565b3297957345044a12aa50d2850ac4e -- filename: packages/contentstack-variants/test/unit/import/experiences.test.ts - checksum: 6142418bafea6454a72b313d933deb494ce4ea1d8cead7ef918c10e283c2c603 -- filename: packages/contentstack-variants/test/unit/export/variant-entries.test.ts - checksum: e150faeefa7b3586b70a6f454c1f68efe05526f582977dba28243b1e47606a42 -- filename: packages/contentstack-variants/test/unit/export/experiences.test.ts - checksum: eb9c989dd14373a90e8866ba3350245d3c06e8ba43cfebb24f40c1106b2e6b95 -- filename: packages/contentstack-variants/test/unit/utils/personalization-api-adapter.test.ts - checksum: d729c9586d3a19e321d79e490790b9d0aa345f81917376199d962de68317fae1 -- filename: packages/contentstack-variants/test/unit/import/variant-entries.test.ts - checksum: 1337a680c98cfa7d4be2e48db284c8e44866f5eee1e72afb4ba52c2264c3850f -- filename: packages/contentstack-export/src/export/modules/taxonomies.ts - checksum: b6d077118280bc88385405f504f921468a9fd490ac37a4a21f741be729fd1ca3 -- filename: packages/contentstack-import/test/unit/import/modules/base-class.test.ts - checksum: fe372852d5f2f3f57ef62c603406c30ccecdb444c17133ac0b21dda399b962c0 -- filename: packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts - checksum: cab2ad4d897d23f04f988c1f018a9583ab7f0ee1815994d7bc9fce23dea70073 -- filename: packages/contentstack-query-export/src/core/query-executor.ts - checksum: 708f8a9bc837ed15342fe73920588978a97cab9002c401dbc6ad7030e0238f48 -- filename: packages/contentstack-bulk-operations/test/unit/commands/bulk-am-assets.test.ts - checksum: f8d21db7db0ca2eebe7cc40af0a59f02e74e1689efb6d50a1072dc5ca3e03e9b -- filename: packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts - checksum: f7d03d649ab0b6d45985fcd9bd8ae374ea34ad5c44cb9365af92b3c9ae4728e6 +- filename: packages/contentstack-import/src/commands/cm/stacks/import.ts + checksum: d50d85534c8779b99e46b0fb8a765c0b4e5a30881007346852c254e8e2604c03 version: '1.0' From 8bb7e4331218cf0045a8d58d89debfd5c471da87 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Tue, 9 Jun 2026 12:21:01 +0530 Subject: [PATCH 10/11] chore: lint fixes --- .../src/commands/cm/stacks/bulk-assets.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts b/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts index 5c4770578..e8bfc9835 100644 --- a/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts +++ b/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts @@ -5,7 +5,15 @@ import { flags, handleAndLogError, log } from '@contentstack/cli-utilities'; import { AssetPublishData, BulkOperationResult, OperationType, ResourceType } from '../../../interfaces'; import { BaseBulkCommand } from '../../../base-bulk-command'; -import { $t, messages, fetchAssets, scanDataDirStats, BATCH_CONSTANTS, categorizeByScanStatus, fillMissingFlags } from '../../../utils'; +import { + $t, + messages, + fetchAssets, + scanDataDirStats, + BATCH_CONSTANTS, + categorizeByScanStatus, + fillMissingFlags, +} from '../../../utils'; import type { DataDirScanStats } from '../../../utils'; import { AssetService } from '../../../services'; From 194d42fce9203f326262515ad97925c7f3be260d Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Thu, 25 Jun 2026 17:54:41 +0530 Subject: [PATCH 11/11] fix: minor fixes in asset scanning flow --- .talismanrc | 6 ++-- .../src/commands/cm/stacks/bulk-assets.ts | 17 +--------- .../src/messages/index.ts | 5 --- .../src/utils/interactive.ts | 34 +++++-------------- .../src/commands/cm/stacks/import.ts | 2 +- .../src/import/modules/assets.ts | 2 +- 6 files changed, 16 insertions(+), 50 deletions(-) diff --git a/.talismanrc b/.talismanrc index b03f6eed4..54870602d 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,6 @@ fileignoreconfig: -- filename: pnpm-lock.yaml - checksum: 7ec6345eb15ed0be001753ee49733421a8a07096dc8a18465cdad1b82562fed8 +- filename: packages/contentstack-import/src/commands/cm/stacks/import.ts + checksum: bbf5ebdec2350c3b6d8118a25046e4e11b68c56434ac6c89be6aaab047a6ac4b +- filename: packages/contentstack-import/src/import/modules/assets.ts + checksum: ce46c32788ea908075873b4dee475bae75975eb58771d0da06a3debb6817bf91 version: '1.0' diff --git a/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts b/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts index e8bfc9835..c50b12766 100644 --- a/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts +++ b/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts @@ -5,15 +5,7 @@ import { flags, handleAndLogError, log } from '@contentstack/cli-utilities'; import { AssetPublishData, BulkOperationResult, OperationType, ResourceType } from '../../../interfaces'; import { BaseBulkCommand } from '../../../base-bulk-command'; -import { - $t, - messages, - fetchAssets, - scanDataDirStats, - BATCH_CONSTANTS, - categorizeByScanStatus, - fillMissingFlags, -} from '../../../utils'; +import { $t, messages, fetchAssets, scanDataDirStats, BATCH_CONSTANTS, categorizeByScanStatus } from '../../../utils'; import type { DataDirScanStats } from '../../../utils'; import { AssetService } from '../../../services'; @@ -67,13 +59,6 @@ export default class BulkAssets extends BaseBulkCommand { protected resourceType: ResourceType = ResourceType.ASSET; - protected async resolveFlagsInteractively(flags: any): Promise { - if (flags['data-dir']) { - return flags; - } - return fillMissingFlags(flags, { promptDataDir: true }); - } - async run(): Promise { try { if (this.bulkOperationConfig.sourceEnv) { diff --git a/packages/contentstack-bulk-operations/src/messages/index.ts b/packages/contentstack-bulk-operations/src/messages/index.ts index c51fe6023..ab7e6f7d9 100644 --- a/packages/contentstack-bulk-operations/src/messages/index.ts +++ b/packages/contentstack-bulk-operations/src/messages/index.ts @@ -379,11 +379,6 @@ const interactiveMsg = { TAXONOMY_UNSUPPORTED_RETRY: 'Retry and revert are not supported for bulk-taxonomies.', TAXONOMY_UNSUPPORTED_CROSS_PUBLISH: 'Cross-publish is not supported for bulk-taxonomies.', - // Data-dir (backup folder) prompt - USE_DATA_DIR_PROMPT: 'Do you want to publish assets using publish details from an import backup folder (data-dir)?', - ENTER_DATA_DIR: 'Enter the path to the import backup folder (data-dir):', - DATA_DIR_REQUIRED: 'Backup folder path is required', - // Errors NO_DELIVERY_TOKENS_FOUND: 'No delivery token aliases found. Add one using: csdx auth:tokens:add -a --delivery-token --api-key --environment --type delivery', diff --git a/packages/contentstack-bulk-operations/src/utils/interactive.ts b/packages/contentstack-bulk-operations/src/utils/interactive.ts index a17485e9a..41e3d319a 100644 --- a/packages/contentstack-bulk-operations/src/utils/interactive.ts +++ b/packages/contentstack-bulk-operations/src/utils/interactive.ts @@ -151,7 +151,7 @@ async function promptForSourceAlias(): Promise { /** * Fills in missing required flags by prompting the user */ -export async function fillMissingFlags(flags: any, options?: { promptDataDir?: boolean }): Promise { +export async function fillMissingFlags(flags: any): Promise { const updatedFlags = { ...flags }; // Skip interactive mode for retry/revert operations @@ -162,17 +162,21 @@ export async function fillMissingFlags(flags: any, options?: { promptDataDir?: b // Track if we prompted for anything let didPrompt = false; + // The presence of --data-dir is what selects the import-backup publish flow: + // environments and locales are then derived per-asset from the backup, so we + // neither prompt for the data-dir path nor for environments/locales here. + const hasDataDir = !!updatedFlags['data-dir']; + // Check if any required fields are missing const needsCredentials = !updatedFlags.alias && !updatedFlags['stack-api-key']; const needsOperation = !updatedFlags.operation; - const needsEnvironments = !updatedFlags.environments || updatedFlags.environments.length === 0; // Check if non-localized filter is used const isNonLocalized = updatedFlags.filter === FilterType.NON_LOCALIZED; - const needsLocales = !isNonLocalized && (!updatedFlags.locales || updatedFlags.locales.length === 0); - const needsDataDir = options?.promptDataDir && !updatedFlags['data-dir']; + const needsEnvironments = !hasDataDir && (!updatedFlags.environments || updatedFlags.environments.length === 0); + const needsLocales = !hasDataDir && !isNonLocalized && (!updatedFlags.locales || updatedFlags.locales.length === 0); // Only show interactive mode header if we need to prompt - if (needsCredentials || needsOperation || needsEnvironments || needsLocales || needsDataDir) { + if (needsCredentials || needsOperation || needsEnvironments || needsLocales) { cliux.print(messages.INTERACTIVE_MODE_START, { color: 'cyan' }); didPrompt = true; } @@ -192,26 +196,6 @@ export async function fillMissingFlags(flags: any, options?: { promptDataDir?: b updatedFlags.operation = await promptForOperation(); } - // 2.5. Data-dir (import backup folder) — alternative to environments/locales for asset publish - if (needsDataDir) { - const useDataDir = await cliux.inquire({ - type: 'confirm', - name: 'useDataDir', - message: messages.USE_DATA_DIR_PROMPT, - default: false, - }); - if (useDataDir) { - updatedFlags['data-dir'] = await cliux.inquire({ - type: 'input', - name: 'dataDir', - message: messages.ENTER_DATA_DIR, - validate: (v: string) => (!v?.trim() ? messages.DATA_DIR_REQUIRED : true), - }); - cliux.print(messages.INTERACTIVE_MODE_COMPLETE, { color: 'green' }); - return updatedFlags; - } - } - // 3. Check for cross-publish mode const isCrossPublish = updatedFlags['source-env'] || updatedFlags['source-alias']; diff --git a/packages/contentstack-import/src/commands/cm/stacks/import.ts b/packages/contentstack-import/src/commands/cm/stacks/import.ts index b1aadbb72..fbd208892 100644 --- a/packages/contentstack-import/src/commands/cm/stacks/import.ts +++ b/packages/contentstack-import/src/commands/cm/stacks/import.ts @@ -162,7 +162,7 @@ export default class ImportCommand extends Command { if (importConfig.assetScanningEnabled) { cliux.print('\nAsset Scanning is enabled — assets were not published.', { color: 'yellow' }); cliux.print(' Once scanning completes, publish your assets using:', { color: 'yellow' }); - cliux.print(` csdx cm:stacks:bulk-assets --data-dir ${backupDir} --stack-api-key ${importConfig.apiKey}`, { color: 'cyan' }); + cliux.print(` csdx cm:stacks:bulk-assets --data-dir ${backupDir} --stack-api-key ${importConfig.apiKey} --operation publish`, { color: 'cyan' }); } this.logSuccessAndBackupMessages(backupDir, importConfig); // Clear progress module setting now that import is complete diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index fbce6e743..f6158d5a8 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -202,7 +202,7 @@ export default class ImportAssets extends BaseClass { log.info(' Asset Scanning is enabled for this stack.', this.importConfig.context); log.info(' Assets cannot be published immediately — scanning must complete first.', this.importConfig.context); log.info(' Once scanning is done, publish your assets using:', this.importConfig.context); - log.info(' csdx cm:stacks:bulk-assets --data-dir ./content --stack-api-key ', this.importConfig.context); + log.info(' csdx cm:stacks:bulk-assets --data-dir ./content --stack-api-key --operation publish', this.importConfig.context); } } catch (error) { this.completeProgress(false, error?.message || 'Asset import failed');