From 785e76296b4a6d1c8aadb7766b4a6c0938c523b6 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Fri, 13 Feb 2026 16:25:30 +0530 Subject: [PATCH 1/3] fix: taxonomy import & entries publishing issue --- .../src/audit-base-command.ts | 2 +- .../src/constants/index.ts | 61 ++++ .../src/export/modules/assets.ts | 5 +- .../src/export/modules/content-types.ts | 11 +- .../src/export/modules/entries.ts | 15 +- .../src/export/modules/stack.ts | 10 +- .../src/export/modules/taxonomies.ts | 2 +- .../test/unit/export/modules/entries.test.ts | 2 +- .../src/constants/index.ts | 61 ++++ .../src/import/modules/assets.ts | 41 ++- .../src/import/modules/composable-studio.ts | 14 +- .../src/import/modules/content-types.ts | 61 +++- .../src/import/modules/custom-roles.ts | 29 +- .../src/import/modules/entries.ts | 100 +++--- .../src/import/modules/environments.ts | 13 +- .../src/import/modules/extensions.ts | 15 +- .../src/import/modules/global-fields.ts | 44 ++- .../src/import/modules/labels.ts | 13 +- .../src/import/modules/locales.ts | 28 +- .../src/import/modules/marketplace-apps.ts | 9 +- .../src/import/modules/stack.ts | 14 +- .../src/import/modules/taxonomies.ts | 336 +++++++++++++----- .../src/import/modules/variant-entries.ts | 5 +- .../src/import/modules/webhooks.ts | 13 +- .../src/import/modules/workflows.ts | 13 +- .../src/utils/common-helper.ts | 18 +- .../src/utils/extension-helper.ts | 22 +- .../src/utils/import-config-handler.ts | 3 - .../test/unit/import/modules/entries.test.ts | 4 +- .../unit/import/modules/taxonomies.test.ts | 64 ++-- .../test/unit/utils/extension-helper.test.ts | 6 +- 31 files changed, 763 insertions(+), 271 deletions(-) create mode 100644 packages/contentstack-export/src/constants/index.ts create mode 100644 packages/contentstack-import/src/constants/index.ts diff --git a/packages/contentstack-audit/src/audit-base-command.ts b/packages/contentstack-audit/src/audit-base-command.ts index 9b24b41800..2d33fbfd07 100644 --- a/packages/contentstack-audit/src/audit-base-command.ts +++ b/packages/contentstack-audit/src/audit-base-command.ts @@ -237,7 +237,7 @@ export abstract class AuditBaseCommand extends BaseCommand { - fsUtil.writeFile(pResolve(this.stackFolderPath, 'settings.json'), resp); + fsUtil.writeFile(pResolve(this.stackFolderPath, PATH_CONSTANTS.FILES.SETTINGS), resp); // Track progress for stack settings completion this.progressManager?.tick(true, 'stack settings', null, PROCESS_NAMES.STACK_SETTINGS); diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index 7ba474f644..885161c51f 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -125,7 +125,7 @@ export default class ExportTaxonomies extends BaseClass { return; } - const taxonomiesFilePath = pResolve(this.taxonomiesFolderPath, 'taxonomies.json'); + const taxonomiesFilePath = pResolve(this.taxonomiesFolderPath, this.taxonomiesConfig.fileName); log.debug(`Writing taxonomies metadata to: ${taxonomiesFilePath}`, this.exportConfig.context); fsUtil.writeFile(taxonomiesFilePath, this.taxonomies); } diff --git a/packages/contentstack-export/test/unit/export/modules/entries.test.ts b/packages/contentstack-export/test/unit/export/modules/entries.test.ts index e747f5802f..68deddc568 100644 --- a/packages/contentstack-export/test/unit/export/modules/entries.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/entries.test.ts @@ -92,7 +92,7 @@ describe('EntriesExport', () => { }, content_types: { dirName: 'content_types', - fileName: 'schema.json', + fileName: 'content_types.json', }, personalize: { baseURL: { diff --git a/packages/contentstack-import/src/constants/index.ts b/packages/contentstack-import/src/constants/index.ts new file mode 100644 index 0000000000..5872ec29ad --- /dev/null +++ b/packages/contentstack-import/src/constants/index.ts @@ -0,0 +1,61 @@ +export const PATH_CONSTANTS = { + /** Root mapper directory (contains module-specific mapper subdirs) */ + MAPPER: 'mapper', + + /** Common mapper file names */ + FILES: { + SUCCESS: 'success.json', + FAILS: 'fails.json', + UID_MAPPING: 'uid-mapping.json', + URL_MAPPING: 'url-mapping.json', + UID_MAPPER: 'uid-mapper.json', + SCHEMA: 'schema.json', + SETTINGS: 'settings.json', + MODIFIED_SCHEMAS: 'modified-schemas.json', + UNIQUE_MAPPING: 'unique-mapping.json', + TAXONOMIES: 'taxonomies.json', + ENVIRONMENTS: 'environments.json', + PENDING_EXTENSIONS: 'pending_extensions.js', + PENDING_GLOBAL_FIELDS: 'pending_global_fields.js', + INDEX: 'index.json', + FOLDER_MAPPING: 'folder-mapping.json', + VERSIONED_ASSETS: 'versioned-assets.json', + }, + + /** Module subdirectory names within mapper */ + MAPPER_MODULES: { + ASSETS: 'assets', + ENTRIES: 'entries', + CONTENT_TYPES: 'content_types', + TAXONOMIES: 'taxonomies', + TAXONOMY_TERMS: 'terms', + GLOBAL_FIELDS: 'global_fields', + EXTENSIONS: 'extensions', + WORKFLOWS: 'workflows', + WEBHOOKS: 'webhooks', + LABELS: 'labels', + ENVIRONMENTS: 'environments', + MARKETPLACE_APPS: 'marketplace_apps', + CUSTOM_ROLES: 'custom-roles', + LANGUAGES: 'languages', + }, + + /** Content directory names (used in both import and export) */ + CONTENT_DIRS: { + ASSETS: 'assets', + ENTRIES: 'entries', + CONTENT_TYPES: 'content_types', + TAXONOMIES: 'taxonomies', + GLOBAL_FIELDS: 'global_fields', + EXTENSIONS: 'extensions', + WEBHOOKS: 'webhooks', + WORKFLOWS: 'workflows', + LABELS: 'labels', + ENVIRONMENTS: 'environments', + STACK: 'stack', + LOCALES: 'locales', + MARKETPLACE_APPS: 'marketplace_apps', + }, +} as const; + +export type PathConstants = typeof PATH_CONSTANTS; diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index e5625e1a3c..007f21875f 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -9,7 +9,12 @@ import { existsSync } from 'node:fs'; import includes from 'lodash/includes'; import { v4 as uuid } from 'uuid'; import { resolve as pResolve, join } from 'node:path'; -import { FsUtility, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { + FsUtility, + log, + handleAndLogError, +} from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import config from '../../config'; import { ModuleClassParams } from '../../types'; @@ -36,15 +41,23 @@ export default class ImportAssets extends BaseClass { this.importConfig.context.module = MODULE_CONTEXTS.ASSETS; this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.ASSETS]; - this.assetsPath = join(this.importConfig.backupDir, 'assets'); - this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'assets'); - this.assetUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); - this.assetUrlMapperPath = join(this.mapperDirPath, 'url-mapping.json'); - this.assetFolderUidMapperPath = join(this.mapperDirPath, 'folder-mapping.json'); + this.assetsPath = join(this.importConfig.backupDir, PATH_CONSTANTS.CONTENT_DIRS.ASSETS); + this.mapperDirPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ASSETS, + ); + this.assetUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING); + this.assetUrlMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.URL_MAPPING); + this.assetFolderUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.FOLDER_MAPPING); this.assetsRootPath = join(this.importConfig.backupDir, this.assetConfig.dirName); this.fs = new FsUtility({ basePath: this.mapperDirPath }); this.environments = this.fs.readFile( - join(this.importConfig.backupDir, 'environments', 'environments.json'), + join( + this.importConfig.backupDir, + PATH_CONSTANTS.CONTENT_DIRS.ENVIRONMENTS, + PATH_CONSTANTS.FILES.ENVIRONMENTS, + ), true, ) as Record; } @@ -208,7 +221,9 @@ export default class ImportAssets extends BaseClass { */ async importAssets(isVersion = false): Promise { const processName = isVersion ? 'import versioned assets' : 'import assets'; - const indexFileName = isVersion ? 'versioned-assets.json' : 'assets.json'; + const indexFileName = isVersion + ? PATH_CONSTANTS.FILES.VERSIONED_ASSETS + : this.assetConfig.fileName; const basePath = isVersion ? join(this.assetsPath, 'versions') : this.assetsPath; const progressProcessName = isVersion ? PROCESS_NAMES.ASSET_VERSIONS : PROCESS_NAMES.ASSET_UPLOAD; @@ -355,7 +370,10 @@ export default class ImportAssets extends BaseClass { * @returns {Promise} Promise */ async publish() { - const fs = new FsUtility({ basePath: this.assetsPath, indexFileName: 'assets.json' }); + const fs = new FsUtility({ + basePath: this.assetsPath, + indexFileName: this.assetConfig.fileName, + }); if (isEmpty(this.assetsUidMap)) { log.debug('Loading asset UID mappings from file', this.importConfig.context); this.assetsUidMap = fs.readFile(this.assetUidMapperPath, true) as any; @@ -563,7 +581,10 @@ export default class ImportAssets extends BaseClass { } private async countPublishableAssets(): Promise { - const fsUtil = new FsUtility({ basePath: this.assetsPath, indexFileName: 'assets.json' }); + const fsUtil = new FsUtility({ + basePath: this.assetsPath, + indexFileName: this.assetConfig.fileName, + }); let count = 0; for (const _ of values(fsUtil.indexFileContent)) { diff --git a/packages/contentstack-import/src/import/modules/composable-studio.ts b/packages/contentstack-import/src/import/modules/composable-studio.ts index 04cd04ef8d..9c05a77e87 100644 --- a/packages/contentstack-import/src/import/modules/composable-studio.ts +++ b/packages/contentstack-import/src/import/modules/composable-studio.ts @@ -7,6 +7,7 @@ import { HttpClient, authenticationHandler, } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import isEmpty from 'lodash/isEmpty'; import { fsUtil, fileHelper } from '../../utils'; @@ -29,9 +30,18 @@ export default class ImportComposableStudio { // Setup paths this.composableStudioPath = join(this.importConfig.backupDir, this.composableStudioConfig.dirName); - this.projectMapperPath = join(this.importConfig.backupDir, 'mapper', this.composableStudioConfig.dirName); + this.projectMapperPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + this.composableStudioConfig.dirName, + ); this.composableStudioFilePath = join(this.composableStudioPath, this.composableStudioConfig.fileName); - this.envUidMapperPath = join(this.importConfig.backupDir, 'mapper', 'environments', 'uid-mapping.json'); + this.envUidMapperPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ENVIRONMENTS, + PATH_CONSTANTS.FILES.UID_MAPPING, + ); this.envUidMapper = {}; // Initialize HttpClient with Studio API base URL diff --git a/packages/contentstack-import/src/import/modules/content-types.ts b/packages/contentstack-import/src/import/modules/content-types.ts index a118a53c80..e187e145fc 100644 --- a/packages/contentstack-import/src/import/modules/content-types.ts +++ b/packages/contentstack-import/src/import/modules/content-types.ts @@ -8,6 +8,7 @@ import * as path from 'path'; import { find, cloneDeep, map } from 'lodash'; import { sanitizePath, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import { ImportConfig, ModuleClassParams } from '../../types'; import BaseClass, { ApiOptions } from './base-class'; import { updateFieldRules } from '../../utils/content-type-helper'; @@ -71,37 +72,51 @@ export default class ContentTypesImport extends BaseClass { this.gFsConfig = importConfig.modules['global-fields']; this.reqConcurrency = this.cTsConfig.writeConcurrency || this.importConfig.writeConcurrency; this.cTsFolderPath = path.join(sanitizePath(this.importConfig.contentDir), sanitizePath(this.cTsConfig.dirName)); - this.cTsMapperPath = path.join(sanitizePath(this.importConfig.contentDir), 'mapper', 'content_types'); - this.cTsSuccessPath = path.join(sanitizePath(this.importConfig.contentDir), 'mapper', 'content_types', 'success.json'); - this.gFsFolderPath = path.resolve(sanitizePath(this.importConfig.contentDir), sanitizePath(this.gFsConfig.dirName)); - this.gFsMapperFolderPath = path.join(sanitizePath(importConfig.contentDir), 'mapper', 'global_fields', 'success.json'); + this.cTsMapperPath = path.join( + sanitizePath(this.importConfig.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.CONTENT_TYPES, + ); + this.cTsSuccessPath = path.join( + sanitizePath(this.importConfig.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.CONTENT_TYPES, + PATH_CONSTANTS.FILES.SUCCESS, + ); + this.gFsFolderPath = path.resolve(sanitizePath(this.importConfig.backupDir), sanitizePath(this.gFsConfig.dirName)); + this.gFsMapperFolderPath = path.join( + sanitizePath(importConfig.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.GLOBAL_FIELDS, + PATH_CONSTANTS.FILES.SUCCESS, + ); this.gFsPendingPath = path.join( - sanitizePath(importConfig.contentDir), - 'mapper', - 'global_fields', - 'pending_global_fields.js', + sanitizePath(importConfig.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.GLOBAL_FIELDS, + PATH_CONSTANTS.FILES.PENDING_GLOBAL_FIELDS, ); this.marketplaceAppMapperPath = path.join( - sanitizePath(this.importConfig.contentDir), - 'mapper', - 'marketplace_apps', - 'uid-mapping.json', + sanitizePath(this.importConfig.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.MARKETPLACE_APPS, + PATH_CONSTANTS.FILES.UID_MAPPING, ); this.ignoredFilesInContentTypesFolder = new Map([ ['__master.json', 'true'], ['__priority.json', 'true'], - ['schema.json', 'true'], + [PATH_CONSTANTS.FILES.SCHEMA, 'true'], ['.DS_Store', 'true'], ]); // Initialize composable studio paths if config exists if (this.importConfig.modules['composable-studio']) { // Use contentDir as fallback if data is not available - const basePath = this.importConfig.data || this.importConfig.contentDir; + const basePath = this.importConfig.contentDir; this.composableStudioSuccessPath = path.join( sanitizePath(basePath), - 'mapper', + PATH_CONSTANTS.MAPPER, this.importConfig.modules['composable-studio'].dirName, this.importConfig.modules['composable-studio'].fileName, ); @@ -124,8 +139,18 @@ export default class ContentTypesImport extends BaseClass { this.createdGFs = []; this.pendingGFs = []; this.pendingExts = []; - this.taxonomiesPath = path.join(sanitizePath(importConfig.contentDir), 'mapper', 'taxonomies', 'success.json'); - this.extPendingPath = path.join(sanitizePath(importConfig.contentDir), 'mapper', 'extensions', 'pending_extensions.js'); + this.taxonomiesPath = path.join( + sanitizePath(importConfig.contentDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.TAXONOMIES, + PATH_CONSTANTS.FILES.SUCCESS, + ); + this.extPendingPath = path.join( + sanitizePath(importConfig.contentDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.EXTENSIONS, + PATH_CONSTANTS.FILES.PENDING_EXTENSIONS, + ); } async start(): Promise { @@ -474,7 +499,7 @@ export default class ContentTypesImport extends BaseClass { const [cts, gfs, pendingGfs, pendingExt] = await this.withLoadingSpinner( 'CONTENT TYPES: Analyzing import data...', async () => { - const cts = fsUtil.readFile(path.join(this.cTsFolderPath, 'schema.json')); + const cts = fsUtil.readFile(path.join(this.cTsFolderPath, PATH_CONSTANTS.FILES.SCHEMA)); const gfs = fsUtil.readFile(path.resolve(this.gFsFolderPath, this.gFsConfig.fileName)); const pendingGfs = fsUtil.readFile(this.gFsPendingPath); const pendingExt = fsUtil.readFile(this.extPendingPath); diff --git a/packages/contentstack-import/src/import/modules/custom-roles.ts b/packages/contentstack-import/src/import/modules/custom-roles.ts index 86603f0388..9678cbc6a2 100644 --- a/packages/contentstack-import/src/import/modules/custom-roles.ts +++ b/packages/contentstack-import/src/import/modules/custom-roles.ts @@ -3,6 +3,7 @@ import values from 'lodash/values'; import { join } from 'node:path'; import { forEach, map } from 'lodash'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import { fsUtil, fileHelper, PROCESS_NAMES, MODULE_CONTEXTS, PROCESS_STATUS, MODULE_NAMES } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; @@ -33,13 +34,25 @@ export default class ImportCustomRoles extends BaseClass { this.importConfig.context.module = MODULE_CONTEXTS.CUSTOM_ROLES; this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.CUSTOM_ROLES]; this.customRolesConfig = importConfig.modules.customRoles; - this.customRolesMapperPath = join(this.importConfig.backupDir, 'mapper', 'custom-roles'); + this.customRolesMapperPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.CUSTOM_ROLES, + ); this.customRolesFolderPath = join(this.importConfig.backupDir, this.customRolesConfig.dirName); - this.customRolesUidMapperPath = join(this.customRolesMapperPath, 'uid-mapping.json'); - this.envUidMapperFolderPath = join(this.importConfig.backupDir, 'mapper', 'environments'); - this.entriesUidMapperFolderPath = join(this.importConfig.backupDir, 'mapper', 'entries'); - this.createdCustomRolesPath = join(this.customRolesMapperPath, 'success.json'); - this.customRolesFailsPath = join(this.customRolesMapperPath, 'fails.json'); + this.customRolesUidMapperPath = join(this.customRolesMapperPath, PATH_CONSTANTS.FILES.UID_MAPPING); + this.envUidMapperFolderPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ENVIRONMENTS, + ); + this.entriesUidMapperFolderPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ENTRIES, + ); + this.createdCustomRolesPath = join(this.customRolesMapperPath, PATH_CONSTANTS.FILES.SUCCESS); + this.customRolesFailsPath = join(this.customRolesMapperPath, PATH_CONSTANTS.FILES.FAILS); this.customRoles = {}; this.failedCustomRoles = []; this.createdCustomRoles = []; @@ -309,11 +322,11 @@ export default class ImportCustomRoles extends BaseClass { this.customRolesUidMapper = this.loadJsonFileIfExists(this.customRolesUidMapperPath, 'custom roles'); this.environmentsUidMap = this.loadJsonFileIfExists( - join(this.envUidMapperFolderPath, 'uid-mapping.json'), + join(this.envUidMapperFolderPath, PATH_CONSTANTS.FILES.UID_MAPPING), 'environments', ); this.entriesUidMap = this.loadJsonFileIfExists( - join(this.entriesUidMapperFolderPath, 'uid-mapping.json'), + join(this.entriesUidMapperFolderPath, PATH_CONSTANTS.FILES.UID_MAPPING), 'entries', ); } diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 4b4706bb4b..2964286dae 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -8,6 +8,7 @@ import * as path from 'path'; import { writeFileSync } from 'fs'; import { isEmpty, values, cloneDeep, find, indexOf, forEach, remove } from 'lodash'; import { FsUtility, sanitizePath, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import { fsUtil, lookupExtension, @@ -68,25 +69,43 @@ export default class EntriesImport extends BaseClass { super({ importConfig, stackAPIClient }); this.importConfig.context.module = MODULE_CONTEXTS.ENTRIES; this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.ENTRIES]; - this.assetUidMapperPath = path.resolve(sanitizePath(importConfig.contentDir), 'mapper', 'assets', 'uid-mapping.json'); - this.assetUrlMapperPath = path.resolve(sanitizePath(importConfig.contentDir), 'mapper', 'assets', 'url-mapping.json'); - this.entriesMapperPath = path.resolve(sanitizePath(importConfig.contentDir), 'mapper', 'entries'); - this.envPath = path.resolve(sanitizePath(importConfig.contentDir), 'environments', 'environments.json'); - this.entriesUIDMapperPath = path.join(sanitizePath(this.entriesMapperPath), 'uid-mapping.json'); - this.uniqueUidMapperPath = path.join(sanitizePath(this.entriesMapperPath), 'unique-mapping.json'); - this.modifiedCTsPath = path.join(sanitizePath(this.entriesMapperPath), 'modified-schemas.json'); + this.assetUidMapperPath = path.resolve( + sanitizePath(importConfig.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ASSETS, + PATH_CONSTANTS.FILES.UID_MAPPING, + ); + this.assetUrlMapperPath = path.resolve( + sanitizePath(importConfig.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ASSETS, + PATH_CONSTANTS.FILES.URL_MAPPING, + ); + this.entriesMapperPath = path.resolve( + sanitizePath(importConfig.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ENTRIES, + ); + this.envPath = path.resolve( + sanitizePath(importConfig.contentDir), + PATH_CONSTANTS.CONTENT_DIRS.ENVIRONMENTS, + PATH_CONSTANTS.FILES.ENVIRONMENTS, + ); + this.entriesUIDMapperPath = path.join(sanitizePath(this.entriesMapperPath), PATH_CONSTANTS.FILES.UID_MAPPING); + this.uniqueUidMapperPath = path.join(sanitizePath(this.entriesMapperPath), PATH_CONSTANTS.FILES.UNIQUE_MAPPING); + this.modifiedCTsPath = path.join(sanitizePath(this.entriesMapperPath), PATH_CONSTANTS.FILES.MODIFIED_SCHEMAS); this.marketplaceAppMapperPath = path.join( - sanitizePath(this.importConfig.contentDir), - 'mapper', - 'marketplace_apps', - 'uid-mapping.json', + sanitizePath(this.importConfig.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.MARKETPLACE_APPS, + PATH_CONSTANTS.FILES.UID_MAPPING, ); this.taxonomiesPath = path.join( - sanitizePath(this.importConfig.contentDir), - 'mapper', - 'taxonomies', - 'terms', - 'success.json', + sanitizePath(this.importConfig.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.TAXONOMIES, + PATH_CONSTANTS.MAPPER_MODULES.TAXONOMY_TERMS, + PATH_CONSTANTS.FILES.SUCCESS, ); this.entriesConfig = importConfig.modules.entries; this.entriesPath = path.resolve(sanitizePath(importConfig.contentDir), sanitizePath(this.entriesConfig.dirName)); @@ -103,11 +122,11 @@ export default class EntriesImport extends BaseClass { // Initialize composable studio paths if config exists if (this.importConfig.modules['composable-studio']) { // Use contentDir as fallback if data is not available - const basePath = this.importConfig.data || this.importConfig.contentDir; + const basePath = this.importConfig.contentDir; this.composableStudioSuccessPath = path.join( sanitizePath(basePath), - 'mapper', + PATH_CONSTANTS.MAPPER, this.importConfig.modules['composable-studio'].dirName, this.importConfig.modules['composable-studio'].fileName, ); @@ -230,7 +249,9 @@ export default class EntriesImport extends BaseClass { return this.withLoadingSpinner('ENTRIES: Analyzing import data...', async () => { log.debug('Loading content types for entry analysis', this.importConfig.context); - this.cTs = fsUtil.readFile(path.join(this.cTsPath, 'schema.json')) as Record[]; + this.cTs = fsUtil.readFile( + path.join(this.cTsPath, PATH_CONSTANTS.FILES.SCHEMA), + ) as Record[]; if (!this.cTs || isEmpty(this.cTs)) { return [0, 0, 0, 0, 0]; } @@ -263,7 +284,7 @@ export default class EntriesImport extends BaseClass { for (let locale of this.locales) { for (let contentType of this.cTs) { const basePath = path.join(this.entriesPath, contentType.uid, locale.code); - const fs = new FsUtility({ basePath, indexFileName: 'index.json' }); + const fs = new FsUtility({ basePath, indexFileName: PATH_CONSTANTS.FILES.INDEX }); const indexer = fs.indexFileContent; const chunksInThisCTLocale = values(indexer).length; totalEntryChunks += chunksInThisCTLocale; @@ -342,7 +363,10 @@ export default class EntriesImport extends BaseClass { } log.debug('Writing entry UID mappings to file', this.importConfig.context); - await fileHelper.writeLargeFile(path.join(this.entriesMapperPath, 'uid-mapping.json'), this.entriesUidMapper); + await fileHelper.writeLargeFile( + path.join(this.entriesMapperPath, PATH_CONSTANTS.FILES.UID_MAPPING), + this.entriesUidMapper, + ); fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries); } @@ -563,7 +587,7 @@ export default class EntriesImport extends BaseClass { async createEntries({ cTUid, locale }: { cTUid: string; locale: string }): Promise { const processName = 'Create Entries'; - const indexFileName = 'index.json'; + const indexFileName = PATH_CONSTANTS.FILES.INDEX; const basePath = path.join(this.entriesPath, cTUid, locale); const fs = new FsUtility({ basePath, indexFileName }); const indexer = fs.indexFileContent; @@ -585,7 +609,7 @@ export default class EntriesImport extends BaseClass { // Write created entries const entriesCreateFileHelper = new FsUtility({ moduleName: 'entries', - indexFileName: 'index.json', + indexFileName: PATH_CONSTANTS.FILES.INDEX, basePath: path.join(this.entriesMapperPath, cTUid, locale), chunkFileSize: this.entriesConfig.chunkFileSize, keepMetadata: false, @@ -595,7 +619,7 @@ export default class EntriesImport extends BaseClass { // create file instance for existing entries const existingEntriesFileHelper = new FsUtility({ moduleName: 'entries', - indexFileName: 'index.json', + indexFileName: PATH_CONSTANTS.FILES.INDEX, basePath: path.join(this.entriesMapperPath, cTUid, locale, 'existing'), chunkFileSize: this.entriesConfig.chunkFileSize, keepMetadata: false, @@ -784,7 +808,7 @@ export default class EntriesImport extends BaseClass { async replaceEntries({ cTUid, locale }: { cTUid: string; locale: string }): Promise { const processName = 'Replace existing Entries'; - const indexFileName = 'index.json'; + const indexFileName = PATH_CONSTANTS.FILES.INDEX; const basePath = path.join(this.entriesMapperPath, cTUid, locale, 'existing'); const fs = new FsUtility({ basePath, indexFileName }); const indexer = fs.indexFileContent; @@ -801,7 +825,7 @@ export default class EntriesImport extends BaseClass { // Write updated entries const entriesReplaceFileHelper = new FsUtility({ moduleName: 'entries', - indexFileName: 'index.json', + indexFileName: PATH_CONSTANTS.FILES.INDEX, basePath: path.join(this.entriesMapperPath, cTUid, locale), chunkFileSize: this.entriesConfig.chunkFileSize, keepMetadata: false, @@ -946,7 +970,7 @@ export default class EntriesImport extends BaseClass { async updateEntriesWithReferences({ cTUid, locale }: { cTUid: string; locale: string }): Promise { const processName = 'Update Entries'; - const indexFileName = 'index.json'; + const indexFileName = PATH_CONSTANTS.FILES.INDEX; const basePath = path.join(this.entriesMapperPath, cTUid, locale); const fs = new FsUtility({ basePath, indexFileName }); const indexer = fs.indexFileContent; @@ -1204,7 +1228,9 @@ export default class EntriesImport extends BaseClass { for (let cTUid of cTsWithFieldRules) { log.debug(`Processing field rules for content type: ${cTUid}`, this.importConfig.context); - const cTs: Record[] = fsUtil.readFile(path.join(this.cTsPath, 'schema.json')) as Record< + const cTs: Record[] = fsUtil.readFile( + path.join(this.cTsPath, PATH_CONSTANTS.FILES.SCHEMA), + ) as Record< string, unknown >[]; @@ -1296,7 +1322,7 @@ export default class EntriesImport extends BaseClass { async publishEntries({ cTUid, locale }: { cTUid: string; locale: string }): Promise { const processName = 'Publish Entries'; - const indexFileName = 'index.json'; + const indexFileName = PATH_CONSTANTS.FILES.INDEX; const basePath = path.join(this.entriesPath, cTUid, locale); const fs = new FsUtility({ basePath, indexFileName }); const indexer = fs.indexFileContent; @@ -1357,19 +1383,9 @@ export default class EntriesImport extends BaseClass { }); if (chunk) { - let apiContent = values(chunk as Record[]); - let apiContentDuplicate: any = []; - apiContentDuplicate = apiContent.flatMap((content: Record) => { - if (content?.publish_details?.length > 0) { - return content.publish_details.map((publish: Record) => ({ - ...content, - locale: publish.locale, - publish_details: [publish], - })); - } - return []; // Return an empty array if publish_details is empty - }); - apiContent = apiContentDuplicate; + const apiContent = values(chunk as Record[]).filter( + (content) => content?.publish_details?.length > 0, + ); log.debug(`Processing ${apiContent.length} publishable entries in chunk ${index}`, this.importConfig.context); diff --git a/packages/contentstack-import/src/import/modules/environments.ts b/packages/contentstack-import/src/import/modules/environments.ts index 4962dd48fb..2e68f8fb47 100644 --- a/packages/contentstack-import/src/import/modules/environments.ts +++ b/packages/contentstack-import/src/import/modules/environments.ts @@ -2,6 +2,7 @@ import isEmpty from 'lodash/isEmpty'; import values from 'lodash/values'; import { join } from 'node:path'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import { fsUtil, fileHelper, PROCESS_NAMES, MODULE_CONTEXTS, PROCESS_STATUS, MODULE_NAMES } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; @@ -24,11 +25,15 @@ export default class ImportEnvironments extends BaseClass { this.importConfig.context.module = MODULE_CONTEXTS.ENVIRONMENTS; this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.ENVIRONMENTS]; this.environmentsConfig = importConfig.modules.environments; - this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'environments'); + this.mapperDirPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ENVIRONMENTS, + ); this.environmentsFolderPath = join(this.importConfig.backupDir, this.environmentsConfig.dirName); - this.envUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); - this.envSuccessPath = join(this.mapperDirPath, 'success.json'); - this.envFailsPath = join(this.mapperDirPath, 'fails.json'); + this.envUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING); + this.envSuccessPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.SUCCESS); + this.envFailsPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.FAILS); this.envFailed = []; this.envSuccess = []; this.envUidMapper = {}; diff --git a/packages/contentstack-import/src/import/modules/extensions.ts b/packages/contentstack-import/src/import/modules/extensions.ts index a2a1836817..e597762887 100644 --- a/packages/contentstack-import/src/import/modules/extensions.ts +++ b/packages/contentstack-import/src/import/modules/extensions.ts @@ -3,6 +3,7 @@ import values from 'lodash/values'; import cloneDeep from 'lodash/cloneDeep'; import { join } from 'node:path'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import { fsUtil, fileHelper, PROCESS_NAMES, MODULE_CONTEXTS, PROCESS_STATUS, MODULE_NAMES } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; @@ -28,12 +29,16 @@ export default class ImportExtensions extends BaseClass { this.importConfig.context.module = MODULE_CONTEXTS.EXTENSIONS; this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.EXTENSIONS]; this.extensionsConfig = importConfig.modules.extensions; - this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'extensions'); + this.mapperDirPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.EXTENSIONS, + ); this.extensionsFolderPath = join(this.importConfig.backupDir, this.extensionsConfig.dirName); - this.extUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); - this.extSuccessPath = join(this.mapperDirPath, 'success.json'); - this.extFailsPath = join(this.mapperDirPath, 'fails.json'); - this.extPendingPath = join(this.mapperDirPath, 'pending_extensions.js'); + this.extUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING); + this.extSuccessPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.SUCCESS); + this.extFailsPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.FAILS); + this.extPendingPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.PENDING_EXTENSIONS); this.extFailed = []; this.extSuccess = []; this.existingExtensions = []; diff --git a/packages/contentstack-import/src/import/modules/global-fields.ts b/packages/contentstack-import/src/import/modules/global-fields.ts index 4a8c909717..ba1f6840e1 100644 --- a/packages/contentstack-import/src/import/modules/global-fields.ts +++ b/packages/contentstack-import/src/import/modules/global-fields.ts @@ -9,6 +9,7 @@ import * as path from 'path'; import { isEmpty, cloneDeep } from 'lodash'; import { GlobalField } from '@contentstack/management/types/stack/globalField'; import { sanitizePath, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import { fsUtil, @@ -63,22 +64,41 @@ export default class ImportGlobalFields extends BaseClass { this.pendingGFs = []; this.existingGFs = []; this.reqConcurrency = this.gFsConfig.writeConcurrency || this.config.writeConcurrency; - this.gFsMapperPath = path.resolve(sanitizePath(this.config.contentDir), 'mapper', 'global_fields'); + this.gFsMapperPath = path.resolve( + sanitizePath(this.config.contentDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.GLOBAL_FIELDS, + ); this.gFsFolderPath = path.resolve(sanitizePath(this.config.contentDir), sanitizePath(this.gFsConfig.dirName)); - this.gFsFailsPath = path.resolve(sanitizePath(this.config.contentDir), 'mapper', 'global_fields', 'fails.json'); - this.gFsSuccessPath = path.resolve(sanitizePath(this.config.contentDir), 'mapper', 'global_fields', 'success.json'); - this.gFsUidMapperPath = path.resolve(sanitizePath(this.config.contentDir), 'mapper', 'global_fields', 'uid-mapping.json'); + this.gFsFailsPath = path.resolve( + sanitizePath(this.config.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.GLOBAL_FIELDS, + PATH_CONSTANTS.FILES.FAILS, + ); + this.gFsSuccessPath = path.resolve( + sanitizePath(this.config.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.GLOBAL_FIELDS, + PATH_CONSTANTS.FILES.SUCCESS, + ); + this.gFsUidMapperPath = path.resolve( + sanitizePath(this.config.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.GLOBAL_FIELDS, + PATH_CONSTANTS.FILES.UID_MAPPING, + ); this.gFsPendingPath = path.resolve( - sanitizePath(this.config.contentDir), - 'mapper', - 'global_fields', - 'pending_global_fields.js', + sanitizePath(this.config.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.GLOBAL_FIELDS, + PATH_CONSTANTS.FILES.PENDING_GLOBAL_FIELDS, ); this.marketplaceAppMapperPath = path.join( - sanitizePath(this.config.contentDir), - 'mapper', - 'marketplace_apps', - 'uid-mapping.json', + sanitizePath(this.config.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.MARKETPLACE_APPS, + PATH_CONSTANTS.FILES.UID_MAPPING, ); } diff --git a/packages/contentstack-import/src/import/modules/labels.ts b/packages/contentstack-import/src/import/modules/labels.ts index 021f33d4ab..020da4341f 100644 --- a/packages/contentstack-import/src/import/modules/labels.ts +++ b/packages/contentstack-import/src/import/modules/labels.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import isEmpty from 'lodash/isEmpty'; import values from 'lodash/values'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import { fsUtil, fileHelper, PROCESS_NAMES, MODULE_CONTEXTS, PROCESS_STATUS, MODULE_NAMES } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; @@ -25,11 +26,15 @@ export default class ImportLabels extends BaseClass { this.importConfig.context.module = MODULE_CONTEXTS.LABELS; this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.LABELS]; this.labelsConfig = importConfig.modules.labels; - this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'labels'); + this.mapperDirPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.LABELS, + ); this.labelsFolderPath = join(this.importConfig.backupDir, this.labelsConfig.dirName); - this.labelUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); - this.createdLabelPath = join(this.mapperDirPath, 'success.json'); - this.labelFailsPath = join(this.mapperDirPath, 'fails.json'); + this.labelUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING); + this.createdLabelPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.SUCCESS); + this.labelFailsPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.FAILS); this.labels = {}; this.failedLabel = []; this.createdLabel = []; diff --git a/packages/contentstack-import/src/import/modules/locales.ts b/packages/contentstack-import/src/import/modules/locales.ts index b18da1400f..f485a80842 100644 --- a/packages/contentstack-import/src/import/modules/locales.ts +++ b/packages/contentstack-import/src/import/modules/locales.ts @@ -8,6 +8,7 @@ import * as path from 'path'; import { values, isEmpty, filter, pick, keys } from 'lodash'; import { cliux, sanitizePath, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import BaseClass from './base-class'; import { @@ -60,11 +61,30 @@ export default class ImportLocales extends BaseClass { this.createdLocales = []; this.failedLocales = []; this.reqConcurrency = this.localeConfig.writeConcurrency || this.config.writeConcurrency; - this.langMapperPath = path.resolve(sanitizePath(this.config.contentDir), 'mapper', 'languages'); + this.langMapperPath = path.resolve( + sanitizePath(this.config.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.LANGUAGES, + ); this.langFolderPath = path.resolve(sanitizePath(this.config.contentDir), sanitizePath(this.localeConfig.dirName)); - this.langFailsPath = path.resolve(sanitizePath(this.config.contentDir), 'mapper', 'languages', 'fails.json'); - this.langSuccessPath = path.resolve(sanitizePath(this.config.contentDir), 'mapper', 'languages', 'success.json'); - this.langUidMapperPath = path.resolve(sanitizePath(this.config.contentDir), 'mapper', 'languages', 'uid-mapper.json'); + this.langFailsPath = path.resolve( + sanitizePath(this.config.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.LANGUAGES, + PATH_CONSTANTS.FILES.FAILS, + ); + this.langSuccessPath = path.resolve( + sanitizePath(this.config.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.LANGUAGES, + PATH_CONSTANTS.FILES.SUCCESS, + ); + this.langUidMapperPath = path.resolve( + sanitizePath(this.config.backupDir), + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.LANGUAGES, + PATH_CONSTANTS.FILES.UID_MAPPER, + ); } async start(): Promise { diff --git a/packages/contentstack-import/src/import/modules/marketplace-apps.ts b/packages/contentstack-import/src/import/modules/marketplace-apps.ts index ceba64f339..70b136cd78 100644 --- a/packages/contentstack-import/src/import/modules/marketplace-apps.ts +++ b/packages/contentstack-import/src/import/modules/marketplace-apps.ts @@ -19,6 +19,7 @@ import { log, handleAndLogError, } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import { askEncryptionKey, getLocationName } from '../../utils/interactive'; import { ModuleClassParams, MarketplaceAppsConfig, ImportConfig, Installation, Manifest } from '../../types'; @@ -62,9 +63,13 @@ export default class ImportMarketplaceApps extends BaseClass { this.importConfig.context.module = MODULE_CONTEXTS.MARKETPLACE_APPS; this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.MARKETPLACE_APPS]; this.marketPlaceAppConfig = importConfig.modules.marketplace_apps; - this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'marketplace_apps'); + this.mapperDirPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.MARKETPLACE_APPS, + ); this.marketPlaceFolderPath = join(this.importConfig.backupDir, this.marketPlaceAppConfig.dirName); - this.marketPlaceUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); + this.marketPlaceUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING); this.appNameMapping = {}; this.appUidMapping = {}; this.appOriginalName = undefined; diff --git a/packages/contentstack-import/src/import/modules/stack.ts b/packages/contentstack-import/src/import/modules/stack.ts index de80a03492..d6d920b0c2 100644 --- a/packages/contentstack-import/src/import/modules/stack.ts +++ b/packages/contentstack-import/src/import/modules/stack.ts @@ -1,5 +1,6 @@ import { join } from 'node:path'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import BaseClass from './base-class'; import { fileHelper, fsUtil, PROCESS_NAMES, MODULE_CONTEXTS, PROCESS_STATUS, MODULE_NAMES } from '../../utils'; @@ -15,8 +16,17 @@ export default class ImportStack extends BaseClass { super({ importConfig, stackAPIClient }); this.importConfig.context.module = MODULE_CONTEXTS.STACK; this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.STACK]; - this.stackSettingsPath = join(this.importConfig.backupDir, 'stack', 'settings.json'); - this.envUidMapperPath = join(this.importConfig.backupDir, 'mapper', 'environments', 'uid-mapping.json'); + this.stackSettingsPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.CONTENT_DIRS.STACK, + PATH_CONSTANTS.FILES.SETTINGS, + ); + this.envUidMapperPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ENVIRONMENTS, + PATH_CONSTANTS.FILES.UID_MAPPING, + ); } /** diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 5fc244b75d..2b1dac376e 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -2,6 +2,7 @@ import { join } from 'node:path'; import values from 'lodash/values'; import isEmpty from 'lodash/isEmpty'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import BaseClass, { ApiOptions } from './base-class'; import { fsUtil, fileHelper, MODULE_CONTEXTS, MODULE_NAMES, PROCESS_STATUS, PROCESS_NAMES } from '../../utils'; @@ -17,6 +18,8 @@ export default class ImportTaxonomies extends BaseClass { private termsMapperDirPath: string; private termsSuccessPath: string; private termsFailsPath: string; + private localesFilePath: string; + private isLocaleBasedStructure: boolean = false; public createdTaxonomies: Record = {}; public failedTaxonomies: Record = {}; public createdTerms: Record> = {}; @@ -27,13 +30,22 @@ export default class ImportTaxonomies extends BaseClass { this.importConfig.context.module = MODULE_CONTEXTS.TAXONOMIES; this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.TAXONOMIES]; this.taxonomiesConfig = importConfig.modules.taxonomies; - this.taxonomiesMapperDirPath = join(importConfig.backupDir, 'mapper', 'taxonomies'); - this.termsMapperDirPath = join(this.taxonomiesMapperDirPath, 'terms'); - this.taxonomiesFolderPath = join(importConfig.backupDir, this.taxonomiesConfig.dirName); - this.taxSuccessPath = join(this.taxonomiesMapperDirPath, 'success.json'); - this.taxFailsPath = join(this.taxonomiesMapperDirPath, 'fails.json'); - this.termsSuccessPath = join(this.termsMapperDirPath, 'success.json'); - this.termsFailsPath = join(this.termsMapperDirPath, 'fails.json'); + this.taxonomiesMapperDirPath = join( + importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.TAXONOMIES, + ); + this.termsMapperDirPath = join(this.taxonomiesMapperDirPath, PATH_CONSTANTS.MAPPER_MODULES.TAXONOMY_TERMS); + this.taxonomiesFolderPath = join(importConfig.contentDir, this.taxonomiesConfig.dirName); + this.taxSuccessPath = join(this.taxonomiesMapperDirPath, PATH_CONSTANTS.FILES.SUCCESS); + this.taxFailsPath = join(this.taxonomiesMapperDirPath, PATH_CONSTANTS.FILES.FAILS); + this.termsSuccessPath = join(this.termsMapperDirPath, PATH_CONSTANTS.FILES.SUCCESS); + this.termsFailsPath = join(this.termsMapperDirPath, PATH_CONSTANTS.FILES.FAILS); + this.localesFilePath = join( + importConfig.backupDir, + importConfig.modules.locales.dirName, + importConfig.modules.locales.fileName, + ); } /** @@ -50,15 +62,25 @@ export default class ImportTaxonomies extends BaseClass { return; } - const progress = this.createSimpleProgress(this.currentModuleName, taxonomiesCount); await this.prepareMapperDirectories(); + + // Check if locale-based structure exists before import + this.isLocaleBasedStructure = this.detectAndScanLocaleStructure(); + + const progress = this.createSimpleProgress(this.currentModuleName, taxonomiesCount); progress.updateStatus(PROCESS_STATUS[PROCESS_NAMES.TAXONOMIES_IMPORT].IMPORTING); log.debug('Starting taxonomies import', this.importConfig.context); - await this.importTaxonomies(); - this.createSuccessAndFailedFile(); - this.completeProgressWithMessage(); + if (this.isLocaleBasedStructure) { + log.debug('Detected locale-based folder structure for taxonomies', this.importConfig.context); + await this.importTaxonomiesByLocale(); + } else { + log.debug('Using legacy folder structure for taxonomies', this.importConfig.context); + await this.importTaxonomiesLegacy(); + } + this.createSuccessAndFailedFile(); + this.completeProgressWithMessage(); } catch (error) { this.completeProgress(false, error?.message || 'Taxonomies import failed'); handleAndLogError(error, { ...this.importConfig.context }); @@ -71,120 +93,252 @@ export default class ImportTaxonomies extends BaseClass { * @async * @returns {Promise} Promise */ - async importTaxonomies(): Promise { - log.debug('Validating taxonomies data', this.importConfig.context); - if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { - log.info('No Taxonomies Found!', this.importConfig.context); - return; - } - - const apiContent = values(this.taxonomies); - log.debug(`Starting to import ${apiContent.length} taxonomies`, this.importConfig.context); - - const onSuccess = ({ apiData }: any) => { - const taxonomyUID = apiData?.taxonomy?.uid; - const taxonomyName = apiData?.taxonomy?.name; - const termsCount = Object.keys(apiData?.terms || {}).length; - - this.createdTaxonomies[taxonomyUID] = apiData?.taxonomy; - this.createdTerms[taxonomyUID] = apiData?.terms; - - this.progressManager?.tick( - true, - null, - `taxonomy: ${taxonomyName || taxonomyUID} (${termsCount} terms)`, - PROCESS_NAMES.TAXONOMIES_IMPORT, - ); - log.success(`Taxonomy '${taxonomyUID}' imported successfully!`, this.importConfig.context); - log.debug( - `Taxonomy '${taxonomyName}' imported with ${termsCount} terms successfully!`, - this.importConfig.context, - ); - }; - - const onReject = ({ error, apiData }: any) => { - const taxonomyUID = apiData?.taxonomy?.uid; - const taxonomyName = apiData?.taxonomy?.name; - if (error?.status === 409 && error?.statusText === 'Conflict') { - log.info(`Taxonomy '${taxonomyUID}' already exists!`, this.importConfig.context); - log.debug(`Adding existing taxonomy '${taxonomyUID}' to created list`, this.importConfig.context); - this.createdTaxonomies[taxonomyUID] = apiData?.taxonomy; - this.createdTerms[taxonomyUID] = apiData?.terms; - this.progressManager?.tick( - true, - null, - `taxonomy: ${taxonomyName || taxonomyUID} already exists`, - PROCESS_NAMES.TAXONOMIES_IMPORT, - ); - } else { - this.failedTaxonomies[taxonomyUID] = apiData?.taxonomy; - this.failedTerms[taxonomyUID] = apiData?.terms; - - this.progressManager?.tick( - false, - `taxonomy: ${taxonomyName || taxonomyUID}`, - error?.message || 'Failed to import taxonomy', - PROCESS_NAMES.TAXONOMIES_IMPORT, - ); - handleAndLogError( - error, - { ...this.importConfig.context, taxonomyUID }, - `Taxonomy '${taxonomyUID}' failed to be imported`, - ); - } - }; + async importTaxonomies({ apiContent, localeCode }: { apiContent: any[]; localeCode?: string }): Promise { + const onSuccess = ({ apiData }: any) => this.handleSuccess(apiData, localeCode); + const onReject = ({ error, apiData }: any) => this.handleFailure(error, apiData, localeCode); - log.debug(`Using concurrency limit: ${this.importConfig.fetchConcurrency || 2}`, this.importConfig.context); await this.makeConcurrentCall( { apiContent, processName: 'import taxonomies', apiParams: { - serializeData: this.serializeTaxonomiesData.bind(this), + serializeData: this.serializeTaxonomy.bind(this), reject: onReject, resolve: onSuccess, entity: 'import-taxonomy', includeParamOnCompletion: true, + queryParam: { + locale: localeCode, + }, }, concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, undefined, false, ); + } - log.debug('Taxonomies import process completed', this.importConfig.context); + /** + * Import taxonomies using legacy structure (taxonomies/{uid}.json) + */ + async importTaxonomiesLegacy(): Promise { + const apiContent = values(this.taxonomies); + await this.importTaxonomies({ apiContent }); } /** - * @method serializeTaxonomiesData - * @param {ApiOptions} apiOptions ApiOptions - * @returns {ApiOptions} ApiOptions + * Import taxonomies using locale-based structure (taxonomies/{locale}/{uid}.json) */ - serializeTaxonomiesData(apiOptions: ApiOptions): ApiOptions { - const { apiData: taxonomyData } = apiOptions; + async importTaxonomiesByLocale(): Promise { + const locales = this.loadAvailableLocales(); + const apiContent = values(this.taxonomies); + + for (const localeCode of Object.keys(locales)) { + await this.importTaxonomies({ apiContent, localeCode }); + } + } + + handleSuccess(apiData: any, locale?: string) { + const { taxonomy, terms } = apiData || {}; + const taxonomyUID = taxonomy?.uid; + const taxonomyName = taxonomy?.name; + const termsCount = Object.keys(terms || {}).length; + + this.createdTaxonomies[taxonomyUID] = taxonomy; + this.createdTerms[taxonomyUID] = terms; + + this.progressManager?.tick( + true, + `taxonomy: ${taxonomyName || taxonomyUID}`, + null, + PROCESS_NAMES.TAXONOMIES_IMPORT, + ); + + log.success( + `Taxonomy '${taxonomyUID}' imported successfully${locale ? ` for locale: ${locale}` : ''}!`, + this.importConfig.context, + ); log.debug( - `Serializing taxonomy: ${taxonomyData.taxonomy?.name} (${taxonomyData.taxonomy?.uid})`, + `Created taxonomy '${taxonomyName}' with ${termsCount} terms${locale ? ` for locale: ${locale}` : ''}`, this.importConfig.context, ); + } - const taxonomyUID = taxonomyData?.uid; - const filePath = join(this.taxonomiesFolderPath, `${taxonomyUID}.json`); + handleFailure(error: any, apiData: any, locale?: string) { + const taxonomyUID = apiData?.taxonomy?.uid; + const taxonomyName = apiData?.taxonomy?.name; - log.debug(`Looking for taxonomy file: ${filePath}`, this.importConfig.context); + if (error?.status === 409 && error?.statusText === 'Conflict') { + this.progressManager?.tick( + true, + null, + `taxonomy: ${taxonomyName || taxonomyUID} (already exists)`, + PROCESS_NAMES.TAXONOMIES_IMPORT, + ); + log.info( + `Taxonomy '${taxonomyUID}' already exists ${locale ? ` for locale: ${locale}` : ''}!`, + this.importConfig.context, + ); + this.createdTaxonomies[taxonomyUID] = apiData?.taxonomy; + this.createdTerms[taxonomyUID] = apiData?.terms; + return; + } - if (fileHelper.fileExistsSync(filePath)) { - const taxonomyDetails = fsUtil.readFile(filePath, true) as Record; - log.debug(`Successfully loaded taxonomy details from ${filePath}`, this.importConfig.context); + const errMsg = error?.errorMessage || error?.errors?.taxonomy || error?.errors?.term || error?.message; + + this.progressManager?.tick( + false, + `taxonomy: ${taxonomyName || taxonomyUID}`, + errMsg || 'Failed to import taxonomy', + PROCESS_NAMES.TAXONOMIES_IMPORT, + ); + + if (errMsg) { + log.error( + `Taxonomy '${taxonomyUID}' failed to import${locale ? ` for locale: ${locale}` : ''}! ${errMsg}`, + this.importConfig.context, + ); + } else { + handleAndLogError( + error, + { ...this.importConfig.context, taxonomyUID, locale }, + `Taxonomy '${taxonomyUID}' failed`, + ); + } + + this.failedTaxonomies[taxonomyUID] = apiData?.taxonomy; + this.failedTerms[taxonomyUID] = apiData?.terms; + } + + /** + * @method serializeTaxonomy + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeTaxonomy(apiOptions: ApiOptions): ApiOptions { + const { + apiData, + queryParam: { locale }, + } = apiOptions; + const taxonomyUID = apiData?.uid; + + if (!taxonomyUID) { + log.debug('No taxonomy UID provided for serialization', this.importConfig.context); + apiOptions.apiData = undefined; + return apiOptions; + } + + const context = locale ? ` for locale: ${locale}` : ''; + log.debug(`Serializing taxonomy: ${taxonomyUID}${context}`, this.importConfig.context); + + // Determine file path - if locale is provided, use it directly, otherwise search + const filePath = locale + ? join(this.taxonomiesFolderPath, locale, `${taxonomyUID}.json`) + : this.findTaxonomyFilePath(taxonomyUID); + + if (!filePath || !fileHelper.fileExistsSync(filePath)) { + log.debug(`Taxonomy file not found for: ${taxonomyUID}${context}`, this.importConfig.context); + apiOptions.apiData = undefined; + return apiOptions; + } + + const taxonomyDetails = this.loadTaxonomyFile(filePath); + if (taxonomyDetails) { const termCount = Object.keys(taxonomyDetails?.terms || {}).length; - log.debug(`Taxonomy has ${termCount} term entries`, this.importConfig.context); - apiOptions.apiData = { filePath, taxonomy: taxonomyDetails?.taxonomy, terms: taxonomyDetails?.terms }; + log.debug(`Taxonomy has ${termCount} term entries${context}`, this.importConfig.context); + + apiOptions.apiData = { + filePath, + taxonomy: taxonomyDetails?.taxonomy, + terms: taxonomyDetails?.terms, + }; } else { - log.debug(`File does not exist for taxonomy: ${taxonomyUID}`, this.importConfig.context); apiOptions.apiData = undefined; } + return apiOptions; } + loadTaxonomyFile(filePath: string): Record | undefined { + if (!fileHelper.fileExistsSync(filePath)) { + log.debug(`File does not exist: ${filePath}`, this.importConfig.context); + return undefined; + } + + try { + const taxonomyDetails = fsUtil.readFile(filePath, true) as Record; + log.debug(`Successfully loaded taxonomy from: ${filePath}`, this.importConfig.context); + return taxonomyDetails; + } catch (error) { + log.debug(`Error loading taxonomy file: ${filePath}`, this.importConfig.context); + return undefined; + } + } + + findTaxonomyFilePath(taxonomyUID: string): string | undefined { + if (this.isLocaleBasedStructure) { + return this.findTaxonomyInLocaleFolders(taxonomyUID); + } + + const legacyPath = join(this.taxonomiesFolderPath, `${taxonomyUID}.json`); + return fileHelper.fileExistsSync(legacyPath) ? legacyPath : undefined; + } + + findTaxonomyInLocaleFolders(taxonomyUID: string): string | undefined { + const locales = this.loadAvailableLocales(); + + for (const localeCode of Object.keys(locales)) { + const filePath = join(this.taxonomiesFolderPath, localeCode, `${taxonomyUID}.json`); + if (fileHelper.fileExistsSync(filePath)) { + return filePath; + } + } + + return undefined; + } + + loadAvailableLocales(): Record { + if (!fileHelper.fileExistsSync(this.localesFilePath)) { + log.debug('No locales file found', this.importConfig.context); + return {}; + } + + try { + const localesData = fsUtil.readFile(this.localesFilePath, true) as Record>; + const locales: Record = {}; + const masterCode = this.importConfig.master_locale?.code || 'en-us'; + locales[masterCode] = masterCode; + + for (const [, locale] of Object.entries(localesData || {})) { + if (locale?.code) { + locales[locale.code] = locale.code; + } + } + + log.debug(`Loaded ${Object.keys(locales).length} locales from file`, this.importConfig.context); + return locales; + } catch (error) { + log.debug('Error loading locales file', this.importConfig.context); + return {}; + } + } + + /** + * Detect if locale-based folder structure exists (taxonomies/{locale}/{uid}.json) + */ + detectAndScanLocaleStructure(): boolean { + const masterLocaleCode = this.importConfig.master_locale?.code || 'en-us'; + const masterLocaleFolder = join(this.taxonomiesFolderPath, masterLocaleCode); + + if (!fileHelper.fileExistsSync(masterLocaleFolder)) { + log.debug('No locale-based folder structure detected', this.importConfig.context); + return false; + } + + log.debug('Locale-based folder structure detected', this.importConfig.context); + return true; + } + /** * create taxonomies success and fail in (mapper/taxonomies) * create terms success and fail in (mapper/taxonomies/terms) diff --git a/packages/contentstack-import/src/import/modules/variant-entries.ts b/packages/contentstack-import/src/import/modules/variant-entries.ts index 0953ebfaf7..938f372b2b 100644 --- a/packages/contentstack-import/src/import/modules/variant-entries.ts +++ b/packages/contentstack-import/src/import/modules/variant-entries.ts @@ -1,5 +1,6 @@ import path from 'path'; import { sanitizePath, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import { Import, ImportHelperMethodsConfig, ProjectStruct } from '@contentstack/cli-variants'; import { ImportConfig, ModuleClassParams } from '../../types'; import { @@ -29,8 +30,8 @@ export default class ImportVariantEntries extends BaseClass { this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.VARIANT_ENTRIES]; this.personalize = importConfig.modules.personalize; this.projectMapperFilePath = path.resolve( - sanitizePath(this.config.contentDir), - 'mapper', + sanitizePath(this.config.backupDir), + PATH_CONSTANTS.MAPPER, sanitizePath(this.personalize.dirName), 'projects', 'projects.json', diff --git a/packages/contentstack-import/src/import/modules/webhooks.ts b/packages/contentstack-import/src/import/modules/webhooks.ts index a9197ce0b1..c02819a12a 100644 --- a/packages/contentstack-import/src/import/modules/webhooks.ts +++ b/packages/contentstack-import/src/import/modules/webhooks.ts @@ -2,6 +2,7 @@ import isEmpty from 'lodash/isEmpty'; import values from 'lodash/values'; import { join } from 'node:path'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import { fsUtil, fileHelper, PROCESS_NAMES, MODULE_CONTEXTS, PROCESS_STATUS, MODULE_NAMES } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; @@ -24,11 +25,15 @@ export default class ImportWebhooks extends BaseClass { this.importConfig.context.module = MODULE_CONTEXTS.WEBHOOKS; this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.WEBHOOKS]; this.webhooksConfig = importConfig.modules.webhooks; - this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'webhooks'); + this.mapperDirPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.WEBHOOKS, + ); this.webhooksFolderPath = join(this.importConfig.backupDir, this.webhooksConfig.dirName); - this.webhookUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); - this.createdWebhooksPath = join(this.mapperDirPath, 'success.json'); - this.failedWebhooksPath = join(this.mapperDirPath, 'fails.json'); + this.webhookUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING); + this.createdWebhooksPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.SUCCESS); + this.failedWebhooksPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.FAILS); this.webhooks = {}; this.failedWebhooks = []; this.createdWebhooks = []; diff --git a/packages/contentstack-import/src/import/modules/workflows.ts b/packages/contentstack-import/src/import/modules/workflows.ts index 5c8ce218d7..454ca5f9c9 100644 --- a/packages/contentstack-import/src/import/modules/workflows.ts +++ b/packages/contentstack-import/src/import/modules/workflows.ts @@ -8,6 +8,7 @@ import isEmpty from 'lodash/isEmpty'; import cloneDeep from 'lodash/cloneDeep'; import findIndex from 'lodash/findIndex'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../../constants'; import BaseClass, { ApiOptions } from './base-class'; import { @@ -38,11 +39,15 @@ export default class ImportWorkflows extends BaseClass { this.importConfig.context.module = MODULE_CONTEXTS.WORKFLOWS; this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.WORKFLOWS]; this.workflowsConfig = importConfig.modules.workflows; - this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'workflows'); + this.mapperDirPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.WORKFLOWS, + ); this.workflowsFolderPath = join(this.importConfig.backupDir, this.workflowsConfig.dirName); - this.workflowUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); - this.createdWorkflowsPath = join(this.mapperDirPath, 'success.json'); - this.failedWorkflowsPath = join(this.mapperDirPath, 'fails.json'); + this.workflowUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING); + this.createdWorkflowsPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.SUCCESS); + this.failedWorkflowsPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.FAILS); this.workflows = {}; this.failedWebhooks = []; this.createdWorkflows = []; diff --git a/packages/contentstack-import/src/utils/common-helper.ts b/packages/contentstack-import/src/utils/common-helper.ts index c1a705797b..6256f89e16 100644 --- a/packages/contentstack-import/src/utils/common-helper.ts +++ b/packages/contentstack-import/src/utils/common-helper.ts @@ -7,7 +7,15 @@ import * as _ from 'lodash'; import * as path from 'path'; -import { HttpClient, managementSDKClient, isAuthenticated, sanitizePath, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { + HttpClient, + managementSDKClient, + isAuthenticated, + sanitizePath, + log, + handleAndLogError, +} from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../constants'; import { readFileSync, readdirSync, readFile, fileExistsSync } from './file-helper'; import chalk from 'chalk'; @@ -161,8 +169,12 @@ export const field_rules_update = (importConfig: ImportConfig, ctPath: string) = if (schema.field_rules[k].conditions[i].operand_field === 'reference') { log.debug(`Processing reference field rule condition`); - let entryMapperPath = path.resolve(importConfig.contentDir, 'mapper', 'entries'); - let entryUidMapperPath = path.join(entryMapperPath, 'uid-mapping.json'); + let entryMapperPath = path.resolve( + importConfig.contentDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ENTRIES, + ); + let entryUidMapperPath = path.join(entryMapperPath, PATH_CONSTANTS.FILES.UID_MAPPING); let fieldRulesValue = schema.field_rules[k].conditions[i].value; let fieldRulesArray = fieldRulesValue.split('.'); let updatedValue = []; diff --git a/packages/contentstack-import/src/utils/extension-helper.ts b/packages/contentstack-import/src/utils/extension-helper.ts index 2dc8ac4e32..e2dab97507 100644 --- a/packages/contentstack-import/src/utils/extension-helper.ts +++ b/packages/contentstack-import/src/utils/extension-helper.ts @@ -9,6 +9,7 @@ */ import { join } from 'node:path'; import { FsUtility, log } from '@contentstack/cli-utilities'; +import { PATH_CONSTANTS } from '../constants'; import { ImportConfig } from '../types'; // eslint-disable-next-line camelcase @@ -21,9 +22,24 @@ export const lookupExtension = function ( log.debug('Starting extension lookup process...'); const fs = new FsUtility({ basePath: config.backupDir }); - const extensionPath = join(config.backupDir, 'mapper/extensions', 'uid-mapping.json'); - const globalfieldsPath = join(config.backupDir, 'mapper/globalfields', 'uid-mapping.json'); - const marketPlaceAppsPath = join(config.backupDir, 'mapper/marketplace_apps', 'uid-mapping.json'); + const extensionPath = join( + config.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.EXTENSIONS, + PATH_CONSTANTS.FILES.UID_MAPPING, + ); + const globalfieldsPath = join( + config.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.GLOBAL_FIELDS, + PATH_CONSTANTS.FILES.UID_MAPPING, + ); + const marketPlaceAppsPath = join( + config.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.MARKETPLACE_APPS, + PATH_CONSTANTS.FILES.UID_MAPPING, + ); log.debug( `Extension mapping paths - Extensions: ${extensionPath}, Global fields: ${globalfieldsPath}, Marketplace apps: ${marketPlaceAppsPath}`, diff --git a/packages/contentstack-import/src/utils/import-config-handler.ts b/packages/contentstack-import/src/utils/import-config-handler.ts index f6604d0530..0eb0ee2972 100644 --- a/packages/contentstack-import/src/utils/import-config-handler.ts +++ b/packages/contentstack-import/src/utils/import-config-handler.ts @@ -126,9 +126,6 @@ const setupConfig = async (importCmdFlags: any): Promise => { config['exclude-global-modules'] = importCmdFlags['exclude-global-modules']; } - // Set progress supported module to check and display console logs - configHandler.set('log.progressSupportedModule', 'import'); - // Add authentication details to config for context tracking config.authenticationMethod = authenticationMethod; log.debug('Import configuration setup completed.', { ...config }); diff --git a/packages/contentstack-import/test/unit/import/modules/entries.test.ts b/packages/contentstack-import/test/unit/import/modules/entries.test.ts index 7fd6818905..a8fa0daa7e 100644 --- a/packages/contentstack-import/test/unit/import/modules/entries.test.ts +++ b/packages/contentstack-import/test/unit/import/modules/entries.test.ts @@ -2136,8 +2136,8 @@ describe('EntriesImport', () => { await entriesImport['publishEntries']({ cTUid: 'simple_ct', locale: 'en-us' }); expect(makeConcurrentCallStub.called).to.be.true; - // Should create multiple entries for each publish detail - expect(makeConcurrentCallStub.getCall(0).args[0].apiContent).to.have.lengthOf(3); // 3 publish details + // Should pass 1 entry with all publish details (serializePublishEntries aggregates them into one API call) + expect(makeConcurrentCallStub.getCall(0).args[0].apiContent).to.have.lengthOf(1); }); it('should handle entries without publish details', async () => { diff --git a/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts b/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts index 3e843a5ada..fc5701115a 100644 --- a/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts +++ b/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts @@ -100,6 +100,9 @@ describe('ImportTaxonomies', () => { expect((importTaxonomies as any).taxFailsPath).to.equal(join(testBackupDir, 'mapper', 'taxonomies', 'fails.json')); expect((importTaxonomies as any).termsSuccessPath).to.equal(join(testBackupDir, 'mapper', 'taxonomies', 'terms', 'success.json')); expect((importTaxonomies as any).termsFailsPath).to.equal(join(testBackupDir, 'mapper', 'taxonomies', 'terms', 'fails.json')); + expect((importTaxonomies as any).localesFilePath).to.equal( + join(testBackupDir, 'locales', 'locales.json'), + ); }); it('should set context module to taxonomies', () => { @@ -272,22 +275,20 @@ describe('ImportTaxonomies', () => { }); it('should handle empty taxonomies data', async () => { - (importTaxonomies as any).taxonomies = {}; const makeConcurrentCallStub = sandbox.stub(importTaxonomies as any, 'makeConcurrentCall').resolves(); - await (importTaxonomies as any).importTaxonomies(); + await (importTaxonomies as any).importTaxonomies({ apiContent: [] }); - // When taxonomies is empty, makeConcurrentCall should not be called + // When apiContent is empty, makeConcurrentCall should not be called expect(makeConcurrentCallStub.called).to.be.false; }); it('should handle undefined taxonomies', async () => { - (importTaxonomies as any).taxonomies = undefined; const makeConcurrentCallStub = sandbox.stub(importTaxonomies as any, 'makeConcurrentCall').resolves(); - await (importTaxonomies as any).importTaxonomies(); + await (importTaxonomies as any).importTaxonomies({ apiContent: undefined as any }); - // When taxonomies is undefined, makeConcurrentCall should not be called + // When apiContent is undefined, makeConcurrentCall should not be called expect(makeConcurrentCallStub.called).to.be.false; }); @@ -306,7 +307,7 @@ describe('ImportTaxonomies', () => { }); }); - describe('serializeTaxonomiesData', () => { + describe('serializeTaxonomy', () => { it('should serialize taxonomy successfully', () => { const mockApiOptions = { entity: 'import-taxonomy' as any, @@ -322,7 +323,7 @@ describe('ImportTaxonomies', () => { terms: { 'term_1': { uid: 'term_1', name: 'Term 1' } } }); - const result = (importTaxonomies as any).serializeTaxonomiesData(mockApiOptions); + const result = (importTaxonomies as any).serializeTaxonomy(mockApiOptions); expect(result).to.have.property('apiData'); expect(result.apiData.taxonomy).to.have.property('uid'); @@ -340,7 +341,7 @@ describe('ImportTaxonomies', () => { (fileHelper.fileExistsSync as any).returns(false); - const result = (importTaxonomies as any).serializeTaxonomiesData(mockApiOptions); + const result = (importTaxonomies as any).serializeTaxonomy(mockApiOptions); expect(result.apiData).to.be.undefined; }); @@ -363,7 +364,7 @@ describe('ImportTaxonomies', () => { } }); - const result = (importTaxonomies as any).serializeTaxonomiesData(mockApiOptions); + const result = (importTaxonomies as any).serializeTaxonomy(mockApiOptions); expect(result.apiData.terms).to.have.property('term_1'); expect(result.apiData.terms).to.have.property('term_2'); @@ -384,7 +385,7 @@ describe('ImportTaxonomies', () => { terms: {} }); - const result = (importTaxonomies as any).serializeTaxonomiesData(mockApiOptions); + const result = (importTaxonomies as any).serializeTaxonomy(mockApiOptions); expect(result.apiData.terms).to.deep.equal({}); }); @@ -780,7 +781,7 @@ describe('ImportTaxonomies', () => { describe('Callback Functions Integration', () => { it('should execute actual onSuccess callback with lines 93-105', async () => { - // Set up file helper to return false so serializeTaxonomiesData gets proper data + // Set up file helper to return false so serializeTaxonomy gets proper data (fileHelper.fileExistsSync as any).returns(false); (fsUtil.readFile as any).returns({}); (fsUtil.makeDirectory as any).resolves(); @@ -839,14 +840,18 @@ describe('ImportTaxonomies', () => { actualOnSuccess = config.apiParams.resolve; actualOnReject = config.apiParams.reject; - // Execute serializeTaxonomiesData to get proper apiData - const serialized = (importTaxonomies as any).serializeTaxonomiesData({ + // Execute serializeTaxonomy to get proper apiData + const apiOptions = { apiData: config.apiContent[0], entity: 'import-taxonomy', queryParam: { locale: config.apiParams.queryParam?.locale }, resolve: actualOnSuccess, reject: actualOnReject - }); + }; + const serialized = (importTaxonomies as any).serializeTaxonomy( + apiOptions, + config.apiParams.queryParam?.locale + ); // Call the ACTUAL onReject callback with 409 error if (serialized.apiData) { @@ -859,7 +864,7 @@ describe('ImportTaxonomies', () => { await (importTaxonomies as any).importTaxonomies({ apiContent: values((importTaxonomies as any).taxonomies) }); - // Verify lines 117-118 executed (adding to createdTaxonomies and createdTerms on 409) + // Verify 409 conflict adds to createdTaxonomies and createdTerms expect((importTaxonomies as any).createdTaxonomies['taxonomy_1']).to.exist; expect((importTaxonomies as any).createdTerms['taxonomy_1']).to.exist; }); @@ -895,14 +900,18 @@ describe('ImportTaxonomies', () => { actualOnSuccess = config.apiParams.resolve; actualOnReject = config.apiParams.reject; - // Execute serializeTaxonomiesData to get proper apiData - const serialized = (importTaxonomies as any).serializeTaxonomiesData({ + // Execute serializeTaxonomy to get proper apiData + const apiOptions = { apiData: config.apiContent[0], entity: 'import-taxonomy', queryParam: { locale: config.apiParams.queryParam?.locale }, resolve: actualOnSuccess, reject: actualOnReject - }); + }; + const serialized = (importTaxonomies as any).serializeTaxonomy( + apiOptions, + config.apiParams.queryParam?.locale + ); // Call the ACTUAL onReject callback with other error if (serialized.apiData) { @@ -915,7 +924,7 @@ describe('ImportTaxonomies', () => { await (importTaxonomies as any).importTaxonomies({ apiContent: values((importTaxonomies as any).taxonomies) }); - // Verify lines 131-132 executed (adding to failedTaxonomies and failedTerms) + // Verify error adds to failedTaxonomies and failedTerms expect((importTaxonomies as any).failedTaxonomies['taxonomy_1']).to.exist; expect((importTaxonomies as any).failedTerms['taxonomy_1']).to.exist; }); @@ -1132,7 +1141,7 @@ describe('ImportTaxonomies', () => { } }); - it('should handle file read errors in serializeTaxonomiesData', () => { + it('should handle file read errors in serializeTaxonomy', () => { const mockApiOptions = { entity: 'import-taxonomy' as any, apiData: { uid: 'taxonomy_1', name: 'Test Taxonomy' }, @@ -1144,15 +1153,10 @@ describe('ImportTaxonomies', () => { (fileHelper.fileExistsSync as any).returns(true); (fsUtil.readFile as any).throws(new Error('File read error')); - // The error will be thrown since serializeTaxonomiesData doesn't catch it - try { - const result = (importTaxonomies as any).serializeTaxonomiesData(mockApiOptions); - // If we get here, the error wasn't thrown (unexpected) - expect.fail('Expected error to be thrown'); - } catch (error: any) { - // Error should be thrown - expect(error.message).to.equal('File read error'); - } + // loadTaxonomyFile catches errors and returns undefined, so apiData becomes undefined + const result = (importTaxonomies as any).serializeTaxonomy(mockApiOptions); + + expect(result.apiData).to.be.undefined; }); }); }); diff --git a/packages/contentstack-import/test/unit/utils/extension-helper.test.ts b/packages/contentstack-import/test/unit/utils/extension-helper.test.ts index e488850444..5eee945eb0 100644 --- a/packages/contentstack-import/test/unit/utils/extension-helper.test.ts +++ b/packages/contentstack-import/test/unit/utils/extension-helper.test.ts @@ -350,7 +350,7 @@ describe('Extension Helper', () => { 'global-field-123': 'mapped-global-field-456', }; - fsUtilityStub.withArgs(path.join(tempDir, 'mapper/globalfields/uid-mapping.json')).returns(globalFieldsMapping); + fsUtilityStub.withArgs(path.join(tempDir, 'mapper/global_fields/uid-mapping.json')).returns(globalFieldsMapping); lookupExtension(config, schema, preserveStackVersion, installedExtensions); @@ -369,7 +369,7 @@ describe('Extension Helper', () => { const preserveStackVersion = false; const installedExtensions = {}; - fsUtilityStub.withArgs(path.join(tempDir, 'mapper/globalfields/uid-mapping.json')).returns({}); + fsUtilityStub.withArgs(path.join(tempDir, 'mapper/global_fields/uid-mapping.json')).returns({}); lookupExtension(config, schema, preserveStackVersion, installedExtensions); @@ -543,7 +543,7 @@ describe('Extension Helper', () => { 'global-1': 'mapped-global-1', }; - fsUtilityStub.withArgs(path.join(tempDir, 'mapper/globalfields/uid-mapping.json')).returns(globalFieldsMapping); + fsUtilityStub.withArgs(path.join(tempDir, 'mapper/global_fields/uid-mapping.json')).returns(globalFieldsMapping); lookupExtension(config, schema, preserveStackVersion, installedExtensions); From a17c05e00d3948beb9c96eba3a1faed3c4062e2b Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Fri, 13 Feb 2026 18:20:45 +0530 Subject: [PATCH 2/3] test: fix taxonomy & variant entries test cases --- .../contentstack-import/src/import/modules/taxonomies.ts | 5 +++++ .../test/unit/import/modules/taxonomies.test.ts | 1 + .../test/unit/import/modules/variant-entries.test.ts | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 2b1dac376e..bd9b1a87c7 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -94,6 +94,11 @@ export default class ImportTaxonomies extends BaseClass { * @returns {Promise} Promise */ async importTaxonomies({ apiContent, localeCode }: { apiContent: any[]; localeCode?: string }): Promise { + if (!apiContent || apiContent?.length === 0) { + log.debug('No taxonomies to import', this.importConfig.context); + return; + } + const onSuccess = ({ apiData }: any) => this.handleSuccess(apiData, localeCode); const onReject = ({ error, apiData }: any) => this.handleFailure(error, apiData, localeCode); diff --git a/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts b/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts index fc5701115a..4ef453fd9d 100644 --- a/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts +++ b/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts @@ -36,6 +36,7 @@ describe('ImportTaxonomies', () => { mockImportConfig = { apiKey: 'test', backupDir: testBackupDir, + contentDir: testBackupDir, context: { module: 'taxonomies' }, concurrency: 2, fetchConcurrency: 3, diff --git a/packages/contentstack-import/test/unit/import/modules/variant-entries.test.ts b/packages/contentstack-import/test/unit/import/modules/variant-entries.test.ts index d629b8f5c0..0aa562e174 100644 --- a/packages/contentstack-import/test/unit/import/modules/variant-entries.test.ts +++ b/packages/contentstack-import/test/unit/import/modules/variant-entries.test.ts @@ -76,6 +76,7 @@ describe('ImportVariantEntries', () => { beforeEach(() => { mockImportConfig = { contentDir: '/test/backup', + backupDir: '/test/backup', apiKey: 'test-api-key', context: { command: 'cm:stacks:import', @@ -528,7 +529,7 @@ describe('ImportVariantEntries', () => { it('should handle different data paths in projectMapperFilePath construction', () => { const customConfig = { ...mockImportConfig, - contentDir: '/custom/backup/path' + backupDir: '/custom/backup/path' }; const customImportVariantEntries = new ImportVariantEntries({ importConfig: customConfig From fbea55525a50d51e2e7ac2203d38909e68dc99de Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Sun, 15 Feb 2026 12:42:17 +0530 Subject: [PATCH 3/3] fix: taxonomy export issue, appended session details in log path --- .../src/commands/cm/stacks/export.ts | 10 +- .../src/export/modules/base-class.ts | 7 +- .../src/export/modules/taxonomies.ts | 193 +++++++++++++++--- .../unit/export/modules/taxonomies.test.ts | 106 +++++----- .../src/commands/cm/stacks/import.ts | 10 +- .../src/import/modules/base-class.ts | 7 +- .../src/import/modules/content-types.ts | 67 +++--- .../src/import/modules/entries.ts | 9 +- packages/contentstack-utilities/src/index.ts | 2 +- .../src/progress-summary/summary-manager.ts | 4 +- 10 files changed, 275 insertions(+), 140 deletions(-) diff --git a/packages/contentstack-export/src/commands/cm/stacks/export.ts b/packages/contentstack-export/src/commands/cm/stacks/export.ts index 49b33a4c2f..af89dec6c1 100644 --- a/packages/contentstack-export/src/commands/cm/stacks/export.ts +++ b/packages/contentstack-export/src/commands/cm/stacks/export.ts @@ -11,7 +11,7 @@ import { configHandler, log, handleAndLogError, - getLogPath, + getSessionLogPath, CLIProgressManager, clearProgressModuleSetting, } from '@contentstack/cli-utilities'; @@ -102,16 +102,17 @@ export default class ExportCommand extends Command { const managementAPIClient: ContentstackClient = await managementSDKClient(exportConfig); const moduleExporter = new ModuleExporter(managementAPIClient, exportConfig); await moduleExporter.start(); + const sessionLogPath = getSessionLogPath(); log.success( `The content of the stack ${exportConfig.apiKey} has been exported successfully!`, ); log.info(`The exported content has been stored at '${exportDir}'`, exportConfig.context); - log.success(`The log has been stored at '${getLogPath()}'`, exportConfig.context); + log.success(`The log has been stored at '${sessionLogPath}'`, exportConfig.context); // Print comprehensive summary at the end if (!exportConfig.branches) CLIProgressManager.printGlobalSummary(); if (!configHandler.get('log')?.showConsoleLogs) { - cliux.print(`The log has been stored at '${getLogPath()}'`, { color: 'green' }); + cliux.print(`The log has been stored at '${sessionLogPath}'`, { color: 'green' }); } // Clear progress module setting now that export is complete clearProgressModuleSetting(); @@ -119,9 +120,10 @@ export default class ExportCommand extends Command { // Clear progress module setting even on error clearProgressModuleSetting(); handleAndLogError(error); + const sessionLogPath = getSessionLogPath(); if (!configHandler.get('log')?.showConsoleLogs) { cliux.print(`Error: ${error}`, { color: 'red' }); - cliux.print(`The log has been stored at '${getLogPath()}'`, { color: 'green' }); + cliux.print(`The log has been stored at '${sessionLogPath}'`, { color: 'green' }); } } } diff --git a/packages/contentstack-export/src/export/modules/base-class.ts b/packages/contentstack-export/src/export/modules/base-class.ts index 32d2a4f98e..c4ea0e0760 100644 --- a/packages/contentstack-export/src/export/modules/base-class.ts +++ b/packages/contentstack-export/src/export/modules/base-class.ts @@ -5,7 +5,7 @@ import chunk from 'lodash/chunk'; import isEmpty from 'lodash/isEmpty'; import entries from 'lodash/entries'; import isEqual from 'lodash/isEqual'; -import { log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; +import { log, CLIProgressManager, configHandler, getSessionLogPath } from '@contentstack/cli-utilities'; import { ExportConfig, ModuleClassParams } from '../../types'; @@ -109,7 +109,7 @@ export default abstract class BaseClass { * - moduleName: The module name to generate the message (e.g., 'Assets', 'Entries') * If not provided, uses this.currentModuleName * - customSuccessMessage: Optional custom success message. If not provided, generates: "{moduleName} have been exported successfully!" - * - customWarningMessage: Optional custom warning message. If not provided, generates: "{moduleName} have been exported with some errors. Please check the logs for details." + * - customWarningMessage: Optional custom warning message. If not provided, generates: "{moduleName} have been exported with some errors. Please check the logs at: {sessionLogPath}" * - context: Optional context for logging */ protected completeProgressWithMessage(options?: CompleteProgressOptions): void { @@ -120,7 +120,8 @@ export default abstract class BaseClass { // Generate default messages if not provided const successMessage = options?.customSuccessMessage || `${name} have been exported successfully!`; - const warningMessage = options?.customWarningMessage || `${name} have been exported with some errors. Please check the logs for details.`; + const sessionLogPath = getSessionLogPath(); + const warningMessage = options?.customWarningMessage || `${name} have been exported with some errors. Please check the logs at: ${sessionLogPath}`; this.completeProgress(true); diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index 885161c51f..6f44a2bdcf 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -40,7 +40,8 @@ export default class ExportTaxonomies extends BaseClass { this.qs = { include_count: true, limit: this.taxonomiesConfig.limit || 100, skip: 0 }; this.applyQueryFilters(this.qs, 'taxonomies'); - this.exportConfig.context.module = 'taxonomies'; + this.exportConfig.context.module = MODULE_CONTEXTS.TAXONOMIES; + this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.TAXONOMIES]; this.localesFilePath = pResolve( sanitizePath(exportConfig.exportDir), sanitizePath(exportConfig.branchName || ''), @@ -50,56 +51,149 @@ export default class ExportTaxonomies extends BaseClass { } async start(): Promise { - log.debug('Starting export process for taxonomies...', this.exportConfig.context); - - //create taxonomies folder - this.taxonomiesFolderPath = pResolve( - this.exportConfig.exportDir, - this.exportConfig.branchName || '', - this.taxonomiesConfig.dirName, - ); - log.debug(`Taxonomies folder path: '${this.taxonomiesFolderPath}'`, this.exportConfig.context); - - await fsUtil.makeDirectory(this.taxonomiesFolderPath); - log.debug('Created taxonomies directory.', this.exportConfig.context); + try { + log.debug('Starting export process for taxonomies...', this.exportConfig.context); - const localesToExport = this.getLocalesToExport(); - log.debug( - `Will attempt to export taxonomies for ${localesToExport.length} locale(s): ${localesToExport.join(', ')}`, - this.exportConfig.context, - ); + const totalCount = await this.initializeExport(); + if (totalCount === 0) { + log.info(messageHandler.parse('TAXONOMY_NOT_FOUND'), this.exportConfig.context); + return; + } - if (localesToExport.length === 0) { - log.warn('No locales found to export', this.exportConfig.context); - return; + const progress = this.setupProgress(totalCount); + const localesToExport = this.getLocalesToExport(); + + if (localesToExport.length === 0) { + log.warn('No locales found to export', this.exportConfig.context); + this.completeProgress(true); + return; + } + + // Start fetch process + progress + .startProcess(PROCESS_NAMES.FETCH_TAXONOMIES) + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.FETCH_TAXONOMIES].FETCHING, PROCESS_NAMES.FETCH_TAXONOMIES); + + // Determine export strategy and fetch taxonomies + await this.determineExportStrategy(this.exportConfig.master_locale?.code); + await this.fetchAllTaxonomies(localesToExport); + progress.completeProcess(PROCESS_NAMES.FETCH_TAXONOMIES, true); + + // Export taxonomies with detailed information + const actualCount = await this.exportAllTaxonomies(progress, localesToExport, totalCount); + + // Write metadata and complete + await this.writeTaxonomiesMetadata(); + log.success(messageHandler.parse('TAXONOMY_EXPORT_COMPLETE', actualCount), this.exportConfig.context); + this.completeProgress(true); + } catch (error) { + handleAndLogError(error, { ...this.exportConfig.context }); + this.completeProgress(false, error?.message || 'Taxonomies export failed'); } + } - // Test locale-based export support with master locale - const masterLocale = this.exportConfig.master_locale?.code; - await this.fetchTaxonomies(masterLocale, true); + /** + * Initialize export setup (create directories, get initial count) + */ + private async initializeExport(): Promise { + return this.withLoadingSpinner('TAXONOMIES: Analyzing taxonomy structure...', async () => { + this.taxonomiesFolderPath = pResolve( + this.exportConfig.exportDir, + this.exportConfig.branchName || '', + this.taxonomiesConfig.dirName, + ); + log.debug(`Taxonomies folder path: '${this.taxonomiesFolderPath}'`, this.exportConfig.context); + + await fsUtil.makeDirectory(this.taxonomiesFolderPath); + log.debug('Created taxonomies directory.', this.exportConfig.context); + + // Get count first for progress tracking + const countResponse = await this.stack + .taxonomy() + .query({ ...this.qs, include_count: true, limit: 1 }) + .find(); + return countResponse.count || 0; + }); + } + + /** + * Setup progress manager with processes + */ + private setupProgress(totalCount: number): any { + const progress = this.createNestedProgress(this.currentModuleName); + // For fetch: count API calls, not individual taxonomies + const fetchApiCallsCount = Math.ceil(totalCount / (this.qs.limit || 100)); + progress.addProcess(PROCESS_NAMES.FETCH_TAXONOMIES, fetchApiCallsCount); + progress.addProcess(PROCESS_NAMES.EXPORT_TAXONOMIES_TERMS, totalCount); + return progress; + } + /** + * Determine if locale-based export is supported + */ + private async determineExportStrategy(masterLocale?: string): Promise { + await this.fetchTaxonomies(masterLocale, true); if (!this.isLocaleBasedExportSupported) { + log.debug('Falling back to legacy export (non-localized)', this.exportConfig.context); this.taxonomies = {}; this.taxonomiesByLocale = {}; - - // Fetch taxonomies without locale parameter - await this.fetchTaxonomies(); - await this.exportTaxonomies(); - await this.writeTaxonomiesMetadata(); } else { - // Process all locales with locale-based export log.debug('Localization enabled, proceeding with locale-based export', this.exportConfig.context); + } + } + /** + * Fetch all taxonomies based on export strategy + */ + private async fetchAllTaxonomies(localesToExport: string[]): Promise { + if (!this.isLocaleBasedExportSupported) { + await this.fetchTaxonomies(); + } else { for (const localeCode of localesToExport) { await this.fetchTaxonomies(localeCode); - await this.processLocaleExport(localeCode); } + } + } - await this.writeTaxonomiesMetadata(); + /** + * Export all taxonomies with detailed information + */ + private async exportAllTaxonomies(progress: any, localesToExport: string[], totalCount: number): Promise { + const actualCount = Object.keys(this.taxonomies || {})?.length; + log.debug( + `Found ${actualCount} taxonomies to export (API reported ${totalCount})`, + this.exportConfig.context, + ); + + if (actualCount === 0) { + log.info('No taxonomies found to export detailed information', this.exportConfig.context); + return 0; + } + + // Update progress total if needed + if (actualCount !== totalCount) { + progress.updateProcessTotal(PROCESS_NAMES.EXPORT_TAXONOMIES_TERMS, actualCount); } - this.completeProgressWithMessage(); + // Start export process + progress + .startProcess(PROCESS_NAMES.EXPORT_TAXONOMIES_TERMS) + .updateStatus( + PROCESS_STATUS[PROCESS_NAMES.EXPORT_TAXONOMIES_TERMS].EXPORTING, + PROCESS_NAMES.EXPORT_TAXONOMIES_TERMS, + ); + // Export based on strategy + if (!this.isLocaleBasedExportSupported) { + await this.exportTaxonomies(); + } else { + for (const localeCode of localesToExport) { + await this.processLocaleExport(localeCode); + } + } + + progress.completeProcess(PROCESS_NAMES.EXPORT_TAXONOMIES_TERMS, true); + return actualCount; } /** @@ -176,6 +270,17 @@ export default class ExportTaxonomies extends BaseClass { } this.sanitizeTaxonomiesAttribs(items, localeCode); + + // Track progress per API call (only for actual fetch, not locale support check) + if (!checkLocaleSupport) { + this.progressManager?.tick( + true, + `fetched ${items.length} taxonomies${localeInfo}`, + null, + PROCESS_NAMES.FETCH_TAXONOMIES, + ); + } + skip += this.qs.limit || 100; if (skip >= taxonomiesCount) { @@ -261,14 +366,34 @@ export default class ExportTaxonomies extends BaseClass { } const onSuccess = ({ response, uid }: any) => { + const taxonomyName = this.taxonomies[uid]?.name; const filePath = pResolve(exportFolderPath, `${uid}.json`); log.debug(`Writing detailed taxonomy data to: ${filePath}`, this.exportConfig.context); fsUtil.writeFile(filePath, response); - log.success(messageHandler.parse('TAXONOMY_EXPORT_SUCCESS', uid), this.exportConfig.context); + + // Track progress for each exported taxonomy + this.progressManager?.tick( + true, + `taxonomy: ${taxonomyName || uid}`, + null, + PROCESS_NAMES.EXPORT_TAXONOMIES_TERMS, + ); + + log.success(messageHandler.parse('TAXONOMY_EXPORT_SUCCESS', taxonomyName || uid), this.exportConfig.context); }; const onReject = ({ error, uid }: any) => { + const taxonomyName = this.taxonomies[uid]?.name; log.debug(`Failed to export detailed data for taxonomy: ${uid}${localeInfo}`, this.exportConfig.context); + + // Track failure + this.progressManager?.tick( + false, + `taxonomy: ${taxonomyName || uid}`, + error?.message || PROCESS_STATUS[PROCESS_NAMES.EXPORT_TAXONOMIES_TERMS].FAILED, + PROCESS_NAMES.EXPORT_TAXONOMIES_TERMS, + ); + handleAndLogError(error, { ...this.exportConfig.context, uid, ...(localeCode && { locale: localeCode }) }); }; diff --git a/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts b/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts index a9d2764b92..71e7dd8bac 100644 --- a/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts @@ -86,6 +86,21 @@ describe('ExportTaxonomies', () => { sinon.stub(FsUtility.prototype, 'writeFile').resolves(); sinon.stub(FsUtility.prototype, 'makeDirectory').resolves(); sinon.stub(FsUtility.prototype, 'readFile').resolves({}); + + // Stub progress manager methods + const mockProgress = { + addProcess: sinon.stub().returnsThis(), + startProcess: sinon.stub().returnsThis(), + updateStatus: sinon.stub().returnsThis(), + completeProcess: sinon.stub().returnsThis(), + updateProcessTotal: sinon.stub().returnsThis(), + }; + sinon.stub(exportTaxonomies, 'withLoadingSpinner').callsFake(async (msg: string, fn: () => Promise) => { + return await fn(); + }); + sinon.stub(exportTaxonomies, 'createNestedProgress').returns(mockProgress); + sinon.stub(exportTaxonomies, 'completeProgress').resolves(); + sinon.stub(exportTaxonomies, 'completeProgressWithMessage').resolves(); }); afterEach(() => { @@ -584,87 +599,82 @@ describe('ExportTaxonomies', () => { describe('start() method - locale-based export scenarios', () => { it('should use legacy export when locale-based export is not supported', async () => { - const mockFetchTaxonomies = sinon - .stub(exportTaxonomies, 'fetchTaxonomies') - .callsFake(async (locale, checkSupport) => { - if (checkSupport) { - exportTaxonomies.isLocaleBasedExportSupported = false; - } + const mockDetermineStrategy = sinon + .stub(exportTaxonomies as any, 'determineExportStrategy') + .callsFake(async () => { + exportTaxonomies.isLocaleBasedExportSupported = false; }); - const mockExportTaxonomies = sinon.stub(exportTaxonomies, 'exportTaxonomies').resolves(); + const mockFetchAll = sinon.stub(exportTaxonomies as any, 'fetchAllTaxonomies').resolves(); + const mockExportAll = sinon.stub(exportTaxonomies as any, 'exportAllTaxonomies').resolves(2); const mockWriteMetadata = sinon.stub(exportTaxonomies, 'writeTaxonomiesMetadata').resolves(); const mockGetLocales = sinon.stub(exportTaxonomies, 'getLocalesToExport').returns(['en-us']); await exportTaxonomies.start(); - // Should use legacy export (no locale parameter) - expect(mockExportTaxonomies.called).to.be.true; - expect(mockExportTaxonomies.calledWith()).to.be.true; // Called without locale + // Should call the helper methods + expect(mockDetermineStrategy.called).to.be.true; + expect(mockFetchAll.called).to.be.true; + expect(mockExportAll.called).to.be.true; expect(mockWriteMetadata.called).to.be.true; - mockFetchTaxonomies.restore(); - mockExportTaxonomies.restore(); + mockDetermineStrategy.restore(); + mockFetchAll.restore(); + mockExportAll.restore(); mockWriteMetadata.restore(); mockGetLocales.restore(); }); it('should clear taxonomies and re-fetch when falling back to legacy export', async () => { - let fetchCallCount = 0; - const mockFetchTaxonomies = sinon - .stub(exportTaxonomies, 'fetchTaxonomies') - .callsFake(async (locale, checkSupport) => { - fetchCallCount++; - if (checkSupport) { - // First call fails locale check - exportTaxonomies.isLocaleBasedExportSupported = false; - exportTaxonomies.taxonomies = { 'partial-data': { uid: 'partial-data' } }; // Simulate partial data - } else { - // Second call should have cleared data - expect(exportTaxonomies.taxonomies).to.deep.equal({}); - } + const mockDetermineStrategy = sinon + .stub(exportTaxonomies as any, 'determineExportStrategy') + .callsFake(async () => { + // Simulate fallback to legacy - clears data + exportTaxonomies.isLocaleBasedExportSupported = false; + exportTaxonomies.taxonomies = {}; + exportTaxonomies.taxonomiesByLocale = {}; }); - const mockExportTaxonomies = sinon.stub(exportTaxonomies, 'exportTaxonomies').resolves(); + const mockFetchAll = sinon.stub(exportTaxonomies as any, 'fetchAllTaxonomies').resolves(); + const mockExportAll = sinon.stub(exportTaxonomies as any, 'exportAllTaxonomies').resolves(2); const mockWriteMetadata = sinon.stub(exportTaxonomies, 'writeTaxonomiesMetadata').resolves(); const mockGetLocales = sinon.stub(exportTaxonomies, 'getLocalesToExport').returns(['en-us']); await exportTaxonomies.start(); - // Should call fetchTaxonomies twice: once for check, once for legacy - expect(fetchCallCount).to.equal(2); - // First call with locale, second without - expect(mockFetchTaxonomies.firstCall.args).to.deep.equal(['en-us', true]); - expect(mockFetchTaxonomies.secondCall.args).to.deep.equal([]); + // Should clear taxonomies and re-fetch + expect(mockDetermineStrategy.called).to.be.true; + expect(mockFetchAll.called).to.be.true; - mockFetchTaxonomies.restore(); - mockExportTaxonomies.restore(); + mockDetermineStrategy.restore(); + mockFetchAll.restore(); + mockExportAll.restore(); mockWriteMetadata.restore(); mockGetLocales.restore(); }); it('should use locale-based export when supported', async () => { - const mockFetchTaxonomies = sinon - .stub(exportTaxonomies, 'fetchTaxonomies') - .callsFake(async (locale, checkSupport) => { - if (checkSupport) { - exportTaxonomies.isLocaleBasedExportSupported = true; - } - if (locale && typeof locale === 'string' && !exportTaxonomies.taxonomiesByLocale[locale]) { - exportTaxonomies.taxonomiesByLocale[locale] = new Set(['taxonomy-1']); - } + const mockDetermineStrategy = sinon + .stub(exportTaxonomies as any, 'determineExportStrategy') + .callsFake(async () => { + exportTaxonomies.isLocaleBasedExportSupported = true; + exportTaxonomies.taxonomiesByLocale['en-us'] = new Set(['taxonomy-1']); + exportTaxonomies.taxonomiesByLocale['es-es'] = new Set(['taxonomy-2']); }); - const mockProcessLocale = sinon.stub(exportTaxonomies, 'processLocaleExport').resolves(); + const mockFetchAll = sinon.stub(exportTaxonomies as any, 'fetchAllTaxonomies').resolves(); + const mockExportAll = sinon.stub(exportTaxonomies as any, 'exportAllTaxonomies').resolves(2); const mockWriteMetadata = sinon.stub(exportTaxonomies, 'writeTaxonomiesMetadata').resolves(); const mockGetLocales = sinon.stub(exportTaxonomies, 'getLocalesToExport').returns(['en-us', 'es-es']); await exportTaxonomies.start(); - // Should process each locale - expect(mockProcessLocale.called).to.be.true; - expect(mockProcessLocale.callCount).to.equal(2); // Two locales + // Should call helper methods + expect(mockDetermineStrategy.called).to.be.true; + expect(mockFetchAll.called).to.be.true; + expect(mockExportAll.called).to.be.true; expect(mockWriteMetadata.called).to.be.true; - mockFetchTaxonomies.restore(); - mockProcessLocale.restore(); + mockDetermineStrategy.restore(); + mockFetchAll.restore(); + mockExportAll.restore(); mockWriteMetadata.restore(); mockGetLocales.restore(); }); diff --git a/packages/contentstack-import/src/commands/cm/stacks/import.ts b/packages/contentstack-import/src/commands/cm/stacks/import.ts index 00bbf07e47..9a086e7801 100644 --- a/packages/contentstack-import/src/commands/cm/stacks/import.ts +++ b/packages/contentstack-import/src/commands/cm/stacks/import.ts @@ -8,7 +8,7 @@ import { log, handleAndLogError, configHandler, - getLogPath, + getSessionLogPath, CLIProgressManager, cliux, clearProgressModuleSetting, @@ -170,8 +170,8 @@ export default class ImportCommand extends Command { private logAndPrintErrorDetails(error: unknown, importConfig: any) { cliux.print('\n'); - const logPath = getLogPath(); - const logMsg = `The log has been stored at '${logPath}'`; + const sessionLogPath = getSessionLogPath(); + const logMsg = `The log has been stored at '${sessionLogPath}'`; const backupDir = importConfig?.backupDir; const backupDirMsg = backupDir @@ -191,8 +191,8 @@ export default class ImportCommand extends Command { private logSuccessAndBackupMessages(backupDir: string, importConfig: any) { cliux.print('\n'); - const logPath = getLogPath(); - const logMsg = `The log has been stored at '${logPath}'`; + const sessionLogPath = getSessionLogPath(); + const logMsg = `The log has been stored at '${sessionLogPath}'`; const backupDirMsg = `The backup content has been stored at '${backupDir}'`; log.success(logMsg, importConfig.context); diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index c22d1e1661..d69628e19e 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -16,7 +16,7 @@ import { LabelData } from '@contentstack/management/types/stack/label'; import { WebhookData } from '@contentstack/management/types/stack/webhook'; import { WorkflowData } from '@contentstack/management/types/stack/workflow'; import { RoleData } from '@contentstack/management/types/stack/role'; -import { log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; +import { log, CLIProgressManager, configHandler, getSessionLogPath } from '@contentstack/cli-utilities'; import { ImportConfig, ModuleClassParams } from '../../types'; import cloneDeep from 'lodash/cloneDeep'; @@ -152,7 +152,7 @@ export default abstract class BaseClass { * - moduleName: The module name to generate the message (e.g., 'Content types', 'Entries') * If not provided, uses this.currentModuleName * - customSuccessMessage: Optional custom success message. If not provided, generates: "{moduleName} have been imported successfully!" - * - customWarningMessage: Optional custom warning message. If not provided, generates: "{moduleName} have been imported with some errors. Please check the logs for details." + * - customWarningMessage: Optional custom warning message. If not provided, generates: "{moduleName} have been imported with some errors. Please check the logs at: {sessionLogPath}" * - context: Optional context for logging */ protected completeProgressWithMessage(options?: CompleteProgressOptions): void { @@ -163,7 +163,8 @@ export default abstract class BaseClass { // Generate default messages if not provided const successMessage = options?.customSuccessMessage || `${name} have been imported successfully!`; - const warningMessage = options?.customWarningMessage || `${name} have been imported with some errors. Please check the logs for details.`; + const sessionLogPath = getSessionLogPath(); + const warningMessage = options?.customWarningMessage || `${name} have been imported with some errors. Please check the logs at: ${sessionLogPath}`; this.completeProgress(true); diff --git a/packages/contentstack-import/src/import/modules/content-types.ts b/packages/contentstack-import/src/import/modules/content-types.ts index e187e145fc..4c2fc7b30d 100644 --- a/packages/contentstack-import/src/import/modules/content-types.ts +++ b/packages/contentstack-import/src/import/modules/content-types.ts @@ -1,4 +1,3 @@ - /* eslint-disable no-prototype-builtins */ /*! * Contentstack Import @@ -111,18 +110,15 @@ export default class ContentTypesImport extends BaseClass { // Initialize composable studio paths if config exists if (this.importConfig.modules['composable-studio']) { - // Use contentDir as fallback if data is not available - const basePath = this.importConfig.contentDir; - this.composableStudioSuccessPath = path.join( - sanitizePath(basePath), + sanitizePath(importConfig.backupDir), PATH_CONSTANTS.MAPPER, this.importConfig.modules['composable-studio'].dirName, this.importConfig.modules['composable-studio'].fileName, ); this.composableStudioExportPath = path.join( - sanitizePath(basePath), + sanitizePath(importConfig.backupDir), this.importConfig.modules['composable-studio'].dirName, this.importConfig.modules['composable-studio'].fileName, ); @@ -162,40 +158,43 @@ export default class ContentTypesImport extends BaseClass { log.info('No content type found to import', this.importConfig.context); return; } - // If success file doesn't exist but export file does, skip the composition content type - // Only check if composable studio paths are configured - if ( - this.composableStudioSuccessPath && - this.composableStudioExportPath && - !fileHelper.fileExistsSync(this.composableStudioSuccessPath) && - fileHelper.fileExistsSync(this.composableStudioExportPath) - ) { - const exportedProject = fileHelper.readFileSync(this.composableStudioExportPath) as { - contentTypeUid: string; - }; - - if (exportedProject?.contentTypeUid) { - const originalCount = this.cTs.length; - this.cTs = this.cTs.filter((ct: Record) => { - const shouldSkip = ct.uid === exportedProject.contentTypeUid; - if (shouldSkip) { - log.info( - `Skipping content type '${ct.uid}' as Composable Studio project was not created successfully`, + // If success file doesn't exist but export file does, skip the composition content type + // Only check if composable studio paths are configured + if ( + this.composableStudioSuccessPath && + this.composableStudioExportPath && + !fileHelper.fileExistsSync(this.composableStudioSuccessPath) && + fileHelper.fileExistsSync(this.composableStudioExportPath) + ) { + const exportedProject = fileHelper.readFileSync(this.composableStudioExportPath) as { + contentTypeUid: string; + }; + + if (exportedProject?.contentTypeUid) { + const originalCount = this.cTs.length; + this.cTs = this.cTs.filter((ct: Record) => { + const shouldSkip = ct.uid === exportedProject.contentTypeUid; + if (shouldSkip) { + log.info( + `Skipping content type '${ct.uid}' as Composable Studio project was not created successfully`, + this.importConfig.context, + ); + } + return !shouldSkip; + }); + + const skippedCount = originalCount - this.cTs.length; + if (skippedCount > 0) { + log.debug( + `Filtered out ${skippedCount} composition content type(s) from import`, this.importConfig.context, ); } - return !shouldSkip; - }); - - const skippedCount = originalCount - this.cTs.length; - if (skippedCount > 0) { - log.debug(`Filtered out ${skippedCount} composition content type(s) from import`, this.importConfig.context); } } - } - await fsUtil.makeDirectory(this.cTsMapperPath); - log.debug('Created content types mapper directory.', this.importConfig.context); + await fsUtil.makeDirectory(this.cTsMapperPath); + log.debug('Created content types mapper directory.', this.importConfig.context); await fsUtil.makeDirectory(this.cTsMapperPath); log.debug('Created content types mapper directory', this.importConfig.context); diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 2964286dae..fc823dd977 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -120,19 +120,16 @@ export default class EntriesImport extends BaseClass { ); // Initialize composable studio paths if config exists - if (this.importConfig.modules['composable-studio']) { - // Use contentDir as fallback if data is not available - const basePath = this.importConfig.contentDir; - + if (this.importConfig.modules['composable-studio']) { this.composableStudioSuccessPath = path.join( - sanitizePath(basePath), + sanitizePath(importConfig.backupDir), PATH_CONSTANTS.MAPPER, this.importConfig.modules['composable-studio'].dirName, this.importConfig.modules['composable-studio'].fileName, ); this.composableStudioExportPath = path.join( - sanitizePath(basePath), + sanitizePath(importConfig.backupDir), this.importConfig.modules['composable-studio'].dirName, this.importConfig.modules['composable-studio'].fileName, ); diff --git a/packages/contentstack-utilities/src/index.ts b/packages/contentstack-utilities/src/index.ts index fce08d2e42..3465b529e6 100644 --- a/packages/contentstack-utilities/src/index.ts +++ b/packages/contentstack-utilities/src/index.ts @@ -77,7 +77,7 @@ export { default as TablePrompt } from './inquirer-table-prompt'; export { Logger }; export { default as authenticationHandler } from './authentication-handler'; -export { v2Logger as log, cliErrorHandler, handleAndLogError, getLogPath } from './logger/log'; +export { v2Logger as log, cliErrorHandler, handleAndLogError, getLogPath, getSessionLogPath } from './logger/log'; export { CLIProgressManager, SummaryManager, diff --git a/packages/contentstack-utilities/src/progress-summary/summary-manager.ts b/packages/contentstack-utilities/src/progress-summary/summary-manager.ts index a8073d5d89..9153968b2b 100644 --- a/packages/contentstack-utilities/src/progress-summary/summary-manager.ts +++ b/packages/contentstack-utilities/src/progress-summary/summary-manager.ts @@ -1,6 +1,6 @@ import chalk from 'chalk'; import { ModuleResult, SummaryOptions } from '../interfaces/index'; -import { getLogPath } from '../logger/log'; +import { getSessionLogPath } from '../logger/log'; export default class SummaryManager { private modules: Map = new Map(); @@ -180,7 +180,7 @@ export default class SummaryManager { }); console.log(chalk.blue('\nšŸ“‹ For detailed error information, check the log files:')); - //console.log(chalk.blue(` ${getLogPath()}`)); + //console.log(chalk.blue(` ${getSessionLogPath()}`)); console.log(chalk.gray(' Recent errors are logged with full context and stack traces.')); }