diff --git a/api/src/models/EntryMapper.ts b/api/src/models/EntryMapper.ts index 7ba3eeb6d..b667f4c90 100644 --- a/api/src/models/EntryMapper.ts +++ b/api/src/models/EntryMapper.ts @@ -8,7 +8,7 @@ import { DATABASE_FILES } from "../constants/index.js"; * Represents an entry mapper object. */ export interface EntryMapper { - entry_mapper: { + entry_mapper: { id: string; projectId: string; contentTypeId: string; @@ -18,6 +18,7 @@ export interface EntryMapper { isUpdate: boolean; contentstackEntryUid: string; isDuplicateEntry: boolean; + language?: string; }[]; } diff --git a/api/src/models/project-lowdb.ts b/api/src/models/project-lowdb.ts index 008efe4e2..0e224f1ba 100644 --- a/api/src/models/project-lowdb.ts +++ b/api/src/models/project-lowdb.ts @@ -72,6 +72,10 @@ interface Project { taxonomies?: any[]; isSSO: boolean; iteration: number; + master_locale?: Record; + locales?: Record; + source_locales?: string[]; + migrated_locales?: string[]; } interface ProjectDocument { diff --git a/api/src/models/uidMapper.ts b/api/src/models/uidMapper.ts index 090de6472..437620351 100644 --- a/api/src/models/uidMapper.ts +++ b/api/src/models/uidMapper.ts @@ -10,9 +10,15 @@ import { DATABASE_FILES } from "../constants/index.js"; interface EntryMapper { entry: Record; assets: Record; + /** + * Per-locale entry uid mapping. Source uid → destination uid, scoped by destination + * locale code. Populated by writePerLocaleEntryUidMapping after each CLI import so the + * delta flow can tell which entries actually have a variant in a given locale. + */ + entryByLocale?: Record>; } -const defaultData: EntryMapper = { entry: {}, assets: {} }; +const defaultData: EntryMapper = { entry: {}, assets: {}, entryByLocale: {} }; /** * Creates and returns a database instance for the field mapper for a specific project. diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index 3ecdf71d0..d16ecd31e 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -31,6 +31,7 @@ import getEntryMapperDb, { EntryMapper } from "../models/EntryMapper.js"; import getContentTypesMapperDb, { ContentTypesMapper } from "../models/contentTypesMapper-lowdb.js"; import getUidMapperDb from "../models/uidMapper.js"; import { isDuplicateEntry } from '../utils/entry-duplicate.utils.js'; +import { getSourceLocaleForDestination } from '../utils/locale-migration.utils.js'; const idCorrector = ({ id }: { id: string }) => { @@ -1952,6 +1953,8 @@ const updateEntryStatus = async (req: Request) => { const EntryMapperModel = getEntryMapperDb(projectId, iteration); await EntryMapperModel.read(); const foundEntry: EntryMapper[] = []; + // Rows in entry_mapper are already per-(entry × source-locale), so each id uniquely + // identifies one locale variant; toggling isUpdate directly is correct. await EntryMapperModel.update((data: any) => { data?.entry_mapper?.forEach((entry: any) => { if (validatedUids.includes(entry?.id)) { @@ -2000,6 +2003,8 @@ const getEntryMapping = async (req: Request) => { const skip: any = req?.params?.skip; const limit: any = req?.params?.limit; const search: string = req?.params?.searchText?.toLowerCase(); + const locale: string | undefined = + (req?.query?.locale as string) || (req?.params as any)?.locale; let result: any[] = []; let filteredResult = []; @@ -2033,6 +2038,11 @@ const getEntryMapping = async (req: Request) => { } const EntryMapperModel = getEntryMapperDb(projectId, iteration); await EntryMapperModel.read(); + // Convert the destination locale param (e.g. "en-in") to its source locale code + // (e.g. "en-IN") so we can filter the per-source-locale entry_mapper rows. + const sourceLocale = locale + ? getSourceLocaleForDestination(projectData ?? {}, locale) + : null; let entryMapping = contentType?.entryMapping?.map?.((mapperUId: any) => { const entryMapper = EntryMapperModel.chain .get("entry_mapper") @@ -2062,16 +2072,25 @@ const getEntryMapping = async (req: Request) => { entryMapping ?? [], ); - if (!isEmpty(enrichedMapping)) { + // entry_mapper rows are already per-source-locale (one row per language variant). + // Filter to just the rows whose `language` matches the selected destination locale's + // source code. Falls open when no locale is provided so legacy callers still work. + const localeFiltered = sourceLocale + ? (enrichedMapping ?? []).filter( + (row: any) => row && (row?.language ?? '') === sourceLocale, + ) + : enrichedMapping; + + if (!isEmpty(localeFiltered)) { if (search) { - filteredResult = enrichedMapping?.filter?.((item: any) => + filteredResult = localeFiltered?.filter?.((item: any) => item?.entryName?.toLowerCase().includes(search) ); totalCount = filteredResult?.length; result = filteredResult?.slice(skip, Number(skip) + Number(limit)); } else { - totalCount = enrichedMapping?.length; - result = enrichedMapping?.slice(skip, Number(skip) + Number(limit)); + totalCount = localeFiltered?.length; + result = localeFiltered?.slice(skip, Number(skip) + Number(limit)); } } return { diff --git a/api/src/services/runCli.service.ts b/api/src/services/runCli.service.ts index c4d09e0a1..8f90b01a2 100644 --- a/api/src/services/runCli.service.ts +++ b/api/src/services/runCli.service.ts @@ -19,7 +19,7 @@ interface TestStack { isMigrated: boolean; } import { setBasicAuthConfig, setOAuthConfig } from '../utils/config-handler.util.js'; -import writeUidMapping from '../utils/uid-mapper.utils.js'; +import writeUidMapping, { writePerLocaleEntryUidMapping } from '../utils/uid-mapper.utils.js'; /** * Determines log level based on message content without removing ANSI codes @@ -277,6 +277,7 @@ export const runCli = async ( .value(); const iteration = projectData?.iteration || 1; await writeUidMapping(backupPath, projectId, iteration); + await writePerLocaleEntryUidMapping(backupPath, projectId, iteration); } // Keep the project status update code: @@ -321,6 +322,19 @@ export const runCli = async ( ProjectModelLowdb.data.projects[projectIndex].current_step = getStepperSteps(ProjectModelLowdb.data.projects[projectIndex]?.iteration).MIGRATION; ProjectModelLowdb.data.projects[projectIndex].status = 5; + // Record every locale that just successfully migrated so the next delta restart can + // tell which locales need a full pass vs delta. Set-union with prior value. + const proj: any = ProjectModelLowdb.data.projects[projectIndex]; + const ranLocales = Array.from( + new Set([ + ...Object.keys(proj?.master_locale ?? {}), + ...Object.keys(proj?.locales ?? {}), + ]), + ); + const existing: string[] = Array.isArray(proj?.migrated_locales) + ? proj.migrated_locales + : []; + proj.migrated_locales = Array.from(new Set([...existing, ...ranLocales])); await ProjectModelLowdb.write(); } } else { diff --git a/api/src/services/wordpress.service.ts b/api/src/services/wordpress.service.ts index ee090dcdb..33a52615d 100644 --- a/api/src/services/wordpress.service.ts +++ b/api/src/services/wordpress.service.ts @@ -1308,6 +1308,39 @@ async function saveEntry(fields: any, entry: any, file_path: string, assetData } return entryData; } +/** + * Replicate the master-locale entries JSON + index into each additional destination locale + * folder so the CLI import creates entry variants under every mapped locale, not just master. + * WP's WXR format doesn't carry per-locale content, so each locale gets the same payload — + * Contentstack's fallback chain handles the read-side behavior and the user can edit the + * variants later. + */ +async function fanOutEntriesToAdditionalLocales( + postFolderPath: string, + masterLocaleCode: string, + contentTypeRoot: string, + project: any, +): Promise { + const additional = project?.locales ?? {}; + const masterFilePath = path.join(postFolderPath, `${masterLocaleCode}.json`); + if (!existsSync(masterFilePath)) return; + const masterContent = await fs.promises.readFile(masterFilePath, 'utf-8'); + for (const destLocale of Object.keys(additional)) { + if (!destLocale || destLocale === masterLocaleCode) continue; + const localeFolderPath = path.join(contentTypeRoot, destLocale); + if (!existsSync(localeFolderPath)) { + await fs.promises.mkdir(localeFolderPath, { recursive: true }); + } + const localeFilePath = path.join(localeFolderPath, `${destLocale}.json`); + await fs.promises.writeFile(localeFilePath, masterContent, 'utf-8'); + await fs.promises.writeFile( + path.join(localeFolderPath, 'index.json'), + JSON.stringify({ '1': `${destLocale}.json` }, null, 4), + 'utf-8', + ); + } +} + async function createEntry(file_path: string, packagePath: string, destinationStackId: string, projectId: string, contentTypes: any, mapperKeys: any, master_locale: string, project: any){ const locale = getLocale(master_locale, project) || master_locale; const Jsondata = await fs.promises.readFile(packagePath, "utf8"); @@ -1355,6 +1388,12 @@ async function createEntry(file_path: string, packagePath: string, destinationSt await fs.promises.writeFile(path.join(postFolderPath, "index.json"), JSON.stringify({ "1": `${locale}.json` }, null, 4), "utf-8" ); + await fanOutEntriesToAdditionalLocales( + postFolderPath, + locale, + path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, postsFolderName), + project, + ); } const termsContentTypes = contentTypes?.filter((contentType: any) => contentType?.contentstackUid === 'terms'); @@ -1376,6 +1415,12 @@ async function createEntry(file_path: string, packagePath: string, destinationSt await fs.promises.writeFile(path.join(termsFolderPath, "index.json"), JSON.stringify({ "1": `${locale}.json` }, null, 4), "utf-8" ); + await fanOutEntriesToAdditionalLocales( + termsFolderPath, + locale, + path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, termsFolderName), + project, + ); } const postContentTypes = contentTypes?.filter( (contentType: any) => @@ -1412,6 +1457,12 @@ async function createEntry(file_path: string, packagePath: string, destinationSt await fs.promises.writeFile(path.join(postFolderPath, "index.json"), JSON.stringify({ "1": `${locale}.json` }, null, 4), "utf-8" ); + await fanOutEntriesToAdditionalLocales( + postFolderPath, + locale, + path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, postsFolderName), + project, + ); console.info(`Processed content for ${contentType?.contentstackTitle}:`, Object?.keys(content)?.length, "items"); } } diff --git a/api/src/utils/entry-update-script.cjs b/api/src/utils/entry-update-script.cjs index fa4d36e3d..33d23456b 100644 --- a/api/src/utils/entry-update-script.cjs +++ b/api/src/utils/entry-update-script.cjs @@ -18,6 +18,8 @@ const FLAT_PAYLOAD_SKIP = new Set([ 'updated_by', '_content_type_uid', 'content', + '__locale', + '__csUid', ]); /** @@ -66,7 +68,7 @@ const resolveAssetField = (fieldName, entryUid, updateValue, stackValue, oldMapp * WordPress (and similar) write migration JSON with fields at the root (email, url, …). * Fetched stack entries keep custom fields under entry.content — merge flat updateData there. */ -const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping, newMapping) => { +const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping, newMapping, updateOpts) => { for (const field of Object.keys(updateData)) { if (FLAT_PAYLOAD_SKIP.has(field)) { continue; @@ -90,7 +92,7 @@ const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping } entry.content[field] = nextVal; } - await entry.update(); + await entry.update(updateOpts); }; module.exports = async ({ @@ -120,13 +122,22 @@ module.exports = async ({ console.info(`Processing content type: ${contentType}, entries: ${entryUids.length}`); for (const entryUid of entryUids) { + const updateData = JSON.parse(JSON.stringify(config[contentType][entryUid])); + // Per-locale config keys are "::" with __locale/__csUid + // on the payload. Fall back to the bare key for legacy single-locale + // configs. + const locale = updateData?.__locale; + const realEntryUid = updateData?.__csUid || entryUid; + delete updateData?.__locale; + delete updateData?.__csUid; + const fetchOpts = locale ? { locale } : undefined; + const updateOpts = locale ? { locale } : undefined; + const entryRef = stackSDKInstance .contentType(contentType) - .entry(entryUid); + .entry(realEntryUid); - - const entry = await entryRef?.fetch(); - const updateData = JSON.parse(JSON.stringify(config[contentType][entryUid])); + const entry = await entryRef?.fetch(fetchOpts); const hasStackContent = entry?.content && typeof entry?.content === 'object'; const hasNestedUpdate = updateData?.content && typeof updateData?.content === 'object'; @@ -145,10 +156,10 @@ module.exports = async ({ } } Object.assign(entry?.content, updateData?.content); - await entry.update(); + await entry.update(updateOpts); } else if (hasStackContent) { - console.info(`[${entryUid}] Merging flat migration payload into entry.content (e.g. WordPress export)`); - await mergeFlatPayloadIntoEntry(entry, entryUid, updateData, oldMapping, newMapping); + console.info(`[${realEntryUid}] Merging flat migration payload into entry.content (e.g. WordPress export)${locale ? ` for locale "${locale}"` : ''}`); + await mergeFlatPayloadIntoEntry(entry, realEntryUid, updateData, oldMapping, newMapping, updateOpts); } else { if (updateData && entry) { for (const field of Object.keys(updateData)) { @@ -165,9 +176,9 @@ module.exports = async ({ } } Object.assign(entry, updateData); - await entry.update(); + await entry.update(updateOpts); } - console.info(`Updated entry: ${entryUid}`); + console.info(`Updated entry: ${realEntryUid}${locale ? ` (locale "${locale}")` : ''}`); } } console.info('All entries updated successfully'); diff --git a/api/src/utils/entry-update.utils.ts b/api/src/utils/entry-update.utils.ts index 7b5aecfcc..445be9401 100644 --- a/api/src/utils/entry-update.utils.ts +++ b/api/src/utils/entry-update.utils.ts @@ -4,6 +4,10 @@ import path from "path"; import fs from "node:fs"; import { MIGRATION_DATA_CONFIG, DATABASE_FILES } from "../constants/index.js"; import { sanitizeStackId, assertResolvedPathUnderBase } from "./sanitize-path.utils.js"; +import { + isFullMigrationForLocale, + getSourceLocaleForDestination, +} from "./locale-migration.utils.js"; /** * Helper function to write log entries to file @@ -75,17 +79,21 @@ export const removeEntriesFromDatabase = async (projectId: string, loggerPath?: entryMapperItems.map((item: { otherCmsEntryUid: string }) => item?.otherCmsEntryUid) ); - // Entries that already exist in Contentstack (have a contentstackEntryUid). These are removed - // from the import data so they are NOT re-created — regardless of isUpdate. - const csUidMap = new Map(); - // Subset of the above marked isUpdate → collected into the update config so they get updated - // in Contentstack. Existing entries that are NOT isUpdate are simply left untouched. - const updateUidMap = new Map(); + // Per (otherCmsEntryUid, sourceLanguage) lookup so we can find the exact row that + // corresponds to a given locale directory. Same source entry may have N rows — one per + // source-locale variant — each with its own isUpdate flag. We also keep a legacy + // single-row-per-uid map for projects/tests where entry_mapper rows aren't tagged with + // a `language` field. + const rowByUidAndLang = new Map(); + const rowByUid = new Map(); + const csUidByOtherCmsUid = new Map(); for (const item of entryMapperItems) { if (item?.contentstackEntryUid) { - csUidMap.set(item?.otherCmsEntryUid, item?.contentstackEntryUid); - if (item?.isUpdate) { - updateUidMap.set(item?.otherCmsEntryUid, item?.contentstackEntryUid); + csUidByOtherCmsUid.set(item?.otherCmsEntryUid, item?.contentstackEntryUid); + const lang = (item as any)?.language ?? ''; + rowByUidAndLang.set(`${item?.otherCmsEntryUid}::${lang}`, item); + if (!rowByUid.has(item?.otherCmsEntryUid)) { + rowByUid.set(item?.otherCmsEntryUid, item); } } } @@ -112,6 +120,25 @@ export const removeEntriesFromDatabase = async (projectId: string, loggerPath?: ?.filter((dirent) => dirent?.isDirectory()); for (const localeDir of localeDirs) { + const localeCode = localeDir.name; + // Skip delta cleanup only when this is a restart AND the locale was never migrated + // before — newly-added locales have no prior state to diff against, so the regular + // full-import pipeline should pick them up untouched. On iteration 1 we always + // process (the function may be a no-op then, but tests/legacy code can still call it). + if ( + (projectData?.iteration ?? 1) > 1 && + isFullMigrationForLocale(projectData ?? {}, localeCode) + ) { + writeLogEntry( + `Skipping delta cleanup for new locale "${localeCode}" — full import.`, + "removeEntriesFromDatabase", + loggerPath, + ); + continue; + } + // entry_mapper rows are tagged with the SOURCE locale code (e.g. "en-IN"); + // directories on disk use the DESTINATION code (e.g. "en-in"). Translate. + const sourceLocale = getSourceLocaleForDestination(projectData ?? {}, localeCode); const localePath = path.join(ctPath, localeDir.name); // Respect index.json — only the chunk files it lists are current. Each import run // writes chunk files with fresh random UUID names and overwrites index.json, but does @@ -152,7 +179,7 @@ export const removeEntriesFromDatabase = async (projectId: string, loggerPath?: let modified = false; for (const key of Object?.keys(data)) { if (sitecoreUids.has(key)) { - const csEntryUid = csUidMap.get(key); + const csEntryUid = csUidByOtherCmsUid.get(key); // No Contentstack entry uid → this entry was never migrated, so leave it in // the import data to be CREATED this iteration. Deleting it would silently // drop the entry (data loss). @@ -160,18 +187,23 @@ export const removeEntriesFromDatabase = async (projectId: string, loggerPath?: continue; } - // Entry already exists in Contentstack. If it's marked isUpdate, collect it - // into the update config so it gets updated; otherwise it's left as-is. - if (updateUidMap.has(key)) { - const entryData = { ...data[key] }; + // Look up the entry_mapper row for THIS source-locale variant. Same source + // entry has separate rows for each source locale; `isUpdate` is per-row. + // When the project has no locale mapping (legacy/test data), fall back to + // a single-row-per-uid match so the legacy delta path still works. + const row = + (sourceLocale && rowByUidAndLang.get(`${key}::${sourceLocale}`)) || + rowByUid.get(key); + if (row?.isUpdate) { + const entryData = { ...data[key], __locale: localeCode, __csUid: csEntryUid }; delete entryData?.uid; if (!entriesToUpdate[contentTypeName]) { entriesToUpdate[contentTypeName] = {}; } - entriesToUpdate[contentTypeName][csEntryUid] = entryData; - writeLogEntry(`Collected update entry "${csEntryUid}" for content type "${contentTypeName}"`, "removeEntriesFromDatabase", loggerPath); - writeLogEntry(`Entry "${key}" has been prepared for update in Contentstack as "${csEntryUid}"`, "removeEntriesFromDatabase", loggerPath); + entriesToUpdate[contentTypeName][`${csEntryUid}::${localeCode}`] = entryData; + writeLogEntry(`Collected update entry "${csEntryUid}" (locale "${localeCode}") for content type "${contentTypeName}"`, "removeEntriesFromDatabase", loggerPath); + writeLogEntry(`Entry "${key}" has been prepared for update in Contentstack as "${csEntryUid}" (locale "${localeCode}")`, "removeEntriesFromDatabase", loggerPath); } // Existing entry → remove from import data so it is NOT re-created. diff --git a/api/src/utils/locale-migration.utils.ts b/api/src/utils/locale-migration.utils.ts new file mode 100644 index 000000000..af5941c56 --- /dev/null +++ b/api/src/utils/locale-migration.utils.ts @@ -0,0 +1,89 @@ +import ProjectModel from '../models/project-lowdb.js'; + +interface ProjectLike { + iteration?: number; + master_locale?: Record; + locales?: Record; + migrated_locales?: string[]; +} + +/** + * Returns every destination locale code mapped on the project (master + additional). + */ +export const getAllMappedLocales = (project: ProjectLike): string[] => { + const master = Object.keys(project?.master_locale ?? {}); + const additional = Object.keys(project?.locales ?? {}); + return Array.from(new Set([...master, ...additional])); +}; + +/** + * Map a destination locale code (e.g. "en-in") to its source locale code + * (e.g. "en-IN") via the project's master_locale + locales lookup. Returns + * null if the destination locale isn't mapped on the project. + */ +export const getSourceLocaleForDestination = ( + project: ProjectLike, + destLocale: string, +): string | null => { + if (!destLocale) return null; + if (project?.master_locale && destLocale in project.master_locale) { + return project.master_locale[destLocale]; + } + if (project?.locales && destLocale in project.locales) { + return project.locales[destLocale]; + } + return null; +}; + +/** + * Returns the project's migrated_locales, lazily backfilling for projects that + * finished a migration before this field existed. For iteration > 1 with no + * migrated_locales recorded, assume every currently-mapped locale was migrated + * previously (so they take the delta path, not a surprise re-run). + */ +export const getMigratedLocales = (project: ProjectLike): string[] => { + if (Array.isArray(project?.migrated_locales)) { + return project.migrated_locales as string[]; + } + if ((project?.iteration ?? 1) > 1) { + return getAllMappedLocales(project); + } + return []; +}; + +/** + * True when this locale should run a full migration on the current run. + * That happens when: this is the first iteration overall, OR this is a + * restart but the locale has no prior migration record (newly-added locale). + */ +export const isFullMigrationForLocale = ( + project: ProjectLike, + localeCode: string, +): boolean => { + if ((project?.iteration ?? 1) <= 1) return true; + return !getMigratedLocales(project).includes(localeCode); +}; + +/** + * Set-union the given locales into project.migrated_locales and persist. + * Idempotent. + */ +export const recordMigratedLocales = async ( + projectId: string, + locales: string[], +): Promise => { + if (!locales?.length) return; + await ProjectModel.read(); + await ProjectModel.update((data: any) => { + const idx = (data?.projects ?? []).findIndex( + (p: any) => p && p.id === projectId, + ); + if (idx < 0) return; + const existing: string[] = Array.isArray(data.projects[idx].migrated_locales) + ? data.projects[idx].migrated_locales + : []; + data.projects[idx].migrated_locales = Array.from( + new Set([...existing, ...locales]), + ); + }); +}; diff --git a/api/src/utils/uid-mapper.utils.ts b/api/src/utils/uid-mapper.utils.ts index 2b46ec6e3..c98a2386b 100644 --- a/api/src/utils/uid-mapper.utils.ts +++ b/api/src/utils/uid-mapper.utils.ts @@ -134,4 +134,123 @@ const writeUidMapping = async ( } }; +/** + * Walk the CLI's per-locale entry mapper output and merge a + * `entryByLocale: { [destLocale]: { [sourceUid]: destUid } }` map into the + * project's uid-mapper lowdb. Leaves the flat `entry` / `assets` maps untouched. + * + * CLI layout this reads from: + * /mapper/entries/uid-mapping.json (flat lookup) + * /mapper/entries///index.json + * /mapper/entries///-entries.json + * /mapper/entries///existing/index.json + * /mapper/entries///existing/-entries.json + * + * Files that are missing or malformed are skipped silently — this is best-effort + * enrichment and must never block the main mapping write. + */ +export const writePerLocaleEntryUidMapping = async ( + backupPath: string, + projectId: string, + iteration: number, +): Promise => { + try { + await projectModelLowdb.read(); + const projectData = projectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + const destinationStackId = projectData?.destination_stack_id; + + const entriesRoot = path.join(backupPath, "mapper", "entries"); + if (!fs.existsSync(entriesRoot)) return; + + // Flat source → dest lookup from the CLI's top-level uid-mapping.json. Used to fill + // dest uids for source uids we discover under each locale directory. + const flatMappingPath = path.join(entriesRoot, "uid-mapping.json"); + let flatMap: Record = {}; + if (fs.existsSync(flatMappingPath)) { + try { + const parsed = JSON.parse(fs.readFileSync(flatMappingPath, "utf-8")); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + flatMap = parsed as Record; + } + } catch { + /* malformed top-level mapping; per-locale map will be empty values */ + } + } + + const isDir = (p: string): boolean => { + try { return fs.statSync(p).isDirectory(); } catch { return false; } + }; + const readJson = (p: string): any => { + try { return JSON.parse(fs.readFileSync(p, "utf-8")); } catch { return null; } + }; + + // Extract source uids from a chunk-index style file ({ "1": "-entries.json", ... }). + const collectSourceUidsFromLocaleDir = (localeDir: string): string[] => { + const sourceUids: string[] = []; + const visit = (indexPath: string, base: string): void => { + const idx = readJson(indexPath); + if (!idx || typeof idx !== "object" || Array.isArray(idx)) return; + for (const file of Object.values(idx as Record)) { + if (typeof file !== "string" || !file.endsWith(".json")) continue; + const chunkPath = path.join(base, path.basename(file)); + const chunk = readJson(chunkPath); + if (chunk && typeof chunk === "object" && !Array.isArray(chunk)) { + sourceUids.push(...Object.keys(chunk)); + } + } + }; + visit(path.join(localeDir, "index.json"), localeDir); + const existingDir = path.join(localeDir, "existing"); + if (isDir(existingDir)) { + visit(path.join(existingDir, "index.json"), existingDir); + } + return sourceUids; + }; + + const entryByLocale: Record> = {}; + + for (const ctName of fs.readdirSync(entriesRoot)) { + const ctPath = path.join(entriesRoot, ctName); + if (!isDir(ctPath)) continue; + for (const localeName of fs.readdirSync(ctPath)) { + const localePath = path.join(ctPath, localeName); + if (!isDir(localePath)) continue; + const sourceUids = collectSourceUidsFromLocaleDir(localePath); + if (!sourceUids.length) continue; + if (!entryByLocale[localeName]) entryByLocale[localeName] = {}; + for (const srcUid of sourceUids) { + const destUid = flatMap[srcUid]; + if (destUid) entryByLocale[localeName][srcUid] = destUid; + } + } + } + + if (!Object.keys(entryByLocale).length) return; + + const UidMapperModelLowdb = getUidMapperDb(projectId, iteration); + await UidMapperModelLowdb.read(); + // Merge into any existing per-locale map rather than overwriting, so a follow-up + // import that touches only one locale doesn't wipe the others. + const existing: Record> = + (UidMapperModelLowdb.data as any)?.entryByLocale ?? {}; + const merged: Record> = { ...existing }; + for (const [loc, m] of Object.entries(entryByLocale)) { + merged[loc] = { ...(existing[loc] ?? {}), ...m }; + } + (UidMapperModelLowdb.data as any).entryByLocale = merged; + await UidMapperModelLowdb.write(); + await customLogger( + projectId, + destinationStackId, + "info", + `Per-locale entry uid mapping written for locales: ${Object.keys(merged).join(", ")}`, + ); + } catch (error) { + console.error("Error writing per-locale uid mapping:", error); + } +}; + export default writeUidMapping; diff --git a/api/tests/unit/utils/locale-migration.utils.test.ts b/api/tests/unit/utils/locale-migration.utils.test.ts new file mode 100644 index 000000000..38d39a9ab --- /dev/null +++ b/api/tests/unit/utils/locale-migration.utils.test.ts @@ -0,0 +1,186 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const { mockProjectRead, mockProjectUpdate } = vi.hoisted(() => ({ + mockProjectRead: vi.fn(), + mockProjectUpdate: vi.fn(), +})); + +vi.mock('../../../src/models/project-lowdb.js', () => ({ + default: { + read: mockProjectRead, + update: mockProjectUpdate, + }, +})); + +import { + getAllMappedLocales, + getSourceLocaleForDestination, + getMigratedLocales, + isFullMigrationForLocale, + recordMigratedLocales, +} from '../../../src/utils/locale-migration.utils'; + +describe('locale-migration.utils', () => { + describe('getAllMappedLocales', () => { + it('returns master + additional locale keys', () => { + const result = getAllMappedLocales({ + master_locale: { 'en-us': 'en-US' }, + locales: { 'en-in': 'en-IN', 'fr-fr': 'fr-FR' }, + }); + expect(result).toEqual(expect.arrayContaining(['en-us', 'en-in', 'fr-fr'])); + expect(result).toHaveLength(3); + }); + + it('dedupes overlap between master and additional', () => { + expect( + getAllMappedLocales({ + master_locale: { 'en-us': 'en' }, + locales: { 'en-us': 'en', 'fr-fr': 'fr' }, + }), + ).toEqual(['en-us', 'fr-fr']); + }); + + it('returns [] when no locales configured', () => { + expect(getAllMappedLocales({})).toEqual([]); + expect(getAllMappedLocales(null as any)).toEqual([]); + }); + }); + + describe('getSourceLocaleForDestination', () => { + const project = { + master_locale: { 'en-us': 'en-US' }, + locales: { 'en-in': 'en-IN' }, + }; + + it('returns master source for master dest locale', () => { + expect(getSourceLocaleForDestination(project, 'en-us')).toBe('en-US'); + }); + + it('returns additional source for additional dest locale', () => { + expect(getSourceLocaleForDestination(project, 'en-in')).toBe('en-IN'); + }); + + it('returns null when dest locale not mapped', () => { + expect(getSourceLocaleForDestination(project, 'fr-fr')).toBeNull(); + }); + + it('returns null for empty input', () => { + expect(getSourceLocaleForDestination(project, '')).toBeNull(); + expect(getSourceLocaleForDestination({}, 'en-us')).toBeNull(); + }); + }); + + describe('getMigratedLocales', () => { + it('returns the explicit array when present', () => { + expect( + getMigratedLocales({ migrated_locales: ['en-us', 'en-in'] }), + ).toEqual(['en-us', 'en-in']); + }); + + it('returns [] on iteration 1 with no record', () => { + expect( + getMigratedLocales({ + iteration: 1, + master_locale: { 'en-us': 'en' }, + }), + ).toEqual([]); + }); + + it('lazy-backfills to all mapped locales for restart projects missing the field', () => { + const result = getMigratedLocales({ + iteration: 2, + master_locale: { 'en-us': 'en' }, + locales: { 'fr-fr': 'fr' }, + }); + expect(result).toEqual(expect.arrayContaining(['en-us', 'fr-fr'])); + }); + }); + + describe('isFullMigrationForLocale', () => { + it('returns true on first iteration regardless of state', () => { + expect( + isFullMigrationForLocale( + { iteration: 1, migrated_locales: ['en-us'] }, + 'en-us', + ), + ).toBe(true); + }); + + it('returns false on restart when locale was previously migrated', () => { + expect( + isFullMigrationForLocale( + { iteration: 2, migrated_locales: ['en-us', 'en-in'] }, + 'en-us', + ), + ).toBe(false); + }); + + it('returns true on restart for a newly-added locale', () => { + expect( + isFullMigrationForLocale( + { iteration: 2, migrated_locales: ['en-us'] }, + 'fr-fr', + ), + ).toBe(true); + }); + + it('returns false on restart with lazy-backfilled migrated_locales', () => { + // No migrated_locales explicitly recorded, but iteration > 1 → backfill + // implies all currently-mapped locales were migrated. + expect( + isFullMigrationForLocale( + { + iteration: 2, + master_locale: { 'en-us': 'en' }, + locales: { 'en-in': 'en-IN' }, + }, + 'en-in', + ), + ).toBe(false); + }); + }); + + describe('recordMigratedLocales', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockProjectRead.mockResolvedValue(undefined); + }); + + it('skips work when locales array is empty', async () => { + await recordMigratedLocales('p1', []); + expect(mockProjectRead).not.toHaveBeenCalled(); + expect(mockProjectUpdate).not.toHaveBeenCalled(); + }); + + it('set-unions into the matching project record', async () => { + const data = { + projects: [ + { id: 'p1', migrated_locales: ['en-us'] }, + { id: 'p2', migrated_locales: ['en-us', 'en-in'] }, + ], + }; + mockProjectUpdate.mockImplementation(async (mut: any) => mut(data)); + await recordMigratedLocales('p1', ['en-in', 'en-us']); + expect(data.projects[0].migrated_locales).toEqual( + expect.arrayContaining(['en-us', 'en-in']), + ); + expect(data.projects[0].migrated_locales).toHaveLength(2); + // Untouched project shouldn't have been written to. + expect(data.projects[1].migrated_locales).toEqual(['en-us', 'en-in']); + }); + + it('initializes migrated_locales when missing on the project', async () => { + const data = { projects: [{ id: 'p1' }] }; + mockProjectUpdate.mockImplementation(async (mut: any) => mut(data)); + await recordMigratedLocales('p1', ['en-us']); + expect((data.projects[0] as any).migrated_locales).toEqual(['en-us']); + }); + + it('no-ops when project id is not found', async () => { + const data = { projects: [{ id: 'other' }] }; + mockProjectUpdate.mockImplementation(async (mut: any) => mut(data)); + await recordMigratedLocales('missing', ['en-us']); + expect((data.projects[0] as any).migrated_locales).toBeUndefined(); + }); + }); +}); diff --git a/api/tests/unit/utils/uid-mapper.utils.test.ts b/api/tests/unit/utils/uid-mapper.utils.test.ts index d4c2623e8..13c861d49 100644 --- a/api/tests/unit/utils/uid-mapper.utils.test.ts +++ b/api/tests/unit/utils/uid-mapper.utils.test.ts @@ -35,14 +35,23 @@ vi.mock('../../../src/utils/custom-logger.utils.js', () => ({ default: mockCustomLogger, })); +const { mockReaddirSync, mockStatSync } = vi.hoisted(() => ({ + mockReaddirSync: vi.fn(), + mockStatSync: vi.fn(), +})); + vi.mock('fs', () => ({ default: { existsSync: mockExistsSync, readFileSync: mockReadFileSync, + readdirSync: mockReaddirSync, + statSync: mockStatSync, }, })); -import writeUidMapping from '../../../src/utils/uid-mapper.utils'; +import writeUidMapping, { + writePerLocaleEntryUidMapping, +} from '../../../src/utils/uid-mapper.utils'; describe('uid-mapper.utils - writeUidMapping', () => { const uidDb = { @@ -151,4 +160,153 @@ describe('uid-mapper.utils - writeUidMapping', () => { ); consoleSpy.mockRestore(); }); +}); + +describe('uid-mapper.utils - writePerLocaleEntryUidMapping', () => { + const uidDb = { + read: mockUidRead, + write: mockUidWrite, + data: {} as Record, + }; + + beforeEach(() => { + vi.clearAllMocks(); + uidDb.data = {}; + mockProjectRead.mockResolvedValue(undefined); + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue({ + id: 'p1', + destination_stack_id: 'stack1', + }), + }), + }); + mockCustomLogger.mockResolvedValue(undefined); + mockUidRead.mockResolvedValue(undefined); + mockUidWrite.mockResolvedValue(undefined); + mockGetUidMapperDb.mockReturnValue(uidDb); + }); + + it('builds entryByLocale by walking content type / locale directories', async () => { + // Filesystem layout the CLI produces: + // /uid-mapping.json + // /article/{en-us,en-in}/{index.json, -entries.json} + mockExistsSync.mockReturnValue(true); + mockStatSync.mockReturnValue({ isDirectory: () => true }); + mockReaddirSync.mockImplementation((dir: string) => { + if (dir.endsWith('/entries')) return ['article']; + if (dir.endsWith('/article')) return ['en-us', 'en-in']; + return []; + }); + mockReadFileSync.mockImplementation((file: string) => { + if (file.endsWith('uid-mapping.json')) { + return JSON.stringify({ src1: 'dst1', src2: 'dst2' }); + } + if (file.endsWith('index.json')) { + return JSON.stringify({ '1': 'chunk.json' }); + } + // chunk file — same source uids regardless of locale (CS uid is shared) + return JSON.stringify({ src1: { title: 'x' }, src2: { title: 'y' } }); + }); + + await writePerLocaleEntryUidMapping('/backup', 'p1', 1); + + expect((uidDb.data as any).entryByLocale).toEqual({ + 'en-us': { src1: 'dst1', src2: 'dst2' }, + 'en-in': { src1: 'dst1', src2: 'dst2' }, + }); + expect(mockUidWrite).toHaveBeenCalled(); + }); + + it('merges with any existing entryByLocale instead of overwriting', async () => { + uidDb.data = { + entryByLocale: { 'fr-fr': { src9: 'dst9' } }, + }; + mockExistsSync.mockReturnValue(true); + mockStatSync.mockReturnValue({ isDirectory: () => true }); + mockReaddirSync.mockImplementation((dir: string) => { + if (dir.endsWith('/entries')) return ['article']; + if (dir.endsWith('/article')) return ['en-us']; + return []; + }); + mockReadFileSync.mockImplementation((file: string) => { + if (file.endsWith('uid-mapping.json')) return JSON.stringify({ src1: 'dst1' }); + if (file.endsWith('index.json')) return JSON.stringify({ '1': 'chunk.json' }); + return JSON.stringify({ src1: {} }); + }); + + await writePerLocaleEntryUidMapping('/backup', 'p1', 1); + + expect((uidDb.data as any).entryByLocale).toEqual({ + 'fr-fr': { src9: 'dst9' }, + 'en-us': { src1: 'dst1' }, + }); + }); + + it('returns early when the entries root does not exist', async () => { + mockExistsSync.mockReturnValue(false); + + await writePerLocaleEntryUidMapping('/backup', 'p1', 1); + + expect(mockUidWrite).not.toHaveBeenCalled(); + }); + + it('skips writing when no source uids were discovered', async () => { + mockExistsSync.mockReturnValue(true); + mockStatSync.mockReturnValue({ isDirectory: () => true }); + mockReaddirSync.mockReturnValue([]); // No content types + mockReadFileSync.mockReturnValue('{}'); + + await writePerLocaleEntryUidMapping('/backup', 'p1', 1); + + expect(mockUidWrite).not.toHaveBeenCalled(); + }); + + it('also reads the `existing/` subdir for entries CLI skipped re-creating', async () => { + mockExistsSync.mockReturnValue(true); + mockStatSync.mockReturnValue({ isDirectory: () => true }); + mockReaddirSync.mockImplementation((dir: string) => { + if (dir.endsWith('/entries')) return ['article']; + if (dir.endsWith('/article')) return ['en-us']; + return []; + }); + mockReadFileSync.mockImplementation((file: string) => { + if (file.endsWith('uid-mapping.json')) { + return JSON.stringify({ srcExisting: 'dstExisting', srcNew: 'dstNew' }); + } + if (file.endsWith('en-us/index.json')) { + return JSON.stringify({ '1': 'new.json' }); + } + if (file.endsWith('existing/index.json')) { + return JSON.stringify({ '1': 'existing.json' }); + } + if (file.endsWith('new.json')) return JSON.stringify({ srcNew: {} }); + if (file.endsWith('existing.json')) return JSON.stringify({ srcExisting: {} }); + return '{}'; + }); + + await writePerLocaleEntryUidMapping('/backup', 'p1', 1); + + expect((uidDb.data as any).entryByLocale).toEqual({ + 'en-us': { srcExisting: 'dstExisting', srcNew: 'dstNew' }, + }); + }); + + it('does not throw on malformed JSON files (best-effort)', async () => { + mockExistsSync.mockReturnValue(true); + mockStatSync.mockReturnValue({ isDirectory: () => true }); + mockReaddirSync.mockImplementation((dir: string) => { + if (dir.endsWith('/entries')) return ['article']; + if (dir.endsWith('/article')) return ['en-us']; + return []; + }); + mockReadFileSync.mockImplementation(() => { + throw new Error('parse error'); + }); + + await expect( + writePerLocaleEntryUidMapping('/backup', 'p1', 1), + ).resolves.toBeUndefined(); + expect(mockUidWrite).not.toHaveBeenCalled(); + }); }); \ No newline at end of file diff --git a/ui/src/components/ContentMapper/entryMapper.tsx b/ui/src/components/ContentMapper/entryMapper.tsx index 462237d0b..ec59e9d27 100644 --- a/ui/src/components/ContentMapper/entryMapper.tsx +++ b/ui/src/components/ContentMapper/entryMapper.tsx @@ -12,6 +12,7 @@ import { cbModal, CircularLoader, EmptyState, + Select, } from '@contentstack/venus-components'; // Services @@ -22,6 +23,7 @@ import { getEntryMapping, updateEntryMapper, } from '../../services/api/migration.service'; +import { getProject } from '../../services/api/project.service'; // Redux import { RootState } from '../../store'; @@ -100,6 +102,11 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { const [isLoadingSaveButton, setisLoadingSaveButton] = useState(false); const [initialRowSelectedData, setInitialRowSelectedData] = useState([]); + // Locale dropdown — sourced from project.json (master_locale + locales) so it reflects the + // user's configured mapping regardless of redux hydration timing on restart. + const [localeOptions, setLocaleOptions] = useState<{ label: string; value: string }[]>([]); + const [selectedLocale, setSelectedLocale] = useState<{ label: string; value: string } | null>(null); + /** ALL HOOKS HERE */ const { projectId = '' } = useParams<{ projectId: string }>(); const navigate = useNavigate(); @@ -130,6 +137,43 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { return () => document.removeEventListener('mousedown', handleClickOutside); }, []); + // Fetch the project's configured locale mapping (master + additional) on mount so the + // dropdown reflects the actual project state, not stale/missing redux. + useEffect(() => { + const orgId = selectedOrganisation?.uid; + if (!orgId || !projectId) return; + (async () => { + try { + const res: any = await getProject(orgId, projectId); + const project = res?.data?.project ?? res?.data ?? res; + const masterMap: Record = project?.master_locale ?? {}; + const additional: Record = project?.locales ?? {}; + const opts: { label: string; value: string }[] = []; + Object.keys(masterMap).forEach((code) => { + opts.push({ label: `${code} (master)`, value: code }); + }); + Object.keys(additional).forEach((code) => { + if (!opts.some((o) => o.value === code)) { + opts.push({ label: code, value: code }); + } + }); + setLocaleOptions(opts); + if (opts.length > 0) setSelectedLocale(opts[0]); + } catch (err) { + console.error('Failed to load project locales', err); + } + })(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [projectId, selectedOrganisation?.uid]); + + // Refetch the entry list when the user switches locale so isUpdate reflects the per-locale flag. + useEffect(() => { + if (contentTypeUid && selectedLocale?.value) { + fetchEntries(contentTypeUid, searchText || ''); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedLocale?.value]); + /********** HELPERS *************/ const buildSelectedRowIds = (entries: EntryMapperType[]) => { return (entries ?? []).reduce((acc, item) => { @@ -178,33 +222,14 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { } }; - // Clear the right-panel entry table + selection state. Called when the left list becomes empty - // (zero search/filter results) so a stale content type's entry table doesn't linger. - const resetEntryTable = () => { - setTableData([]); - setRowIds({}); - setPersistedRowIds({}); - setInitialRowSelectedData([]); - setTotalCounts(0); - setOtherCmsTitle(''); - setContentTypeUid(''); - setOtherCmsUid(''); - setActive(null); - }; - // Search content types in the left list const handleSearch = async (searchCT: string) => { setSearchContentType(searchCT); try { const { data } = await getContentTypes(projectId, 0, 1000, searchCT || '', 'old'); - const nextContentTypes = data?.contentTypes ?? []; - setContentTypes(nextContentTypes); - setFilteredContentTypes(nextContentTypes); - setCount(nextContentTypes?.length ?? 0); - // No matching content types → clear the right panel so the previous CT's table doesn't stick around. - if (!nextContentTypes?.length) { - resetEntryTable(); - } + setContentTypes(data?.contentTypes ?? []); + setFilteredContentTypes(data?.contentTypes ?? []); + setCount(data?.contentTypes?.length ?? 0); } catch (error) { console.error(error); return error; @@ -266,12 +291,6 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { if (value !== 'All') { setFilteredContentTypes(filteredCT); setCount(filteredCT?.length); - // Filter yielded no content types → clear the right panel so a stale entry table doesn't linger. - if (!filteredCT?.length) { - resetEntryTable(); - setShowFilter(false); - return; - } const selectedIndex = filteredCT.findIndex((ct) => ct?.otherCmsUid === otherCmsUid); setActive(selectedIndex >= 0 ? selectedIndex : null); } else { @@ -298,7 +317,7 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { setItemStatusMap(itemStatusMapLocal); setLoading(true); - const { data } = await getEntryMapping(ctId || '', 0, 1000, searchVal, projectId); + const { data } = await getEntryMapping(ctId || '', 0, 1000, searchVal, projectId, selectedLocale?.value); for (let index = 0; index <= 1000; index++) { itemStatusMapLocal[index] = 'loaded'; @@ -340,7 +359,7 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { setItemStatusMap({ ...itemStatusMapCopy }); setLoading(true); - const { data } = await getEntryMapping(contentTypeUid || '', skip, limit, search || '', projectId); + const { data } = await getEntryMapping(contentTypeUid || '', skip, limit, search || '', projectId, selectedLocale?.value); const updated: ItemStatusMapProp = { ...itemStatusMap }; for (let index = startIndex; index <= stopIndex; index++) { @@ -395,7 +414,8 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { const orgId = selectedOrganisation?.uid; if (orgId && contentTypeUid) { - const dataCs = { ids: changedUids }; + const dataCs: Record = { ids: changedUids }; + if (selectedLocale?.value) dataCs.locale = selectedLocale.value; try { if (changedUids.length === 0) { setisLoadingSaveButton(false); @@ -485,7 +505,7 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { const tableHeight = calcHeight(); const modalProps = { - body: 'An error occurred while generating the content mapper. Please go to the Legacy CMS step and validate the file again.', + body: 'There is something error occured while generating content mapper. Please go to Legacy Cms step and validate the file again.', }; return ( @@ -609,7 +629,22 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { {/* Entry Mapping Table */}
-
+
1 ? ' has-locale-select' : ''}`}> + {localeOptions?.length > 1 && ( +
+