From e3a156faf4cd93a8b8f6b876549e554679eaf088 Mon Sep 17 00:00:00 2001 From: chetan-contentstack Date: Tue, 16 Jun 2026 16:55:13 +0530 Subject: [PATCH 1/3] [CMG-998] Add locale-based delta migration support --- api/src/models/EntryMapper.ts | 3 +- api/src/models/project-lowdb.ts | 4 + api/src/models/uidMapper.ts | 8 +- api/src/services/contentMapper.service.ts | 27 +- api/src/services/runCli.service.ts | 16 +- api/src/services/wordpress.service.ts | 51 + api/src/utils/entry-update-script.cjs | 33 +- api/src/utils/entry-update.utils.ts | 55 +- api/src/utils/locale-migration.utils.ts | 89 ++ api/src/utils/uid-mapper.utils.ts | 119 ++ .../components/ContentMapper/entryMapper.tsx | 322 ++++- ui/src/components/ContentMapper/index.scss | 14 + ui/src/services/api/migration.service.ts | 6 +- upload-api/migration-wordpress/index.ts | 4 +- upload-api/package-lock.json | 1103 +---------------- upload-api/package.json | 4 + upload-api/src/controllers/wordpress/index.ts | 11 +- 17 files changed, 728 insertions(+), 1141 deletions(-) create mode 100644 api/src/utils/locale-migration.utils.ts diff --git a/api/src/models/EntryMapper.ts b/api/src/models/EntryMapper.ts index 7ba3eeb6d..b667f4c90 100644 --- a/api/src/models/EntryMapper.ts +++ b/api/src/models/EntryMapper.ts @@ -8,7 +8,7 @@ import { DATABASE_FILES } from "../constants/index.js"; * Represents an entry mapper object. */ export interface EntryMapper { - entry_mapper: { + entry_mapper: { id: string; projectId: string; contentTypeId: string; @@ -18,6 +18,7 @@ export interface EntryMapper { isUpdate: boolean; contentstackEntryUid: string; isDuplicateEntry: boolean; + language?: string; }[]; } diff --git a/api/src/models/project-lowdb.ts b/api/src/models/project-lowdb.ts index 008efe4e2..0e224f1ba 100644 --- a/api/src/models/project-lowdb.ts +++ b/api/src/models/project-lowdb.ts @@ -72,6 +72,10 @@ interface Project { taxonomies?: any[]; isSSO: boolean; iteration: number; + master_locale?: Record; + locales?: Record; + source_locales?: string[]; + migrated_locales?: string[]; } interface ProjectDocument { diff --git a/api/src/models/uidMapper.ts b/api/src/models/uidMapper.ts index 090de6472..437620351 100644 --- a/api/src/models/uidMapper.ts +++ b/api/src/models/uidMapper.ts @@ -10,9 +10,15 @@ import { DATABASE_FILES } from "../constants/index.js"; interface EntryMapper { entry: Record; assets: Record; + /** + * Per-locale entry uid mapping. Source uid → destination uid, scoped by destination + * locale code. Populated by writePerLocaleEntryUidMapping after each CLI import so the + * delta flow can tell which entries actually have a variant in a given locale. + */ + entryByLocale?: Record>; } -const defaultData: EntryMapper = { entry: {}, assets: {} }; +const defaultData: EntryMapper = { entry: {}, assets: {}, entryByLocale: {} }; /** * Creates and returns a database instance for the field mapper for a specific project. diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index bc46345d4..e7fe0b938 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -31,6 +31,7 @@ import getEntryMapperDb, { EntryMapper } from "../models/EntryMapper.js"; import getContentTypesMapperDb, { ContentTypesMapper } from "../models/contentTypesMapper-lowdb.js"; import getUidMapperDb from "../models/uidMapper.js"; import { isDuplicateEntry } from '../utils/entry-duplicate.utils.js'; +import { getSourceLocaleForDestination } from '../utils/locale-migration.utils.js'; const idCorrector = ({ id }: { id: string }) => { @@ -1880,6 +1881,8 @@ const updateEntryStatus = async (req: Request) => { const EntryMapperModel = getEntryMapperDb(projectId, iteration); await EntryMapperModel.read(); const foundEntry: EntryMapper[] = []; + // Rows in entry_mapper are already per-(entry × source-locale), so each id uniquely + // identifies one locale variant; toggling isUpdate directly is correct. await EntryMapperModel.update((data: any) => { data?.entry_mapper?.forEach((entry: any) => { if (validatedUids.includes(entry?.id)) { @@ -1928,6 +1931,8 @@ const getEntryMapping = async (req: Request) => { const skip: any = req?.params?.skip; const limit: any = req?.params?.limit; const search: string = req?.params?.searchText?.toLowerCase(); + const locale: string | undefined = + (req?.query?.locale as string) || (req?.params as any)?.locale; let result: any[] = []; let filteredResult = []; @@ -1961,6 +1966,11 @@ const getEntryMapping = async (req: Request) => { } const EntryMapperModel = getEntryMapperDb(projectId, iteration); await EntryMapperModel.read(); + // Convert the destination locale param (e.g. "en-in") to its source locale code + // (e.g. "en-IN") so we can filter the per-source-locale entry_mapper rows. + const sourceLocale = locale + ? getSourceLocaleForDestination(projectData ?? {}, locale) + : null; let entryMapping = contentType?.entryMapping?.map?.((mapperUId: any) => { const entryMapper = EntryMapperModel.chain .get("entry_mapper") @@ -1990,16 +2000,25 @@ const getEntryMapping = async (req: Request) => { entryMapping ?? [], ); - if (!isEmpty(enrichedMapping)) { + // entry_mapper rows are already per-source-locale (one row per language variant). + // Filter to just the rows whose `language` matches the selected destination locale's + // source code. Falls open when no locale is provided so legacy callers still work. + const localeFiltered = sourceLocale + ? (enrichedMapping ?? []).filter( + (row: any) => row && (row?.language ?? '') === sourceLocale, + ) + : enrichedMapping; + + if (!isEmpty(localeFiltered)) { if (search) { - filteredResult = enrichedMapping?.filter?.((item: any) => + filteredResult = localeFiltered?.filter?.((item: any) => item?.entryName?.toLowerCase().includes(search) ); totalCount = filteredResult?.length; result = filteredResult?.slice(skip, Number(skip) + Number(limit)); } else { - totalCount = enrichedMapping?.length; - result = enrichedMapping?.slice(skip, Number(skip) + Number(limit)); + totalCount = localeFiltered?.length; + result = localeFiltered?.slice(skip, Number(skip) + Number(limit)); } } return { diff --git a/api/src/services/runCli.service.ts b/api/src/services/runCli.service.ts index c814117f8..8ac60ebed 100644 --- a/api/src/services/runCli.service.ts +++ b/api/src/services/runCli.service.ts @@ -19,7 +19,7 @@ interface TestStack { isMigrated: boolean; } import { setBasicAuthConfig, setOAuthConfig } from '../utils/config-handler.util.js'; -import writeUidMapping from '../utils/uid-mapper.utils.js'; +import writeUidMapping, { writePerLocaleEntryUidMapping } from '../utils/uid-mapper.utils.js'; /** * Determines log level based on message content without removing ANSI codes @@ -277,6 +277,7 @@ export const runCli = async ( .value(); const iteration = projectData?.iteration || 1; await writeUidMapping(backupPath, projectId, iteration); + await writePerLocaleEntryUidMapping(backupPath, projectId, iteration); } // Keep the project status update code: @@ -319,6 +320,19 @@ export const runCli = async ( false; ProjectModelLowdb.data.projects[projectIndex].current_step = 5; ProjectModelLowdb.data.projects[projectIndex].status = 5; + // Record every locale that just successfully migrated so the next delta restart can + // tell which locales need a full pass vs delta. Set-union with prior value. + const proj: any = ProjectModelLowdb.data.projects[projectIndex]; + const ranLocales = Array.from( + new Set([ + ...Object.keys(proj?.master_locale ?? {}), + ...Object.keys(proj?.locales ?? {}), + ]), + ); + const existing: string[] = Array.isArray(proj?.migrated_locales) + ? proj.migrated_locales + : []; + proj.migrated_locales = Array.from(new Set([...existing, ...ranLocales])); await ProjectModelLowdb.write(); } } else { diff --git a/api/src/services/wordpress.service.ts b/api/src/services/wordpress.service.ts index ee090dcdb..33a52615d 100644 --- a/api/src/services/wordpress.service.ts +++ b/api/src/services/wordpress.service.ts @@ -1308,6 +1308,39 @@ async function saveEntry(fields: any, entry: any, file_path: string, assetData } return entryData; } +/** + * Replicate the master-locale entries JSON + index into each additional destination locale + * folder so the CLI import creates entry variants under every mapped locale, not just master. + * WP's WXR format doesn't carry per-locale content, so each locale gets the same payload — + * Contentstack's fallback chain handles the read-side behavior and the user can edit the + * variants later. + */ +async function fanOutEntriesToAdditionalLocales( + postFolderPath: string, + masterLocaleCode: string, + contentTypeRoot: string, + project: any, +): Promise { + const additional = project?.locales ?? {}; + const masterFilePath = path.join(postFolderPath, `${masterLocaleCode}.json`); + if (!existsSync(masterFilePath)) return; + const masterContent = await fs.promises.readFile(masterFilePath, 'utf-8'); + for (const destLocale of Object.keys(additional)) { + if (!destLocale || destLocale === masterLocaleCode) continue; + const localeFolderPath = path.join(contentTypeRoot, destLocale); + if (!existsSync(localeFolderPath)) { + await fs.promises.mkdir(localeFolderPath, { recursive: true }); + } + const localeFilePath = path.join(localeFolderPath, `${destLocale}.json`); + await fs.promises.writeFile(localeFilePath, masterContent, 'utf-8'); + await fs.promises.writeFile( + path.join(localeFolderPath, 'index.json'), + JSON.stringify({ '1': `${destLocale}.json` }, null, 4), + 'utf-8', + ); + } +} + async function createEntry(file_path: string, packagePath: string, destinationStackId: string, projectId: string, contentTypes: any, mapperKeys: any, master_locale: string, project: any){ const locale = getLocale(master_locale, project) || master_locale; const Jsondata = await fs.promises.readFile(packagePath, "utf8"); @@ -1355,6 +1388,12 @@ async function createEntry(file_path: string, packagePath: string, destinationSt await fs.promises.writeFile(path.join(postFolderPath, "index.json"), JSON.stringify({ "1": `${locale}.json` }, null, 4), "utf-8" ); + await fanOutEntriesToAdditionalLocales( + postFolderPath, + locale, + path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, postsFolderName), + project, + ); } const termsContentTypes = contentTypes?.filter((contentType: any) => contentType?.contentstackUid === 'terms'); @@ -1376,6 +1415,12 @@ async function createEntry(file_path: string, packagePath: string, destinationSt await fs.promises.writeFile(path.join(termsFolderPath, "index.json"), JSON.stringify({ "1": `${locale}.json` }, null, 4), "utf-8" ); + await fanOutEntriesToAdditionalLocales( + termsFolderPath, + locale, + path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, termsFolderName), + project, + ); } const postContentTypes = contentTypes?.filter( (contentType: any) => @@ -1412,6 +1457,12 @@ async function createEntry(file_path: string, packagePath: string, destinationSt await fs.promises.writeFile(path.join(postFolderPath, "index.json"), JSON.stringify({ "1": `${locale}.json` }, null, 4), "utf-8" ); + await fanOutEntriesToAdditionalLocales( + postFolderPath, + locale, + path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, postsFolderName), + project, + ); console.info(`Processed content for ${contentType?.contentstackTitle}:`, Object?.keys(content)?.length, "items"); } } diff --git a/api/src/utils/entry-update-script.cjs b/api/src/utils/entry-update-script.cjs index 29f667772..1178752d9 100644 --- a/api/src/utils/entry-update-script.cjs +++ b/api/src/utils/entry-update-script.cjs @@ -18,6 +18,8 @@ const FLAT_PAYLOAD_SKIP = new Set([ 'updated_by', '_content_type_uid', 'content', + '__locale', + '__csUid', ]); /** @@ -66,7 +68,7 @@ const resolveAssetField = (fieldName, entryUid, updateValue, stackValue, oldMapp * WordPress (and similar) write migration JSON with fields at the root (email, url, …). * Fetched stack entries keep custom fields under entry.content — merge flat updateData there. */ -const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping, newMapping) => { +const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping, newMapping, updateOpts) => { for (const field of Object.keys(updateData)) { if (FLAT_PAYLOAD_SKIP.has(field)) { continue; @@ -90,7 +92,7 @@ const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping } entry.content[field] = nextVal; } - await entry.update(); + await entry.update(updateOpts); }; module.exports = async ({ @@ -120,13 +122,22 @@ module.exports = async ({ console.info(`Processing content type: ${contentType}, entries: ${entryUids.length}`); for (const entryUid of entryUids) { + const updateData = JSON.parse(JSON.stringify(config[contentType][entryUid])); + // Per-locale config keys are "::" with __locale/__csUid + // on the payload. Fall back to the bare key for legacy single-locale + // configs. + const locale = updateData?.__locale; + const realEntryUid = updateData?.__csUid || entryUid; + delete updateData?.__locale; + delete updateData?.__csUid; + const fetchOpts = locale ? { locale } : undefined; + const updateOpts = locale ? { locale } : undefined; + const entryRef = stackSDKInstance .contentType(contentType) - .entry(entryUid); + .entry(realEntryUid); - - const entry = await entryRef?.fetch(); - const updateData = JSON.parse(JSON.stringify(config[contentType][entryUid])); + const entry = await entryRef?.fetch(fetchOpts); const hasStackContent = entry?.content && typeof entry?.content === 'object'; const hasNestedUpdate = updateData?.content && typeof updateData?.content === 'object'; @@ -145,10 +156,10 @@ module.exports = async ({ } } Object.assign(entry?.content, updateData?.content); - await entry.update(); + await entry.update(updateOpts); } else if (hasStackContent) { - console.info(`[${entryUid}] Merging flat migration payload into entry.content (e.g. WordPress export)`); - await mergeFlatPayloadIntoEntry(entry, entryUid, updateData, oldMapping, newMapping); + console.info(`[${realEntryUid}] Merging flat migration payload into entry.content (e.g. WordPress export)${locale ? ` for locale "${locale}"` : ''}`); + await mergeFlatPayloadIntoEntry(entry, realEntryUid, updateData, oldMapping, newMapping, updateOpts); } else { if (updateData && entry) { for (const field of Object.keys(updateData)) { @@ -166,9 +177,9 @@ module.exports = async ({ } } Object.assign(entry, updateData); - await entry.update(); + await entry.update(updateOpts); } - console.info(`Updated entry: ${entryUid}`); + console.info(`Updated entry: ${realEntryUid}${locale ? ` (locale "${locale}")` : ''}`); } } console.info('All entries updated successfully'); diff --git a/api/src/utils/entry-update.utils.ts b/api/src/utils/entry-update.utils.ts index 489fc4c1b..1da8ce694 100644 --- a/api/src/utils/entry-update.utils.ts +++ b/api/src/utils/entry-update.utils.ts @@ -4,6 +4,10 @@ import path from "path"; import fs from "node:fs"; import { MIGRATION_DATA_CONFIG, DATABASE_FILES } from "../constants/index.js"; import { sanitizeStackId, assertResolvedPathUnderBase } from "./sanitize-path.utils.js"; +import { + isFullMigrationForLocale, + getSourceLocaleForDestination, +} from "./locale-migration.utils.js"; /** * Helper function to write log entries to file @@ -75,10 +79,16 @@ export const removeEntriesFromDatabase = async (projectId: string, loggerPath?: entryMapperItems.map((item: { otherCmsEntryUid: string }) => item?.otherCmsEntryUid) ); - const updateUidMap = new Map(); + // Per (otherCmsEntryUid, sourceLanguage) lookup so we can find the exact row that + // corresponds to a given locale directory. Same source entry may have N rows — one per + // source-locale variant — each with its own isUpdate flag. + const rowByUidAndLang = new Map(); + const csUidByOtherCmsUid = new Map(); for (const item of entryMapperItems) { - if (item.isUpdate) { - updateUidMap.set(item?.otherCmsEntryUid, item?.contentstackEntryUid); + if (item?.contentstackEntryUid) { + csUidByOtherCmsUid.set(item?.otherCmsEntryUid, item?.contentstackEntryUid); + const lang = (item as any)?.language ?? ''; + rowByUidAndLang.set(`${item?.otherCmsEntryUid}::${lang}`, item); } } @@ -104,6 +114,21 @@ export const removeEntriesFromDatabase = async (projectId: string, loggerPath?: ?.filter((dirent) => dirent?.isDirectory()); for (const localeDir of localeDirs) { + const localeCode = localeDir.name; + // Newly-added locales have no prior migration state to diff against. Leave their + // import data alone (so the regular full-import pipeline picks them up) and skip + // generating any update payloads for them. + if (isFullMigrationForLocale(projectData ?? {}, localeCode)) { + writeLogEntry( + `Skipping delta cleanup for new locale "${localeCode}" — full import.`, + "removeEntriesFromDatabase", + loggerPath, + ); + continue; + } + // entry_mapper rows are tagged with the SOURCE locale code (e.g. "en-IN"); + // directories on disk use the DESTINATION code (e.g. "en-in"). Translate. + const sourceLocale = getSourceLocaleForDestination(projectData ?? {}, localeCode); const localePath = path.join(ctPath, localeDir.name); // Respect index.json — only the chunk files it lists are current. Each import run // writes chunk files with fresh random UUID names and overwrites index.json, but does @@ -144,17 +169,29 @@ export const removeEntriesFromDatabase = async (projectId: string, loggerPath?: let modified = false; for (const key of Object?.keys(data)) { if (sitecoreUids.has(key)) { - const csEntryUid = updateUidMap.get(key); - if (csEntryUid) { - const entryData = { ...data[key] }; + const csEntryUid = csUidByOtherCmsUid.get(key); + // No Contentstack entry uid → this entry was never migrated, so leave it in + // the import data to be CREATED this iteration. Deleting it would silently + // drop the entry (data loss). + if (!csEntryUid) { + continue; + } + + // Look up the entry_mapper row for THIS source-locale variant. Same source + // entry has separate rows for each source locale; `isUpdate` is per-row. + const row = sourceLocale + ? rowByUidAndLang.get(`${key}::${sourceLocale}`) + : undefined; + if (row?.isUpdate) { + const entryData = { ...data[key], __locale: localeCode, __csUid: csEntryUid }; delete entryData?.uid; if (!entriesToUpdate[contentTypeName]) { entriesToUpdate[contentTypeName] = {}; } - entriesToUpdate[contentTypeName][csEntryUid] = entryData; - writeLogEntry(`Collected update entry "${csEntryUid}" for content type "${contentTypeName}"`, "removeEntriesFromDatabase", loggerPath); - writeLogEntry(`Entry "${key}" has been prepared for update in Contentstack as "${csEntryUid}"`, "removeEntriesFromDatabase", loggerPath); + entriesToUpdate[contentTypeName][`${csEntryUid}::${localeCode}`] = entryData; + writeLogEntry(`Collected update entry "${csEntryUid}" (locale "${localeCode}") for content type "${contentTypeName}"`, "removeEntriesFromDatabase", loggerPath); + writeLogEntry(`Entry "${key}" has been prepared for update in Contentstack as "${csEntryUid}" (locale "${localeCode}")`, "removeEntriesFromDatabase", loggerPath); } delete data[key]; diff --git a/api/src/utils/locale-migration.utils.ts b/api/src/utils/locale-migration.utils.ts new file mode 100644 index 000000000..af5941c56 --- /dev/null +++ b/api/src/utils/locale-migration.utils.ts @@ -0,0 +1,89 @@ +import ProjectModel from '../models/project-lowdb.js'; + +interface ProjectLike { + iteration?: number; + master_locale?: Record; + locales?: Record; + migrated_locales?: string[]; +} + +/** + * Returns every destination locale code mapped on the project (master + additional). + */ +export const getAllMappedLocales = (project: ProjectLike): string[] => { + const master = Object.keys(project?.master_locale ?? {}); + const additional = Object.keys(project?.locales ?? {}); + return Array.from(new Set([...master, ...additional])); +}; + +/** + * Map a destination locale code (e.g. "en-in") to its source locale code + * (e.g. "en-IN") via the project's master_locale + locales lookup. Returns + * null if the destination locale isn't mapped on the project. + */ +export const getSourceLocaleForDestination = ( + project: ProjectLike, + destLocale: string, +): string | null => { + if (!destLocale) return null; + if (project?.master_locale && destLocale in project.master_locale) { + return project.master_locale[destLocale]; + } + if (project?.locales && destLocale in project.locales) { + return project.locales[destLocale]; + } + return null; +}; + +/** + * Returns the project's migrated_locales, lazily backfilling for projects that + * finished a migration before this field existed. For iteration > 1 with no + * migrated_locales recorded, assume every currently-mapped locale was migrated + * previously (so they take the delta path, not a surprise re-run). + */ +export const getMigratedLocales = (project: ProjectLike): string[] => { + if (Array.isArray(project?.migrated_locales)) { + return project.migrated_locales as string[]; + } + if ((project?.iteration ?? 1) > 1) { + return getAllMappedLocales(project); + } + return []; +}; + +/** + * True when this locale should run a full migration on the current run. + * That happens when: this is the first iteration overall, OR this is a + * restart but the locale has no prior migration record (newly-added locale). + */ +export const isFullMigrationForLocale = ( + project: ProjectLike, + localeCode: string, +): boolean => { + if ((project?.iteration ?? 1) <= 1) return true; + return !getMigratedLocales(project).includes(localeCode); +}; + +/** + * Set-union the given locales into project.migrated_locales and persist. + * Idempotent. + */ +export const recordMigratedLocales = async ( + projectId: string, + locales: string[], +): Promise => { + if (!locales?.length) return; + await ProjectModel.read(); + await ProjectModel.update((data: any) => { + const idx = (data?.projects ?? []).findIndex( + (p: any) => p && p.id === projectId, + ); + if (idx < 0) return; + const existing: string[] = Array.isArray(data.projects[idx].migrated_locales) + ? data.projects[idx].migrated_locales + : []; + data.projects[idx].migrated_locales = Array.from( + new Set([...existing, ...locales]), + ); + }); +}; diff --git a/api/src/utils/uid-mapper.utils.ts b/api/src/utils/uid-mapper.utils.ts index 2b46ec6e3..c98a2386b 100644 --- a/api/src/utils/uid-mapper.utils.ts +++ b/api/src/utils/uid-mapper.utils.ts @@ -134,4 +134,123 @@ const writeUidMapping = async ( } }; +/** + * Walk the CLI's per-locale entry mapper output and merge a + * `entryByLocale: { [destLocale]: { [sourceUid]: destUid } }` map into the + * project's uid-mapper lowdb. Leaves the flat `entry` / `assets` maps untouched. + * + * CLI layout this reads from: + * /mapper/entries/uid-mapping.json (flat lookup) + * /mapper/entries///index.json + * /mapper/entries///-entries.json + * /mapper/entries///existing/index.json + * /mapper/entries///existing/-entries.json + * + * Files that are missing or malformed are skipped silently — this is best-effort + * enrichment and must never block the main mapping write. + */ +export const writePerLocaleEntryUidMapping = async ( + backupPath: string, + projectId: string, + iteration: number, +): Promise => { + try { + await projectModelLowdb.read(); + const projectData = projectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + const destinationStackId = projectData?.destination_stack_id; + + const entriesRoot = path.join(backupPath, "mapper", "entries"); + if (!fs.existsSync(entriesRoot)) return; + + // Flat source → dest lookup from the CLI's top-level uid-mapping.json. Used to fill + // dest uids for source uids we discover under each locale directory. + const flatMappingPath = path.join(entriesRoot, "uid-mapping.json"); + let flatMap: Record = {}; + if (fs.existsSync(flatMappingPath)) { + try { + const parsed = JSON.parse(fs.readFileSync(flatMappingPath, "utf-8")); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + flatMap = parsed as Record; + } + } catch { + /* malformed top-level mapping; per-locale map will be empty values */ + } + } + + const isDir = (p: string): boolean => { + try { return fs.statSync(p).isDirectory(); } catch { return false; } + }; + const readJson = (p: string): any => { + try { return JSON.parse(fs.readFileSync(p, "utf-8")); } catch { return null; } + }; + + // Extract source uids from a chunk-index style file ({ "1": "-entries.json", ... }). + const collectSourceUidsFromLocaleDir = (localeDir: string): string[] => { + const sourceUids: string[] = []; + const visit = (indexPath: string, base: string): void => { + const idx = readJson(indexPath); + if (!idx || typeof idx !== "object" || Array.isArray(idx)) return; + for (const file of Object.values(idx as Record)) { + if (typeof file !== "string" || !file.endsWith(".json")) continue; + const chunkPath = path.join(base, path.basename(file)); + const chunk = readJson(chunkPath); + if (chunk && typeof chunk === "object" && !Array.isArray(chunk)) { + sourceUids.push(...Object.keys(chunk)); + } + } + }; + visit(path.join(localeDir, "index.json"), localeDir); + const existingDir = path.join(localeDir, "existing"); + if (isDir(existingDir)) { + visit(path.join(existingDir, "index.json"), existingDir); + } + return sourceUids; + }; + + const entryByLocale: Record> = {}; + + for (const ctName of fs.readdirSync(entriesRoot)) { + const ctPath = path.join(entriesRoot, ctName); + if (!isDir(ctPath)) continue; + for (const localeName of fs.readdirSync(ctPath)) { + const localePath = path.join(ctPath, localeName); + if (!isDir(localePath)) continue; + const sourceUids = collectSourceUidsFromLocaleDir(localePath); + if (!sourceUids.length) continue; + if (!entryByLocale[localeName]) entryByLocale[localeName] = {}; + for (const srcUid of sourceUids) { + const destUid = flatMap[srcUid]; + if (destUid) entryByLocale[localeName][srcUid] = destUid; + } + } + } + + if (!Object.keys(entryByLocale).length) return; + + const UidMapperModelLowdb = getUidMapperDb(projectId, iteration); + await UidMapperModelLowdb.read(); + // Merge into any existing per-locale map rather than overwriting, so a follow-up + // import that touches only one locale doesn't wipe the others. + const existing: Record> = + (UidMapperModelLowdb.data as any)?.entryByLocale ?? {}; + const merged: Record> = { ...existing }; + for (const [loc, m] of Object.entries(entryByLocale)) { + merged[loc] = { ...(existing[loc] ?? {}), ...m }; + } + (UidMapperModelLowdb.data as any).entryByLocale = merged; + await UidMapperModelLowdb.write(); + await customLogger( + projectId, + destinationStackId, + "info", + `Per-locale entry uid mapping written for locales: ${Object.keys(merged).join(", ")}`, + ); + } catch (error) { + console.error("Error writing per-locale uid mapping:", error); + } +}; + export default writeUidMapping; diff --git a/ui/src/components/ContentMapper/entryMapper.tsx b/ui/src/components/ContentMapper/entryMapper.tsx index 29fa5f0bc..1db60c8f3 100644 --- a/ui/src/components/ContentMapper/entryMapper.tsx +++ b/ui/src/components/ContentMapper/entryMapper.tsx @@ -6,7 +6,10 @@ import { Button, InfiniteScrollTable, Notification, - + cbModal, + CircularLoader, + EmptyState, + Select, } from '@contentstack/venus-components'; // Services @@ -16,6 +19,7 @@ import { getEntryMapping, updateEntryMapper, } from '../../services/api/migration.service'; +import { getProject } from '../../services/api/project.service'; // Redux import { RootState } from '../../store'; @@ -68,36 +72,44 @@ const EntryMapper = ({selectedContentTypeId, tableHeight}: {selectedContentTypeI const [isLoadingSaveButton, setisLoadingSaveButton] = useState(false); const [initialRowSelectedData, setInitialRowSelectedData] = useState([]); - - - /********** ALL USEEFFECT HERE *************/ + // Fetch the project's configured locale mapping (master + additional) on mount so the + // dropdown reflects the actual project state, not stale/missing redux. useEffect(() => { - //check if offline CMS data field is set to true, if then read data from cms data file. - getCMSDataFromFile(CS_ENTRIES.CONTENT_MAPPING) - .then((data) => { - //Check for null - if (!data) { - dispatch(updateMigrationData({ contentMappingData: DEFAULT_CONTENT_MAPPING_DATA })); - return; - } - - dispatch(updateMigrationData({ contentMappingData: data })); - }) - .catch((err) => { - console.error(err); - }); - - fetchContentTypes(searchText || ''); - }, []); + const orgId = selectedOrganisation?.uid; + if (!orgId || !projectId) return; + (async () => { + try { + const res: any = await getProject(orgId, projectId); + const project = res?.data?.project ?? res?.data ?? res; + const masterMap: Record = project?.master_locale ?? {}; + const additional: Record = project?.locales ?? {}; + const opts: { label: string; value: string }[] = []; + Object.keys(masterMap).forEach((code) => { + opts.push({ label: `${code} (master)`, value: code }); + }); + Object.keys(additional).forEach((code) => { + if (!opts.some((o) => o.value === code)) { + opts.push({ label: code, value: code }); + } + }); + setLocaleOptions(opts); + if (opts.length > 0) setSelectedLocale(opts[0]); + } catch (err) { + console.error('Failed to load project locales', err); + } + })(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [projectId, selectedOrganisation?.uid]); + // Refetch the entry list when the user switches locale so isUpdate reflects the per-locale flag. useEffect(() => { - if (selectedContentTypeId) { - fetchEntries(selectedContentTypeId?.id || '', searchText); - setOtherCmsTitle(selectedContentTypeId?.otherCmsTitle); + if (contentTypeUid && selectedLocale?.value) { + fetchEntries(contentTypeUid, searchText || ''); } - - },[selectedContentTypeId]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedLocale?.value]); + /********** HELPERS *************/ const buildSelectedRowIds = (entries: EntryMapperType[]) => { return (entries ?? []).reduce((acc, item) => { if (item?._canSelect && item?.isUpdate) { @@ -151,9 +163,9 @@ const EntryMapper = ({selectedContentTypeId, tableHeight}: {selectedContentTypeI setItemStatusMap(itemStatusMap); setLoading(true); - - const { data } = await getEntryMapping(contentTypeId || '', 0, 1000, searchText, projectId); - + + const { data } = await getEntryMapping(ctId || '', 0, 1000, searchVal, projectId, selectedLocale?.value); + for (let index = 0; index <= 1000; index++) { itemStatusMap[index] = 'loaded'; } @@ -197,9 +209,7 @@ const EntryMapper = ({selectedContentTypeId, tableHeight}: {selectedContentTypeI setItemStatusMap({ ...itemStatusMapCopy }); setLoading(true); - const { data } = await getEntryMapping(contentTypeUid || '', skip, limit, searchText || '', projectId); - - const updateditemStatusMapCopy: ItemStatusMapProp = { ...itemStatusMap }; + const { data } = await getEntryMapping(contentTypeUid || '', skip, limit, search || '', projectId, selectedLocale?.value); for (let index = startIndex; index <= stopIndex; index++) { updateditemStatusMapCopy[index] = 'loaded'; @@ -232,27 +242,30 @@ const EntryMapper = ({selectedContentTypeId, tableHeight}: {selectedContentTypeI selectedObj[uid] = true; }); - setRowIds(selectedObj); - setTableData((prev) => applySelectionToEntries(prev ?? [], selectedObj)); - }; - - const handleSaveContentType = async () => { - console.info("handleSaveContentType", rowIds); - setisLoadingSaveButton(true); - const allKeys = new Set([ - ...Object.keys(rowIds ?? {}), - ...Object.keys(persistedRowIds ?? {}), - ]); - const changedUids = Array.from(allKeys).filter( - (uid) => !!rowIds?.[uid] !== !!persistedRowIds?.[uid], - ); - const orgId = selectedOrganisation?.uid; - // const projectID = projectId; - - if (orgId && contentTypeUid) { - const dataCs = { - ids: changedUids - }; + // Handle selected entries + const handleSelectedEntries = (singleSelectedRowIds: string[]) => { + const selectedObj: UidMap = {}; + singleSelectedRowIds?.forEach((uid: string) => { + selectedObj[uid] = true; + }); + setRowIds(selectedObj); + setTableData((prev) => applySelectionToEntries(prev ?? [], selectedObj)); + }; + + const handleSaveContentType = async () => { + setisLoadingSaveButton(true); + const allKeys = new Set([ + ...Object.keys(rowIds ?? {}), + ...Object.keys(persistedRowIds ?? {}), + ]); + const changedUids = Array.from(allKeys).filter( + (uid) => !!rowIds?.[uid] !== !!persistedRowIds?.[uid], + ); + const orgId = selectedOrganisation?.uid; + + if (orgId && contentTypeUid) { + const dataCs: Record = { ids: changedUids }; + if (selectedLocale?.value) dataCs.locale = selectedLocale.value; try { if (changedUids.length === 0) { setisLoadingSaveButton(false); @@ -403,12 +416,203 @@ const EntryMapper = ({selectedContentTypeId, tableHeight}: {selectedContentTypeI plural: `${totalCounts === 0 ? 'Count' : ''}` }} - /> -
-
Total Entries: {totalCounts}
- + {showFilter && ( +
+
    + {Object.keys(CONTENT_MAPPING_STATUS)?.map?.((key, keyInd) => ( +
  • + +
    • + ))} +
+
+ )} +
+ + + {filteredContentTypes && validateArray(filteredContentTypes) + ?
+
    + {filteredContentTypes?.map?.((content: ContentType, index: number) => { + const icon = STATUS_ICON_Mapping[content?.status] || ''; + const format = (str: string) => { + const frags = str?.split('_'); + for (let i = 0; i < frags?.length; i++) { + frags[i] = frags?.[i]?.charAt?.(0)?.toUpperCase() + frags?.[i]?.slice(1); + } + return frags?.join?.(' '); + }; + return ( +
  • + +
    + + {icon && ( + + + + )} + + + + + + +
    +
    • + ); + })} +
+
+ :
No Content Types Found.
+ } + + + {/* Entry Mapping Table */} +
+
+
1 ? ' has-locale-select' : ''}`}> + {localeOptions?.length > 1 && ( +
+