diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4219fdb6568..264ffdd01d3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -65,15 +65,12 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 # Add timeout to prevent hanging jobs steps: - - name: Setup SSH agent for Docker build + - name: Setup SSH agent with build and deploy keys uses: webfactory/ssh-agent@v0.9.0 with: - ssh-private-key: ${{ secrets.DOCKER_BUILD_SSH_KEY || '' }} - - - name: Add deployment SSH key to agent - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} + ssh-private-key: | + ${{ secrets.DOCKER_BUILD_SSH_KEY }} + ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} - name: Configure git to use SSH for submodules run: | @@ -520,6 +517,13 @@ jobs: echo "IMAGE=$IMAGE_BY_TAG" >> $GITHUB_ENV fi + - name: Debug SSH agent + if: steps.check_image.outputs.SKIP_BUILD == 'false' + run: | + echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" + ssh-add -l || echo "No keys in agent" + ssh -o StrictHostKeyChecking=no -T git@github.com 2>&1 || true + - name: Build Docker image if: steps.check_image.outputs.SKIP_BUILD == 'false' env: @@ -533,7 +537,7 @@ jobs: # Build SSH args - only add if SSH_AUTH_SOCK is available (from ssh-agent) SSH_ARGS="" if [ -n "$SSH_AUTH_SOCK" ]; then - SSH_ARGS="--ssh default" + SSH_ARGS="--ssh default=$SSH_AUTH_SOCK" fi # Collect cache sources for better layer reuse diff --git a/amp/Dockerfile b/amp/Dockerfile index 06687fe9ba2..167999bbd7a 100644 --- a/amp/Dockerfile +++ b/amp/Dockerfile @@ -178,6 +178,10 @@ COPY --from=compile-amp-filter /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-fi COPY --from=compile-amp-settings /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-settings ../ampTemplate/node_modules/amp-settings # Copy package files for dependency installation COPY TEMPLATE/reampv2/package*.json ./ +COPY TEMPLATE/reampv2/packages/ampoffline/package.json ./packages/ampoffline/package.json +COPY TEMPLATE/reampv2/packages/container/package.json ./packages/container/package.json +COPY TEMPLATE/reampv2/packages/reampv2-app/package.json ./packages/reampv2-app/package.json +COPY TEMPLATE/reampv2/packages/user-manager/package.json ./packages/user-manager/package.json RUN --mount=type=cache,target=/root/.npm \ --mount=type=ssh \ npm-install-with-retry.sh @@ -252,4 +256,4 @@ LABEL "branch"=$AMP_BRANCH ENV AMP_REGISTRY_PRIVATE_KEY $AMP_REGISTRY_PRIVATE_KEY RUN rm -fr /usr/local/tomcat/webapps/ROOT -COPY --from=compile-mvn /tmp/amp/exploded /usr/local/tomcat/webapps/ROOT/ +COPY --from=compile-mvn /tmp/amp/exploded /usr/local/tomcat/webapps/ROOT/ \ No newline at end of file diff --git a/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java b/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java index 43bcfe05720..d7b6ef38acc 100644 --- a/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java +++ b/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java @@ -387,8 +387,7 @@ public final class ArConstants { // public final static String EXECUTING_AGENCY_PERCENTAGE="Eexecuting Agency Percentage"; - //burkina -// public final static String PROGRAM_PERCENTAGE="Program Percentage"; + public final static String PROGRAM_PERCENTAGE="Program Percentage"; diff --git a/amp/src/main/java/org/dgfoundation/amp/onepager/util/ActivityUtil.java b/amp/src/main/java/org/dgfoundation/amp/onepager/util/ActivityUtil.java index 207728fe54b..cdb597b8182 100644 --- a/amp/src/main/java/org/dgfoundation/amp/onepager/util/ActivityUtil.java +++ b/amp/src/main/java/org/dgfoundation/amp/onepager/util/ActivityUtil.java @@ -212,9 +212,14 @@ public static AmpActivityVersion saveActivityNewVersion(AmpActivityVersion a, AmpActivityGroup tmpGroup = a.getAmpActivityGroup(); a = ActivityVersionUtil.cloneActivity(a); - //keeping session.clear() only for acitivity form as it was before - if (isActivityForm) - session.clear(); + // Always clear the session after cloning. When running in a batch context (e.g. Excel importer), + // validateAndImport executes queries with FlushMode.AUTO which can cascade-save new child entities + // (fundings, etc.) into the action queue. The subsequent session.evict(oldA) then cascade-evicts + // those children from the persistence context while they remain in the action queue. The flush + // below would find them in the queue but not in the persistence context. + // Clearing the session here discards those stale queued actions; downstream session.save()/ + // saveOrUpdate() re-registers everything cleanly before the real INSERTs are flushed. + session.clear(); a.setMember(new HashSet<>()); if (tmpGroup == null) { //we need to create a group for this activity diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityImporter.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityImporter.java index 03f3185c301..6ccd6bd896f 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityImporter.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityImporter.java @@ -28,6 +28,7 @@ import org.digijava.kernel.ampapi.filters.AmpClientModeHolder; import org.digijava.kernel.exception.DgException; import org.digijava.kernel.persistence.DBPersistenceTransactionManager; +import org.digijava.kernel.persistence.PersistenceManager; import org.digijava.kernel.persistence.PersistenceTransactionManager; import org.digijava.kernel.request.TLSUtils; import org.digijava.kernel.user.User; @@ -246,6 +247,16 @@ private void importOrUpdateActivity(Long activityId) { oldActivity.setAmpId(newActivity.getAmpId()); oldActivity.setAmpActivityGroup(newActivity.getAmpActivityGroup().clone()); + // Evict newActivity from the Hibernate session before validateAndImport processes JSON. + // loadActivity() eagerly initializes the funding set (Hibernate.initialize), making it a + // managed PersistentSet. Without this eviction, validateAndImport replaces newActivity.funding + // with JSON-derived objects (different Java identity). Hibernate's all-delete-orphan cascade + // then orphan-deletes the original DB fundings during any auto-flush triggered by queries + // inside validateAndImport. Evicting detaches newActivity so Hibernate stops tracking those + // collection changes, preventing the spurious DELETE that causes StaleStateException later + // when saveOrUpdate tries to UPDATE the already-deleted funding rows. + PersistenceManager.getRequestDBSession().evict(newActivity); + // newActivity.getAmpActivityGroup().setVersion(-1L); // TODO AMP-28993: remove explicitly resetting createdBy since it is cleared during init if (!rules.isTrackEditors()) { diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityInterchangeUtils.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityInterchangeUtils.java index 54fd5c23802..0928a762337 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityInterchangeUtils.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityInterchangeUtils.java @@ -79,6 +79,7 @@ public static JsonApiResponse importActivity(Map newJson) { if (AmpClientModeHolder.isOfflineClient()) { Workspace team = TeamUtil.getWorkspace(Long.parseLong(newJson.get("team").toString())); diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/DashboardsService.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/DashboardsService.java index 7e25886cbe9..55fab08ccdd 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/DashboardsService.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/DashboardsService.java @@ -657,10 +657,11 @@ public static String calculateSumarizedTotals(double total, ReportSpecificationI private static String getKMB(String lang, int exp) { String ret = ""; + int normalizedExp = Math.max(EXP_1, Math.min(exp, EXP_6)); if (lang.equals("en") || lang.equals("sp")) { - ret = "kMBTPE".charAt(exp - 1) + ""; + ret = "kMBTPE".charAt(normalizedExp - 1) + ""; } else if (lang.equals("fr")) { - switch (exp) { + switch (normalizedExp) { case EXP_1: ret = "m"; break; diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java index 0f062b3d42a..e143bf1e1a9 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java @@ -26,12 +26,12 @@ public class SectorSchemeDTO { private final SectorDTO[] children; public SectorSchemeDTO(AmpSectorScheme scheme, SectorDTO[] children) { - this.ampSecSchemeId = scheme.getAmpSecSchemeId(); - this.secSchemeCode = scheme.getSecSchemeCode(); - this.secSchemeName = scheme.getSecSchemeName(); - this.showInRMFilters = scheme.getShowInRMFilters(); - this.used = scheme.isUsed(); - this.children = children; + this.ampSecSchemeId = scheme != null ? scheme.getAmpSecSchemeId() : null; + this.secSchemeCode = scheme != null ? scheme.getSecSchemeCode() : null; + this.secSchemeName = scheme != null ? scheme.getSecSchemeName() : null; + this.showInRMFilters = Boolean.TRUE.equals(scheme != null ? scheme.getShowInRMFilters() : null); + this.used = scheme != null && scheme.isUsed(); + this.children = children != null ? children : new SectorDTO[0]; } public Long getAmpSecSchemeId() { diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java index fa6b9c2e4bc..0acd3b49a38 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java @@ -79,6 +79,88 @@ public MEIndicatorDTO getMEIndicatorById(final Long indicatorId) { throw new ApiRuntimeException(BAD_REQUEST, ApiError.toError("Indicator with id " + indicatorId + " not found")); } + public MEIndicatorDTO getMeIndicatorByNameAndProgramName(String name, String programName) { + Session session = PersistenceManager.getSession(); + AmpIndicator indicator; + if (programName==null){ + indicator = (AmpIndicator) session.createCriteria(AmpIndicator.class) + .add(Restrictions.eq("name", name)) + .setMaxResults(1) + .uniqueResult(); + } + else { + indicator = (AmpIndicator) session.createCriteria(AmpIndicator.class) + .add(Restrictions.eq("name", name)) + .createAlias("program", "p") + .add(Restrictions.eq("p.name", programName)) + .setMaxResults(1) + .uniqueResult(); + } + + + + if (indicator != null) { + return new MEIndicatorDTO(indicator); + } + + throw new ApiRuntimeException(BAD_REQUEST, + ApiError.toError("Indicator with name " + name + " and program name " + programName + " not found")); + } + + /** + * Returns the indicator by name and optional program name, or null if not found. + * Use this when you need to look up an indicator without throwing (e.g. data import). + * Tries exact name match first; if not found, looks for an indicator whose name contains + * the given name (e.g. DB "1.2.1 - Number of ... - 1.2.1" matches file "Number of ..."). + */ + public MEIndicatorDTO getMeIndicatorByNameAndProgramNameOptional(String name, String programName) { + if (name == null || name.trim().isEmpty()) { + return null; + } + String trimmedName = name.trim(); + String trimmedProgram = programName != null && !programName.trim().isEmpty() ? programName.trim() : null; + try { + return getMeIndicatorByNameAndProgramName(trimmedName, trimmedProgram); + } catch (ApiRuntimeException e) { + logger.info("getMeIndicatorByNameAndProgramNameOptional: exact match not found, trying substring match for name='" + trimmedName + "' programName='" + trimmedProgram + "'"); + + // exact match not found, try substring match (e.g. DB "1.2.1 - X - 1.2.1" vs file "X") + } + return getMeIndicatorByNameSubstringOptional(trimmedName, trimmedProgram); + } + + /** + * Finds an indicator whose stored name contains the given name (e.g. "1.2.1 - X - 1.2.1" contains "X"). + * Returns null if none or multiple matches (when program not specified), or the match when program is specified. + */ + private MEIndicatorDTO getMeIndicatorByNameSubstringOptional(String name, String programName) { + Session session = PersistenceManager.getSession(); + String escaped = escapeForLike(name); + String pattern = "%" + escaped + "%"; + String hql = "from " + AmpIndicator.class.getName() + " i where i.name like :name escape '\\'"; + if (programName != null) { + hql += " and i.program is not null and i.program.name = :programName"; + } + org.hibernate.query.Query query = session.createQuery(hql); + query.setParameter("name", pattern); + if (programName != null) { + query.setParameter("programName", programName); + } + @SuppressWarnings("unchecked") + List list = (List) query.list(); + if (list == null || list.isEmpty()) { + return null; + } + if (list.size() > 1 && programName == null) { + logger.warn("getMeIndicatorByNameSubstringOptional: multiple indicators contain name substring '" + name + "', returning first"); + } + return new MEIndicatorDTO(list.get(0)); + } + + private static String escapeForLike(String s) { + if (s == null) return null; + return s.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_"); + } public MEIndicatorDTO createMEIndicator(final MEIndicatorDTO indicatorRequest) { Session session = PersistenceManager.getSession(); @@ -167,6 +249,12 @@ private void validateYear(MEIndicatorDTO value) { } private void validateYearRange(String startYear, String endYear, AmpIndicatorGlobalValue value, String error){ + if (value == null) { + return; + } + if (startYear == null || endYear == null) { + return; + } String startInString = "01/01/" + startYear; DateTime dateTime = DateTime.parse(startInString, formatter); diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/util/DateFilterUtils.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/util/DateFilterUtils.java index 31334834d3c..9c554f79dc8 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/util/DateFilterUtils.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/util/DateFilterUtils.java @@ -134,8 +134,12 @@ private static FilterRule getDatesListFilterRule(ElementType elemType, List R doInTransaction(Function fn) { PersistenceManager.closeSession(session); } } -} +} \ No newline at end of file diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java index cc5244d9b79..28625f69a70 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -1,11 +1,36 @@ package org.digijava.module.aim.action.dataimporter; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.opencsv.CSVParser; -import com.opencsv.CSVParserBuilder; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvValidationException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; @@ -16,45 +41,59 @@ import org.apache.struts.action.ActionForward; import org.apache.struts.action.ActionMapping; import org.digijava.kernel.persistence.PersistenceManager; +import org.digijava.kernel.translator.TranslatorWorker; +import static org.digijava.module.aim.action.dataimporter.ExcelImporter.processExcelFileInBatches; import org.digijava.module.aim.action.dataimporter.dbentity.DataImporterConfig; import org.digijava.module.aim.action.dataimporter.dbentity.DataImporterConfigValues; import org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus; import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; import org.digijava.module.aim.action.dataimporter.util.ImportedFileUtil; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.ConstantsMap; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.isFileContentValid; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.isFileReadable; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.removeMapItem; + +import org.digijava.module.aim.action.dataimporter.util.ImporterConstants; +import org.digijava.module.aim.dbentity.AmpOrgGroup; import org.digijava.module.aim.form.DataImporterForm; +import org.digijava.module.aim.util.DbUtil; +import org.digijava.module.aim.util.DynLocationManagerUtil; +import org.digijava.module.aim.util.LocationUtil; +import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; +import org.digijava.module.aim.dbentity.AmpCategoryValueLocations; +import org.digijava.module.categorymanager.util.CategoryConstants; +import org.digijava.module.categorymanager.util.CategoryManagerUtil; import org.hibernate.Query; import org.hibernate.Session; import org.hibernate.type.StringType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.*; -import java.nio.file.Files; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDateTime; -import java.util.*; -import java.util.stream.Collectors; - -import static org.digijava.module.aim.action.dataimporter.ExcelImporter.processExcelFileInBatches; -import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.*; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.exceptions.CsvValidationException; public class DataImporter extends Action { static Logger logger = LoggerFactory.getLogger(DataImporter.class); @Override public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServletRequest request, HttpServletResponse response) throws Exception { - // List of fields - List fieldsInfo = getEntityFieldsInfo(); + // List of fields - map of original to translated + Map fieldsInfo = getEntityFieldsInfo(); request.setAttribute("fieldsInfo", fieldsInfo); + request.setAttribute("fieldsInfoList", new ArrayList<>(fieldsInfo.values())); List configNames= getConfigNames(); request.setAttribute("configNames", configNames); + List orgGroups = DbUtil.getAllOrgGroups(); + request.setAttribute("orgGroups", orgGroups); + request.setAttribute("activityStatuses", getActivityStatuses()); + List availableLocations = getAvailableLocations(); + request.setAttribute("availableLocations", availableLocations); + AmpCategoryValueLocations defaultLocation = DynLocationManagerUtil.getDefaultCountry(); + request.setAttribute("defaultLocationId", defaultLocation != null ? defaultLocation.getId() : null); DataImporterForm dataImporterForm = (DataImporterForm) form; if (Objects.equals(request.getParameter("action"), "configByName")) { @@ -77,208 +116,340 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } - if (Objects.equals(request.getParameter("action"), "uploadTemplate")) { + if (Objects.equals(request.getParameter("action"), "uploadTemplate")) { logger.info(" this is the action " + request.getParameter("action")); if (request.getParameter("uploadTemplate") != null) { logger.info(" this is the action " + request.getParameter("uploadTemplate")); - Set headersSet = new HashSet<>(); + response.setCharacterEncoding("UTF-8"); + + // Only clear in-memory column pairs when no existing named config is active. + // If the user has a configName selected, preserve its pairs so uploading a new + // template just lets them add more column mappings without losing existing ones. + String uploadConfigName = request.getParameter("configName"); + if (uploadConfigName == null || uploadConfigName.trim().isEmpty()) { + dataImporterForm.getColumnPairs().clear(); + } else { + // Reload from DB so the in-memory map reflects the latest saved state + Map savedPairs = getConfigByName(uploadConfigName.trim()); + dataImporterForm.getColumnPairs().clear(); + dataImporterForm.getColumnPairs().putAll(savedPairs); + } if (request.getParameter("fileType") != null) { InputStream fileInputStream = dataImporterForm.getTemplateFile().getInputStream(); - if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { - Workbook workbook = new XSSFWorkbook(fileInputStream); - int numberOfSheets = workbook.getNumberOfSheets(); - for (int i = 0; i < numberOfSheets; i++) { - Sheet sheet = workbook.getSheetAt(i); - Row headerRow = sheet.getRow(0); - Iterator cellIterator = headerRow.cellIterator(); - while (cellIterator.hasNext()) { - Cell cell = cellIterator.next(); - headersSet.add(cell.getStringCellValue()); + if (Objects.equals(request.getParameter("fileType"), "excel")) { + // Excel: return sheet names and columns per sheet for template configuration + List sheetNames = new ArrayList<>(); + Map> columnsBySheet = new HashMap<>(); + try (Workbook workbook = new XSSFWorkbook(fileInputStream)) { + int numberOfSheets = workbook.getNumberOfSheets(); + for (int i = 0; i < numberOfSheets; i++) { + Sheet sheet = workbook.getSheetAt(i); + String sheetName = sheet.getSheetName(); + sheetNames.add(sheetName); + List columns = new ArrayList<>(); + Row headerRow = sheet.getRow(0); + if (headerRow != null) { + Iterator cellIterator = headerRow.cellIterator(); + while (cellIterator.hasNext()) { + Cell cell = cellIterator.next(); + String val = cell.getStringCellValue(); + if (val != null && !val.trim().isEmpty()) { + columns.add(val.trim()); + } + } + } + columnsBySheet.put(sheetName, columns.stream().sorted().collect(Collectors.toList())); } - } - workbook.close(); - - + Map jsonResponse = new HashMap<>(); + jsonResponse.put("sheetNames", sheetNames); + jsonResponse.put("columnsBySheet", columnsBySheet); + response.setContentType("application/json"); + new ObjectMapper().writeValue(response.getWriter(), jsonResponse); + } else if (Objects.equals(request.getParameter("fileType"), "csv")) { + Set headersSet = new HashSet<>(); + try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(fileInputStream)).build()) { + String[] headers = reader.readNext(); + if (headers != null) { + headersSet.addAll(Arrays.asList(headers)); + } + } catch (IOException | CsvValidationException e) { + logger.error("An error occurred during extraction of headers.", e); + } + headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); + StringBuilder headers = new StringBuilder(); + headers.append(" \n"); + response.setContentType("text/html;charset=UTF-8"); + response.getWriter().write(headers.toString()); } else if (Objects.equals(request.getParameter("fileType"), "text")) { - CSVParser parser = new CSVParserBuilder().withSeparator(request.getParameter("dataSeparator").charAt(0)).build(); - + Set headersSet = new HashSet<>(); + String sep = request.getParameter("dataSeparator"); + char separator = (sep != null && !sep.isEmpty()) ? sep.charAt(0) : ','; + CSVParser parser = new CSVParserBuilder().withSeparator(separator).build(); try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(fileInputStream)).withCSVParser(parser).build()) { - String[] headers = reader.readNext(); // Read the first line which contains headers - + String[] headers = reader.readNext(); if (headers != null) { - // Print each header headersSet.addAll(Arrays.asList(headers)); } else { logger.info("File is empty or does not contain headers."); } } catch (IOException | CsvValidationException e) { - logger.error("An error occurred during extraction of headers.",e); + logger.error("An error occurred during extraction of headers.", e); } - + headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); + StringBuilder headers = new StringBuilder(); + headers.append(" \n"); + response.setContentType("text/html;charset=UTF-8"); + response.getWriter().write(headers.toString()); } } - headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); - StringBuilder headers = new StringBuilder(); - headers.append(" \n"); - response.setCharacterEncoding("UTF-8"); - response.setContentType("text/html;charset=UTF-8"); - - response.getWriter().write(headers.toString()); response.setHeader("updatedMap", ""); - - dataImporterForm.getColumnPairs().clear(); - } return null; } - if (Objects.equals(request.getParameter("action"), "addField")) { - logger.info(" this is the action " + request.getParameter("action")); + if (Objects.equals(request.getParameter("action"), "addField")) { + logger.info(" this is the action " + request.getParameter("action")); - String columnName = request.getParameter("columnName"); - String selectedField = request.getParameter("selectedField"); - dataImporterForm.getColumnPairs().put(columnName, selectedField); - logger.info("Column Pairs:" + dataImporterForm.getColumnPairs()); + String columnName = request.getParameter("columnName"); + String selectedField = request.getParameter("selectedField"); + String configName = request.getParameter("configName"); + Map columnPairs = dataImporterForm.getColumnPairs(); - ObjectMapper objectMapper = new ObjectMapper(); - String json = objectMapper.writeValueAsString(dataImporterForm.getColumnPairs()); + if (configName != null && !configName.trim().isEmpty()) { + addColumnPairToConfig(configName.trim(), columnName, selectedField); + columnPairs = getConfigByName(configName.trim()); + } else { + columnPairs.put(columnName, selectedField); + } + logger.info("Column Pairs:" + columnPairs); - // Send response - response.setContentType("application/json"); - response.getWriter().write(json); - response.setCharacterEncoding("UTF-8"); + ObjectMapper objectMapper = new ObjectMapper(); + String json = objectMapper.writeValueAsString(columnPairs); - return null; + response.setContentType("application/json"); + response.getWriter().write(json); + response.setCharacterEncoding("UTF-8"); + + return null; + + } + + + if (Objects.equals(request.getParameter("action"), "removeField")) { + logger.info(" this is the action " + request.getParameter("action")); + String columnName = request.getParameter("columnName"); + String selectedField = request.getParameter("selectedField"); + String configName = request.getParameter("configName"); + Map columnPairs = dataImporterForm.getColumnPairs(); + + if (configName != null && !configName.trim().isEmpty()) { + removeColumnPairFromConfig(configName.trim(), columnName); + columnPairs = getConfigByName(configName.trim()); + } else { + removeMapItem(columnPairs, columnName, selectedField); } + logger.info("Column Pairs:" + columnPairs); + ObjectMapper objectMapper = new ObjectMapper(); + String json = objectMapper.writeValueAsString(columnPairs); - if (Objects.equals(request.getParameter("action"), "removeField")) { - logger.info(" this is the action " + request.getParameter("action")); + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + response.getWriter().write(json); - String columnName = request.getParameter("columnName"); - String selectedField = request.getParameter("selectedField"); - dataImporterForm.getColumnPairs().put(columnName, selectedField); - removeMapItem(dataImporterForm.getColumnPairs(), columnName, selectedField); - logger.info("Column Pairs:" + dataImporterForm.getColumnPairs()); + return null; - ObjectMapper objectMapper = new ObjectMapper(); - String json = objectMapper.writeValueAsString(dataImporterForm.getColumnPairs()); + } - // Send response + if (Objects.equals(request.getParameter("action"), "getDataFileSheets")) { + logger.info("This is the action getDataFileSheets"); + if (dataImporterForm.getDataFile() == null || dataImporterForm.getDataFile().getFileSize() == 0) { + response.setStatus(400); response.setContentType("application/json"); - response.setCharacterEncoding("UTF-8"); - response.getWriter().write(json); - + response.getWriter().write("{\"error\":\"No file provided\"}"); return null; - } - - if (Objects.equals(request.getParameter("action"), "uploadDataFile")) { - logger.info("This is the action " + request.getParameter("action")); - Instant start = Instant.now(); - String fileName = dataImporterForm.getDataFile().getFileName(); - String tempDirPath = System.getProperty("java.io.tmpdir"); - File tempDir = new File(tempDirPath); - if (!tempDir.exists()) { - tempDir.mkdirs(); + String fileType = request.getParameter("fileType"); + if (!Objects.equals(fileType, "excel")) { + response.setContentType("application/json"); + new ObjectMapper().writeValue(response.getWriter(), Collections.emptyList()); + return null; + } + List sheetNames = new ArrayList<>(); + try (InputStream is = dataImporterForm.getDataFile().getInputStream(); + Workbook workbook = new XSSFWorkbook(is)) { + int n = workbook.getNumberOfSheets(); + for (int i = 0; i < n; i++) { + sheetNames.add(workbook.getSheetAt(i).getSheetName()); } - String tempFilePath = tempDirPath + File.separator + fileName; - try (InputStream inputStream = dataImporterForm.getDataFile().getInputStream(); - FileOutputStream outputStream = new FileOutputStream(tempFilePath)) { - byte[] buffer = new byte[8192]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } + } + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + new ObjectMapper().writeValue(response.getWriter(), sheetNames); + return null; + } + + if (Objects.equals(request.getParameter("action"), "uploadDataFile")) { + logger.info("This is the action " + request.getParameter("action")); + Instant start = Instant.now(); + String fileName = dataImporterForm.getDataFile().getFileName(); + String tempDirPath = System.getProperty("java.io.tmpdir"); + File tempDir = new File(tempDirPath); + if (!tempDir.exists()) { + tempDir.mkdirs(); + } + String tempFilePath = tempDirPath + File.separator + fileName; + try (InputStream inputStream = dataImporterForm.getDataFile().getInputStream(); + FileOutputStream outputStream = new FileOutputStream(tempFilePath)) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); } + } - // Check if the file is readable and has correct content - File tempFile = new File(tempFilePath); - List similarFiles = ImportedFileUtil.getSimilarFiles(tempFile); - if (similarFiles != null && !similarFiles.isEmpty()) { - for (ImportedFilesRecord similarFilesRecord : similarFiles) { - logger.info("Similar file: " + similarFilesRecord); - if (similarFilesRecord.getImportStatus().equals(ImportStatus.IN_PROGRESS)) { - response.setHeader("errorMessage", "You have a similar file in progress. Please try again later."); - response.setStatus(400); - return mapping.findForward("importData"); - } + // Check if the file is readable and has correct content + File tempFile = new File(tempFilePath); + List similarFiles = ImportedFileUtil.getSimilarFiles(tempFile); + if (similarFiles != null && !similarFiles.isEmpty()) { + for (ImportedFilesRecord similarFilesRecord : similarFiles) { + logger.info("Similar file: " + similarFilesRecord); + if (similarFilesRecord.getImportStatus().equals(ImportStatus.IN_PROGRESS)) { + response.setHeader("errorMessage", "You have a similar file in progress. Please try again later."); + response.setStatus(400); + return mapping.findForward("importData"); } } - if (dataImporterForm.getColumnPairs().isEmpty() ||(!dataImporterForm.getColumnPairs().containsValue("Project Title") && !dataImporterForm.getColumnPairs().containsValue("Project Code"))) { + } + // Resolve which config to use: saved config by name (from load/edit) or form's column pairs + String existingConfig = request.getParameter("existingConfig"); + String configNameParam = request.getParameter("configName"); + String configNameToUse = (configNameParam != null && !configNameParam.trim().isEmpty()) + ? configNameParam.trim() + : (existingConfig != null && !existingConfig.isEmpty() && !"0".equals(existingConfig) && !"1".equals(existingConfig) ? existingConfig.trim() : null); + Map columnPairsToUse = (configNameToUse != null) + ? getConfigByName(configNameToUse) + : dataImporterForm.getColumnPairs(); + + if (columnPairsToUse.isEmpty() || (!columnPairsToUse.containsValue(ImporterConstants.PROJECT_TITLE) && !columnPairsToUse.containsValue(ImporterConstants.PROJECT_CODE))) { + response.setHeader("errorMessage", "You must have at least the 'Project Title' or 'Project Code' column in your config."); + response.setStatus(400); + return mapping.findForward("importData"); + } + if (!isFileReadable(tempFile) || !isFileContentValid(tempFile)) { + // Handle invalid file + logger.error("Invalid file or content."); + response.setHeader("errorMessage", "Unable to parse the file. Please check the file format/content and try again."); + response.setStatus(400); + return mapping.findForward("importData"); + + } else { + logger.info("Existing configuration: {}", existingConfig); + if (configNameToUse == null) { + saveImportConfig(request, fileName, dataImporterForm.getColumnPairs()); + } - response.setHeader("errorMessage", "You must have at least the 'Project Title' or 'Project Code' column in your config."); + int res = 0; + ImportedFilesRecord importedFilesRecord = ImportedFileUtil.saveFile(tempFile, fileName); + logger.info("Saved file record: {}",importedFilesRecord); + boolean isInternal= dataImporterForm.isInternal(); + boolean skipExisting = dataImporterForm.isSkipExisting(); + boolean validateActivities = dataImporterForm.isValidateActivities(); + boolean addDisbursementForCommitment = dataImporterForm.isAddDisbursementForCommitment(); + boolean skipRecordsWithoutTransactions = dataImporterForm.isSkipRecordsWithoutTransactions(); + boolean createMissingOrgs = dataImporterForm.isCreateMissingOrgs(); + boolean createMissingSectors = dataImporterForm.isCreateMissingSectors(); + boolean createMissingOrgGroups = dataImporterForm.isCreateMissingOrgGroups(); + Long orgGroupId = dataImporterForm.getOrgGroupId(); + Long defaultActivityStatusId = dataImporterForm.getDefaultActivityStatusId(); + Long defaultLocationId = dataImporterForm.getDefaultLocationId(); + logger.info("Internal: "+ isInternal); + logger.info("Skip existing: "+ skipExisting); + logger.info("Validate activities: "+ validateActivities); + logger.info("Add disbursement for commitment: "+ addDisbursementForCommitment); + logger.info("Skip records without transactions: " + skipRecordsWithoutTransactions); + logger.info("Create missing orgs: "+ createMissingOrgs); + logger.info("Create missing sectors: {}", createMissingSectors); + logger.info("Create missing org groups: " + createMissingOrgGroups); + logger.info("Org group id: "+ orgGroupId); + logger.info("Default activity status id: {}", defaultActivityStatusId); + logger.info("Default location id: {}", defaultLocationId); + if (createMissingOrgs && orgGroupId == null && !createMissingOrgGroups + && !columnPairsToUse.containsValue(ImporterConstants.ORG_GROUP)) { + response.setHeader("errorMessage", + "Creating missing organizations requires a fallback Organization Group, the 'Create missing org groups' option, or an 'Organization Group' column mapping."); response.setStatus(400); return mapping.findForward("importData"); } - if (!isFileReadable(tempFile) || !isFileContentValid(tempFile)) { - // Handle invalid file - logger.error("Invalid file or content."); - response.setHeader("errorMessage", "Unable to parse the file. Please check the file format/content and try again."); + if (isInternal) { + columnPairsToUse = new HashMap<>(columnPairsToUse); + columnPairsToUse.put("Donor Agency", "Donor Agency"); + } + if (!columnPairsToUse.containsValue(ImporterConstants.PROJECT_LOCATION) + && defaultLocationId == null) { + response.setHeader("errorMessage", "Please select a fallback location when no 'Project Location' column is mapped."); response.setStatus(400); return mapping.findForward("importData"); - - } else { - // Proceed with processing the file - String existingConfig = request.getParameter("existingConfig"); - logger.info("Existing configuration: {}",existingConfig); - if (!Objects.equals(existingConfig, "1")) { - saveImportConfig(request, fileName, dataImporterForm.getColumnPairs()); - } - - int res = 0; - ImportedFilesRecord importedFilesRecord = ImportedFileUtil.saveFile(tempFile, fileName); - logger.info("Saved file record: {}",importedFilesRecord); - boolean isInternal= dataImporterForm.isInternal(); - logger.info("Internal: "+ isInternal); - if (isInternal) - { - dataImporterForm.getColumnPairs().put("Donor Agency", "Donor Agency"); - } - logger.info("Configuration"+ dataImporterForm.getColumnPairs()); + } + logger.info("Configuration: {}", columnPairsToUse); + try { if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { - - // Process the file in batches - res = processExcelFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs(), isInternal); + String dataSheetChoice = request.getParameter("dataSheetChoice"); + String dataSheetName = request.getParameter("dataSheetName"); + boolean useSpecificSheet = "sheet".equals(dataSheetChoice) && dataSheetName != null && !dataSheetName.trim().isEmpty(); + res = processExcelFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal, skipExisting, useSpecificSheet ? dataSheetName : null, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); } else if ( Objects.equals(request.getParameter("fileType"), "text")) { - res=TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs(), isInternal); - } - if (res != 1) { - // Handle error - logger.info("Error processing file " + tempFile); - response.setHeader("errorMessage", "Unable to parse the file. Please check the file format/content and try again."); - response.setStatus(400); - return mapping.findForward("importData"); + res = TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); } + } catch (Exception e) { + ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED); + throw e; + } finally { + Instant finish = Instant.now(); + long timeElapsedMillis = Duration.between(start, finish).toMillis(); + ImportedFileUtil.updateFileProcessingTime(importedFilesRecord, timeElapsedMillis); + long minutes = timeElapsedMillis / 60000; + long seconds = (timeElapsedMillis % 60000) / 1000; + logger.info("Time Elapsed: " + minutes + "m " + seconds + "s"); + } + if (res != 1) { + // Handle error + logger.info("Error processing file " + tempFile); + ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED); + response.setHeader("errorMessage", "Unable to parse the file. Please check the file format/content and try again."); + response.setStatus(400); + return mapping.findForward("importData"); + } - // Clean up - ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.SUCCESS); - Files.delete(tempFile.toPath()); - logger.info("Cache map size: " + ConstantsMap.size()); - ConstantsMap.clear(); - logger.info("File path is " + tempFilePath + " and size is " + tempFile.length() / (1024 * 1024) + " mb"); - logger.info("Start time: " + start); - Instant finish = Instant.now(); - long timeElapsed = Duration.between(start, finish).toMillis(); - logger.info("Time Elapsed: " + timeElapsed); + // Clean up + ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.SUCCESS); + Files.delete(tempFile.toPath()); + logger.info("Cache map size: " + ConstantsMap.size()); + ConstantsMap.clear(); + logger.info("File path is " + tempFilePath + " and size is " + tempFile.length() / (1024 * 1024) + " mb"); + logger.info("Start time: " + start); - // Send response - response.setHeader("updatedMap", ""); - dataImporterForm.getColumnPairs().clear(); - } - return null; + // Send response + response.setHeader("updatedMap", ""); + dataImporterForm.getColumnPairs().clear(); } + return null; + } - return mapping.findForward("importData"); + return mapping.findForward("importData"); } private static List getConfigNames() { @@ -295,31 +466,76 @@ private static List getConfigNames() } + private static List getAvailableLocations() { + return LocationUtil.getAllCountriesAndRegions().stream() + .filter(Objects::nonNull) + .filter(location -> !location.isSoftDeleted()) + .sorted(Comparator.comparing(AmpCategoryValueLocations::getHierarchicalName, String.CASE_INSENSITIVE_ORDER)) + .collect(Collectors.toList()); + } + private static Map getConfigByName(String configName) { logger.info("Getting import config for configName: {}", configName); Session session = PersistenceManager.getRequestDBSession(); Map configValues = new HashMap<>(); + // Query DataImporterConfigValues directly to bypass Hibernate first-level cache + // (querying through the parent entity's collection returns stale cached results + // when new pairs were added in the same session via addColumnPairToConfig) + String hql = "SELECT cv.configKey, cv.configValue FROM " + + DataImporterConfigValues.class.getName() + + " cv WHERE cv.dataImporterConfig.configName = :configName"; + Query query = session.createQuery(hql); + query.setParameter("configName", configName, StringType.INSTANCE); + List rows = query.list(); + logger.info("Config rows found for '{}': {}", configName, rows.size()); + for (Object[] row : rows) { + configValues.put((String) row[0], (String) row[1]); + } + return configValues; + } - String hql = "FROM DataImporterConfig WHERE configName = :configName"; - Query query = session.createQuery(hql); - query.setParameter("configName", configName, StringType.INSTANCE); - query.setMaxResults(1); - - List resultList = query.list(); - logger.info("Configs found: {}",resultList); - + private static DataImporterConfig getConfigEntityByName(String configName) { + Session session = PersistenceManager.getRequestDBSession(); + String hql = "FROM DataImporterConfig WHERE configName = :configName"; + Query query = session.createQuery(hql); + query.setParameter("configName", configName, StringType.INSTANCE); + query.setMaxResults(1); + List list = query.list(); + return list.isEmpty() ? null : list.get(0); + } - if (!resultList.isEmpty()) { - Set values = resultList.get(0).getConfigValues(); - logger.info("Config Values found: {}",values); + private static void addColumnPairToConfig(String configName, String columnName, String selectedField) { + DataImporterConfig config = getConfigEntityByName(configName); + if (config == null) { + logger.warn("Config not found for name: {}", configName); + return; + } + Session session = PersistenceManager.getRequestDBSession(); + DataImporterConfigValues cv = new DataImporterConfigValues(columnName, selectedField, config); + session.save(cv); + config.getConfigValues().add(cv); + session.flush(); + } - if (!values.isEmpty()) - { - values.forEach(value-> configValues.put(value.getConfigKey(),value.getConfigValue())); - } + private static void removeColumnPairFromConfig(String configName, String columnName) { + DataImporterConfig config = getConfigEntityByName(configName); + if (config == null) { + logger.warn("Config not found for name: {}", configName); + return; + } + DataImporterConfigValues toRemove = null; + for (DataImporterConfigValues v : config.getConfigValues()) { + if (columnName.equals(v.getConfigKey())) { + toRemove = v; + break; } - - return configValues; + } + if (toRemove != null) { + config.getConfigValues().remove(toRemove); + Session session = PersistenceManager.getRequestDBSession(); + session.delete(toRemove); + session.flush(); + } } public static void saveImportConfig(HttpServletRequest request, String fileName, Map config) { @@ -384,40 +600,84 @@ public static void saveImportConfig(HttpServletRequest request, String fileName, - private List getEntityFieldsInfo() { + private Map getEntityFieldsInfo() { List fieldsInfos = new ArrayList<>(); - fieldsInfos.add("Project Title"); - fieldsInfos.add("Project Code"); - fieldsInfos.add("Project Description"); - fieldsInfos.add("Primary Sector"); - fieldsInfos.add("Secondary Sector"); - fieldsInfos.add("Project Location"); - fieldsInfos.add("Project Start Date"); - fieldsInfos.add("Project End Date"); - fieldsInfos.add("Donor Agency"); - fieldsInfos.add("Exchange Rate"); - fieldsInfos.add("Donor Agency Code"); - fieldsInfos.add("Responsible Organization"); - fieldsInfos.add("Responsible Organization Code"); - fieldsInfos.add("Executing Agency"); - fieldsInfos.add("Implementing Agency"); - fieldsInfos.add("Actual Disbursement"); - fieldsInfos.add("Actual Commitment"); - fieldsInfos.add("Actual Expenditure"); - fieldsInfos.add("Planned Disbursement"); - fieldsInfos.add("Planned Commitment"); - fieldsInfos.add("Planned Expenditure"); - fieldsInfos.add("Funding Item"); - fieldsInfos.add("Transaction Date"); - fieldsInfos.add("Financing Instrument"); - fieldsInfos.add("Type Of Assistance"); - fieldsInfos.add("Secondary Subsector"); - fieldsInfos.add("Primary Subsector"); - fieldsInfos.add("Currency"); - fieldsInfos.add("Component Name"); - fieldsInfos.add("Component Code"); - fieldsInfos.add("Beneficiary Agency"); - return fieldsInfos.stream().sorted().collect(Collectors.toList()); + fieldsInfos.add(ImporterConstants.PROJECT_TITLE); + fieldsInfos.add(ImporterConstants.PROJECT_CODE); + fieldsInfos.add(ImporterConstants.OBJECTIVE); + fieldsInfos.add(ImporterConstants.PROJECT_DESCRIPTION); + fieldsInfos.add(ImporterConstants.PRIMARY_SECTOR); + fieldsInfos.add(ImporterConstants.SECONDARY_SECTOR); + fieldsInfos.add(ImporterConstants.PROJECT_LOCATION); + fieldsInfos.add(ImporterConstants.PROJECT_START_DATE); + fieldsInfos.add(ImporterConstants.PROJECT_END_DATE); + fieldsInfos.add(ImporterConstants.DONOR_AGENCY); + fieldsInfos.add(ImporterConstants.EXCHANGE_RATE); + fieldsInfos.add(ImporterConstants.DONOR_AGENCY_CODE); + fieldsInfos.add(ImporterConstants.ORG_GROUP); + fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION); + fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE); + fieldsInfos.add(ImporterConstants.EXECUTING_AGENCY); + fieldsInfos.add(ImporterConstants.IMPLEMENTING_AGENCY); + fieldsInfos.add(ImporterConstants.ACTUAL_DISBURSEMENT); + fieldsInfos.add(ImporterConstants.ACTUAL_COMMITMENT); + fieldsInfos.add(ImporterConstants.ACTUAL_EXPENDITURE); + fieldsInfos.add(ImporterConstants.PLANNED_DISBURSEMENT); + fieldsInfos.add(ImporterConstants.PLANNED_COMMITMENT); + fieldsInfos.add(ImporterConstants.PLANNED_EXPENDITURE); + fieldsInfos.add(ImporterConstants.TRANSACTION_AMOUNT); + fieldsInfos.add(ImporterConstants.MEASURE_TYPE); + fieldsInfos.add(ImporterConstants.TRANSACTION_DATE); + fieldsInfos.add(ImporterConstants.FINANCING_INSTRUMENT); + fieldsInfos.add(ImporterConstants.TYPE_OF_ASSISTANCE); + fieldsInfos.add(ImporterConstants.SECONDARY_SUBSECTOR); + fieldsInfos.add(ImporterConstants.PRIMARY_SUBSECTOR); + fieldsInfos.add(ImporterConstants.CURRENCY); + fieldsInfos.add(ImporterConstants.COMPONENT_NAME); + fieldsInfos.add(ImporterConstants.COMPONENT_CODE); + fieldsInfos.add(ImporterConstants.BENEFICIARY_AGENCY); + fieldsInfos.add(ImporterConstants.PROJECT_STATUS); + fieldsInfos.add(ImporterConstants.PROCUREMENT_SYSTEM); + // Indicator columns for M&E import + fieldsInfos.add(ImporterConstants.INDICATOR_NAME); + fieldsInfos.add(ImporterConstants.PROGRAM_NAME); + fieldsInfos.add(ImporterConstants.INDICATOR_LOCATION); + fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE); + fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE_DATE); + fieldsInfos.add(ImporterConstants.REVISED_BASE_VALUE); + fieldsInfos.add(ImporterConstants.REVISED_BASE_VALUE_DATE); + fieldsInfos.add(ImporterConstants.ORIGINAL_TARGET_VALUE); + fieldsInfos.add(ImporterConstants.ORIGINAL_TARGET_VALUE_DATE); + fieldsInfos.add(ImporterConstants.REVISED_TARGET_VALUE); + fieldsInfos.add(ImporterConstants.REVISED_TARGET_VALUE_DATE); + fieldsInfos.add(ImporterConstants.ACTUAL_VALUE); + fieldsInfos.add(ImporterConstants.ACTUAL_VALUE_DATE); + fieldsInfos.add(ImporterConstants.UNIT_OF_MEASURE); + + // Create map of original field names to translated field names + Map fieldMap = new LinkedHashMap<>(); + for (String field : fieldsInfos) { + String translated = org.digijava.kernel.translator.TranslatorWorker.translateText(field); + fieldMap.put(field, translated); + + } + + + return fieldMap.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .collect(Collectors.toMap(Map.Entry::getKey, + Map.Entry::getValue, + (e1, e2) -> e1, + LinkedHashMap::new)); + } + + private List getActivityStatuses() { + List activityStatuses = new ArrayList<>( + CategoryManagerUtil.getAmpCategoryValueCollectionByKey(CategoryConstants.ACTIVITY_STATUS_KEY)); + activityStatuses.sort(Comparator + .comparing(AmpCategoryValue::getIndex, Comparator.nullsLast(Integer::compareTo)) + .thenComparing(AmpCategoryValue::getValue, String.CASE_INSENSITIVE_ORDER)); + return activityStatuses; } } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java index d41c5e64c98..3a7b8797d4e 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java @@ -1,6 +1,17 @@ package org.digijava.module.aim.action.dataimporter; -import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.File; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import javax.servlet.http.HttpServletRequest; + import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.openxml4j.exceptions.InvalidOperationException; import org.apache.poi.ss.usermodel.Cell; @@ -15,24 +26,18 @@ import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; import org.digijava.module.aim.action.dataimporter.model.Funding; import org.digijava.module.aim.action.dataimporter.model.ImportDataModel; +import org.digijava.module.aim.action.dataimporter.util.ImporterConstants; import org.digijava.module.aim.action.dataimporter.util.ImportedFileUtil; +import org.digijava.module.aim.action.dataimporter.util.ImporterUtil; import org.digijava.module.aim.dbentity.AmpActivityVersion; +import org.digijava.module.categorymanager.util.CategoryConstants; import org.digijava.module.aim.util.FeaturesUtil; import org.digijava.module.aim.util.TeamMemberUtil; import org.hibernate.Session; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServletRequest; -import java.io.File; -import java.io.IOException; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import com.fasterxml.jackson.core.JsonProcessingException; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.*; @@ -41,26 +46,41 @@ public class ExcelImporter { private static final int BATCH_SIZE = 1000; public static int processExcelFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal) { - int res=0; + return processExcelFileInBatches(importedFilesRecord, file, request, config, isInternal, false, null, false, false, null, false, false, false, false, null, null); + } + + public static int processExcelFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal, boolean skipExisting, String sheetNameToProcess, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId) { + int res = 0; ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.IN_PROGRESS); try (Workbook workbook = new XSSFWorkbook(file)) { int numberOfSheets = workbook.getNumberOfSheets(); logger.info("Number of sheets: {}", numberOfSheets); - // Process each sheet in the workbook - for (int i = 0; i < numberOfSheets; i++) { - logger.info("Sheet number: {}", i); - Sheet sheet = workbook.getSheetAt(i); + if (sheetNameToProcess != null && !sheetNameToProcess.trim().isEmpty()) { + Sheet sheet = workbook.getSheet(sheetNameToProcess); + if (sheet == null) { + logger.error("Sheet not found: {}", sheetNameToProcess); + ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED); + return 0; + } if (isInternal) { addDonorAgencyColumn(sheet, FeaturesUtil.getGlobalSettingValue("Internal Ecowas Donor")); - } - - processSheetInBatches(sheet, request,config, importedFilesRecord); + processSheetInBatches(sheet, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); + } else { + // Process each sheet in the workbook + for (int i = 0; i < numberOfSheets; i++) { + logger.info("Sheet number: {}", i); + Sheet sheet = workbook.getSheetAt(i); + if (isInternal) { + addDonorAgencyColumn(sheet, FeaturesUtil.getGlobalSettingValue("Internal Ecowas Donor")); + } + processSheetInBatches(sheet, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); + } } logger.info("Closing the workbook..."); - res =1; + res = 1; } catch (IOException e) { ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED); logger.error("Error processing Excel file: {}", e.getMessage(), e); @@ -99,7 +119,7 @@ private static void addDonorAgencyColumn(Sheet sheet, String donorAgencyValue) { } - public static void processSheetInBatches(Sheet sheet, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord) throws JsonProcessingException { + public static void processSheetInBatches(Sheet sheet, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId) throws JsonProcessingException { // Get the number of rows in the sheet int rowCount = sheet.getPhysicalNumberOfRows(); logger.info("There are {} rows in sheet {} " , rowCount, sheet.getSheetName()); @@ -121,137 +141,261 @@ public static void processSheetInBatches(Sheet sheet, HttpServletRequest request } // Process the batch - processBatch(batch, sheet, request,config, importedFilesRecord); + processBatch(batch, sheet, request,config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); } } - public static void processBatch(List batch,Sheet sheet, HttpServletRequest request, Map config, ImportedFilesRecord importedFilesRecord) throws JsonProcessingException { + public static void processBatch(List batch,Sheet sheet, HttpServletRequest request, Map config, ImportedFilesRecord importedFilesRecord, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId) throws JsonProcessingException { // Process the batch of rows SessionUtil.extendSessionIfNeeded(request); - Session session = PersistenceManager.getRequestDBSession(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); for (Row row : batch) { if (row != null) { + final Row rowRef = row; ImportedProject importedProject = new ImportedProject(); importedProject.setImportedFilesRecord(importedFilesRecord); List fundings = new ArrayList<>(); ImportDataModel importDataModel = new ImportDataModel(); importDataModel.setModified_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); - importDataModel.setCreated_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); + // created_by is set in ensureCreatedBySet when building the API map (correct for new vs existing) importDataModel.setTeam(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeam().getAmpTeamId()); - importDataModel.setIs_draft(true); + importDataModel.setIs_draft(!validateActivities); OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); importDataModel.setCreation_date(now.format(formatter)); - setStatus(importDataModel); - int componentCodeColumn = getColumnIndexByName(sheet, getKey(config, "Component Code")); - String componentCode = componentCodeColumn >= 0 ? getStringValueFromCell(row.getCell(componentCodeColumn),true) : null; + int componentCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.COMPONENT_CODE)); + String componentCode = componentCodeColumn >= 0 ? getStringValueFromCell(rowRef.getCell(componentCodeColumn),true) : null; - int componentNameColumn = getColumnIndexByName(sheet, getKey(config, "Component Name")); - String componentName = componentNameColumn >= 0 ? getStringValueFromCell(row.getCell(componentNameColumn),true): null; + int componentNameColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.COMPONENT_NAME)); + String componentName = componentNameColumn >= 0 ? getStringValueFromCell(rowRef.getCell(componentNameColumn),true): null; - int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, "Donor Agency Code")); - String donorAgencyCode = donorAgencyCodeColumn >= 0 ? getStringValueFromCell(row.getCell(donorAgencyCodeColumn),true) : null; + int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.DONOR_AGENCY_CODE)); + String donorAgencyCode = donorAgencyCodeColumn >= 0 ? getStringValueFromCell(rowRef.getCell(donorAgencyCodeColumn),true) : null; - int responsibleOrgCodeColumn = getColumnIndexByName(sheet, getKey(config, "Responsible Organization Code")); - String responsibleOrgCode = responsibleOrgCodeColumn >= 0 ? getStringValueFromCell(row.getCell(responsibleOrgCodeColumn),true) : null; + int responsibleOrgCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE)); + String responsibleOrgCode = responsibleOrgCodeColumn >= 0 ? getStringValueFromCell(rowRef.getCell(responsibleOrgCodeColumn),true) : null; - int primarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, "Primary Subsector")); - String primarySubSector = primarySubSectorColumn >= 0 ? getStringValueFromCell(row.getCell(primarySubSectorColumn),true) : null; + int primarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PRIMARY_SUBSECTOR)); + String primarySubSector = primarySubSectorColumn >= 0 ? getStringValueFromCell(rowRef.getCell(primarySubSectorColumn),true) : null; - int secondarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, "Secondary Subsector")); - String secondarySubSector = secondarySubSectorColumn >= 0 ? getStringValueFromCell(row.getCell(secondarySubSectorColumn),true) : null; + int secondarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.SECONDARY_SUBSECTOR)); + String secondarySubSector = secondarySubSectorColumn >= 0 ? getStringValueFromCell(rowRef.getCell(secondarySubSectorColumn),true) : null; - int projectCodeColumn = getColumnIndexByName(sheet, getKey(config, "Project Code")); - String projectCode = projectCodeColumn >= 0 ? getStringValueFromCell(row.getCell(projectCodeColumn),false) : ""; + int projectCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PROJECT_CODE)); + String projectCode = projectCodeColumn >= 0 ? getStringValueFromCell(rowRef.getCell(projectCodeColumn),false) : ""; importDataModel.setProject_code(projectCode); - int projectTitleColumn = getColumnIndexByName(sheet, getKey(config, "Project Title")); - String projectTitle = projectTitleColumn >= 0 ? getStringValueFromCell(row.getCell(projectTitleColumn),false) : ""; + int projectTitleColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PROJECT_TITLE)); + String projectTitle = projectTitleColumn >= 0 ? getStringValueFromCell(rowRef.getCell(projectTitleColumn),false) : ""; importDataModel.setProject_title(projectTitle); + int objectiveColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.OBJECTIVE)); + String objective = objectiveColumn >= 0 ? getStringValueFromCell(rowRef.getCell(objectiveColumn),false) : null; + importDataModel.setObjective(objective); - int projectDescColumn = getColumnIndexByName(sheet, getKey(config, "Project Description")); - String projectDesc = projectDescColumn >= 0 ? getStringValueFromCell(row.getCell(projectDescColumn),false) : null; + int projectDescColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PROJECT_DESCRIPTION)); + String projectDesc = projectDescColumn >= 0 ? getStringValueFromCell(rowRef.getCell(projectDescColumn),false) : null; importDataModel.setDescription(projectDesc); - AmpActivityVersion existing = existingActivity(projectTitle, projectCode, session); - Long responsibleOrgId = null; - -// if (existing!=null && SKIP_EXISTING) -// { -// logger.info("Instructed to skip existing activities"); -// importedProject.setImportStatus(ImportStatus.SKIPPED); -// continue; -// } - - logger.info("Row Number: {}, Sheet Name: {}", row.getRowNum(), sheet.getSheetName()); - for (Map.Entry entry : config.entrySet()) { - Funding fundingItem = new Funding(); - - int columnIndex = getColumnIndexByName(sheet, entry.getKey()); - - if (columnIndex >= 0) { - Cell cell = row.getCell(columnIndex); - switch (entry.getValue()) { - case "Project Location": - updateLocations(importDataModel,Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(),session); - break; - case "Primary Sector": - updateSectors(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), session, true, primarySubSector); - break; - case "Secondary Sector": - updateSectors(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), session, false, secondarySubSector); - break; - case "Donor Agency": - logger.info("Getting donor"); - updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), donorAgencyCode, session, "donor"); - break; - case "Responsible Organization": - responsibleOrgId = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, "responsibleOrg"); - break; - case "Beneficiary Agency": - responsibleOrgId = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, "beneficiaryAgency"); - break; - case "Funding Item": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, true, true, false,"Actual", fundingItem, existing); - break; - case "Planned Commitment": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, true, false,false, "Planned", fundingItem, existing); - break; - case "Planned Disbursement": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, true, false,"Planned", fundingItem, existing); - break; - case "Planned Expenditure": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, false,true, "Planned", fundingItem, existing); - break; - case "Actual Commitment": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, true, false, false,"Actual", fundingItem, existing); - break; - case "Actual Disbursement": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, true, false,"Actual", fundingItem, existing); - break; - case "Actual Expenditure": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, false,true, "Actual", fundingItem, existing); - break; - case "Reporting Date": - default: - logger.error("Unexpected value: " + entry.getValue()); - break; + String importedOrgGroupName; + if (config.containsValue(ImporterConstants.ORG_GROUP)) { + String configuredOrgGroupName = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.ORG_GROUP); + if (configuredOrgGroupName != null && !configuredOrgGroupName.trim().isEmpty()) { + importedOrgGroupName = configuredOrgGroupName.trim(); + } else { + importedOrgGroupName = null; + } + } else { + importedOrgGroupName = null; + } + // Use holder arrays to capture values from lambda (for effectively final requirement) + final Long[] existingActivityIdHolder = new Long[1]; // Store only the ID, not the entity + final Long[] responsibleOrgIdHolder = new Long[1]; + + // Phase 1: Data preparation - use transaction for reading/preparing data ONLY + try { + PersistenceManager.inTransaction(() -> { + Session session = PersistenceManager.getRequestDBSession(); + + if (config.containsValue(ImporterConstants.PROJECT_STATUS)) { + String projectStatusStr = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.PROJECT_STATUS); + if (projectStatusStr != null && !projectStatusStr.trim().isEmpty()) { + Long statusId = ImporterUtil.getOrCreateActivityStatusCategoryValue(projectStatusStr.trim(), session); + if (statusId != null) { + importDataModel.setActivity_status(statusId); + } + } + } + if (config.containsValue(ImporterConstants.PROCUREMENT_SYSTEM)) { + String procurementSystemStr = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.PROCUREMENT_SYSTEM); + if (procurementSystemStr != null && !procurementSystemStr.trim().isEmpty()) { + Long procId = ImporterUtil.getCategoryValueByName(CategoryConstants.PROCUREMENT_SYSTEM_KEY, procurementSystemStr.trim(), session); + if (procId != null) { + importDataModel.setProcurement_system(procId); + } + } + } + setStatus(importDataModel, validateActivities, defaultActivityStatusId); + + AmpActivityVersion existing = existingActivity(projectTitle, projectCode, session); + existingActivityIdHolder[0] = existing != null ? existing.getAmpActivityId() : null; + if (existing != null && skipExisting) { + logger.info("Skipping existing activity: {}", existing.getAmpActivityId()); + importedProject.setImportStatus(ImportStatus.SKIPPED); + return; } - + logger.info("Row Number: {}, Sheet Name: {}", rowRef.getRowNum(), sheet.getSheetName()); + for (Map.Entry entry : config.entrySet()) { + + int columnIndex = getColumnIndexByName(sheet, entry.getKey()); + + if (columnIndex >= 0) { + Cell cell = rowRef.getCell(columnIndex); + switch (entry.getValue()) { + case ImporterConstants.PROJECT_LOCATION: + updateLocations(importDataModel,Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(),session); + break; + case ImporterConstants.PRIMARY_SECTOR: + updateSectors(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), + session, true, primarySubSector, createMissingSectors, + ImporterConstants.PRIMARY_SECTOR); + break; + case ImporterConstants.SECONDARY_SECTOR: + updateSectors(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), + session, false, secondarySubSector, createMissingSectors, + ImporterConstants.SECONDARY_SECTOR); + break; + case ImporterConstants.DONOR_AGENCY: + logger.info("Getting donor"); + updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), donorAgencyCode, session, ImporterConstants.ORG_TYPE_DONOR, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.RESPONSIBLE_ORGANIZATION: + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_RESPONSIBLE_ORG, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.BENEFICIARY_AGENCY: + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_BENEFICIARY_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.EXECUTING_AGENCY: + updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), null, session, ImporterConstants.ORG_TYPE_EXECUTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.IMPLEMENTING_AGENCY: + updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), null, session, ImporterConstants.ORG_TYPE_IMPLEMENTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.CONTRACTING_AGENCY: + updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), null, session, ImporterConstants.ORG_TYPE_CONTRACTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.TRANSACTION_AMOUNT: { + boolean commitment = true, disbursement = true, expenditure = false; + String adjustmentType = ImporterConstants.ADJUSTMENT_TYPE_ACTUAL; + if (config.containsValue(ImporterConstants.MEASURE_TYPE)) { + String measureTypeStr = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.MEASURE_TYPE); + ImporterUtil.MeasureTypeResult parsed = parseMeasureType(measureTypeStr); + if (parsed != null) { + commitment = parsed.commitment; + disbursement = parsed.disbursement; + expenditure = parsed.expenditure; + adjustmentType = parsed.adjustmentType; + } + } + fundings.addAll(setFundingItemsForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, commitment, disbursement, expenditure, adjustmentType, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + } + case ImporterConstants.PLANNED_COMMITMENT: + fundings.addAll(setFundingItemsForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, true, false,false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.PLANNED_DISBURSEMENT: + fundings.addAll(setFundingItemsForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.PLANNED_EXPENDITURE: + fundings.addAll(setFundingItemsForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, false, false,true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.ACTUAL_COMMITMENT: + fundings.addAll(setFundingItemsForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.ACTUAL_DISBURSEMENT: + fundings.addAll(setFundingItemsForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.ACTUAL_EXPENDITURE: + fundings.addAll(setFundingItemsForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, false, false,true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.MEASURE_TYPE: + break; + case ImporterConstants.ORG_GROUP: + break; + case ImporterConstants.PROJECT_STATUS: + break; + case ImporterConstants.PROCUREMENT_SYSTEM: + break; + case ImporterConstants.REPORTING_DATE: + default: + logger.error("Unexpected value: " + entry.getValue()); + break; + } + } + } + if (!config.containsValue(ImporterConstants.PROJECT_LOCATION) && defaultLocationId != null) { + applyDefaultLocation(importDataModel, defaultLocationId, session); + } + }); + } catch (RuntimeException e) { + Throwable cause = e.getCause(); + if (cause instanceof JsonProcessingException) { + throw (JsonProcessingException) cause; } - fundings.add(fundingItem); + importedProject.setImportStatus(ImportStatus.FAILED); + persistImportedProjectStatus(importedProject); + logger.error("Error preparing data for row " + rowRef.getRowNum() + " in sheet " + sheet.getSheetName() + ": " + e.getMessage(), e); + continue; + } + if (importedProject.getImportStatus() == ImportStatus.SKIPPED) { + persistImportedProjectStatus(importedProject); + continue; + } + if (skipRecordsWithoutTransactions && !hasTransactions(fundings)) { + importedProject.setImportStatus(ImportStatus.SKIPPED); + persistImportedProjectStatus(importedProject); + logger.info("Skipping row {} in sheet {} because no non-zero transactions were found", rowRef.getRowNum(), sheet.getSheetName()); + continue; } + // Clear session after Phase 1 to avoid contamination of Phase 2's transaction + // Phase 1's committed changes may leave pending actions in the session that conflict with ActivityGatekeeper + Session currentSession = PersistenceManager.getRequestDBSession(); + if (currentSession != null && currentSession.isOpen()) { + currentSession.clear(); + } - importTheData(importDataModel, session, importedProject, componentName, componentCode, responsibleOrgId, fundings, existing); + // Phase 2: Activity import - DO NOT wrap in transaction, let ActivityGatekeeper handle it + // This avoids nested transaction issues when ActivityGatekeeper.doWithLock creates its own transaction + Long activityId = null; + if (importedProject.getImportStatus() != ImportStatus.SKIPPED) { + try { + // Pass only the ID, not the entity - importTheData will re-fetch in its own transaction context + activityId = importTheData(importDataModel, null, importedProject, componentName, componentCode, responsibleOrgIdHolder[0], fundings, existingActivityIdHolder[0], validateActivities); + } catch (JsonProcessingException e) { + throw e; + } + } + // Phase 3: Indicator import - use separate transaction + if (activityId != null && config.containsValue(ImporterConstants.INDICATOR_NAME)) { + logger.info("Adding indicator data for activity " + activityId); + try { + final Long activityIdFinal = activityId; + PersistenceManager.inTransaction(() -> { + Session s = PersistenceManager.getRequestDBSession(); + addIndicatorDataToActivity(activityIdFinal, rowRef, sheet, config, s); + logger.info("Indicator data added for activity " + activityIdFinal); + }); + } catch (Exception e) { + logger.error("Failed to add indicator data for activity " + activityId, e); + } + } } } } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 1c735c6f9bb..18f2b32aee3 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -12,7 +12,9 @@ import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; import org.digijava.module.aim.action.dataimporter.model.Funding; +import org.digijava.module.aim.action.dataimporter.util.ImporterConstants; import org.digijava.module.aim.action.dataimporter.model.ImportDataModel; +import org.digijava.module.aim.action.dataimporter.util.ImporterUtil; import org.digijava.module.aim.dbentity.AmpActivityVersion; import org.digijava.module.aim.util.FeaturesUtil; import org.digijava.module.aim.util.TeamMemberUtil; @@ -38,7 +40,7 @@ public class TxtDataImporter { private static final Logger logger = LoggerFactory.getLogger(TxtDataImporter.class); - public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal) + public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId) { logger.info("Processing txt file: " + file.getName()); CSVParser parser = new CSVParserBuilder().withSeparator(request.getParameter("dataSeparator").charAt(0)).build(); @@ -57,7 +59,7 @@ public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecor logger.info("Batch number here: {}",batchNumber); // Process the batch - processBatch(batch, request,config,importedFilesRecord); + processBatch(batch, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); // Clear the batch for the next set of rows batch.clear(); batchNumber+=1; @@ -67,7 +69,7 @@ public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecor // Process any remaining rows in the batch if (!batch.isEmpty()) { logger.info("Processing last batch of size {}", batch.size()); - processBatch(batch, request,config,importedFilesRecord); + processBatch(batch, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); } } catch (IOException | CsvValidationException e) { logger.error("Error processing txt file "+e.getMessage(),e); @@ -77,103 +79,199 @@ public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecor } - private static void processBatch(List> batch, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord) throws JsonProcessingException { + private static void processBatch(List> batch, HttpServletRequest request, Map config, ImportedFilesRecord importedFilesRecord, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId) throws JsonProcessingException { logger.info("Processing txt batch"); SessionUtil.extendSessionIfNeeded(request); - Session session = PersistenceManager.getRequestDBSession(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); for (Map row : batch) { + final Map rowRef = row; ImportedProject importedProject= new ImportedProject(); importedProject.setImportedFilesRecord(importedFilesRecord); List fundings= new ArrayList<>(); - ImportDataModel importDataModel = new ImportDataModel(); importDataModel.setModified_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); - importDataModel.setCreated_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); importDataModel.setTeam(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeam().getAmpTeamId()); - importDataModel.setIs_draft(true); + importDataModel.setIs_draft(!validateActivities); OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); importDataModel.setCreation_date(now.format(formatter)); - setStatus(importDataModel); - String componentName= row.get(getKey(config, "Component Name")); - String componentCode= row.get(getKey(config, "Component Code")); - String projectCode= row.get(getKey(config, "Project Code")); - String projectTitle= row.get(getKey(config, "Project Title")); - String projectDesc= row.get(getKey(config, "Project Description")); - String primarySubSector= row.get(getKey(config, "Primary Subsector")); - String secondarySubSector= row.get(getKey(config, "Secondary Subsector")); - AmpActivityVersion existing = existingActivity(projectTitle,projectCode,session); - if (existing!=null && SKIP_EXISTING) - { - logger.info("Instructed to skip existing activities"); - importedProject.setImportStatus(ImportStatus.SKIPPED); - continue; + String componentName= rowRef.get(getKey(config, ImporterConstants.COMPONENT_NAME)); + String componentCode= rowRef.get(getKey(config, ImporterConstants.COMPONENT_CODE)); + String projectCode= rowRef.get(getKey(config, ImporterConstants.PROJECT_CODE)); + String projectTitle= rowRef.get(getKey(config, ImporterConstants.PROJECT_TITLE)); + String projectDesc= rowRef.get(getKey(config, ImporterConstants.PROJECT_DESCRIPTION)); + String objective= rowRef.get(getKey(config, ImporterConstants.OBJECTIVE)); + String primarySubSector= rowRef.get(getKey(config, ImporterConstants.PRIMARY_SUBSECTOR)); + String secondarySubSector= rowRef.get(getKey(config, ImporterConstants.SECONDARY_SUBSECTOR)); + String projectStatusStr = rowRef.get(getKey(config, ImporterConstants.PROJECT_STATUS)); + String importedOrgGroupName; + if (config.containsValue(ImporterConstants.ORG_GROUP)) { + String configuredOrgGroupName = rowRef.get(getKey(config, ImporterConstants.ORG_GROUP)); + if (configuredOrgGroupName != null && !configuredOrgGroupName.trim().isEmpty()) { + importedOrgGroupName = configuredOrgGroupName.trim(); + } else { + importedOrgGroupName = null; + } + } else { + importedOrgGroupName = null; } - importDataModel.setProject_title(projectTitle); - importDataModel.setProject_code(projectCode); - importDataModel.setDescription(projectDesc); - - String donorAgencyCode= row.get(getKey(config, "Donor Agency Code")); - String responsibleOrgCode= row.get(getKey(config, "Responsible Organization Code")); - Long responsibleOrgId=null; - - logger.info("Configuration: "+config); - for (Map.Entry entry : config.entrySet()) { - Funding fundingItem = new Funding(); - switch (entry.getValue()) { - case "Project Location": - updateLocations(importDataModel, row.get(entry.getKey().trim()),session); - break; - case "Primary Sector": - updateSectors(importDataModel, row.get(entry.getKey().trim()), session, true, primarySubSector); - break; - case "Secondary Sector": - updateSectors(importDataModel, row.get(entry.getKey().trim()), session, false, secondarySubSector); - break; - case "Donor Agency": - updateOrgs(importDataModel,row.get(entry.getKey().trim()),donorAgencyCode, session, "donor"); - break; - case "Responsible Organization": - responsibleOrgId=updateOrgs(importDataModel,row.get(entry.getKey().trim()),responsibleOrgCode, session, "responsibleOrg"); - break; - case "Beneficiary Agency": - responsibleOrgId=updateOrgs(importDataModel,row.get(entry.getKey().trim()),responsibleOrgCode, session, "beneficiaryAgency"); - break; - case "Funding Item": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), true, true, false,"Actual", fundingItem, existing); - break; - case "Planned Commitment": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), true, false, false,"Planned", fundingItem, existing); - break; - case "Planned Disbursement": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, true, false,"Planned", fundingItem, existing); - break; - case "Planned Expenditure": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, false,true, "Planned", fundingItem, existing); - break; - case "Actual Commitment": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), true, false, false,"Actual", fundingItem, existing); - break; - case "Actual Disbursement": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, true, false,"Actual", fundingItem, existing); - break; - case "Actual Expenditure": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, false,true, "Actual", fundingItem, existing); - break; - default: - logger.error("Unexpected value: " + entry.getValue()); - break; + // Use holder arrays to capture values from lambda (for effectively final requirement) + final Long[] existingActivityIdHolder = new Long[1]; // Store only the ID, not the entity + final Long[] responsibleOrgIdHolder = new Long[1]; + + // Phase 1: Data preparation - use transaction for reading/preparing data ONLY + try { + PersistenceManager.inTransaction(() -> { + Session session = PersistenceManager.getRequestDBSession(); + + AmpActivityVersion existing = existingActivity(projectTitle, projectCode, session); + existingActivityIdHolder[0] = existing != null ? existing.getAmpActivityId() : null; + if (existing != null && skipExisting) { + logger.info("Instructed to skip existing activities"); + importedProject.setImportStatus(ImportStatus.SKIPPED); + return; + } + + importDataModel.setProject_title(projectTitle); + importDataModel.setObjective(objective); + importDataModel.setProject_code(projectCode); + importDataModel.setDescription(projectDesc); + + if (projectStatusStr != null && !projectStatusStr.trim().isEmpty()) { + Long statusId = getOrCreateActivityStatusCategoryValue(projectStatusStr.trim(), session); + if (statusId != null) { + importDataModel.setActivity_status(statusId); + } + } + setStatus(importDataModel, validateActivities, defaultActivityStatusId); + + String donorAgencyCode = rowRef.get(getKey(config, ImporterConstants.DONOR_AGENCY_CODE)); + String responsibleOrgCode = rowRef.get(getKey(config, ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE)); + + logger.info("Configuration: " + config); + for (Map.Entry entry : config.entrySet()) { + switch (entry.getValue()) { + case ImporterConstants.PROJECT_LOCATION: + updateLocations(importDataModel, rowRef.get(entry.getKey().trim()), session); + break; + case ImporterConstants.PRIMARY_SECTOR: + updateSectors(importDataModel, rowRef.get(entry.getKey().trim()), session, + true, primarySubSector, createMissingSectors, + ImporterConstants.PRIMARY_SECTOR); + break; + case ImporterConstants.SECONDARY_SECTOR: + updateSectors(importDataModel, rowRef.get(entry.getKey().trim()), session, + false, secondarySubSector, createMissingSectors, + ImporterConstants.SECONDARY_SECTOR); + break; + case ImporterConstants.DONOR_AGENCY: + updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), donorAgencyCode, session, ImporterConstants.ORG_TYPE_DONOR, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.RESPONSIBLE_ORGANIZATION: + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_RESPONSIBLE_ORG, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.BENEFICIARY_AGENCY: + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_BENEFICIARY_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.EXECUTING_AGENCY: + updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), null, session, ImporterConstants.ORG_TYPE_EXECUTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.IMPLEMENTING_AGENCY: + updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), null, session, ImporterConstants.ORG_TYPE_IMPLEMENTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.CONTRACTING_AGENCY: + updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), null, session, ImporterConstants.ORG_TYPE_CONTRACTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + break; + case ImporterConstants.TRANSACTION_AMOUNT: { + boolean commitment = true, disbursement = true, expenditure = false; + String adjustmentType = ImporterConstants.ADJUSTMENT_TYPE_ACTUAL; + if (config.containsValue(ImporterConstants.MEASURE_TYPE)) { + String measureTypeStr = rowRef.get(getKey(config, ImporterConstants.MEASURE_TYPE)); + ImporterUtil.MeasureTypeResult parsed = parseMeasureType(measureTypeStr); + if (parsed != null) { + commitment = parsed.commitment; + disbursement = parsed.disbursement; + expenditure = parsed.expenditure; + adjustmentType = parsed.adjustmentType; + } + } + fundings.addAll(setFundingItemsForTxt(rowRef, config, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), commitment, disbursement, expenditure, adjustmentType, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + } + case ImporterConstants.PLANNED_COMMITMENT: + fundings.addAll(setFundingItemsForTxt(rowRef, config, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.PLANNED_DISBURSEMENT: + fundings.addAll(setFundingItemsForTxt(rowRef, config, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.PLANNED_EXPENDITURE: + fundings.addAll(setFundingItemsForTxt(rowRef, config, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), false, false, true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.ACTUAL_COMMITMENT: + fundings.addAll(setFundingItemsForTxt(rowRef, config, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.ACTUAL_DISBURSEMENT: + fundings.addAll(setFundingItemsForTxt(rowRef, config, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.ACTUAL_EXPENDITURE: + fundings.addAll(setFundingItemsForTxt(rowRef, config, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), false, false, true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, null, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups, addDisbursementForCommitment)); + break; + case ImporterConstants.MEASURE_TYPE: + break; + case ImporterConstants.ORG_GROUP: + break; + case ImporterConstants.PROJECT_STATUS: + break; + default: + logger.error("Unexpected value: " + entry.getValue()); + break; + } + } + if (!config.containsValue(ImporterConstants.PROJECT_LOCATION) && defaultLocationId != null) { + applyDefaultLocation(importDataModel, defaultLocationId, session); + } + logger.info("Funding items :{}", fundings); + }); + } catch (RuntimeException e) { + Throwable cause = e.getCause(); + if (cause instanceof JsonProcessingException) { + throw (JsonProcessingException) cause; } - fundings.add(fundingItem); - logger.info("Funding items :{}",fundings); + importedProject.setImportStatus(ImportStatus.FAILED); + persistImportedProjectStatus(importedProject); + logger.error("Error preparing txt row for project {}: {}", projectCode, e.getMessage(), e); + continue; + } + + if (importedProject.getImportStatus() == ImportStatus.SKIPPED) { + persistImportedProjectStatus(importedProject); + continue; + } + if (skipRecordsWithoutTransactions && !hasTransactions(fundings)) { + importedProject.setImportStatus(ImportStatus.SKIPPED); + persistImportedProjectStatus(importedProject); + logger.info("Skipping txt row for project {} because no non-zero transactions were found", projectCode); + continue; } - importTheData(importDataModel, session, importedProject, componentName, componentCode,responsibleOrgId,fundings,existing); + // Clear session after Phase 1 to avoid contamination of Phase 2's transaction + // Phase 1's committed changes may leave pending actions in the session that conflict with ActivityGatekeeper + Session currentSession = PersistenceManager.getRequestDBSession(); + if (currentSession != null && currentSession.isOpen()) { + currentSession.clear(); + } + // Phase 2: Activity import - DO NOT wrap in transaction, let ActivityGatekeeper handle it + // This avoids nested transaction issues when ActivityGatekeeper.doWithLock creates its own transaction + try { + // Pass only the ID, not the entity - importTheData will re-fetch in its own transaction context + importTheData(importDataModel, null, importedProject, componentName, componentCode, responsibleOrgIdHolder[0], fundings, existingActivityIdHolder[0], validateActivities); + } catch (JsonProcessingException e) { + throw e; + } } } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ViewImportProgress.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ViewImportProgress.java index 8cf785f3686..5bdb1f54eb8 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ViewImportProgress.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ViewImportProgress.java @@ -48,6 +48,7 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet data.put("importedProjects", importedProjects); data.put("failedProjects", variousCounts.get("failedProjects")); data.put("successfulProjects", variousCounts.get("successfulProjects")); + data.put("skippedProjects", variousCounts.get("skippedProjects")); // data.put("totalPages", variousCounts.get("totalPages")); data.put("totalProjects", variousCounts.get("totalProjects")); @@ -85,6 +86,9 @@ private Map getVariousCounts(Long importedFilesRecordId) { List resultList = query.list(); Map countsMap = new HashMap<>(); + countsMap.put("failedProjects", 0L); + countsMap.put("successfulProjects", 0L); + countsMap.put("skippedProjects", 0L); for (Object[] row : resultList) { ImportStatus importStatus = (ImportStatus) row[0]; @@ -94,6 +98,8 @@ private Map getVariousCounts(Long importedFilesRecordId) { countsMap.put("failedProjects", count); } else if (importStatus.equals(ImportStatus.SUCCESS)) { countsMap.put("successfulProjects", count); + } else if (importStatus.equals(ImportStatus.SKIPPED)) { + countsMap.put("skippedProjects", count); } } long totalProjects= getTotalProjects(importedFilesRecordId); @@ -154,7 +160,7 @@ private List getImportedProjects(int pageNumber, int pageSize, public List getAllImportFileRecords() { Session session = PersistenceManager.getRequestDBSession(); - String hql = "FROM ImportedFilesRecord"; + String hql = "FROM ImportedFilesRecord ORDER BY uploadedAt DESC, id DESC"; Query query = session.createQuery(hql); List importedFilesRecords = query.list(); logger.info("importedFilesRecords: " + importedFilesRecords); diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.hbm.xml b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.hbm.xml index 2b5d5a2e674..247293ebaff 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.hbm.xml +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.hbm.xml @@ -17,6 +17,10 @@ + + + + org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.java index 6a9a872a3fd..c424f607021 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedFilesRecord.java @@ -1,11 +1,15 @@ package org.digijava.module.aim.action.dataimporter.dbentity; import java.io.Serializable; +import java.text.SimpleDateFormat; +import java.util.Date; public class ImportedFilesRecord implements Serializable { private Long id; private String fileName; private String fileHash; +private Long processingTimeMillis; +private Date uploadedAt; private ImportStatus importStatus; @@ -33,6 +37,42 @@ public void setFileHash(String fileHash) { this.fileHash = fileHash; } + public Long getProcessingTimeMillis() { + return processingTimeMillis; + } + + public void setProcessingTimeMillis(Long processingTimeMillis) { + this.processingTimeMillis = processingTimeMillis; + } + + public Date getUploadedAt() { + return uploadedAt; + } + + public void setUploadedAt(Date uploadedAt) { + this.uploadedAt = uploadedAt; + } + + public String getFormattedUploadedAt() { + if (uploadedAt == null) { + return "-"; + } + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(uploadedAt); + } + + public Long getUploadedAtEpochMillis() { + return uploadedAt != null ? uploadedAt.getTime() : null; + } + + public String getFormattedProcessingTime() { + if (processingTimeMillis == null) { + return "-"; + } + long minutes = processingTimeMillis / 60000; + long seconds = (processingTimeMillis % 60000) / 1000; + return minutes + "m " + seconds + "s"; + } + public ImportStatus getImportStatus() { return importStatus; } @@ -47,6 +87,8 @@ public String toString() { "id=" + id + ", fileName='" + fileName + '\'' + ", fileHash='" + fileHash + '\'' + + ", processingTimeMillis=" + processingTimeMillis + + ", uploadedAt=" + uploadedAt + ", importStatus=" + importStatus + '}'; } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProjectCurrency.hbm.xml b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProjectCurrency.hbm.xml index 577485f4d84..4fc39b2a539 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProjectCurrency.hbm.xml +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProjectCurrency.hbm.xml @@ -10,7 +10,7 @@ IMPORTED_PROJECT_CURRENCY_ID - + diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/ImportDataModel.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/ImportDataModel.java index a88a1bc9ccd..bb17179b9fe 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/ImportDataModel.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/ImportDataModel.java @@ -28,6 +28,8 @@ public class ImportDataModel { private Set responsible_organization=new HashSet<>(); private Set beneficiary_agency=new HashSet<>(); private Set executing_agency=new HashSet<>(); + private Set contracting_agency = new HashSet<>(); + private Set implementing_agency = new HashSet<>(); private Set activity_internal_ids; private Set fundings= new HashSet<>(); private Set issues; @@ -51,8 +53,8 @@ public class ImportDataModel { private Object approval_date; private Integer approval_status; private Object archived; - private Set indicators; - private Set activity_documents; +// private Set indicators; +// private Set activity_documents; private Long activity_status; private Long activity_budget; private Long implementation_location; @@ -64,6 +66,7 @@ public class ImportDataModel { private ActivityGroup activity_group; private Long modified_by; private Long activity_type; + private Long procurement_system; public Long getInternal_id() { return internal_id; @@ -393,21 +396,21 @@ public void setArchived(Object archived) { this.archived = archived; } - public Set getIndicators() { - return indicators; - } +// public Set getIndicators() { +// return indicators; +// } - public void setIndicators(Set indicators) { - this.indicators = indicators; - } +// public void setIndicators(Set indicators) { +// this.indicators = indicators; +// } - public Set getActivity_documents() { - return activity_documents; - } +// public Set getActivity_documents() { +// return activity_documents; +// } - public void setActivity_documents(Set activity_documents) { - this.activity_documents = activity_documents; - } +// public void setActivity_documents(Set activity_documents) { +// this.activity_documents = activity_documents; +// } public Long getActivity_status() { return activity_status; @@ -502,6 +505,14 @@ public void setActivity_type(Long activity_type) { this.activity_type = activity_type; } + public Long getProcurement_system() { + return procurement_system; + } + + public void setProcurement_system(Long procurement_system) { + this.procurement_system = procurement_system; + } + @Override public String toString() { return "ImportDataModel{" + @@ -545,8 +556,8 @@ public String toString() { ", approval_date=" + approval_date + ", approval_status=" + approval_status + ", archived=" + archived + - ", indicators=" + indicators + - ", activity_documents=" + activity_documents + +// ", indicators=" + indicators + +// ", activity_documents=" + activity_documents + ", activity_status=" + activity_status + ", activity_budget=" + activity_budget + ", implementation_level=" + implementation_level + @@ -559,9 +570,27 @@ public String toString() { ", activity_group=" + activity_group + ", modified_by=" + modified_by + ", activity_type=" + activity_type + + ", procurement_system=" + procurement_system + + ", implementing_agency=" + implementing_agency + '}'; } + public Set getContracting_agency() { + return contracting_agency; + } + + public void setContracting_agency(Set contracting_agency) { + this.contracting_agency = contracting_agency; + } + + public Set getImplementing_agency() { + return implementing_agency; + } + + public void setImplementing_agency(Set implementing_agency) { + this.implementing_agency = implementing_agency; + } + // Getters and setters } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/Sector.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/Sector.java index 5ff9c9656ac..8743504148d 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/Sector.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/Sector.java @@ -44,12 +44,19 @@ public String toString() { @Override public boolean equals(Object o) { if (!(o instanceof Sector)) return false; - Sector sector = (Sector) o; - return Objects.equals(getId(), sector.getId()); + Sector other = (Sector) o; + // Two Sector objects referencing the same amp_sector entity are equal, + // regardless of whether the activity-sector PK (id) has been populated yet. + if (this.sector != null && other.sector != null) { + return Objects.equals(this.sector, other.sector); + } + return Objects.equals(this.id, other.id); } @Override public int hashCode() { - return Objects.hashCode(getId()); + // Always hash by the sector entity id so hashCode is consistent with equals + // when one object has a populated activity-sector PK and the other does not. + return Objects.hashCode(sector); } } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/Transaction.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/Transaction.java index 69537c3e6d9..c83e4d4c87c 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/Transaction.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/Transaction.java @@ -1,5 +1,7 @@ package org.digijava.module.aim.action.dataimporter.model; +import com.fasterxml.jackson.annotation.JsonIgnore; + public class Transaction { private Long transaction_id; private Long adjustment_type; @@ -9,6 +11,8 @@ public class Transaction { private double transaction_amount; private Long currency; private Object fixed_exchange_rate; + @JsonIgnore + private boolean inferredTransactionDate; public Long getTransaction_id() { return transaction_id; @@ -74,6 +78,14 @@ public void setFixed_exchange_rate(Object fixed_exchange_rate) { this.fixed_exchange_rate = fixed_exchange_rate; } + public boolean isInferredTransactionDate() { + return inferredTransactionDate; + } + + public void setInferredTransactionDate(boolean inferredTransactionDate) { + this.inferredTransactionDate = inferredTransactionDate; + } + @Override public String toString() { return "Transaction{" + diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImportedFileUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImportedFileUtil.java index 910b99149fe..ec180266814 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImportedFileUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImportedFileUtil.java @@ -43,7 +43,7 @@ public static ImportedFilesRecord saveFile(File file, String filename) throws IO String generatedHash = generateSHA256Hash(file); logger.info("Saving File hash is " + generatedHash); long generatedId=0l; - String sql = "INSERT INTO IMPORTED_FILES_RECORD (id, file_name, file_hash, import_status) VALUES (nextval('IMPORTED_FILES_RECORD_SEQ'), ?, ?, ?) RETURNING id"; + String sql = "INSERT INTO IMPORTED_FILES_RECORD (id, file_name, file_hash, import_status, uploaded_at) VALUES (nextval('IMPORTED_FILES_RECORD_SEQ'), ?, ?, ?, CURRENT_TIMESTAMP) RETURNING id"; try (Connection connection = PersistenceManager.getJdbcConnection(); PreparedStatement preparedStatement = connection.prepareStatement(sql)) { @@ -90,6 +90,24 @@ public static void updateFileStatus(ImportedFilesRecord importedFilesRecord, Imp logger.error("Error updating file status", e); } } + + public static void updateFileProcessingTime(ImportedFilesRecord importedFilesRecord, long processingTimeMillis) { + logger.info("Updating file processing time to {} ms", processingTimeMillis); + + Session session = PersistenceManager.getRequestDBSession(); + + try { + String sql = "UPDATE IMPORTED_FILES_RECORD SET processing_time_millis = :processingTimeMillis WHERE id = :fileId"; + Query query = session.createNativeQuery(sql); + query.setParameter("processingTimeMillis", processingTimeMillis); + query.setParameter("fileId", importedFilesRecord.getId()); + query.executeUpdate(); + session.getTransaction().commit(); + importedFilesRecord.setProcessingTimeMillis(processingTimeMillis); + } catch (Exception e) { + logger.error("Error updating file processing time", e); + } + } public static List getSimilarFiles(File file) throws IOException, NoSuchAlgorithmException { String hash = generateSHA256Hash(file); logger.info("Checking File hash is {}", hash); diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java new file mode 100644 index 00000000000..8637a6fbcf5 --- /dev/null +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java @@ -0,0 +1,84 @@ +package org.digijava.module.aim.action.dataimporter.util; + +/** + * Central place for all string constants used by the Data Importer (Excel, Txt, config). + * These are the field names used in column-to-entity mapping and in switch/case logic. + */ +public final class ImporterConstants { + + private ImporterConstants() { + } + + // ----- Adjustment types (funding: Actual vs Planned) ----- + public static final String ADJUSTMENT_TYPE_ACTUAL = "Actual"; + public static final String ADJUSTMENT_TYPE_PLANNED = "Planned"; + + // ----- Organization role types (for updateOrgs) ----- + public static final String ORG_TYPE_DONOR = "donor"; + public static final String ORG_TYPE_RESPONSIBLE_ORG = "responsibleOrg"; + public static final String ORG_TYPE_BENEFICIARY_AGENCY = "beneficiaryAgency"; + public static final String ORG_TYPE_EXECUTING_AGENCY = "executingAgency"; + public static final String ORG_TYPE_IMPLEMENTING_AGENCY = "implementingAgency"; + public static final String ORG_TYPE_CONTRACTING_AGENCY = "contractingAgency"; + + // ----- Entity / column field names (template mapping) ----- + public static final String PROJECT_TITLE = "Project Title"; + public static final String PROJECT_CODE = "Project Code"; + public static final String OBJECTIVE = "Objective"; + public static final String PROJECT_DESCRIPTION = "Project Description"; + public static final String PRIMARY_SECTOR = "Primary Sector"; + public static final String SECONDARY_SECTOR = "Secondary Sector"; + public static final String PROJECT_LOCATION = "Project Location"; + public static final String PROJECT_START_DATE = "Project Start Date"; + public static final String PROJECT_END_DATE = "Project End Date"; + public static final String DONOR_AGENCY = "Donor Agency"; + public static final String DONOR_AGENCY_CODE = "Donor Agency Code"; + public static final String EXCHANGE_RATE = "Exchange Rate"; + public static final String ORG_GROUP = "Organization Group"; + public static final String RESPONSIBLE_ORGANIZATION = "Responsible Organization"; + public static final String RESPONSIBLE_ORGANIZATION_CODE = "Responsible Organization Code"; + public static final String EXECUTING_AGENCY = "Executing Agency"; + public static final String IMPLEMENTING_AGENCY = "Implementing Agency"; + public static final String CONTRACTING_AGENCY = "Contracting Agency"; + public static final String BENEFICIARY_AGENCY = "Beneficiary Agency"; + + public static final String ACTUAL_DISBURSEMENT = "Actual Disbursement"; + public static final String ACTUAL_COMMITMENT = "Actual Commitment"; + public static final String ACTUAL_EXPENDITURE = "Actual Expenditure"; + public static final String PLANNED_DISBURSEMENT = "Planned Disbursement"; + public static final String PLANNED_COMMITMENT = "Planned Commitment"; + public static final String PLANNED_EXPENDITURE = "Planned Expenditure"; + public static final String TRANSACTION_AMOUNT = "Transaction Amount"; + public static final String MEASURE_TYPE = "Measure Type"; + public static final String TRANSACTION_DATE = "Transaction Date"; + public static final String FINANCING_INSTRUMENT = "Financing Instrument"; + public static final String TYPE_OF_ASSISTANCE = "Type Of Assistance"; + public static final String PRIMARY_SUBSECTOR = "Primary Subsector"; + public static final String SECONDARY_SUBSECTOR = "Secondary Subsector"; + public static final String CURRENCY = "Currency"; + public static final String COMPONENT_NAME = "Component Name"; + public static final String COMPONENT_CODE = "Component Code"; + + public static final String REPORTING_DATE = "Reporting Date"; + public static final String PROJECT_STATUS = "Project Status"; + public static final String PROCUREMENT_SYSTEM = "Procurement System"; + + // ----- Indicator (M&E) columns ----- + public static final String INDICATOR_NAME = "Indicator Name"; + public static final String PROGRAM_NAME = "Program Name"; + /** Used for project-level location (e.g. Project Location). */ + public static final String LOCATION = "Location"; + /** Used for matching indicator value to activity location; distinct from project Location. */ + public static final String INDICATOR_LOCATION = "Indicator Location"; + public static final String ORIGINAL_BASE_VALUE = "Original Base Value"; + public static final String ORIGINAL_BASE_VALUE_DATE = "Original Base Value Date"; + public static final String REVISED_BASE_VALUE = "Revised Base Value"; + public static final String REVISED_BASE_VALUE_DATE = "Revised Base Value Date"; + public static final String ORIGINAL_TARGET_VALUE = "Original Target Value"; + public static final String ORIGINAL_TARGET_VALUE_DATE = "Original Target Value Date"; + public static final String REVISED_TARGET_VALUE = "Revised Target Value"; + public static final String REVISED_TARGET_VALUE_DATE = "Revised Target Value Date"; + public static final String ACTUAL_VALUE = "Actual Value"; + public static final String ACTUAL_VALUE_DATE = "Actual Value Date"; + public static final String UNIT_OF_MEASURE = "Unit of Measure"; +} diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index 357fc6c0c79..f96aebe3ec3 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -9,10 +9,14 @@ import org.apache.poi.ss.usermodel.DateUtil; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; +import org.dgfoundation.amp.ar.ArConstants; +import org.dgfoundation.amp.ar.ARUtil; import org.digijava.kernel.ampapi.endpoints.activity.ActivityImportRules; import org.digijava.kernel.ampapi.endpoints.activity.ActivityInterchangeUtils; import org.digijava.kernel.ampapi.endpoints.activity.dto.ActivitySummary; import org.digijava.kernel.ampapi.endpoints.common.JsonApiResponse; +import org.digijava.kernel.ampapi.endpoints.indicator.manager.IndicatorManagerService; +import org.digijava.kernel.ampapi.endpoints.indicator.manager.MEIndicatorDTO; import org.digijava.kernel.persistence.PersistenceManager; import org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus; import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; @@ -20,8 +24,16 @@ import org.digijava.module.aim.action.dataimporter.model.*; import org.digijava.module.aim.dbentity.*; import org.digijava.module.aim.util.CurrencyUtil; +import org.digijava.module.aim.util.DbUtil; +import org.digijava.module.aim.util.FeaturesUtil; +import org.digijava.module.aim.util.ProgramUtil; +import org.digijava.module.aim.util.SectorUtil; +import org.digijava.module.aim.util.TeamUtil; +import org.digijava.module.categorymanager.dbentity.AmpCategoryClass; import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; import org.digijava.module.categorymanager.util.CategoryConstants; +import org.digijava.module.categorymanager.util.CategoryManagerUtil; +import org.hibernate.Hibernate; import org.hibernate.Query; import org.hibernate.Session; import org.hibernate.type.StringType; @@ -33,8 +45,6 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.math.BigDecimal; -import java.math.RoundingMode; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -67,21 +77,21 @@ private static Double parseDouble(String number) { } - public static Funding setAFundingItemForExcel(Sheet sheet, Map config, Row row, Map.Entry entry, ImportDataModel importDataModel, Session session, Cell cell, boolean commitment, boolean disbursement, boolean expenditure, String - adjustmentType, Funding fundingItem, AmpActivityVersion existingActivity) { - int detailColumn = getColumnIndexByName(sheet, getKey(config, "Financing Instrument")); + public static List setFundingItemsForExcel(Sheet sheet, Map config, Row row, Map.Entry entry, ImportDataModel importDataModel, Session session, Cell cell, boolean commitment, boolean disbursement, boolean expenditure, String + adjustmentType, AmpActivityVersion existingActivity, boolean createMissingOrgs, Long orgGroupId, String importedOrgGroupName, boolean createMissingOrgGroups, boolean addDisbursementForCommitment) { + int detailColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.FINANCING_INSTRUMENT)); String finInstrument = detailColumn >= 0 ? getStringValueFromCell(row.getCell(detailColumn), false) : ""; - detailColumn = getColumnIndexByName(sheet, getKey(config, "Exchange Rate")); + detailColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.EXCHANGE_RATE)); String exchangeRate = detailColumn >= 0 ? getStringValueFromCell(row.getCell(detailColumn), false) : ""; Double exchangeRateValue = !exchangeRate.isEmpty() ? parseDouble(exchangeRate) : Double.valueOf(0.0); - detailColumn = getColumnIndexByName(sheet, getKey(config, "Type Of Assistance")); + detailColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.TYPE_OF_ASSISTANCE)); String typeOfAss = detailColumn >= 0 ? getStringValueFromCell(row.getCell(detailColumn), false) : ""; - int separateFundingDateColumn = getColumnIndexByName(sheet, getKey(config, "Transaction Date")); + int separateFundingDateColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.TRANSACTION_DATE)); String separateFundingDate = separateFundingDateColumn >= 0 ? getDateFromExcel(row, separateFundingDateColumn) : null; - int currencyCodeColumn = getColumnIndexByName(sheet, getKey(config, "Currency")); + int currencyCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.CURRENCY)); String currencyCode = currencyCodeColumn >= 0 ? getStringValueFromCell(row.getCell(currencyCodeColumn), true) : CurrencyUtil.getDefaultCurrency().getCurrencyCode(); if (existingActivity != null) { String existingActivityCurrencyCode = getCurrencyCodeFromExistingImported(existingActivity.getName()); @@ -90,41 +100,59 @@ public static Funding setAFundingItemForExcel(Sheet sheet, Map c } } saveCurrencyCode(currencyCode, importDataModel.getProject_title()); - Funding funding; - int componentNameColumn = getColumnIndexByName(sheet, getKey(config, "Component Name")); + List fundings = new ArrayList<>(); + int componentNameColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.COMPONENT_NAME)); String componentName = componentNameColumn >= 0 ? getStringValueFromCell(row.getCell(componentNameColumn), true) : null; if (importDataModel.getDonor_organization() == null || importDataModel.getDonor_organization().isEmpty()) { - if (!config.containsValue("Donor Agency")) { - funding = updateFunding(fundingItem, importDataModel, getNumericValueFromCell(cell), entry.getKey(), separateFundingDate, getRandomOrg(session), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue); + if (!config.containsValue(ImporterConstants.DONOR_AGENCY)) { + Funding f = new Funding(); + updateFunding(f, importDataModel, getNumericValueFromCell(cell), entry.getKey(), separateFundingDate, getRandomOrg(session), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue, addDisbursementForCommitment); + + fundings.add(f); } else { - int columnIndex1 = getColumnIndexByName(sheet, getKey(config, "Donor Agency")); - int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, "Donor Agency Code")); + int columnIndex1 = getColumnIndexByName(sheet, getKey(config, ImporterConstants.DONOR_AGENCY)); + int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.DONOR_AGENCY_CODE)); String donorAgencyCode = donorAgencyCodeColumn >= 0 ? getStringValueFromCell(row.getCell(donorAgencyCodeColumn), true) : null; - updateOrgs(importDataModel, columnIndex1 >= 0 ? Objects.requireNonNull(getStringValueFromCell(row.getCell(columnIndex1), false)).trim() : "no org", donorAgencyCode, session, "donor"); - funding = updateFunding(fundingItem, importDataModel, getNumericValueFromCell(cell), entry.getKey(), separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue); + updateOrgs(importDataModel, columnIndex1 >= 0 ? Objects.requireNonNull(getStringValueFromCell(row.getCell(columnIndex1), false)).trim() : "no org", donorAgencyCode, session, "donor", createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + List donors = new ArrayList<>(importDataModel.getDonor_organization()); + List splits = splitAmounts(getNumericValueFromCell(cell).doubleValue(), donors.size()); + for (int i = 0; i < donors.size(); i++) { + Funding f = new Funding(); + updateFunding(f, importDataModel, splits.get(i), entry.getKey(), separateFundingDate, donors.get(i).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue, addDisbursementForCommitment); + + fundings.add(f); + } + } } else { - funding = updateFunding(fundingItem, importDataModel, getNumericValueFromCell(cell), entry.getKey(), separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue); + List donors = new ArrayList<>(importDataModel.getDonor_organization()); + List splits = splitAmounts(getNumericValueFromCell(cell).doubleValue(), donors.size()); + for (int i = 0; i < donors.size(); i++) { + Funding f = new Funding(); + updateFunding(f, importDataModel, splits.get(i), entry.getKey(), separateFundingDate, donors.get(i).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue, addDisbursementForCommitment); + + fundings.add(f); + } } - return funding; + return fundings; } - public static Funding setAFundingItemForTxt(Map row, Map config, Map.Entry entry, ImportDataModel importDataModel, Session session, Number value, boolean commitment, boolean disbursement, boolean expenditure, String - adjustmentType, Funding fundingItem, AmpActivityVersion existingActivity) { - String finInstrument = row.get(getKey(config, "Financing Instrument")); + public static List setFundingItemsForTxt(Map row, Map config, Map.Entry entry, ImportDataModel importDataModel, Session session, Number value, boolean commitment, boolean disbursement, boolean expenditure, String + adjustmentType, AmpActivityVersion existingActivity, boolean createMissingOrgs, Long orgGroupId, String importedOrgGroupName, boolean createMissingOrgGroups, boolean addDisbursementForCommitment) { + String finInstrument = row.get(getKey(config, ImporterConstants.FINANCING_INSTRUMENT)); finInstrument = finInstrument != null ? finInstrument : ""; - String typeOfAss = row.get(getKey(config, "Type Of Assistance")); + String typeOfAss = row.get(getKey(config, ImporterConstants.TYPE_OF_ASSISTANCE)); typeOfAss = typeOfAss != null ? typeOfAss : ""; - Funding funding; + List fundings = new ArrayList<>(); - String separateFundingDate = row.get(getKey(config, "Transaction Date")); + String separateFundingDate = row.get(getKey(config, ImporterConstants.TRANSACTION_DATE)); separateFundingDate = separateFundingDate != null ? separateFundingDate : ""; - String currencyCode = row.get(getKey(config, "Currency")); + String currencyCode = row.get(getKey(config, ImporterConstants.CURRENCY)); currencyCode = currencyCode != null ? currencyCode : CurrencyUtil.getDefaultCurrency().getCurrencyCode(); if (existingActivity != null) { String existingActivityCurrencyCode = getCurrencyCodeFromExistingImported(existingActivity.getName()); @@ -133,32 +161,48 @@ public static Funding setAFundingItemForTxt(Map row, Map(importDataModel.getDonor_organization()).get(0).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue); + String donorColumn = row.get(getKey(config, ImporterConstants.DONOR_AGENCY)); + String donorAgencyCode = row.get(getKey(config, ImporterConstants.DONOR_AGENCY_CODE)); + + updateOrgs(importDataModel, donorColumn != null && !donorColumn.isEmpty() ? donorColumn.trim() : "no org", donorAgencyCode, session, "donor", createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + List donors = new ArrayList<>(importDataModel.getDonor_organization()); + List splits = splitAmounts(value != null ? value.doubleValue() : 0.0, donors.size()); + for (int i = 0; i < donors.size(); i++) { + Funding f = new Funding(); + updateFunding(f, importDataModel, splits.get(i), entry.getKey(), separateFundingDate, donors.get(i).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue, addDisbursementForCommitment); + + fundings.add(f); + } } } else { - funding = updateFunding(fundingItem, importDataModel, value, entry.getKey(), separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue); + List donors = new ArrayList<>(importDataModel.getDonor_organization()); + List splits = splitAmounts(value != null ? value.doubleValue() : 0.0, donors.size()); + for (int i = 0; i < donors.size(); i++) { + Funding f = new Funding(); + updateFunding(f, importDataModel, splits.get(i), entry.getKey(), separateFundingDate, donors.get(i).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue, addDisbursementForCommitment); + fundings.add(f); + } } - return funding; + return fundings; } public static String getStringValueFromCell(Cell cell, boolean nullable) { @@ -181,6 +225,13 @@ public static String getStringValueFromCell(Cell cell, boolean nullable) { public static Number getNumericValueFromCell(Cell cell) { try { + if (cell.getCellType() == Cell.CELL_TYPE_STRING) { + String raw = cell.getStringCellValue().trim().replace(",", ""); + if (!raw.isEmpty()) { + return Double.parseDouble(raw); + } + return 0; + } return cell.getNumericCellValue(); } catch (Exception e) { logger.error("Error getting cell {} value: ", cell, e); @@ -209,8 +260,13 @@ private static String extractDateFromStringCell(Cell cell) { return null; } - if (rawValue.matches("\\d+")) { + if (rawValue.matches("\\d+(\\.0+)?")) { double numericValue = Double.parseDouble(rawValue); + int intVal = (int) numericValue; + // Year-only: whole number in reasonable year range (e.g. 2023 or 2023.0) -> use as calendar year, not Excel serial days + if (intVal >= 1800 && intVal <= 2700 && numericValue == Math.floor(numericValue)) { + return intVal + "-12-31"; + } if (numericValue > 59) { // Excel bug: after 28 Feb 1900, 60+ is valid Date date = DateUtil.getJavaDate(numericValue); return outputFormat.format(date); @@ -271,6 +327,13 @@ private static Session getSession() { private static String getFundingDate(String dateString) { + if (dateString == null || dateString.trim().isEmpty()) { + return LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + } + if (dateString != null && dateString.trim().matches("\\d{4}")) { + int year = Integer.parseInt(dateString.trim()); + return LocalDate.of(year, 12, 31).format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + } LocalDate date = LocalDate.now(); if (isCommonDateFormat(dateString)) { List formatters = Arrays.asList( @@ -326,9 +389,9 @@ private static String formatDateFromDateObject(String date) { // Check if date is in year-only format (e.g., "2024") if (Pattern.matches("\\d{4}", date)) { try { - // Parse the year and create a Date object for January 1 of that year - Date januaryFirst = new SimpleDateFormat("yyyy-MM-dd").parse(date + "-01-01"); - return new SimpleDateFormat("yyyy-MM-dd").format(januaryFirst); // Return as "yyyy-MM-dd" + // Parse the year and create a Date object for December 31 of that year + Date decemberLast = new SimpleDateFormat("yyyy-MM-dd").parse(date + "-12-31"); + return new SimpleDateFormat("yyyy-MM-dd").format(decemberLast); // Return as "yyyy-MM-dd" } catch (Exception e) { logger.info("Error parsing date", e); } @@ -387,6 +450,58 @@ public static K getKey(Map map, V value) { return null; } + /** + * Result of parsing a Measure Type string (e.g. "PC - Planned Commitment"). + * Used to set commitment/disbursement/expenditure and Actual/Planned for funding. + */ + public static class MeasureTypeResult { + public final boolean commitment; + public final boolean disbursement; + public final boolean expenditure; + public final String adjustmentType; // "Actual" or "Planned" + + public MeasureTypeResult(boolean commitment, boolean disbursement, boolean expenditure, String adjustmentType) { + this.commitment = commitment; + this.disbursement = disbursement; + this.expenditure = expenditure; + this.adjustmentType = adjustmentType; + } + } + + /** + * Parses a Measure Type value from the template (e.g. "PC - Planned Commitment", "AC", or "Actual Commitment"). + * @return MeasureTypeResult or null if not recognized + */ + public static MeasureTypeResult parseMeasureType(String value) { + if (value == null) return null; + String s = value.trim(); + if (s.isEmpty()) return null; + // AC / PC / AD / PD / AE / PE + if (s.equalsIgnoreCase("AC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase("PC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase("AD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase("PD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase("AE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase("PE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + // Full form "PC - Planned Commitment" or label only "Planned Commitment" + if (s.contains(" - ")) { + String code = s.substring(0, s.indexOf(" - ")).trim(); + if (code.equalsIgnoreCase("AC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (code.equalsIgnoreCase("PC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (code.equalsIgnoreCase("AD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (code.equalsIgnoreCase("PD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (code.equalsIgnoreCase("AE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (code.equalsIgnoreCase("PE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + } + if (s.equalsIgnoreCase(ImporterConstants.ACTUAL_COMMITMENT)) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase(ImporterConstants.PLANNED_COMMITMENT)) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase(ImporterConstants.ACTUAL_DISBURSEMENT)) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase(ImporterConstants.PLANNED_DISBURSEMENT)) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase(ImporterConstants.ACTUAL_EXPENDITURE)) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase(ImporterConstants.PLANNED_EXPENDITURE)) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + return null; + } + public static String findYearSubstring(String text) { Pattern pattern = Pattern.compile("(?:19|20)\\d{2}"); Matcher matcher = pattern.matcher(text); @@ -425,8 +540,8 @@ public static boolean isFileContentValid(File file) { } - private static Funding updateFunding(Funding fundingItem, ImportDataModel importDataModel, Number amount, String columnHeaderContainingYear, String separateFundingDate, Long orgId, String assistanceType, String finInst, boolean commitment, boolean disbursement, boolean expenditure, String - adjustmentType, String currencyCode, String componentName, Double exchangeRate) { + private static Funding updateFunding(Funding fundingItem, ImportDataModel importDataModel, Number amount, String columnHeaderContainingYear, String separateFundingDate, Long orgId, String assistanceType, String finInst, boolean commitment, boolean disbursement, boolean expenditure, String + adjustmentType, String currencyCode, String componentName, Double exchangeRate, boolean addDisbursementForCommitment) { // TODO: 27/06/2024 pick Month from file and use it in funding Session session = getSession(); Long currencyId = getCurrencyId(session, currencyCode); @@ -438,17 +553,28 @@ private static Funding updateFunding(Funding fundingItem, ImportDataModel import String yearString; String fundingDate; + boolean inferredFundingDate = false; if (separateFundingDate != null) { if (isCommonDateFormat(separateFundingDate)) { fundingDate = getFundingDate(separateFundingDate); } else { yearString = findYearSubstring(separateFundingDate); - fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); + if (yearString != null) { + fundingDate = getFundingDate(yearString); + } else { + fundingDate = getFundingDate(null); + inferredFundingDate = true; + } } } else { yearString = findYearSubstring(columnHeaderContainingYear); - fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); + if (yearString != null) { + fundingDate = getFundingDate(yearString); + } else { + fundingDate = getFundingDate(null); + inferredFundingDate = true; + } } @@ -462,24 +588,31 @@ private static Funding updateFunding(Funding fundingItem, ImportDataModel import transaction.setAdjustment_type(adjType); transaction.setTransaction_amount(amount != null ? amount.doubleValue() : 0.0); transaction.setTransaction_date(fundingDate); + transaction.setInferredTransactionDate(inferredFundingDate); transaction.setFixed_exchange_rate(exchangeRate); + + // Check for duplicate transactions by currency, amount, and date before adding if (commitment) { - fundingItem.getCommitments().add(transaction); + if (!transactionExists(fundingItem.getCommitments(), transaction)) { + fundingItem.getCommitments().add(transaction); + } } if (disbursement) { - fundingItem.getDisbursements().add(transaction); + if (!transactionExists(fundingItem.getDisbursements(), transaction)) { + fundingItem.getDisbursements().add(transaction); + } } if (expenditure) { - if (transaction.getTransaction_amount() == 0) { - transaction.setTransaction_amount(-1); + if (!transactionExists(fundingItem.getExpenditures(), transaction)) { + fundingItem.getExpenditures().add(transaction); } - if (transaction.getTransaction_amount() > 0) { - transaction.setTransaction_amount(-transaction.getTransaction_amount()); - } - fundingItem.getCommitments().add(transaction); } + if (addDisbursementForCommitment && commitment && !disbursement) { + createCorrespondingDisbursement(fundingItem, adjustmentType); + } + createDonorOrg(importDataModel,orgId); if (componentName == null || componentName.isEmpty()) { @@ -489,6 +622,59 @@ private static Funding updateFunding(Funding fundingItem, ImportDataModel import return fundingItem; } + /** + * Check if a transaction with the same currency, amount, and date already exists in the list + * @param transactions List of existing transactions + * @param newTransaction Transaction to check for duplicates + * @return true if a duplicate exists, false otherwise + */ + private static boolean transactionExists(List transactions, Transaction newTransaction) { + if (transactions == null || newTransaction == null) { + return false; + } + + return transactions.stream().anyMatch(existing -> + existing.getCurrency() != null && existing.getCurrency().equals(newTransaction.getCurrency()) && + Double.compare(existing.getTransaction_amount(), newTransaction.getTransaction_amount()) == 0 && + ((existing.isInferredTransactionDate() || newTransaction.isInferredTransactionDate()) + || Objects.equals(existing.getTransaction_date(), newTransaction.getTransaction_date())) && + Objects.equals(existing.getAdjustment_type(), newTransaction.getAdjustment_type()) + ); + } + + /** + * Creates a corresponding disbursement transaction for each commitment transaction. + * The disbursement will have the same amount, currency, and date as the commitment. + * @param funding The funding object containing commitments + * @param adjustmentType The adjustment type ("actual" or "planned") + */ + private static void createCorrespondingDisbursement(Funding funding, String adjustmentType) { + if (funding == null || funding.getCommitments() == null || funding.getCommitments().isEmpty()) { + return; + } + + for (Transaction commitment : funding.getCommitments()) { + Transaction disbursement = new Transaction(); + disbursement.setCurrency(commitment.getCurrency()); + disbursement.setTransaction_amount(commitment.getTransaction_amount()); + disbursement.setTransaction_date(commitment.getTransaction_date()); + disbursement.setInferredTransactionDate(commitment.isInferredTransactionDate()); + disbursement.setFixed_exchange_rate(commitment.getFixed_exchange_rate()); + + // Set the adjustment type for disbursement + Session session = getSession(); + Long adjType = getCategoryValue("adjustmentType", CategoryConstants.ADJUSTMENT_TYPE_KEY, adjustmentType); + disbursement.setAdjustment_type(adjType); + + // Add the disbursement if it doesn't already exist + if (!transactionExists(funding.getDisbursements(), disbursement)) { + funding.getDisbursements().add(disbursement); + logger.info("Created corresponding disbursement transaction: amount={}, date={}, currency={}", + disbursement.getTransaction_amount(), disbursement.getTransaction_date(), disbursement.getCurrency()); + } + } + } + private static Long getOrganizationRole(Session session) { if (ConstantsMap.containsKey("orgRole")) { @@ -511,26 +697,40 @@ private static Long getOrganizationRole(Session session) { } private static Long getCurrencyId(Session session, String currencyCode) { - - if (ConstantsMap.containsKey("currencyId")) { - Long val = ConstantsMap.get("currencyId"); + if (currencyCode == null) { + currencyCode = "USD"; + } + String cacheKey = "currencyId_" + currencyCode; + if (ConstantsMap.containsKey(cacheKey)) { + Long val = ConstantsMap.get(cacheKey); logger.info("In cache... currency: " + val); return val; - } if (!session.isOpen()) { session = PersistenceManager.getRequestDBSession(); } - if (currencyCode == null) { - currencyCode = "USD"; - } String hql = "SELECT ac.ampCurrencyId FROM " + AmpCurrency.class.getName() + " ac " + "WHERE ac.currencyCode = :currencyCode"; Query query = session.createQuery(hql); query.setString("currencyCode", currencyCode); Long currencyId = (Long) query.uniqueResult(); - ConstantsMap.put("currencyId", currencyId); + + // If currency not found, create it + if (currencyId == null) { + logger.info("Currency not found: {}. Creating new currency.", currencyCode); + AmpCurrency newCurrency = new AmpCurrency(); + newCurrency.setCurrencyCode(currencyCode); + newCurrency.setCurrencyName(currencyCode); // Use code as name if not specified + newCurrency.setActiveFlag(1); // Active by default + newCurrency.setVirtual(false); + session.save(newCurrency); + session.flush(); + currencyId = newCurrency.getAmpCurrencyId(); + logger.info("Created new currency: {} (id={})", currencyCode, currencyId); + } + + ConstantsMap.put(cacheKey, currencyId); return currencyId; } @@ -563,6 +763,95 @@ private static Long getCategoryValue(String constantKey, String categoryKey, Str return categoryId; } + /** + * Looks up a category value ID by its category key and value name. + * @param categoryKey the category class key (e.g. "procurement_system") + * @param valueName the value name from the file (e.g. "National Competitive Bidding") + * @param session current Hibernate session + * @return category value id, or null if not found + */ + public static Long getCategoryValueByName(String categoryKey, String valueName, Session session) { + if (valueName == null || valueName.trim().isEmpty()) { + return null; + } + String cacheKey = "catVal_" + categoryKey + "_" + valueName; + if (ConstantsMap.containsKey(cacheKey)) { + return ConstantsMap.get(cacheKey); + } + String hql = "SELECT s FROM " + AmpCategoryValue.class.getName() + " s JOIN s.ampCategoryClass c WHERE c.keyName = :categoryKey"; + Query query = session.createQuery(hql); + query.setParameter("categoryKey", categoryKey); + List values = query.list(); + for (Object val : values) { + AmpCategoryValue cv = (AmpCategoryValue) val; + if (cv.getValue() != null && cv.getValue().trim().equalsIgnoreCase(valueName.trim())) { + ConstantsMap.put(cacheKey, cv.getId()); + logger.info("Found category value: " + cv.getValue() + " (id=" + cv.getId() + ") for key=" + categoryKey); + return cv.getId(); + } + } + logger.warn("Category value not found for key=" + categoryKey + ", value=" + valueName); + return null; + } + + /** + * Resolves activity (project) status by value: looks up existing category value for ACTIVITY_STATUS_KEY; + * if not found in DB, creates a new category value and returns its id. + * @param statusValue value from the file (e.g. "Ongoing", "Completed") + * @param session current session (used for create and flush) + * @return category value id, or null if statusValue is null/empty + */ + public static Long getOrCreateActivityStatusCategoryValue(String statusValue, Session session) { + if (statusValue == null || statusValue.trim().isEmpty()) return null; + String trimmed = statusValue.trim(); + String cacheKey = "statusId_" + trimmed; + if (ConstantsMap.containsKey(cacheKey)) { + return ConstantsMap.get(cacheKey); + } + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + String hql = "SELECT s FROM " + AmpCategoryValue.class.getName() + " s JOIN s.ampCategoryClass c WHERE c.keyName = :categoryKey"; + Query query = session.createQuery(hql); + query.setParameter("categoryKey", CategoryConstants.ACTIVITY_STATUS_KEY); + @SuppressWarnings("unchecked") + List values = (List) query.list(); + if (values != null) { + for (AmpCategoryValue cv : values) { + if (cv.getValue() != null && cv.getValue().equalsIgnoreCase(trimmed)) { + Long id = cv.getId(); + ConstantsMap.put(cacheKey, id); + return id; + } + } + } + AmpCategoryClass categoryClass = CategoryManagerUtil.loadAmpCategoryClassByKey(CategoryConstants.ACTIVITY_STATUS_KEY); + if (categoryClass == null) { + logger.warn("Activity status category class not found; cannot create value: " + trimmed); + return null; + } + try { + AmpCategoryValue newValue = new AmpCategoryValue(); + newValue.setValue(trimmed); + newValue.setAmpCategoryClass(categoryClass); + if (categoryClass.getPossibleValues() == null) { + categoryClass.setPossibleValues(new java.util.ArrayList<>()); + } + newValue.setIndex(categoryClass.getPossibleValues().size()); + session.save(newValue); + session.flush(); + Long id = newValue.getId(); + if (id != null) { + ConstantsMap.put(cacheKey, id); + logger.info("Created new activity status category value: " + trimmed + " (id=" + id + ")"); + return id; + } + } catch (Exception e) { + logger.warn("Failed to create activity status value: " + trimmed, e); + } + return null; + } + public static AmpActivityVersion existingActivity(String projectTitle, String projectCode, Session session) { if ((projectTitle == null || projectTitle.trim().isEmpty()) && (projectCode == null || projectCode.trim().isEmpty())) { @@ -571,43 +860,126 @@ public static AmpActivityVersion existingActivity(String projectTitle, String pr if (!session.isOpen()) { session = PersistenceManager.getRequestDBSession(); } - String hql = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a " + - "WHERE a.name = :name"; - Query query = session.createQuery(hql); - query.setCacheable(true); - query.setParameter("name", projectTitle, StringType.INSTANCE); -// query.setString("projectCode", projectCode); - List ampActivityVersions = query.list(); - return !ampActivityVersions.isEmpty() ? ampActivityVersions.get(ampActivityVersions.size() - 1) : null; + // Prefer project code if provided + if (projectCode != null && !projectCode.trim().isEmpty()) { + String hqlByCode = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a LEFT JOIN FETCH a.activityCreator WHERE a.projectCode = :projectCode"; + Query queryByCode = session.createQuery(hqlByCode); + queryByCode.setCacheable(true); + queryByCode.setParameter("projectCode", projectCode.trim(), StringType.INSTANCE); + List byCode = queryByCode.list(); + if (!byCode.isEmpty()) { + return byCode.get(byCode.size() - 1); + } + } + // Fall back to project title (name) + if (projectTitle != null && !projectTitle.trim().isEmpty()) { + String hql = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a LEFT JOIN FETCH a.activityCreator WHERE a.name = :name"; + Query query = session.createQuery(hql); + query.setCacheable(true); + query.setParameter("name", projectTitle.trim(), StringType.INSTANCE); + List ampActivityVersions = query.list(); + return !ampActivityVersions.isEmpty() ? ampActivityVersions.get(ampActivityVersions.size() - 1) : null; + } + return null; } - public static void setStatus(ImportDataModel importDataModel) { - Long statusId = getCategoryValue("statusId", CategoryConstants.ACTIVITY_STATUS_KEY, ""); - importDataModel.setActivity_status(statusId); - importDataModel.setApproval_status(ApprovalStatus.started.getId()); + /** + * Sets default activity status and approval status on the import model. + * If activity_status is already set (e.g. from Project Status column), it is left unchanged. + */ + public static void setStatus(ImportDataModel importDataModel, boolean validateActivities) { + setStatus(importDataModel, validateActivities, null); } - public static void importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject, String componentName, String componentCode, Long responsibleOrgId, List fundings, AmpActivityVersion existing) throws JsonProcessingException { - if (!session.isOpen()) { + /** + * Sets default activity status and approval status on the import model. + * If activity_status is already set (e.g. from Project Status column), it is left unchanged. + */ + public static void setStatus(ImportDataModel importDataModel, boolean validateActivities, Long defaultActivityStatusId) { + if (importDataModel.getActivity_status() == null) { + Long statusId = defaultActivityStatusId != null + ? defaultActivityStatusId + : getCategoryValue("statusId", CategoryConstants.ACTIVITY_STATUS_KEY, ""); + importDataModel.setActivity_status(statusId); + } + if (validateActivities) { + logger.info("validateActivities=true: setting approval_status=approved in setStatus"); + importDataModel.setApproval_status(ApprovalStatus.approved.getId()); + } else { + importDataModel.setApproval_status(ApprovalStatus.started.getId()); + } + } + + private static final String CREATED_BY_KEY = "created_by"; + + /** + * Ensures created_by in the activity map is set to a valid team member id when null, + * so the activity API validator does not reject with "(Invalid field value) created_by". + * For new activities uses current user; for updates uses existing activity's creator only + * when that creator is present (never overwrite with current user for existing activities). + */ + private static void ensureCreatedBySet(Map map, AmpActivityVersion existing) { + if (existing != null) { + AmpTeamMember creator = existing.getActivityCreator(); + if (creator == null) { + // Existing activity has no creator (legacy); API expects null. + map.put(CREATED_BY_KEY, null); + return; + } + map.put(CREATED_BY_KEY, creator.getAmpTeamMemId()); + return; + } + Object createdBy = map.get(CREATED_BY_KEY); + if (createdBy != null) { + return; + } + AmpTeamMember currentMember = TeamUtil.getCurrentAmpTeamMember(); + if (currentMember != null) { + map.put(CREATED_BY_KEY, currentMember.getAmpTeamMemId()); + } + } + + /** @return activity ID on success, null on skip or failure */ + public static Long importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject, String componentName, String componentCode, Long responsibleOrgId, List fundings, Long existingActivityId, boolean validateActivities) throws JsonProcessingException { + if (session == null || !session.isOpen()) { session = PersistenceManager.getRequestDBSession(); } + + // Re-fetch existing activity in this transaction if ID is provided to avoid detached entity issues + AmpActivityVersion existing = null; + if (existingActivityId != null) { + existing = session.get(AmpActivityVersion.class, existingActivityId); + } ActivityImportRules rules = new ActivityImportRules(true, false, true); ObjectMapper objectMapper = new ObjectMapper(); objectMapper.configure(ESCAPE_NON_ASCII, false); // Disable escaping of non-ASCII characters during serialization objectMapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true); + normalizeLocationPercentages(importDataModel); Map map = objectMapper .convertValue(importDataModel, new TypeReference>() { }); + // Remove null values and "null" strings from the map to avoid API validation errors + map.entrySet().removeIf(entry -> entry.getValue() == null || "null".equals(String.valueOf(entry.getValue()))); + + // Do not send indicators in the payload so activity/update does not replace or clear existing indicators. + // Indicator data is appended separately in addIndicatorDataToActivity. + map.remove("indicators"); JsonApiResponse response; logger.info("Data model object: " + importDataModel); if (importDataModel.getProject_title().trim().isEmpty() && importDataModel.getProject_code().trim().isEmpty()) { logger.info("Project title and code are empty. Skipping import"); importedProject.setImportStatus(ImportStatus.SKIPPED); - return; + return null; } if (existing == null) { + ensureCreatedBySet(map, null); + if (validateActivities) { + logger.info("validateActivities=true: setting approval_status=approved and is_draft=false for new activity"); + map.put("approval_status", "approved"); + map.put("is_draft", false); + } logger.info("New activity"); importedProject.setNewProject(true); response = ActivityInterchangeUtils.importActivity(map, false, rules, "activity/new"); @@ -616,103 +988,321 @@ public static void importTheData(ImportDataModel importDataModel, Session sessio importedProject.setNewProject(false); importDataModel.setInternal_id(existing.getAmpActivityId()); importDataModel.setAmp_id(existing.getAmpId()); - ActivityGroup activityGroup = new ActivityGroup(); - activityGroup.setVersion(existing.getAmpActivityGroup().getVersion()); - importDataModel.setActivity_group(activityGroup); - importDataModel.setProject_title(existing.getName()); - importDataModel.setProject_code(!Objects.equals(importDataModel.getProject_code(), "") ? importDataModel.getProject_code() : existing.getProjectCode()); + // Only set activity group if it exists and has the data we need + if (existing.getAmpActivityGroup() != null) { + ActivityGroup activityGroup = new ActivityGroup(); + activityGroup.setVersion(existing.getAmpActivityGroup().getVersion()); + importDataModel.setActivity_group(activityGroup); + } + importDataModel.setProject_title(existing.getName() != null ? existing.getName() : ""); + importDataModel.setProject_code(!Objects.equals(importDataModel.getProject_code(), "") ? importDataModel.getProject_code() : (existing.getProjectCode() != null ? existing.getProjectCode() : "")); updateFundingOrgsAndSectorsWithAlreadyExisting(existing, importDataModel); + // Merge existing activity locations into payload so we only add (row + existing), never remove + mergeExistingActivityLocationsIntoImport(existing, importDataModel); + ensureImplementationLevelWhenHasLocations(importDataModel, session); + normalizeLocationPercentages(importDataModel); map = objectMapper .convertValue(importDataModel, new TypeReference>() { }); - response = ActivityInterchangeUtils.importActivity(map, true, rules, "activity/update"); + // Remove null values and "null" strings from the map to avoid API validation errors + map.entrySet().removeIf(entry -> entry.getValue() == null || "null".equals(String.valueOf(entry.getValue()))); + + map.remove("indicators"); // preserve existing indicators; we append in addIndicatorDataToActivity + // Do not replace programs; avoids StaleStateException when deleting AMP_ACTIVITY_PROGRAM rows + map.remove("national_plan_objective"); + map.remove("primary_programs"); + map.remove("secondary_programs"); + map.remove("tertiary_programs"); + // Avoid triggering merge of contacts/documents that may reference deleted rows (ObjectNotFoundException) + map.remove("activity_contacts"); + map.remove("activityContacts"); + map.remove("donor_contact_information"); + map.remove("project_coordinator_contact_information"); + map.remove("sector_ministry_contact_information"); + map.remove("mofed_contact_information"); + map.remove("implementing_executing_agency_contact_information"); + evictActivityFromSecondLevelCache(existing.getAmpActivityId()); + ensureCreatedBySet(map, existing); + if (validateActivities) { + logger.info("validateActivities=true: setting approval_status=approved and is_draft=false for existing activity"); + map.put("approval_status", "approved"); + map.put("is_draft", false); + } + // All data from 'existing' has been extracted into 'map'. Clear the session first-level + // cache before handing off to ActivityGatekeeper so its internal doInTransaction starts + // with a clean context. Without this, entities loaded above remain in the session action + // queue and Hibernate raises HHH000099 (possible non-threadsafe access to session) when + // EntityInsertAction.execute() checks the persistence context during flush. + session.clear(); + try { + response = ActivityInterchangeUtils.importActivity(map, true, rules, "activity/update"); + } catch (Exception e) { + logger.error("Activity import failed for row", e); + importedProject.setImportStatus(ImportStatus.FAILED); + Map> errMap = new LinkedHashMap<>(); + errMap.put("1", Collections.singletonList("Internal Error : [" + (e.getMessage() != null ? e.getMessage() : "Activity import failed") + "]")); + response = new JsonApiResponse<>(errMap, null, null, null); + } } + Long activityId = null; if (response != null) { if (!response.getErrors().isEmpty()) { importedProject.setImportStatus(ImportStatus.FAILED); } else { importedProject.setImportStatus(ImportStatus.SUCCESS); + activityId = existing != null ? existing.getAmpActivityId() : (Long) response.getContent().getAmpActivityId(); logger.info("Successfully imported the project. Now adding component if present"); logger.info("--------------------------------"); logger.info("Component name at start: " + componentName); if (componentName != null && !componentName.isEmpty()) { addComponentsAndProjectCode(response, componentName, componentCode, responsibleOrgId, fundings, importDataModel.getProject_code()); } -// logger.info("Updating expenditures ................"); -// updateExpendituresIfAny(response); - } } String resp = objectMapper.writeValueAsString(response); importedProject.setImportResponse(resp); - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); + try { + if (session == null || !session.isOpen()) { + // After importActivityInNewSession fails, thread's current session may be closed; use a fresh transaction to save status + PersistenceManager.doInTransaction(s -> { + s.saveOrUpdate(importedProject); + s.flush(); + }); + } else { + session.saveOrUpdate(importedProject); + session.flush(); + } + } catch (Exception e) { + logger.warn("Could not save import status for imported project (response already set): {}", e.getMessage()); } - session.saveOrUpdate(importedProject); - session.flush(); logger.info("Imported project: " + importedProject); + return activityId; } private static void updateFundingOrgsAndSectorsWithAlreadyExisting(AmpActivityVersion ampActivityVersion, ImportDataModel importDataModel) { if (ampActivityVersion.getFunding() != null) { + Hibernate.initialize(ampActivityVersion.getFunding()); Long adjType = getCategoryValue("adjustmentType", CategoryConstants.ADJUSTMENT_TYPE_KEY, ""); Long assType = getCategoryValue("assistanceType", CategoryConstants.TYPE_OF_ASSISTENCE_KEY, ""); Long finInstrument = getCategoryValue("finInstrument", CategoryConstants.FINANCING_INSTRUMENT_KEY, ""); + if (importDataModel.getFundings() == null) importDataModel.setFundings(new HashSet<>()); for (AmpFunding ampFunding : ampActivityVersion.getFunding()) { - Funding funding = new Funding(); - funding.setDonor_organization_id(ampFunding.getAmpDonorOrgId().getAmpOrgId()); - funding.setType_of_assistance(ampFunding.getTypeOfAssistance() != null ? ampFunding.getTypeOfAssistance().getId() : assType); - funding.setFinancing_instrument(ampFunding.getFinancingInstrument() != null ? ampFunding.getFinancingInstrument().getId() : finInstrument); - funding.setSource_role(ampFunding.getSourceRole().getAmpRoleId()); - for (AmpFundingDetail ampFundingDetail : ampFunding.getFundingDetails()) { - Transaction transaction = new Transaction(); - transaction.setCurrency(ampFundingDetail.getAmpCurrencyId().getAmpCurrencyId()); - transaction.setAdjustment_type(ampFundingDetail.getAdjustmentType() != null ? ampFundingDetail.getAdjustmentType().getId() : adjType); - transaction.setTransaction_amount(ampFundingDetail.getTransactionAmount()); - if (ampFundingDetail.getTransactionDate() != null) { - - transaction.setTransaction_date(getFundingDate(ampFundingDetail.getTransactionDate().toInstant() - .atZone(ZoneId.systemDefault()) - .toLocalDate().toString())); + Long existingId = ampFunding.getAmpFundingId(); + Long donorOrgId = ampFunding.getAmpDonorOrgId().getAmpOrgId(); + Long typeOfAssistance = ampFunding.getTypeOfAssistance() != null ? ampFunding.getTypeOfAssistance().getId() : assType; + Long financingInstrument = ampFunding.getFinancingInstrument() != null ? ampFunding.getFinancingInstrument().getId() : finInstrument; + Long sourceRole = ampFunding.getSourceRole().getAmpRoleId(); + + // Check if the Excel importer already added a new funding entry (no ID yet) for the same + // donor+role+type combination. If so, mark it with the existing funding_id so the API + // updates the existing DB record instead of inserting a duplicate. + Funding matchedNewFunding = null; + for (Funding newFunding : importDataModel.getFundings()) { + if (newFunding.getFunding_id() == null + && Objects.equals(donorOrgId, newFunding.getDonor_organization_id()) + && Objects.equals(sourceRole, newFunding.getSource_role()) + && Objects.equals(typeOfAssistance, newFunding.getType_of_assistance()) + && Objects.equals(financingInstrument, newFunding.getFinancing_instrument())) { + matchedNewFunding = newFunding; + break; } - transaction.setFixed_exchange_rate(ampFundingDetail.getFixedExchangeRate()); - if (ampFundingDetail.getTransactionType() == 0) { - funding.getCommitments().add(transaction); - } else if (ampFundingDetail.getTransactionType() == 1) { - funding.getDisbursements().add(transaction); + } + + if (matchedNewFunding != null) { + // Stamp the existing DB funding_id so the API updates the existing record instead of inserting. + matchedNewFunding.setFunding_id(existingId); + // Also merge the existing DB transactions into the Excel entry so the API's + // removeByIdExcept doesn't delete them (DB transactions carry transaction_id; + // the Excel transactions have none, so they'd be the only ones in jsonIds={null}, + // causing all DB transactions to be removed on flush). + // transactionExists guards against re-adding a transaction already supplied by Excel. + if (ampFunding.getFundingDetails() != null) { + Hibernate.initialize(ampFunding.getFundingDetails()); + for (AmpFundingDetail ampFundingDetail : ampFunding.getFundingDetails()) { + Transaction transaction = new Transaction(); + if (ampFundingDetail.getAmpFundDetailId() != null) transaction.setTransaction_id(ampFundingDetail.getAmpFundDetailId()); + transaction.setCurrency(ampFundingDetail.getAmpCurrencyId().getAmpCurrencyId()); + transaction.setAdjustment_type(ampFundingDetail.getAdjustmentType() != null ? ampFundingDetail.getAdjustmentType().getId() : adjType); + transaction.setTransaction_amount(ampFundingDetail.getTransactionAmount()); + if (ampFundingDetail.getTransactionDate() != null) { + transaction.setTransaction_date(getFundingDate(ampFundingDetail.getTransactionDate().toInstant() + .atZone(ZoneId.systemDefault()) + .toLocalDate().toString())); + } + transaction.setFixed_exchange_rate(ampFundingDetail.getFixedExchangeRate()); + if (ampFundingDetail.getTransactionType() == 0) { + if (!transactionExists(matchedNewFunding.getCommitments(), transaction)) { + matchedNewFunding.getCommitments().add(transaction); + } + } else if (ampFundingDetail.getTransactionType() == 1) { + if (!transactionExists(matchedNewFunding.getDisbursements(), transaction)) { + matchedNewFunding.getDisbursements().add(transaction); + } + } else if (ampFundingDetail.getTransactionType() == 2) { + if (!transactionExists(matchedNewFunding.getExpenditures(), transaction)) { + matchedNewFunding.getExpenditures().add(transaction); + } + } + } } + continue; } + // No Excel entry for this existing funding — preserve it so it is not deleted. + Funding funding = new Funding(); + if (existingId != null) funding.setFunding_id(existingId); + funding.setDonor_organization_id(donorOrgId); + funding.setType_of_assistance(typeOfAssistance); + funding.setFinancing_instrument(financingInstrument); + funding.setSource_role(sourceRole); + if (ampFunding.getFundingDetails() != null) { + Hibernate.initialize(ampFunding.getFundingDetails()); + for (AmpFundingDetail ampFundingDetail : ampFunding.getFundingDetails()) { + Transaction transaction = new Transaction(); + if (ampFundingDetail.getAmpFundDetailId() != null) transaction.setTransaction_id(ampFundingDetail.getAmpFundDetailId()); + transaction.setCurrency(ampFundingDetail.getAmpCurrencyId().getAmpCurrencyId()); + transaction.setAdjustment_type(ampFundingDetail.getAdjustmentType() != null ? ampFundingDetail.getAdjustmentType().getId() : adjType); + transaction.setTransaction_amount(ampFundingDetail.getTransactionAmount()); + if (ampFundingDetail.getTransactionDate() != null) { + transaction.setTransaction_date(getFundingDate(ampFundingDetail.getTransactionDate().toInstant() + .atZone(ZoneId.systemDefault()) + .toLocalDate().toString())); + } + transaction.setFixed_exchange_rate(ampFundingDetail.getFixedExchangeRate()); + if (ampFundingDetail.getTransactionType() == 0) { + funding.getCommitments().add(transaction); + } else if (ampFundingDetail.getTransactionType() == 1) { + funding.getDisbursements().add(transaction); + } else if (ampFundingDetail.getTransactionType() == 2) { + funding.getExpenditures().add(transaction); + } + } + } + importDataModel.getFundings().add(funding); } } if (ampActivityVersion.getOrgrole() != null && !ampActivityVersion.getOrgrole().isEmpty()) { for (AmpOrgRole ampOrgRole : ampActivityVersion.getOrgrole()) { - if (ampOrgRole.getRole().getRoleCode().equalsIgnoreCase("DN")) { - createDonorOrg(importDataModel,ampOrgRole.getOrganisation().getAmpOrgId()); - } else if (ampOrgRole.getRole().getRoleCode().equalsIgnoreCase("EA")) { - Organization responsibleOrg = new Organization(); - responsibleOrg.setOrganization(ampOrgRole.getOrganisation().getAmpOrgId()); - importDataModel.getResponsible_organization().add(responsibleOrg); - } else if (ampOrgRole.getRole().getRoleCode().equalsIgnoreCase("BA")) { - Organization beneficiaryAgency = new Organization(); - beneficiaryAgency.setOrganization(ampOrgRole.getOrganisation().getAmpOrgId()); - importDataModel.getBeneficiary_agency().add(beneficiaryAgency); - + if (ampOrgRole.getRole() == null) continue; + String roleCode = ampOrgRole.getRole().getRoleCode(); + if (roleCode == null) continue; + Long orgId = ampOrgRole.getOrganisation().getAmpOrgId(); + Long orgRoleId = ampOrgRole.getAmpOrgRoleId(); + if (roleCode.equalsIgnoreCase("DN")) { + createDonorOrg(importDataModel, orgId, orgRoleId); + } else if (roleCode.equalsIgnoreCase("RO")) { + Organization org = new Organization(); + org.setOrganization(orgId); + if (orgRoleId != null) org.setId(orgRoleId); + importDataModel.getResponsible_organization().add(org); + } else if (roleCode.equalsIgnoreCase("BA")) { + Organization org = new Organization(); + org.setOrganization(orgId); + if (orgRoleId != null) org.setId(orgRoleId); + importDataModel.getBeneficiary_agency().add(org); + } else if (roleCode.equalsIgnoreCase("EA")) { + Organization org = new Organization(); + org.setOrganization(orgId); + if (orgRoleId != null) org.setId(orgRoleId); + importDataModel.getExecuting_agency().add(org); + } else if (roleCode.equalsIgnoreCase("IA")) { + Organization org = new Organization(); + org.setOrganization(orgId); + if (orgRoleId != null) org.setId(orgRoleId); + importDataModel.getImplementing_agency().add(org); + } else if (roleCode.equalsIgnoreCase("CA")) { + Organization org = new Organization(); + org.setOrganization(orgId); + if (orgRoleId != null) org.setId(orgRoleId); + importDataModel.getContracting_agency().add(org); } } } - if (ampActivityVersion.getSectors() != null && !ampActivityVersion.getSectors().isEmpty()) { + Hibernate.initialize(ampActivityVersion.getSectors()); for (AmpActivitySector ampActivitySector : ampActivityVersion.getSectors()) { - createSector(importDataModel,ampActivitySector.getClassificationConfig().getName().equalsIgnoreCase("primary"),ampActivitySector.getSectorId().getAmpSectorId()); + if (ampActivitySector.getSectorId() == null) continue; + boolean primary = ampActivitySector.getClassificationConfig() != null && ampActivitySector.getClassificationConfig().isPrimary(); + createSector(importDataModel, primary, ampActivitySector.getSectorId().getAmpSectorId(), ampActivitySector.getAmpActivitySectorId()); } } } + /** + * For an existing activity, merges its current locations into the import payload so we only add locations + * (row locations + existing), never remove. Any existing activity location not already in importDataModel + * is added. This avoids activity/update deleting locations (e.g. those referenced by indicator connections). + */ + private static void mergeExistingActivityLocationsIntoImport(AmpActivityVersion existing, ImportDataModel importDataModel) { + if (existing == null || importDataModel == null) return; + if (existing.getLocations() == null) return; + Hibernate.initialize(existing.getLocations()); + Set alreadyInImport = new HashSet<>(); + if (importDataModel.getLocations() != null) { + for (Location loc : importDataModel.getLocations()) { + if (loc != null && loc.getLocation() != null) alreadyInImport.add(loc.getLocation()); + } + } + for (AmpActivityLocation aal : existing.getLocations()) { + AmpCategoryValueLocations loc = aal.getLocation(); + if (loc == null) continue; + Long locId = loc.getId(); + if (locId == null || alreadyInImport.contains(locId)) continue; + if (importDataModel.getLocations() == null) importDataModel.setLocations(new HashSet<>()); + double pct = aal.getLocationPercentage() != null ? aal.getLocationPercentage().doubleValue() : 100.0; + // Include aal.getId() (amp_activity_location_id) so the API matches and keeps this row; otherwise removeByIdExcept drops it and Hibernate deletes it (FK violation if referenced by amp_indicator_connection). + Long aalId = aal.getId(); + importDataModel.getLocations().add(aalId != null ? new Location(aalId, locId, pct) : new Location(locId, pct)); + alreadyInImport.add(locId); + } + } + + /** + * Scales location percentages so they sum to 100, as required by activity validation. + * If there are no locations or sum is 0, does nothing. + */ + private static void normalizeLocationPercentages(ImportDataModel importDataModel) { + if (importDataModel == null || importDataModel.getLocations() == null || importDataModel.getLocations().isEmpty()) + return; + Set locs = importDataModel.getLocations(); + double sum = 0; + for (Location loc : locs) { + Double pct = loc.getLocation_percentage(); + sum += (pct != null ? pct : 0); + } + if (sum <= 0) return; + if (Math.abs(sum - 100.0) < 0.001) return; // already 100 + List list = new ArrayList<>(locs); + double scale = 100.0 / sum; + double running = 0; + for (int i = 0; i < list.size(); i++) { + Location loc = list.get(i); + double v; + if (i == list.size() - 1) { + v = 100.0 - running; // last one gets remainder so total is exactly 100 + } else { + Double pct = loc.getLocation_percentage(); + v = (pct != null ? pct : 0) * scale; + running += v; + } + loc.setLocation_percentage(v); + } + } + + /** + * When the payload has locations, implementation level is required. Sets default if missing (e.g. after merging locations for existing activity). + */ + private static void ensureImplementationLevelWhenHasLocations(ImportDataModel importDataModel, Session session) { + if (importDataModel == null || importDataModel.getLocations() == null || importDataModel.getLocations().isEmpty()) + return; + if (importDataModel.getImplementation_level() != null) return; + updateImpLevels(importDataModel, session); + } + static void updateExpendituresIfAny(JsonApiResponse response) { Long activityId = (Long) response.getContent().getAmpActivityId(); Session session = PersistenceManager.getRequestDBSession(); @@ -950,11 +1540,19 @@ protected static AmpCurrency getAmpCurrencyById(Long id) { } - public static void updateSectors(ImportDataModel importDataModel, String name, Session session, boolean primary, String subSector) { + public static void updateSectors(ImportDataModel importDataModel, String name, Session session, boolean primary, + String subSector, boolean createMissingSectors, String importerSectorField) { if (subSector!=null && !subSector.isEmpty()) { name = subSector; } + for (String sectorName : splitMultiValues(name)) { + updateSingleSector(importDataModel, sectorName, session, primary, createMissingSectors, importerSectorField); + } + } + + private static void updateSingleSector(ImportDataModel importDataModel, String name, Session session, boolean primary, + boolean createMissingSectors, String importerSectorField) { if (ConstantsMap.containsKey("sector_" + name)) { Long sectorId = ConstantsMap.get("sector_" + name); logger.info("In cache... sector " + "sector_" + name + ":" + sectorId); @@ -965,16 +1563,31 @@ public static void updateSectors(ImportDataModel importDataModel, String name, S } String finalName = name; + String classificationName = getClassificationNameForImporterField(importerSectorField, primary); + final Long[] foundSectorId = new Long[1]; session.doWork(connection -> { - String query = primary ? "SELECT ams.amp_sector_id AS amp_sector_id, ams.name AS name FROM amp_sector ams JOIN amp_classification_config acc ON ams.amp_sec_scheme_id=acc.classification_id WHERE LOWER(ams.name) = LOWER(?) AND acc.name='Primary'" : "SELECT ams.amp_sector_id AS amp_sector_id, ams.name AS name FROM amp_sector ams JOIN amp_classification_config acc ON ams.amp_sec_scheme_id=acc.classification_id WHERE LOWER(ams.name) = LOWER(?) AND acc.name='Secondary'"; + String query = primary + ? "SELECT ams.amp_sector_id AS amp_sector_id, ams.name AS name " + + "FROM amp_sector ams " + + "JOIN amp_classification_config acc ON ams.amp_sec_scheme_id = acc.classification_id " + + "WHERE LOWER(ams.name) = LOWER(?) " + + "AND (acc.is_primary_sector = TRUE OR LOWER(acc.name) = LOWER(?))" + : "SELECT ams.amp_sector_id AS amp_sector_id, ams.name AS name " + + "FROM amp_sector ams " + + "JOIN amp_classification_config acc ON ams.amp_sec_scheme_id = acc.classification_id " + + "WHERE LOWER(ams.name) = LOWER(?) " + + "AND LOWER(acc.name) = LOWER(?)"; try (PreparedStatement statement = connection.prepareStatement(query)) { // Set the name as a parameter to the prepared statement statement.setString(1, finalName); + statement.setString(2, classificationName); + // Execute the query and process the results try (ResultSet resultSet = statement.executeQuery()) { while (resultSet.next()) { Long ampSectorId = resultSet.getLong("amp_sector_id"); + foundSectorId[0] = ampSectorId; createSector(importDataModel, primary, ampSectorId); ConstantsMap.put("sector_" + finalName, ampSectorId); } @@ -984,50 +1597,211 @@ public static void updateSectors(ImportDataModel importDataModel, String name, S logger.error("Error getting sectors", e); } }); + + if (foundSectorId[0] == null && createMissingSectors) { + Long createdSectorId = createMissingSector(finalName, session, primary, importerSectorField); + if (createdSectorId != null) { + createSector(importDataModel, primary, createdSectorId); + ConstantsMap.put("sector_" + finalName, createdSectorId); + } + } } + } + private static Long createMissingSector(String name, Session session, boolean primary, String importerSectorField) { + if (name == null || name.trim().isEmpty()) { + return null; + } + String classificationName = getClassificationNameForImporterField(importerSectorField, primary); + AmpClassificationConfiguration classificationConfig = resolveClassificationConfig(session, primary, classificationName); + if (classificationConfig == null || classificationConfig.getClassification() == null) { + logger.warn("No classification configuration found for {} sector; cannot create '{}'.", + classificationName, name); + return null; + } + try { + AmpSector newSector = new AmpSector(); + newSector.setParentSectorId(null); + newSector.setAmpOrgId(null); + newSector.setAmpSecSchemeId(SectorUtil.getAmpSectorScheme( + classificationConfig.getClassification().getAmpSecSchemeId())); + newSector.setSectorCode("101"); + newSector.setSectorCodeOfficial(name); + newSector.setName(name); + newSector.setDescription(" "); + // Keep the source sector type (Primary/Secondary/etc.) based on the mapped import column. + newSector.setType(classificationConfig.getName()); + newSector.setLanguage(null); + newSector.setVersion(null); + newSector.setDeleted(false); + DbUtil.add(newSector); + logger.info("Created missing {} sector '{}' with id={}", + classificationConfig.getName(), name, newSector.getAmpSectorId()); + return newSector.getAmpSectorId(); + } catch (Exception e) { + logger.error("Failed creating missing sector '{}': {}", name, e.getMessage(), e); + return null; + } } - public static void updateLocations(ImportDataModel importDataModel, String locationName, Session session) { - logger.info("Updating locations"); + private static AmpClassificationConfiguration resolveClassificationConfig(Session session, boolean primary, + String classificationName) { + String hql = primary + ? "FROM " + AmpClassificationConfiguration.class.getName() + " c WHERE c.primary = true" + : "FROM " + AmpClassificationConfiguration.class.getName() + " c WHERE LOWER(c.name) = LOWER(:name)"; + Query query = session.createQuery(hql); + if (!primary) { + query.setParameter("name", classificationName, StringType.INSTANCE); + } + query.setMaxResults(1); + return (AmpClassificationConfiguration) query.uniqueResult(); + } - if (ConstantsMap.containsKey("location_" + locationName)) { - Long location = ConstantsMap.get("location_" + locationName); - logger.info("In cache... location " + "location_" + locationName + ":" + location); - importDataModel.getLocations().add(new Location(location, 100.00)); + private static String getClassificationNameForImporterField(String importerSectorField, boolean primary) { + if (ImporterConstants.PRIMARY_SECTOR.equals(importerSectorField)) { + return AmpClassificationConfiguration.PRIMARY_CLASSIFICATION_CONFIGURATION_NAME; + } + if (ImporterConstants.SECONDARY_SECTOR.equals(importerSectorField)) { + return AmpClassificationConfiguration.SECONDARY_CLASSIFICATION_CONFIGURATION_NAME; + } + return primary + ? AmpClassificationConfiguration.PRIMARY_CLASSIFICATION_CONFIGURATION_NAME + : AmpClassificationConfiguration.SECONDARY_CLASSIFICATION_CONFIGURATION_NAME; + } - } else { - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); + private static List splitMultiValues(String value) { + if (value == null || value.trim().isEmpty()) return Collections.emptyList(); + List result = new ArrayList<>(); + // Support standard and locale-specific semicolons used in spreadsheet exports. + for (String part : value.split("[;\\u061B\\uFF1B]")) { + String trimmed = part.trim(); + if (trimmed.length() >= 2 && trimmed.startsWith("\"") && trimmed.endsWith("\"")) { + trimmed = trimmed.substring(1, trimmed.length() - 1).trim(); } + if (!trimmed.isEmpty()) result.add(trimmed); + } + return result; + } - session.doWork(connection -> { - String query = "SELECT acvl.id AS location_id FROM amp_category_value_location acvl WHERE LOWER(acvl.location_name) = LOWER(?)"; - try (PreparedStatement statement = connection.prepareStatement(query)) { - statement.setString(1, locationName); - try (ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - Long location = resultSet.getLong("location_id"); - logger.info("Location:" + location); - importDataModel.getLocations().add(new Location(location, 100.00)); -// importDataModel.setImplementation_location(location); - ConstantsMap.put("location_" + locationName, location); - } - } + private static List splitLocationNames(String locationNames) { + if (locationNames == null || locationNames.trim().isEmpty()) return Collections.emptyList(); + List result = new ArrayList<>(); + for (String part : locationNames.split("[,;]")) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) result.add(trimmed); + } + return result; + } - } catch (SQLException e) { - logger.error("Error getting locations", e); + public static void updateLocations(ImportDataModel importDataModel, String locationNames, Session session) { + logger.info("Updating locations"); + if (locationNames == null || locationNames.trim().isEmpty()) return; + for (String locationName : splitLocationNames(locationNames)) { + if (ConstantsMap.containsKey("location_" + locationName)) { + Long location = ConstantsMap.get("location_" + locationName); + logger.info("In cache... location " + "location_" + locationName + ":" + location); + importDataModel.getLocations().add(new Location(location, 100.00)); + + } else { + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); } - }); + final String locationNameFinal = locationName; + session.doWork(connection -> { + String query = "SELECT acvl.id AS location_id FROM amp_category_value_location acvl WHERE LOWER(acvl.location_name) = LOWER(?)"; + try (PreparedStatement statement = connection.prepareStatement(query)) { + statement.setString(1, locationNameFinal); + + try (ResultSet resultSet = statement.executeQuery()) { + while (resultSet.next()) { + Long location = resultSet.getLong("location_id"); + logger.info("Location:" + location); + importDataModel.getLocations().add(new Location(location, 100.00)); + ConstantsMap.put("location_" + locationNameFinal, location); + } + } + } catch (SQLException e) { + logger.error("Error getting locations", e); + } + }); + } } - updateImpLevels(importDataModel,session); + updateImpLevels(importDataModel, session); + } + public static void applyDefaultLocation(ImportDataModel importDataModel, Long locationId, Session session) { + if (importDataModel == null || locationId == null) { + return; + } + if (importDataModel.getLocations() == null) { + importDataModel.setLocations(new HashSet<>()); + } + boolean alreadyPresent = importDataModel.getLocations().stream() + .filter(Objects::nonNull) + .anyMatch(location -> Objects.equals(location.getLocation(), locationId)); + if (!alreadyPresent) { + importDataModel.getLocations().add(new Location(locationId, 100.00)); + } + updateImpLevels(importDataModel, session); + } + /** + * Ensures the activity has an activity location for the given location name (for indicator location). + * If the location is not already on the activity, resolves it by name and adds it. + * @return the AmpActivityLocation for the name, or null if the location name cannot be resolved + */ + private static AmpActivityLocation getOrAddActivityLocationForName(AmpActivityVersion activity, String locationName, Session session) { + if (activity == null || locationName == null || locationName.trim().isEmpty()) return null; + locationName = locationName.trim(); + if (ConstantsMap.containsKey("location_" + locationName)) { + Long locationId = ConstantsMap.get("location_" + locationName); + AmpCategoryValueLocations loc = session.get(AmpCategoryValueLocations.class, locationId); + if (loc == null) return null; + AmpActivityLocation aal = new AmpActivityLocation(); + aal.setActivity(activity); + aal.setLocation(loc); + aal.setLocationPercentage(100f); + if (activity.getLocations() == null) activity.setLocations(new HashSet<>()); + activity.getLocations().add(aal); + session.save(aal); + session.flush(); + return aal; + } + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + final String locationNameFinal = locationName; + final Long[] foundId = new Long[1]; + session.doWork(connection -> { + String query = "SELECT acvl.id AS location_id FROM amp_category_value_location acvl WHERE LOWER(acvl.location_name) = LOWER(?)"; + try (PreparedStatement statement = connection.prepareStatement(query)) { + statement.setString(1, locationNameFinal); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + foundId[0] = resultSet.getLong("location_id"); + ConstantsMap.put("location_" + locationNameFinal, foundId[0]); + } + } + } catch (SQLException e) { + logger.error("Error resolving location by name: " + locationNameFinal, e); + } + }); + if (foundId[0] == null) return null; + AmpCategoryValueLocations loc = session.get(AmpCategoryValueLocations.class, foundId[0]); + if (loc == null) return null; + AmpActivityLocation aal = new AmpActivityLocation(); + aal.setActivity(activity); + aal.setLocation(loc); + aal.setLocationPercentage(100f); + if (activity.getLocations() == null) activity.setLocations(new HashSet<>()); + activity.getLocations().add(aal); + session.save(aal); + return aal; } public static void updateImpLevels(ImportDataModel importDataModel, Session session) @@ -1063,23 +1837,39 @@ public static void updateImpLevels(ImportDataModel importDataModel, Session sess } private static void createSector(ImportDataModel importDataModel, boolean primary, Long ampSectorId) { - Sector sector1 = new Sector(); + createSector(importDataModel, primary, ampSectorId, null); + } + private static void createSector(ImportDataModel importDataModel, boolean primary, Long ampSectorId, Long sectorPkId) { + Sector sector1 = new Sector(); sector1.setSector(ampSectorId); + if (sectorPkId != null) sector1.setId(sectorPkId); if (primary) { - importDataModel.getPrimary_sectors().add(sector1); + boolean added = importDataModel.getPrimary_sectors().add(sector1); + if (!added && sectorPkId != null) { + // Sector already present from import row (id=null). Stamp the existing + // activity-sector PK so the API updates rather than inserts. + importDataModel.getPrimary_sectors().stream() + .filter(s -> Objects.equals(s.getSector(), ampSectorId)) + .findFirst() + .ifPresent(s -> s.setId(sectorPkId)); + } Map percentages = divide100(importDataModel.getPrimary_sectors().size()); - int index=0; + int index = 0; for (Sector sec : importDataModel.getPrimary_sectors()) { sec.setSector_percentage(percentages.get(index)); index++; } - } - else - { - importDataModel.getSecondary_sectors().add(sector1); + } else { + boolean added = importDataModel.getSecondary_sectors().add(sector1); + if (!added && sectorPkId != null) { + importDataModel.getSecondary_sectors().stream() + .filter(s -> Objects.equals(s.getSector(), ampSectorId)) + .findFirst() + .ifPresent(s -> s.setId(sectorPkId)); + } Map percentages = divide100(importDataModel.getSecondary_sectors().size()); - int index=0; + int index = 0; for (Sector sec : importDataModel.getSecondary_sectors()) { sec.setSector_percentage(percentages.get(index)); index++; @@ -1108,67 +1898,289 @@ private static Long getRandomOrg(Session session) } - public static Long updateOrgs(ImportDataModel importDataModel, String name, String code, Session session, String type) + public static Long updateOrgs(ImportDataModel importDataModel, String name, String code, Session session, String type) { + return updateOrgs(importDataModel, name, code, session, type, false, null, null, false); + } + + public static Long updateOrgs(ImportDataModel importDataModel, String name, String code, Session session, String type, boolean createMissingOrgs, Long orgGroupId, String importedOrgGroupName, boolean createMissingOrgGroups) + { + List names = splitMultiValues(name); + List codes = splitMultiValues(code); + Long lastOrgId = null; + for (int i = 0; i < names.size(); i++) { + String singleName = names.get(i); + String singleCode = i < codes.size() ? codes.get(i) : null; + lastOrgId = updateSingleOrg(importDataModel, singleName, singleCode, session, type, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + } + return lastOrgId; + } + + /** + * Strips any parenthetical suffix and percentage annotation from a name. + * e.g. "World Bank (IDA)" → "World Bank" + * "IsDB (BID) - 92%" → "IsDB" + * "State of Senegal - 8%" → "State of Senegal" + */ + private static String stripParenthetical(String value) { + if (value == null) return null; + String trimmed = value.trim(); + int parenIdx = trimmed.indexOf('('); + if (parenIdx > 0) { + trimmed = trimmed.substring(0, parenIdx).trim(); + } + // Strip percentage suffix (e.g. "- 8%") for names without parentheses + return trimmed.replaceAll("\\s*-\\s*\\d+(\\.\\d+)?%\\s*$", "").trim(); + } + + private static Long updateSingleOrg(ImportDataModel importDataModel, String name, String code, Session session, String type, boolean createMissingOrgs, Long orgGroupId, String importedOrgGroupName, boolean createMissingOrgGroups) { Long orgId; + String cleanName = stripParenthetical(name); - if (ConstantsMap.containsKey("org_"+name+"_"+code)) { - orgId = ConstantsMap.get("org_"+name+"_"+code); - logger.info("In cache... organisation "+"org_"+name+"_"+code+":"+orgId); + if (ConstantsMap.containsKey("org_"+cleanName+"_"+code)) { + orgId = ConstantsMap.get("org_"+cleanName+"_"+code); + logger.info("In cache... organisation "+"org_"+cleanName+"_"+code+":"+orgId); } else { if (!session.isOpen()) { session = PersistenceManager.getRequestDBSession(); } - String hql = ""; + + String hql; Query query; List organisations= new ArrayList<>(); - if (name!=null) { - hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o WHERE LOWER(o.name)=LOWER(:name) OR LOWER(o.acronym)=LOWER(:name)"; + + if (cleanName!=null) { + hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o WHERE LOWER(o.name)=LOWER(:name) OR LOWER(o.acronym)=LOWER(:name) OR LOWER(o.orgCode)=LOWER(:name)"; query = session.createQuery(hql); - query.setParameter("name", name); + query.setParameter("name", cleanName); organisations = query.list(); } - if (organisations.isEmpty() && (code!=null)) { - hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o WHERE LOWER(o.orgCode)=LOWER(:code)"; - query = session.createQuery(hql); - query.setParameter("code", code); - organisations = query.list(); - - } if (!organisations.isEmpty()) { orgId = organisations.get(0); + } else if (createMissingOrgs) { + // Create the organisation if it does not exist and the user opted in + logger.info("Organisation not found, creating new: " + cleanName); + AmpOrganisation newOrg = new AmpOrganisation(); + newOrg.setName(cleanName); + if (code != null) { + newOrg.setOrgCode(code); + } + AmpOrgGroup orgGroup = resolveOrgGroup(session, orgGroupId, importedOrgGroupName, createMissingOrgGroups, cleanName); + if (orgGroup == null) { + throw new IllegalStateException("Unable to resolve an organization group for new organization '" + cleanName + "'"); + } + logger.info("Group being set for the new org is: " + orgGroup.getName()); + newOrg.setOrgGrpId(orgGroup); + session.save(newOrg); + session.flush(); + orgId = newOrg.getAmpOrgId(); + logger.info("Created new organisation: " + cleanName + " with id: " + orgId); } else { + // Fallback to "Undefined Agency" hql = "SELECT o.ampOrgId FROM " + AmpOrganisation.class.getName() + " o where o.name= :name"; - query = session.createQuery(hql).setParameter("name", "Undefined Agency", StringType.INSTANCE).setMaxResults(1); orgId = (Long) query.uniqueResult(); + logger.info("Organisation not found, using Undefined Agency for: " + cleanName); } - ConstantsMap.put("org_"+name+"_"+code, orgId); + ConstantsMap.put("org_"+cleanName+"_"+code, orgId); } logger.info("Organisation: " + orgId); if (Objects.equals(type, "donor")) { createDonorOrg(importDataModel, orgId); } - else if (Objects.equals(type, "responsibleOrg")) + else if (Objects.equals(type, ImporterConstants.ORG_TYPE_RESPONSIBLE_ORG)) { Organization responsibleOrg = new Organization(); responsibleOrg.setOrganization(orgId); importDataModel.getResponsible_organization().add(responsibleOrg); } - else if (Objects.equals(type, "beneficiaryAgency")) + else if (Objects.equals(type, ImporterConstants.ORG_TYPE_BENEFICIARY_AGENCY)) { Organization beneficiaryAgency = new Organization(); beneficiaryAgency.setOrganization(orgId); importDataModel.getBeneficiary_agency().add(beneficiaryAgency); - + } + else if (Objects.equals(type, ImporterConstants.ORG_TYPE_EXECUTING_AGENCY)) + { + Organization executingAgency = new Organization(); + executingAgency.setOrganization(orgId); + importDataModel.getExecuting_agency().add(executingAgency); + } + else if (Objects.equals(type, ImporterConstants.ORG_TYPE_IMPLEMENTING_AGENCY)) + { + Organization implementingAgency = new Organization(); + implementingAgency.setOrganization(orgId); + importDataModel.getImplementing_agency().add(implementingAgency); + } + else if (Objects.equals(type, ImporterConstants.ORG_TYPE_CONTRACTING_AGENCY)) + { + Organization contractingAgency = new Organization(); + contractingAgency.setOrganization(orgId); + importDataModel.getContracting_agency().add(contractingAgency); } return orgId; + } + + public static boolean hasTransactions(List fundings) { + if (fundings == null || fundings.isEmpty()) { + return false; + } + for (Funding funding : fundings) { + if (funding == null) { + continue; + } + if (hasNonZeroTransaction(funding.getCommitments()) + || hasNonZeroTransaction(funding.getDisbursements()) + || hasNonZeroTransaction(funding.getExpenditures())) { + return true; + } + } + return false; + } + + private static boolean hasNonZeroTransaction(List transactions) { + if (transactions == null || transactions.isEmpty()) { + return false; + } + for (Transaction transaction : transactions) { + if (transaction != null && Double.compare(transaction.getTransaction_amount(), 0.0d) != 0) { + return true; + } + } + return false; + } + + public static void persistImportedProjectStatus(ImportedProject importedProject) { + if (importedProject == null) { + return; + } + Session session = PersistenceManager.getRequestDBSession(); + try { + if (session == null || !session.isOpen()) { + PersistenceManager.doInTransaction(s -> { + s.saveOrUpdate(importedProject); + s.flush(); + }); + return; + } + session.saveOrUpdate(importedProject); + session.flush(); + } catch (Exception e) { + logger.warn("Could not save imported project status: {}", e.getMessage()); + } + } + + private static AmpOrgGroup resolveOrgGroup(Session session, Long orgGroupId, String importedOrgGroupName, boolean createMissingOrgGroups, String organizationName) { + AmpOrgGroup importedOrgGroup = findOrgGroupByNameOrCode(session, importedOrgGroupName); + if (importedOrgGroup != null) { + return importedOrgGroup; + } + + if (importedOrgGroupName != null && createMissingOrgGroups) { + return getOrCreateOrgGroup(session, importedOrgGroupName, orgGroupId); + } + + if (orgGroupId != null) { + return (AmpOrgGroup) session.get(AmpOrgGroup.class, orgGroupId); + } + if (createMissingOrgGroups && organizationName != null && !organizationName.trim().isEmpty()) { + return getOrCreateOrgGroup(session, organizationName.trim(), null); + } + + return null; + } + private static AmpOrgGroup findOrgGroupByNameOrCode(Session session, String orgGroupValue) { + if (orgGroupValue == null || orgGroupValue.trim().isEmpty()) { + return null; + } + String normalizedValue = orgGroupValue.trim(); + String orgGroupNameHql = AmpOrgGroup.hqlStringForName("grp"); + String hql = "SELECT grp FROM " + AmpOrgGroup.class.getName() + " grp WHERE (grp.deleted IS NULL OR grp.deleted = false) " + + "AND (LOWER(" + orgGroupNameHql + ") = LOWER(:value) OR LOWER(grp.orgGrpCode) = LOWER(:value))"; + return (AmpOrgGroup) session.createQuery(hql) + .setParameter("value", normalizedValue, StringType.INSTANCE) + .setMaxResults(1) + .uniqueResult(); + } + + private static AmpOrgGroup getOrCreateOrgGroup(Session session, String orgGroupName, Long fallbackOrgGroupId) { + AmpOrgGroup existingGroup = findOrgGroupByNameOrCode(session, orgGroupName); + if (existingGroup != null) { + return existingGroup; + } + + AmpOrgGroup newOrgGroup = new AmpOrgGroup(); + newOrgGroup.setOrgGrpName(orgGroupName.trim()); + + AmpOrgType orgType = null; + if (fallbackOrgGroupId != null) { + AmpOrgGroup fallbackGroup = (AmpOrgGroup) session.get(AmpOrgGroup.class, fallbackOrgGroupId); + if (fallbackGroup != null) { + orgType = fallbackGroup.getOrgType(); + } + } + if (orgType == null) { + orgType = getDefaultOrgType(); + } + + newOrgGroup.setOrgType(orgType); + ARUtil.clearOrgGroupTypeDimensions(); + DbUtil.add(newOrgGroup); + session.flush(); + return newOrgGroup; + } + + private static AmpOrgType getDefaultOrgType() { + List allAmpOrgTypes = DbUtil.getAmpOrgTypes(); + AmpOrgType otherOrgType = null; + AmpOrgType bilateralOrgType = null; + + if (allAmpOrgTypes == null || allAmpOrgTypes.isEmpty()) { + throw new IllegalStateException("No organization types are available to create a new organization group"); + } + + for (AmpOrgType type : allAmpOrgTypes) { + if (type.getOrgType() != null && type.getOrgType().equalsIgnoreCase("other")) { + otherOrgType = type; + } + if (type.getOrgType() != null && type.getOrgType().equalsIgnoreCase("bilateral")) { + bilateralOrgType = type; + } + } + + if (otherOrgType != null) { + return otherOrgType; + } + if (bilateralOrgType != null) { + return bilateralOrgType; + } + return allAmpOrgTypes.get(0); + } + + /** + * Splits a total amount evenly across n donors, using the same whole-number percentages + * as {@link #divide100}. The last donor absorbs any floating-point remainder so the + * individual amounts always sum exactly to totalAmount. + */ + private static List splitAmounts(double totalAmount, int n) { + if (n <= 0) return Collections.emptyList(); + if (n == 1) return Collections.singletonList(totalAmount); + Map percentages = divide100(n); + List amounts = new ArrayList<>(); + double allocated = 0; + for (int i = 0; i < n - 1; i++) { + double share = totalAmount * percentages.get(i) / 100.0; + amounts.add(share); + allocated += share; + } + amounts.add(totalAmount - allocated); + return amounts; } public static Map divide100(int n) { @@ -1195,11 +2207,16 @@ public static Map divide100(int n) { } private static void createDonorOrg(ImportDataModel importDataModel, Long orgId) { + createDonorOrg(importDataModel, orgId, null); + } + + private static void createDonorOrg(ImportDataModel importDataModel, Long orgId, Long orgRoleId) { DonorOrganization donorOrganization = new DonorOrganization(); donorOrganization.setOrganization(orgId); + if (orgRoleId != null) donorOrganization.setId(orgRoleId); importDataModel.getDonor_organization().add(donorOrganization); Map percentages = divide100(importDataModel.getDonor_organization().size()); - int index=0; + int index = 0; for (DonorOrganization donorOrganization1 : importDataModel.getDonor_organization()) { donorOrganization1.setPercentage(percentages.get(index)); index++; @@ -1223,4 +2240,573 @@ public static int getColumnIndexByName(Sheet sheet, String columnName) { } } + + /** Parse date from Excel cell or string; returns today if null/empty/invalid. */ + public static Date parseDateDefaultToday(Row row, Sheet sheet, Map config, String columnName) { + String key = getKey(config, columnName); + if (key == null) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + int col = getColumnIndexByName(sheet, key); + if (col < 0) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + Cell cell = row.getCell(col); + if (cell == null) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + String dateStr = extractDateFromStringCell(cell); + if (dateStr == null || dateStr.isEmpty()) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + try { + return java.sql.Date.valueOf(dateStr); + } catch (Exception e) { + return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + } + } + + /** Add indicator data to an activity from the current row. Called after importTheData when indicator columns are mapped. */ + public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sheet, Map config, Session session) { + logger.info("addIndicatorDataToActivity: activityId={}, row={}", activityId, row != null ? row.getRowNum() : null); + if (activityId == null || config == null || row == null || sheet == null) { + logger.info("addIndicatorDataToActivity: skipping - activityId, config, row or sheet is null"); + return; + } + // importTheData runs inside ActivityGatekeeper.doWithLock which commits and closes the session; use a fresh one if closed + if (session == null || !session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + logger.info("addIndicatorDataToActivity: obtained fresh session"); + } + String locationConfigKey = getKey(config, ImporterConstants.INDICATOR_LOCATION) != null + ? ImporterConstants.INDICATOR_LOCATION + : ImporterConstants.LOCATION; + if (getKey(config, ImporterConstants.INDICATOR_NAME) == null || getKey(config, locationConfigKey) == null || getKey(config, ImporterConstants.ACTUAL_VALUE) == null) { + logger.info("addIndicatorDataToActivity: skipping - missing config for indicator name, location or actual value"); + return; + } + String indicatorName = getCellValueByConfig(row, sheet, config, ImporterConstants.INDICATOR_NAME); + String locationNamesStr = getCellValueByConfig(row, sheet, config, locationConfigKey); + if (indicatorName == null || indicatorName.trim().isEmpty() || locationNamesStr == null || locationNamesStr.trim().isEmpty()) { + logger.info("addIndicatorDataToActivity: skipping - indicatorName or locationNamesStr empty (indicatorName='{}', locationNamesStr='{}')", indicatorName, locationNamesStr); + return; + } + indicatorName = indicatorName.trim(); + List locationNames = splitLocationNames(locationNamesStr); + logger.info("addIndicatorDataToActivity: indicator='{}', locations(count={}): {}", indicatorName, locationNames.size(), locationNames); + if (locationNames.isEmpty()) { + logger.debug("addIndicatorDataToActivity: no location names after split"); + return; + } + + String programName = getCellValueByConfig(row, sheet, config, ImporterConstants.PROGRAM_NAME); + if (programName != null) programName = programName.trim(); + + AmpTheme programTheme = null; + if (programName != null && !programName.isEmpty()) { + try { + programTheme = getOrCreateProgramByName(programName, session); + } catch (Exception e) { + logger.error("Could not resolve or create program by name: " + programName, e); + } + } + + IndicatorManagerService indicatorService = new IndicatorManagerService(); + MEIndicatorDTO indicatorDto = indicatorService.getMeIndicatorByNameAndProgramNameOptional(indicatorName, (programName == null || programName.isEmpty()) ? null : programName); + AmpIndicator indicator; + if (indicatorDto == null) { + logger.info("addIndicatorDataToActivity: creating new indicator '{}' (program={})", indicatorName, programName); + MEIndicatorDTO createDto = new MEIndicatorDTO(); + createDto.setName(indicatorName); + createDto.setCode(indicatorName + "_" + System.currentTimeMillis()); + createDto.setCreationDate(new Date()); + createDto.setAscending(true); + createDto.setSectorIds(new ArrayList<>()); + if (programTheme != null && programTheme.getAmpThemeId() != null) { + createDto.setProgramId(programTheme.getAmpThemeId()); + } + try { + indicatorDto = indicatorService.createMEIndicator(createDto); + } catch (Exception e) { + logger.error("Failed to create indicator: " + indicatorName, e); + return; + } + indicator = session.get(AmpIndicator.class, indicatorDto.getId()); + logger.info("addIndicatorDataToActivity: created indicator id={}", indicator != null ? indicator.getIndicatorId() : null); + } else { + indicator = session.get(AmpIndicator.class, indicatorDto.getId()); + logger.info("addIndicatorDataToActivity: using existing indicator id={} name='{}'", indicator != null ? indicator.getIndicatorId() : null, indicatorName); + } + if (indicator == null) { + logger.info("addIndicatorDataToActivity: indicator is null after lookup/create"); + return; + } + + AmpActivityVersion activity = session.get(AmpActivityVersion.class, activityId); + if (activity == null) { + logger.info("addIndicatorDataToActivity: activity not found for activityId={}", activityId); + return; + } + // Force-load indicators so we append to existing; avoid replacing due to lazy/uninitialized collection + if (activity.getIndicators() != null) { + Hibernate.initialize(activity.getIndicators()); + } + int indicatorsCountBefore = activity.getIndicators() == null ? 0 : activity.getIndicators().size(); + logger.info("addIndicatorDataToActivity: activityId={} existing indicators count={}", activityId, indicatorsCountBefore); + + if (programTheme != null) { + addProgramToActivityIfMissing(activity, programTheme, session); + } + + AmpIndicatorGlobalValue existingBase = indicator.getBaseValue(); + double origBase = parseDoubleFromConfig(row, sheet, config, ImporterConstants.ORIGINAL_BASE_VALUE); + boolean hasOrigBase = getKey(config, ImporterConstants.ORIGINAL_BASE_VALUE) != null && !Double.isNaN(origBase); + double revBase = parseDoubleFromConfig(row, sheet, config, ImporterConstants.REVISED_BASE_VALUE); + boolean hasRevBase = getKey(config, ImporterConstants.REVISED_BASE_VALUE) != null && !Double.isNaN(revBase); + double origTarget = parseDoubleFromConfig(row, sheet, config, ImporterConstants.ORIGINAL_TARGET_VALUE); + double revTarget = parseDoubleFromConfig(row, sheet, config, ImporterConstants.REVISED_TARGET_VALUE); + String actualValueConfigKey = getKey(config, ImporterConstants.ACTUAL_VALUE); + double actualVal = parseDoubleFromConfig(row, sheet, config, ImporterConstants.ACTUAL_VALUE); + if (Double.isNaN(actualVal)) actualVal = 0.0; + logger.info("addIndicatorDataToActivity: actual value configKey='{}' parsed actualVal={} (NaN->0)", actualValueConfigKey, actualVal); + + Date origBaseDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.ORIGINAL_BASE_VALUE_DATE); + Date revBaseDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.REVISED_BASE_VALUE_DATE); + Date origTargetDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.ORIGINAL_TARGET_VALUE_DATE); + Date revTargetDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.REVISED_TARGET_VALUE_DATE); + Date actualDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.ACTUAL_VALUE_DATE); + logger.info("addIndicatorDataToActivity: actualDate={}", actualDate); + + double baseOrigVal = hasOrigBase ? origBase : (existingBase != null && existingBase.getOriginalValue() != null ? existingBase.getOriginalValue() : 0.0); + double baseRevVal = hasRevBase ? revBase : (existingBase != null && existingBase.getRevisedValue() != null ? existingBase.getRevisedValue() : 0.0); + double targetOrigVal = Double.isNaN(origTarget) ? 0.0 : origTarget; + double targetRevVal = Double.isNaN(revTarget) ? 0.0 : revTarget; + String unit = getCellValueByConfig(row, sheet, config, ImporterConstants.UNIT_OF_MEASURE); + if (unit != null) unit = unit.trim(); + String actualComment = (unit != null && !unit.isEmpty()) ? "Unit: " + unit : null; + + int merged = 0, created = 0, skipped = 0; + for (String locationName : locationNames) { + logger.info("addIndicatorDataToActivity: processing location '{}' for activityId={} indicator='{}'", locationName, activityId, indicatorName); + AmpActivityLocation activityLocation = null; + if (activity.getLocations() != null) { + for (AmpActivityLocation aal : activity.getLocations()) { + if (aal.getLocation() != null && locationName.equalsIgnoreCase(aal.getLocation().getName())) { + activityLocation = aal; + break; + } + } + } + if (activityLocation == null) { + activityLocation = getOrAddActivityLocationForName(activity, locationName, session); + } + if (activityLocation == null) { + logger.info("addIndicatorDataToActivity: could not resolve or add location '{}' for activityId={}, skipping", locationName, activityId); + skipped++; + continue; + } + logger.info("addIndicatorDataToActivity: activityLocation id={} for '{}'", activityLocation.getLocation() != null ? activityLocation.getLocation().getId() : null, locationName); + + IndicatorActivity ia = findExistingIndicatorActivity(activity, indicator, activityLocation); + if (ia != null) { + logger.info("addIndicatorDataToActivity: merging into existing IndicatorActivity for location '{}' (actual={})", locationName, actualVal); + mergeIndicatorValuesIntoExisting(ia, activityLocation, session, config, + baseOrigVal, origBaseDate, baseRevVal, revBaseDate, + targetOrigVal, origTargetDate, targetRevVal, revTargetDate, + actualVal, actualDate, actualComment); + session.flush(); + merged++; + continue; + } + + logger.info("addIndicatorDataToActivity: creating new IndicatorActivity for activityId={} indicator='{}' location='{}' (actual={})", activityId, indicatorName, locationName, actualVal); + ia = new IndicatorActivity(); + ia.setActivity(activity); + ia.setIndicator(indicator); + ia.setActivityLocation(activityLocation); + + Set values = new HashSet<>(); + if (getKey(config, ImporterConstants.ORIGINAL_BASE_VALUE) != null || getKey(config, ImporterConstants.REVISED_BASE_VALUE) != null || existingBase != null) { + AmpIndicatorValue baseOrig = new AmpIndicatorValue(AmpIndicatorValue.BASE); + baseOrig.setValue(baseOrigVal); + baseOrig.setValueDate(origBaseDate); + baseOrig.setIndicatorConnection(ia); + baseOrig.setActivityLocation(activityLocation); + values.add(baseOrig); + AmpIndicatorValue baseRev = new AmpIndicatorValue(AmpIndicatorValue.REVISED); + baseRev.setValue(baseRevVal); + baseRev.setValueDate(revBaseDate); + baseRev.setIndicatorConnection(ia); + baseRev.setActivityLocation(activityLocation); + values.add(baseRev); + } + if (getKey(config, ImporterConstants.ORIGINAL_TARGET_VALUE) != null || getKey(config, ImporterConstants.REVISED_TARGET_VALUE) != null) { + AmpIndicatorValue tOrig = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tOrig.setValue(targetOrigVal); + tOrig.setValueDate(origTargetDate); + tOrig.setIndicatorConnection(ia); + tOrig.setActivityLocation(activityLocation); + values.add(tOrig); + AmpIndicatorValue tRev = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tRev.setValue(targetRevVal); + tRev.setValueDate(revTargetDate); + tRev.setIndicatorConnection(ia); + tRev.setActivityLocation(activityLocation); + values.add(tRev); + } + AmpIndicatorValue actual = new AmpIndicatorValue(AmpIndicatorValue.ACTUAL); + actual.setValue(actualVal); + actual.setValueDate(actualDate); + actual.setIndicatorConnection(ia); + actual.setActivityLocation(activityLocation); // required for OnePager form to show value (filters by activityLocation) + if (actualComment != null) actual.setComment(actualComment); + values.add(actual); + logger.info("addIndicatorDataToActivity: new IndicatorActivity - created ACTUAL value: value={}, valueDate={}, saving child", actualVal, actualDate); + + ia.setValues(values); + if (activity.getIndicators() == null) activity.setIndicators(new HashSet<>()); + activity.getIndicators().add(ia); + session.save(ia); + session.save(actual); + logger.info("addIndicatorDataToActivity: saved IndicatorActivity id={} and ACTUAL AmpIndicatorValue", ia.getId()); + created++; + } + session.flush(); + int indicatorsCountAfter = activity.getIndicators() == null ? 0 : activity.getIndicators().size(); + logger.info("addIndicatorDataToActivity: done for activityId={} indicator='{}' - merged={}, created={}, skipped={}, indicatorsCount before={} after={}", activityId, indicatorName, merged, created, skipped, indicatorsCountBefore, indicatorsCountAfter); + } + + /** + * Finds an existing activity–indicator connection for the same activity, indicator, and location. + * Match is by indicator id and activity location (both null or same location). + */ + private static IndicatorActivity findExistingIndicatorActivity(AmpActivityVersion activity, AmpIndicator indicator, + AmpActivityLocation activityLocation) { + if (activity.getIndicators() == null) return null; + Long indicatorId = indicator != null ? indicator.getIndicatorId() : null; + Long locationId = activityLocation != null && activityLocation.getLocation() != null + ? activityLocation.getLocation().getId() : null; + for (IndicatorActivity ia : activity.getIndicators()) { + if (ia.getIndicator() == null) continue; + if (!Objects.equals(ia.getIndicator().getIndicatorId(), indicatorId)) continue; + Long existingLocId = ia.getActivityLocation() != null && ia.getActivityLocation().getLocation() != null + ? ia.getActivityLocation().getLocation().getId() : null; + if (Objects.equals(existingLocId, locationId)) return ia; + } + return null; + } + + /** + * Merges imported values into an existing indicator connection: updates existing values by type where present, + * adds new values only for types that are missing. Sets activityLocation on values so OnePager form can display them. + */ + private static void mergeIndicatorValuesIntoExisting(IndicatorActivity ia, AmpActivityLocation activityLocation, Session session, Map config, + double baseOrigVal, Date origBaseDate, double baseRevVal, Date revBaseDate, + double targetOrigVal, Date origTargetDate, double targetRevVal, Date revTargetDate, + double actualVal, Date actualDate, String actualComment) { + logger.debug("mergeIndicatorValuesIntoExisting: indicatorConnection id={}, actualVal={}", ia.getId(), actualVal); + Set existing = ia.getValues(); + if (existing == null) { + existing = new HashSet<>(); + ia.setValues(existing); + } + boolean hasBase = getKey(config, ImporterConstants.ORIGINAL_BASE_VALUE) != null || getKey(config, ImporterConstants.REVISED_BASE_VALUE) != null; + boolean hasTarget = getKey(config, ImporterConstants.ORIGINAL_TARGET_VALUE) != null || getKey(config, ImporterConstants.REVISED_TARGET_VALUE) != null; + + // Add a new ACTUAL value only if we don't already have the same value on the same date for this location + if (hasActualWithSameValueAndDate(existing, activityLocation, actualVal, actualDate)) { + logger.info("mergeIndicatorValuesIntoExisting: skipping ACTUAL - same value {} and date {} already present for this location", actualVal, actualDate); + } else { + logger.info("mergeIndicatorValuesIntoExisting: adding new ACTUAL value: value={} valueDate={}", actualVal, actualDate); + AmpIndicatorValue actual = new AmpIndicatorValue(AmpIndicatorValue.ACTUAL); + actual.setValue(actualVal); + actual.setValueDate(actualDate); + if (actualComment != null) actual.setComment(actualComment); + actual.setIndicatorConnection(ia); + actual.setActivityLocation(activityLocation); + existing.add(actual); + session.save(actual); + } + + if (hasBase) { + AmpIndicatorValue existingBase = findValueByType(existing, AmpIndicatorValue.BASE); + if (existingBase != null) { + existingBase.setValue(baseOrigVal); + existingBase.setValueDate(origBaseDate); + if (activityLocation != null && existingBase.getActivityLocation() == null) existingBase.setActivityLocation(activityLocation); + } else { + AmpIndicatorValue baseOrig = new AmpIndicatorValue(AmpIndicatorValue.BASE); + baseOrig.setValue(baseOrigVal); + baseOrig.setValueDate(origBaseDate); + baseOrig.setIndicatorConnection(ia); + baseOrig.setActivityLocation(activityLocation); + existing.add(baseOrig); + session.save(baseOrig); + } + AmpIndicatorValue existingRev = findValueByType(existing, AmpIndicatorValue.REVISED); + if (existingRev != null) { + existingRev.setValue(baseRevVal); + existingRev.setValueDate(revBaseDate); + if (activityLocation != null && existingRev.getActivityLocation() == null) existingRev.setActivityLocation(activityLocation); + } else { + AmpIndicatorValue baseRev = new AmpIndicatorValue(AmpIndicatorValue.REVISED); + baseRev.setValue(baseRevVal); + baseRev.setValueDate(revBaseDate); + baseRev.setIndicatorConnection(ia); + baseRev.setActivityLocation(activityLocation); + existing.add(baseRev); + session.save(baseRev); + } + } + + if (hasTarget) { + List targets = getValuesByType(existing, AmpIndicatorValue.TARGET); + if (targets.size() >= 2) { + targets.get(0).setValue(targetOrigVal); + targets.get(0).setValueDate(origTargetDate); + if (activityLocation != null && targets.get(0).getActivityLocation() == null) targets.get(0).setActivityLocation(activityLocation); + targets.get(1).setValue(targetRevVal); + targets.get(1).setValueDate(revTargetDate); + if (activityLocation != null && targets.get(1).getActivityLocation() == null) targets.get(1).setActivityLocation(activityLocation); + } else if (targets.size() == 1) { + targets.get(0).setValue(targetOrigVal); + targets.get(0).setValueDate(origTargetDate); + if (activityLocation != null && targets.get(0).getActivityLocation() == null) targets.get(0).setActivityLocation(activityLocation); + AmpIndicatorValue tRev = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tRev.setValue(targetRevVal); + tRev.setValueDate(revTargetDate); + tRev.setIndicatorConnection(ia); + tRev.setActivityLocation(activityLocation); + existing.add(tRev); + session.save(tRev); + } else { + AmpIndicatorValue tOrig = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tOrig.setValue(targetOrigVal); + tOrig.setValueDate(origTargetDate); + tOrig.setIndicatorConnection(ia); + tOrig.setActivityLocation(activityLocation); + existing.add(tOrig); + session.save(tOrig); + AmpIndicatorValue tRev = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tRev.setValue(targetRevVal); + tRev.setValueDate(revTargetDate); + tRev.setIndicatorConnection(ia); + tRev.setActivityLocation(activityLocation); + existing.add(tRev); + session.save(tRev); + } + } + evictIndicatorConnectionFromSecondLevelCache(ia); + } + + /** + * Evicts the activity from the second-level cache before an update. Avoids ObjectNotFoundException + * when the cached activity (or its activityContacts) references deleted entities (e.g. AmpActivityDocument). + */ + private static void evictActivityFromSecondLevelCache(Long activityId) { + if (activityId == null) return; + try { + org.hibernate.SessionFactory sessionFactory = org.digijava.kernel.persistence.PersistenceManager.sf(); + if (sessionFactory == null) return; + org.hibernate.Cache cache = sessionFactory.getCache(); + if (cache == null) return; + cache.evictEntityData(AmpActivityVersion.class, activityId); + } catch (Exception e) { + logger.debug("Could not evict activity from cache: {}", e.getMessage()); + } + } + + /** + * Evicts the indicator connection and its values from the second-level cache so the activity form + * sees updated values on next load (otherwise cached stale values can be shown). + */ + private static void evictIndicatorConnectionFromSecondLevelCache(IndicatorActivity ia) { + try { + org.hibernate.SessionFactory sessionFactory = org.digijava.kernel.persistence.PersistenceManager.sf(); + if (sessionFactory == null) return; + org.hibernate.Cache cache = sessionFactory.getCache(); + if (cache == null) return; + if (ia.getId() != null) cache.evictEntityData(IndicatorConnection.class, ia.getId()); + if (ia.getValues() != null) { + for (AmpIndicatorValue v : ia.getValues()) { + if (v != null && v.getIndValId() != null) cache.evictEntityData(AmpIndicatorValue.class, v.getIndValId()); + } + } + } catch (Exception e) { + logger.debug("Could not evict indicator connection from cache: {}", e.getMessage()); + } + } + + private static AmpIndicatorValue findValueByType(Set values, int valueType) { + if (values == null) return null; + for (AmpIndicatorValue v : values) { + if (v.getValueType() == valueType) return v; + } + return null; + } + + /** Returns true if there is already an ACTUAL value for this location with the same value and same date (calendar day). */ + private static boolean hasActualWithSameValueAndDate(Set values, AmpActivityLocation activityLocation, double value, Date valueDate) { + if (values == null) return false; + Long wantLocId = activityLocation != null && activityLocation.getLocation() != null ? activityLocation.getLocation().getId() : null; + for (AmpIndicatorValue v : values) { + if (v.getValueType() != AmpIndicatorValue.ACTUAL) continue; + Long vLocId = v.getActivityLocation() != null && v.getActivityLocation().getLocation() != null + ? v.getActivityLocation().getLocation().getId() : null; + if (!Objects.equals(vLocId, wantLocId)) continue; + if (Double.compare(v.getValue(), value) != 0) continue; + if (!isSameDay(v.getValueDate(), valueDate)) continue; + return true; + } + return false; + } + + /** + * Converts a Date to LocalDate, handling both java.util.Date and java.sql.Date. + * java.sql.Date doesn't support toInstant(), so we use toLocalDate() for it. + */ + private static LocalDate toLocalDate(Date date) { + if (date instanceof java.sql.Date) { + return ((java.sql.Date) date).toLocalDate(); + } + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + } + + private static boolean isSameDay(Date a, Date b) { + if (a == null && b == null) return true; + if (a == null || b == null) return false; + return toLocalDate(a).equals(toLocalDate(b)); + } + + /** Finds ACTUAL value matching this location; falls back to ACTUAL with null location if none match (for backward compat). */ + private static AmpIndicatorValue findValueByTypeAndLocation(Set values, int valueType, AmpActivityLocation activityLocation) { + if (values == null) return null; + AmpIndicatorValue fallbackNull = null; + Long wantLocId = activityLocation != null && activityLocation.getLocation() != null ? activityLocation.getLocation().getId() : null; + for (AmpIndicatorValue v : values) { + if (v.getValueType() != valueType) continue; + Long vLocId = v.getActivityLocation() != null && v.getActivityLocation().getLocation() != null + ? v.getActivityLocation().getLocation().getId() : null; + if (Objects.equals(vLocId, wantLocId)) return v; + if (v.getActivityLocation() == null) fallbackNull = v; + } + return fallbackNull; + } + + private static List getValuesByType(Set values, int valueType) { + if (values == null) return Collections.emptyList(); + List list = new ArrayList<>(); + for (AmpIndicatorValue v : values) { + if (v.getValueType() == valueType) list.add(v); + } + list.sort(Comparator.comparing(AmpIndicatorValue::getValueDate, Comparator.nullsLast(Comparator.naturalOrder()))); + return list; + } + + /** + * Adds the program (theme) to the activity's programs if not already present. + * When the Program Percentage field is enabled, percentages are recalculated and + * divided evenly among all activity programs (including the one just added). + */ + public static void addProgramToActivityIfMissing(AmpActivityVersion activity, AmpTheme program, Session session) { + if (activity == null || program == null) return; + Set actPrograms = activity.getActPrograms(); + if (actPrograms == null) { + actPrograms = new HashSet<>(); + activity.setActPrograms(actPrograms); + } + for (AmpActivityProgram ap : actPrograms) { + if (ap.getProgram() != null && program.getAmpThemeId() != null + && program.getAmpThemeId().equals(ap.getProgram().getAmpThemeId())) { + return; + } + } + AmpActivityProgram activityProgram = new AmpActivityProgram(); + activityProgram.setActivity(activity); + activityProgram.setProgram(program); + activityProgram.setProgramPercentage(100f); + actPrograms.add(activityProgram); + session.save(activityProgram); + + // If Program Percentage field is enabled, distribute 100% evenly among all programs + boolean percentageEnabled = false; + try { + percentageEnabled = FeaturesUtil.isVisibleField(ArConstants.PROGRAM_PERCENTAGE); + } catch (Exception e) { + // No request/session (e.g. batch) – skip percentage redistribution + logger.error("Could not determine if Program Percentage field is enabled; skipping percentage redistribution", e); + } + if (percentageEnabled && !actPrograms.isEmpty()) { + List list = new ArrayList<>(actPrograms); + int n = list.size(); + Map percentages = divide100(n); + for (int i = 0; i < n; i++) { + list.get(i).setProgramPercentage(percentages.get(i)); + } + } + } + + /** + * Returns the program (theme) by name, or creates a new root-level program if it does not exist. + * @param programName program name (must be non-empty) + * @param session current session (used for create and flush) + * @return AmpTheme or null if programName is null/empty or creation fails + */ + public static AmpTheme getOrCreateProgramByName(String programName, Session session) { + if (programName == null || programName.trim().isEmpty()) return null; + programName = programName.trim(); + AmpTheme theme = ProgramUtil.getTheme(programName); + if (theme != null) return theme; + try { + AmpTheme newTheme = new AmpTheme(); + newTheme.setName(programName); + String code = programName.replaceAll("[^a-zA-Z0-9_-]", "_").replaceAll("_+", "_").trim(); + if (code.length() > 45) code = code.substring(0, 45); + newTheme.setThemeCode("IMP_" + code + "_" + System.currentTimeMillis()); + newTheme.setIndlevel(0); + newTheme.setParentThemeId(null); + session.save(newTheme); + session.flush(); + return newTheme; + } catch (Exception e) { + logger.warn("Failed to create program: " + programName, e); + return null; + } + } + + public static String getCellValueByConfig(Row row, Sheet sheet, Map config, String fieldName) { + String key = getKey(config, fieldName); + if (key == null) return null; + int col = getColumnIndexByName(sheet, key); + if (col < 0) return null; + return getStringValueFromCell(row.getCell(col), true); + } + + private static double parseDoubleFromConfig(Row row, Sheet sheet, Map config, String fieldName) { + String key = getKey(config, fieldName); + if (key == null) { + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: no config key for field '{}'", fieldName); + return Double.NaN; + } + int col = getColumnIndexByName(sheet, key); + if (col < 0) { + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: column not found for key '{}' in sheet", key); + return Double.NaN; + } + Cell cell = row.getCell(col); + if (cell == null) { + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: cell is null for col={} key='{}'", col, key); + return Double.NaN; + } + try { + if (cell.getCellType() == Cell.CELL_TYPE_NUMERIC) { + double v = cell.getNumericCellValue(); + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: ACTUAL_VALUE from numeric cell col='{}' value={}", key, v); + return v; + } + String s = getStringValueFromCell(cell, true); + if (s == null || s.trim().isEmpty()) { + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: ACTUAL_VALUE cell empty for col='{}'", key); + return Double.NaN; + } + double v = Double.parseDouble(s.trim()); + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: ACTUAL_VALUE from string cell col='{}' raw='{}' parsed={}", key, s, v); + return v; + } catch (Exception e) { + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: ACTUAL_VALUE parse failed for col='{}' cellType={} error={}", key, cell.getCellType(), e.getMessage()); + return Double.NaN; + } + } } diff --git a/amp/src/main/java/org/digijava/module/aim/dbentity/IndicatorConnection.java b/amp/src/main/java/org/digijava/module/aim/dbentity/IndicatorConnection.java index 21e575d1449..ff0fa8f8f3c 100644 --- a/amp/src/main/java/org/digijava/module/aim/dbentity/IndicatorConnection.java +++ b/amp/src/main/java/org/digijava/module/aim/dbentity/IndicatorConnection.java @@ -28,11 +28,19 @@ public class IndicatorConnection implements Serializable, Comparable getValues() { @@ -96,5 +105,17 @@ public AmpActivityLocation getActivityLocation() { public void setActivityLocation(AmpActivityLocation activityLocation) { this.activityLocation = activityLocation; + this.indicatorLocationKey = null; // reset so getter recomputes + } + + /** Composite key for uniqueness: indicator id + location id so same indicator with different locations is allowed. */ + public String getIndicatorLocationKey() { + if (indicatorLocationKey == null) { + Long indId = indicator != null ? indicator.getIndicatorId() : null; + Long locId = activityLocation != null && activityLocation.getLocation() != null + ? activityLocation.getLocation().getId() : null; + indicatorLocationKey = (indId != null ? indId : "") + "_" + (locId != null ? locId : ""); + } + return indicatorLocationKey; } } diff --git a/amp/src/main/java/org/digijava/module/aim/form/DataImporterForm.java b/amp/src/main/java/org/digijava/module/aim/form/DataImporterForm.java index 37add978278..c1b46a0869f 100644 --- a/amp/src/main/java/org/digijava/module/aim/form/DataImporterForm.java +++ b/amp/src/main/java/org/digijava/module/aim/form/DataImporterForm.java @@ -12,6 +12,96 @@ public class DataImporterForm extends ActionForm { private boolean internal; + private boolean skipExisting; + private boolean skipRecordsWithoutTransactions; + private boolean createMissingOrgs; + private boolean createMissingSectors; + private boolean createMissingOrgGroups; + private Long orgGroupId; + private Long defaultActivityStatusId; + private Long defaultLocationId; + private boolean validateActivities; + private boolean addDisbursementForCommitment; + + public boolean isSkipExisting() { + return skipExisting; + } + + public void setSkipExisting(boolean skipExisting) { + this.skipExisting = skipExisting; + } + + public boolean isSkipRecordsWithoutTransactions() { + return skipRecordsWithoutTransactions; + } + + public void setSkipRecordsWithoutTransactions(boolean skipRecordsWithoutTransactions) { + this.skipRecordsWithoutTransactions = skipRecordsWithoutTransactions; + } + + public boolean isValidateActivities() { + return validateActivities; + } + + public void setValidateActivities(boolean validateActivities) { + this.validateActivities = validateActivities; + } + + public boolean isAddDisbursementForCommitment() { + return addDisbursementForCommitment; + } + + public void setAddDisbursementForCommitment(boolean addDisbursementForCommitment) { + this.addDisbursementForCommitment = addDisbursementForCommitment; + } + + public boolean isCreateMissingOrgs() { + return createMissingOrgs; + } + + public void setCreateMissingOrgs(boolean createMissingOrgs) { + this.createMissingOrgs = createMissingOrgs; + } + + public boolean isCreateMissingSectors() { + return createMissingSectors; + } + + public void setCreateMissingSectors(boolean createMissingSectors) { + this.createMissingSectors = createMissingSectors; + } + + public boolean isCreateMissingOrgGroups() { + return createMissingOrgGroups; + } + + public void setCreateMissingOrgGroups(boolean createMissingOrgGroups) { + this.createMissingOrgGroups = createMissingOrgGroups; + } + + public Long getOrgGroupId() { + return orgGroupId; + } + + public void setOrgGroupId(Long orgGroupId) { + this.orgGroupId = orgGroupId; + } + + public Long getDefaultActivityStatusId() { + return defaultActivityStatusId; + } + + public void setDefaultActivityStatusId(Long defaultActivityStatusId) { + this.defaultActivityStatusId = defaultActivityStatusId; + } + + public Long getDefaultLocationId() { + return defaultLocationId; + } + + public void setDefaultLocationId(Long defaultLocationId) { + this.defaultLocationId = defaultLocationId; + } public Set getFileHeaders() { return fileHeaders; diff --git a/amp/src/main/java/org/digijava/module/aim/util/ActivityUtil.java b/amp/src/main/java/org/digijava/module/aim/util/ActivityUtil.java index 82799e16667..56db388a1c6 100644 --- a/amp/src/main/java/org/digijava/module/aim/util/ActivityUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/util/ActivityUtil.java @@ -450,6 +450,12 @@ public static AmpActivityVersion loadActivity(Long id) throws DgException { Hibernate.initialize(str.getType()); Hibernate.initialize(str.getCoordinates()); } + Hibernate.initialize(result.getIndicators()); + if (result.getIndicators() != null) { + for (IndicatorActivity ia : result.getIndicators()) { + Hibernate.initialize(ia.getValues()); + } + } // AMPOFFLINE-1528 ActivityUtil.setCurrentWorkspacePrefixIntoRequest(result); diff --git a/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java b/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java index 3e28ec4261f..84c625badb7 100644 --- a/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java +++ b/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java @@ -485,7 +485,7 @@ public static void sendReportsToServer(List ampDashboardFundin // Convert to JSON using a JSON library (e.g., Gson) Gson gson = new Gson(); String jsonData = gson.toJson(submissionData); - logger.info("JSON data: " + jsonData); +// logger.info("JSON data: " + jsonData); // Get the output stream of the connection try (OutputStream os = connection.getOutputStream()) { diff --git a/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v3.xml b/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v3.xml deleted file mode 100644 index fca110892ba..00000000000 --- a/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v3.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - AMP-30885 - menu - bmokandu - Add menu entry for Data Importer - - - - diff --git a/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v4.xml b/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v4.xml new file mode 100644 index 00000000000..91a3ed4bea8 --- /dev/null +++ b/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v4.xml @@ -0,0 +1,67 @@ + + + AMP-30885 + menu + bmokandu + Add menu entry for Data Importer + + + + diff --git a/amp/src/main/resources/xmlpatches/4.0/Increase-ImportedProjectCurrency-Name-Length.xml b/amp/src/main/resources/xmlpatches/4.0/Increase-ImportedProjectCurrency-Name-Length.xml new file mode 100644 index 00000000000..7e8e36233d7 --- /dev/null +++ b/amp/src/main/resources/xmlpatches/4.0/Increase-ImportedProjectCurrency-Name-Length.xml @@ -0,0 +1,20 @@ + + + AMP-DATAIMPORT + Increase imported_project_name column length to support long project names + brianbrix + + Increases the IMPORTED_PROJECT_CURRENCY.imported_project_name column from varchar(255) to varchar(2000) + to accommodate lengthy project names that may exceed 255 characters. + + + + + diff --git a/amp/src/main/resources/xmlpatches/4.0/Update-Import-Tables-v4.xml b/amp/src/main/resources/xmlpatches/4.0/Update-Import-Tables-v4.xml new file mode 100644 index 00000000000..3485a20f517 --- /dev/null +++ b/amp/src/main/resources/xmlpatches/4.0/Update-Import-Tables-v4.xml @@ -0,0 +1,15 @@ + + + AMP- + Add processing time tracking to imported files records + GitHub Copilot + + + + + + AMP- + Add uploaded timestamp to imported files records + GitHub Copilot + + + + \ No newline at end of file diff --git a/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp b/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp index c24461951be..b269dd68844 100644 --- a/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp +++ b/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp @@ -2,13 +2,150 @@ <%@ taglib prefix="html" uri="http://struts.apache.org/tags-html" %> <%@ taglib prefix="bean" uri="http://struts.apache.org/tags-bean" %> <%@ taglib prefix="logic" uri="http://struts.apache.org/tags-logic" %> +<%@ taglib uri="http://digijava.org/digi" prefix="digi" %> - Data Importer + <digi:trn>Data Importer</digi:trn> - - - -

Data Importer

- - - - -

Upload File

-

Data file configuration

- - -
- - - - -
- - + input[type="button"]:hover, + button:hover { + background: var(--accent-deep); + border-color: var(--accent-deep); + } + .remove-row { + background: #6f5a5a; + border-color: #6f5a5a; + } + .inline-field, + .toggle-grid, + .sheet-choice-card { + margin-top: 16px; + } -
- + .toggle-grid { + display: grid; + gap: 10px; + margin-top: 18px; + padding: 18px; + border-radius: var(--radius-md); + background: var(--surface-muted); + border: 1px solid var(--panel-border); + } -
- -

- -
+ .toggle-item { + display: flex; + align-items: center; + gap: 10px; + color: var(--text-strong); + } + .toggle-item input { + width: auto; + margin: 0; + } + .fields-table, + #import-projects-table, + .records-table { + width: 100%; + border-collapse: separate; + border-spacing: 0; + overflow: hidden; + border-radius: 18px; + background: #fff; + border: 1px solid rgba(22, 53, 67, 0.1); + } + table th, + table td { + text-align: left; + padding: 14px 16px; + border-bottom: 1px solid rgba(22, 53, 67, 0.08); + } + table th { + background: #eef1f3; + color: var(--text-strong); + font-size: 12px; + letter-spacing: 0.12em; + text-transform: uppercase; + } + table tr:nth-child(even) { + background: var(--row-alt); + } -