unanalyzedCommits,
for (int i = 0; i < maxCommits; i++) {
String hash = unanalyzedCommits.get(i);
try {
- String commitDiff = operationsService.getCommitDiff(
- client, vcsRepoInfoImpl.workspace(), vcsRepoInfoImpl.repoSlug(), hash);
+ String commitDiff = client.getCommitDiff(
+ vcsRepoInfoImpl.workspace(), vcsRepoInfoImpl.repoSlug(), hash);
if (commitDiff != null && !commitDiff.isBlank()) {
aggregatedDiff.append(commitDiff);
if (!commitDiff.endsWith("\n")) {
diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsService.java
index 54cb26ef..1d458d74 100644
--- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsService.java
+++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsService.java
@@ -1,29 +1,30 @@
package org.rostilos.codecrow.analysisengine.service.branch;
-import okhttp3.OkHttpClient;
import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest;
import org.rostilos.codecrow.analysisengine.processor.VcsRepoInfoImpl;
import org.rostilos.codecrow.analysisengine.service.BranchArchiveService;
-import org.rostilos.codecrow.analysisengine.service.vcs.VcsOperationsService;
-import org.rostilos.codecrow.analysisengine.service.vcs.VcsServiceFactory;
+import org.rostilos.codecrow.analysisengine.service.VcsFileRetrievalPolicy;
import org.rostilos.codecrow.core.model.branch.Branch;
import org.rostilos.codecrow.filecontent.model.BranchFile;
import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis;
import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysisIssue;
import org.rostilos.codecrow.core.model.project.Project;
-import org.rostilos.codecrow.core.model.vcs.EVcsProvider;
+import org.rostilos.codecrow.core.model.project.config.ProjectConfig;
import org.rostilos.codecrow.filecontent.persistence.BranchFileRepository;
import org.rostilos.codecrow.core.persistence.repository.branch.BranchIssueRepository;
import org.rostilos.codecrow.core.persistence.repository.branch.BranchRepository;
import org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisIssueRepository;
import org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisRepository;
+import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository;
import org.rostilos.codecrow.filecontent.service.FileSnapshotService;
+import org.rostilos.codecrow.vcsclient.VcsClient;
import org.rostilos.codecrow.vcsclient.VcsClientProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
/**
@@ -37,33 +38,36 @@ public class BranchFileOperationsService {
private final BranchFileRepository branchFileRepository;
private final BranchRepository branchRepository;
+ private final ProjectRepository projectRepository;
private final BranchIssueRepository branchIssueRepository;
private final CodeAnalysisIssueRepository codeAnalysisIssueRepository;
private final CodeAnalysisRepository codeAnalysisRepository;
- private final VcsServiceFactory vcsServiceFactory;
private final VcsClientProvider vcsClientProvider;
private final FileSnapshotService fileSnapshotService;
private final BranchArchiveService branchArchiveService;
+ private final VcsFileRetrievalPolicy fileRetrievalPolicy;
public BranchFileOperationsService(
BranchFileRepository branchFileRepository,
BranchRepository branchRepository,
+ ProjectRepository projectRepository,
BranchIssueRepository branchIssueRepository,
CodeAnalysisIssueRepository codeAnalysisIssueRepository,
CodeAnalysisRepository codeAnalysisRepository,
- VcsServiceFactory vcsServiceFactory,
VcsClientProvider vcsClientProvider,
FileSnapshotService fileSnapshotService,
- BranchArchiveService branchArchiveService) {
+ BranchArchiveService branchArchiveService,
+ VcsFileRetrievalPolicy fileRetrievalPolicy) {
this.branchFileRepository = branchFileRepository;
this.branchRepository = branchRepository;
+ this.projectRepository = projectRepository;
this.branchIssueRepository = branchIssueRepository;
this.codeAnalysisIssueRepository = codeAnalysisIssueRepository;
this.codeAnalysisRepository = codeAnalysisRepository;
- this.vcsServiceFactory = vcsServiceFactory;
this.vcsClientProvider = vcsClientProvider;
this.fileSnapshotService = fileSnapshotService;
this.branchArchiveService = branchArchiveService;
+ this.fileRetrievalPolicy = fileRetrievalPolicy;
}
// ────────────────────────── Archive download ──────────────────────────────
@@ -73,18 +77,41 @@ public BranchFileOperationsService(
*
* Replaces per-file VCS API calls with a single archive download,
* avoiding rate-limiting issues (e.g. Bitbucket HTTP 429).
- * Returns an empty map on failure — callers must handle graceful fallback.
+ * Legacy content-only view. New orchestration code must use
+ * {@link #downloadBranchFileSnapshot(VcsRepoInfoImpl, String, Set)} so an
+ * unavailable archive cannot be confused with a successful empty snapshot.
*/
public Map downloadBranchArchive(VcsRepoInfoImpl vcsRepoInfoImpl, String branchOrCommit,
Set neededFiles) {
+ return downloadBranchFileSnapshot(vcsRepoInfoImpl, branchOrCommit, neededFiles).contents();
+ }
+
+ /**
+ * Downloads one authoritative repository snapshot for a bounded file set.
+ * An unavailable archive is represented explicitly so downstream stages do
+ * not mistake an acquisition failure for an empty repository.
+ */
+ public BranchFileSnapshot downloadBranchFileSnapshot(
+ VcsRepoInfoImpl vcsRepoInfoImpl,
+ String branchOrCommit,
+ Set neededFiles) {
+ Set requestedFiles = neededFiles != null
+ ? new LinkedHashSet<>(neededFiles)
+ : Collections.emptySet();
try {
- return branchArchiveService.downloadAndExtractFiles(
+ BranchArchiveService.ArchiveSnapshot snapshot = branchArchiveService.downloadSnapshot(
vcsRepoInfoImpl.vcsConnection(), vcsRepoInfoImpl.workspace(), vcsRepoInfoImpl.repoSlug(),
- branchOrCommit, neededFiles);
+ branchOrCommit, requestedFiles);
+ return BranchFileSnapshot.fromArchive(snapshot);
} catch (Exception e) {
- log.warn("Failed to download branch archive — will fall back to per-file API calls: {}",
- e.getMessage());
- return Collections.emptyMap();
+ boolean allowPerFileFallback =
+ fileRetrievalPolicy.allowPerFileFallback(requestedFiles.size());
+ log.warn("Failed to download branch archive for {} requested files: {}. {}",
+ requestedFiles.size(), e.getMessage(),
+ allowPerFileFallback
+ ? "A bounded per-file fallback remains available"
+ : "Per-file fallback is disabled above the shared archive threshold");
+ return BranchFileSnapshot.unavailable(allowPerFileFallback, e.getMessage());
}
}
@@ -93,40 +120,49 @@ public Map downloadBranchArchive(VcsRepoInfoImpl vcsRepoInfoImpl
/**
* Updates branch file records for changed files.
*
- * When {@code archiveContents} is non-empty, file existence is determined
- * from the archive map instead of per-file API calls (avoids rate-limiting).
+ * Prefer the {@link BranchFileSnapshot} overload. It keeps archive
+ * acquisition state and path presence separate from extracted text.
*
* @param archiveContents pre-downloaded archive contents (filePath → content).
- * May be empty — in that case, falls back to per-file API.
+ * May be empty; retained only for legacy callers.
* @return the set of file paths confirmed to exist in the branch
*/
public Set updateBranchFiles(Set changedFiles, Project project,
String branchName, Map archiveContents) {
+ return updateBranchFiles(
+ changedFiles, project, branchName, BranchFileSnapshot.legacy(archiveContents));
+ }
+
+ public Set updateBranchFiles(Set changedFiles, Project project,
+ String branchName, BranchFileSnapshot fileSnapshot) {
Set filesExistingInBranch = new HashSet<>();
- boolean useArchive = archiveContents != null && !archiveContents.isEmpty();
+ BranchFileSnapshot snapshot = fileSnapshot != null
+ ? fileSnapshot
+ : BranchFileSnapshot.unavailable(true, "snapshot not provided");
Branch branchEntity = branchRepository
.findByProjectIdAndBranchName(project.getId(), branchName).orElse(null);
- // Always initialise VCS client — used as fallback when a file is missing
- // from the archive (binary filter, size limit, path encoding mismatch).
- VcsOperationsService operationsService = null;
- OkHttpClient client = null;
+ // Resolve provider clients only when a bounded per-file fallback is
+ // actually allowed. An authoritative archive snapshot (including its
+ // binary/large-file presence set) must never trigger extra API calls.
+ VcsClient client = null;
String workspace = null;
String repoSlug = null;
var vcsRepoInfo = project.getEffectiveVcsRepoInfo();
- if (vcsRepoInfo != null && vcsRepoInfo.getVcsConnection() != null) {
- EVcsProvider provider = vcsRepoInfo.getVcsConnection().getProviderType();
- operationsService = vcsServiceFactory.getOperationsService(provider);
- client = vcsClientProvider.getHttpClient(vcsRepoInfo.getVcsConnection());
+ if (!snapshot.archiveAvailable()
+ && snapshot.allowPerFileFallback()
+ && vcsRepoInfo != null
+ && vcsRepoInfo.getVcsConnection() != null) {
+ client = vcsClientProvider.getClient(vcsRepoInfo.getVcsConnection());
workspace = vcsRepoInfo.getRepoWorkspace();
repoSlug = vcsRepoInfo.getRepoSlug();
}
for (String filePath : changedFiles) {
boolean fileExists = resolveFileExistence(
- filePath, branchName, useArchive, archiveContents,
- operationsService, client, workspace, repoSlug);
+ filePath, branchName, snapshot,
+ client, workspace, repoSlug);
if (!fileExists) {
log.debug("Skipping file {} - does not exist in branch {}", filePath, branchName);
@@ -156,7 +192,33 @@ public Branch createOrUpdateProjectBranch(Project project, BranchProcessRequest
branch.setBranchName(request.getTargetBranchName());
}
branch.setCommitHash(request.getCommitHash());
- return branchRepository.save(branch);
+ Branch savedBranch = branchRepository.save(branch);
+
+ ProjectConfig config = project.getConfiguration();
+ String configuredMainBranch = config != null ? config.mainBranch() : null;
+ if ((configuredMainBranch == null || configuredMainBranch.isBlank())
+ && project.getVcsRepoBinding() != null) {
+ configuredMainBranch = project.getVcsRepoBinding().getDefaultBranch();
+ }
+
+ boolean isConfiguredMainBranch = configuredMainBranch != null
+ && configuredMainBranch.equals(savedBranch.getBranchName());
+ boolean shouldSelectBranch = project.getDefaultBranch() == null
+ || (isConfiguredMainBranch
+ && !Objects.equals(savedBranch.getId(), project.getDefaultBranch().getId()));
+
+ if (shouldSelectBranch) {
+ project.setDefaultBranch(savedBranch);
+ if (config != null && (config.mainBranch() == null || config.mainBranch().isBlank())) {
+ config.setMainBranch(savedBranch.getBranchName());
+ config.ensureMainBranchInPatterns();
+ }
+ projectRepository.save(project);
+ log.info("Selected branch {} as the default analysis branch for project {}",
+ savedBranch.getBranchName(), project.getId());
+ }
+
+ return savedBranch;
}
// ──────────────────── File snapshot updates ──────────────────────────────
@@ -171,12 +233,20 @@ public Branch createOrUpdateProjectBranch(Project project, BranchProcessRequest
* preserve the file content at the time each issue was originally detected,
* which is critical for the Source Context viewer.
*
- * When {@code archiveContents} is non-empty, file content is read from the
- * map directly (no API calls). When empty, falls back to per-file VCS API.
+ * Prefer the {@link BranchFileSnapshot} overload so optional snapshot
+ * persistence follows the same bounded acquisition decision as existence
+ * reconciliation.
*/
public void updateFileSnapshotsForBranch(Set existingFiles, Project project,
BranchProcessRequest request,
Map archiveContents) {
+ updateFileSnapshotsForBranch(
+ existingFiles, project, request, BranchFileSnapshot.legacy(archiveContents));
+ }
+
+ public void updateFileSnapshotsForBranch(Set existingFiles, Project project,
+ BranchProcessRequest request,
+ BranchFileSnapshot fileSnapshot) {
if (existingFiles.isEmpty()) return;
try {
@@ -190,7 +260,7 @@ public void updateFileSnapshotsForBranch(Set existingFiles, Project proj
Branch branch = branchOpt.get();
Map fileContents = buildFileContentsMap(
- existingFiles, project, request, archiveContents);
+ existingFiles, project, request, fileSnapshot);
if (!fileContents.isEmpty()) {
int updated = fileSnapshotService.persistSnapshotsForBranch(
@@ -222,44 +292,28 @@ public Set getBranchFilePaths(Long projectId, String branchName) {
// ──────────────────── Private helpers ────────────────────────────────────
private boolean resolveFileExistence(String filePath, String branchName,
- boolean useArchive, Map archiveContents,
- VcsOperationsService operationsService, OkHttpClient client,
+ BranchFileSnapshot snapshot,
+ VcsClient client,
String workspace, String repoSlug) {
- if (useArchive) {
- if (archiveContents.containsKey(filePath)) {
- return true;
- }
- // Archive miss — the file might still exist (binary filter, size limit,
- // path encoding mismatch between diff parser and archive extractor).
- // Verify via per-file API before declaring it deleted.
- if (operationsService != null && client != null) {
- try {
- boolean existsViaApi = operationsService.checkFileExistsInBranch(
- client, workspace, repoSlug, branchName, filePath);
- if (existsViaApi) {
- log.warn("File {} not in archive but EXISTS via API — archive extraction missed it "
- + "(binary/size/path mismatch?). Treating as existing in branch {}.",
- filePath, branchName);
- } else {
- log.debug("File {} confirmed deleted via API (branch {})", filePath, branchName);
- }
- return existsViaApi;
- } catch (Exception e) {
- log.warn("File {} not in archive and API check failed for branch {}: {}. "
- + "Assuming file exists to avoid false deletion resolution.",
- filePath, branchName, e.getMessage());
- return true; // Fail-open: assume exists to prevent false resolutions
- }
- }
- log.debug("File {} not found in archive and no API fallback available — "
- + "treating as deleted from branch {}", filePath, branchName);
- return false;
+ if (snapshot.archiveAvailable()) {
+ return snapshot.presentFiles().contains(filePath);
+ }
+ if (!snapshot.allowContentApiFallback()) {
+ log.debug("Per-file fallback unavailable for {} in branch {} — "
+ + "assuming it exists (fail-open)", filePath, branchName);
+ return true;
+ }
+ if (client == null) {
+ log.debug("No VCS fallback available for {} — assuming it exists in branch {}",
+ filePath, branchName);
+ return true;
}
try {
- return operationsService.checkFileExistsInBranch(
- client, workspace, repoSlug, branchName, filePath);
+ return client.fileExists(workspace, repoSlug, branchName, filePath);
} catch (Exception e) {
- log.warn("Failed to check file existence for {} in branch {}: {}. Proceeding anyway.",
+ snapshot.stopPerFileFallback();
+ log.warn("File-existence fallback stopped after provider failure for {} in branch {}: {}. "
+ + "Remaining files will be treated as existing without additional VCS calls.",
filePath, branchName, e.getMessage());
return true;
}
@@ -311,38 +365,102 @@ private void persistBranchFile(Project project, String branchName, String filePa
private Map buildFileContentsMap(Set existingFiles, Project project,
BranchProcessRequest request,
- Map archiveContents) {
+ BranchFileSnapshot fileSnapshot) {
Map fileContents = new LinkedHashMap<>();
- boolean useArchive = archiveContents != null && !archiveContents.isEmpty();
+ BranchFileSnapshot snapshot = fileSnapshot != null
+ ? fileSnapshot
+ : BranchFileSnapshot.unavailable(true, "snapshot not provided");
- if (useArchive) {
+ if (snapshot.archiveAvailable()) {
for (String filePath : existingFiles) {
- String content = archiveContents.get(filePath);
+ String content = snapshot.contents().get(filePath);
if (content != null) {
fileContents.put(filePath, content);
}
}
- } else {
+ } else if (snapshot.allowContentApiFallback()) {
var vcsRepoInfo = project.getEffectiveVcsRepoInfo();
if (vcsRepoInfo == null || vcsRepoInfo.getVcsConnection() == null) return fileContents;
- EVcsProvider provider = vcsRepoInfo.getVcsConnection().getProviderType();
- VcsOperationsService operationsService = vcsServiceFactory.getOperationsService(provider);
- OkHttpClient client = vcsClientProvider.getHttpClient(vcsRepoInfo.getVcsConnection());
+ VcsClient client = vcsClientProvider.getClient(vcsRepoInfo.getVcsConnection());
for (String filePath : existingFiles) {
+ if (!snapshot.allowContentApiFallback()) {
+ break;
+ }
try {
- String content = operationsService.getFileContent(
- client, vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(),
- request.getCommitHash(), filePath);
+ String content = client.getFileContent(
+ vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(),
+ filePath, request.getCommitHash());
if (content != null) {
fileContents.put(filePath, content);
}
} catch (Exception e) {
- log.debug("Could not fetch content for {} (snapshot update): {}", filePath, e.getMessage());
+ snapshot.stopPerFileFallback();
+ log.warn("Stopping per-file snapshot fallback after provider failure for {}: {}",
+ filePath, e.getMessage());
+ break;
}
}
+ } else {
+ log.warn("Skipping branch snapshot content fallback for {} files because the archive "
+ + "was unavailable and the shared per-file threshold was exceeded",
+ existingFiles.size());
}
return fileContents;
}
+
+ public record BranchFileSnapshot(
+ Map contents,
+ Set presentFiles,
+ boolean archiveAvailable,
+ boolean allowPerFileFallback,
+ String diagnostic,
+ AtomicBoolean providerUnavailable
+ ) {
+ public BranchFileSnapshot {
+ contents = Collections.unmodifiableMap(new LinkedHashMap<>(
+ contents != null ? contents : Collections.emptyMap()));
+ presentFiles = Collections.unmodifiableSet(new LinkedHashSet<>(
+ presentFiles != null ? presentFiles : Collections.emptySet()));
+ providerUnavailable = providerUnavailable != null
+ ? providerUnavailable
+ : new AtomicBoolean(false);
+ }
+
+ public static BranchFileSnapshot fromArchive(
+ BranchArchiveService.ArchiveSnapshot snapshot) {
+ return new BranchFileSnapshot(
+ snapshot.contents(), snapshot.presentFiles(), true, false, null,
+ new AtomicBoolean(false));
+ }
+
+ public static BranchFileSnapshot unavailable(
+ boolean allowPerFileFallback, String diagnostic) {
+ return new BranchFileSnapshot(
+ Collections.emptyMap(), Collections.emptySet(), false,
+ allowPerFileFallback, diagnostic, new AtomicBoolean(false));
+ }
+
+ public static BranchFileSnapshot legacy(Map contents) {
+ Map safeContents =
+ contents != null ? contents : Collections.emptyMap();
+ if (safeContents.isEmpty()) {
+ return unavailable(true, "legacy empty archive contents");
+ }
+ return new BranchFileSnapshot(
+ safeContents, safeContents.keySet(), true, false, null,
+ new AtomicBoolean(false));
+ }
+
+ public boolean allowContentApiFallback() {
+ return !archiveAvailable
+ && allowPerFileFallback
+ && !providerUnavailable.get();
+ }
+
+ public void stopPerFileFallback() {
+ providerUnavailable.set(true);
+ }
+ }
}
diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationService.java
index b53c325f..f1b85722 100644
--- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationService.java
+++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationService.java
@@ -153,13 +153,14 @@ public Map fullReconcile(
+ allUnresolved.size() + " unresolved issues");
VcsRepoInfoImpl vcsRepoInfoImpl = ProjectVcsInfoRetriever.getVcsInfo(project);
- Map archiveContents = branchFileOperationsService.downloadBranchArchive(
+ BranchFileOperationsService.BranchFileSnapshot branchFileSnapshot =
+ branchFileOperationsService.downloadBranchFileSnapshot(
vcsRepoInfoImpl, commitHash, allFilePaths);
log.info("Full reconciliation archive: {} files extracted for {} requested",
- archiveContents.size(), allFilePaths.size());
+ branchFileSnapshot.contents().size(), allFilePaths.size());
Set filesExistingInBranch = branchFileOperationsService.updateBranchFiles(
- allFilePaths, project, branchName, archiveContents);
+ allFilePaths, project, branchName, branchFileSnapshot);
EventNotificationEmitter.emitStatus(consumer, "updating_snapshots",
"Updating file content snapshots for " + filesExistingInBranch.size()
@@ -168,7 +169,7 @@ public Map fullReconcile(
BranchProcessRequest syntheticRequest = buildSyntheticRequest(projectId, branchName, commitHash);
branchFileOperationsService.updateFileSnapshotsForBranch(
- filesExistingInBranch, project, syntheticRequest, archiveContents);
+ filesExistingInBranch, project, syntheticRequest, branchFileSnapshot);
EventNotificationEmitter.emitStatus(consumer, "reanalyzing_issues",
"Reanalyzing " + allUnresolved.size() + " issues across "
@@ -176,7 +177,8 @@ public Map fullReconcile(
branchIssueReconciliationService.reanalyzeCandidateIssues(
allFilePaths, filesExistingInBranch, branch, project,
- syntheticRequest, consumer, archiveContents);
+ syntheticRequest, consumer, branchFileSnapshot.contents(), null,
+ branchFileSnapshot.allowContentApiFallback());
Branch branchForVerify = branchRepository.findByProjectIdAndBranchName(projectId, branchName)
.orElse(branch);
@@ -273,10 +275,11 @@ private MergedPrSyncResult syncMergedPrIssuesIntoBranch(
long beforeCount = branchIssueRepository.countAllByBranchId(branch.getId());
VcsRepoInfoImpl vcsRepoInfoImpl = ProjectVcsInfoRetriever.getVcsInfo(project);
- Map archiveContents = branchFileOperationsService.downloadBranchArchive(
+ BranchFileOperationsService.BranchFileSnapshot branchFileSnapshot =
+ branchFileOperationsService.downloadBranchFileSnapshot(
vcsRepoInfoImpl, commitHash, allPrIssuePaths);
Set filesExistingInBranch = branchFileOperationsService.updateBranchFiles(
- allPrIssuePaths, project, branch.getBranchName(), archiveContents);
+ allPrIssuePaths, project, branch.getBranchName(), branchFileSnapshot);
for (Map.Entry> entry : issuePathsByPr.entrySet()) {
Set existingPrPaths = entry.getValue().stream()
diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationService.java
index 781c7bdc..d1a308b3 100644
--- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationService.java
+++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationService.java
@@ -1,6 +1,5 @@
package org.rostilos.codecrow.analysisengine.service.branch;
-import okhttp3.OkHttpClient;
import org.rostilos.codecrow.analysisengine.aiclient.AiAnalysisClient;
import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest;
import org.rostilos.codecrow.analysisengine.dto.request.ai.AiAnalysisRequest;
@@ -11,7 +10,6 @@
import org.rostilos.codecrow.astparser.model.ParsedTree;
import org.rostilos.codecrow.astparser.api.ScopeResolver;
import org.rostilos.codecrow.analysisengine.service.vcs.VcsAiClientService;
-import org.rostilos.codecrow.analysisengine.service.vcs.VcsOperationsService;
import org.rostilos.codecrow.analysisengine.service.vcs.VcsServiceFactory;
import org.rostilos.codecrow.analysisengine.util.DiffParsingUtils;
import org.rostilos.codecrow.analysisengine.util.AnalysisScopeFilter;
@@ -25,6 +23,7 @@
import org.rostilos.codecrow.filecontent.service.FileSnapshotService;
import org.rostilos.codecrow.core.util.tracking.LineHashSequence;
import org.rostilos.codecrow.core.util.tracking.TrackingConfidence;
+import org.rostilos.codecrow.vcsclient.VcsClient;
import org.rostilos.codecrow.vcsclient.VcsClientProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -335,9 +334,7 @@ public int sweepDeterministicResolutions(
// Prepare VCS fallback for files not in the archive
var vcsRepoInfo = project.getEffectiveVcsRepoInfo();
- EVcsProvider provider = vcsRepoInfo.getVcsConnection().getProviderType();
- VcsOperationsService operationsService = vcsServiceFactory.getOperationsService(provider);
- OkHttpClient client = vcsClientProvider.getHttpClient(vcsRepoInfo.getVcsConnection());
+ VcsClient client = vcsClientProvider.getClient(vcsRepoInfo.getVcsConnection());
int resolvedCount = 0;
@@ -355,9 +352,9 @@ public int sweepDeterministicResolutions(
try {
String fileContent = archiveContents != null ? archiveContents.get(filePath) : null;
if (fileContent == null) {
- fileContent = operationsService.getFileContent(
- client, vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(),
- request.getCommitHash(), filePath);
+ fileContent = client.getFileContent(
+ vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(),
+ filePath, request.getCommitHash());
}
if (fileContent == null) {
// Content unavailable — verify file existence explicitly.
@@ -365,8 +362,8 @@ public int sweepDeterministicResolutions(
// not actual file deletion. Must confirm before resolving (fail-open).
boolean confirmedDeleted = false;
try {
- confirmedDeleted = !operationsService.checkFileExistsInBranch(
- client, vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(),
+ confirmedDeleted = !client.fileExists(
+ vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(),
request.getTargetBranchName(), filePath);
} catch (Exception ex) {
log.warn("Sweep: file existence check failed for {} — skipping (fail-open): {}",
@@ -469,7 +466,7 @@ public void reanalyzeCandidateIssues(Set changedFiles,
Consumer> consumer,
Map archiveContents) {
reanalyzeCandidateIssues(changedFiles, filesExistingInBranch, branch, project,
- request, consumer, archiveContents, null);
+ request, consumer, archiveContents, null, true);
}
/**
@@ -491,6 +488,18 @@ public void reanalyzeCandidateIssues(Set changedFiles,
Consumer> consumer,
Map archiveContents,
String rawDiff) {
+ reanalyzeCandidateIssues(changedFiles, filesExistingInBranch, branch, project,
+ request, consumer, archiveContents, rawDiff, true);
+ }
+
+ public void reanalyzeCandidateIssues(Set changedFiles,
+ Set filesExistingInBranch,
+ Branch branch, Project project,
+ BranchProcessRequest request,
+ Consumer> consumer,
+ Map archiveContents,
+ String rawDiff,
+ boolean allowVcsContentFallback) {
List candidateBranchIssues = collectCandidateIssues(changedFiles, branch);
if (candidateBranchIssues.isEmpty()) {
log.info("No pre-existing issues to re-analyze (Branch: {})", request.getTargetBranchName());
@@ -535,7 +544,8 @@ public void reanalyzeCandidateIssues(Set changedFiles,
+ partitioned.autoResolved.size() + " auto-resolved)"));
TrackingResult tracking = performDeterministicTracking(
- partitioned.needsTracking, project, request, archiveContents);
+ partitioned.needsTracking, project, request, archiveContents,
+ allowVcsContentFallback);
// Resolve deterministically-resolved issues
if (!tracking.resolved.isEmpty()) {
@@ -555,7 +565,8 @@ public void reanalyzeCandidateIssues(Set changedFiles,
// Stage 3: AI reconciliation for ambiguous issues
if (!tracking.needsAi.isEmpty()) {
performAiReconciliation(tracking.needsAi, tracking.fetchedFileContents,
- branch, project, request, consumer, archiveContents, rawDiff);
+ branch, project, request, consumer, archiveContents, rawDiff,
+ allowVcsContentFallback && !tracking.providerUnavailable);
}
updateBranchCountsAfterReconciliation(branch);
@@ -685,18 +696,22 @@ private PartitionResult partitionByFileExistence(List candidates,
private record TrackingResult(List confirmed,
List resolved,
List needsAi,
- Map fetchedFileContents) {
+ Map fetchedFileContents,
+ boolean providerUnavailable) {
}
private TrackingResult performDeterministicTracking(
List needsTracking, Project project,
- BranchProcessRequest request, Map archiveContents) {
+ BranchProcessRequest request, Map archiveContents,
+ boolean allowVcsContentFallback) {
- // Resolve VCS clients for fallback file content fetching
var vcsRepoInfo = project.getEffectiveVcsRepoInfo();
- EVcsProvider provider = vcsRepoInfo.getVcsConnection().getProviderType();
- VcsOperationsService operationsService = vcsServiceFactory.getOperationsService(provider);
- OkHttpClient client = vcsClientProvider.getHttpClient(vcsRepoInfo.getVcsConnection());
+ VcsClient client = null;
+ if (allowVcsContentFallback
+ && vcsRepoInfo != null
+ && vcsRepoInfo.getVcsConnection() != null) {
+ client = vcsClientProvider.getClient(vcsRepoInfo.getVcsConnection());
+ }
// Group issues by file
Map> issuesByFile = new LinkedHashMap<>();
@@ -710,6 +725,7 @@ private TrackingResult performDeterministicTracking(
List confirmed = new ArrayList<>();
List resolved = new ArrayList<>();
List needsAi = new ArrayList<>();
+ boolean providerUnavailable = false;
// Pre-populate file contents from archive
Map fetchedFileContents = new LinkedHashMap<>();
@@ -731,10 +747,20 @@ private TrackingResult performDeterministicTracking(
LineHashSequence currentHashes;
try {
String fileContent = fetchedFileContents.get(filePath);
- if (fileContent == null) {
- fileContent = operationsService.getFileContent(
- client, vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(),
- request.getCommitHash(), filePath);
+ if (fileContent == null
+ && allowVcsContentFallback
+ && !providerUnavailable
+ && client != null) {
+ try {
+ fileContent = client.getFileContent(
+ vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(),
+ filePath, request.getCommitHash());
+ } catch (Exception providerFailure) {
+ providerUnavailable = true;
+ log.warn("Stopping per-file reconciliation content fallback after provider "
+ + "failure for {}: {}",
+ filePath, providerFailure.getMessage());
+ }
}
currentHashes = fileContent != null
? LineHashSequence.from(fileContent)
@@ -761,7 +787,8 @@ private TrackingResult performDeterministicTracking(
confirmed, resolved, needsAi, filePath, fetchedFileContents.get(filePath));
}
- return new TrackingResult(confirmed, resolved, needsAi, fetchedFileContents);
+ return new TrackingResult(
+ confirmed, resolved, needsAi, fetchedFileContents, providerUnavailable);
}
/**
@@ -824,7 +851,8 @@ private void performAiReconciliation(List needsAiReconciliation,
BranchProcessRequest request,
Consumer> consumer,
Map archiveContents,
- String rawDiff) {
+ String rawDiff,
+ boolean allowVcsContentFallback) {
consumer.accept(Map.of(
"type", "status",
"state", "reanalyzing_issues",
@@ -838,15 +866,17 @@ private void performAiReconciliation(List needsAiReconciliation,
var vcsRepoInfo = project.getEffectiveVcsRepoInfo();
EVcsProvider provider = vcsRepoInfo.getVcsConnection().getProviderType();
VcsAiClientService aiClientService = vcsServiceFactory.getAiClientService(provider);
- VcsOperationsService operationsService = vcsServiceFactory.getOperationsService(provider);
- OkHttpClient client = vcsClientProvider.getHttpClient(vcsRepoInfo.getVcsConnection());
+ VcsClient client = allowVcsContentFallback
+ ? vcsClientProvider.getClient(vcsRepoInfo.getVcsConnection())
+ : null;
// Build file contents for AI — only files that have issues needing
// reconciliation (+ cross-file context from issue descriptions — Fix 3)
Map aiFileContents = buildAiFileContents(
needsAiReconciliation, fetchedFileContents, archiveContents,
- operationsService, client, vcsRepoInfo.getRepoWorkspace(),
- vcsRepoInfo.getRepoSlug(), request.getCommitHash());
+ client, vcsRepoInfo.getRepoWorkspace(),
+ vcsRepoInfo.getRepoSlug(), request.getCommitHash(),
+ allowVcsContentFallback);
// Build relevant per-file diff filtered to only files with AI-bound issues
// This gives the LLM "before → after" context for recognising applied fixes
@@ -903,8 +933,9 @@ private void performAiReconciliation(List needsAiReconciliation,
private Map buildAiFileContents(
List issues, Map fetchedFileContents,
Map archiveContents,
- VcsOperationsService operationsService, OkHttpClient client,
- String workspace, String repoSlug, String commitHash) {
+ VcsClient client,
+ String workspace, String repoSlug, String commitHash,
+ boolean allowVcsContentFallback) {
Map aiFileContents = new LinkedHashMap<>();
@@ -925,19 +956,23 @@ private Map buildAiFileContents(
}
// Fetch any missing primary files — archive first, then per-file API
+ boolean providerUnavailable = false;
for (String fp : primaryFiles) {
if (!aiFileContents.containsKey(fp)) {
if (archiveContents != null && archiveContents.containsKey(fp)) {
aiFileContents.put(fp, archiveContents.get(fp));
- } else {
+ } else if (allowVcsContentFallback && !providerUnavailable
+ && client != null) {
try {
- String content = operationsService.getFileContent(
- client, workspace, repoSlug, commitHash, fp);
+ String content = client.getFileContent(
+ workspace, repoSlug, fp, commitHash);
if (content != null) {
aiFileContents.put(fp, content);
}
} catch (Exception e) {
- log.warn("Failed to fetch file content for AI reconciliation: {}", fp);
+ providerUnavailable = true;
+ log.warn("Stopping per-file AI reconciliation content fallback after "
+ + "provider failure for {}: {}", fp, e.getMessage());
}
}
}
@@ -960,12 +995,15 @@ private Map buildAiFileContents(
if (content == null && fetchedFileContents.containsKey(refPath)) {
content = fetchedFileContents.get(refPath);
}
- if (content == null) {
+ if (content == null && allowVcsContentFallback && !providerUnavailable
+ && client != null) {
try {
- content = operationsService.getFileContent(
- client, workspace, repoSlug, commitHash, refPath);
+ content = client.getFileContent(
+ workspace, repoSlug, refPath, commitHash);
} catch (Exception e) {
- log.debug("Cross-file fetch skipped for {}: {}", refPath, e.getMessage());
+ providerUnavailable = true;
+ log.warn("Stopping per-file cross-file context fallback after provider "
+ + "failure for {}: {}", refPath, e.getMessage());
}
}
if (content != null) {
diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrFileEnrichmentService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrFileEnrichmentService.java
index c92a49cf..86fbeca0 100644
--- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrFileEnrichmentService.java
+++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrFileEnrichmentService.java
@@ -5,6 +5,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import okhttp3.*;
import org.rostilos.codecrow.analysisengine.dto.request.ai.enrichment.*;
+import org.rostilos.codecrow.analysisengine.util.TextFileEligibility;
import org.rostilos.codecrow.vcsclient.VcsClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -230,33 +231,6 @@ public PrEnrichmentDataDto fetchFileContentsOnly(
}
}
- /**
- * Blacklist of binary / non-text file extensions that should be excluded
- * from enrichment. Everything else is allowed.
- */
- private static final Set EXCLUDED_EXTENSIONS = Set.of(
- // Raster images. SVG is XML source and remains eligible.
- ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".webp", ".tiff", ".tif",
- // Fonts
- ".woff", ".woff2", ".ttf", ".otf", ".eot",
- // Compiled / bytecode
- ".class", ".pyc", ".pyo", ".o", ".obj", ".dll", ".so", ".dylib", ".exe",
- ".war", ".ear", ".nar",
- // Archives
- ".jar", ".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar",
- // Documents / media
- ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
- ".mp3", ".mp4", ".avi", ".mov", ".wav", ".flac", ".ogg", ".webm",
- // Data blobs / certs
- ".bin", ".dat", ".db", ".sqlite", ".sqlite3",
- ".p12", ".pfx", ".jks", ".keystore", ".der", ".cer",
- // Lock files (large, auto-generated, no review value)
- ".lockb",
- // Misc binary. Source maps and minified JS/CSS remain text; selected
- // plugin file policy decides whether they are generated or reviewable.
- ".wasm"
- );
-
/**
* Filter out known binary / non-text files. Everything not blacklisted is
* allowed through for enrichment and file snapshots.
@@ -264,11 +238,7 @@ public PrEnrichmentDataDto fetchFileContentsOnly(
private List filterSupportedFiles(List files, Map skipReasons) {
List supported = new ArrayList<>();
for (String file : files) {
- String lower = file.toLowerCase();
- boolean isExcluded = EXCLUDED_EXTENSIONS.stream()
- .anyMatch(lower::endsWith);
-
- if (isExcluded) {
+ if (!TextFileEligibility.isTextCandidate(file)) {
skipReasons.merge("binary_or_non_text", 1, Integer::sum);
} else {
supported.add(file);
diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsOperationsService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsOperationsService.java
deleted file mode 100644
index ae4ac8c7..00000000
--- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsOperationsService.java
+++ /dev/null
@@ -1,116 +0,0 @@
-package org.rostilos.codecrow.analysisengine.service.vcs;
-
-import okhttp3.OkHttpClient;
-import org.rostilos.codecrow.core.model.pullrequest.PullRequestState;
-import org.rostilos.codecrow.core.model.vcs.EVcsProvider;
-
-import java.io.IOException;
-import java.util.Optional;
-
-/**
- * Generic interface for VCS operations that vary by provider.
- * Implementations handle provider-specific API calls for operations like
- * fetching diffs and checking file existence.
- */
-public interface VcsOperationsService {
-
- /**
- * @return the VCS provider this service handles
- */
- EVcsProvider getProvider();
-
- /**
- * Fetches the raw diff for a commit.
- *
- * @param client authorized HTTP client
- * @param workspace workspace or team/organization slug
- * @param repoSlug repository slug
- * @param commitHash commit hash
- * @return raw unified diff as returned by VCS API
- * @throws IOException on network / parsing errors
- */
- String getCommitDiff(OkHttpClient client, String workspace, String repoSlug, String commitHash) throws IOException;
-
- /**
- * Fetches the raw diff for a pull request.
- * This returns ALL files changed in the PR, not just the merge commit.
- *
- * @param client authorized HTTP client
- * @param workspace workspace or team/organization slug
- * @param repoSlug repository slug
- * @param prNumber pull request number
- * @return raw unified diff as returned by VCS API
- * @throws IOException on network / parsing errors
- */
- String getPullRequestDiff(OkHttpClient client, String workspace, String repoSlug, String prNumber) throws IOException;
-
- /**
- * Fetches the diff between two commits (delta diff for incremental analysis).
- * This is used to get only the changes made since the last analyzed commit.
- *
- * @param client authorized HTTP client
- * @param workspace workspace or team/organization slug
- * @param repoSlug repository slug
- * @param baseCommitHash the base commit (previously analyzed commit)
- * @param headCommitHash the head commit (current commit to analyze)
- * @return raw unified diff between the two commits
- * @throws IOException on network / parsing errors
- */
- String getCommitRangeDiff(OkHttpClient client, String workspace, String repoSlug, String baseCommitHash, String headCommitHash) throws IOException;
-
- /**
- * Checks if a file exists in the specified branch.
- *
- * @param client authorized HTTP client
- * @param workspace workspace or team/organization slug
- * @param repoSlug repository slug
- * @param branchName branch name (or commit hash)
- * @param filePath file path relative to repository root
- * @return true if file exists in the branch, false otherwise
- * @throws IOException on network errors
- */
- boolean checkFileExistsInBranch(OkHttpClient client, String workspace, String repoSlug, String branchName, String filePath) throws IOException;
-
- /**
- * Finds the pull request number that introduced a specific commit to the repository.
- * This is useful for branch reconciliation when we need to track which PR resolved an issue.
- *
- * @param client authorized HTTP client
- * @param workspace workspace or team/organization slug
- * @param repoSlug repository slug
- * @param commitHash the commit hash to look up
- * @return the PR/MR number that introduced this commit, or null if not found or commit wasn't from a PR
- * @throws IOException on network errors
- */
- Long findPullRequestForCommit(OkHttpClient client, String workspace, String repoSlug, String commitHash) throws IOException;
-
- /**
- * Fetches the current lifecycle state of a pull request from the VCS.
- *
- * @param client authorized HTTP client
- * @param workspace workspace or team/organization slug
- * @param repoSlug repository slug
- * @param prNumber pull request number
- * @return mapped CodeCrow pull request state, or empty when the provider returns an unknown state
- * @throws IOException on network / parsing errors
- */
- Optional getPullRequestState(
- OkHttpClient client,
- String workspace,
- String repoSlug,
- Long prNumber) throws IOException;
-
- /**
- * Fetches the raw content of a file at a specific branch or commit.
- * Used for computing line hashes during branch analysis reconciliation.
- *
- * @param client authorized HTTP client
- * @param workspace workspace or team/organization slug
- * @param repoSlug repository slug
- * @param branchOrCommit branch name or commit hash
- * @param filePath file path relative to repository root
- * @return the raw file content as a string, or null if file does not exist
- * @throws IOException on network errors
- */
- String getFileContent(OkHttpClient client, String workspace, String repoSlug, String branchOrCommit, String filePath) throws IOException;
-}
diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsReportingService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsReportingService.java
index 21a48829..bd8fdca1 100644
--- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsReportingService.java
+++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsReportingService.java
@@ -138,12 +138,13 @@ default void updateComment(
/**
* Post a reply to an existing comment with additional context.
- * For platforms that don't support threading (GitHub), this will format
+ * For provider comment types that don't support threading, this will format
* the reply with a quote and mention.
*
* @param project The project entity
* @param pullRequestNumber The PR number
* @param parentCommentId The ID of the comment to reply to
+ * @param inlineComment Whether the triggering comment belongs to a diff thread
* @param content The reply content (markdown)
* @param originalAuthorUsername Username of original comment author (for @mention)
* @param originalCommentBody Original comment body (for quoting)
@@ -153,6 +154,7 @@ default String postCommentReplyWithContext(
Project project,
Long pullRequestNumber,
String parentCommentId,
+ boolean inlineComment,
String content,
String originalAuthorUsername,
String originalCommentBody
diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsServiceFactory.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsServiceFactory.java
index 52e3002f..a030cb02 100644
--- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsServiceFactory.java
+++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsServiceFactory.java
@@ -16,19 +16,15 @@ public class VcsServiceFactory {
private final Map aiClientServices;
private final Map reportingServices;
- private final Map operationsServices;
public VcsServiceFactory(
List aiClientServiceList,
- List reportingServiceList,
- List operationsServiceList
+ List reportingServiceList
) {
this.aiClientServices = aiClientServiceList.stream()
.collect(Collectors.toMap(VcsAiClientService::getProvider, Function.identity()));
this.reportingServices = reportingServiceList.stream()
.collect(Collectors.toMap(VcsReportingService::getProvider, Function.identity()));
- this.operationsServices = operationsServiceList.stream()
- .collect(Collectors.toMap(VcsOperationsService::getProvider, Function.identity()));
}
public VcsAiClientService getAiClientService(EVcsProvider provider) {
@@ -46,12 +42,4 @@ public VcsReportingService getReportingService(EVcsProvider provider) {
}
return service;
}
-
- public VcsOperationsService getOperationsService(EVcsProvider provider) {
- VcsOperationsService service = operationsServices.get(provider);
- if (service == null) {
- throw new UnsupportedOperationException("No operations service registered for provider: " + provider);
- }
- return service;
- }
}
diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/AnalysisLimitEnforcer.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/AnalysisLimitEnforcer.java
index b86b95bc..ae96f4d3 100644
--- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/AnalysisLimitEnforcer.java
+++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/AnalysisLimitEnforcer.java
@@ -31,16 +31,17 @@ public AnalysisLimitsConfig effectiveLimits(Project project) {
}
public void enforce(Project project, Long pullRequestId, String rawDiff) {
- if (rawDiff == null || rawDiff.isBlank()) {
+ String scopedDiff = AnalysisScopeFilter.filterDiff(rawDiff, project);
+ if (scopedDiff == null || scopedDiff.isBlank()) {
return;
}
AnalysisLimitsConfig limits = effectiveLimits(project);
- List sections = DiffParsingUtils.parseFileChanges(rawDiff);
+ List sections = DiffParsingUtils.parseFileChanges(scopedDiff);
if (sections.size() > limits.maxFiles()) {
throw exceeded(LimitType.FILES, sections.size(), limits.maxFiles(), project, pullRequestId, null);
}
- long totalBytes = rawDiff.getBytes(StandardCharsets.UTF_8).length;
+ long totalBytes = scopedDiff.getBytes(StandardCharsets.UTF_8).length;
if (totalBytes > limits.maxTotalDiffSizeBytes()) {
throw exceeded(LimitType.TOTAL_DIFF_SIZE, totalBytes, limits.maxTotalDiffSizeBytes(),
project, pullRequestId, null);
@@ -55,7 +56,7 @@ public void enforce(Project project, Long pullRequestId, String rawDiff) {
}
}
- int estimatedTokens = TokenEstimator.estimateTokens(rawDiff);
+ int estimatedTokens = TokenEstimator.estimateTokens(scopedDiff);
if (estimatedTokens > limits.maxTotalTokens()) {
throw exceeded(LimitType.TOKENS, estimatedTokens, limits.maxTotalTokens(),
project, pullRequestId, null);
diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/AnalysisScopeFilter.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/AnalysisScopeFilter.java
index 3eb2a37b..ad61e425 100644
--- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/AnalysisScopeFilter.java
+++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/AnalysisScopeFilter.java
@@ -1,5 +1,6 @@
package org.rostilos.codecrow.analysisengine.util;
+import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -7,29 +8,66 @@
import org.rostilos.codecrow.core.model.project.config.AnalysisScopeConfig;
import org.rostilos.codecrow.core.model.project.config.ProjectConfig;
-/** Applies project analysis include/exclude globs to unified diffs and file sets. */
+/**
+ * Applies the effective analysis file scope to unified diffs and file sets.
+ * Project exclusions are combined with generated artifacts that are never useful
+ * review input. This is the shared scope for PR review, branch review,
+ * reconciliation and hard-limit accounting.
+ */
public final class AnalysisScopeFilter {
+ public static final List DEFAULT_EXCLUDE_PATTERNS = List.of(
+ "*.min.js",
+ "*.min.mjs",
+ "*.min.cjs",
+ "*.min.css",
+ "*.js.map",
+ "*.mjs.map",
+ "*.cjs.map",
+ "*.jsx.map",
+ "*.ts.map",
+ "*.tsx.map",
+ "*.css.map",
+ "*.scss.map",
+ "*.sass.map",
+ "*.less.map",
+ "*.vue.map",
+ "*.svelte.map",
+ "*.dart.map",
+ "*.wasm.map");
+
private AnalysisScopeFilter() {}
public static AnalysisScopeConfig scope(Project project) {
- if (project == null) return new AnalysisScopeConfig();
- ProjectConfig config = project.getEffectiveConfig();
- return config != null ? config.analysisScope() : new AnalysisScopeConfig();
+ AnalysisScopeConfig configured = new AnalysisScopeConfig();
+ if (project != null) {
+ ProjectConfig config = project.getEffectiveConfig();
+ if (config != null && config.analysisScope() != null) {
+ configured = config.analysisScope();
+ }
+ }
+
+ List effectiveExclusions = new ArrayList<>(
+ DEFAULT_EXCLUDE_PATTERNS.size() + configured.excludePatterns().size());
+ effectiveExclusions.addAll(DEFAULT_EXCLUDE_PATTERNS);
+ effectiveExclusions.addAll(configured.excludePatterns());
+ return new AnalysisScopeConfig(configured.includePatterns(), effectiveExclusions);
}
public static String filterDiff(String rawDiff, Project project) {
if (rawDiff == null || rawDiff.isBlank()) return rawDiff;
AnalysisScopeConfig scope = scope(project);
- if (scope.includePatterns().isEmpty() && scope.excludePatterns().isEmpty()) return rawDiff;
StringBuilder result = new StringBuilder();
+ boolean excludedAny = false;
List changes = DiffParsingUtils.parseFileChanges(rawDiff);
for (DiffParsingUtils.FileChange change : changes) {
if (scope.includesChange(change.oldPath(), change.newPath())) {
result.append(change.diff());
+ } else {
+ excludedAny = true;
}
}
- return result.toString();
+ return excludedAny ? result.toString() : rawDiff;
}
public static void retainIncluded(Set paths, Project project) {
diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/TextFileEligibility.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/TextFileEligibility.java
new file mode 100644
index 00000000..e0149fff
--- /dev/null
+++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/TextFileEligibility.java
@@ -0,0 +1,63 @@
+package org.rostilos.codecrow.analysisengine.util;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
+import java.util.Set;
+
+/**
+ * Shared host-level classification for files that cannot provide text context.
+ *
+ * This is intentionally a conservative binary/non-text blacklist rather
+ * than a source-language allowlist. Language and framework plugins remain free
+ * to decide which text files are useful; the host only prevents known binary
+ * assets from entering text acquisition and indexing paths.
+ */
+public final class TextFileEligibility {
+
+ public static final long MAX_TEXT_CONTENT_BYTES = 10L * 1024L * 1024L;
+
+ private static final Set EXCLUDED_EXTENSIONS = Set.of(
+ // Raster images. SVG is XML source and remains eligible.
+ ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".webp", ".tiff", ".tif",
+ // Fonts
+ ".woff", ".woff2", ".ttf", ".otf", ".eot",
+ // Compiled / bytecode
+ ".class", ".pyc", ".pyo", ".o", ".obj", ".dll", ".so", ".dylib", ".exe",
+ ".war", ".ear", ".nar",
+ // Archives
+ ".jar", ".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar",
+ // Documents / media
+ ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
+ ".mp3", ".mp4", ".avi", ".mov", ".wav", ".flac", ".ogg", ".webm",
+ // Data blobs / certificates
+ ".bin", ".dat", ".db", ".sqlite", ".sqlite3",
+ ".p12", ".pfx", ".jks", ".keystore", ".der", ".cer",
+ // Lock files and other binary build products
+ ".lockb", ".wasm"
+ );
+
+ private TextFileEligibility() {
+ }
+
+ public static boolean isTextCandidate(String path) {
+ if (path == null || path.isBlank()) {
+ return false;
+ }
+ String normalized = path.toLowerCase(Locale.ROOT);
+ return EXCLUDED_EXTENSIONS.stream().noneMatch(normalized::endsWith);
+ }
+
+ /**
+ * Applies the same bounded-text expectation to provider responses whose
+ * extension alone does not reveal whether they are indexable.
+ */
+ public static boolean isBoundedTextContent(String content) {
+ if (content == null || content.indexOf('\0') >= 0) {
+ return false;
+ }
+ if (content.length() > MAX_TEXT_CONTENT_BYTES) {
+ return false;
+ }
+ return content.getBytes(StandardCharsets.UTF_8).length <= MAX_TEXT_CONTENT_BYTES;
+ }
+}
diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/VcsDiffUtils.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/VcsDiffUtils.java
index 0209ecd6..6a4e51ba 100644
--- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/VcsDiffUtils.java
+++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/VcsDiffUtils.java
@@ -37,9 +37,8 @@ private VcsDiffUtils() {
/**
* Provider-agnostic callback for obtaining the raw diff between two commits.
*
- * Implementations typically delegate to a VCS-specific action class
- * (e.g. {@code GetCommitRangeDiffAction}) or to
- * {@code VcsOperationsService.getCommitRangeDiff}.
+ * Implementations delegate to the authorized
+ * {@code VcsClient.getCommitRangeDiff} operation.
*/
@FunctionalInterface
public interface CommitRangeDiffFetcher {
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java
index 822e2276..ec305311 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java
@@ -207,6 +207,8 @@ public Map getReconciliationFileContents() {
void setUp() throws Exception {
objectMapper = new ObjectMapper();
client = new AiAnalysisClient(restTemplate, queueService, objectMapper);
+ lenient().when(queueService.hasKey(AiAnalysisClient.CONSUMER_HEARTBEAT_KEY))
+ .thenReturn(true);
System.setProperty(PromptDryRunMode.ENABLED_KEY, "false");
System.clearProperty(PromptDryRunMode.PROJECT_IDS_KEY);
}
@@ -510,9 +512,8 @@ void shouldHandleIntermediateEvents() throws Exception {
verify(queueService, times(3)).rightPop(anyString(), anyLong());
assertThat(response.get("comment")).isEqualTo("All good");
- assertThat(capturedEvents).hasSize(2);
+ assertThat(capturedEvents).hasSize(1);
assertThat(capturedEvents.get(0).get("type")).isEqualTo("progress");
- assertThat(capturedEvents.get(1).get("type")).isEqualTo("result");
}
@Test
@@ -555,8 +556,8 @@ void shouldAllowLongReviewWhileWorkerRemainsActive() throws Exception {
}
@Test
- @DisplayName("should keep waiting while the exact job remains durably queued")
- void shouldTreatExactQueueMembershipAsAdmissionLiveness() throws Exception {
+ @DisplayName("should report durable queue wait while a live consumer can still admit it")
+ void shouldReportDurableQueueWaitWithLiveConsumer() throws Exception {
AiAnalysisClient shortInactivityClient = new AiAnalysisClient(
restTemplate, queueService, objectMapper, 20L);
String terminal = objectMapper.writeValueAsString(Map.of(
@@ -586,6 +587,9 @@ void shouldTreatExactQueueMembershipAsAdmissionLiveness() throws Exception {
assertThat(event).containsEntry("type", "status");
assertThat(event).containsEntry("state", "queued");
});
+ assertThat(capturedEvents)
+ .noneMatch(event -> "final".equals(event.get("type"))
+ || "result".equals(event.get("type")));
verify(queueService).listContains(
eq("codecrow:analysis:jobs"),
anyString());
@@ -596,6 +600,66 @@ void shouldTreatExactQueueMembershipAsAdmissionLiveness() throws Exception {
@DisplayName("performAnalysis() error paths")
class PerformAnalysisErrorTests {
+ @Test
+ @DisplayName("should fail before enqueue when no review consumer is alive")
+ void shouldFailWhenInferenceOrchestratorIsUnavailable() {
+ when(queueService.hasKey(AiAnalysisClient.CONSUMER_HEARTBEAT_KEY))
+ .thenReturn(false);
+
+ assertThatThrownBy(() -> client.performAnalysis(mockRequest))
+ .isInstanceOf(IOException.class)
+ .hasMessageContaining("no live review queue consumer");
+
+ verify(queueService, never()).leftPush(anyString(), anyString());
+ }
+
+ @Test
+ @DisplayName("should fail queued work when the review consumer heartbeat expires")
+ void shouldFailWhenConsumerDisappearsBeforeAdmission() {
+ AiAnalysisClient shortAdmissionClient = new AiAnalysisClient(
+ restTemplate,
+ queueService,
+ objectMapper,
+ 1_000L,
+ 1_000L);
+ when(queueService.hasKey(AiAnalysisClient.CONSUMER_HEARTBEAT_KEY))
+ .thenReturn(true, true, false);
+ when(queueService.rightPop(anyString(), anyLong()))
+ .thenReturn(null);
+
+ assertThatThrownBy(() ->
+ shortAdmissionClient.performAnalysis(mockRequest))
+ .isInstanceOf(IOException.class)
+ .hasMessageContaining(
+ "became unavailable before the review job was admitted");
+
+ verify(queueService).removeFromList(
+ eq("codecrow:analysis:jobs"),
+ anyString());
+ }
+
+ @Test
+ @DisplayName("should fail work that exceeds the bounded admission wait")
+ void shouldFailWhenWorkerCapacityDoesNotAdmitJobInTime() {
+ AiAnalysisClient shortAdmissionClient = new AiAnalysisClient(
+ restTemplate,
+ queueService,
+ objectMapper,
+ 1_000L,
+ 10L);
+ when(queueService.rightPop(anyString(), anyLong()))
+ .thenAnswer(invocation -> {
+ Thread.sleep(12L);
+ return null;
+ });
+
+ assertThatThrownBy(() ->
+ shortAdmissionClient.performAnalysis(mockRequest))
+ .isInstanceOf(IOException.class)
+ .hasMessageContaining(
+ "was not admitted by Inference Orchestrator");
+ }
+
@Test
@DisplayName("should throw IOException when AI service returns error event")
void shouldThrowWhenErrorEventReceived() throws Exception {
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientRecordsTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientRecordsTest.java
index 276d2c93..e1e6f74c 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientRecordsTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientRecordsTest.java
@@ -21,7 +21,7 @@ void shouldCreateWithAllFields() {
AiCommandClient.SummarizeRequest request = new AiCommandClient.SummarizeRequest(
1L, "workspace", "repo-slug", "project-workspace", "namespace",
"openai", "gpt-4", "api-key", null, 42L, "feature", "main", "abc123",
- "oauth-client", "oauth-secret", "access-token", true, 4096, "bitbucket"
+ "oauth-client", "oauth-secret", "access-token", true, 4096, "bitbucket", null
);
assertThat(request.projectId()).isEqualTo(1L);
@@ -56,7 +56,7 @@ void shouldCreateWithAllFields() {
1L, "workspace", "repo-slug", "project-workspace", "namespace",
"anthropic", "claude-3", "api-key", null, "What is this code doing?",
42L, "abc123", "oauth-client", "oauth-secret", "access-token",
- 8192, "github", "analysis context", List.of("issue-1", "issue-2")
+ 8192, "github", null, "analysis context", List.of("issue-1", "issue-2")
);
assertThat(request.projectId()).isEqualTo(1L);
@@ -76,7 +76,7 @@ void shouldSupportNullOptionalFields() {
1L, "workspace", "repo-slug", null, null,
"openai", "gpt-4", "api-key", null, "question",
null, null, null, null, null,
- null, "bitbucket", null, null
+ null, "bitbucket", null, null, null
);
assertThat(request.pullRequestId()).isNull();
@@ -147,7 +147,7 @@ void shouldCreateWithAllFields() {
AiCommandClient.ReviewRequest request = new AiCommandClient.ReviewRequest(
1L, "workspace", "repo-slug", "project-workspace", "namespace",
"openai", "gpt-4", "api-key", null, 42L, "feature", "main", "abc123",
- "oauth-client", "oauth-secret", "access-token", 4096, "bitbucket"
+ "oauth-client", "oauth-secret", "access-token", 4096, "bitbucket", null
);
assertThat(request.projectId()).isEqualTo(1L);
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java
index d614992a..a187e6b8 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java
@@ -42,7 +42,7 @@ private AiCommandClient.SummarizeRequest createSummarizeRequest() {
return new AiCommandClient.SummarizeRequest(
1L, "workspace", "repo-slug", "project-workspace", "namespace",
"openai", "gpt-4", "api-key", null, 42L, "feature", "main", "abc123",
- "oauth-client", "oauth-secret", "access-token", true, 4096, "bitbucket");
+ "oauth-client", "oauth-secret", "access-token", true, 4096, "bitbucket", null);
}
private AiCommandClient.AskRequest createAskRequest() {
@@ -50,14 +50,14 @@ private AiCommandClient.AskRequest createAskRequest() {
1L, "workspace", "repo-slug", "project-workspace", "namespace",
"openai", "gpt-4", "api-key", null, "What is this code doing?",
42L, "abc123", "oauth-client", "oauth-secret", "access-token",
- 4096, "bitbucket", null, null);
+ 4096, "bitbucket", null, null, null);
}
private AiCommandClient.ReviewRequest createReviewRequest() {
return new AiCommandClient.ReviewRequest(
1L, "workspace", "repo-slug", "project-workspace", "namespace",
"openai", "gpt-4", "api-key", null, 42L, "feature", "main", "abc123",
- "oauth-client", "oauth-secret", "access-token", 4096, "bitbucket");
+ "oauth-client", "oauth-secret", "access-token", 4096, "bitbucket", null);
}
@Nested
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java
index 9cf54e83..f987a06e 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java
@@ -1,6 +1,5 @@
package org.rostilos.codecrow.analysisengine.processor.analysis;
-import okhttp3.OkHttpClient;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
@@ -26,11 +25,11 @@
import org.rostilos.codecrow.commitgraph.service.AnalyzedCommitService;
import org.rostilos.codecrow.analysisengine.service.AstScopeEnricher;
import org.rostilos.codecrow.analysisengine.service.AnalysisLockService;
+import org.rostilos.codecrow.analysisengine.service.BranchArchiveService;
import org.rostilos.codecrow.analysisengine.service.ProjectValidationService;
import org.rostilos.codecrow.analysisengine.service.PullRequestService;
import org.rostilos.codecrow.analysisengine.service.PullRequestStatusSyncService;
import org.rostilos.codecrow.commitgraph.service.CommitCoverageService;
-import org.rostilos.codecrow.analysisengine.service.vcs.VcsOperationsService;
import org.rostilos.codecrow.analysisengine.service.vcs.VcsServiceFactory;
import org.rostilos.codecrow.analysisengine.util.DiffParsingUtils;
import org.rostilos.codecrow.analysisengine.util.AnalysisLimitEnforcer;
@@ -46,6 +45,7 @@
import org.rostilos.codecrow.vcsclient.VcsClient;
import org.rostilos.codecrow.vcsclient.VcsClientProvider;
import org.rostilos.codecrow.vcsclient.model.VcsCommit;
+import org.springframework.test.util.ReflectionTestUtils;
import java.io.IOException;
import java.util.*;
@@ -127,9 +127,6 @@ class BranchAnalysisProcessorTest {
@Mock
private RagOperationsService ragOperationsService;
- @Mock
- private VcsOperationsService operationsService;
-
@Mock
private Project project;
@@ -137,7 +134,7 @@ class BranchAnalysisProcessorTest {
private VcsConnection vcsConnection;
@Mock
- private OkHttpClient httpClient;
+ private VcsClient authorizedClient;
@Mock
private Branch branch;
@@ -146,6 +143,7 @@ class BranchAnalysisProcessorTest {
@BeforeEach
void setUp() {
+ when(vcsClientProvider.getClient(vcsConnection)).thenReturn(authorizedClient);
processor = new BranchAnalysisProcessor(
projectService,
branchRepository,
@@ -405,23 +403,21 @@ void shouldCompleteFullHappyPath() throws Exception {
when(repoInfo.getRepoWorkspace()).thenReturn("ws");
when(repoInfo.getRepoSlug()).thenReturn("repo");
when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD);
- when(vcsClientProvider.getHttpClient(vcsConnection)).thenReturn(httpClient);
- when(vcsServiceFactory.getOperationsService(EVcsProvider.BITBUCKET_CLOUD)).thenReturn(operationsService);
-
// DAG sync
when(branchCommitService.resolveCommitRange(any(), any(), any(), any()))
.thenReturn(new CommitRangeContext(Collections.emptyList(), null, true));
// Diff fetcher returns the raw diff
String rawDiff = "diff --git a/src/App.java b/src/App.java\n+new code\n";
- when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any(), any()))
+ when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any()))
.thenReturn(rawDiff);
// Support services return values
Map archiveContents = Map.of("src/App.java", "file content");
- when(branchFileOperationsService.downloadBranchArchive(any(), eq("new-commit"), anySet()))
- .thenReturn(archiveContents);
- when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveContents)))
+ var archiveSnapshot = archiveSnapshot(archiveContents);
+ when(branchFileOperationsService.downloadBranchFileSnapshot(any(), eq("new-commit"), anySet()))
+ .thenReturn(archiveSnapshot);
+ when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveSnapshot)))
.thenReturn(Set.of("src/App.java"));
Branch savedBranch = mock(Branch.class);
@@ -443,17 +439,21 @@ void shouldCompleteFullHappyPath() throws Exception {
assertThat(result).containsEntry("cached", false);
// Verify orchestration calls to support services
- verify(branchFileOperationsService).downloadBranchArchive(any(), eq("new-commit"), anySet());
- verify(branchFileOperationsService).updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveContents));
+ verify(branchFileOperationsService).downloadBranchFileSnapshot(any(), eq("new-commit"), anySet());
+ verify(branchFileOperationsService).updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveSnapshot));
verify(branchFileOperationsService).createOrUpdateProjectBranch(eq(project), eq(request), any());
verify(branchDiffFetcher).fetchDiff(
- any(), any(), any(), any(), any(), any(), isNull(), any());
+ any(), any(), any(), any(), any(), isNull(), any());
verify(branchIssueMappingService).mapCodeAnalysisIssuesToBranch(
anySet(), anySet(), eq(savedBranch), eq(project), eq(Set.of(40L, 41L, 42L)));
verify(branchIssueReconciliationService).reconcileIssueLineNumbers(eq(rawDiff), anySet(), eq(savedBranch));
verify(branchIssueReconciliationService).reanalyzeCandidateIssues(
- anySet(), anySet(), eq(savedBranch), eq(project), eq(request), eq(consumer), eq(archiveContents), eq(rawDiff));
- verify(branchFileOperationsService).updateFileSnapshotsForBranch(anySet(), eq(project), eq(request), eq(archiveContents));
+ anySet(), anySet(), eq(savedBranch), eq(project), eq(request), eq(consumer),
+ eq(archiveContents), eq(rawDiff), eq(false));
+ verify(branchIssueReconciliationService, never()).sweepDeterministicResolutions(
+ anySet(), any(), any(), any(), anyMap());
+ verify(branchFileOperationsService).updateFileSnapshotsForBranch(
+ anySet(), eq(project), eq(request), eq(archiveSnapshot));
verify(branchIssueReconciliationService).verifyIssueLineNumbersWithSnippets(anySet(), eq(project), any());
verify(analysisLockService).releaseLock("lock-key");
}
@@ -484,23 +484,21 @@ void shouldUseDeltaDiffWhenLastSuccessfulCommitExists() throws Exception {
when(repoInfo.getRepoWorkspace()).thenReturn("ws");
when(repoInfo.getRepoSlug()).thenReturn("repo");
when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD);
- when(vcsClientProvider.getHttpClient(vcsConnection)).thenReturn(httpClient);
- when(vcsServiceFactory.getOperationsService(EVcsProvider.BITBUCKET_CLOUD)).thenReturn(operationsService);
-
// DAG sync
when(branchCommitService.resolveCommitRange(any(), any(), any(), any()))
.thenReturn(new CommitRangeContext(Collections.emptyList(), null, false));
// Diff fetcher returns delta diff
String rawDiff = "diff --git a/src/App.java b/src/App.java\n+delta change\n";
- when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any(), any()))
+ when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any()))
.thenReturn(rawDiff);
// Support services
Map archiveContents = Map.of("src/App.java", "content");
- when(branchFileOperationsService.downloadBranchArchive(any(), eq("new-commit"), anySet()))
- .thenReturn(archiveContents);
- when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveContents)))
+ var archiveSnapshot = archiveSnapshot(archiveContents);
+ when(branchFileOperationsService.downloadBranchFileSnapshot(any(), eq("new-commit"), anySet()))
+ .thenReturn(archiveSnapshot);
+ when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveSnapshot)))
.thenReturn(Set.of("src/App.java"));
when(branchFileOperationsService.createOrUpdateProjectBranch(eq(project), eq(request), any()))
.thenReturn(existingBranch);
@@ -511,7 +509,7 @@ void shouldUseDeltaDiffWhenLastSuccessfulCommitExists() throws Exception {
Map result = processor.process(request, consumer);
assertThat(result).containsEntry("status", "accepted");
- verify(branchDiffFetcher).fetchDiff(any(), any(), any(), any(), any(), any(), any(), any());
+ verify(branchDiffFetcher).fetchDiff(any(), any(), any(), any(), any(), any(), any());
}
@Test
@@ -547,10 +545,6 @@ void shouldScopeOversizedDirectPushToFilesWithPreviousIssues() throws Exception
when(repoInfo.getRepoWorkspace()).thenReturn("ws");
when(repoInfo.getRepoSlug()).thenReturn("repo");
when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD);
- when(vcsClientProvider.getHttpClient(vcsConnection)).thenReturn(httpClient);
- when(vcsServiceFactory.getOperationsService(EVcsProvider.BITBUCKET_CLOUD))
- .thenReturn(operationsService);
-
when(branchCommitService.resolveCommitRange(any(), any(), any(), any()))
.thenReturn(new CommitRangeContext(List.of("new-commit"), "old-commit", false));
@@ -568,7 +562,7 @@ void shouldScopeOversizedDirectPushToFilesWithPreviousIssues() throws Exception
-old unrelated code
+new unrelated code
""";
- when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any(), any()))
+ when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any()))
.thenReturn(rawDiff);
doThrow(new DiffTooLargeException(
DiffTooLargeException.LimitType.FILES, 851, 150, 1L, null, null))
@@ -578,10 +572,11 @@ void shouldScopeOversizedDirectPushToFilesWithPreviousIssues() throws Exception
.thenReturn(Set.of("src/Issue.java"));
Map archiveContents = Map.of("src/Issue.java", "new issue code");
- when(branchFileOperationsService.downloadBranchArchive(any(), eq("new-commit"), anySet()))
- .thenReturn(archiveContents);
+ var archiveSnapshot = archiveSnapshot(archiveContents);
+ when(branchFileOperationsService.downloadBranchFileSnapshot(any(), eq("new-commit"), anySet()))
+ .thenReturn(archiveSnapshot);
when(branchFileOperationsService.updateBranchFiles(
- anySet(), eq(project), eq("feature-x"), eq(archiveContents)))
+ anySet(), eq(project), eq("feature-x"), eq(archiveSnapshot)))
.thenReturn(Set.of("src/Issue.java"));
when(branchFileOperationsService.createOrUpdateProjectBranch(eq(project), eq(request), any()))
.thenReturn(existingBranch);
@@ -592,7 +587,6 @@ void shouldScopeOversizedDirectPushToFilesWithPreviousIssues() throws Exception
when(ragOperationsService.isRagIndexReady(project)).thenReturn(true);
when(ragOperationsService.getBaseBranch(project)).thenReturn("main");
when(ragOperationsService.isRagPipelineHealthy()).thenReturn(true);
-
Map result = processor.process(request, events::add);
assertThat(result).containsEntry("status", "accepted");
@@ -600,13 +594,13 @@ void shouldScopeOversizedDirectPushToFilesWithPreviousIssues() throws Exception
assertThat(event).containsEntry("state", "direct_push_analysis_skipped_limit"));
verify(aiAnalysisClient, never()).performAnalysis(any(), any());
verify(commitCoverageService, never()).checkCoverage(any(), any(), anyList());
- verify(branchFileOperationsService).downloadBranchArchive(
+ verify(branchFileOperationsService).downloadBranchFileSnapshot(
any(), eq("new-commit"), eq(Set.of("src/Issue.java")));
verify(branchIssueReconciliationService).reanalyzeCandidateIssues(
eq(Set.of("src/Issue.java")), eq(Set.of("src/Issue.java")),
eq(existingBranch), eq(project), eq(request), any(), eq(archiveContents),
argThat(diff -> diff.contains("src/Issue.java")
- && !diff.contains("src/Unrelated.java")));
+ && !diff.contains("src/Unrelated.java")), eq(false));
verify(branchIssueReconciliationService, never()).sweepDeterministicResolutions(
anySet(), any(), any(), any(), anyMap());
verify(ragOperationsService).triggerIncrementalUpdate(
@@ -638,23 +632,21 @@ void shouldFallBackToCommitDiff() throws Exception {
when(repoInfo.getRepoWorkspace()).thenReturn("ws");
when(repoInfo.getRepoSlug()).thenReturn("repo");
when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD);
- when(vcsClientProvider.getHttpClient(vcsConnection)).thenReturn(httpClient);
- when(vcsServiceFactory.getOperationsService(EVcsProvider.BITBUCKET_CLOUD)).thenReturn(operationsService);
-
// DAG sync
when(branchCommitService.resolveCommitRange(any(), any(), any(), any()))
.thenReturn(new CommitRangeContext(Collections.emptyList(), null, false));
// Diff fetcher returns commit diff
String rawDiff = "diff --git a/README.md b/README.md\n+updated\n";
- when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any(), any()))
+ when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any()))
.thenReturn(rawDiff);
// Support services
Map archiveContents = Map.of("README.md", "content");
- when(branchFileOperationsService.downloadBranchArchive(any(), eq("new-commit"), anySet()))
- .thenReturn(archiveContents);
- when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveContents)))
+ var archiveSnapshot = archiveSnapshot(archiveContents);
+ when(branchFileOperationsService.downloadBranchFileSnapshot(any(), eq("new-commit"), anySet()))
+ .thenReturn(archiveSnapshot);
+ when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveSnapshot)))
.thenReturn(Set.of("README.md"));
Branch savedBranch = mock(Branch.class);
@@ -669,7 +661,7 @@ void shouldFallBackToCommitDiff() throws Exception {
Map result = processor.process(request, consumer);
assertThat(result).containsEntry("status", "accepted");
- verify(branchDiffFetcher).fetchDiff(any(), any(), any(), any(), any(), any(), any(), any());
+ verify(branchDiffFetcher).fetchDiff(any(), any(), any(), any(), any(), any(), any());
}
@Test
@@ -693,21 +685,19 @@ void shouldPerformRagUpdateOnMainBranch() throws Exception {
when(repoInfo.getRepoWorkspace()).thenReturn("ws");
when(repoInfo.getRepoSlug()).thenReturn("repo");
when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD);
- when(vcsClientProvider.getHttpClient(vcsConnection)).thenReturn(httpClient);
- when(vcsServiceFactory.getOperationsService(EVcsProvider.BITBUCKET_CLOUD)).thenReturn(operationsService);
-
// DAG sync
when(branchCommitService.resolveCommitRange(any(), any(), any(), any()))
.thenReturn(new CommitRangeContext(Collections.emptyList(), null, false));
String rawDiff = "diff --git a/f.java b/f.java\n+x\n";
- when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any(), any()))
+ when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any()))
.thenReturn(rawDiff);
Map archiveContents = Map.of("f.java", "content");
- when(branchFileOperationsService.downloadBranchArchive(any(), eq("new-commit"), anySet()))
- .thenReturn(archiveContents);
- when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveContents)))
+ var archiveSnapshot = archiveSnapshot(archiveContents);
+ when(branchFileOperationsService.downloadBranchFileSnapshot(any(), eq("new-commit"), anySet()))
+ .thenReturn(archiveSnapshot);
+ when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveSnapshot)))
.thenReturn(Set.of("f.java"));
Branch savedBranch = mock(Branch.class);
@@ -723,6 +713,9 @@ void shouldPerformRagUpdateOnMainBranch() throws Exception {
when(ragOperationsService.isRagIndexReady(project)).thenReturn(true);
when(ragOperationsService.getBaseBranch(project)).thenReturn("main");
when(ragOperationsService.isRagPipelineHealthy()).thenReturn(true);
+ when(ragOperationsService.triggerIncrementalUpdate(
+ eq(project), eq("main"), eq("new-commit"), eq(rawDiff), any()))
+ .thenReturn(true);
// Final markHealthy
when(branchRepository.findByProjectIdAndBranchName(1L, "main"))
@@ -734,6 +727,38 @@ void shouldPerformRagUpdateOnMainBranch() throws Exception {
verify(ragOperationsService).triggerIncrementalUpdate(eq(project), eq("main"), eq("new-commit"), eq(rawDiff), any());
}
+ @Test
+ @DisplayName("should not emit RAG success after an incremental failure")
+ void shouldNotEmitRagSuccessAfterIncrementalFailure() {
+ BranchProcessRequest request = createRequest();
+ request.commitHash = "failed-commit";
+ request.targetBranchName = "main";
+ String rawDiff = "diff --git a/f.java b/f.java\n+x\n";
+ List> events = new ArrayList<>();
+
+ when(project.getId()).thenReturn(1L);
+ when(ragOperationsService.isRagEnabled(project)).thenReturn(true);
+ when(ragOperationsService.isRagIndexReady(project)).thenReturn(true);
+ when(ragOperationsService.isRagPipelineHealthy()).thenReturn(true);
+ when(ragOperationsService.getBaseBranch(project)).thenReturn("main");
+ when(ragOperationsService.triggerIncrementalUpdate(
+ eq(project), eq("main"), eq("failed-commit"), eq(rawDiff), any()))
+ .thenReturn(false);
+
+ ReflectionTestUtils.invokeMethod(
+ processor,
+ "performIncrementalRagUpdate",
+ request,
+ project,
+ rawDiff,
+ (Consumer>) events::add,
+ false);
+
+ assertThat(events)
+ .noneSatisfy(event ->
+ assertThat(event).containsEntry("state", "rag_update_complete"));
+ }
+
@Test
@DisplayName("should call updateBranchIndex for non-main branch RAG update")
void shouldCallUpdateBranchIndexForNonMainBranch() throws Exception {
@@ -756,21 +781,19 @@ void shouldCallUpdateBranchIndexForNonMainBranch() throws Exception {
when(repoInfo.getRepoWorkspace()).thenReturn("ws");
when(repoInfo.getRepoSlug()).thenReturn("repo");
when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD);
- when(vcsClientProvider.getHttpClient(vcsConnection)).thenReturn(httpClient);
- when(vcsServiceFactory.getOperationsService(EVcsProvider.BITBUCKET_CLOUD)).thenReturn(operationsService);
-
// DAG sync
when(branchCommitService.resolveCommitRange(any(), any(), any(), any()))
.thenReturn(new CommitRangeContext(Collections.emptyList(), null, false));
String rawDiff = "diff --git a/f.java b/f.java\n+x\n";
- when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any(), any()))
+ when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any()))
.thenReturn(rawDiff);
Map archiveContents = Map.of("f.java", "content");
- when(branchFileOperationsService.downloadBranchArchive(any(), eq("new-commit"), anySet()))
- .thenReturn(archiveContents);
- when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("feature-x"), eq(archiveContents)))
+ var archiveSnapshot = archiveSnapshot(archiveContents);
+ when(branchFileOperationsService.downloadBranchFileSnapshot(any(), eq("new-commit"), anySet()))
+ .thenReturn(archiveSnapshot);
+ when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("feature-x"), eq(archiveSnapshot)))
.thenReturn(Set.of("f.java"));
Branch savedBranch = mock(Branch.class);
@@ -822,22 +845,20 @@ void shouldFallBackToPrDiffWhenDeltaDiffFails() throws Exception {
when(repoInfo.getRepoWorkspace()).thenReturn("ws");
when(repoInfo.getRepoSlug()).thenReturn("repo");
when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD);
- when(vcsClientProvider.getHttpClient(vcsConnection)).thenReturn(httpClient);
- when(vcsServiceFactory.getOperationsService(EVcsProvider.BITBUCKET_CLOUD)).thenReturn(operationsService);
-
// DAG sync
when(branchCommitService.resolveCommitRange(any(), any(), any(), any()))
.thenReturn(new CommitRangeContext(Collections.emptyList(), null, false));
// Diff fetcher returns diff (handles fallback internally)
String rawDiff = "diff --git a/f.java b/f.java\n+x\n";
- when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any(), any()))
+ when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any()))
.thenReturn(rawDiff);
Map archiveContents = Map.of("f.java", "content");
- when(branchFileOperationsService.downloadBranchArchive(any(), eq("new-commit"), anySet()))
- .thenReturn(archiveContents);
- when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveContents)))
+ var archiveSnapshot = archiveSnapshot(archiveContents);
+ when(branchFileOperationsService.downloadBranchFileSnapshot(any(), eq("new-commit"), anySet()))
+ .thenReturn(archiveSnapshot);
+ when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveSnapshot)))
.thenReturn(Set.of("f.java"));
when(branchFileOperationsService.createOrUpdateProjectBranch(eq(project), eq(request), any()))
.thenReturn(existingBranch);
@@ -847,7 +868,7 @@ void shouldFallBackToPrDiffWhenDeltaDiffFails() throws Exception {
processor.process(request, consumer);
- verify(branchDiffFetcher).fetchDiff(any(), any(), any(), any(), any(), any(), any(), any());
+ verify(branchDiffFetcher).fetchDiff(any(), any(), any(), any(), any(), any(), any());
}
@Test
@@ -876,21 +897,16 @@ void shouldRecoverSourcePrFromLocalReviewedMergeParent() throws Exception {
when(repoInfo.getRepoWorkspace()).thenReturn("ws");
when(repoInfo.getRepoSlug()).thenReturn("repo");
when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD);
- when(vcsClientProvider.getHttpClient(vcsConnection)).thenReturn(httpClient);
- when(vcsServiceFactory.getOperationsService(EVcsProvider.BITBUCKET_CLOUD)).thenReturn(operationsService);
-
when(branchCommitService.resolveCommitRange(any(), any(), any(), any()))
.thenReturn(new CommitRangeContext(List.of("source-head"), "old-commit", false));
- VcsClient vcsClient = mock(VcsClient.class);
- when(vcsClientProvider.getClient(vcsConnection)).thenReturn(vcsClient);
- when(vcsClient.getCommitHistory("ws", "repo", "merge-commit", 1))
+ when(authorizedClient.getCommitHistory("ws", "repo", "merge-commit", 1))
.thenReturn(List.of(new VcsCommit(
"merge-commit", "Merge PR", null, null, null,
List.of("target-parent", "source-head"))));
- when(operationsService.findPullRequestForCommit(httpClient, "ws", "repo", "merge-commit"))
+ when(authorizedClient.findPullRequestForCommit("ws", "repo", "merge-commit"))
.thenReturn(null);
- when(operationsService.findPullRequestForCommit(httpClient, "ws", "repo", "source-head"))
+ when(authorizedClient.findPullRequestForCommit("ws", "repo", "source-head"))
.thenReturn(null);
when(codeAnalysisService.findReviewedPrNumberByCommitHash(1L, "main", "merge-commit"))
.thenReturn(Optional.empty());
@@ -898,13 +914,14 @@ void shouldRecoverSourcePrFromLocalReviewedMergeParent() throws Exception {
.thenReturn(Optional.of(42L));
String rawDiff = "diff --git a/src/App.java b/src/App.java\n+change\n";
- when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any(), any()))
+ when(branchDiffFetcher.fetchDiff(any(), any(), any(), any(), any(), any(), any()))
.thenReturn(rawDiff);
Map archiveContents = Map.of("src/App.java", "content");
- when(branchFileOperationsService.downloadBranchArchive(any(), eq("merge-commit"), anySet()))
- .thenReturn(archiveContents);
- when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveContents)))
+ var archiveSnapshot = archiveSnapshot(archiveContents);
+ when(branchFileOperationsService.downloadBranchFileSnapshot(any(), eq("merge-commit"), anySet()))
+ .thenReturn(archiveSnapshot);
+ when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveSnapshot)))
.thenReturn(Set.of("src/App.java"));
when(branchFileOperationsService.createOrUpdateProjectBranch(eq(project), eq(request), any()))
.thenReturn(existingBranch);
@@ -914,7 +931,7 @@ void shouldRecoverSourcePrFromLocalReviewedMergeParent() throws Exception {
processor.process(request, consumer);
- verify(branchDiffFetcher).fetchDiff(any(), any(), any(), any(), any(), any(), eq(42L), any());
+ verify(branchDiffFetcher).fetchDiff(any(), any(), any(), any(), any(), eq(42L), any());
verify(branchIssueMappingService).mapCodeAnalysisIssuesToBranch(
anySet(), anySet(), eq(existingBranch), eq(project), eq(42L));
verify(pullRequestService).markPullRequestMerged(1L, 42L);
@@ -949,6 +966,12 @@ class RagUpdateTests {
// main branch vs non-main branch are all covered indirectly.
}
+ private static BranchFileOperationsService.BranchFileSnapshot archiveSnapshot(
+ Map contents) {
+ return BranchFileOperationsService.BranchFileSnapshot.fromArchive(
+ new BranchArchiveService.ArchiveSnapshot(contents, contents.keySet()));
+ }
+
@Nested
@DisplayName("Constructor")
class ConstructorTests {
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java
index 4fb36dea..5560b540 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java
@@ -28,16 +28,19 @@
import org.rostilos.codecrow.core.model.vcs.VcsConnection;
import org.rostilos.codecrow.core.model.vcs.VcsRepoInfo;
import org.rostilos.codecrow.core.service.CodeAnalysisService;
+import org.rostilos.codecrow.core.service.TaskImplementationEvidenceService;
import org.rostilos.codecrow.filecontent.service.FileSnapshotService;
import org.rostilos.codecrow.analysisengine.service.AstScopeEnricher;
import org.rostilos.codecrow.analysisengine.service.pr.PrIssueTrackingService;
import org.rostilos.codecrow.analysisengine.util.PromptDryRunMode;
import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.test.util.ReflectionTestUtils;
import java.io.IOException;
import java.util.*;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.*;
@@ -52,6 +55,9 @@ class PullRequestAnalysisProcessorTest {
@Mock
private CodeAnalysisService codeAnalysisService;
+ @Mock
+ private TaskImplementationEvidenceService taskImplementationEvidenceService;
+
@Mock
private AiAnalysisClient aiAnalysisClient;
@@ -109,9 +115,15 @@ class PullRequestAnalysisProcessorTest {
void setUp() {
System.setProperty(PromptDryRunMode.ENABLED_KEY, "false");
System.clearProperty(PromptDryRunMode.PROJECT_IDS_KEY);
+ lenient().when(taskImplementationEvidenceService.persistFromAnalysisResponse(
+ any(), any()))
+ .thenReturn(TaskImplementationEvidenceService.PersistenceResult.empty());
+ lenient().when(taskImplementationEvidenceService.copyForAnalysis(any(), any()))
+ .thenReturn(TaskImplementationEvidenceService.PersistenceResult.empty());
processor = new PullRequestAnalysisProcessor(
pullRequestService,
codeAnalysisService,
+ taskImplementationEvidenceService,
aiAnalysisClient,
vcsServiceFactory,
analysisLockService,
@@ -235,15 +247,31 @@ void shouldSuccessfullyProcessPRAnalysis() throws Exception {
"task_key", "PROJ-123",
"task_summary", "Build export"));
+ Map taskEvidence = Map.of(
+ "taskKey", "PROJ-123",
+ "source", "DETERMINISTIC_PR_LEDGER",
+ "fullEvidenceComplete", true,
+ "items", List.of(Map.of(
+ "evidenceRef", "PRF001",
+ "filePath", "src/Export.java",
+ "hunkId", "hunk-1",
+ "lineStart", 10,
+ "lineEnd", 12,
+ "excerpt", "exportService.run();")));
Map aiResponse = Map.of(
"comment", "Review comment",
- "issues", List.of());
+ "issues", List.of(),
+ "taskEvidence", taskEvidence);
when(aiAnalysisClient.performAnalysis(any(), any())).thenReturn(aiResponse);
when(codeAnalysisService.createAnalysisFromAiResponse(
any(), any(), anyLong(), anyString(), anyString(), anyString(), any(), any(),
any(), any(), any(), any()))
.thenReturn(codeAnalysis);
+ when(taskImplementationEvidenceService.persistFromAnalysisResponse(
+ codeAnalysis, taskEvidence))
+ .thenReturn(new TaskImplementationEvidenceService.PersistenceResult(
+ 1, 0, 0));
Map result = processor.process(request, consumer, project);
@@ -265,6 +293,27 @@ void shouldSuccessfullyProcessPRAnalysis() throws Exception {
anyMap(),
eq("PROJ-123"),
eq("Build export"));
+ verify(taskImplementationEvidenceService)
+ .persistFromAnalysisResponse(codeAnalysis, taskEvidence);
+ }
+
+ @Test
+ @DisplayName("should fail open when auxiliary task evidence persistence is unavailable")
+ void shouldFailOpenWhenTaskEvidencePersistenceFails() {
+ Map taskEvidence = Map.of(
+ "taskKey", "PROJ-123",
+ "source", "DETERMINISTIC_PR_LEDGER",
+ "items", List.of());
+ when(taskImplementationEvidenceService.persistFromAnalysisResponse(
+ codeAnalysis, taskEvidence))
+ .thenThrow(new RuntimeException("database unavailable"));
+
+ assertThatCode(() -> ReflectionTestUtils.invokeMethod(
+ processor,
+ "persistTaskImplementationEvidence",
+ codeAnalysis,
+ taskEvidence))
+ .doesNotThrowAnyException();
}
@Test
@@ -556,6 +605,8 @@ void shouldReturnCachedByCommitWhenCommitHashCacheHits() throws Exception {
assertThat(result).containsEntry("cached", true);
verify(codeAnalysisService).cloneAnalysisForPr(eq(sourceAnalysis), eq(project), eq(42L),
eq("abc123"), eq("main"), eq("feature-branch"), eq(reviewIdentity));
+ verify(taskImplementationEvidenceService)
+ .copyForAnalysis(sourceAnalysis, clonedAnalysis);
verify(reportingService).postAnalysisResults(eq(clonedAnalysis), any(), anyLong(), any(),
any());
verify(analysisLockService).releaseLock("lock-key");
@@ -613,6 +664,8 @@ void shouldReturnCachedByFingerprintWhenDiffFingerprintMatches() throws Exceptio
assertThat(result).containsEntry("status", "cached_by_fingerprint");
assertThat(result).containsEntry("cached", true);
+ verify(taskImplementationEvidenceService)
+ .copyForAnalysis(fingerprintSource, clonedAnalysis);
verify(analysisLockService).releaseLock("lock-key");
}
@@ -859,6 +912,7 @@ void shouldWorkWithoutOptionalDependencies() {
PullRequestAnalysisProcessor processorWithoutOptional = new PullRequestAnalysisProcessor(
pullRequestService,
codeAnalysisService,
+ taskImplementationEvidenceService,
aiAnalysisClient,
vcsServiceFactory,
analysisLockService,
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java
index 784e8a4b..d0e8e16d 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java
@@ -185,6 +185,75 @@ void shouldSkipBinaryFiles(@TempDir Path tempDir) throws Exception {
assertThat(result).doesNotContainKey("binary.dat");
}
+ @Test
+ void snapshotReportsBinaryPathPresenceWithoutLoadingItsContent(@TempDir Path tempDir)
+ throws Exception {
+ VcsConnection conn = new VcsConnection();
+ when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (ZipOutputStream zos = new ZipOutputStream(baos)) {
+ zos.putNextEntry(new ZipEntry("root/binary.dat"));
+ zos.write(new byte[]{0, 1, 2, 3});
+ zos.closeEntry();
+ zos.putNextEntry(new ZipEntry("root/text.txt"));
+ zos.write("hello".getBytes(StandardCharsets.UTF_8));
+ zos.closeEntry();
+ }
+ byte[] zipBytes = baos.toByteArray();
+ when(vcsClient.downloadRepositoryArchiveToFile(
+ anyString(), anyString(), anyString(), any(Path.class)))
+ .thenAnswer(inv -> {
+ Files.write(inv.getArgument(3, Path.class), zipBytes);
+ return (long) zipBytes.length;
+ });
+
+ BranchArchiveService.ArchiveSnapshot snapshot = service.downloadSnapshot(
+ conn, "ws", "repo", "main",
+ Set.of("binary.dat", "text.txt", "missing.txt"));
+
+ assertThat(snapshot.presentFiles()).containsExactlyInAnyOrder(
+ "binary.dat", "text.txt");
+ assertThat(snapshot.contents()).containsEntry("text.txt", "hello");
+ assertThat(snapshot.contents()).doesNotContainKey("binary.dat");
+ assertThat(snapshot.presentFiles()).doesNotContain("missing.txt");
+ }
+
+ @Test
+ void directorySnapshotReportsPresentBinaryPathWithoutExtractingIt(@TempDir Path tempDir)
+ throws Exception {
+ VcsConnection conn = new VcsConnection();
+ when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (ZipOutputStream zos = new ZipOutputStream(baos)) {
+ zos.putNextEntry(new ZipEntry("root/image.asset"));
+ zos.write(new byte[]{0, 1, 2, 3});
+ zos.closeEntry();
+ zos.putNextEntry(new ZipEntry("root/source.xml"));
+ zos.write("".getBytes(StandardCharsets.UTF_8));
+ zos.closeEntry();
+ }
+ byte[] zipBytes = baos.toByteArray();
+ when(vcsClient.downloadRepositoryArchiveToFile(
+ anyString(), anyString(), anyString(), any(Path.class)))
+ .thenAnswer(inv -> {
+ Files.write(inv.getArgument(3, Path.class), zipBytes);
+ return (long) zipBytes.length;
+ });
+
+ BranchArchiveService.ArchiveDirectorySnapshot snapshot =
+ service.downloadAndExtractSnapshotToDirectory(
+ conn, "ws", "repo", "main",
+ Set.of("image.asset", "source.xml"), tempDir.resolve("repository"));
+
+ assertThat(snapshot.presentFiles())
+ .containsExactlyInAnyOrder("image.asset", "source.xml");
+ assertThat(snapshot.extractedFiles()).containsExactly("source.xml");
+ assertThat(tempDir.resolve("repository/image.asset")).doesNotExist();
+ assertThat(tempDir.resolve("repository/source.xml")).exists();
+ }
+
@Test
void vcsClientThrows_shouldPropagate() throws Exception {
VcsConnection conn = new VcsConnection();
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/PullRequestStatusSyncServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/PullRequestStatusSyncServiceTest.java
index f12e04a5..fc6af987 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/PullRequestStatusSyncServiceTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/PullRequestStatusSyncServiceTest.java
@@ -1,21 +1,18 @@
package org.rostilos.codecrow.analysisengine.service;
-import okhttp3.OkHttpClient;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
-import org.rostilos.codecrow.analysisengine.service.vcs.VcsOperationsService;
-import org.rostilos.codecrow.analysisengine.service.vcs.VcsServiceFactory;
import org.rostilos.codecrow.core.model.project.Project;
import org.rostilos.codecrow.core.model.pullrequest.PullRequest;
import org.rostilos.codecrow.core.model.pullrequest.PullRequestState;
-import org.rostilos.codecrow.core.model.vcs.EVcsProvider;
import org.rostilos.codecrow.core.model.vcs.VcsConnection;
import org.rostilos.codecrow.core.model.vcs.VcsRepoInfo;
import org.rostilos.codecrow.core.persistence.repository.pullrequest.PullRequestRepository;
+import org.rostilos.codecrow.vcsclient.VcsClient;
import org.rostilos.codecrow.vcsclient.VcsClientProvider;
import java.io.IOException;
@@ -36,11 +33,9 @@ class PullRequestStatusSyncServiceTest {
@Mock private PullRequestRepository pullRequestRepository;
@Mock private VcsClientProvider vcsClientProvider;
- @Mock private VcsServiceFactory vcsServiceFactory;
- @Mock private VcsOperationsService operationsService;
+ @Mock private VcsClient vcsClient;
@Mock private Project project;
@Mock private VcsConnection vcsConnection;
- @Mock private OkHttpClient httpClient;
private PullRequestStatusSyncService service;
@@ -48,8 +43,7 @@ class PullRequestStatusSyncServiceTest {
void setUp() {
service = new PullRequestStatusSyncService(
pullRequestRepository,
- vcsClientProvider,
- vcsServiceFactory);
+ vcsClientProvider);
}
@Test
@@ -66,13 +60,13 @@ void shouldRepairStaleOpenPrStatesFromVcsAcrossWholeProject() throws IOException
.thenReturn(List.of(merged, stillOpen, declined, unknown));
mockVcsInfo();
- when(operationsService.getPullRequestState(httpClient, "ws", "repo", 11L))
+ when(vcsClient.getPullRequestState("ws", "repo", 11L))
.thenReturn(Optional.of(PullRequestState.MERGED));
- when(operationsService.getPullRequestState(httpClient, "ws", "repo", 12L))
+ when(vcsClient.getPullRequestState("ws", "repo", 12L))
.thenReturn(Optional.of(PullRequestState.OPEN));
- when(operationsService.getPullRequestState(httpClient, "ws", "repo", 13L))
+ when(vcsClient.getPullRequestState("ws", "repo", 13L))
.thenReturn(Optional.of(PullRequestState.DECLINED));
- when(operationsService.getPullRequestState(httpClient, "ws", "repo", 14L))
+ when(vcsClient.getPullRequestState("ws", "repo", 14L))
.thenReturn(Optional.empty());
PullRequestStatusSyncService.SyncResult result =
@@ -99,9 +93,7 @@ private void mockVcsInfo() {
when(repoInfo.getVcsConnection()).thenReturn(vcsConnection);
when(repoInfo.getRepoWorkspace()).thenReturn("ws");
when(repoInfo.getRepoSlug()).thenReturn("repo");
- when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.GITHUB);
- when(vcsClientProvider.getHttpClient(vcsConnection)).thenReturn(httpClient);
- when(vcsServiceFactory.getOperationsService(EVcsProvider.GITHUB)).thenReturn(operationsService);
+ when(vcsClientProvider.getClient(vcsConnection)).thenReturn(vcsClient);
}
private PullRequest pullRequest(Long prNumber) {
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/VcsFileRetrievalPolicyTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/VcsFileRetrievalPolicyTest.java
new file mode 100644
index 00000000..63186fd5
--- /dev/null
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/VcsFileRetrievalPolicyTest.java
@@ -0,0 +1,39 @@
+package org.rostilos.codecrow.analysisengine.service;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class VcsFileRetrievalPolicyTest {
+
+ @Test
+ void usesPerFileRetrievalUpToThresholdAndArchiveAboveIt() {
+ VcsFileRetrievalPolicy policy = new VcsFileRetrievalPolicy(25);
+
+ assertThat(policy.shouldUseArchive(25)).isFalse();
+ assertThat(policy.allowPerFileFallback(25)).isTrue();
+ assertThat(policy.shouldUseArchive(26)).isTrue();
+ assertThat(policy.allowPerFileFallback(26)).isFalse();
+ }
+
+ @Test
+ void clampsNegativeThresholdToArchiveOnly() {
+ VcsFileRetrievalPolicy policy = new VcsFileRetrievalPolicy(-10);
+
+ assertThat(policy.archiveFileThreshold()).isZero();
+ assertThat(policy.shouldUseArchive(1)).isTrue();
+ }
+
+ @Test
+ void recognizesWrappedProviderRateLimits() {
+ VcsFileRetrievalPolicy policy = new VcsFileRetrievalPolicy(25);
+ IOException failure = new IOException(
+ "file retrieval failed",
+ new IOException("Unexpected response 429: Too Many Requests"));
+
+ assertThat(policy.isRateLimited(failure)).isTrue();
+ assertThat(policy.isRateLimited(new IOException("connection reset"))).isFalse();
+ }
+}
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchDiffFetcherTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchDiffFetcherTest.java
index 7798e45c..8c1efc2d 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchDiffFetcherTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchDiffFetcherTest.java
@@ -1,6 +1,5 @@
package org.rostilos.codecrow.analysisengine.service.branch;
-import okhttp3.OkHttpClient;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
@@ -9,9 +8,9 @@
import org.mockito.junit.jupiter.MockitoExtension;
import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest;
import org.rostilos.codecrow.analysisengine.processor.VcsRepoInfoImpl;
-import org.rostilos.codecrow.analysisengine.service.vcs.VcsOperationsService;
import org.rostilos.codecrow.commitgraph.dag.CommitRangeContext;
import org.rostilos.codecrow.core.model.branch.Branch;
+import org.rostilos.codecrow.vcsclient.VcsClient;
import java.io.IOException;
import java.util.List;
@@ -23,8 +22,7 @@
@ExtendWith(MockitoExtension.class)
class BranchDiffFetcherTest {
- @Mock private VcsOperationsService operationsService;
- @Mock private OkHttpClient client;
+ @Mock private VcsClient client;
private BranchDiffFetcher fetcher;
private VcsRepoInfoImpl vcsRepoInfo;
@@ -51,11 +49,11 @@ class FirstAnalysis {
@Test
void noPrNumber_shouldUseCommitDiff() throws IOException {
BranchProcessRequest request = makeRequest("main", "abc123");
- when(operationsService.getCommitDiff(client, "ws", "repo", "abc123"))
+ when(client.getCommitDiff("ws", "repo", "abc123"))
.thenReturn("diff content");
String diff = fetcher.fetchDiff(request, null, CommitRangeContext.firstAnalysis("abc123"),
- operationsService, client, vcsRepoInfo, null, List.of("abc123"));
+ client, vcsRepoInfo, null, List.of("abc123"));
assertThat(diff).isEqualTo("diff content");
}
@@ -63,26 +61,26 @@ void noPrNumber_shouldUseCommitDiff() throws IOException {
@Test
void withPrNumber_shouldUsePrDiff() throws IOException {
BranchProcessRequest request = makeRequest("main", "abc123");
- when(operationsService.getPullRequestDiff(client, "ws", "repo", "42"))
+ when(client.getPullRequestDiff("ws", "repo", 42L))
.thenReturn("pr diff");
String diff = fetcher.fetchDiff(request, null, CommitRangeContext.firstAnalysis("abc123"),
- operationsService, client, vcsRepoInfo, 42L, List.of("abc123"));
+ client, vcsRepoInfo, 42L, List.of("abc123"));
assertThat(diff).isEqualTo("pr diff");
- verify(operationsService, never()).getCommitDiff(any(), any(), any(), any());
+ verify(client, never()).getCommitDiff(any(), any(), any());
}
@Test
void prDiffEmpty_shouldFallBackToCommitDiff() throws IOException {
BranchProcessRequest request = makeRequest("main", "abc123");
- when(operationsService.getPullRequestDiff(client, "ws", "repo", "42"))
+ when(client.getPullRequestDiff("ws", "repo", 42L))
.thenReturn("");
- when(operationsService.getCommitDiff(client, "ws", "repo", "abc123"))
+ when(client.getCommitDiff("ws", "repo", "abc123"))
.thenReturn("commit diff");
String diff = fetcher.fetchDiff(request, null, CommitRangeContext.firstAnalysis("abc123"),
- operationsService, client, vcsRepoInfo, 42L, List.of("abc123"));
+ client, vcsRepoInfo, 42L, List.of("abc123"));
assertThat(diff).isEqualTo("commit diff");
}
@@ -90,13 +88,13 @@ void prDiffEmpty_shouldFallBackToCommitDiff() throws IOException {
@Test
void prDiffThrows_shouldFallBackToCommitDiff() throws IOException {
BranchProcessRequest request = makeRequest("main", "abc123");
- when(operationsService.getPullRequestDiff(client, "ws", "repo", "42"))
+ when(client.getPullRequestDiff("ws", "repo", 42L))
.thenThrow(new IOException("network error"));
- when(operationsService.getCommitDiff(client, "ws", "repo", "abc123"))
+ when(client.getCommitDiff("ws", "repo", "abc123"))
.thenReturn("commit diff");
String diff = fetcher.fetchDiff(request, null, CommitRangeContext.firstAnalysis("abc123"),
- operationsService, client, vcsRepoInfo, 42L, List.of("abc123"));
+ client, vcsRepoInfo, 42L, List.of("abc123"));
assertThat(diff).isEqualTo("commit diff");
}
@@ -119,11 +117,11 @@ void setUp() {
void withPrNumber_shouldUsePrDiff() throws IOException {
BranchProcessRequest request = makeRequest("main", "new-head");
CommitRangeContext ctx = new CommitRangeContext(List.of("new-head"), "old-head", false);
- when(operationsService.getPullRequestDiff(client, "ws", "repo", "42"))
+ when(client.getPullRequestDiff("ws", "repo", 42L))
.thenReturn("pr diff");
String diff = fetcher.fetchDiff(request, existingBranch, ctx,
- operationsService, client, vcsRepoInfo, 42L, List.of("new-head"));
+ client, vcsRepoInfo, 42L, List.of("new-head"));
assertThat(diff).isEqualTo("pr diff");
}
@@ -132,11 +130,11 @@ void withPrNumber_shouldUsePrDiff() throws IOException {
void noPr_tier1RangeDiffAvailable_shouldUseRangeDiff() throws IOException {
BranchProcessRequest request = makeRequest("main", "new-head");
CommitRangeContext ctx = new CommitRangeContext(List.of("new-head"), "old-head", false);
- when(operationsService.getCommitRangeDiff(client, "ws", "repo", "old-head", "new-head"))
+ when(client.getCommitRangeDiff("ws", "repo", "old-head", "new-head"))
.thenReturn("range diff");
String diff = fetcher.fetchDiff(request, existingBranch, ctx,
- operationsService, client, vcsRepoInfo, null, List.of("new-head"));
+ client, vcsRepoInfo, null, List.of("new-head"));
assertThat(diff).isEqualTo("range diff");
}
@@ -145,13 +143,13 @@ void noPr_tier1RangeDiffAvailable_shouldUseRangeDiff() throws IOException {
void noPr_tier1Empty_tier2Available_shouldUseDeltaDiff() throws IOException {
BranchProcessRequest request = makeRequest("main", "new-head");
CommitRangeContext ctx = new CommitRangeContext(List.of("new-head"), "old-head", false);
- when(operationsService.getCommitRangeDiff(client, "ws", "repo", "old-head", "new-head"))
+ when(client.getCommitRangeDiff("ws", "repo", "old-head", "new-head"))
.thenReturn(""); // blank → skip
- when(operationsService.getCommitRangeDiff(client, "ws", "repo", "prev-success", "new-head"))
+ when(client.getCommitRangeDiff("ws", "repo", "prev-success", "new-head"))
.thenReturn("delta diff");
String diff = fetcher.fetchDiff(request, existingBranch, ctx,
- operationsService, client, vcsRepoInfo, null, List.of("new-head"));
+ client, vcsRepoInfo, null, List.of("new-head"));
assertThat(diff).isEqualTo("delta diff");
}
@@ -162,17 +160,17 @@ void noPr_tier1Null_tier2Null_tier2_5_shouldAggregateCommitDiffs() throws IOExce
CommitRangeContext ctx = new CommitRangeContext(List.of("c1", "c2"), "old-head", false);
// Tier 1: null diffBase would cause skip since old-head != new-head
- when(operationsService.getCommitRangeDiff(client, "ws", "repo", "old-head", "new-head"))
+ when(client.getCommitRangeDiff("ws", "repo", "old-head", "new-head"))
.thenThrow(new IOException("not reachable"));
- when(operationsService.getCommitRangeDiff(client, "ws", "repo", "prev-success", "new-head"))
+ when(client.getCommitRangeDiff("ws", "repo", "prev-success", "new-head"))
.thenThrow(new IOException("not reachable"));
- when(operationsService.getCommitDiff(client, "ws", "repo", "c1"))
+ when(client.getCommitDiff("ws", "repo", "c1"))
.thenReturn("diff1");
- when(operationsService.getCommitDiff(client, "ws", "repo", "c2"))
+ when(client.getCommitDiff("ws", "repo", "c2"))
.thenReturn("diff2");
String diff = fetcher.fetchDiff(request, existingBranch, ctx,
- operationsService, client, vcsRepoInfo, null, List.of("c1", "c2"));
+ client, vcsRepoInfo, null, List.of("c1", "c2"));
assertThat(diff).contains("diff1").contains("diff2");
}
@@ -184,11 +182,11 @@ void allTiersFail_shouldFallBackToSingleCommitDiff() throws IOException {
CommitRangeContext ctx = new CommitRangeContext(List.of("new-head"), null, false);
existingBranch.setLastSuccessfulCommitHash(null);
- when(operationsService.getCommitDiff(client, "ws", "repo", "new-head"))
+ when(client.getCommitDiff("ws", "repo", "new-head"))
.thenReturn("last resort diff");
String diff = fetcher.fetchDiff(request, existingBranch, ctx,
- operationsService, client, vcsRepoInfo, null, List.of("new-head"));
+ client, vcsRepoInfo, null, List.of("new-head"));
assertThat(diff).isEqualTo("last resort diff");
}
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsServiceTest.java
new file mode 100644
index 00000000..a9e84305
--- /dev/null
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsServiceTest.java
@@ -0,0 +1,183 @@
+package org.rostilos.codecrow.analysisengine.service.branch;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.rostilos.codecrow.analysisengine.processor.VcsRepoInfoImpl;
+import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest;
+import org.rostilos.codecrow.analysisengine.service.BranchArchiveService;
+import org.rostilos.codecrow.analysisengine.service.VcsFileRetrievalPolicy;
+import org.rostilos.codecrow.core.model.branch.Branch;
+import org.rostilos.codecrow.core.model.project.Project;
+import org.rostilos.codecrow.core.model.project.config.ProjectConfig;
+import org.rostilos.codecrow.core.model.vcs.VcsConnection;
+import org.rostilos.codecrow.core.model.vcs.VcsRepoInfo;
+import org.rostilos.codecrow.core.persistence.repository.branch.BranchIssueRepository;
+import org.rostilos.codecrow.core.persistence.repository.branch.BranchRepository;
+import org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisIssueRepository;
+import org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisRepository;
+import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository;
+import org.rostilos.codecrow.filecontent.persistence.BranchFileRepository;
+import org.rostilos.codecrow.filecontent.service.FileSnapshotService;
+import org.rostilos.codecrow.vcsclient.VcsClient;
+import org.rostilos.codecrow.vcsclient.VcsClientProvider;
+
+import java.io.IOException;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class BranchFileOperationsServiceTest {
+
+ @Mock private BranchFileRepository branchFileRepository;
+ @Mock private BranchRepository branchRepository;
+ @Mock private ProjectRepository projectRepository;
+ @Mock private BranchIssueRepository branchIssueRepository;
+ @Mock private CodeAnalysisIssueRepository codeAnalysisIssueRepository;
+ @Mock private CodeAnalysisRepository codeAnalysisRepository;
+ @Mock private VcsClientProvider vcsClientProvider;
+ @Mock private FileSnapshotService fileSnapshotService;
+ @Mock private BranchArchiveService branchArchiveService;
+ @Mock private VcsFileRetrievalPolicy fileRetrievalPolicy;
+ @Mock private Project project;
+ @Mock private VcsConnection vcsConnection;
+ @Mock private VcsRepoInfo vcsRepoInfo;
+ @Mock private VcsClient vcsClient;
+
+ private BranchFileOperationsService service;
+
+ @BeforeEach
+ void setUp() {
+ service = new BranchFileOperationsService(
+ branchFileRepository,
+ branchRepository,
+ projectRepository,
+ branchIssueRepository,
+ codeAnalysisIssueRepository,
+ codeAnalysisRepository,
+ vcsClientProvider,
+ fileSnapshotService,
+ branchArchiveService,
+ fileRetrievalPolicy);
+ }
+
+ @Test
+ void preservesArchiveFailureWithoutEnablingLargePerFileFallback() throws Exception {
+ Set requestedFiles = new LinkedHashSet<>();
+ for (int i = 0; i < 26; i++) {
+ requestedFiles.add("src/File" + i + ".java");
+ }
+ VcsRepoInfoImpl repoInfo = new VcsRepoInfoImpl(vcsConnection, "workspace", "repo");
+ when(branchArchiveService.downloadSnapshot(
+ vcsConnection, "workspace", "repo", "commit", requestedFiles))
+ .thenThrow(new IOException("archive unavailable"));
+ when(fileRetrievalPolicy.allowPerFileFallback(26)).thenReturn(false);
+
+ BranchFileOperationsService.BranchFileSnapshot snapshot =
+ service.downloadBranchFileSnapshot(repoInfo, "commit", requestedFiles);
+
+ assertThat(snapshot.archiveAvailable()).isFalse();
+ assertThat(snapshot.allowPerFileFallback()).isFalse();
+ assertThat(snapshot.diagnostic()).contains("archive unavailable");
+ }
+
+ @Test
+ void usesArchivePathPresenceWithoutAnyProviderExistenceCalls() {
+ configureProjectRepository();
+ when(branchRepository.findByProjectIdAndBranchName(1L, "main"))
+ .thenReturn(Optional.empty());
+ when(branchFileRepository.findByProjectIdAndBranchNameAndFilePath(
+ anyLong(), anyString(), anyString())).thenReturn(Optional.empty());
+ when(codeAnalysisIssueRepository.findByProjectIdAndFilePath(anyLong(), anyString()))
+ .thenReturn(List.of());
+
+ BranchFileOperationsService.BranchFileSnapshot snapshot =
+ BranchFileOperationsService.BranchFileSnapshot.fromArchive(
+ new BranchArchiveService.ArchiveSnapshot(
+ Map.of("src/Text.java", "class Text {}"),
+ Set.of("src/Text.java", "assets/logo.png")));
+
+ Set existing = service.updateBranchFiles(
+ new LinkedHashSet<>(List.of(
+ "src/Text.java", "assets/logo.png", "src/Deleted.java")),
+ project,
+ "main",
+ snapshot);
+
+ assertThat(existing).containsExactlyInAnyOrder("src/Text.java", "assets/logo.png");
+ verify(branchFileRepository, org.mockito.Mockito.times(2)).save(any());
+ verifyNoInteractions(vcsClientProvider, vcsClient);
+ }
+
+ @Test
+ void stopsPerFileExistenceChecksAfterFirstProviderFailure() throws Exception {
+ configureProjectRepository();
+ when(branchRepository.findByProjectIdAndBranchName(1L, "main"))
+ .thenReturn(Optional.empty());
+ when(branchFileRepository.findByProjectIdAndBranchNameAndFilePath(
+ anyLong(), anyString(), anyString())).thenReturn(Optional.empty());
+ when(codeAnalysisIssueRepository.findByProjectIdAndFilePath(anyLong(), anyString()))
+ .thenReturn(List.of());
+ when(vcsRepoInfo.getVcsConnection()).thenReturn(vcsConnection);
+ when(vcsRepoInfo.getRepoWorkspace()).thenReturn("workspace");
+ when(vcsRepoInfo.getRepoSlug()).thenReturn("repo");
+ when(vcsClientProvider.getClient(vcsConnection)).thenReturn(vcsClient);
+ when(vcsClient.fileExists(
+ "workspace", "repo", "main", "src/A.java"))
+ .thenThrow(new IOException("Unexpected response 429"));
+
+ BranchFileOperationsService.BranchFileSnapshot snapshot =
+ BranchFileOperationsService.BranchFileSnapshot.unavailable(
+ true, "archive unavailable");
+ Set existing = service.updateBranchFiles(
+ new LinkedHashSet<>(List.of("src/A.java", "src/B.java")),
+ project,
+ "main",
+ snapshot);
+
+ assertThat(existing).containsExactlyInAnyOrder("src/A.java", "src/B.java");
+ assertThat(snapshot.allowContentApiFallback()).isFalse();
+ verify(vcsClient).fileExists(
+ "workspace", "repo", "main", "src/A.java");
+ verify(vcsClient, never()).fileExists(
+ "workspace", "repo", "main", "src/B.java");
+ }
+
+ @Test
+ void selectsTheFirstAnalyzedBranchAsTheProjectDefault() {
+ ProjectConfig config = new ProjectConfig(false, "main");
+ when(project.getConfiguration()).thenReturn(config);
+ when(project.getDefaultBranch()).thenReturn(null);
+ when(branchRepository.save(any(Branch.class)))
+ .thenAnswer(invocation -> invocation.getArgument(0));
+
+ BranchProcessRequest request = new BranchProcessRequest();
+ request.targetBranchName = "feature/first-analysis";
+ request.commitHash = "abc123";
+
+ Branch saved = service.createOrUpdateProjectBranch(project, request, null);
+
+ assertThat(saved.getBranchName()).isEqualTo("feature/first-analysis");
+ verify(project).setDefaultBranch(saved);
+ verify(projectRepository).save(project);
+ }
+
+ private void configureProjectRepository() {
+ when(project.getId()).thenReturn(1L);
+ when(project.getEffectiveVcsRepoInfo()).thenReturn(vcsRepoInfo);
+ }
+}
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationServiceTest.java
index f4d7a6ad..1aad8ec5 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationServiceTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationServiceTest.java
@@ -9,6 +9,7 @@
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.rostilos.codecrow.analysisengine.service.AnalysisLockService;
+import org.rostilos.codecrow.analysisengine.service.BranchArchiveService;
import org.rostilos.codecrow.analysisengine.service.ProjectValidationService;
import org.rostilos.codecrow.analysisengine.service.PullRequestStatusSyncService;
import org.rostilos.codecrow.core.model.branch.Branch;
@@ -36,6 +37,7 @@
import static org.mockito.ArgumentMatchers.anySet;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -151,9 +153,12 @@ void shouldSyncStaleOpenPrStatusesBeforeImportingMergedPrIssues() throws IOExcep
.thenReturn(Set.of("src/App.java"));
Map syncArchive = Map.of("src/App.java", "dangerousCall();");
- when(branchFileOperationsService.downloadBranchArchive(any(), eq("abc123"), eq(Set.of("src/App.java"))))
- .thenReturn(syncArchive);
- when(branchFileOperationsService.updateBranchFiles(eq(Set.of("src/App.java")), eq(project), eq("main"), eq(syncArchive)))
+ var syncSnapshot = archiveSnapshot(syncArchive);
+ when(branchFileOperationsService.downloadBranchFileSnapshot(
+ any(), eq("abc123"), eq(Set.of("src/App.java"))))
+ .thenReturn(syncSnapshot);
+ when(branchFileOperationsService.updateBranchFiles(
+ eq(Set.of("src/App.java")), eq(project), eq("main"), eq(syncSnapshot)))
.thenReturn(Set.of("src/App.java"));
when(branchIssueRepository.countAllByBranchId(10L)).thenReturn(0L, 1L);
@@ -228,9 +233,11 @@ void shouldReportNewlyResolvedCountNotCumulativeResolvedCount() throws IOExcepti
Map archiveContents = Map.of(
"src/App.java", "safeCall();",
"src/Open.java", "dangerousCall();");
- when(branchFileOperationsService.downloadBranchArchive(any(), eq("abc123"), anySet()))
- .thenReturn(archiveContents);
- when(branchFileOperationsService.updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveContents)))
+ var archiveSnapshot = archiveSnapshot(archiveContents);
+ when(branchFileOperationsService.downloadBranchFileSnapshot(any(), eq("abc123"), anySet()))
+ .thenReturn(archiveSnapshot);
+ when(branchFileOperationsService.updateBranchFiles(
+ anySet(), eq(project), eq("main"), eq(archiveSnapshot)))
.thenReturn(Set.of("src/App.java", "src/Open.java"));
Map result = service.fullReconcile(1L, "main", consumer);
@@ -244,7 +251,8 @@ void shouldReportNewlyResolvedCountNotCumulativeResolvedCount() throws IOExcepti
assertThat(result).containsEntry("totalIssues", 1L);
assertThat((String) result.get("message")).contains("1 newly resolved");
verify(branchIssueReconciliationService).reanalyzeCandidateIssues(
- anySet(), anySet(), eq(existingBranch), eq(project), any(), eq(consumer), eq(archiveContents));
+ anySet(), anySet(), eq(existingBranch), eq(project), any(), eq(consumer),
+ eq(archiveContents), isNull(), eq(false));
verify(analysisLockService).releaseLock("lock-key");
}
}
@@ -257,6 +265,12 @@ private void mockVcsInfo() {
when(repoInfo.getRepoSlug()).thenReturn("repo");
}
+ private static BranchFileOperationsService.BranchFileSnapshot archiveSnapshot(
+ Map contents) {
+ return BranchFileOperationsService.BranchFileSnapshot.fromArchive(
+ new BranchArchiveService.ArchiveSnapshot(contents, contents.keySet()));
+ }
+
private Branch branch(String branchName, String commitHash, Long id) {
Branch branch = new Branch();
branch.setBranchName(branchName);
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsReportingServiceDefaultMethodsTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsReportingServiceDefaultMethodsTest.java
index 7a3346b1..8cf2626a 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsReportingServiceDefaultMethodsTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsReportingServiceDefaultMethodsTest.java
@@ -94,7 +94,7 @@ void postCommentReplyWithContextShouldFallBackToBasicReply() {
// Since postCommentReply throws UnsupportedOperationException,
// postCommentReplyWithContext should also throw
assertThatThrownBy(() -> service.postCommentReplyWithContext(
- mockProject, 1L, "parent-id", "content", "author", "original body"))
+ mockProject, 1L, "parent-id", true, "content", "author", "original body"))
.isInstanceOf(UnsupportedOperationException.class)
.hasMessageContaining("postCommentReply not implemented");
}
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsServiceFactoryTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsServiceFactoryTest.java
index 4cce7f20..33a263ae 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsServiceFactoryTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/vcs/VcsServiceFactoryTest.java
@@ -30,12 +30,6 @@ class VcsServiceFactoryTest {
@Mock
private VcsReportingService gitlabReportingService;
- @Mock
- private VcsOperationsService githubOperationsService;
-
- @Mock
- private VcsOperationsService gitlabOperationsService;
-
private VcsServiceFactory factory;
@BeforeEach
@@ -44,14 +38,10 @@ void setUp() {
when(gitlabAiService.getProvider()).thenReturn(EVcsProvider.GITLAB);
when(githubReportingService.getProvider()).thenReturn(EVcsProvider.GITHUB);
when(gitlabReportingService.getProvider()).thenReturn(EVcsProvider.GITLAB);
- when(githubOperationsService.getProvider()).thenReturn(EVcsProvider.GITHUB);
- when(gitlabOperationsService.getProvider()).thenReturn(EVcsProvider.GITLAB);
-
List aiServices = Arrays.asList(githubAiService, gitlabAiService);
List reportingServices = Arrays.asList(githubReportingService, gitlabReportingService);
- List operationsServices = Arrays.asList(githubOperationsService, gitlabOperationsService);
- factory = new VcsServiceFactory(aiServices, reportingServices, operationsServices);
+ factory = new VcsServiceFactory(aiServices, reportingServices);
}
@Test
@@ -96,31 +86,9 @@ void testGetReportingService_UnknownProvider_ThrowsException() {
.hasMessageContaining("No reporting service registered for provider: BITBUCKET_CLOUD");
}
- @Test
- void testGetOperationsService_GitHub_ReturnsGitHubService() {
- VcsOperationsService result = factory.getOperationsService(EVcsProvider.GITHUB);
-
- assertThat(result).isEqualTo(githubOperationsService);
- }
-
- @Test
- void testGetOperationsService_GitLab_ReturnsGitLabService() {
- VcsOperationsService result = factory.getOperationsService(EVcsProvider.GITLAB);
-
- assertThat(result).isEqualTo(gitlabOperationsService);
- }
-
- @Test
- void testGetOperationsService_UnknownProvider_ThrowsException() {
- assertThatThrownBy(() -> factory.getOperationsService(EVcsProvider.BITBUCKET_CLOUD))
- .isInstanceOf(UnsupportedOperationException.class)
- .hasMessageContaining("No operations service registered for provider: BITBUCKET_CLOUD");
- }
-
@Test
void testFactoryWithEmptyLists_ThrowsExceptionForAnyProvider() {
VcsServiceFactory emptyFactory = new VcsServiceFactory(
- Collections.emptyList(),
Collections.emptyList(),
Collections.emptyList()
);
@@ -129,16 +97,13 @@ void testFactoryWithEmptyLists_ThrowsExceptionForAnyProvider() {
.isInstanceOf(UnsupportedOperationException.class);
assertThatThrownBy(() -> emptyFactory.getReportingService(EVcsProvider.GITHUB))
.isInstanceOf(UnsupportedOperationException.class);
- assertThatThrownBy(() -> emptyFactory.getOperationsService(EVcsProvider.GITHUB))
- .isInstanceOf(UnsupportedOperationException.class);
}
@Test
void testFactoryWithOnlyGitHub_GitLabNotAvailable() {
VcsServiceFactory githubOnlyFactory = new VcsServiceFactory(
List.of(githubAiService),
- List.of(githubReportingService),
- List.of(githubOperationsService)
+ List.of(githubReportingService)
);
assertThat(githubOnlyFactory.getAiClientService(EVcsProvider.GITHUB))
@@ -153,10 +118,8 @@ void testFactoryWithOnlyGitHub_GitLabNotAvailable() {
void testAllServicesForSameProvider_ReturnsConsistently() {
VcsAiClientService aiService = factory.getAiClientService(EVcsProvider.GITHUB);
VcsReportingService reportingService = factory.getReportingService(EVcsProvider.GITHUB);
- VcsOperationsService operationsService = factory.getOperationsService(EVcsProvider.GITHUB);
assertThat(aiService.getProvider()).isEqualTo(EVcsProvider.GITHUB);
assertThat(reportingService.getProvider()).isEqualTo(EVcsProvider.GITHUB);
- assertThat(operationsService.getProvider()).isEqualTo(EVcsProvider.GITHUB);
}
}
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/AnalysisLimitEnforcerTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/AnalysisLimitEnforcerTest.java
index a00bfc60..4404f199 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/AnalysisLimitEnforcerTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/AnalysisLimitEnforcerTest.java
@@ -1,13 +1,18 @@
package org.rostilos.codecrow.analysisengine.util;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
import org.junit.jupiter.api.Test;
import org.rostilos.codecrow.analysisengine.exception.DiffTooLargeException;
import org.rostilos.codecrow.analysisengine.exception.DiffTooLargeException.LimitType;
import org.rostilos.codecrow.core.model.project.Project;
import org.rostilos.codecrow.core.model.project.config.AnalysisLimitsConfig;
+import org.rostilos.codecrow.core.model.project.config.AnalysisScopeConfig;
import org.rostilos.codecrow.core.model.project.config.ProjectConfig;
import org.rostilos.codecrow.core.model.workspace.Workspace;
@@ -77,6 +82,58 @@ void reportsPathForOversizedSingleFileDiff() {
});
}
+ @Test
+ void projectExclusionsAreAppliedBeforeEveryLimitMetric() {
+ assertExcludedContentDoesNotConsumeAnyLimit(
+ "vendor/generated.js",
+ new AnalysisScopeConfig(List.of(), List.of("vendor/**")));
+ }
+
+ @Test
+ void defaultGeneratedArtifactExclusionsAreAppliedBeforeEveryLimitMetric() {
+ for (String path : List.of(
+ "public/app.min.js",
+ "public/app.min.mjs",
+ "public/app.min.cjs",
+ "public/styles.min.css",
+ "public/app.js.map",
+ "public/styles.css.map",
+ "public/component.vue.map")) {
+ assertExcludedContentDoesNotConsumeAnyLimit(path, new AnalysisScopeConfig());
+ }
+ }
+
+ private void assertExcludedContentDoesNotConsumeAnyLimit(
+ String excludedPath,
+ AnalysisScopeConfig scope) {
+ String included = section("src/App.java", "small");
+ String excluded = section(excludedPath, "x".repeat(20_000));
+ String rawDiff = excluded + included;
+ long includedBytes = included.getBytes(StandardCharsets.UTF_8).length;
+ int includedTokens = TokenEstimator.estimateTokens(included);
+
+ List limitsByMetric = List.of(
+ new AnalysisLimitsConfig(1, 100_000L, 200_000L, 100_000),
+ new AnalysisLimitsConfig(10, includedBytes + 32, 200_000L, 100_000),
+ new AnalysisLimitsConfig(10, 100_000L, includedBytes + 32, 100_000),
+ new AnalysisLimitsConfig(10, 100_000L, 200_000L, includedTokens + 10));
+
+ for (AnalysisLimitsConfig limits : limitsByMetric) {
+ Project project = projectWithLimits(AnalysisLimitsConfig.empty(), limits);
+ project.getConfiguration().setAnalysisScope(scope);
+
+ assertThatCode(() -> enforcer.enforce(project, 42L, rawDiff))
+ .as("excluded path %s must not consume limits %s", excludedPath, limits)
+ .doesNotThrowAnyException();
+ }
+ }
+
+ private String section(String path, String addedContent) {
+ return "diff --git a/" + path + " b/" + path + "\n"
+ + "--- a/" + path + "\n+++ b/" + path + "\n@@ -1 +1 @@\n-old\n+"
+ + addedContent + "\n";
+ }
+
private Project projectWithLimits(AnalysisLimitsConfig workspaceLimits, AnalysisLimitsConfig projectLimits) {
Workspace workspace = new Workspace("test", "Test", null);
workspace.setAnalysisLimits(workspaceLimits);
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/AnalysisScopeFilterTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/AnalysisScopeFilterTest.java
index 72525133..2d3c2db7 100644
--- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/AnalysisScopeFilterTest.java
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/AnalysisScopeFilterTest.java
@@ -2,7 +2,9 @@
import static org.assertj.core.api.Assertions.assertThat;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import org.junit.jupiter.api.Test;
import org.rostilos.codecrow.core.model.project.Project;
@@ -29,7 +31,72 @@ void returnsOriginalDiffWhenScopeIsEmpty() {
Project project = new Project();
String diff = section("src/App.java");
- assertThat(AnalysisScopeFilter.filterDiff(diff, project)).isSameAs(diff);
+ assertThat(AnalysisScopeFilter.filterDiff(diff, project)).isEqualTo(diff);
+ }
+
+ @Test
+ void excludesMinifiedAssetsAndSourceMapsByDefault() {
+ Project project = new Project();
+ String diff = section("src/App.java")
+ + section("assets/level.map")
+ + section("public/app.min.js")
+ + section("public/module.min.mjs")
+ + section("public/legacy.min.cjs")
+ + section("public/styles.min.css")
+ + section("public/app.js.map")
+ + section("public/styles.css.map")
+ + section("public/module.mjs.map")
+ + section("public/legacy.cjs.map")
+ + section("public/component.jsx.map")
+ + section("public/component.ts.map")
+ + section("public/component.tsx.map")
+ + section("public/styles.scss.map")
+ + section("public/styles.sass.map")
+ + section("public/styles.less.map")
+ + section("public/widget.vue.map")
+ + section("public/widget.svelte.map")
+ + section("public/app.dart.map")
+ + section("public/module.wasm.map");
+
+ String filtered = AnalysisScopeFilter.filterDiff(diff, project);
+
+ assertThat(filtered).contains("src/App.java", "assets/level.map");
+ assertThat(filtered).doesNotContain(
+ "app.min.js",
+ "module.min.mjs",
+ "legacy.min.cjs",
+ "styles.min.css",
+ "app.js.map",
+ "styles.css.map",
+ "module.mjs.map",
+ "legacy.cjs.map",
+ "component.jsx.map",
+ "component.ts.map",
+ "component.tsx.map",
+ "styles.scss.map",
+ "styles.sass.map",
+ "styles.less.map",
+ "widget.vue.map",
+ "widget.svelte.map",
+ "app.dart.map",
+ "module.wasm.map");
+ }
+
+ @Test
+ void appliesDefaultAndProjectExclusionsToFileSets() {
+ ProjectConfig config = new ProjectConfig();
+ config.setAnalysisScope(new AnalysisScopeConfig(List.of(), List.of("vendor/**")));
+ Project project = new Project();
+ project.setConfiguration(config);
+ Set paths = new LinkedHashSet<>(List.of(
+ "src/App.java",
+ "vendor/library.js",
+ "public/app.min.js",
+ "public/app.js.map"));
+
+ AnalysisScopeFilter.retainIncluded(paths, project);
+
+ assertThat(paths).containsExactly("src/App.java");
}
private String section(String path) {
diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/TextFileEligibilityTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/TextFileEligibilityTest.java
new file mode 100644
index 00000000..a0ee4955
--- /dev/null
+++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/TextFileEligibilityTest.java
@@ -0,0 +1,33 @@
+package org.rostilos.codecrow.analysisengine.util;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class TextFileEligibilityTest {
+
+ @Test
+ void excludesKnownBinaryAssetsCaseInsensitively() {
+ assertThat(TextFileEligibility.isTextCandidate("images/Box.jpg")).isFalse();
+ assertThat(TextFileEligibility.isTextCandidate("images/Label.JPEG")).isFalse();
+ assertThat(TextFileEligibility.isTextCandidate("fonts/storefront.woff2")).isFalse();
+ assertThat(TextFileEligibility.isTextCandidate("artifacts/module.jar")).isFalse();
+ }
+
+ @Test
+ void retainsFrameworkAndSourceTextWithoutAnAllowlist() {
+ assertThat(TextFileEligibility.isTextCandidate("app/etc/di.xml")).isTrue();
+ assertThat(TextFileEligibility.isTextCandidate("templates/item.phtml")).isTrue();
+ assertThat(TextFileEligibility.isTextCandidate("web/images/logo.svg")).isTrue();
+ assertThat(TextFileEligibility.isTextCandidate(".htaccess")).isTrue();
+ }
+
+ @Test
+ void classifiesProviderContentUsingBinaryAndSizeBounds() {
+ assertThat(TextFileEligibility.isBoundedTextContent("plain text")).isTrue();
+ assertThat(TextFileEligibility.isBoundedTextContent("binary\0payload")).isFalse();
+ assertThat(TextFileEligibility.isBoundedTextContent(
+ "x".repeat((int) TextFileEligibility.MAX_TEXT_CONTENT_BYTES + 1)))
+ .isFalse();
+ }
+}
diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/gitlab/GitLabDTO.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/gitlab/GitLabDTO.java
index 106c77a2..840d7458 100644
--- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/gitlab/GitLabDTO.java
+++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/gitlab/GitLabDTO.java
@@ -17,7 +17,8 @@ public record GitLabDTO(
Boolean hasAccessToken,
LocalDateTime updatedAt,
EVcsConnectionType connectionType,
- String repositoryPath
+ String repositoryPath,
+ String baseUrl
) {
public static GitLabDTO fromVcsConnection(VcsConnection vcsConnection) {
if (vcsConnection.getProviderType() != EVcsProvider.GITLAB) {
@@ -34,7 +35,8 @@ public static GitLabDTO fromVcsConnection(VcsConnection vcsConnection) {
vcsConnection.getAccessToken() != null && !vcsConnection.getAccessToken().isBlank(),
vcsConnection.getUpdatedAt(),
vcsConnection.getConnectionType(),
- vcsConnection.getRepositoryPath()
+ vcsConnection.getRepositoryPath(),
+ effectiveBaseUrl(vcsConnection)
);
}
@@ -48,7 +50,14 @@ public static GitLabDTO fromVcsConnection(VcsConnection vcsConnection) {
config.accessToken() != null && !config.accessToken().isBlank(),
vcsConnection.getUpdatedAt(),
vcsConnection.getConnectionType(),
- vcsConnection.getRepositoryPath()
+ vcsConnection.getRepositoryPath(),
+ config.effectiveBaseUrl()
);
}
+
+ private static String effectiveBaseUrl(VcsConnection connection) {
+ return connection.getConfiguration() instanceof GitLabConfig config
+ ? config.effectiveBaseUrl()
+ : GitLabConfig.DEFAULT_BASE_URL;
+ }
}
diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java
index 1d952511..20d54bfa 100644
--- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java
+++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java
@@ -10,6 +10,7 @@
import org.rostilos.codecrow.core.model.project.config.TaskManagementConfig;
import org.rostilos.codecrow.core.model.vcs.VcsConnection;
import org.rostilos.codecrow.core.model.vcs.VcsRepoInfo;
+import org.rostilos.codecrow.core.model.vcs.config.gitlab.GitLabConfig;
import java.util.List;
@@ -21,6 +22,7 @@ public record ProjectDTO(
Long vcsConnectionId,
String vcsConnectionType,
String vcsProvider,
+ String vcsBaseUrl,
String projectVcsWorkspace,
String projectVcsRepoSlug,
Long aiConnectionId,
@@ -46,6 +48,7 @@ public static ProjectDTO fromProject(Project project) {
Long vcsConnectionId = null;
String vcsConnectionType = null;
String vcsProvider = null;
+ String vcsBaseUrl = null;
String vcsWorkspace = null;
String repoSlug = null;
@@ -61,6 +64,11 @@ public static ProjectDTO fromProject(Project project) {
}
if (conn.getProviderType() != null) {
vcsProvider = conn.getProviderType().name();
+ if (conn.getProviderType() == org.rostilos.codecrow.core.model.vcs.EVcsProvider.GITLAB) {
+ vcsBaseUrl = conn.getConfiguration() instanceof GitLabConfig gitLabConfig
+ ? gitLabConfig.effectiveBaseUrl()
+ : GitLabConfig.DEFAULT_BASE_URL;
+ }
}
}
vcsWorkspace = vcsInfo.getRepoWorkspace();
@@ -165,6 +173,7 @@ public static ProjectDTO fromProject(Project project) {
vcsConnectionId,
vcsConnectionType,
vcsProvider,
+ vcsBaseUrl,
vcsWorkspace,
repoSlug,
aiConnectionId,
diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/TaskImplementationEvidence.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/TaskImplementationEvidence.java
new file mode 100644
index 00000000..4c68c795
--- /dev/null
+++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/TaskImplementationEvidence.java
@@ -0,0 +1,203 @@
+package org.rostilos.codecrow.core.model.codeanalysis;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.FetchType;
+import jakarta.persistence.GeneratedValue;
+import jakarta.persistence.GenerationType;
+import jakarta.persistence.Id;
+import jakarta.persistence.JoinColumn;
+import jakarta.persistence.ManyToOne;
+import jakarta.persistence.Table;
+import jakarta.persistence.UniqueConstraint;
+import org.rostilos.codecrow.core.model.project.Project;
+
+import java.time.OffsetDateTime;
+
+/**
+ * Positive, deterministic implementation evidence associated with one PR
+ * analysis and task. This is internal analysis state and is never rendered in
+ * a VCS or task-management comment.
+ */
+@Entity
+@Table(
+ name = "task_implementation_evidence",
+ uniqueConstraints = @UniqueConstraint(
+ name = "uq_task_implementation_evidence_analysis_fingerprint",
+ columnNames = {"analysis_id", "content_fingerprint"}
+ )
+)
+public class TaskImplementationEvidence {
+
+ @Id
+ @GeneratedValue(strategy = GenerationType.IDENTITY)
+ @Column(nullable = false, updatable = false)
+ private Long id;
+
+ @ManyToOne(fetch = FetchType.LAZY, optional = false)
+ @JoinColumn(name = "analysis_id", nullable = false)
+ private CodeAnalysis analysis;
+
+ @ManyToOne(fetch = FetchType.LAZY, optional = false)
+ @JoinColumn(name = "project_id", nullable = false)
+ private Project project;
+
+ @Column(name = "task_id", nullable = false, length = 128)
+ private String taskId;
+
+ @Column(name = "pr_number", nullable = false)
+ private Long prNumber;
+
+ @Column(name = "commit_hash", nullable = false, length = 64)
+ private String commitHash;
+
+ @Column(name = "source", nullable = false, length = 40)
+ private String source;
+
+ @Column(name = "evidence_ref", nullable = false, length = 32)
+ private String evidenceRef;
+
+ @Column(name = "file_path", nullable = false, length = 2048)
+ private String filePath;
+
+ @Column(name = "hunk_id", nullable = false, length = 160)
+ private String hunkId;
+
+ @Column(name = "line_start", nullable = false)
+ private Integer lineStart;
+
+ @Column(name = "line_end", nullable = false)
+ private Integer lineEnd;
+
+ @Column(name = "excerpt", nullable = false, columnDefinition = "TEXT")
+ private String excerpt;
+
+ @Column(name = "full_evidence_complete", nullable = false)
+ private boolean fullEvidenceComplete;
+
+ @Column(name = "content_fingerprint", nullable = false, length = 64)
+ private String contentFingerprint;
+
+ @Column(name = "created_at", nullable = false, updatable = false)
+ private OffsetDateTime createdAt = OffsetDateTime.now();
+
+ public Long getId() {
+ return id;
+ }
+
+ public CodeAnalysis getAnalysis() {
+ return analysis;
+ }
+
+ public void setAnalysis(CodeAnalysis analysis) {
+ this.analysis = analysis;
+ }
+
+ public Project getProject() {
+ return project;
+ }
+
+ public void setProject(Project project) {
+ this.project = project;
+ }
+
+ public String getTaskId() {
+ return taskId;
+ }
+
+ public void setTaskId(String taskId) {
+ this.taskId = taskId;
+ }
+
+ public Long getPrNumber() {
+ return prNumber;
+ }
+
+ public void setPrNumber(Long prNumber) {
+ this.prNumber = prNumber;
+ }
+
+ public String getCommitHash() {
+ return commitHash;
+ }
+
+ public void setCommitHash(String commitHash) {
+ this.commitHash = commitHash;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
+ public void setSource(String source) {
+ this.source = source;
+ }
+
+ public String getEvidenceRef() {
+ return evidenceRef;
+ }
+
+ public void setEvidenceRef(String evidenceRef) {
+ this.evidenceRef = evidenceRef;
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+
+ public void setFilePath(String filePath) {
+ this.filePath = filePath;
+ }
+
+ public String getHunkId() {
+ return hunkId;
+ }
+
+ public void setHunkId(String hunkId) {
+ this.hunkId = hunkId;
+ }
+
+ public Integer getLineStart() {
+ return lineStart;
+ }
+
+ public void setLineStart(Integer lineStart) {
+ this.lineStart = lineStart;
+ }
+
+ public Integer getLineEnd() {
+ return lineEnd;
+ }
+
+ public void setLineEnd(Integer lineEnd) {
+ this.lineEnd = lineEnd;
+ }
+
+ public String getExcerpt() {
+ return excerpt;
+ }
+
+ public void setExcerpt(String excerpt) {
+ this.excerpt = excerpt;
+ }
+
+ public boolean isFullEvidenceComplete() {
+ return fullEvidenceComplete;
+ }
+
+ public void setFullEvidenceComplete(boolean fullEvidenceComplete) {
+ this.fullEvidenceComplete = fullEvidenceComplete;
+ }
+
+ public String getContentFingerprint() {
+ return contentFingerprint;
+ }
+
+ public void setContentFingerprint(String contentFingerprint) {
+ this.contentFingerprint = contentFingerprint;
+ }
+
+ public OffsetDateTime getCreatedAt() {
+ return createdAt;
+ }
+}
diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/vcs/config/gitlab/GitLabConfig.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/vcs/config/gitlab/GitLabConfig.java
index 08d53f31..a5656334 100644
--- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/vcs/config/gitlab/GitLabConfig.java
+++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/vcs/config/gitlab/GitLabConfig.java
@@ -16,6 +16,7 @@ public record GitLabConfig(
List allowedRepos,
String baseUrl // For self-hosted GitLab instances (e.g., "https://gitlab.mycompany.com")
) implements VcsConnectionConfig {
+ public static final String DEFAULT_BASE_URL = "https://gitlab.com";
/**
* Constructor for backward compatibility (without baseUrl).
@@ -28,6 +29,21 @@ public GitLabConfig(String accessToken, String groupId, List allowedRepo
* Returns the effective base URL (defaults to gitlab.com if not specified).
*/
public String effectiveBaseUrl() {
- return (baseUrl != null && !baseUrl.isBlank()) ? baseUrl : "https://gitlab.com";
+ return normalizeBaseUrl(baseUrl);
+ }
+
+ /**
+ * Normalize a persisted or process-provided GitLab instance root.
+ */
+ public static String normalizeBaseUrl(String baseUrl) {
+ if (baseUrl == null || baseUrl.isBlank()) {
+ return DEFAULT_BASE_URL;
+ }
+
+ String normalized = baseUrl.trim().replaceAll("/+$", "");
+ if (normalized.endsWith("/api/v4")) {
+ normalized = normalized.substring(0, normalized.length() - "/api/v4".length());
+ }
+ return normalized.isBlank() ? DEFAULT_BASE_URL : normalized;
}
}
diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java
index 6ae6d47a..f96cc670 100644
--- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java
+++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java
@@ -19,6 +19,8 @@ public interface BranchRepository extends JpaRepository {
List findByProjectId(Long projectId);
+ Optional findFirstByProjectIdOrderByIdAsc(Long projectId);
+
void deleteByProjectId(Long projectId);
@Query("SELECT b FROM Branch b LEFT JOIN FETCH b.issues WHERE b.id = :id")
diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/codeanalysis/TaskImplementationEvidenceRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/codeanalysis/TaskImplementationEvidenceRepository.java
new file mode 100644
index 00000000..3202a50f
--- /dev/null
+++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/codeanalysis/TaskImplementationEvidenceRepository.java
@@ -0,0 +1,37 @@
+package org.rostilos.codecrow.core.persistence.repository.codeanalysis;
+
+import org.rostilos.codecrow.core.model.codeanalysis.TaskImplementationEvidence;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
+import org.springframework.stereotype.Repository;
+
+import java.util.Collection;
+import java.util.List;
+
+@Repository
+public interface TaskImplementationEvidenceRepository
+ extends JpaRepository {
+
+ @Query("SELECT e.contentFingerprint FROM TaskImplementationEvidence e " +
+ "WHERE e.analysis.id = :analysisId")
+ List findFingerprintsByAnalysisId(@Param("analysisId") Long analysisId);
+
+ @Query("SELECT e FROM TaskImplementationEvidence e " +
+ "WHERE e.analysis.id IN :analysisIds " +
+ "ORDER BY e.analysis.id ASC, e.id ASC")
+ List findByAnalysisIds(
+ @Param("analysisIds") Collection analysisIds);
+
+ @Query("SELECT e FROM TaskImplementationEvidence e " +
+ "WHERE e.project.id = :projectId " +
+ "AND e.taskId = :taskId " +
+ "AND (:excludedPrNumber IS NULL OR e.prNumber <> :excludedPrNumber) " +
+ "ORDER BY e.createdAt DESC, e.id DESC")
+ List findForTaskHistory(
+ @Param("projectId") Long projectId,
+ @Param("taskId") String taskId,
+ @Param("excludedPrNumber") Long excludedPrNumber,
+ Pageable pageable);
+}
diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java
index 8418eafe..8712ae2e 100644
--- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java
+++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java
@@ -168,6 +168,48 @@ boolean existsNewerBranchAnalysisJob(
"AND j.startedAt < :threshold")
List findStuckJobs(@Param("threshold") OffsetDateTime threshold);
+ @Query("SELECT j FROM Job j WHERE " +
+ "j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " +
+ "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " +
+ "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED) " +
+ "AND j.updatedAt < :threshold ORDER BY j.createdAt ASC")
+ List findRecoverableWebhookJobs(
+ @Param("threshold") OffsetDateTime threshold,
+ Pageable pageable);
+
+ @Query("SELECT j FROM Job j WHERE " +
+ "j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " +
+ "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.RUNNING " +
+ "AND j.updatedAt < :threshold ORDER BY j.updatedAt ASC")
+ List findAbandonedRunningWebhookJobs(
+ @Param("threshold") OffsetDateTime threshold,
+ Pageable pageable);
+
+ @Modifying(clearAutomatically = true, flushAutomatically = true)
+ @Query("UPDATE Job j SET j.status = org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " +
+ "j.updatedAt = :claimedAt WHERE j.id = :jobId " +
+ "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " +
+ "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED) " +
+ "AND j.updatedAt < :threshold")
+ int claimRecoverableWebhookJob(
+ @Param("jobId") Long jobId,
+ @Param("threshold") OffsetDateTime threshold,
+ @Param("claimedAt") OffsetDateTime claimedAt);
+
+ @Modifying
+ @Query("UPDATE Job j SET j.updatedAt = :activityAt WHERE j.id = :jobId")
+ int touchJob(@Param("jobId") Long jobId, @Param("activityAt") OffsetDateTime activityAt);
+
+ @Modifying(clearAutomatically = true, flushAutomatically = true)
+ @Query("UPDATE Job j SET j.status = org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " +
+ "j.updatedAt = :claimedAt WHERE j.id = :jobId " +
+ "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.RUNNING " +
+ "AND j.updatedAt < :threshold")
+ int claimAbandonedRunningWebhookJob(
+ @Param("jobId") Long jobId,
+ @Param("threshold") OffsetDateTime threshold,
+ @Param("claimedAt") OffsetDateTime claimedAt);
+
@Query("SELECT COUNT(j) FROM Job j WHERE j.project.id = :projectId " +
"AND j.status NOT IN (org.rostilos.codecrow.core.model.job.JobStatus.COMPLETED, " +
"org.rostilos.codecrow.core.model.job.JobStatus.FAILED, " +
diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java
index b975fad2..23190f90 100644
--- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java
+++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java
@@ -31,6 +31,7 @@
public class JobService {
private static final Logger log = LoggerFactory.getLogger(JobService.class);
+ private static final String WEBHOOK_DISPATCH_STEP = "webhook_dispatch";
private final JobRepository jobRepository;
private final JobLogRepository jobLogRepository;
@@ -124,6 +125,45 @@ public Job createBranchAnalysisJob(
return job;
}
+ /**
+ * Persist the executable webhook work before handing it to an async worker.
+ * A process restart can then replay the same normalized payload.
+ */
+ @Transactional
+ public Job queueWebhookJob(Job job, String webhookPayload) {
+ Job persisted = jobRepository.findById(job.getId()).orElseThrow(
+ () -> new IllegalArgumentException("Job not found: " + job.getId()));
+ persisted.setStatus(JobStatus.QUEUED);
+ persisted.setCurrentStep("Waiting for pipeline worker");
+ persisted = jobRepository.save(persisted);
+ addLog(persisted, JobLogLevel.INFO, "queued",
+ "Webhook work persisted and queued for pipeline processing");
+ addLog(persisted, JobLogLevel.DEBUG, WEBHOOK_DISPATCH_STEP,
+ "Durable webhook dispatch payload",
+ Map.of("payload", webhookPayload));
+ return persisted;
+ }
+
+ public Optional findWebhookDispatchPayload(Long jobId) {
+ List entries =
+ jobLogRepository.findByJobIdAndStep(jobId, WEBHOOK_DISPATCH_STEP);
+ if (entries.isEmpty()) {
+ return Optional.empty();
+ }
+ String metadata = entries.get(entries.size() - 1).getMetadata();
+ if (metadata == null || metadata.isBlank()) {
+ return Optional.empty();
+ }
+ try {
+ return Optional.ofNullable(
+ objectMapper.readTree(metadata).path("payload").textValue());
+ } catch (JsonProcessingException malformedMetadata) {
+ log.warn("Could not read durable webhook payload for job {}: {}",
+ jobId, malformedMetadata.getMessage());
+ return Optional.empty();
+ }
+ }
+
/**
* Create a new job for branch reconciliation (full reconcile triggered from UI).
*/
@@ -433,6 +473,7 @@ public JobLog addLog(Job job, JobLogLevel level, String step, String message) {
logEntry.setSequenceNumber(jobLogRepository.getNextSequenceNumber(job.getId()));
logEntry = jobLogRepository.save(logEntry);
+ touchJobActivity(job);
// Notify SSE subscribers
notifySubscribers(job.getExternalId(), logEntry);
@@ -456,6 +497,7 @@ public JobLog addLogInNewTransaction(Job job, JobLogLevel level, String step, St
logEntry.setSequenceNumber(jobLogRepository.getNextSequenceNumber(job.getId()));
logEntry = jobLogRepository.save(logEntry);
+ touchJobActivity(job);
// Notify SSE subscribers
notifySubscribers(job.getExternalId(), logEntry);
@@ -485,6 +527,7 @@ public JobLog addLog(Job job, JobLogLevel level, String step, String message, Ma
}
logEntry = jobLogRepository.save(logEntry);
+ touchJobActivity(job);
notifySubscribers(job.getExternalId(), logEntry);
return logEntry;
@@ -666,4 +709,38 @@ public List findAndFailStuckJobs(OffsetDateTime threshold, String reason) {
}
return stuckJobs;
}
+
+ public List findRecoverableWebhookJobs(OffsetDateTime threshold, int limit) {
+ return jobRepository.findRecoverableWebhookJobs(
+ threshold, PageRequest.of(0, Math.max(1, limit)));
+ }
+
+ public List findAbandonedRunningWebhookJobs(OffsetDateTime threshold, int limit) {
+ return jobRepository.findAbandonedRunningWebhookJobs(
+ threshold, PageRequest.of(0, Math.max(1, limit)));
+ }
+
+ @Transactional
+ public boolean claimRecoverableWebhookJob(Long jobId, OffsetDateTime threshold) {
+ return jobRepository.claimRecoverableWebhookJob(
+ jobId, threshold, OffsetDateTime.now()) == 1;
+ }
+
+ @Transactional
+ public boolean claimAbandonedRunningWebhookJob(Long jobId, OffsetDateTime threshold) {
+ return jobRepository.claimAbandonedRunningWebhookJob(
+ jobId, threshold, OffsetDateTime.now()) == 1;
+ }
+
+ private void touchJobActivity(Job job) {
+ if (job == null || job.getId() == null) {
+ return;
+ }
+ try {
+ jobRepository.touchJob(job.getId(), OffsetDateTime.now());
+ } catch (Exception touchError) {
+ log.debug("Could not update activity timestamp for job {}: {}",
+ job.getExternalId(), touchError.getMessage());
+ }
+ }
}
diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/TaskImplementationEvidenceService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/TaskImplementationEvidenceService.java
new file mode 100644
index 00000000..3b91ef95
--- /dev/null
+++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/TaskImplementationEvidenceService.java
@@ -0,0 +1,336 @@
+package org.rostilos.codecrow.core.service;
+
+import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis;
+import org.rostilos.codecrow.core.model.codeanalysis.TaskImplementationEvidence;
+import org.rostilos.codecrow.core.persistence.repository.codeanalysis.TaskImplementationEvidenceRepository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Owns validation, persistence, retrieval, and cache-copy behavior for bounded
+ * deterministic task implementation evidence.
+ */
+@Service
+public class TaskImplementationEvidenceService {
+
+ public static final String SOURCE_DETERMINISTIC_PR_LEDGER = "DETERMINISTIC_PR_LEDGER";
+
+ private static final Logger log =
+ LoggerFactory.getLogger(TaskImplementationEvidenceService.class);
+ private static final int MAX_ITEMS = 8;
+ private static final int MAX_TOTAL_EXCERPT_CHARS = 2_400;
+ private static final int MAX_EXCERPT_CHARS = 420;
+ private static final int MAX_FILE_PATH_CHARS = 2_048;
+ private static final int MAX_HUNK_ID_CHARS = 160;
+ private static final int MAX_EVIDENCE_REF_CHARS = 32;
+
+ private final TaskImplementationEvidenceRepository repository;
+
+ public TaskImplementationEvidenceService(
+ TaskImplementationEvidenceRepository repository) {
+ this.repository = repository;
+ }
+
+ /**
+ * Validate and persist the structured {@code taskEvidence} analysis output.
+ * Existing fingerprints make webhook retries idempotent.
+ */
+ @Transactional
+ public PersistenceResult persistFromAnalysisResponse(
+ CodeAnalysis analysis,
+ Object rawPayload) {
+ if (analysis == null || analysis.getId() == null || rawPayload == null) {
+ return PersistenceResult.empty();
+ }
+ if (analysis.getTaskId() == null || analysis.getTaskId().isBlank()
+ || analysis.getPrNumber() == null
+ || analysis.getCommitHash() == null
+ || analysis.getCommitHash().isBlank()) {
+ return PersistenceResult.empty();
+ }
+ if (!(rawPayload instanceof Map payload)) {
+ return new PersistenceResult(0, 1, 0);
+ }
+
+ String taskKey = normalizedString(payload.get("taskKey"), 128);
+ String source = normalizedString(payload.get("source"), 40);
+ if (!analysis.getTaskId().equals(taskKey)
+ || !SOURCE_DETERMINISTIC_PR_LEDGER.equals(source)) {
+ log.warn(
+ "Skipping task evidence for analysis {}: task/source mismatch",
+ analysis.getId());
+ return new PersistenceResult(0, 1, 0);
+ }
+
+ Object rawItems = payload.get("items");
+ if (!(rawItems instanceof Collection items) || items.isEmpty()) {
+ return PersistenceResult.empty();
+ }
+
+ boolean fullEvidenceComplete = booleanValue(payload.get("fullEvidenceComplete"));
+ Set existingFingerprints = new HashSet<>(
+ repository.findFingerprintsByAnalysisId(analysis.getId()));
+ List accepted = new ArrayList<>();
+ int rejected = 0;
+ int duplicate = 0;
+ int usedExcerptChars = 0;
+
+ for (Object rawItem : items) {
+ if (accepted.size() >= MAX_ITEMS) {
+ rejected++;
+ continue;
+ }
+ if (!(rawItem instanceof Map item)) {
+ rejected++;
+ continue;
+ }
+
+ String evidenceRef = normalizedString(
+ item.get("evidenceRef"), MAX_EVIDENCE_REF_CHARS);
+ String filePath = normalizedPath(item.get("filePath"));
+ String hunkId = normalizedString(item.get("hunkId"), MAX_HUNK_ID_CHARS);
+ Integer lineStart = positiveInteger(item.get("lineStart"));
+ Integer lineEnd = positiveInteger(item.get("lineEnd"));
+ String excerpt = normalizedExcerpt(item.get("excerpt"));
+
+ if (evidenceRef == null || filePath == null || hunkId == null
+ || lineStart == null || lineEnd == null
+ || lineEnd < lineStart || excerpt == null
+ || usedExcerptChars + excerpt.length() > MAX_TOTAL_EXCERPT_CHARS) {
+ rejected++;
+ continue;
+ }
+
+ String fingerprint = fingerprint(
+ source, evidenceRef, filePath, hunkId,
+ lineStart, lineEnd, excerpt);
+ if (!existingFingerprints.add(fingerprint)) {
+ duplicate++;
+ continue;
+ }
+
+ TaskImplementationEvidence evidence = new TaskImplementationEvidence();
+ evidence.setAnalysis(analysis);
+ evidence.setProject(analysis.getProject());
+ evidence.setTaskId(taskKey);
+ evidence.setPrNumber(analysis.getPrNumber());
+ evidence.setCommitHash(analysis.getCommitHash());
+ evidence.setSource(source);
+ evidence.setEvidenceRef(evidenceRef);
+ evidence.setFilePath(filePath);
+ evidence.setHunkId(hunkId);
+ evidence.setLineStart(lineStart);
+ evidence.setLineEnd(lineEnd);
+ evidence.setExcerpt(excerpt);
+ evidence.setFullEvidenceComplete(fullEvidenceComplete);
+ evidence.setContentFingerprint(fingerprint);
+ accepted.add(evidence);
+ usedExcerptChars += excerpt.length();
+ }
+
+ if (!accepted.isEmpty()) {
+ repository.saveAll(accepted);
+ }
+ return new PersistenceResult(accepted.size(), rejected, duplicate);
+ }
+
+ /**
+ * Load evidence for a bounded set of analyses. Failure is deliberately
+ * observable and fail-open because this is optional prompt enrichment.
+ */
+ @Transactional(readOnly = true)
+ public List findForAnalyses(
+ Collection analysisIds) {
+ if (analysisIds == null || analysisIds.isEmpty()) {
+ return List.of();
+ }
+ List distinctIds = analysisIds.stream()
+ .filter(id -> id != null)
+ .distinct()
+ .toList();
+ if (distinctIds.isEmpty()) {
+ return List.of();
+ }
+ try {
+ return repository.findByAnalysisIds(distinctIds);
+ } catch (RuntimeException e) {
+ log.warn("Task evidence lookup failed; continuing without persisted evidence: {}",
+ e.getMessage());
+ return List.of();
+ }
+ }
+
+ /**
+ * Load the newest distinct evidence receipts across prior PR analyses for a
+ * task. Evidence remains available when the latest analysis iteration could
+ * not rebuild optional full-PR enrichment.
+ */
+ @Transactional(readOnly = true)
+ public List findForTaskHistory(
+ Long projectId,
+ String taskId,
+ Long excludedPrNumber,
+ int maxRecords) {
+ if (projectId == null || taskId == null || taskId.isBlank()
+ || maxRecords <= 0) {
+ return List.of();
+ }
+ int boundedMax = Math.min(maxRecords, 40);
+ try {
+ List candidates =
+ repository.findForTaskHistory(
+ projectId,
+ taskId,
+ excludedPrNumber,
+ PageRequest.of(0, Math.min(160, boundedMax * 4)));
+ Map distinct =
+ new LinkedHashMap<>();
+ for (TaskImplementationEvidence evidence : candidates) {
+ String key = evidence.getPrNumber()
+ + ":" + evidence.getContentFingerprint();
+ distinct.putIfAbsent(key, evidence);
+ if (distinct.size() >= boundedMax) {
+ break;
+ }
+ }
+ return List.copyOf(distinct.values());
+ } catch (RuntimeException e) {
+ log.warn(
+ "Task evidence history lookup failed for project {} task {}; "
+ + "continuing without persisted evidence: {}",
+ projectId,
+ taskId,
+ e.getMessage());
+ return List.of();
+ }
+ }
+
+ /**
+ * Copy immutable evidence when a cached analysis is cloned for another PR.
+ */
+ @Transactional
+ public PersistenceResult copyForAnalysis(
+ CodeAnalysis sourceAnalysis,
+ CodeAnalysis targetAnalysis) {
+ if (sourceAnalysis == null || targetAnalysis == null
+ || sourceAnalysis.getId() == null || targetAnalysis.getId() == null
+ || targetAnalysis.getTaskId() == null
+ || targetAnalysis.getTaskId().isBlank()) {
+ return PersistenceResult.empty();
+ }
+ List sourceEvidence =
+ repository.findByAnalysisIds(List.of(sourceAnalysis.getId()));
+ if (sourceEvidence.isEmpty()) {
+ return PersistenceResult.empty();
+ }
+
+ Map payload = Map.of(
+ "taskKey", targetAnalysis.getTaskId(),
+ "source", SOURCE_DETERMINISTIC_PR_LEDGER,
+ "fullEvidenceComplete", sourceEvidence.stream()
+ .allMatch(TaskImplementationEvidence::isFullEvidenceComplete),
+ "items", sourceEvidence.stream().map(evidence -> Map.of(
+ "evidenceRef", evidence.getEvidenceRef(),
+ "filePath", evidence.getFilePath(),
+ "hunkId", evidence.getHunkId(),
+ "lineStart", evidence.getLineStart(),
+ "lineEnd", evidence.getLineEnd(),
+ "excerpt", evidence.getExcerpt()
+ )).toList()
+ );
+ return persistFromAnalysisResponse(targetAnalysis, payload);
+ }
+
+ private String normalizedPath(Object value) {
+ String path = normalizedString(value, MAX_FILE_PATH_CHARS);
+ if (path == null) {
+ return null;
+ }
+ return path.replace('\\', '/');
+ }
+
+ private String normalizedExcerpt(Object value) {
+ String excerpt = normalizedString(value, MAX_EXCERPT_CHARS);
+ if (excerpt == null) {
+ return null;
+ }
+ return excerpt
+ .replace('\u0000', ' ')
+ .replaceAll("[\\t\\x0B\\f\\r ]+", " ")
+ .trim();
+ }
+
+ private String normalizedString(Object value, int maxChars) {
+ if (!(value instanceof String text)) {
+ return null;
+ }
+ String normalized = text.replace('\u0000', ' ').trim();
+ if (normalized.isEmpty() || normalized.length() > maxChars) {
+ return null;
+ }
+ return normalized;
+ }
+
+ private Integer positiveInteger(Object value) {
+ if (value instanceof Number number) {
+ try {
+ int result = new BigDecimal(number.toString()).intValueExact();
+ return result > 0 ? result : null;
+ } catch (ArithmeticException | NumberFormatException ignored) {
+ return null;
+ }
+ }
+ return null;
+ }
+
+ private boolean booleanValue(Object value) {
+ return value instanceof Boolean bool && bool;
+ }
+
+ private String fingerprint(
+ String source,
+ String evidenceRef,
+ String filePath,
+ String hunkId,
+ int lineStart,
+ int lineEnd,
+ String excerpt) {
+ String canonical = String.join("\n",
+ source.toUpperCase(Locale.ROOT),
+ evidenceRef,
+ filePath,
+ hunkId,
+ Integer.toString(lineStart),
+ Integer.toString(lineEnd),
+ excerpt);
+ try {
+ byte[] digest = MessageDigest.getInstance("SHA-256")
+ .digest(canonical.getBytes(StandardCharsets.UTF_8));
+ return java.util.HexFormat.of().formatHex(digest);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("SHA-256 is not available", e);
+ }
+ }
+
+ public record PersistenceResult(int persisted, int rejected, int duplicate) {
+ public static PersistenceResult empty() {
+ return new PersistenceResult(0, 0, 0);
+ }
+ }
+}
diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/R__task_implementation_evidence.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/R__task_implementation_evidence.sql
new file mode 100644
index 00000000..5f5c5d1d
--- /dev/null
+++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/R__task_implementation_evidence.sql
@@ -0,0 +1,44 @@
+CREATE TABLE IF NOT EXISTS task_implementation_evidence (
+ id BIGSERIAL PRIMARY KEY,
+ analysis_id BIGINT NOT NULL,
+ project_id BIGINT NOT NULL,
+ task_id VARCHAR(128) NOT NULL,
+ pr_number BIGINT NOT NULL,
+ commit_hash VARCHAR(64) NOT NULL,
+ source VARCHAR(40) NOT NULL,
+ evidence_ref VARCHAR(32) NOT NULL,
+ file_path VARCHAR(2048) NOT NULL,
+ hunk_id VARCHAR(160) NOT NULL,
+ line_start INTEGER NOT NULL,
+ line_end INTEGER NOT NULL,
+ excerpt TEXT NOT NULL,
+ full_evidence_complete BOOLEAN NOT NULL DEFAULT FALSE,
+ content_fingerprint VARCHAR(64) NOT NULL,
+ created_at TIMESTAMPTZ(6) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+ CONSTRAINT fk_task_implementation_evidence_analysis
+ FOREIGN KEY (analysis_id) REFERENCES code_analysis (id)
+ ON DELETE CASCADE,
+ CONSTRAINT fk_task_implementation_evidence_project
+ FOREIGN KEY (project_id) REFERENCES project (id)
+ ON DELETE CASCADE,
+ CONSTRAINT uq_task_implementation_evidence_analysis_fingerprint
+ UNIQUE (analysis_id, content_fingerprint),
+ CONSTRAINT ck_task_implementation_evidence_lines
+ CHECK (line_start > 0 AND line_end >= line_start),
+ CONSTRAINT ck_task_implementation_evidence_source
+ CHECK (source = 'DETERMINISTIC_PR_LEDGER'),
+ CONSTRAINT ck_task_implementation_evidence_fingerprint
+ CHECK (content_fingerprint ~ '^[0-9a-f]{64}$')
+);
+
+CREATE INDEX IF NOT EXISTS idx_task_implementation_evidence_task_history
+ ON task_implementation_evidence (
+ project_id,
+ task_id,
+ created_at DESC,
+ id DESC
+ );
+
+COMMENT ON TABLE task_implementation_evidence IS
+ 'Bounded positive implementation evidence for task-aware review context; never comment content.';
diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/gitlab/GitLabDTOTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/gitlab/GitLabDTOTest.java
index dc4aa714..d9a55839 100644
--- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/gitlab/GitLabDTOTest.java
+++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/gitlab/GitLabDTOTest.java
@@ -32,7 +32,8 @@ void recordShouldStoreAllFieldsCorrectly() {
true,
now,
EVcsConnectionType.PERSONAL_TOKEN,
- "/path/to/repo"
+ "/path/to/repo",
+ "https://gitlab.example.com"
);
assertThat(dto.id()).isEqualTo(1L);
@@ -44,6 +45,7 @@ void recordShouldStoreAllFieldsCorrectly() {
assertThat(dto.updatedAt()).isEqualTo(now);
assertThat(dto.connectionType()).isEqualTo(EVcsConnectionType.PERSONAL_TOKEN);
assertThat(dto.repositoryPath()).isEqualTo("/path/to/repo");
+ assertThat(dto.baseUrl()).isEqualTo("https://gitlab.example.com");
}
@Test
@@ -51,10 +53,12 @@ void recordShouldStoreAllFieldsCorrectly() {
void recordsWithSameValuesShouldBeEqual() {
LocalDateTime now = LocalDateTime.now();
GitLabDTO dto1 = new GitLabDTO(
- 1L, "name", "group", 5, EVcsSetupStatus.CONNECTED, true, now, EVcsConnectionType.APP, "/path"
+ 1L, "name", "group", 5, EVcsSetupStatus.CONNECTED, true, now,
+ EVcsConnectionType.APP, "/path", "https://gitlab.com"
);
GitLabDTO dto2 = new GitLabDTO(
- 1L, "name", "group", 5, EVcsSetupStatus.CONNECTED, true, now, EVcsConnectionType.APP, "/path"
+ 1L, "name", "group", 5, EVcsSetupStatus.CONNECTED, true, now,
+ EVcsConnectionType.APP, "/path", "https://gitlab.com"
);
assertThat(dto1).isEqualTo(dto2);
diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/project/ProjectDTOTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/project/ProjectDTOTest.java
index f4a33a6f..b11be02e 100644
--- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/project/ProjectDTOTest.java
+++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/project/ProjectDTOTest.java
@@ -18,6 +18,7 @@
import org.rostilos.codecrow.core.model.vcs.EVcsProvider;
import org.rostilos.codecrow.core.model.vcs.VcsConnection;
import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding;
+import org.rostilos.codecrow.core.model.vcs.config.gitlab.GitLabConfig;
import java.lang.reflect.Field;
import java.util.List;
@@ -44,7 +45,7 @@ void shouldCreateWithAllFields() {
ProjectDTO dto = new ProjectDTO(
1L, "Test Project", "Description", true,
10L, "OAUTH_MANUAL", "BITBUCKET_CLOUD",
- "workspace", "repo-slug",
+ null, "workspace", "repo-slug",
20L, "namespace", "main", "main",
100L, stats, ragConfig,
true, false, "WEBHOOK",
@@ -80,7 +81,7 @@ void shouldCreateWithAllFields() {
void shouldCreateWithNullOptionalFields() {
ProjectDTO dto = new ProjectDTO(
1L, "Test", null, true,
- null, null, null, null, null,
+ null, null, null, null, null, null,
null, null, null, null, null, null,
null, null, null, null, null, null, null, null, null, null, null, null, null);
@@ -129,6 +130,33 @@ void shouldConvertProjectWithVcsBinding() {
assertThat(dto.webhooksConfigured()).isTrue();
}
+ @Test
+ @DisplayName("should expose the normalized GitLab instance URL")
+ void shouldExposeGitLabInstanceUrl() {
+ Project project = createProjectWithVcsBinding();
+ VcsConnection connection = project.getVcsRepoBinding().getVcsConnection();
+ connection.setProviderType(EVcsProvider.GITLAB);
+ connection.setConfiguration(new GitLabConfig(
+ null, "test-workspace", null, "https://gitlab.example.com/api/v4/"));
+
+ ProjectDTO dto = ProjectDTO.fromProject(project);
+
+ assertThat(dto.vcsBaseUrl()).isEqualTo("https://gitlab.example.com");
+ }
+
+ @Test
+ @DisplayName("should default legacy GitLab connections to GitLab.com")
+ void shouldDefaultLegacyGitLabInstanceUrl() {
+ Project project = createProjectWithVcsBinding();
+ VcsConnection connection = project.getVcsRepoBinding().getVcsConnection();
+ connection.setProviderType(EVcsProvider.GITLAB);
+ connection.setConfiguration(null);
+
+ ProjectDTO dto = ProjectDTO.fromProject(project);
+
+ assertThat(dto.vcsBaseUrl()).isEqualTo(GitLabConfig.DEFAULT_BASE_URL);
+ }
+
@Test
@DisplayName("should convert project with AI binding")
void shouldConvertProjectWithAiBinding() {
diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/vcs/config/gitlab/GitLabConfigTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/vcs/config/gitlab/GitLabConfigTest.java
index d2b37f10..658d75f0 100644
--- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/vcs/config/gitlab/GitLabConfigTest.java
+++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/vcs/config/gitlab/GitLabConfigTest.java
@@ -37,6 +37,13 @@ void testEffectiveBaseUrl_WithCustomUrl() {
assertThat(config.effectiveBaseUrl()).isEqualTo("https://gitlab.mycompany.com");
}
+ @Test
+ void testEffectiveBaseUrl_NormalizesApiSuffixAndTrailingSlash() {
+ GitLabConfig config = new GitLabConfig(
+ "token", "group-id", null, " https://gitlab.mycompany.com/api/v4/ ");
+ assertThat(config.effectiveBaseUrl()).isEqualTo("https://gitlab.mycompany.com");
+ }
+
@Test
void testEffectiveBaseUrl_WithNullBaseUrl() {
GitLabConfig config = new GitLabConfig("token", "group-id", null, null);
diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java
index f0dc5f00..1c717b3b 100644
--- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java
+++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java
@@ -126,6 +126,31 @@ void shouldRetainMergedPrNumber() {
}
}
+ @Nested
+ @DisplayName("queueWebhookJob()")
+ class QueueWebhookJobTests {
+
+ @Test
+ void persistsExecutablePayloadBeforeAsyncDispatch() {
+ Job job = new Job();
+ setField(job, "id", 104L);
+ when(jobRepository.findById(104L)).thenReturn(Optional.of(job));
+ when(jobRepository.save(job)).thenReturn(job);
+ when(jobLogRepository.save(any(JobLog.class)))
+ .thenAnswer(inv -> inv.getArgument(0));
+
+ Job queued = jobService.queueWebhookJob(job, "{\"eventType\":\"push\"}");
+
+ assertThat(queued.getStatus()).isEqualTo(JobStatus.QUEUED);
+ assertThat(queued.getCurrentStep()).isEqualTo("Waiting for pipeline worker");
+ verify(jobLogRepository).save(argThat(log ->
+ "queued".equals(log.getStep())));
+ verify(jobLogRepository).save(argThat(log ->
+ "webhook_dispatch".equals(log.getStep())
+ && log.getMetadata().contains("\\\"eventType\\\":\\\"push\\\"")));
+ }
+ }
+
@Nested
@DisplayName("createRagIndexJob()")
class CreateRagIndexJobTests {
diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/TaskImplementationEvidenceServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/TaskImplementationEvidenceServiceTest.java
new file mode 100644
index 00000000..9c6a1d24
--- /dev/null
+++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/TaskImplementationEvidenceServiceTest.java
@@ -0,0 +1,228 @@
+package org.rostilos.codecrow.core.service;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis;
+import org.rostilos.codecrow.core.model.codeanalysis.TaskImplementationEvidence;
+import org.rostilos.codecrow.core.model.project.Project;
+import org.rostilos.codecrow.core.persistence.repository.codeanalysis.TaskImplementationEvidenceRepository;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+@DisplayName("TaskImplementationEvidenceService")
+class TaskImplementationEvidenceServiceTest {
+
+ @Mock
+ private TaskImplementationEvidenceRepository repository;
+
+ private TaskImplementationEvidenceService service;
+
+ @BeforeEach
+ void setUp() {
+ service = new TaskImplementationEvidenceService(repository);
+ }
+
+ @Test
+ @DisplayName("persists bounded structured evidence without comment parsing")
+ void persistsStructuredEvidence() {
+ CodeAnalysis analysis = analysis(101L, "SHOP-42");
+ when(repository.findFingerprintsByAnalysisId(101L)).thenReturn(List.of());
+
+ TaskImplementationEvidenceService.PersistenceResult result =
+ service.persistFromAnalysisResponse(analysis, payload(
+ "SHOP-42",
+ Map.of(
+ "evidenceRef", "PRF001",
+ "filePath", "src\\Checkout\\NewRelicTracker.php",
+ "hunkId", "hunk-1",
+ "lineStart", 18,
+ "lineEnd", 21,
+ "excerpt", "recordCustomEvent('CouponApplied', $payload);"
+ )));
+
+ assertThat(result.persisted()).isEqualTo(1);
+ assertThat(result.rejected()).isZero();
+ ArgumentCaptor> captor =
+ ArgumentCaptor.forClass(List.class);
+ verify(repository).saveAll(captor.capture());
+ TaskImplementationEvidence saved = captor.getValue().get(0);
+ assertThat(saved.getAnalysis()).isSameAs(analysis);
+ assertThat(saved.getProject()).isSameAs(analysis.getProject());
+ assertThat(saved.getTaskId()).isEqualTo("SHOP-42");
+ assertThat(saved.getFilePath()).isEqualTo("src/Checkout/NewRelicTracker.php");
+ assertThat(saved.getLineStart()).isEqualTo(18);
+ assertThat(saved.getLineEnd()).isEqualTo(21);
+ assertThat(saved.getContentFingerprint()).matches("[0-9a-f]{64}");
+ }
+
+ @Test
+ @DisplayName("rejects evidence associated with a different task")
+ void rejectsTaskMismatch() {
+ CodeAnalysis analysis = analysis(101L, "SHOP-42");
+
+ TaskImplementationEvidenceService.PersistenceResult result =
+ service.persistFromAnalysisResponse(
+ analysis,
+ payload("OTHER-9", validItem()));
+
+ assertThat(result.persisted()).isZero();
+ assertThat(result.rejected()).isEqualTo(1);
+ verifyNoInteractions(repository);
+ }
+
+ @Test
+ @DisplayName("skips duplicate evidence on webhook retry")
+ void skipsDuplicateEvidence() {
+ CodeAnalysis analysis = analysis(101L, "SHOP-42");
+ when(repository.findFingerprintsByAnalysisId(101L)).thenReturn(List.of());
+ service.persistFromAnalysisResponse(
+ analysis,
+ payload("SHOP-42", validItem()));
+
+ ArgumentCaptor> captor =
+ ArgumentCaptor.forClass(List.class);
+ verify(repository).saveAll(captor.capture());
+ String fingerprint = captor.getValue().get(0).getContentFingerprint();
+
+ when(repository.findFingerprintsByAnalysisId(101L))
+ .thenReturn(List.of(fingerprint));
+ TaskImplementationEvidenceService.PersistenceResult retry =
+ service.persistFromAnalysisResponse(
+ analysis,
+ payload("SHOP-42", validItem()));
+
+ assertThat(retry.persisted()).isZero();
+ assertThat(retry.duplicate()).isEqualTo(1);
+ verify(repository).saveAll(anyList());
+ }
+
+ @Test
+ @DisplayName("fails open when optional history evidence lookup is unavailable")
+ void lookupFailureFailsOpen() {
+ when(repository.findForTaskHistory(
+ 7L, "SHOP-42", 99L, PageRequest.of(0, 160)))
+ .thenThrow(new RuntimeException("database temporarily unavailable"));
+
+ assertThat(service.findForTaskHistory(
+ 7L, "SHOP-42", 99L, 40)).isEmpty();
+ }
+
+ @Test
+ @DisplayName("keeps earlier task evidence when newer analysis repeats the same receipt")
+ void deduplicatesTaskHistoryAcrossAnalysisIterations() {
+ TaskImplementationEvidence newest = storedEvidence(42L, "fingerprint-a");
+ TaskImplementationEvidence repeated = storedEvidence(42L, "fingerprint-a");
+ TaskImplementationEvidence earlier = storedEvidence(42L, "fingerprint-b");
+ when(repository.findForTaskHistory(
+ 7L, "SHOP-42", 99L, PageRequest.of(0, 160)))
+ .thenReturn(List.of(newest, repeated, earlier));
+
+ List result =
+ service.findForTaskHistory(7L, "SHOP-42", 99L, 40);
+
+ assertThat(result).containsExactly(newest, earlier);
+ }
+
+ @Test
+ @DisplayName("does not persist malformed evidence rows")
+ void rejectsMalformedRows() {
+ CodeAnalysis analysis = analysis(101L, "SHOP-42");
+ when(repository.findFingerprintsByAnalysisId(101L)).thenReturn(List.of());
+
+ TaskImplementationEvidenceService.PersistenceResult result =
+ service.persistFromAnalysisResponse(
+ analysis,
+ payload("SHOP-42", Map.of(
+ "evidenceRef", "PRF001",
+ "filePath", "src/File.php",
+ "hunkId", "hunk-1",
+ "lineStart", 21,
+ "lineEnd", 18,
+ "excerpt", "invalid range"
+ )));
+
+ assertThat(result.persisted()).isZero();
+ assertThat(result.rejected()).isEqualTo(1);
+ verify(repository, never()).saveAll(anyList());
+ }
+
+ @Test
+ @DisplayName("rejects fractional evidence line numbers instead of truncating them")
+ void rejectsFractionalLineNumbers() {
+ CodeAnalysis analysis = analysis(101L, "SHOP-42");
+ when(repository.findFingerprintsByAnalysisId(101L)).thenReturn(List.of());
+
+ TaskImplementationEvidenceService.PersistenceResult result =
+ service.persistFromAnalysisResponse(
+ analysis,
+ payload("SHOP-42", Map.of(
+ "evidenceRef", "PRF001",
+ "filePath", "src/File.php",
+ "hunkId", "hunk-1",
+ "lineStart", 18.5,
+ "lineEnd", 21,
+ "excerpt", "fractional start line"
+ )));
+
+ assertThat(result.persisted()).isZero();
+ assertThat(result.rejected()).isEqualTo(1);
+ verify(repository, never()).saveAll(anyList());
+ }
+
+ private Map payload(
+ String taskKey,
+ Map item) {
+ return Map.of(
+ "taskKey", taskKey,
+ "source", TaskImplementationEvidenceService.SOURCE_DETERMINISTIC_PR_LEDGER,
+ "fullEvidenceComplete", true,
+ "items", List.of(item));
+ }
+
+ private Map validItem() {
+ return Map.of(
+ "evidenceRef", "PRF001",
+ "filePath", "src/Checkout/NewRelicTracker.php",
+ "hunkId", "hunk-1",
+ "lineStart", 18,
+ "lineEnd", 21,
+ "excerpt", "recordCustomEvent('CouponApplied', $payload);");
+ }
+
+ private CodeAnalysis analysis(Long id, String taskId) {
+ Project project = new Project();
+ ReflectionTestUtils.setField(project, "id", 7L);
+ CodeAnalysis analysis = new CodeAnalysis();
+ ReflectionTestUtils.setField(analysis, "id", id);
+ analysis.setProject(project);
+ analysis.setTaskId(taskId);
+ analysis.setPrNumber(99L);
+ analysis.setCommitHash("0123456789012345678901234567890123456789");
+ return analysis;
+ }
+
+ private TaskImplementationEvidence storedEvidence(
+ Long prNumber,
+ String fingerprint) {
+ TaskImplementationEvidence evidence = new TaskImplementationEvidence();
+ evidence.setPrNumber(prNumber);
+ evidence.setContentFingerprint(fingerprint);
+ return evidence;
+ }
+}
diff --git a/java-ecosystem/libs/queue/src/main/java/org/rostilos/codecrow/queue/RedisQueueService.java b/java-ecosystem/libs/queue/src/main/java/org/rostilos/codecrow/queue/RedisQueueService.java
index 3e1075f0..4019fea6 100644
--- a/java-ecosystem/libs/queue/src/main/java/org/rostilos/codecrow/queue/RedisQueueService.java
+++ b/java-ecosystem/libs/queue/src/main/java/org/rostilos/codecrow/queue/RedisQueueService.java
@@ -27,6 +27,14 @@ public boolean listContains(String queueKey, String payload) {
return redisTemplate.opsForList().indexOf(queueKey, payload) != null;
}
+ public void removeFromList(String queueKey, String payload) {
+ redisTemplate.opsForList().remove(queueKey, 1, payload);
+ }
+
+ public boolean hasKey(String key) {
+ return Boolean.TRUE.equals(redisTemplate.hasKey(key));
+ }
+
public void setExpiry(String key, long timeoutMinutes) {
redisTemplate.expire(key, timeoutMinutes, TimeUnit.MINUTES);
}
diff --git a/java-ecosystem/libs/queue/src/test/java/org/rostilos/codecrow/queue/RedisQueueServiceTest.java b/java-ecosystem/libs/queue/src/test/java/org/rostilos/codecrow/queue/RedisQueueServiceTest.java
index f571ff03..be2603c3 100644
--- a/java-ecosystem/libs/queue/src/test/java/org/rostilos/codecrow/queue/RedisQueueServiceTest.java
+++ b/java-ecosystem/libs/queue/src/test/java/org/rostilos/codecrow/queue/RedisQueueServiceTest.java
@@ -102,6 +102,20 @@ void listContains_shouldReturnFalseWhenPayloadIsNotQueued() {
assertThat(service.listContains("q", "payload")).isFalse();
}
+ @Test
+ void removeFromList_shouldDeleteOnlyTheExactQueuedPayload() {
+ service.removeFromList("q", "payload");
+
+ verify(listOperations).remove("q", 1, "payload");
+ }
+
+ @Test
+ void hasKey_shouldRequireAnExistingHeartbeatKey() {
+ when(redisTemplate.hasKey("heartbeat")).thenReturn(true);
+
+ assertThat(service.hasKey("heartbeat")).isTrue();
+ }
+
// ── setExpiry ────────────────────────────────────────────────────────
@Test
diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java
index 353b12c6..631134a6 100644
--- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java
+++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java
@@ -1,6 +1,8 @@
package org.rostilos.codecrow.ragengine.service;
import org.rostilos.codecrow.analysisengine.service.BranchArchiveService;
+import org.rostilos.codecrow.analysisengine.service.VcsFileRetrievalPolicy;
+import org.rostilos.codecrow.analysisengine.util.TextFileEligibility;
import org.rostilos.codecrow.core.model.project.Project;
import org.rostilos.codecrow.core.model.project.config.ProjectConfig;
import org.rostilos.codecrow.core.model.vcs.VcsConnection;
@@ -18,6 +20,8 @@
import java.nio.file.attribute.PosixFilePermissions;
import java.util.*;
import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
@Service
public class IncrementalRagUpdateService {
@@ -27,6 +31,7 @@ public class IncrementalRagUpdateService {
private final RagPipelineClient ragPipelineClient;
private final RagIndexTrackingService ragIndexTrackingService;
private final BranchArchiveService branchArchiveService;
+ private final VcsFileRetrievalPolicy fileRetrievalPolicy;
@Value("${codecrow.rag.api.enabled:true}")
private boolean ragApiEnabled;
@@ -34,9 +39,6 @@ public class IncrementalRagUpdateService {
@Value("${codecrow.rag.parallel.requests:10}")
private int parallelRequests;
- @Value("${codecrow.rag.incremental.archive-file-threshold:25}")
- private int archiveFileThreshold;
-
@Value("${codecrow.rag.incremental.max-attempts:3}")
private int ragApiMaxAttempts;
@@ -47,11 +49,13 @@ public IncrementalRagUpdateService(
VcsClientProvider vcsClientProvider,
RagPipelineClient ragPipelineClient,
RagIndexTrackingService ragIndexTrackingService,
- BranchArchiveService branchArchiveService) {
+ BranchArchiveService branchArchiveService,
+ VcsFileRetrievalPolicy fileRetrievalPolicy) {
this.vcsClientProvider = vcsClientProvider;
this.ragPipelineClient = ragPipelineClient;
this.ragIndexTrackingService = ragIndexTrackingService;
this.branchArchiveService = branchArchiveService;
+ this.fileRetrievalPolicy = fileRetrievalPolicy;
}
public boolean shouldPerformIncrementalUpdate(Project project) {
@@ -91,10 +95,15 @@ public Map performIncrementalUpdate(
Set addedFiles,
Set modifiedFiles,
Set deletedFiles) throws IOException {
- int addedOrModifiedSize = addedFiles.size() + modifiedFiles.size();
+ Set skippedNonTextFiles = new LinkedHashSet<>();
+ Set indexableAddedFiles = filterTextCandidates(addedFiles, skippedNonTextFiles);
+ Set indexableModifiedFiles = filterTextCandidates(modifiedFiles, skippedNonTextFiles);
+ int addedOrModifiedSize = indexableAddedFiles.size() + indexableModifiedFiles.size();
log.info(
- "Starting incremental RAG update for project {} branch {}: {} files to update ({} added), {} to delete",
- project.getName(), branch, addedOrModifiedSize, addedFiles.size(), deletedFiles.size());
+ "Starting incremental RAG update for project {} branch {}: {} text files to update "
+ + "({} added), {} to delete, {} non-text files skipped",
+ project.getName(), branch, addedOrModifiedSize, indexableAddedFiles.size(),
+ deletedFiles.size(), skippedNonTextFiles.size());
String projectWorkspace = project.getWorkspace().getName();
String projectNamespace = project.getNamespace();
@@ -102,11 +111,13 @@ public Map performIncrementalUpdate(
Map result = new HashMap<>();
result.put("branch", branch);
result.put("commitHash", commitHash);
+ result.put("skippedFiles", skippedNonTextFiles.size());
Set addedOrModifiedFiles = new LinkedHashSet<>();
- addedOrModifiedFiles.addAll(addedFiles);
- addedOrModifiedFiles.addAll(modifiedFiles);
- List orderedAddedOrModifiedFiles = sortedList(addedOrModifiedFiles);
+ addedOrModifiedFiles.addAll(indexableAddedFiles);
+ addedOrModifiedFiles.addAll(indexableModifiedFiles);
+ List orderedAddedOrModifiedFiles =
+ new ArrayList<>(sortedList(addedOrModifiedFiles));
List orderedDeletedFiles = sortedList(deletedFiles);
if (orderedAddedOrModifiedFiles.isEmpty() && orderedDeletedFiles.isEmpty()) {
result.put("status", "completed");
@@ -121,30 +132,72 @@ public Map performIncrementalUpdate(
tempDir = Files.createTempDirectory("codecrow-rag-incremental-",
PosixFilePermissions.asFileAttribute(
PosixFilePermissions.fromString("rwxrwxrwx")));
- int effectiveArchiveFileThreshold = Math.max(0, archiveFileThreshold);
- boolean useArchive = orderedAddedOrModifiedFiles.size() > effectiveArchiveFileThreshold;
+ int effectiveArchiveFileThreshold = fileRetrievalPolicy.archiveFileThreshold();
+ boolean useArchive =
+ fileRetrievalPolicy.shouldUseArchive(orderedAddedOrModifiedFiles.size());
Set fetchedFilePaths;
+ Set presentFilePaths = Collections.emptySet();
+ String fileFetchMode;
if (useArchive) {
log.info("Using one repository archive at revision {} for {} incremental RAG files "
+ "(threshold: {})",
revision, orderedAddedOrModifiedFiles.size(), effectiveArchiveFileThreshold);
- fetchedFilePaths = branchArchiveService.downloadAndExtractFilesToDirectory(
+ BranchArchiveService.ArchiveDirectorySnapshot archiveSnapshot =
+ branchArchiveService.downloadAndExtractSnapshotToDirectory(
vcsConnection,
workspaceSlug,
repoSlug,
revision,
new LinkedHashSet<>(orderedAddedOrModifiedFiles),
tempDir);
+ fetchedFilePaths = archiveSnapshot.extractedFiles();
+ presentFilePaths = archiveSnapshot.presentFiles();
+ fileFetchMode = "archive";
} else {
log.info("Using per-file VCS retrieval for {} incremental RAG files (threshold: {})",
orderedAddedOrModifiedFiles.size(), effectiveArchiveFileThreshold);
- fetchedFilePaths = fetchFilesToTempDir(
+ PerFileFetchResult perFileResult = fetchFilesToTempDir(
vcsConnection,
workspaceSlug,
repoSlug,
revision,
orderedAddedOrModifiedFiles,
tempDir);
+ if (perFileResult.rateLimited()) {
+ log.warn("Per-file VCS retrieval hit a provider rate limit after {} of {} files; "
+ + "switching the batch to one repository archive",
+ perFileResult.fetchedFiles().size(),
+ orderedAddedOrModifiedFiles.size());
+ BranchArchiveService.ArchiveDirectorySnapshot archiveSnapshot =
+ branchArchiveService.downloadAndExtractSnapshotToDirectory(
+ vcsConnection,
+ workspaceSlug,
+ repoSlug,
+ revision,
+ new LinkedHashSet<>(orderedAddedOrModifiedFiles),
+ tempDir);
+ fetchedFilePaths = archiveSnapshot.extractedFiles();
+ presentFilePaths = archiveSnapshot.presentFiles();
+ fileFetchMode = "archive-after-rate-limit";
+ } else {
+ fetchedFilePaths = perFileResult.fetchedFiles();
+ skippedNonTextFiles.addAll(perFileResult.skippedNonIndexableFiles());
+ orderedAddedOrModifiedFiles.removeAll(
+ perFileResult.skippedNonIndexableFiles());
+ fileFetchMode = "per-file";
+ }
+ }
+ Set presentButNotText = orderedAddedOrModifiedFiles.stream()
+ .filter(presentFilePaths::contains)
+ .filter(path -> !fetchedFilePaths.contains(path))
+ .collect(Collectors.toCollection(LinkedHashSet::new));
+ if (!presentButNotText.isEmpty()) {
+ skippedNonTextFiles.addAll(presentButNotText);
+ orderedAddedOrModifiedFiles.removeAll(presentButNotText);
+ log.warn("Skipping {} changed files that are present at revision {} but "
+ + "cannot produce text chunks: {}",
+ presentButNotText.size(), revision,
+ String.join(", ", sortedList(presentButNotText)));
}
List missingFiles = orderedAddedOrModifiedFiles.stream()
.filter(path -> !fetchedFilePaths.contains(path))
@@ -157,15 +210,20 @@ public Map performIncrementalUpdate(
}
repoBase = tempDir.toString();
result.put("updatedFiles", orderedAddedOrModifiedFiles.size());
- result.put("addedFilesCount", addedFiles.size());
- result.put("fileFetchMode", useArchive ? "archive" : "per-file");
+ int appliedAddedFiles = (int) orderedAddedOrModifiedFiles.stream()
+ .filter(indexableAddedFiles::contains)
+ .count();
+ result.put("addedFilesCount", appliedAddedFiles);
+ result.put("skippedFiles", skippedNonTextFiles.size());
+ result.put("fileFetchMode", fileFetchMode);
}
String changeSetRepoBase = repoBase;
+ List filesToUpdate = List.copyOf(orderedAddedOrModifiedFiles);
Map updateResult = executeWithRetry(
"apply incremental RAG change set",
() -> ragPipelineClient.applyChanges(
- orderedAddedOrModifiedFiles,
+ filesToUpdate,
orderedDeletedFiles,
changeSetRepoBase,
projectWorkspace,
@@ -175,9 +233,10 @@ public Map performIncrementalUpdate(
result.put("deletedFiles", orderedDeletedFiles.size());
result.putAll(updateResult);
log.info(
- "Applied one incremental RAG change set: {} updated, {} deleted",
- orderedAddedOrModifiedFiles.size(),
- orderedDeletedFiles.size());
+ "Applied one incremental RAG change set: {} updated, {} deleted, {} skipped",
+ filesToUpdate.size(),
+ orderedDeletedFiles.size(),
+ skippedNonTextFiles.size());
} finally {
if (tempDir != null) {
deleteDirectory(tempDir.toFile());
@@ -248,7 +307,24 @@ private List sortedList(Collection values) {
.toList();
}
- private Set fetchFilesToTempDir(
+ private Set filterTextCandidates(
+ Collection paths,
+ Set skippedNonTextFiles) {
+ Set eligible = new LinkedHashSet<>();
+ if (paths == null) {
+ return eligible;
+ }
+ for (String path : paths) {
+ if (TextFileEligibility.isTextCandidate(path)) {
+ eligible.add(path);
+ } else if (path != null && !path.isBlank()) {
+ skippedNonTextFiles.add(path);
+ }
+ }
+ return eligible;
+ }
+
+ private PerFileFetchResult fetchFilesToTempDir(
VcsConnection vcsConnection,
String workspaceSlug,
String repoSlug,
@@ -259,15 +335,27 @@ private Set fetchFilesToTempDir(
int workerCount = Math.min(Math.max(1, parallelRequests), filePaths.size());
ExecutorService executor = Executors.newFixedThreadPool(workerCount);
+ AtomicBoolean rateLimited = new AtomicBoolean(false);
+ Set skippedNonIndexableFiles = ConcurrentHashMap.newKeySet();
try {
List