diff --git a/deployment/.env.sample b/deployment/.env.sample index e580a2cc..2d14fa68 100644 --- a/deployment/.env.sample +++ b/deployment/.env.sample @@ -29,6 +29,11 @@ PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES=20971520 # The same bound prevents branch reconciliation from falling back to an # unbounded sequence of provider file requests when an archive is unavailable. VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD=25 +# Repository archives are rejected before indexing if any uncompressed member, +# total uncompressed content, or file count crosses these per-job bounds. +RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES=268435456 +RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES=4294967296 +RAG_ARCHIVE_MAX_ENTRIES=500000 # Testing-only: route normal analyses through full prompt capture instead of an # LLM. Empty project IDs means every project; a comma-separated list scopes it. diff --git a/deployment/config/inference-orchestrator/.env.sample b/deployment/config/inference-orchestrator/.env.sample index 6f4524f5..95f6ef62 100644 --- a/deployment/config/inference-orchestrator/.env.sample +++ b/deployment/config/inference-orchestrator/.env.sample @@ -11,7 +11,7 @@ SERVICE_SECRET=change-me-to-a-random-secret # AI_CLIENT_HOST=0.0.0.0 # AI_CLIENT_PORT=8000 # REDIS_URL=redis://localhost:6379/1 -# MAX_CONCURRENT_REVIEWS=4 +# MAX_CONCURRENT_REVIEWS=20 # ANALYSIS_QUEUE_HEARTBEAT_SECONDS=30 # ANALYSIS_CONSUMER_HEARTBEAT_SECONDS=5 # MAX_CONCURRENT_COMMANDS=10 diff --git a/deployment/config/java-shared/application.properties.sample b/deployment/config/java-shared/application.properties.sample index 2cc383e7..c788fb31 100644 --- a/deployment/config/java-shared/application.properties.sample +++ b/deployment/config/java-shared/application.properties.sample @@ -181,6 +181,17 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF #codecrow.rag.api.timeout.read=120 # RAG indexing timeout - 4 hours for large repositories #codecrow.rag.api.timeout.indexing=14400 +# Per-project fan-out for an explicit Refresh all. Two independent branch +# snapshots (for example main and develop) run in parallel without allowing a +# project with many retained branches to occupy all service slots. +#codecrow.rag.branch-build.parallelism=2 +# Dedicated service-wide capacity for full branch snapshot builds. This pool is +# separate from PR, branch-analysis, webhook and inference executors. Size it to +# the available RAG replicas and memory; it does not cap ordinary analyses. +#codecrow.rag.branch-build.global-parallelism=4 +# Repair interval for readable Qdrant current-branch aliases of active generations. +#codecrow.rag.operator-alias.reconcile-interval-ms=300000 +#codecrow.rag.operator-alias.reconcile-initial-delay-ms=15000 # Shared VCS acquisition threshold for incremental RAG and reconciliation # fallback. The legacy codecrow.rag.incremental.archive-file-threshold property # remains a fallback when this property is not set. @@ -202,6 +213,11 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF # Lock cleanup interval - how often to clean up expired locks (in milliseconds) #analysis.lock.cleanup.interval.ms=300000 +# Durable webhook execution. The executor has no in-memory queue: accepted work +# beyond max capacity stays QUEUED in PostgreSQL and is recovered later. +#webhook.executor.core-pool-size=8 +#webhook.executor.max-pool-size=20 + # Hard PR-wide spending limits are supplied to pipeline-agent through deployment/.env: # ANALYSIS_MAX_FILES=150 # ANALYSIS_MAX_FILE_SIZE_BYTES=5242880 diff --git a/deployment/config/rag-pipeline/.env.sample b/deployment/config/rag-pipeline/.env.sample index ca9b97ad..8d678290 100644 --- a/deployment/config/rag-pipeline/.env.sample +++ b/deployment/config/rag-pipeline/.env.sample @@ -55,10 +55,19 @@ SERVICE_SECRET=change-me-to-a-random-secret # === Queue and Server Runtime === # REDIS_URL=redis://redis:6379/1 # MAX_CONCURRENT_RAG_JOBS=2 -# UVICORN_WORKERS=4 +# Keep one API process by default: every worker loads its own embedding/indexing state. +# Increase only when the host has enough memory for another complete runtime. +# UVICORN_WORKERS=1 # Project mutation coordination is correctness-critical in multi-worker setups. # RAG_MUTATION_LEASE_SECONDS=300 # RAG_MUTATION_ACQUIRE_TIMEOUT_SECONDS=5 +# Exact-generation integrity scans are streamed, cached per immutable physical +# collection, and single-flight. This cap applies only to cold verification in +# one RAG service process; it does not cap indexing or application-wide reviews. +# RAG_REVISION_PREFLIGHT_CACHE_ENTRIES=512 +# 0 retains immutable positive receipts until LRU eviction or process restart. +# RAG_REVISION_PREFLIGHT_CACHE_TTL_SECONDS=0 +# RAG_REVISION_PREFLIGHT_MAX_CONCURRENCY=2 # Expired pending collections are retained for six hours by default. # RAG_PENDING_COLLECTION_MAX_AGE_SECONDS=21600 # RAG_PENDING_JANITOR_INTERVAL_SECONDS=3600 diff --git a/deployment/docker-compose.prod.yml b/deployment/docker-compose.prod.yml index 831377ab..96f2c71d 100644 --- a/deployment/docker-compose.prod.yml +++ b/deployment/docker-compose.prod.yml @@ -167,6 +167,9 @@ services: PR_ENRICHMENT_MAX_FILE_SIZE_BYTES: ${PR_ENRICHMENT_MAX_FILE_SIZE_BYTES:-5242880} PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES: ${PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES:-20971520} CODECROW_VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD: ${VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD:-25} + RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES: ${RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES:-268435456} + RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES: ${RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES:-4294967296} + RAG_ARCHIVE_MAX_ENTRIES: ${RAG_ARCHIVE_MAX_ENTRIES:-500000} ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES: ${ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES:-15} ANALYSIS_QUEUE_ADMISSION_TIMEOUT_MINUTES: ${ANALYSIS_QUEUE_ADMISSION_TIMEOUT_MINUTES:-5} ANALYSIS_PROMPT_DRY_RUN_ENABLED: ${ANALYSIS_PROMPT_DRY_RUN_ENABLED:-false} diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 1a0c5297..306e6891 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -176,6 +176,9 @@ services: PR_ENRICHMENT_MAX_FILE_SIZE_BYTES: ${PR_ENRICHMENT_MAX_FILE_SIZE_BYTES:-5242880} PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES: ${PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES:-20971520} CODECROW_VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD: ${VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD:-25} + RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES: ${RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES:-268435456} + RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES: ${RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES:-4294967296} + RAG_ARCHIVE_MAX_ENTRIES: ${RAG_ARCHIVE_MAX_ENTRIES:-500000} ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES: ${ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES:-15} ANALYSIS_QUEUE_ADMISSION_TIMEOUT_MINUTES: ${ANALYSIS_QUEUE_ADMISSION_TIMEOUT_MINUTES:-5} ANALYSIS_PROMPT_DRY_RUN_ENABLED: ${ANALYSIS_PROMPT_DRY_RUN_ENABLED:-false} @@ -278,7 +281,7 @@ services: SERVICE_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} QDRANT_API_KEY: ${QDRANT_API_KEY:?QDRANT_API_KEY must be set in .env} REDIS_URL: redis://redis:6379/1 - #UVICORN_WORKERS: 1 + UVICORN_WORKERS: ${RAG_UVICORN_WORKERS:-1} ports: - "127.0.0.1:${RAG_PIPELINE_HOST_PORT:-8004}:8001" #- "127.0.0.1:5678:5678" diff --git a/frontend b/frontend index baa2853c..cc7c44a3 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit baa2853c665ba99b00861501ad1f5f984b4dd20a +Subproject commit cc7c44a3c0623f3e9259e5f388a27bdea0527100 diff --git a/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java b/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java index 86d4341a..3b882325 100644 --- a/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java +++ b/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java @@ -118,6 +118,20 @@ default boolean shouldHaveBranchIndex(Project project, String branchName) { : null; return config.ragConfig().shouldHaveBranchIndex(branchName, branchPushPatterns); } + + /** + * Whether an eligible PR target that is not retained may receive a temporary, + * revision-pinned branch snapshot. This never makes branch pushes retain data. + */ + default boolean shouldCreateTransientBranchIndex(Project project, String branchName) { + var config = project.getConfiguration(); + if (config == null || config.ragConfig() == null || branchName == null) { + return false; + } + return config.ragConfig().isTransientBranchIndexesEnabled() + && !branchName.equals(getBaseBranch(project)) + && !shouldHaveBranchIndex(project, branchName); + } /** * Get the authoritative base branch for RAG indexing. @@ -203,11 +217,9 @@ default void createOrUpdateBranchIndex( } /** - * Update branch index by calculating diff between base branch and target branch. - * - * This method always recalculates the full diff between the base branch (e.g., "master") - * and the target branch (e.g., "release/1.0"), then indexes all changed files with - * the target branch in their metadata. + * Update an already retained branch from its completed checkpoint. A first + * legacy branch seed may still compare it with the primary branch; exact + * generation implementations replace that seed path with a complete snapshot. * * Use this when a push happens to a non-main branch and you need to update * the RAG index to reflect the current state of that branch. diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java index 7de7ef5c..58642f6d 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java @@ -10,6 +10,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.rostilos.codecrow.queue.RedisQueueService; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; @@ -35,6 +37,9 @@ public class AiAnalysisClient { private final RedisQueueService queueService; private final ObjectMapper objectMapper; + @Autowired(required = false) + private RagBranchIndexGenerationRepository branchGenerationRepository; + static final String INACTIVITY_TIMEOUT_MINUTES_KEY = "ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES"; static final String ADMISSION_TIMEOUT_MINUTES_KEY = @@ -348,6 +353,25 @@ private Map buildSerializableRequestPayload(AiAnalysisRequest re payload.put("previousCommitHash", request.getPreviousCommitHash()); payload.put("currentCommitHash", request.getCurrentCommitHash()); payload.put("baseCommitHash", request.getBaseCommitHash()); + if (branchGenerationRepository != null + && request.getProjectId() != null + && request.getTargetBranchName() != null + && request.getBaseCommitHash() != null) { + branchGenerationRepository.findAvailableExactGeneration( + request.getProjectId(), + request.getTargetBranchName(), + request.getBaseCommitHash(), + List.of( + RagBranchIndexGenerationStatus.ACTIVE, + RagBranchIndexGenerationStatus.SUPERSEDED)) + .stream() + .findFirst() + .ifPresent(generation -> { + payload.put("ragCollectionTarget", generation.getCollectionName()); + payload.put("ragBaseGenerationManifestSha256", + generation.getManifestDigest()); + }); + } payload.put("previousCodeAnalysisIssues", request.getPreviousCodeAnalysisIssues()); payload.put("reconciliationFileContents", request.getReconciliationFileContents()); payload.put("projectCapabilities", request.getProjectCapabilities()); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java index 1084fa83..ea3e8f93 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java @@ -153,13 +153,33 @@ public BranchAnalysisProcessor( public Map process( BranchProcessRequest request, Consumer> consumer) throws IOException { + return process(request, consumer, false); + } + + /** + * Execute a branch job whose persisted cross-analysis dependencies were + * already resolved by the dispatcher. + */ + public Map processAfterDependencyGate( + BranchProcessRequest request, + Consumer> consumer) throws IOException { + return process(request, consumer, true); + } + + private Map process( + BranchProcessRequest request, + Consumer> consumer, + boolean dependencyGateSatisfied) throws IOException { Project project = projectService.getProjectWithConnections(request.getProjectId()); - // PR jobs are registered before async processing starts and remain active - // until their source-branch lock is released and analysis is persisted. - branchAnalysisGateService.awaitPrAnalysis( - project.getId(), request.getTargetBranchName(), - request.getSourcePrNumber(), consumer); + if (!dependencyGateSatisfied) { + // Scheduled/direct callers without a durable dispatch job retain the + // broad compatibility barrier. Webhook and pipeline dispatchers use the + // job-id snapshot barrier before calling processAfterDependencyGate(). + branchAnalysisGateService.awaitPrAnalysis( + project.getId(), request.getTargetBranchName(), + request.getSourcePrNumber(), consumer); + } refreshMergedBranchHead(project, request); Optional lockKey = analysisLockService.acquireLockWithWait( @@ -724,8 +744,12 @@ private void performDirectPushAnalysisIfNeeded( } // Check commit coverage by open/merged PRs + boolean exactTargetBranchCoverage = project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); CommitCoverageService.CoverageResult coverage = commitCoverageService.checkCoverage( - project.getId(), request.getTargetBranchName(), unanalyzedCommits); + project.getId(), request.getTargetBranchName(), unanalyzedCommits, + exactTargetBranchCoverage); switch (coverage.status()) { case FULLY_COVERED: @@ -858,10 +882,19 @@ private void performIncrementalRagUpdate(BranchProcessRequest request, Project p return; } - String targetBranch = request.getTargetBranchName(); - String baseBranch = ragOperationsService.getBaseBranch(project); + String targetBranch = request.getTargetBranchName(); + String baseBranch = ragOperationsService.getBaseBranch(project); + + if (!targetBranch.equals(baseBranch) + && !ragOperationsService.shouldHaveBranchIndex(project, targetBranch)) { + log.info("Skipping RAG update for non-retained branch: project={}, branch={}", + project.getId(), targetBranch); + EventNotificationEmitter.emitStatus(consumer, "rag_skipped", + "Branch is analyzed but is not configured as a retained RAG branch"); + return; + } - // Health check: verify RAG pipeline is reachable before starting + // Health check: verify RAG pipeline is reachable before starting if (!ragOperationsService.isRagPipelineHealthy()) { log.warn("RAG pipeline is not reachable — skipping incremental update for project={}", project.getId()); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java index 023c7a0f..bb6cfc6b 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java @@ -4,6 +4,7 @@ import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis; import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysisIssue; +import org.rostilos.codecrow.core.model.codeanalysis.AnalysisType; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.pullrequest.PullRequest; import org.rostilos.codecrow.core.model.vcs.EVcsProvider; @@ -52,6 +53,9 @@ import org.rostilos.codecrow.analysisengine.util.PromptDryRunMode; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.rostilos.codecrow.vcsclient.model.VcsCommit; +import org.rostilos.codecrow.scmevidence.service.ScmEvidenceService; +import org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisIssueRepository; /** * Generic service that handles pull request analysis. @@ -61,6 +65,7 @@ @Service public class PullRequestAnalysisProcessor { private static final Logger log = LoggerFactory.getLogger(PullRequestAnalysisProcessor.class); + private static final int SCM_EVIDENCE_COMMIT_LIMIT = 1000; private final CodeAnalysisService codeAnalysisService; private final TaskImplementationEvidenceService taskImplementationEvidenceService; @@ -76,6 +81,12 @@ public class PullRequestAnalysisProcessor { private final PrIssueTrackingService prIssueTrackingService; private final AstScopeEnricher astScopeEnricher; + @Autowired(required = false) + private ScmEvidenceService scmEvidenceService; + + @Autowired(required = false) + private CodeAnalysisIssueRepository codeAnalysisIssueRepository; + public PullRequestAnalysisProcessor( PullRequestService pullRequestService, CodeAnalysisService codeAnalysisService, @@ -357,7 +368,7 @@ public Map process( } // === DAG: Mark PR commits as ANALYZED === - markPrCommitsAnalyzed(project, request.getSourceBranchName(), request.getCommitHash(), newAnalysis); + markPrCommitsAnalyzed(project, request, newAnalysis); // Publish successful completion event publishAnalysisCompletedEvent(project, request, correlationId, startTime, @@ -741,13 +752,84 @@ private static void putIdentity(Map target, String key, Object v * @param commitHash the HEAD commit of the source branch * @param analysis the CodeAnalysis to link, or null for cache-hit scenarios */ - private void markPrCommitsAnalyzed(Project project, String sourceBranch, String commitHash, CodeAnalysis analysis) { + private void markPrCommitsAnalyzed( + Project project, + PrProcessRequest request, + CodeAnalysis analysis) { try { + String sourceBranch = request.getSourceBranchName(); + String targetBranch = request.getTargetBranchName(); + String commitHash = request.getCommitHash(); if (commitHash == null) return; + String targetBaseRevision = null; + List analyzedHashes = List.of(commitHash); + if (scmEvidenceService != null) { + try { + VcsClient client = vcsClientProvider.getClient( + project.getEffectiveVcsRepoInfo().getVcsConnection()); + String workspace = project.getEffectiveVcsRepoInfo().getRepoWorkspace(); + String repository = project.getEffectiveVcsRepoInfo().getRepoSlug(); + var pullRequest = client.getPullRequest( + workspace, repository, request.getPullRequestId()); + targetBaseRevision = pullRequest != null + ? pullRequest.baseCommit() : null; + List newestFirst = client.getCommitHistory( + workspace, repository, sourceBranch, + SCM_EVIDENCE_COMMIT_LIMIT); + List prCommits = selectPrEvidenceCommits( + newestFirst, commitHash, targetBaseRevision); + Collections.reverse(prCommits); + if (!prCommits.isEmpty()) { + scmEvidenceService.capture( + project.getId(), client, workspace, repository, + prCommits); + analyzedHashes = prCommits.stream() + .map(VcsCommit::hash) + .toList(); + } + scmEvidenceService.recordAnalysisReceipts( + project.getId(), analyzedHashes, + sourceBranch, targetBranch, targetBaseRevision, + analysis != null ? analysis.getId() : null, + AnalysisType.PR_REVIEW.name()); + if (analysis != null && codeAnalysisIssueRepository != null) { + for (CodeAnalysisIssue issue : analysis.getIssues()) { + scmEvidenceService.resolveIssueProvenance( + project.getId(), analyzedHashes, + issue.getFilePath(), issue.getLineNumber(), + issue.getCodeSnippet()) + .ifPresent(provenance -> { + issue.setIntroducingCommitHash( + provenance.commitHash()); + issue.setIntroducingAuthorName( + provenance.authorName()); + issue.setIntroducingAuthorEmail( + provenance.authorEmail()); + issue.setAuthorProvenanceConfidence( + provenance.confidence()); + }); + } + codeAnalysisIssueRepository.saveAll(analysis.getIssues()); + } + } catch (Exception evidenceFailure) { + log.warn("SCM promotion/provenance evidence unavailable for PR #{}: {}", + request.getPullRequestId(), evidenceFailure.getMessage()); + } + } + // Record the PR's HEAD commit as analyzed - analyzedCommitService.recordPrCommitsAnalyzed( - project, List.of(commitHash), analysis); + boolean multiBranch = project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); + if (multiBranch) { + analyzedCommitService.recordPrCommitsAnalyzed( + project, analyzedHashes, analysis, + sourceBranch, targetBranch, targetBaseRevision); + } else { + analyzedCommitService.recordPrCommitsAnalyzed( + project, List.of(commitHash), analysis); + } log.info("Recorded PR commit {} as analyzed (branch={}, analysis={})", commitHash.substring(0, Math.min(7, commitHash.length())), @@ -755,8 +837,51 @@ private void markPrCommitsAnalyzed(Project project, String sourceBranch, String analysis != null ? analysis.getId() : "none"); } catch (Exception e) { log.warn("Failed to record PR commit as analyzed (non-critical): branch={}, error={}", - sourceBranch, e.getMessage()); + request.getSourceBranchName(), e.getMessage()); + } + } + + /** + * Select only the ancestry that belongs to the revision actually reviewed. + * Provider history is newest-first and may already contain commits pushed + * after the webhook event, so collection cannot begin until the requested + * PR head is reached. + */ + static List selectPrEvidenceCommits( + List newestFirst, + String requestedRevision, + String targetBaseRevision) { + if (newestFirst == null || newestFirst.isEmpty() + || requestedRevision == null || requestedRevision.isBlank()) { + return new java.util.ArrayList<>(); + } + List selected = new java.util.ArrayList<>(); + boolean requestedRevisionReached = false; + boolean baseRevisionReached = targetBaseRevision == null; + for (VcsCommit commit : newestFirst) { + if (!requestedRevisionReached) { + if (!requestedRevision.equals(commit.hash())) { + continue; + } + requestedRevisionReached = true; + } + if (targetBaseRevision != null + && targetBaseRevision.equals(commit.hash())) { + baseRevisionReached = true; + break; + } + selected.add(commit); + if (targetBaseRevision == null) { + break; + } + } + if (!baseRevisionReached && selected.size() > 1) { + // The bounded provider window did not prove where PR ancestry ends. + // Retain the reviewed head receipt, but do not claim older commits + // that may predate the PR base. + return new java.util.ArrayList<>(selected.subList(0, 1)); } + return selected; } /** diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java index 9863323a..01bca47b 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java @@ -5,6 +5,7 @@ import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.*; @@ -29,16 +30,40 @@ public class BranchArchiveService { private static final Logger log = LoggerFactory.getLogger(BranchArchiveService.class); - /** - * Maximum single-file size to extract from archive (10 MB). - * Files larger than this are skipped to avoid memory pressure. - */ + /** Maximum single-file size retained by the in-memory extraction API. */ private static final long MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; + static final long DEFAULT_MAX_ARCHIVE_ENTRY_SIZE_BYTES = 256L * 1024 * 1024; + static final long DEFAULT_MAX_ARCHIVE_EXTRACTED_SIZE_BYTES = 4L * 1024 * 1024 * 1024; + static final int DEFAULT_MAX_ARCHIVE_ENTRIES = 500_000; private final VcsClientProvider vcsClientProvider; + private final long maxArchiveEntrySizeBytes; + private final long maxArchiveExtractedSizeBytes; + private final int maxArchiveEntries; + @Autowired public BranchArchiveService(VcsClientProvider vcsClientProvider) { + this( + vcsClientProvider, + envLong("RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES", DEFAULT_MAX_ARCHIVE_ENTRY_SIZE_BYTES), + envLong("RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES", DEFAULT_MAX_ARCHIVE_EXTRACTED_SIZE_BYTES), + envInt("RAG_ARCHIVE_MAX_ENTRIES", DEFAULT_MAX_ARCHIVE_ENTRIES)); + } + + BranchArchiveService( + VcsClientProvider vcsClientProvider, + long maxArchiveEntrySizeBytes, + long maxArchiveExtractedSizeBytes, + int maxArchiveEntries) { this.vcsClientProvider = vcsClientProvider; + if (maxArchiveEntrySizeBytes <= 0 + || maxArchiveExtractedSizeBytes <= 0 + || maxArchiveEntries <= 0) { + throw new IllegalArgumentException("Archive extraction limits must be positive"); + } + this.maxArchiveEntrySizeBytes = maxArchiveEntrySizeBytes; + this.maxArchiveExtractedSizeBytes = maxArchiveExtractedSizeBytes; + this.maxArchiveEntries = maxArchiveEntries; } /** @@ -164,12 +189,11 @@ public ArchiveDirectorySnapshot downloadAndExtractSnapshotToDirectory( repoSlug, branchOrCommit, archiveFile -> { - Set extractedFiles = extractFilesFromArchive( + Set extractedFiles = extractFilesToDirectory( archiveFile, neededFiles, presentFiles, - (relativePath, bytes) -> - writeFile(normalizedTarget, relativePath, bytes)); + normalizedTarget); return new ArchiveDirectorySnapshot(extractedFiles, presentFiles); }); } @@ -223,6 +247,7 @@ private Set extractFilesFromArchive( int skippedLarge = 0; int skippedNotNeeded = 0; int skippedUnsafe = 0; + ArchiveExtractionBudget budget = new ArchiveExtractionBudget(); try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(archiveFile))) { ZipEntry entry; @@ -233,27 +258,32 @@ private Set extractFilesFromArchive( } String relativePath = stripArchiveRoot(entry.getName()); + ArchiveEntryBudget entryBudget = budget.beginEntry(relativePath, entry.getSize()); // Skip files we don't need if (neededFiles != null && !neededFiles.isEmpty() && !neededFiles.contains(relativePath)) { skippedNotNeeded++; + drainEntry(zis, entryBudget); zis.closeEntry(); continue; } presentFiles.add(relativePath); - // Read the entry content - byte[] bytes = readZipEntry(zis); + // The in-memory API must not retain an unbounded archive + // member. RAG uses extractFilesToDirectory below instead, + // which streams every accepted entry to disk. + ZipEntryContent content = readZipEntryBounded(zis, entryBudget); - // Skip very large files - if (bytes.length > MAX_FILE_SIZE_BYTES) { - log.debug("Skipping large file {} ({} bytes)", relativePath, bytes.length); + if (content.tooLarge()) { + log.debug("Skipping large file {} (more than {} bytes)", + relativePath, MAX_FILE_SIZE_BYTES); skippedLarge++; zis.closeEntry(); continue; } + byte[] bytes = content.bytes(); // Skip binary files (null bytes in first 1 KB) if (isBinary(bytes)) { @@ -285,17 +315,101 @@ private Set extractFilesFromArchive( return extractedFiles; } - private boolean writeFile(Path targetDirectory, String relativePath, byte[] bytes) throws IOException { + /** + * Streams archive entries directly into an isolated repository directory. + * This is the RAG path: it intentionally has no archive-size or entry-size + * ceiling, so large repositories do not require correspondingly large JVM + * heaps. Only a small prefix and a fixed copy buffer are held in memory. + */ + private Set extractFilesToDirectory( + Path archiveFile, + Set neededFiles, + Set presentFiles, + Path targetDirectory + ) throws IOException { + Set extractedFiles = new LinkedHashSet<>(); + int skippedBinary = 0; + int skippedNotNeeded = 0; + int skippedUnsafe = 0; + ArchiveExtractionBudget budget = new ArchiveExtractionBudget(); + + try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(archiveFile))) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (entry.isDirectory()) { + zis.closeEntry(); + continue; + } + + String relativePath = stripArchiveRoot(entry.getName()); + ArchiveEntryBudget entryBudget = budget.beginEntry(relativePath, entry.getSize()); + if (neededFiles != null && !neededFiles.isEmpty() + && !neededFiles.contains(relativePath)) { + skippedNotNeeded++; + drainEntry(zis, entryBudget); + zis.closeEntry(); + continue; + } + + presentFiles.add(relativePath); + Path targetFile = resolveTargetFile(targetDirectory, relativePath); + if (targetFile == null) { + skippedUnsafe++; + drainEntry(zis, entryBudget); + zis.closeEntry(); + continue; + } + + byte[] prefix = zis.readNBytes(1024); + entryBudget.consume(prefix.length); + if (isBinary(prefix, prefix.length)) { + skippedBinary++; + drainEntry(zis, entryBudget); + zis.closeEntry(); + continue; + } + + try (OutputStream output = Files.newOutputStream(targetFile)) { + output.write(prefix); + byte[] buffer = new byte[8192]; + int length; + while ((length = zis.read(buffer)) > 0) { + entryBudget.consume(length); + output.write(buffer, 0, length); + } + } catch (IOException extractionFailure) { + Files.deleteIfExists(targetFile); + throw extractionFailure; + } + setWorldReadable(targetFile, false); + extractedFiles.add(relativePath); + zis.closeEntry(); + + if (neededFiles != null && !neededFiles.isEmpty() + && extractedFiles.size() >= neededFiles.size()) { + break; + } + } + } + + log.info("Archive extraction: {} files streamed to disk, {} skipped (not needed), " + + "{} binary, {} unsafe. Requested: {}", + extractedFiles.size(), skippedNotNeeded, skippedBinary, skippedUnsafe, + neededFiles != null ? neededFiles.size() : "all"); + return extractedFiles; + } + + private Path resolveTargetFile(Path targetDirectory, String relativePath) throws IOException { Path targetFile; try { targetFile = targetDirectory.resolve(relativePath).normalize(); } catch (RuntimeException e) { log.warn("Skipping invalid archive entry path: {}", relativePath); - return false; + return null; } if (!targetFile.startsWith(targetDirectory) || targetFile.equals(targetDirectory)) { log.warn("Skipping archive entry outside target directory: {}", relativePath); - return false; + return null; } Path parent = targetFile.getParent(); @@ -305,9 +419,7 @@ private boolean writeFile(Path targetDirectory, String relativePath, byte[] byte directory = directory.getParent()) { setWorldReadable(directory, true); } - Files.write(targetFile, bytes); - setWorldReadable(targetFile, false); - return true; + return targetFile; } private void setWorldReadable(Path path, boolean directory) { @@ -338,14 +450,53 @@ static String stripArchiveRoot(String entryPath) { return entryPath; } - private byte[] readZipEntry(ZipInputStream zis) throws IOException { + private ZipEntryContent readZipEntryBounded( + ZipInputStream zis, + ArchiveEntryBudget entryBudget + ) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); byte[] buffer = new byte[8192]; + long totalBytes = 0; + boolean tooLarge = false; int len; while ((len = zis.read(buffer)) > 0) { + entryBudget.consume(len); + totalBytes += len; + if (totalBytes > MAX_FILE_SIZE_BYTES) { + tooLarge = true; + continue; + } baos.write(buffer, 0, len); } - return baos.toByteArray(); + return tooLarge + ? ZipEntryContent.TOO_LARGE + : new ZipEntryContent(baos.toByteArray(), false); + } + + private void drainEntry(ZipInputStream zis, ArchiveEntryBudget entryBudget) throws IOException { + byte[] buffer = new byte[8192]; + int length; + while ((length = zis.read(buffer)) > 0) { + entryBudget.consume(length); + } + } + + private static long envLong(String name, long fallback) { + try { + long value = Long.parseLong(System.getenv().getOrDefault(name, String.valueOf(fallback))); + return value > 0 ? value : fallback; + } catch (NumberFormatException ignored) { + return fallback; + } + } + + private static int envInt(String name, int fallback) { + try { + int value = Integer.parseInt(System.getenv().getOrDefault(name, String.valueOf(fallback))); + return value > 0 ? value : fallback; + } catch (NumberFormatException ignored) { + return fallback; + } } /** @@ -353,7 +504,11 @@ private byte[] readZipEntry(ZipInputStream zis) throws IOException { * in the first 1024 bytes. */ private boolean isBinary(byte[] data) { - int checkLimit = Math.min(data.length, 1024); + return isBinary(data, data.length); + } + + private boolean isBinary(byte[] data, int length) { + int checkLimit = Math.min(length, 1024); for (int i = 0; i < checkLimit; i++) { if (data[i] == 0) return true; } @@ -376,6 +531,65 @@ private interface ExtractedFileConsumer { boolean accept(String relativePath, byte[] bytes) throws IOException; } + private record ZipEntryContent(byte[] bytes, boolean tooLarge) { + private static final ZipEntryContent TOO_LARGE = new ZipEntryContent(null, true); + } + + private final class ArchiveExtractionBudget { + private long extractedBytes; + private int entries; + + private ArchiveEntryBudget beginEntry(String relativePath, long declaredSize) throws IOException { + entries++; + if (entries > maxArchiveEntries) { + throw new IOException("Repository archive exceeds RAG_ARCHIVE_MAX_ENTRIES=" + + maxArchiveEntries); + } + if (declaredSize > maxArchiveEntrySizeBytes) { + throw new IOException("Repository archive entry '" + relativePath + + "' exceeds RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES=" + + maxArchiveEntrySizeBytes); + } + if (declaredSize >= 0 + && declaredSize > maxArchiveExtractedSizeBytes - extractedBytes) { + throw new IOException("Repository archive exceeds RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES=" + + maxArchiveExtractedSizeBytes); + } + return new ArchiveEntryBudget(relativePath, this); + } + + private void consume(String relativePath, long entryBytes, int length) throws IOException { + if (length > maxArchiveEntrySizeBytes - entryBytes) { + throw new IOException("Repository archive entry '" + relativePath + + "' exceeds RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES=" + + maxArchiveEntrySizeBytes); + } + if (length > maxArchiveExtractedSizeBytes - extractedBytes) { + throw new IOException("Repository archive exceeds RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES=" + + maxArchiveExtractedSizeBytes); + } + extractedBytes += length; + } + } + + private static final class ArchiveEntryBudget { + private final String relativePath; + private final ArchiveExtractionBudget archiveBudget; + private long bytes; + + private ArchiveEntryBudget( + String relativePath, + ArchiveExtractionBudget archiveBudget) { + this.relativePath = relativePath; + this.archiveBudget = archiveBudget; + } + + private void consume(int length) throws IOException { + archiveBudget.consume(relativePath, bytes, length); + bytes += length; + } + } + public record ArchiveSnapshot( Map contents, Set presentFiles diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java index 2314f764..6718f620 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java @@ -42,6 +42,36 @@ public BranchAnalysisGateService(JobRepository jobRepository) { this.jobRepository = jobRepository; } + /** + * Apply the dependency barrier for a durably accepted analysis job. + * Jobs only wait for older work on the same target branch, which preserves + * repository/RAG ordering without serializing independent PR reviews. + */ + public GateResult awaitDependencies( + Long projectId, + Job job, + Consumer> consumer) { + if (job == null || job.getId() == null) { + return GateResult.READY; + } + if (job.getJobType() == JobType.BRANCH_ANALYSIS) { + return awaitTurn( + projectId, + job.getBranchName(), + job.getId(), + job.getPrNumber(), + consumer); + } + if (job.getJobType() == JobType.PR_ANALYSIS) { + awaitOlderBranchAnalyses( + projectId, + job.getBranchName(), + job.getId(), + consumer); + } + return GateResult.READY; + } + /** * Wait for the relevant target-branch PR job. When the merge PR number is * known, only its newest analysis attempt can block reconciliation. When it @@ -117,6 +147,36 @@ public void awaitPrAnalysis( awaitTurn(projectId, branchName, null, sourcePrNumber, consumer); } + public void awaitOlderBranchAnalyses( + Long projectId, + String branchName, + Long currentPrJobId, + Consumer> consumer) { + if (branchName == null || branchName.isBlank() || currentPrJobId == null) { + return; + } + + long timeoutNanos = TimeUnit.MINUTES.toNanos(Math.max(1, waitTimeoutMinutes)); + long startedAt = System.nanoTime(); + while (jobRepository.existsActiveBranchAnalysisJobBefore( + projectId, branchName, currentPrJobId)) { + long waitedNanos = System.nanoTime() - startedAt; + if (waitedNanos >= timeoutNanos) { + log.warn("Timed out waiting for target branch update: project={}, branch={}, prJob={}, waited={}m", + projectId, branchName, currentPrJobId, + TimeUnit.NANOSECONDS.toMinutes(waitedNanos)); + throw new AnalysisLockedException( + AnalysisLockType.BRANCH_ANALYSIS.name(), branchName, projectId); + } + + emitBranchWait(consumer, branchName, waitedNanos); + if (!pause()) { + throw new AnalysisLockedException( + AnalysisLockType.BRANCH_ANALYSIS.name(), branchName, projectId); + } + } + } + private boolean hasBlockingPrAnalysis( Long projectId, String branchName, @@ -166,6 +226,26 @@ private void emitWait( } } + private void emitBranchWait( + Consumer> consumer, + String branchName, + long waitedNanos) { + if (consumer == null) { + return; + } + try { + consumer.accept(Map.of( + "type", "branch_analysis_wait", + "state", "waiting_for_target_branch", + "message", "Waiting for the earlier " + branchName + + " update and its RAG publication to finish", + "branchName", branchName, + "waitedSeconds", TimeUnit.NANOSECONDS.toSeconds(waitedNanos))); + } catch (Exception e) { + log.debug("Could not emit branch barrier status: {}", e.getMessage()); + } + } + private boolean pause() { if (pollIntervalMillis <= 0) { return true; diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java index 5591e31f..5831375f 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java @@ -2,10 +2,13 @@ import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest; import org.rostilos.codecrow.commitgraph.service.AnalyzedCommitService; +import org.rostilos.codecrow.core.model.codeanalysis.AnalysisType; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.persistence.repository.branch.BranchRepository; +import org.rostilos.codecrow.scmevidence.service.ScmEvidenceService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.List; @@ -17,6 +20,9 @@ public class BranchHealthService { private final BranchRepository branchRepository; private final AnalyzedCommitService analyzedCommitService; + @Autowired(required = false) + private ScmEvidenceService scmEvidenceService; + public BranchHealthService( BranchRepository branchRepository, AnalyzedCommitService analyzedCommitService @@ -37,7 +43,22 @@ public void recordCommitsAnalyzed(Project project, List unanalyzedCommit String branchName) { if (unanalyzedCommits.isEmpty()) return; try { - analyzedCommitService.recordBranchCommitsAnalyzed(project, unanalyzedCommits); + boolean multiBranch = project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); + if (multiBranch) { + analyzedCommitService.recordBranchCommitsAnalyzed( + project, unanalyzedCommits, branchName); + } else { + analyzedCommitService.recordBranchCommitsAnalyzed( + project, unanalyzedCommits); + } + if (scmEvidenceService != null) { + scmEvidenceService.recordAnalysisReceipts( + project.getId(), unanalyzedCommits, branchName, branchName, + unanalyzedCommits.get(unanalyzedCommits.size() - 1), + null, AnalysisType.BRANCH_ANALYSIS.name()); + } log.info("Recorded {} commits as analyzed after successful branch analysis (branch={})", unanalyzedCommits.size(), branchName); } catch (Exception e) { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java index ec305311..69ed3606 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java @@ -318,6 +318,47 @@ void shouldIncludeSourceAndTargetBranchNamesInQueuedRequestPayload() throws Exce assertThat(requestPayload.get("ragEnabled")).isEqualTo(true); } + @Test + @DisplayName("should bind exact target branch generation to queued review") + void shouldBindExactTargetBranchGenerationToQueuedReview() throws Exception { + var repository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexGenerationRepository.class); + var generation = mock(org.rostilos.codecrow.core.model.rag + .RagBranchIndexGeneration.class); + when(generation.getCollectionName()).thenReturn("opaque-master-generation"); + when(generation.getManifestDigest()).thenReturn("master-manifest"); + when(repository.findAvailableExactGeneration( + eq(1L), eq("main"), eq("master-base"), anyList())) + .thenReturn(List.of(generation)); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchGenerationRepository", repository); + AiAnalysisRequest exactRequest = new TestAiAnalysisRequest() { + @Override + public String getBaseCommitHash() { + return "master-base"; + } + }; + Map finalEvent = Map.of( + "type", "final", + "result", Map.of("comment", "ok", "issues", List.of())); + when(queueService.rightPop(anyString(), anyLong())) + .thenReturn(objectMapper.writeValueAsString(finalEvent)); + + client.performAnalysis(exactRequest); + + var payloadCaptor = org.mockito.ArgumentCaptor.forClass(String.class); + verify(queueService).leftPush(eq("codecrow:analysis:jobs"), payloadCaptor.capture()); + @SuppressWarnings("unchecked") + Map queued = objectMapper.readValue( + payloadCaptor.getValue(), Map.class); + @SuppressWarnings("unchecked") + Map requestPayload = + (Map) queued.get("request"); + assertThat(requestPayload) + .containsEntry("ragCollectionTarget", "opaque-master-generation") + .containsEntry("ragBaseGenerationManifestSha256", "master-manifest"); + } + @Test @DisplayName("should include task context in queued request payload") void shouldIncludeTaskContextInQueuedRequestPayload() throws Exception { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java index f987a06e..5781c71f 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java @@ -586,6 +586,7 @@ void shouldScopeOversizedDirectPushToFilesWithPreviousIssues() throws Exception when(ragOperationsService.isRagEnabled(project)).thenReturn(true); when(ragOperationsService.isRagIndexReady(project)).thenReturn(true); when(ragOperationsService.getBaseBranch(project)).thenReturn("main"); + when(ragOperationsService.shouldHaveBranchIndex(project, "feature-x")).thenReturn(true); when(ragOperationsService.isRagPipelineHealthy()).thenReturn(true); Map result = processor.process(request, events::add); @@ -807,6 +808,7 @@ void shouldCallUpdateBranchIndexForNonMainBranch() throws Exception { when(ragOperationsService.isRagEnabled(project)).thenReturn(true); when(ragOperationsService.isRagIndexReady(project)).thenReturn(true); when(ragOperationsService.getBaseBranch(project)).thenReturn("main"); + when(ragOperationsService.shouldHaveBranchIndex(project, "feature-x")).thenReturn(true); when(ragOperationsService.isRagPipelineHealthy()).thenReturn(true); when(branchRepository.findByProjectIdAndBranchName(1L, "feature-x")) @@ -819,6 +821,36 @@ void shouldCallUpdateBranchIndexForNonMainBranch() throws Exception { verify(ragOperationsService, never()).triggerIncrementalUpdate(any(), any(), any(), any(), any()); } + @Test + @DisplayName("should not mutate RAG for an analyzed branch that is not retained") + void shouldSkipRagForNonRetainedBranch() { + BranchProcessRequest request = createRequest(); + request.targetBranchName = "release/preview"; + request.commitHash = "release-commit"; + List> events = new ArrayList<>(); + + when(project.getId()).thenReturn(1L); + when(ragOperationsService.isRagEnabled(project)).thenReturn(true); + when(ragOperationsService.isRagIndexReady(project)).thenReturn(true); + when(ragOperationsService.getBaseBranch(project)).thenReturn("master"); + when(ragOperationsService.shouldHaveBranchIndex(project, "release/preview")).thenReturn(false); + + ReflectionTestUtils.invokeMethod( + processor, + "performIncrementalRagUpdate", + request, + project, + "diff --git a/f.java b/f.java\n+x\n", + (Consumer>) events::add, + false); + + verify(ragOperationsService, never()).isRagPipelineHealthy(); + verify(ragOperationsService, never()).updateBranchIndex(any(), any(), any()); + verify(ragOperationsService, never()).triggerIncrementalUpdate(any(), any(), any(), any(), any()); + assertThat(events).anySatisfy(event -> + assertThat(event).containsEntry("state", "rag_skipped")); + } + @Test @DisplayName("should handle delta diff failure and fall back to PR diff") void shouldFallBackToPrDiffWhenDeltaDiffFails() throws Exception { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java index 5560b540..13668ba2 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java @@ -30,6 +30,7 @@ import org.rostilos.codecrow.core.service.CodeAnalysisService; import org.rostilos.codecrow.core.service.TaskImplementationEvidenceService; import org.rostilos.codecrow.filecontent.service.FileSnapshotService; +import org.rostilos.codecrow.vcsclient.model.VcsCommit; import org.rostilos.codecrow.analysisengine.service.AstScopeEnricher; import org.rostilos.codecrow.analysisengine.service.pr.PrIssueTrackingService; import org.rostilos.codecrow.analysisengine.util.PromptDryRunMode; @@ -152,6 +153,54 @@ private PrProcessRequest createRequest() { return request; } + @Test + @DisplayName("SCM evidence starts at the reviewed PR revision") + void scmEvidenceExcludesCommitsPushedAfterReviewedHead() { + List selected = PullRequestAnalysisProcessor.selectPrEvidenceCommits( + List.of( + commit("newer-2"), + commit("newer-1"), + commit("reviewed-head"), + commit("pr-parent"), + commit("target-base"), + commit("older")), + "reviewed-head", + "target-base"); + + assertThat(selected).extracting(VcsCommit::hash) + .containsExactly("reviewed-head", "pr-parent"); + } + + @Test + @DisplayName("SCM evidence stays empty when reviewed head is outside provider history") + void scmEvidenceDoesNotClaimUnknownHistoryWindow() { + List selected = PullRequestAnalysisProcessor.selectPrEvidenceCommits( + List.of(commit("newer"), commit("target-base")), + "reviewed-head", + "target-base"); + + assertThat(selected).isEmpty(); + } + + @Test + @DisplayName("SCM evidence keeps only reviewed head when PR base is outside provider history") + void scmEvidenceDoesNotClaimCommitsPastUnknownBase() { + List selected = PullRequestAnalysisProcessor.selectPrEvidenceCommits( + List.of( + commit("reviewed-head"), + commit("unknown-parent"), + commit("older")), + "reviewed-head", + "target-base"); + + assertThat(selected).extracting(VcsCommit::hash) + .containsExactly("reviewed-head"); + } + + private static VcsCommit commit(String hash) { + return new VcsCommit(hash, hash, "author", "author@example.test", null, List.of()); + } + @Nested @DisplayName("process()") class ProcessTests { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java index d0e8e16d..b01f9902 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java @@ -185,6 +185,109 @@ void shouldSkipBinaryFiles(@TempDir Path tempDir) throws Exception { assertThat(result).doesNotContainKey("binary.dat"); } + @Test + void shouldStreamLargeArchiveEntriesToDiskWithoutRetainingThem(@TempDir Path tempDir) + throws Exception { + VcsConnection conn = new VcsConnection(); + when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); + + byte[] largeFile = new byte[10 * 1024 * 1024 + 1]; + java.util.Arrays.fill(largeFile, (byte) 'x'); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + zos.putNextEntry(new ZipEntry("root/large.generated")); + zos.write(largeFile); + zos.closeEntry(); + zos.putNextEntry(new ZipEntry("root/src/StillIndexed.java")); + zos.write("class StillIndexed {}".getBytes(StandardCharsets.UTF_8)); + zos.closeEntry(); + } + byte[] zipBytes = baos.toByteArray(); + when(vcsClient.downloadRepositoryArchiveToFile(anyString(), anyString(), anyString(), any(Path.class))) + .thenAnswer(inv -> { + Files.write(inv.getArgument(3, Path.class), zipBytes); + return (long) zipBytes.length; + }); + + Set extracted = service.downloadAndExtractFilesToDirectory( + conn, "ws", "repo", "main", null, tempDir.resolve("repository")); + + assertThat(extracted).containsExactlyInAnyOrder("large.generated", "src/StillIndexed.java"); + assertThat(tempDir.resolve("repository/large.generated")).hasSize(largeFile.length); + assertThat(tempDir.resolve("repository/src/StillIndexed.java")).exists(); + } + + @Test + void extractToDirectory_rejectsOversizedEntryAndDeletesPartialFile(@TempDir Path tempDir) + throws Exception { + service = new BranchArchiveService(vcsClientProvider, 32, 1024, 10); + VcsConnection conn = new VcsConnection(); + when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); + byte[] zipBytes = createZip(Map.of( + "root/oversized.txt", "x".repeat(64) + )); + when(vcsClient.downloadRepositoryArchiveToFile( + anyString(), anyString(), anyString(), any(Path.class))) + .thenAnswer(inv -> { + Files.write(inv.getArgument(3, Path.class), zipBytes); + return (long) zipBytes.length; + }); + Path targetDirectory = tempDir.resolve("repository"); + + assertThatThrownBy(() -> service.downloadAndExtractFilesToDirectory( + conn, "ws", "repo", "main", null, targetDirectory)) + .isInstanceOf(IOException.class) + .hasMessageContaining("RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES=32"); + assertThat(targetDirectory.resolve("oversized.txt")).doesNotExist(); + } + + @Test + void extractToDirectory_rejectsArchiveOverTotalExtractedLimit(@TempDir Path tempDir) + throws Exception { + service = new BranchArchiveService(vcsClientProvider, 64, 40, 10); + VcsConnection conn = new VcsConnection(); + when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); + byte[] zipBytes = createZip(Map.of( + "root/first.txt", "a".repeat(24), + "root/second.txt", "b".repeat(24) + )); + when(vcsClient.downloadRepositoryArchiveToFile( + anyString(), anyString(), anyString(), any(Path.class))) + .thenAnswer(inv -> { + Files.write(inv.getArgument(3, Path.class), zipBytes); + return (long) zipBytes.length; + }); + + assertThatThrownBy(() -> service.downloadAndExtractFilesToDirectory( + conn, "ws", "repo", "main", null, tempDir.resolve("repository"))) + .isInstanceOf(IOException.class) + .hasMessageContaining("RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES=40"); + } + + @Test + void extractionCountsUnrequestedEntriesAgainstArchiveEntryLimit(@TempDir Path tempDir) + throws Exception { + service = new BranchArchiveService(vcsClientProvider, 64, 1024, 1); + VcsConnection conn = new VcsConnection(); + when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); + byte[] zipBytes = createZip(Map.of( + "root/requested.txt", "requested", + "root/unrequested.txt", "unrequested" + )); + when(vcsClient.downloadRepositoryArchiveToFile( + anyString(), anyString(), anyString(), any(Path.class))) + .thenAnswer(inv -> { + Files.write(inv.getArgument(3, Path.class), zipBytes); + return (long) zipBytes.length; + }); + + assertThatThrownBy(() -> service.downloadAndExtractFilesToDirectory( + conn, "ws", "repo", "main", Set.of("missing.txt"), + tempDir.resolve("repository"))) + .isInstanceOf(IOException.class) + .hasMessageContaining("RAG_ARCHIVE_MAX_ENTRIES=1"); + } + @Test void snapshotReportsBinaryPathPresenceWithoutLoadingItsContent(@TempDir Path tempDir) throws Exception { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java index 59cf8ba0..98bd2709 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java @@ -147,6 +147,28 @@ void processorLevelBarrierUsesLatestAttemptForTheMergedPr() { verify(jobRepository, times(0)).existsActivePrAnalysisJob(1L, "main"); } + @Test + void prWaitsOnlyForOlderBranchJobsOnItsTargetBranch() { + @SuppressWarnings("unchecked") + Consumer> consumer = mock(Consumer.class); + Job prJob = job(JobStatus.RUNNING); + ReflectionTestUtils.setField(prJob, "id", 104L); + prJob.setBranchName("main"); + + when(jobRepository.existsActiveBranchAnalysisJobBefore(1L, "main", 104L)) + .thenReturn(true, true, false); + + BranchAnalysisGateService.GateResult result = service.awaitDependencies( + 1L, prJob, consumer); + + assertThat(result).isEqualTo(BranchAnalysisGateService.GateResult.READY); + verify(jobRepository, times(3)) + .existsActiveBranchAnalysisJobBefore(1L, "main", 104L); + verify(consumer, times(2)).accept(org.mockito.ArgumentMatchers.argThat( + event -> "branch_analysis_wait".equals(event.get("type")) + && "main".equals(event.get("branchName")))); + } + private static Job job(JobStatus status) { Job job = new Job(); job.setJobType(JobType.PR_ANALYSIS); diff --git a/java-ecosystem/libs/commit-graph/pom.xml b/java-ecosystem/libs/commit-graph/pom.xml index a598ff59..40fda6bd 100644 --- a/java-ecosystem/libs/commit-graph/pom.xml +++ b/java-ecosystem/libs/commit-graph/pom.xml @@ -33,6 +33,11 @@ codecrow-vcs-client + + org.rostilos.codecrow + codecrow-scm-evidence + + org.springframework.boot diff --git a/java-ecosystem/libs/commit-graph/src/main/java/module-info.java b/java-ecosystem/libs/commit-graph/src/main/java/module-info.java index 903c7bbe..fd08bc42 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/module-info.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/module-info.java @@ -10,6 +10,7 @@ requires org.rostilos.codecrow.core; requires org.rostilos.codecrow.vcs; + requires org.rostilos.codecrow.scmevidence; // Model exports exports org.rostilos.codecrow.commitgraph.model; diff --git a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/model/AnalyzedCommit.java b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/model/AnalyzedCommit.java index 2cb28629..16c711f4 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/model/AnalyzedCommit.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/model/AnalyzedCommit.java @@ -19,12 +19,6 @@ @Entity @Table( name = "analyzed_commit", - uniqueConstraints = { - @UniqueConstraint( - name = "uq_analyzed_commit_project_hash", - columnNames = {"project_id", "commit_hash"} - ) - }, indexes = { @Index(name = "idx_analyzed_commit_project", columnList = "project_id"), @Index(name = "idx_analyzed_commit_hash", columnList = "commit_hash") @@ -61,6 +55,15 @@ public class AnalyzedCommit { @Column(name = "analysis_type", length = 30) private AnalysisType analysisType; + @Column(name = "source_branch", length = 256) + private String sourceBranch; + + @Column(name = "target_branch", length = 256) + private String targetBranch; + + @Column(name = "target_base_revision", length = 64) + private String targetBaseRevision; + // ── Constructors ─────────────────────────────────────────────────── public AnalyzedCommit() { @@ -97,4 +100,10 @@ public AnalyzedCommit(Project project, String commitHash, Long analysisId, Analy public AnalysisType getAnalysisType() { return analysisType; } public void setAnalysisType(AnalysisType analysisType) { this.analysisType = analysisType; } + public String getSourceBranch() { return sourceBranch; } + public void setSourceBranch(String sourceBranch) { this.sourceBranch = sourceBranch; } + public String getTargetBranch() { return targetBranch; } + public void setTargetBranch(String targetBranch) { this.targetBranch = targetBranch; } + public String getTargetBaseRevision() { return targetBaseRevision; } + public void setTargetBaseRevision(String targetBaseRevision) { this.targetBaseRevision = targetBaseRevision; } } diff --git a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/persistence/AnalyzedCommitRepository.java b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/persistence/AnalyzedCommitRepository.java index 47460eab..b4f5243a 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/persistence/AnalyzedCommitRepository.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/persistence/AnalyzedCommitRepository.java @@ -18,6 +18,9 @@ public interface AnalyzedCommitRepository extends JpaRepository findAnalyzedHashesByProjectIdAndCommitHashIn( @Param("projectId") Long projectId, @Param("hashes") List hashes); + @Query(""" + SELECT ac.commitHash FROM AnalyzedCommit ac + WHERE ac.project.id = :projectId + AND ac.targetBranch = :targetBranch + AND ac.commitHash IN :hashes + """) + Set findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + @Param("projectId") Long projectId, + @Param("targetBranch") String targetBranch, + @Param("hashes") List hashes); + /** * Find all analyzed commits for a project. */ diff --git a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/AnalyzedCommitService.java b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/AnalyzedCommitService.java index 31a9c93d..3ff715b7 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/AnalyzedCommitService.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/AnalyzedCommitService.java @@ -41,15 +41,29 @@ public AnalyzedCommitService(AnalyzedCommitRepository analyzedCommitRepository) */ @Transactional public void recordBranchCommitsAnalyzed(Project project, List hashes) { + recordBranchCommitsAnalyzed(project, hashes, null); + } + + @Transactional + public void recordBranchCommitsAnalyzed( + Project project, List hashes, String targetBranch) { if (hashes == null || hashes.isEmpty()) return; - Set alreadyAnalyzed = analyzedCommitRepository - .findAnalyzedHashesByProjectIdAndCommitHashIn(project.getId(), hashes); + Set alreadyAnalyzed = targetBranch == null + ? analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndCommitHashIn( + project.getId(), hashes) + : analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + project.getId(), targetBranch, hashes); List toSave = new ArrayList<>(); for (String hash : hashes) { if (!alreadyAnalyzed.contains(hash)) { - toSave.add(new AnalyzedCommit(project, hash, AnalysisType.BRANCH_ANALYSIS)); + AnalyzedCommit analyzed = new AnalyzedCommit( + project, hash, AnalysisType.BRANCH_ANALYSIS); + analyzed.setTargetBranch(targetBranch); + toSave.add(analyzed); } } @@ -69,16 +83,37 @@ public void recordBranchCommitsAnalyzed(Project project, List hashes) { */ @Transactional public void recordPrCommitsAnalyzed(Project project, List hashes, CodeAnalysis analysis) { + recordPrCommitsAnalyzed(project, hashes, analysis, null, null, null); + } + + @Transactional + public void recordPrCommitsAnalyzed( + Project project, + List hashes, + CodeAnalysis analysis, + String sourceBranch, + String targetBranch, + String targetBaseRevision) { if (hashes == null || hashes.isEmpty()) return; - Set alreadyAnalyzed = analyzedCommitRepository - .findAnalyzedHashesByProjectIdAndCommitHashIn(project.getId(), hashes); + Set alreadyAnalyzed = targetBranch == null + ? analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndCommitHashIn( + project.getId(), hashes) + : analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + project.getId(), targetBranch, hashes); Long analysisId = analysis != null ? analysis.getId() : null; List toSave = new ArrayList<>(); for (String hash : hashes) { if (!alreadyAnalyzed.contains(hash)) { - toSave.add(new AnalyzedCommit(project, hash, analysisId, AnalysisType.PR_REVIEW)); + AnalyzedCommit analyzed = new AnalyzedCommit( + project, hash, analysisId, AnalysisType.PR_REVIEW); + analyzed.setSourceBranch(sourceBranch); + analyzed.setTargetBranch(targetBranch); + analyzed.setTargetBaseRevision(targetBaseRevision); + toSave.add(analyzed); } } @@ -107,10 +142,26 @@ public List filterUnanalyzed(Long projectId, List allHashes) { .toList(); } + public List filterUnanalyzed( + Long projectId, String targetBranch, List allHashes) { + if (allHashes == null || allHashes.isEmpty()) return List.of(); + Set analyzed = analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + projectId, targetBranch, allHashes); + return allHashes.stream().filter(hash -> !analyzed.contains(hash)).toList(); + } + /** * Check if a specific commit has been analyzed. */ public boolean isAnalyzed(Long projectId, String commitHash) { return analyzedCommitRepository.existsByProjectIdAndCommitHash(projectId, commitHash); } + + public boolean isAnalyzed( + Long projectId, String targetBranch, String commitHash) { + return analyzedCommitRepository + .existsByProjectIdAndTargetBranchAndCommitHash( + projectId, targetBranch, commitHash); + } } diff --git a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java index 64668b77..ecc423a3 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java @@ -8,12 +8,15 @@ import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.rostilos.codecrow.vcsclient.model.VcsCommit; +import org.rostilos.codecrow.scmevidence.service.ScmEvidenceService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Autowired; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; /** @@ -34,20 +37,34 @@ public class BranchCommitService { private static final Logger log = LoggerFactory.getLogger(BranchCommitService.class); - private static final int DEFAULT_COMMIT_FETCH_LIMIT = 100; + // Preserve evidence for long-lived branches instead of collapsing a 400+ + // commit promotion to a HEAD-only fallback. + private static final int DEFAULT_COMMIT_FETCH_LIMIT = 1000; private final VcsClientProvider vcsClientProvider; private final AnalyzedCommitService analyzedCommitService; private final BranchRepository branchRepository; + private final ScmEvidenceService scmEvidenceService; public BranchCommitService( VcsClientProvider vcsClientProvider, AnalyzedCommitService analyzedCommitService, BranchRepository branchRepository + ) { + this(vcsClientProvider, analyzedCommitService, branchRepository, null); + } + + @Autowired + public BranchCommitService( + VcsClientProvider vcsClientProvider, + AnalyzedCommitService analyzedCommitService, + BranchRepository branchRepository, + ScmEvidenceService scmEvidenceService ) { this.vcsClientProvider = vcsClientProvider; this.analyzedCommitService = analyzedCommitService; this.branchRepository = branchRepository; + this.scmEvidenceService = scmEvidenceService; } /** @@ -82,7 +99,15 @@ public CommitRangeContext resolveCommitRange( // ── Same commit: nothing to do ─────────────────────────────────── if (lastKnownHead.equals(commitHash)) { // Check if this commit is already analyzed - if (analyzedCommitService.isAnalyzed(project.getId(), commitHash)) { + boolean multiBranch = project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); + boolean analyzed = multiBranch + ? analyzedCommitService.isAnalyzed( + project.getId(), targetBranchName, commitHash) + : analyzedCommitService.isAnalyzed( + project.getId(), commitHash); + if (analyzed) { log.info("HEAD commit {} is already analyzed — skipping", shortHash(commitHash)); return CommitRangeContext.skip(); } @@ -130,9 +155,41 @@ public CommitRangeContext resolveCommitRange( // Reverse to chronological order (oldest first) Collections.reverse(commitsSinceLastHead); + if (scmEvidenceService != null) { + try { + Map commitsByHash = commits.stream() + .collect(java.util.stream.Collectors.toMap( + VcsCommit::hash, + commit -> commit, + (first, ignored) -> first)); + List evidenceCommits = commitsSinceLastHead.stream() + .map(commitsByHash::get) + .filter(java.util.Objects::nonNull) + .toList(); + scmEvidenceService.capture( + project.getId(), vcsClient, workspace, slug, + evidenceCommits); + var promotion = scmEvidenceService.planPromotion( + project.getId(), commitsSinceLastHead, + targetBranchName, lastKnownHead); + log.info("SCM promotion evidence for branch {}: reuse={}, targetContextAnalysis={}", + targetBranchName, promotion.reuseKind(), + promotion.requiresTargetContextAnalysis()); + } catch (Exception evidenceFailure) { + log.warn("SCM evidence enrichment unavailable for branch {}: {}", + targetBranchName, evidenceFailure.getMessage()); + } + } + // Subtract already-analyzed commits - List unanalyzed = analyzedCommitService.filterUnanalyzed( - project.getId(), commitsSinceLastHead); + boolean multiBranch = project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); + List unanalyzed = multiBranch + ? analyzedCommitService.filterUnanalyzed( + project.getId(), targetBranchName, commitsSinceLastHead) + : analyzedCommitService.filterUnanalyzed( + project.getId(), commitsSinceLastHead); if (unanalyzed.isEmpty()) { log.info("All {} commits since lastKnownHead are already analyzed — skipping", diff --git a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageService.java b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageService.java index a49d7daf..50d68efa 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageService.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageService.java @@ -78,13 +78,30 @@ public enum CoverageStatus { */ public CoverageResult checkCoverage(Long projectId, String targetBranchName, List unanalyzedCommits) { + return checkCoverage(projectId, targetBranchName, unanalyzedCommits, false); + } + + /** + * Check coverage while optionally limiting durable receipts to the exact + * target branch. This prevents an analysis on {@code develop} from + * suppressing analysis of the same commits in a different {@code master} + * context. + */ + public CoverageResult checkCoverage(Long projectId, String targetBranchName, + List unanalyzedCommits, + boolean exactTargetBranch) { if (unanalyzedCommits == null || unanalyzedCommits.isEmpty()) { return new CoverageResult(CoverageStatus.FULLY_COVERED, Collections.emptyList()); } // Tier 1: Check the analyzed_commit table directly (covers both branch & PR analyses) - Set alreadyRecorded = analyzedCommitRepository - .findAnalyzedHashesByProjectIdAndCommitHashIn(projectId, unanalyzedCommits); + Set alreadyRecorded = exactTargetBranch + ? analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + projectId, targetBranchName, unanalyzedCommits) + : analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndCommitHashIn( + projectId, unanalyzedCommits); List notInTable = unanalyzedCommits.stream() .filter(h -> !alreadyRecorded.contains(h)) diff --git a/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java b/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java index 9472c5c8..696e056e 100644 --- a/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java +++ b/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java @@ -148,7 +148,7 @@ void resolveCommitRange_normalCase_shouldResolveNewCommits() throws Exception { VcsConnection conn = new VcsConnection(); when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); - when(vcsClient.getCommitHistory("ws", "repo", "main", 100)) + when(vcsClient.getCommitHistory("ws", "repo", "main", 1000)) .thenReturn(List.of( commit("new3"), commit("new2"), commit("new1"), commit("old-head") )); @@ -175,7 +175,7 @@ void resolveCommitRange_allCommitsAlreadyAnalyzed_shouldSkip() throws Exception VcsConnection conn = new VcsConnection(); when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); - when(vcsClient.getCommitHistory("ws", "repo", "main", 100)) + when(vcsClient.getCommitHistory("ws", "repo", "main", 1000)) .thenReturn(List.of(commit("new1"), commit("old-head"))); when(analyzedCommitService.filterUnanalyzed(eq(1L), anyList())) @@ -199,7 +199,7 @@ void resolveCommitRange_vcsReturnsNull_shouldFallBackToHeadOnly() throws Excepti VcsConnection conn = new VcsConnection(); when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); - when(vcsClient.getCommitHistory("ws", "repo", "main", 100)) + when(vcsClient.getCommitHistory("ws", "repo", "main", 1000)) .thenReturn(null); CommitRangeContext ctx = service.resolveCommitRange(project, conn, "main", "new-head"); @@ -220,7 +220,7 @@ void resolveCommitRange_vcsReturnsEmpty_shouldFallBackToHeadOnly() throws Except VcsConnection conn = new VcsConnection(); when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); - when(vcsClient.getCommitHistory("ws", "repo", "main", 100)) + when(vcsClient.getCommitHistory("ws", "repo", "main", 1000)) .thenReturn(List.of()); CommitRangeContext ctx = service.resolveCommitRange(project, conn, "main", "new-head"); @@ -240,7 +240,7 @@ void resolveCommitRange_lastKnownHeadNotInWindow_shouldFallBackToHeadOnly() thro VcsConnection conn = new VcsConnection(); when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); - when(vcsClient.getCommitHistory("ws", "repo", "main", 100)) + when(vcsClient.getCommitHistory("ws", "repo", "main", 1000)) .thenReturn(List.of(commit("c3"), commit("c2"), commit("c1"))); CommitRangeContext ctx = service.resolveCommitRange(project, conn, "main", "c3"); diff --git a/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageServiceTest.java b/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageServiceTest.java index 4b31362f..c86470e3 100644 --- a/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageServiceTest.java +++ b/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageServiceTest.java @@ -77,6 +77,25 @@ void checkCoverage_allInAnalyzedCommitTable_shouldReturnFullyCovered() { verifyNoInteractions(pullRequestRepository); // tier 2 never reached } + @Test + void checkCoverage_exactTarget_doesNotReuseDevelopReceiptForMaster() { + when(analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + 1L, "master", List.of("shared-commit"))) + .thenReturn(Set.of()); + when(pullRequestRepository.findByProjectIdAndTargetBranchNameAndStateIn( + eq(1L), eq("master"), anyList())).thenReturn(List.of()); + + CommitCoverageService.CoverageResult result = service.checkCoverage( + 1L, "master", List.of("shared-commit"), true); + + assertThat(result.status()) + .isEqualTo(CommitCoverageService.CoverageStatus.NOT_COVERED); + assertThat(result.uncoveredCommits()).containsExactly("shared-commit"); + verify(analyzedCommitRepository, never()) + .findAnalyzedHashesByProjectIdAndCommitHashIn(anyLong(), anyList()); + } + // ── Tier 2: PR coverage ────────────────────────────────────────────── @Test diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java index fa35ef06..e3c6098d 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java @@ -46,9 +46,39 @@ public record IssueDTO ( String issueScope, Integer endLineNumber, // Origin issue – the CodeAnalysisIssue this branch issue was cloned from (null for PR-level issues) - Long originIssueId + Long originIssueId, + // Deterministic introducing-commit provenance (distinct from PR/comment author) + String introducingCommitHash, + String introducingAuthorName, + String introducingAuthorEmail, + String authorProvenanceConfidence ) { + /** Backward-compatible constructor for callers predating SCM provenance. */ + public IssueDTO( + String id, String type, String severity, String title, + String description, String suggestedFixDescription, + String suggestedFixDiff, String file, Integer line, + Integer column, String rule, String branch, + String pullRequestId, String status, OffsetDateTime createdAt, + String issueCategory, Long analysisId, Long prNumber, + String commitHash, OffsetDateTime detectedAt, + String resolvedDescription, Long resolvedByPr, + String resolvedCommitHash, Long resolvedAnalysisId, + OffsetDateTime resolvedAt, String resolvedBy, + String vcsAuthorId, String vcsAuthorUsername, + String detectionSource, String issueScope, + Integer endLineNumber, Long originIssueId) { + this(id, type, severity, title, description, + suggestedFixDescription, suggestedFixDiff, file, line, + column, rule, branch, pullRequestId, status, createdAt, + issueCategory, analysisId, prNumber, commitHash, detectedAt, + resolvedDescription, resolvedByPr, resolvedCommitHash, + resolvedAnalysisId, resolvedAt, resolvedBy, vcsAuthorId, + vcsAuthorUsername, detectionSource, issueScope, + endLineNumber, originIssueId, null, null, null, null); + } + /** * Create an IssueDTO from an independent {@link BranchIssue}. * Reads all data from BranchIssue's own fields — never dereferences to CodeAnalysisIssue. @@ -108,7 +138,11 @@ public static IssueDTO fromBranchIssue(BranchIssue bi) { bi.getIssueScope() != null ? bi.getIssueScope().name() : null, bi.getCurrentEndLineNumber() != null ? bi.getCurrentEndLineNumber() : bi.getEndLineNumber(), - bi.getOriginIssue() != null ? bi.getOriginIssue().getId() : null + bi.getOriginIssue() != null ? bi.getOriginIssue().getId() : null, + bi.getIntroducingCommitHash(), + bi.getIntroducingAuthorName(), + bi.getIntroducingAuthorEmail(), + bi.getAuthorProvenanceConfidence() ); } @@ -162,7 +196,11 @@ public static IssueDTO fromEntity(CodeAnalysisIssue issue) { issue.getDetectionSource() != null ? issue.getDetectionSource().name() : null, issue.getIssueScope() != null ? issue.getIssueScope().name() : null, issue.getEndLineNumber(), - null // CodeAnalysisIssue has no origin + null, // CodeAnalysisIssue has no origin + issue.getIntroducingCommitHash(), + issue.getIntroducingAuthorName(), + issue.getIntroducingAuthorEmail(), + issue.getAuthorProvenanceConfidence() ); } } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java index 20d54bfa..8854b757 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java @@ -118,7 +118,9 @@ public static ProjectDTO fromProject(Project project) { rc.includePatterns(), rc.excludePatterns(), rc.multiBranchEnabled(), - rc.branchRetentionDays()); + rc.branchRetentionDays(), + rc.indexedBranches(), + rc.transientBranchIndexesEnabled()); } if (config.prAnalysisEnabled() != null) { prAnalysisEnabled = config.prAnalysisEnabled(); @@ -212,13 +214,26 @@ public record RagConfigDTO( java.util.List includePatterns, java.util.List excludePatterns, Boolean multiBranchEnabled, - Integer branchRetentionDays) { + Integer branchRetentionDays, + java.util.List indexedBranches, + Boolean transientBranchIndexesEnabled) { + public RagConfigDTO( + boolean enabled, + String branch, + java.util.List includePatterns, + java.util.List excludePatterns, + Boolean multiBranchEnabled, + Integer branchRetentionDays) { + this(enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, + branchRetentionDays, null, null); + } + /** * Backward-compatible constructor without include patterns and multi-branch * fields. */ public RagConfigDTO(boolean enabled, String branch, java.util.List excludePatterns) { - this(enabled, branch, null, excludePatterns, null, null); + this(enabled, branch, null, excludePatterns, null, null, null, null); } } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java index 45997213..21bbcb13 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java @@ -136,6 +136,18 @@ public class BranchIssue implements ReconcilableIssue { @Column(name = "vcs_author_username", length = 100) private String vcsAuthorUsername; + @Column(name = "introducing_commit_hash", length = 64) + private String introducingCommitHash; + + @Column(name = "introducing_author_name", length = 200) + private String introducingAuthorName; + + @Column(name = "introducing_author_email", length = 320) + private String introducingAuthorEmail; + + @Column(name = "author_provenance_confidence", length = 32) + private String authorProvenanceConfidence; + // ── Content-based tracking fields ─────────────────────────────────── /** MD5 hex of the whitespace-normalized source line at detection time. */ @@ -242,6 +254,10 @@ public static BranchIssue fromCodeAnalysisIssue(CodeAnalysisIssue cai, Branch br // VCS author bi.setVcsAuthorId(cai.getVcsAuthorId()); bi.setVcsAuthorUsername(cai.getVcsAuthorUsername()); + bi.setIntroducingCommitHash(cai.getIntroducingCommitHash()); + bi.setIntroducingAuthorName(cai.getIntroducingAuthorName()); + bi.setIntroducingAuthorEmail(cai.getIntroducingAuthorEmail()); + bi.setAuthorProvenanceConfidence(cai.getAuthorProvenanceConfidence()); // Tracking hashes bi.setLineHash(cai.getLineHash()); @@ -387,6 +403,15 @@ public void setResolved(boolean resolved) { public String getVcsAuthorUsername() { return vcsAuthorUsername; } public void setVcsAuthorUsername(String vcsAuthorUsername) { this.vcsAuthorUsername = vcsAuthorUsername; } + public String getIntroducingCommitHash() { return introducingCommitHash; } + public void setIntroducingCommitHash(String introducingCommitHash) { this.introducingCommitHash = introducingCommitHash; } + public String getIntroducingAuthorName() { return introducingAuthorName; } + public void setIntroducingAuthorName(String introducingAuthorName) { this.introducingAuthorName = introducingAuthorName; } + public String getIntroducingAuthorEmail() { return introducingAuthorEmail; } + public void setIntroducingAuthorEmail(String introducingAuthorEmail) { this.introducingAuthorEmail = introducingAuthorEmail; } + public String getAuthorProvenanceConfidence() { return authorProvenanceConfidence; } + public void setAuthorProvenanceConfidence(String authorProvenanceConfidence) { this.authorProvenanceConfidence = authorProvenanceConfidence; } + public String getLineHash() { return lineHash; } public void setLineHash(String lineHash) { this.lineHash = lineHash; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java index 577ad0d9..e161d677 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java @@ -87,6 +87,18 @@ public class CodeAnalysisIssue implements ReconcilableIssue { @Column(name = "vcs_author_username", length = 100) private String vcsAuthorUsername; + @Column(name = "introducing_commit_hash", length = 64) + private String introducingCommitHash; + + @Column(name = "introducing_author_name", length = 200) + private String introducingAuthorName; + + @Column(name = "introducing_author_email", length = 320) + private String introducingAuthorEmail; + + @Column(name = "author_provenance_confidence", length = 32) + private String authorProvenanceConfidence; + // --- Content-based tracking fields --- /** MD5 hex of the whitespace-normalized source line at detection time. */ @@ -213,6 +225,15 @@ public class CodeAnalysisIssue implements ReconcilableIssue { public String getVcsAuthorUsername() { return vcsAuthorUsername; } public void setVcsAuthorUsername(String vcsAuthorUsername) { this.vcsAuthorUsername = vcsAuthorUsername; } + public String getIntroducingCommitHash() { return introducingCommitHash; } + public void setIntroducingCommitHash(String introducingCommitHash) { this.introducingCommitHash = introducingCommitHash; } + public String getIntroducingAuthorName() { return introducingAuthorName; } + public void setIntroducingAuthorName(String introducingAuthorName) { this.introducingAuthorName = introducingAuthorName; } + public String getIntroducingAuthorEmail() { return introducingAuthorEmail; } + public void setIntroducingAuthorEmail(String introducingAuthorEmail) { this.introducingAuthorEmail = introducingAuthorEmail; } + public String getAuthorProvenanceConfidence() { return authorProvenanceConfidence; } + public void setAuthorProvenanceConfidence(String authorProvenanceConfidence) { this.authorProvenanceConfidence = authorProvenanceConfidence; } + public String getLineHash() { return lineHash; } public void setLineHash(String lineHash) { this.lineHash = lineHash; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java index 7af91c0f..08bc1409 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java @@ -278,7 +278,9 @@ public void setMainBranch(String mainBranch) { this.ragConfig.includePatterns(), this.ragConfig.excludePatterns(), this.ragConfig.multiBranchEnabled(), - this.ragConfig.branchRetentionDays()); + this.ragConfig.branchRetentionDays(), + this.ragConfig.indexedBranches(), + this.ragConfig.transientBranchIndexesEnabled()); } } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java index 014d69fc..43132e6e 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java @@ -1,5 +1,6 @@ package org.rostilos.codecrow.core.model.project.config; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; @@ -18,6 +19,10 @@ * be retained. PR retrieval still selects only the immutable VCS target branch; * source changes come from the exact PR overlay rather than a second branch. * - branchRetentionDays: how long to keep branch index metadata before auto-cleanup (default: 90 days) + * - indexedBranches: explicit non-primary branches whose complete snapshots are retained. + * A null/empty value preserves the legacy branchPushPatterns interpretation. + * - transientBranchIndexesEnabled: whether an analyzed PR target that is not retained + * may receive a revision-pinned temporary snapshot. */ @JsonIgnoreProperties(ignoreUnknown = true) public record RagConfig( @@ -26,33 +31,51 @@ public record RagConfig( @JsonProperty("includePatterns") List includePatterns, @JsonProperty("excludePatterns") List excludePatterns, @JsonProperty("multiBranchEnabled") Boolean multiBranchEnabled, - @JsonProperty("branchRetentionDays") Integer branchRetentionDays + @JsonProperty("branchRetentionDays") Integer branchRetentionDays, + @JsonProperty("indexedBranches") List indexedBranches, + @JsonProperty("transientBranchIndexesEnabled") Boolean transientBranchIndexesEnabled ) { public static final int DEFAULT_BRANCH_RETENTION_DAYS = 90; public RagConfig() { - this(false, null, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS); + this(false, null, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); } public RagConfig(boolean enabled) { - this(enabled, null, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS); + this(enabled, null, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); } public RagConfig(boolean enabled, String branch) { - this(enabled, branch, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS); + this(enabled, branch, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); } public RagConfig(boolean enabled, String branch, List excludePatterns) { - this(enabled, branch, null, excludePatterns, false, DEFAULT_BRANCH_RETENTION_DAYS); + this(enabled, branch, null, excludePatterns, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); } public RagConfig(boolean enabled, String branch, List includePatterns, List excludePatterns) { - this(enabled, branch, includePatterns, excludePatterns, false, DEFAULT_BRANCH_RETENTION_DAYS); + this(enabled, branch, includePatterns, excludePatterns, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); + } + + /** + * Backward-compatible constructor for configurations written before explicit + * retained and transient branch ownership was introduced. + */ + public RagConfig( + boolean enabled, + String branch, + List includePatterns, + List excludePatterns, + Boolean multiBranchEnabled, + Integer branchRetentionDays) { + this(enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, + branchRetentionDays, null, false); } /** * Check if multi-branch context is enabled for PR analysis. */ + @JsonIgnore public boolean isMultiBranchEnabled() { return multiBranchEnabled != null && multiBranchEnabled; } @@ -60,9 +83,32 @@ public boolean isMultiBranchEnabled() { /** * Get effective branch retention days. */ + @JsonIgnore public int getEffectiveBranchRetentionDays() { return branchRetentionDays != null ? branchRetentionDays : DEFAULT_BRANCH_RETENTION_DAYS; } + + public boolean hasExplicitIndexedBranches() { + return indexedBranches != null && indexedBranches.stream() + .anyMatch(value -> value != null && !value.isBlank()); + } + + @JsonIgnore + public List getEffectiveIndexedBranches() { + if (indexedBranches == null) { + return List.of(); + } + return indexedBranches.stream() + .filter(value -> value != null && !value.isBlank()) + .map(String::trim) + .distinct() + .toList(); + } + + @JsonIgnore + public boolean isTransientBranchIndexesEnabled() { + return isMultiBranchEnabled() && Boolean.TRUE.equals(transientBranchIndexesEnabled); + } /** * Check if a branch should have indexed context based on branchPushPatterns. @@ -71,7 +117,13 @@ public int getEffectiveBranchRetentionDays() { * @return true if branch matches any pattern and multi-branch is enabled */ public boolean shouldHaveBranchIndex(String branchName, List branchPushPatterns) { - if (!isMultiBranchEnabled() || branchPushPatterns == null || branchPushPatterns.isEmpty()) { + if (!isMultiBranchEnabled() || branchName == null || branchName.isBlank()) { + return false; + } + if (hasExplicitIndexedBranches()) { + return getEffectiveIndexedBranches().contains(branchName.trim()); + } + if (branchPushPatterns == null || branchPushPatterns.isEmpty()) { return false; } return branchPushPatterns.stream() diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java index 8cea6307..89c5b5c9 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java @@ -48,6 +48,27 @@ public class RagBranchIndex { @Column(name = "commit_hash", length = 64) private String commitHash; + @Enumerated(EnumType.STRING) + @Column(name = "index_kind", nullable = false, length = 24) + private RagBranchIndexKind indexKind = RagBranchIndexKind.LEGACY; + + @Enumerated(EnumType.STRING) + @Column(name = "lifecycle_status", nullable = false, length = 24) + private RagBranchIndexLifecycleStatus lifecycleStatus = RagBranchIndexLifecycleStatus.READY; + + @Column(name = "desired_commit_hash", length = 64) + private String desiredCommitHash; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "active_generation_id") + private RagBranchIndexGeneration activeGeneration; + + @Column(name = "last_accessed_at") + private OffsetDateTime lastAccessedAt; + + @Column(name = "error_message", columnDefinition = "TEXT") + private String errorMessage; + /** * Files that were deleted in this branch (for query-time filtering). * These files should be excluded when querying the branch's context. @@ -82,6 +103,39 @@ public RagBranchIndex(Project project, String branchName) { this.branchName = branchName; } + public RagBranchIndex(Project project, String branchName, RagBranchIndexKind indexKind) { + this(project, branchName); + this.indexKind = indexKind; + this.lifecycleStatus = RagBranchIndexLifecycleStatus.PENDING; + } + + public void requestRevision(String desiredCommitHash) { + this.desiredCommitHash = desiredCommitHash; + this.lifecycleStatus = RagBranchIndexLifecycleStatus.BUILDING; + this.errorMessage = null; + } + + public void activate(RagBranchIndexGeneration generation) { + this.activeGeneration = generation; + this.commitHash = generation.getRevision(); + this.desiredCommitHash = generation.getRevision(); + this.chunkCount = generation.getChunkCount(); + this.lifecycleStatus = RagBranchIndexLifecycleStatus.READY; + this.errorMessage = null; + this.lastAccessedAt = OffsetDateTime.now(); + } + + public void failUpdate(String errorMessage) { + this.lifecycleStatus = activeGeneration == null + ? RagBranchIndexLifecycleStatus.FAILED + : RagBranchIndexLifecycleStatus.READY; + this.errorMessage = errorMessage; + } + + public void markAccessed() { + this.lastAccessedAt = OffsetDateTime.now(); + } + public Long getId() { return id; } @@ -114,6 +168,54 @@ public void setCommitHash(String commitHash) { this.commitHash = commitHash; } + public RagBranchIndexKind getIndexKind() { + return indexKind; + } + + public void setIndexKind(RagBranchIndexKind indexKind) { + this.indexKind = indexKind; + } + + public RagBranchIndexLifecycleStatus getLifecycleStatus() { + return lifecycleStatus; + } + + public void setLifecycleStatus(RagBranchIndexLifecycleStatus lifecycleStatus) { + this.lifecycleStatus = lifecycleStatus; + } + + public String getDesiredCommitHash() { + return desiredCommitHash; + } + + public void setDesiredCommitHash(String desiredCommitHash) { + this.desiredCommitHash = desiredCommitHash; + } + + public RagBranchIndexGeneration getActiveGeneration() { + return activeGeneration; + } + + public void setActiveGeneration(RagBranchIndexGeneration activeGeneration) { + this.activeGeneration = activeGeneration; + } + + public OffsetDateTime getLastAccessedAt() { + return lastAccessedAt; + } + + public void setLastAccessedAt(OffsetDateTime lastAccessedAt) { + this.lastAccessedAt = lastAccessedAt; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + public Set getDeletedFiles() { return deletedFiles; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGeneration.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGeneration.java new file mode 100644 index 00000000..9bb40f90 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGeneration.java @@ -0,0 +1,149 @@ +package org.rostilos.codecrow.core.model.rag; + +import jakarta.persistence.*; + +import java.time.OffsetDateTime; + +/** + * One immutable physical representation of a branch at an exact repository + * revision. A generation is published by pointing its owning branch index at it; + * failed or incomplete generations never replace the active generation. + */ +@Entity +@Table(name = "rag_branch_index_generation", indexes = { + @Index(name = "idx_rag_branch_generation_revision", columnList = "branch_index_id, revision"), + @Index(name = "idx_rag_branch_generation_status", columnList = "branch_index_id, status") +}) +public class RagBranchIndexGeneration { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "branch_index_id", nullable = false) + private RagBranchIndex branchIndex; + + @Column(name = "revision", nullable = false, length = 64, updatable = false) + private String revision; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "parent_generation_id", updatable = false) + private RagBranchIndexGeneration parentGeneration; + + @Column(name = "seed_revision", length = 64, updatable = false) + private String seedRevision; + + @Column(name = "collection_name", nullable = false, length = 300, updatable = false) + private String collectionName; + + @Enumerated(EnumType.STRING) + @Column(name = "status", nullable = false, length = 24) + private RagBranchIndexGenerationStatus status = RagBranchIndexGenerationStatus.BUILDING; + + @Column(name = "manifest_digest", length = 128) + private String manifestDigest; + + @Column(name = "representation_fingerprint", length = 128, updatable = false) + private String representationFingerprint; + + @Column(name = "file_count") + private Integer fileCount; + + @Column(name = "chunk_count") + private Integer chunkCount; + + @Column(name = "created_at", nullable = false, updatable = false) + private OffsetDateTime createdAt = OffsetDateTime.now(); + + @Column(name = "activated_at") + private OffsetDateTime activatedAt; + + @Column(name = "superseded_at") + private OffsetDateTime supersededAt; + + @Column(name = "error_message", columnDefinition = "TEXT") + private String errorMessage; + + public RagBranchIndexGeneration() { + } + + public RagBranchIndexGeneration( + RagBranchIndex branchIndex, + String revision, + String collectionName, + RagBranchIndexGeneration parentGeneration, + String seedRevision, + String representationFingerprint) { + this.branchIndex = branchIndex; + this.revision = revision; + this.collectionName = collectionName; + this.parentGeneration = parentGeneration; + this.seedRevision = seedRevision; + this.representationFingerprint = representationFingerprint; + } + + public void activate(String manifestDigest, int fileCount, int chunkCount) { + this.manifestDigest = manifestDigest; + this.fileCount = fileCount; + this.chunkCount = chunkCount; + this.status = RagBranchIndexGenerationStatus.ACTIVE; + this.activatedAt = OffsetDateTime.now(); + this.errorMessage = null; + } + + public void supersede() { + this.status = RagBranchIndexGenerationStatus.SUPERSEDED; + this.supersededAt = OffsetDateTime.now(); + } + + public void fail(String errorMessage) { + this.status = RagBranchIndexGenerationStatus.FAILED; + this.errorMessage = errorMessage; + } + + /** Reopen an unpublished failed generation for an idempotent retry. */ + public void retry() { + if (status != RagBranchIndexGenerationStatus.FAILED) { + throw new IllegalStateException("Only a failed generation can be retried"); + } + status = RagBranchIndexGenerationStatus.BUILDING; + errorMessage = null; + activatedAt = null; + supersededAt = null; + manifestDigest = null; + fileCount = null; + chunkCount = null; + } + + public Long getId() { return id; } + public void setId(Long id) { this.id = id; } + public RagBranchIndex getBranchIndex() { return branchIndex; } + public void setBranchIndex(RagBranchIndex branchIndex) { this.branchIndex = branchIndex; } + public String getRevision() { return revision; } + public void setRevision(String revision) { this.revision = revision; } + public RagBranchIndexGeneration getParentGeneration() { return parentGeneration; } + public void setParentGeneration(RagBranchIndexGeneration parentGeneration) { this.parentGeneration = parentGeneration; } + public String getSeedRevision() { return seedRevision; } + public void setSeedRevision(String seedRevision) { this.seedRevision = seedRevision; } + public String getCollectionName() { return collectionName; } + public void setCollectionName(String collectionName) { this.collectionName = collectionName; } + public RagBranchIndexGenerationStatus getStatus() { return status; } + public void setStatus(RagBranchIndexGenerationStatus status) { this.status = status; } + public String getManifestDigest() { return manifestDigest; } + public void setManifestDigest(String manifestDigest) { this.manifestDigest = manifestDigest; } + public String getRepresentationFingerprint() { return representationFingerprint; } + public void setRepresentationFingerprint(String representationFingerprint) { this.representationFingerprint = representationFingerprint; } + public Integer getFileCount() { return fileCount; } + public void setFileCount(Integer fileCount) { this.fileCount = fileCount; } + public Integer getChunkCount() { return chunkCount; } + public void setChunkCount(Integer chunkCount) { this.chunkCount = chunkCount; } + public OffsetDateTime getCreatedAt() { return createdAt; } + public void setCreatedAt(OffsetDateTime createdAt) { this.createdAt = createdAt; } + public OffsetDateTime getActivatedAt() { return activatedAt; } + public void setActivatedAt(OffsetDateTime activatedAt) { this.activatedAt = activatedAt; } + public OffsetDateTime getSupersededAt() { return supersededAt; } + public void setSupersededAt(OffsetDateTime supersededAt) { this.supersededAt = supersededAt; } + public String getErrorMessage() { return errorMessage; } + public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationStatus.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationStatus.java new file mode 100644 index 00000000..0545531b --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationStatus.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.core.model.rag; + +/** Immutable generation publication state. */ +public enum RagBranchIndexGenerationStatus { + BUILDING, + ACTIVE, + SUPERSEDED, + FAILED +} + diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexKind.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexKind.java new file mode 100644 index 00000000..3c652844 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexKind.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.core.model.rag; + +/** Ownership and retention policy for a branch index. */ +public enum RagBranchIndexKind { + PRIMARY, + DURABLE, + TRANSIENT, + LEGACY +} + diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexLifecycleStatus.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexLifecycleStatus.java new file mode 100644 index 00000000..9053a65d --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexLifecycleStatus.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.core.model.rag; + +/** Branch-level readiness independent from any individual build attempt. */ +public enum RagBranchIndexLifecycleStatus { + PENDING, + BUILDING, + READY, + FAILED +} + diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperation.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperation.java new file mode 100644 index 00000000..120b59c1 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperation.java @@ -0,0 +1,137 @@ +package org.rostilos.codecrow.core.model.rag; + +import jakarta.persistence.*; +import org.rostilos.codecrow.core.model.project.Project; + +import java.time.OffsetDateTime; + +/** Durable, idempotent request to move one branch index to a desired revision. */ +@Entity +@Table(name = "rag_index_operation", uniqueConstraints = { + @UniqueConstraint(name = "uq_rag_index_operation_key", columnNames = {"project_id", "operation_key"}) +}) +public class RagIndexOperation { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "project_id", nullable = false) + private Project project; + + @Column(name = "branch_name", nullable = false, length = 256) + private String branchName; + + @Column(name = "from_revision", length = 64) + private String fromRevision; + + @Column(name = "to_revision", nullable = false, length = 64) + private String toRevision; + + @Column(name = "operation_key", nullable = false, length = 128) + private String operationKey; + + @Enumerated(EnumType.STRING) + @Column(name = "status", nullable = false, length = 24) + private RagIndexOperationStatus status = RagIndexOperationStatus.PENDING; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "generation_id") + private RagBranchIndexGeneration generation; + + @Column(name = "job_id") + private Long jobId; + + @Column(name = "attempt_count", nullable = false) + private int attemptCount; + + @Column(name = "created_at", nullable = false, updatable = false) + private OffsetDateTime createdAt = OffsetDateTime.now(); + + @Column(name = "updated_at", nullable = false) + private OffsetDateTime updatedAt = OffsetDateTime.now(); + + @Column(name = "completed_at") + private OffsetDateTime completedAt; + + @Column(name = "error_message", columnDefinition = "TEXT") + private String errorMessage; + + public RagIndexOperation() { + } + + public RagIndexOperation( + Project project, + String branchName, + String fromRevision, + String toRevision, + String operationKey) { + this.project = project; + this.branchName = branchName; + this.fromRevision = fromRevision; + this.toRevision = toRevision; + this.operationKey = operationKey; + } + + @PreUpdate + void onUpdate() { + updatedAt = OffsetDateTime.now(); + } + + public void start() { + status = RagIndexOperationStatus.RUNNING; + attemptCount++; + errorMessage = null; + completedAt = null; + } + + public void succeed(RagBranchIndexGeneration generation) { + this.generation = generation; + status = RagIndexOperationStatus.SUCCEEDED; + completedAt = OffsetDateTime.now(); + errorMessage = null; + } + + public void fail(String errorMessage) { + status = RagIndexOperationStatus.FAILED; + completedAt = OffsetDateTime.now(); + this.errorMessage = errorMessage; + } + + public void heartbeat() { + if (status == RagIndexOperationStatus.PENDING + || status == RagIndexOperationStatus.RUNNING) { + updatedAt = OffsetDateTime.now(); + } + } + + public Long getId() { return id; } + public void setId(Long id) { this.id = id; } + public Project getProject() { return project; } + public void setProject(Project project) { this.project = project; } + public String getBranchName() { return branchName; } + public void setBranchName(String branchName) { this.branchName = branchName; } + public String getFromRevision() { return fromRevision; } + public void setFromRevision(String fromRevision) { this.fromRevision = fromRevision; } + public String getToRevision() { return toRevision; } + public void setToRevision(String toRevision) { this.toRevision = toRevision; } + public String getOperationKey() { return operationKey; } + public void setOperationKey(String operationKey) { this.operationKey = operationKey; } + public RagIndexOperationStatus getStatus() { return status; } + public void setStatus(RagIndexOperationStatus status) { this.status = status; } + public RagBranchIndexGeneration getGeneration() { return generation; } + public void setGeneration(RagBranchIndexGeneration generation) { this.generation = generation; } + public Long getJobId() { return jobId; } + public void setJobId(Long jobId) { this.jobId = jobId; } + public int getAttemptCount() { return attemptCount; } + public void setAttemptCount(int attemptCount) { this.attemptCount = attemptCount; } + public OffsetDateTime getCreatedAt() { return createdAt; } + public void setCreatedAt(OffsetDateTime createdAt) { this.createdAt = createdAt; } + public OffsetDateTime getUpdatedAt() { return updatedAt; } + public void setUpdatedAt(OffsetDateTime updatedAt) { this.updatedAt = updatedAt; } + public OffsetDateTime getCompletedAt() { return completedAt; } + public void setCompletedAt(OffsetDateTime completedAt) { this.completedAt = completedAt; } + public String getErrorMessage() { return errorMessage; } + public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperationStatus.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperationStatus.java new file mode 100644 index 00000000..42500fa3 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperationStatus.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.core.model.rag; + +/** Durable branch-index operation state used for restart recovery. */ +public enum RagIndexOperationStatus { + PENDING, + RUNNING, + SUCCEEDED, + FAILED +} + diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java index 8712ae2e..28570720 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java @@ -125,6 +125,26 @@ boolean existsActivePrAnalysisJobBefore( @Param("beforeJobId") Long beforeJobId ); + /** + * A PR accepted after a target-branch update must wait until that older + * branch job has reconciled repository state and published its RAG update. + * Later branch jobs wait for the PR instead, so the persisted job id gives + * both directions one deadlock-free ordering rule. + */ + @Query("SELECT CASE WHEN COUNT(j) > 0 THEN true ELSE false END FROM Job j " + + "WHERE j.project.id = :projectId AND j.branchName = :branchName " + + "AND j.jobType = org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS " + + "AND j.id < :beforeJobId " + + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + + "org.rostilos.codecrow.core.model.job.JobStatus.RUNNING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.WAITING)") + boolean existsActiveBranchAnalysisJobBefore( + @Param("projectId") Long projectId, + @Param("branchName") String branchName, + @Param("beforeJobId") Long beforeJobId + ); + /** * Return only the newest analysis attempt for a PR. An abandoned older * attempt must not poison branch reconciliation after a newer attempt has diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java new file mode 100644 index 00000000..a7347032 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java @@ -0,0 +1,32 @@ +package org.rostilos.codecrow.core.persistence.repository.rag; + +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Optional; + +@Repository +public interface RagBranchIndexGenerationRepository extends JpaRepository { + + Optional findFirstByBranchIndexIdAndRevisionAndStatusInOrderByCreatedAtDesc( + Long branchIndexId, + String revision, + List statuses); + + List findByBranchIndexIdOrderByCreatedAtDesc(Long branchIndexId); + + @Query("SELECT g FROM RagBranchIndexGeneration g " + + "JOIN g.branchIndex b WHERE b.project.id = :projectId " + + "AND b.branchName = :branchName AND g.revision = :revision " + + "AND g.status IN :statuses ORDER BY g.createdAt DESC") + List findAvailableExactGeneration( + @Param("projectId") Long projectId, + @Param("branchName") String branchName, + @Param("revision") String revision, + @Param("statuses") List statuses); +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java index 09743ea0..5660ee45 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java @@ -1,7 +1,10 @@ package org.rostilos.codecrow.core.persistence.repository.rag; +import jakarta.persistence.LockModeType; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -18,8 +21,20 @@ public interface RagBranchIndexRepository extends JpaRepository findByProjectIdAndBranchName(Long projectId, String branchName); + @Lock(LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT b FROM RagBranchIndex b WHERE b.project.id = :projectId AND b.branchName = :branchName") + Optional findByProjectIdAndBranchNameForUpdate( + @Param("projectId") Long projectId, + @Param("branchName") String branchName); + + @Lock(LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT b FROM RagBranchIndex b WHERE b.id = :id") + Optional findByIdForPublication(@Param("id") Long id); + List findByProjectId(Long projectId); + List findByIndexKind(RagBranchIndexKind indexKind); + @Query("SELECT CASE WHEN COUNT(b) > 0 THEN true ELSE false END FROM RagBranchIndex b " + "WHERE b.project.id = :projectId AND b.branchName = :branchName") boolean existsByProjectIdAndBranchName(@Param("projectId") Long projectId, @Param("branchName") String branchName); diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java new file mode 100644 index 00000000..92d4b4c1 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java @@ -0,0 +1,20 @@ +package org.rostilos.codecrow.core.persistence.repository.rag; + +import org.rostilos.codecrow.core.model.rag.RagIndexOperation; +import org.rostilos.codecrow.core.model.rag.RagIndexOperationStatus; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; + +@Repository +public interface RagIndexOperationRepository extends JpaRepository { + + Optional findByProjectIdAndOperationKey(Long projectId, String operationKey); + + List findByStatusInAndUpdatedAtBefore( + List statuses, + OffsetDateTime updatedBefore); +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java index 4bfc6618..043b4fbe 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java @@ -32,6 +32,26 @@ public interface AnalysisJobService { */ Job createRagIndexJob(Project project, boolean isInitial, JobTriggerSource triggerSource); + /** + * Create a branch-bound RAG indexing job. Hosts that persist jobs should + * override this method so the branch and revision are durable before the + * worker starts; the default keeps non-persistent host implementations + * source-compatible. + */ + default Job createRagIndexJob( + Project project, + boolean isInitial, + JobTriggerSource triggerSource, + String branchName, + String commitHash) { + Job job = createRagIndexJob(project, isInitial, triggerSource); + if (job != null) { + job.setBranchName(branchName); + job.setCommitHash(commitHash); + } + return job; + } + /** * Start a job. * @param job The job to start diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java index aea9361b..414ad18f 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java @@ -519,6 +519,10 @@ public CodeAnalysis cloneAnalysisForPr( issueClone.setResolvedDescription(srcIssue.getResolvedDescription()); issueClone.setVcsAuthorId(srcIssue.getVcsAuthorId()); issueClone.setVcsAuthorUsername(srcIssue.getVcsAuthorUsername()); + issueClone.setIntroducingCommitHash(srcIssue.getIntroducingCommitHash()); + issueClone.setIntroducingAuthorName(srcIssue.getIntroducingAuthorName()); + issueClone.setIntroducingAuthorEmail(srcIssue.getIntroducingAuthorEmail()); + issueClone.setAuthorProvenanceConfidence(srcIssue.getAuthorProvenanceConfidence()); // Copy content-based tracking hashes issueClone.setLineHash(srcIssue.getLineHash()); issueClone.setLineHashContext(srcIssue.getLineHashContext()); diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java index fd530e29..9ae74079 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java @@ -21,6 +21,9 @@ *

*

De-duplication identities

*
    + *
  1. Exact narrative identity — same normalized title and complete + * root-cause reason in the same file. Category, severity, and a stale line + * anchor do not make the same finding independent.
    2. *
  2. Issue fingerprint — exact match on * {@link org.rostilos.codecrow.core.util.tracking.IssueFingerprint} * (category + anchored line-content hash + normalized title).
    3. @@ -38,7 +41,8 @@ *
      *
    • The highest severity among the group
      • *
    • The best (longest valid) suggested fix diff
      • - *
    • The lowest line number (most specific location)
      • + *
    • The strongest concrete source anchor
      • + *
    • Additional concrete locations for exact narrative occurrences
      • *
    * *

    Thread safety

    @@ -105,6 +109,10 @@ public List deduplicateAtIngestion(List is int before = active.size(); + // Exact root narrative catches duplicated history/current output even + // when line anchors and classifications drifted before ingestion. + active = exactNarrativeDedup(active, filePath); + // Exact category-aware identity. active = fingerprintDedup(active, filePath); @@ -130,7 +138,127 @@ public List deduplicateAtIngestion(List is return result; } - // ── Tier 1: category-aware fingerprint ─────────────────────────────── + // ── Tier 1: complete root-narrative identity ───────────────────────── + + private List exactNarrativeDedup( + List issues, + String filePath + ) { + if (issues.size() < 2) { + return issues; + } + + Map byNarrative = new LinkedHashMap<>(); + int noIdentityOrdinal = 0; + int duplicateCount = 0; + for (CodeAnalysisIssue issue : issues) { + String title = normalizeNarrative(issue.getTitle()); + String reason = normalizeNarrative(rootReason(issue.getReason())); + // Short boilerplate (for example "same issue" / "test reason") is + // not a safe root identity. The inference tier handles ambiguous + // prose; this ingestion fallback requires a complete explanation. + if (title.isBlank() || reason.length() < 40) { + byNarrative.put("__no_narrative_" + noIdentityOrdinal++, issue); + continue; + } + + String identity = title + "\u0000" + reason; + CodeAnalysisIssue existing = byNarrative.get(identity); + if (existing == null) { + byNarrative.put(identity, issue); + continue; + } + + CodeAnalysisIssue winner = pickBest(existing, issue); + preserveAdditionalNarrativeLocation(winner, existing, issue); + byNarrative.put(identity, winner); + duplicateCount++; + } + + if (duplicateCount > 0) { + log.debug("Exact-narrative dedup: removed {} in {}", duplicateCount, filePath); + } + return new ArrayList<>(byNarrative.values()); + } + + private String normalizeNarrative(String value) { + if (value == null) { + return ""; + } + return value.strip().replaceAll("\\s+", " ").toLowerCase(Locale.ROOT); + } + + private String rootReason(String reason) { + if (reason == null || reason.isBlank()) { + return ""; + } + return Arrays.stream(reason.split("\\R")) + .filter(line -> !line.strip().toLowerCase(Locale.ROOT).startsWith("also affects:")) + .reduce((left, right) -> left + "\n" + right) + .orElse("") + .strip(); + } + + private void preserveAdditionalNarrativeLocation( + CodeAnalysisIssue winner, + CodeAnalysisIssue first, + CodeAnalysisIssue second + ) { + Set locations = new TreeSet<>(); + collectGeneratedLocations(first.getReason(), locations); + collectGeneratedLocations(second.getReason(), locations); + addConcreteSecondaryLocation(winner, first, locations); + addConcreteSecondaryLocation(winner, second, locations); + + String primary = issueLocation(winner); + locations.remove(primary); + String baseReason = rootReason(winner.getReason()); + winner.setReason(locations.isEmpty() + ? baseReason + : baseReason + "\n\nAlso affects: " + String.join(", ", locations)); + } + + private void collectGeneratedLocations(String reason, Set locations) { + if (reason == null) { + return; + } + for (String line : reason.split("\\R")) { + String stripped = line.strip(); + if (!stripped.toLowerCase(Locale.ROOT).startsWith("also affects:")) { + continue; + } + String value = stripped.substring(stripped.indexOf(':') + 1); + Arrays.stream(value.split(",")) + .map(String::strip) + .filter(item -> !item.isBlank()) + .forEach(locations::add); + } + } + + private void addConcreteSecondaryLocation( + CodeAnalysisIssue winner, + CodeAnalysisIssue candidate, + Set locations + ) { + if (candidate == winner || candidate.getLineNumber() == null + || candidate.getLineNumber() <= 1) { + // Line 1 is the legacy FILE-scope/stale-anchor fallback and is not + // safe to publish as an additional concrete occurrence. + return; + } + String location = issueLocation(candidate); + if (!location.isBlank()) { + locations.add(location); + } + } + + private String issueLocation(CodeAnalysisIssue issue) { + String file = issue.getFilePath() != null ? issue.getFilePath() : ""; + Integer line = issue.getLineNumber(); + return line != null && line > 0 ? file + ":" + line : file; + } + + // ── Tier 2: category-aware fingerprint ─────────────────────────────── /** * De-duplicate by issue fingerprint — catches issues at different lines @@ -214,38 +342,44 @@ private List contentFingerprintDedup(List /** * Pick the best representation of two issues that already have the same exact - * identity. Higher severity wins, then the longer valid diff, then the lower - * anchored line number. This method never decides whether two issues are equal. + * identity, then promote useful metadata from the other representation. This + * method never decides whether two issues are equal. */ private CodeAnalysisIssue pickBest(CodeAnalysisIssue a, CodeAnalysisIssue b) { - int sevA = severityRank(a.getSeverity()); - int sevB = severityRank(b.getSeverity()); - if (sevA != sevB) { - return sevA >= sevB ? promoteSeverity(a, b) : promoteSeverity(b, a); - } + CodeAnalysisIssue winner = representationRank(a) >= representationRank(b) ? a : b; + CodeAnalysisIssue loser = winner == a ? b : a; + promoteMergedMetadata(winner, loser); + return winner; + } - int diffLenA = validDiffLength(a.getSuggestedFixDiff()); - int diffLenB = validDiffLength(b.getSuggestedFixDiff()); - if (diffLenA != diffLenB) { - return diffLenA >= diffLenB ? a : b; + private long representationRank(CodeAnalysisIssue issue) { + long rank = 0; + if (issue.getCodeSnippet() != null && !issue.getCodeSnippet().isBlank()) { + rank += 1_000_000_000L; } - - int lineA = a.getLineNumber() != null ? a.getLineNumber() : Integer.MAX_VALUE; - int lineB = b.getLineNumber() != null ? b.getLineNumber() : Integer.MAX_VALUE; - return lineA <= lineB ? a : b; + if (issue.getLineNumber() != null && issue.getLineNumber() > 1) { + rank += 100_000_000L; + } else if (issue.getLineNumber() != null && issue.getLineNumber() > 0) { + rank += 10_000_000L; + } + rank += Math.min(validDiffLength(issue.getSuggestedFixDiff()), 1_000_000); + rank += Math.min(issue.getReason() != null ? issue.getReason().length() : 0, 100_000); + return rank; } - /** - * Return the winner but ensure it carries the highest severity from either issue. - */ - private CodeAnalysisIssue promoteSeverity(CodeAnalysisIssue winner, CodeAnalysisIssue loser) { - // Winner already has higher severity, but adopt loser's diff if winner lacks one - if (validDiffLength(winner.getSuggestedFixDiff()) == 0 - && validDiffLength(loser.getSuggestedFixDiff()) > 0) { + private void promoteMergedMetadata(CodeAnalysisIssue winner, CodeAnalysisIssue loser) { + if (severityRank(loser.getSeverity()) > severityRank(winner.getSeverity())) { + winner.setSeverity(loser.getSeverity()); + } + if (validDiffLength(loser.getSuggestedFixDiff()) + > validDiffLength(winner.getSuggestedFixDiff())) { winner.setSuggestedFixDiff(loser.getSuggestedFixDiff()); winner.setSuggestedFixDescription(loser.getSuggestedFixDescription()); + } else if ((winner.getSuggestedFixDescription() == null + || winner.getSuggestedFixDescription().isBlank()) + && loser.getSuggestedFixDescription() != null) { + winner.setSuggestedFixDescription(loser.getSuggestedFixDescription()); } - return winner; } private int severityRank(IssueSeverity severity) { diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java index 23190f90..5382c051 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java @@ -198,17 +198,39 @@ public Job createRagIndexJob( boolean isInitial, JobTriggerSource triggerSource, User triggeredBy + ) { + return createRagIndexJob( + project, isInitial, triggerSource, triggeredBy, null, null); + } + + /** Create a durable RAG job bound to the branch revision it builds. */ + @Transactional + public Job createRagIndexJob( + Project project, + boolean isInitial, + JobTriggerSource triggerSource, + User triggeredBy, + String branchName, + String commitHash ) { Job job = new Job(); job.setProject(project); job.setJobType(isInitial ? JobType.RAG_INITIAL_INDEX : JobType.RAG_INCREMENTAL_INDEX); job.setTriggerSource(triggerSource); job.setTriggeredBy(triggeredBy); - job.setTitle(isInitial ? "Initial RAG Indexing" : "Incremental RAG Update"); + job.setBranchName(branchName); + job.setCommitHash(commitHash); + String operation = isInitial ? "Initial RAG Indexing" : "Incremental RAG Update"; + job.setTitle(branchName == null || branchName.isBlank() + ? operation + : operation + ": " + branchName); job.setStatus(JobStatus.PENDING); job = jobRepository.save(job); - addLog(job, JobLogLevel.INFO, "init", "RAG indexing job created"); + addLog(job, JobLogLevel.INFO, "init", + branchName == null || branchName.isBlank() + ? "RAG indexing job created" + : "RAG indexing job created for branch: " + branchName); return job; } diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.23.0__exact_branch_index_registry.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.23.0__exact_branch_index_registry.sql new file mode 100644 index 00000000..f21e78f4 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.23.0__exact_branch_index_registry.sql @@ -0,0 +1,126 @@ +-- Establish durable ownership and immutable generation state for exact branch indexes. +-- The early multi-branch migration dropped rag_delta_index but did not create its +-- replacement, so this forward-only migration safely handles both fresh and +-- Hibernate-created installations. + +CREATE TABLE IF NOT EXISTS rag_branch_index ( + id BIGSERIAL PRIMARY KEY, + project_id BIGINT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + branch_name VARCHAR(256) NOT NULL, + commit_hash VARCHAR(64), + chunk_count INTEGER, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_rag_branch_index_project_branch UNIQUE (project_id, branch_name) +); + +ALTER TABLE rag_branch_index + ADD COLUMN IF NOT EXISTS index_kind VARCHAR(24) NOT NULL DEFAULT 'LEGACY', + ADD COLUMN IF NOT EXISTS lifecycle_status VARCHAR(24) NOT NULL DEFAULT 'READY', + ADD COLUMN IF NOT EXISTS desired_commit_hash VARCHAR(64), + ADD COLUMN IF NOT EXISTS active_generation_id BIGINT, + ADD COLUMN IF NOT EXISTS last_accessed_at TIMESTAMP WITH TIME ZONE, + ADD COLUMN IF NOT EXISTS error_message TEXT; + +CREATE INDEX IF NOT EXISTS idx_rag_branch_project + ON rag_branch_index(project_id); +CREATE INDEX IF NOT EXISTS idx_rag_branch_name + ON rag_branch_index(branch_name); +CREATE INDEX IF NOT EXISTS idx_rag_branch_lifecycle + ON rag_branch_index(project_id, lifecycle_status); + +CREATE TABLE IF NOT EXISTS rag_branch_deleted_files ( + branch_index_id BIGINT NOT NULL REFERENCES rag_branch_index(id) ON DELETE CASCADE, + file_path VARCHAR(512) NOT NULL, + CONSTRAINT uq_rag_branch_deleted_file UNIQUE (branch_index_id, file_path) +); + +CREATE TABLE IF NOT EXISTS rag_branch_index_generation ( + id BIGSERIAL PRIMARY KEY, + branch_index_id BIGINT NOT NULL REFERENCES rag_branch_index(id) ON DELETE CASCADE, + revision VARCHAR(64) NOT NULL, + parent_generation_id BIGINT REFERENCES rag_branch_index_generation(id) ON DELETE SET NULL, + seed_revision VARCHAR(64), + collection_name VARCHAR(300) NOT NULL, + status VARCHAR(24) NOT NULL, + manifest_digest VARCHAR(128), + representation_fingerprint VARCHAR(128), + file_count INTEGER, + chunk_count INTEGER, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + activated_at TIMESTAMP WITH TIME ZONE, + superseded_at TIMESTAMP WITH TIME ZONE, + error_message TEXT, + CONSTRAINT uq_rag_branch_generation_collection UNIQUE (branch_index_id, collection_name), + CONSTRAINT ck_rag_branch_generation_status + CHECK (status IN ('BUILDING', 'ACTIVE', 'SUPERSEDED', 'FAILED')), + CONSTRAINT ck_rag_branch_generation_counts + CHECK ((file_count IS NULL OR file_count >= 0) AND (chunk_count IS NULL OR chunk_count >= 0)) +); + +CREATE INDEX IF NOT EXISTS idx_rag_branch_generation_revision + ON rag_branch_index_generation(branch_index_id, revision); +CREATE INDEX IF NOT EXISTS idx_rag_branch_generation_status + ON rag_branch_index_generation(branch_index_id, status); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'fk_rag_branch_active_generation' + AND conrelid = 'rag_branch_index'::regclass + ) THEN + ALTER TABLE rag_branch_index + ADD CONSTRAINT fk_rag_branch_active_generation + FOREIGN KEY (active_generation_id) + REFERENCES rag_branch_index_generation(id) + ON DELETE SET NULL; + END IF; +END; +$$; + +CREATE TABLE IF NOT EXISTS rag_index_operation ( + id BIGSERIAL PRIMARY KEY, + project_id BIGINT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + branch_name VARCHAR(256) NOT NULL, + from_revision VARCHAR(64), + to_revision VARCHAR(64) NOT NULL, + operation_key VARCHAR(128) NOT NULL, + status VARCHAR(24) NOT NULL, + generation_id BIGINT REFERENCES rag_branch_index_generation(id) ON DELETE SET NULL, + job_id BIGINT, + attempt_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP WITH TIME ZONE, + error_message TEXT, + CONSTRAINT uq_rag_index_operation_key UNIQUE (project_id, operation_key), + CONSTRAINT ck_rag_index_operation_status + CHECK (status IN ('PENDING', 'RUNNING', 'SUCCEEDED', 'FAILED')), + CONSTRAINT ck_rag_index_operation_attempts CHECK (attempt_count >= 0) +); + +CREATE INDEX IF NOT EXISTS idx_rag_index_operation_recovery + ON rag_index_operation(status, updated_at); +CREATE INDEX IF NOT EXISTS idx_rag_index_operation_branch + ON rag_index_operation(project_id, branch_name, to_revision); + +-- Existing rows represent the shared-collection implementation. Only the row +-- matching the authoritative RagIndexStatus branch can be identified as primary; +-- every other legacy row remains unverified until rebuilt as an exact generation. +UPDATE rag_branch_index branch_index +SET index_kind = CASE + WHEN EXISTS ( + SELECT 1 + FROM rag_index_status status + WHERE status.project_id = branch_index.project_id + AND status.indexed_branch = branch_index.branch_name + ) THEN 'PRIMARY' + ELSE 'LEGACY' + END, + lifecycle_status = CASE + WHEN branch_index.commit_hash IS NULL THEN 'PENDING' + ELSE 'READY' + END +WHERE active_generation_id IS NULL; + diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.24.0__scm_evidence_and_branch_analysis_scope.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.24.0__scm_evidence_and_branch_analysis_scope.sql new file mode 100644 index 00000000..b3a62ed0 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.24.0__scm_evidence_and_branch_analysis_scope.sql @@ -0,0 +1,76 @@ +-- Deterministic commit evidence is reusable across promotions, while analysis +-- receipts remain scoped to the exact target branch context. + +ALTER TABLE analyzed_commit + ADD COLUMN IF NOT EXISTS source_branch VARCHAR(256), + ADD COLUMN IF NOT EXISTS target_branch VARCHAR(256), + ADD COLUMN IF NOT EXISTS target_base_revision VARCHAR(64); + +ALTER TABLE analyzed_commit + DROP CONSTRAINT IF EXISTS uq_analyzed_commit_project_hash; + +CREATE UNIQUE INDEX IF NOT EXISTS uq_analyzed_commit_project_target_hash + ON analyzed_commit(project_id, COALESCE(target_branch, ''), commit_hash); +CREATE INDEX IF NOT EXISTS idx_analyzed_commit_target + ON analyzed_commit(project_id, target_branch, commit_hash); + +CREATE TABLE scm_commit_evidence ( + id BIGSERIAL PRIMARY KEY, + project_id BIGINT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + commit_hash VARCHAR(64) NOT NULL, + patch_id VARCHAR(64) NOT NULL, + author_name VARCHAR(200), + author_email VARCHAR(320), + captured_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_scm_commit_evidence_project_hash + UNIQUE (project_id, commit_hash), + CONSTRAINT ck_scm_commit_patch_id CHECK (patch_id ~ '^[0-9a-f]{64}$') +); +CREATE INDEX idx_scm_commit_patch + ON scm_commit_evidence(project_id, patch_id); + +CREATE TABLE scm_added_line_evidence ( + id BIGSERIAL PRIMARY KEY, + commit_evidence_id BIGINT NOT NULL + REFERENCES scm_commit_evidence(id) ON DELETE CASCADE, + project_id BIGINT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + file_path VARCHAR(1024) NOT NULL, + new_line_number INTEGER NOT NULL, + line_hash VARCHAR(64) NOT NULL, + CONSTRAINT ck_scm_added_line_number CHECK (new_line_number > 0), + CONSTRAINT ck_scm_added_line_hash CHECK (line_hash ~ '^[0-9a-f]{64}$') +); +CREATE INDEX idx_scm_added_line_lookup + ON scm_added_line_evidence(project_id, file_path, line_hash); + +CREATE TABLE scm_analysis_receipt ( + id BIGSERIAL PRIMARY KEY, + project_id BIGINT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + commit_evidence_id BIGINT NOT NULL + REFERENCES scm_commit_evidence(id) ON DELETE CASCADE, + source_branch VARCHAR(256), + target_branch VARCHAR(256) NOT NULL, + target_base_revision VARCHAR(64), + analysis_id BIGINT, + analysis_type VARCHAR(40) NOT NULL, + context_key VARCHAR(64) NOT NULL, + analyzed_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_scm_analysis_receipt_context + UNIQUE (project_id, commit_evidence_id, context_key), + CONSTRAINT ck_scm_analysis_context_key + CHECK (context_key ~ '^[0-9a-f]{64}$') +); +CREATE INDEX idx_scm_analysis_receipt_patch + ON scm_analysis_receipt(project_id, commit_evidence_id, target_branch); + +ALTER TABLE code_analysis_issue + ADD COLUMN IF NOT EXISTS introducing_commit_hash VARCHAR(64), + ADD COLUMN IF NOT EXISTS introducing_author_name VARCHAR(200), + ADD COLUMN IF NOT EXISTS introducing_author_email VARCHAR(320), + ADD COLUMN IF NOT EXISTS author_provenance_confidence VARCHAR(32); + +ALTER TABLE branch_issue + ADD COLUMN IF NOT EXISTS introducing_commit_hash VARCHAR(64), + ADD COLUMN IF NOT EXISTS introducing_author_name VARCHAR(200), + ADD COLUMN IF NOT EXISTS introducing_author_email VARCHAR(320), + ADD COLUMN IF NOT EXISTS author_provenance_confidence VARCHAR(32); diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java index 1e0ccc99..d769c9e3 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java @@ -1,5 +1,6 @@ package org.rostilos.codecrow.core.model.project.config; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import java.util.List; @@ -104,4 +105,66 @@ void shouldSupportInequality() { void shouldHaveDefaultBranchRetentionDaysConstant() { assertThat(RagConfig.DEFAULT_BRANCH_RETENTION_DAYS).isEqualTo(90); } + + @Test + void explicitIndexedBranchesShouldOverrideLegacyPushPatterns() { + RagConfig config = new RagConfig( + true, + "master", + null, + null, + true, + 30, + List.of(" develop ", "support/1.x", "develop", " "), + true); + + assertThat(config.getEffectiveIndexedBranches()).containsExactly("develop", "support/1.x"); + assertThat(config.shouldHaveBranchIndex("develop", List.of("feature/**"))).isTrue(); + assertThat(config.shouldHaveBranchIndex("feature/one", List.of("feature/**"))).isFalse(); + assertThat(config.isTransientBranchIndexesEnabled()).isTrue(); + } + + @Test + void legacyConfigurationShouldContinueUsingPushPatterns() { + RagConfig config = new RagConfig(true, "master", null, null, true, 30); + + assertThat(config.hasExplicitIndexedBranches()).isFalse(); + assertThat(config.shouldHaveBranchIndex("develop", List.of("develop", "support/**"))).isTrue(); + assertThat(config.shouldHaveBranchIndex("support/1.x", List.of("develop", "support/**"))).isTrue(); + assertThat(config.shouldHaveBranchIndex("feature/one", List.of("develop", "support/**"))).isFalse(); + assertThat(config.isTransientBranchIndexesEnabled()).isFalse(); + } + + @Test + void transientIndexesRequireMultiBranchOwnership() { + RagConfig config = new RagConfig( + true, "master", null, null, false, 30, List.of("develop"), true); + + assertThat(config.isTransientBranchIndexesEnabled()).isFalse(); + assertThat(config.shouldHaveBranchIndex("develop", List.of("develop"))).isFalse(); + } + + @Test + void serializesOnlyPersistedFieldsAndRoundTripsRetainedBranches() throws Exception { + ProjectConfig projectConfig = new ProjectConfig(); + projectConfig.setRagConfig(new RagConfig( + true, "master", null, null, true, 30, + List.of("develop", "release/1.x"), true)); + + ObjectMapper mapper = new ObjectMapper(); + String json = mapper.writeValueAsString(projectConfig); + + assertThat(json) + .doesNotContain("effectiveIndexedBranches") + .doesNotContain("effectiveBranchRetentionDays") + .doesNotContain("isMultiBranchEnabled") + .doesNotContain("isTransientBranchIndexesEnabled"); + + ProjectConfig restored = mapper.readValue(json, ProjectConfig.class); + assertThat(restored.ragConfig().indexedBranches()) + .containsExactly("develop", "release/1.x"); + assertThat(restored.ragConfig().getEffectiveIndexedBranches()) + .containsExactly("develop", "release/1.x"); + assertThat(restored.ragConfig().isTransientBranchIndexesEnabled()).isTrue(); + } } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationTest.java new file mode 100644 index 00000000..41fcb48d --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationTest.java @@ -0,0 +1,78 @@ +package org.rostilos.codecrow.core.model.rag; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.project.Project; + +import static org.assertj.core.api.Assertions.assertThat; + +class RagBranchIndexGenerationTest { + + @Test + void activatesExactGenerationAndMakesItTheBranchCheckpoint() { + RagBranchIndex branchIndex = new RagBranchIndex( + new Project(), "develop", RagBranchIndexKind.DURABLE); + branchIndex.requestRevision("develop-002"); + RagBranchIndexGeneration generation = new RagBranchIndexGeneration( + branchIndex, + "develop-002", + "tenant-1-project-2-develop-generation-2", + null, + "master-100", + "representation-digest"); + + generation.activate("manifest-digest", 231, 840); + branchIndex.activate(generation); + + assertThat(generation.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.ACTIVE); + assertThat(generation.getActivatedAt()).isNotNull(); + assertThat(branchIndex.getActiveGeneration()).isSameAs(generation); + assertThat(branchIndex.getCommitHash()).isEqualTo("develop-002"); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("develop-002"); + assertThat(branchIndex.getChunkCount()).isEqualTo(840); + assertThat(branchIndex.getLifecycleStatus()).isEqualTo(RagBranchIndexLifecycleStatus.READY); + } + + @Test + void failedReplacementPreservesPreviouslyActiveGeneration() { + RagBranchIndex branchIndex = new RagBranchIndex( + new Project(), "master", RagBranchIndexKind.PRIMARY); + RagBranchIndexGeneration active = new RagBranchIndexGeneration( + branchIndex, "master-100", "master-generation-100", null, + null, "representation-digest"); + active.activate("manifest-100", 400, 1200); + branchIndex.activate(active); + + branchIndex.requestRevision("master-101"); + RagBranchIndexGeneration replacement = new RagBranchIndexGeneration( + branchIndex, "master-101", "master-generation-101", active, + "master-100", "representation-digest"); + replacement.fail("Qdrant unavailable"); + branchIndex.failUpdate(replacement.getErrorMessage()); + + assertThat(branchIndex.getActiveGeneration()).isSameAs(active); + assertThat(branchIndex.getCommitHash()).isEqualTo("master-100"); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("master-101"); + assertThat(branchIndex.getLifecycleStatus()).isEqualTo(RagBranchIndexLifecycleStatus.READY); + assertThat(branchIndex.getErrorMessage()).isEqualTo("Qdrant unavailable"); + assertThat(replacement.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.FAILED); + } + + @Test + void durableOperationTracksAttemptsAndCompletion() { + Project project = new Project(); + RagIndexOperation operation = new RagIndexOperation( + project, "support/1.x", "support-9", "support-10", "operation-digest"); + RagBranchIndex branchIndex = new RagBranchIndex(project, "support/1.x", RagBranchIndexKind.DURABLE); + RagBranchIndexGeneration generation = new RagBranchIndexGeneration( + branchIndex, "support-10", "support-generation-10", null, + "master-100", "representation-digest"); + + operation.start(); + operation.succeed(generation); + + assertThat(operation.getAttemptCount()).isEqualTo(1); + assertThat(operation.getStatus()).isEqualTo(RagIndexOperationStatus.SUCCEEDED); + assertThat(operation.getGeneration()).isSameAs(generation); + assertThat(operation.getCompletedAt()).isNotNull(); + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/IssueDeduplicationServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/IssueDeduplicationServiceTest.java index ac30e0b1..3bdbea4b 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/IssueDeduplicationServiceTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/IssueDeduplicationServiceTest.java @@ -299,6 +299,87 @@ void shouldMergeSameAnchoredIssueWhenCategoryDrifts() { } } + @Nested + @DisplayName("Exact narrative safety net") + class ExactNarrativeDedup { + + @Test + @DisplayName("should collapse recreated history despite anchor and category drift") + void shouldCollapseRecreatedHistoryWithCurrentAnchor() { + CodeAnalysisIssue staleHistory = createIssue( + "src/RetryService.java", 1, IssueCategory.CODE_QUALITY, + IssueSeverity.MEDIUM, "Unbounded retry loop can exhaust workers"); + staleHistory.setReason("The retry loop has no terminal attempt limit."); + staleHistory.setCodeSnippet("public class RetryService {"); + staleHistory.setIssueFingerprint(null); + staleHistory.setContentFingerprint(null); + + CodeAnalysisIssue current = createIssue( + "src/RetryService.java", 1233, IssueCategory.BUG_RISK, + IssueSeverity.HIGH, "Unbounded retry loop can exhaust workers"); + current.setReason("The retry loop has no terminal attempt limit."); + current.setCodeSnippet("while (shouldRetry(response)) {"); + current.setIssueFingerprint(null); + current.setContentFingerprint(null); + + List result = service.deduplicateAtIngestion( + new ArrayList<>(List.of(staleHistory, current))); + + assertThat(result).hasSize(1); + assertThat(result.get(0).getLineNumber()).isEqualTo(1233); + assertThat(result.get(0).getCodeSnippet()) + .isEqualTo("while (shouldRetry(response)) {"); + assertThat(result.get(0).getSeverity()).isEqualTo(IssueSeverity.HIGH); + assertThat(result.get(0).getReason()).doesNotContain("RetryService.java:1"); + } + + @Test + @DisplayName("should retain every concrete occurrence location") + void shouldRetainAdditionalOccurrenceLocation() { + CodeAnalysisIssue first = createIssue( + "src/Policy.java", 10, IssueCategory.SECURITY, + IssueSeverity.HIGH, "Authorization check is missing"); + CodeAnalysisIssue second = createIssue( + "src/Policy.java", 20, IssueCategory.BUG_RISK, + IssueSeverity.MEDIUM, "Authorization check is missing"); + first.setReason("The update executes without checking workspace ownership."); + second.setReason("The update executes without checking workspace ownership."); + first.setIssueFingerprint(null); + first.setContentFingerprint(null); + second.setIssueFingerprint(null); + second.setContentFingerprint(null); + + List result = service.deduplicateAtIngestion( + new ArrayList<>(List.of(first, second))); + + assertThat(result).hasSize(1); + assertThat(result.get(0).getReason()) + .contains("Also affects: src/Policy.java:20"); + } + + @Test + @DisplayName("should keep independent root causes at one anchor") + void shouldKeepIndependentRootCausesAtOneAnchor() { + CodeAnalysisIssue authorization = createIssue( + "src/Policy.java", 25, IssueCategory.BUG_RISK, + IssueSeverity.HIGH, "Request processing defect"); + CodeAnalysisIssue transaction = createIssue( + "src/Policy.java", 25, IssueCategory.BUG_RISK, + IssueSeverity.HIGH, "Request processing defect"); + authorization.setReason("Workspace ownership is never checked."); + transaction.setReason("The database transaction commits too early."); + authorization.setIssueFingerprint(null); + authorization.setContentFingerprint(null); + transaction.setIssueFingerprint(null); + transaction.setContentFingerprint(null); + + List result = service.deduplicateAtIngestion( + new ArrayList<>(List.of(authorization, transaction))); + + assertThat(result).hasSize(2); + } + } + // ── Resolved issues ────────────────────────────────────────────────── @Nested diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java index 1c717b3b..d7b46d87 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java @@ -190,6 +190,31 @@ void shouldCreateIncrementalRagIndexJob() { assertThat(job.getJobType()).isEqualTo(JobType.RAG_INCREMENTAL_INDEX); assertThat(job.getTitle()).contains("Incremental"); } + + @Test + @DisplayName("should persist branch and revision for a branch RAG job") + void shouldCreateBranchBoundRagIndexJob() { + Project project = createProject(1L, "Test"); + + when(jobRepository.save(any(Job.class))).thenAnswer(inv -> { + Job j = inv.getArgument(0); + setField(j, "id", 104L); + return j; + }); + when(jobLogRepository.save(any(JobLog.class))).thenAnswer(inv -> inv.getArgument(0)); + + Job job = jobService.createRagIndexJob( + project, + false, + JobTriggerSource.UI, + null, + "develop", + "abc123"); + + assertThat(job.getBranchName()).isEqualTo("develop"); + assertThat(job.getCommitHash()).isEqualTo("abc123"); + assertThat(job.getTitle()).isEqualTo("Incremental RAG Update: develop"); + } } @Nested diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java new file mode 100644 index 00000000..b014abc3 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java @@ -0,0 +1,28 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.Executor; + +/** Dedicated service capacity for independent configured branch snapshots. */ +@Configuration +public class BranchIndexBuildExecutorConfiguration { + + @Bean(name = "branchIndexBuildExecutor") + public Executor branchIndexBuildExecutor( + @Value("${codecrow.rag.branch-build.global-parallelism:4}") int parallelism) { + int workers = Math.max(1, parallelism); + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(workers); + executor.setMaxPoolSize(workers); + executor.setQueueCapacity(50); + executor.setThreadNamePrefix("rag-branch-build-"); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(300); + executor.initialize(); + return executor; + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java new file mode 100644 index 00000000..fa465d3d --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java @@ -0,0 +1,279 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; +import org.rostilos.codecrow.core.model.rag.RagIndexOperationStatus; +import org.rostilos.codecrow.core.model.vcs.VcsConnection; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Builds one exact branch generation without depending on analysis processors. + * VCS acquisition, vector storage, and registry publication meet only through + * this small orchestration boundary. + */ +@Service +public class BranchIndexGenerationBuildService { + private static final Logger log = LoggerFactory.getLogger( + BranchIndexGenerationBuildService.class); + private static final long HEARTBEAT_INTERVAL_SECONDS = 15; + + private final BranchArchiveService archiveService; + private final RagPipelineClient pipelineClient; + private final RagBranchIndexRegistryService registryService; + private final ScheduledExecutorService heartbeatExecutor; + + public BranchIndexGenerationBuildService( + BranchArchiveService archiveService, + RagPipelineClient pipelineClient, + RagBranchIndexRegistryService registryService) { + this.archiveService = archiveService; + this.pipelineClient = pipelineClient; + this.registryService = registryService; + this.heartbeatExecutor = Executors.newSingleThreadScheduledExecutor(new HeartbeatThreadFactory()); + } + + public Map build( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns) throws IOException { + return build(project, connection, vcsWorkspace, repoSlug, branch, + revision, kind, includePatterns, excludePatterns, null); + } + + public Map build( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + Long jobId, + Consumer> progressEvents) throws IOException { + return buildInternal(project, connection, vcsWorkspace, repoSlug, branch, + revision, kind, includePatterns, excludePatterns, jobId, progressEvents, false); + } + + /** + * Builds a fresh immutable generation even when this revision already has + * a successful generation. This is reserved for an explicit operator + * refresh; automatic maintenance remains idempotent. + */ + public Map rebuild( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + Long jobId, + Consumer> progressEvents) throws IOException { + return buildInternal(project, connection, vcsWorkspace, repoSlug, branch, + revision, kind, includePatterns, excludePatterns, jobId, progressEvents, true); + } + + public Map build( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + Long jobId) throws IOException { + return buildInternal(project, connection, vcsWorkspace, repoSlug, branch, + revision, kind, includePatterns, excludePatterns, jobId, null, false); + } + + private Map buildInternal( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + Long jobId, + Consumer> progressEvents, + boolean forceRebuild) throws IOException { + var registration = registryService.registerBuild( + project, branch, kind, null, revision, + forceRebuild ? "operator-refresh:" + requireJobId(jobId) : null); + if (registration.existingOperation() + && registration.operation().getStatus() + == RagIndexOperationStatus.SUCCEEDED) { + return Map.of( + "status", "reused", + "collection_target", registration.generation().getCollectionName(), + "generation_manifest_sha256", registration.generation().getManifestDigest()); + } + + registryService.startBuild(registration.operation().getId(), jobId); + ScheduledFuture heartbeat = heartbeatExecutor.scheduleAtFixedRate( + () -> heartbeat(registration.operation().getId()), + HEARTBEAT_INTERVAL_SECONDS, + HEARTBEAT_INTERVAL_SECONDS, + TimeUnit.SECONDS); + Path snapshot = null; + try { + snapshot = Files.createTempDirectory("codecrow-rag-branch-generation-"); + archiveService.downloadAndExtractSnapshotToDirectory( + connection, + vcsWorkspace, + repoSlug, + revision, + null, + snapshot); + boolean publishBranchAlias = kind == RagBranchIndexKind.PRIMARY + || kind == RagBranchIndexKind.DURABLE; + boolean publishLegacyProjectAlias = kind == RagBranchIndexKind.PRIMARY; + Map result = progressEvents == null + ? pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, registration.generation().getCollectionName(), + false, false) + : pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, registration.generation().getCollectionName(), + false, false, + progressEvents); + Object manifest = result.get("generation_manifest_sha256"); + if (!(manifest instanceof String digest) || digest.isBlank()) { + throw new IOException("RAG full branch generation has no manifest digest"); + } + var published = registryService.publish( + registration.operation().getId(), + digest, + number(result.get("document_count")), + number(result.get("chunk_count"))); + publishReadableAliasesIfActive( + project, branch, revision, registration.generation().getCollectionName(), + published, publishBranchAlias, publishLegacyProjectAlias); + return result; + } catch (Throwable failure) { + registryService.fail( + registration.operation().getId(), + failure.getMessage() != null + ? failure.getMessage() + : failure.getClass().getSimpleName()); + if (failure instanceof IOException ioFailure) { + throw ioFailure; + } + if (failure instanceof Error error) { + throw error; + } + throw new IOException("Failed to build exact branch generation", failure); + } finally { + heartbeat.cancel(false); + if (snapshot != null) { + deleteTree(snapshot); + } + } + } + + private void publishReadableAliasesIfActive( + Project project, + String branch, + String revision, + String collectionTarget, + org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration published, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias) { + if (published == null + || published.getStatus() != RagBranchIndexGenerationStatus.ACTIVE + || !publishBranchAlias) { + return; + } + try { + pipelineClient.publishGenerationAliases( + project.getWorkspace().getName(), project.getNamespace(), + branch, revision, collectionTarget, + true, publishLegacyProjectAlias); + } catch (IOException aliasFailure) { + // Readable aliases are operator convenience. Exact retrieval uses + // the registry target, and reconciliation repairs this alias later. + log.warn("Could not publish readable aliases for RAG generation {}: {}", + published.getId(), aliasFailure.getMessage()); + } + } + + private static int number(Object value) { + return value instanceof Number number ? number.intValue() : 0; + } + + private static String requireJobId(Long jobId) { + if (jobId == null) { + throw new IllegalArgumentException("An operator refresh requires a durable job id"); + } + return jobId.toString(); + } + + private void heartbeat(long operationId) { + try { + registryService.heartbeatBuild(operationId); + } catch (Exception ignored) { + // A later heartbeat may still succeed. If the producer actually + // stops, recovery turns the durable operation into a failure. + } + } + + private static final class HeartbeatThreadFactory implements ThreadFactory { + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread(runnable, "rag-generation-heartbeat"); + thread.setDaemon(true); + return thread; + } + } + + private static void deleteTree(Path root) { + try { + if (!Files.exists(root)) { + return; + } + try (var paths = Files.walk(root)) { + for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) { + Files.deleteIfExists(path); + } + } + } catch (IOException ignored) { + // Temporary cleanup must not replace the indexing result. + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java new file mode 100644 index 00000000..d5813aed --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java @@ -0,0 +1,292 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; +import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; +import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobLogLevel; +import org.rostilos.codecrow.core.model.job.JobTriggerSource; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.vcs.VcsConnection; +import org.rostilos.codecrow.core.service.AnalysisJobService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.rostilos.codecrow.vcsclient.VcsClient; +import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.Executor; +import java.util.function.Consumer; + +/** + * Explicit operator-triggered rebuilds for configured RAG branches. + * + * This is deliberately separate from webhook reconciliation: it always builds + * an exact complete snapshot for the requested revision, and only accepts the + * primary branch or an explicitly retained target. Each branch build has its + * own durable operation and job, so an all-branch run is observable and safe + * to repeat after a partial failure. + */ +@Service +public class BranchIndexMaintenanceService { + private final RagOperationsService ragOperationsService; + private final VcsClientProvider vcsClientProvider; + private final BranchIndexGenerationBuildService generationBuildService; + private final RagIndexTrackingService trackingService; + private final AnalysisLockService lockService; + private final AnalysisJobService jobService; + private final Executor branchIndexBuildExecutor; + private final int perProjectParallelism; + + public BranchIndexMaintenanceService( + RagOperationsService ragOperationsService, + VcsClientProvider vcsClientProvider, + BranchIndexGenerationBuildService generationBuildService, + RagIndexTrackingService trackingService, + AnalysisLockService lockService, + AnalysisJobService jobService, + @Qualifier("branchIndexBuildExecutor") Executor branchIndexBuildExecutor, + @Value("${codecrow.rag.branch-build.parallelism:2}") int perProjectParallelism) { + this.ragOperationsService = ragOperationsService; + this.vcsClientProvider = vcsClientProvider; + this.generationBuildService = generationBuildService; + this.trackingService = trackingService; + this.lockService = lockService; + this.jobService = jobService; + this.branchIndexBuildExecutor = branchIndexBuildExecutor; + this.perProjectParallelism = Math.max(1, perProjectParallelism); + } + + public Map rebuild(Project project, String requestedBranch, boolean allConfiguredBranches, + Consumer> events) { + List branches = resolveBranches(project, requestedBranch, allConfiguredBranches); + List completed = new ArrayList<>(); + Map failures = new LinkedHashMap<>(); + + // Obtaining a provider client can refresh a shared installation token. Do + // that small VCS preparation phase once at a time, then let the expensive + // archive download and RAG mutation for every resolved branch run in + // parallel. Concurrent token refreshes previously allowed one branch to + // disappear before it had a durable job or operation to report. + List plans = new ArrayList<>(); + for (String branch : branches) { + try { + plans.add(prepareBuild(project, branch)); + } catch (RuntimeException failure) { + String message = failure.getMessage() != null + ? failure.getMessage() : failure.getClass().getSimpleName(); + failures.put(branch, message); + events.accept(Map.of("type", "progress", "stage", "branch_failed", + "branch", branch, + "message", "RAG snapshot failed for branch '" + branch + "': " + message)); + } + } + // Limit one project's fan-out independently from the service capacity. + // A project with many retained branches can therefore use at most its + // configured share while builds for other tenants still occupy the + // remaining dedicated RAG slots. Each wave is fully parallel. + for (int from = 0; from < plans.size(); from += perProjectParallelism) { + int to = Math.min(plans.size(), from + perProjectParallelism); + Map> wave = new LinkedHashMap<>(); + for (BranchBuildPlan plan : plans.subList(from, to)) { + String branch = plan.branch(); + wave.put(branch, CompletableFuture.runAsync(() -> { + events.accept(Map.of("type", "progress", "stage", "branch", + "branch", branch, + "message", "Building exact RAG snapshot for branch '" + branch + "'")); + rebuildOne(project, plan, events); + }, branchIndexBuildExecutor)); + } + for (Map.Entry> build : wave.entrySet()) { + try { + build.getValue().join(); + completed.add(build.getKey()); + } catch (CompletionException failure) { + Throwable cause = failure.getCause() != null ? failure.getCause() : failure; + String message = cause.getMessage() != null + ? cause.getMessage() : cause.getClass().getSimpleName(); + failures.put(build.getKey(), message); + events.accept(Map.of("type", "progress", "stage", "branch_failed", + "branch", build.getKey(), + "message", "RAG snapshot failed for branch '" + build.getKey() + + "': " + message)); + } + } + } + if (completed.isEmpty()) { + throw new IllegalStateException("No configured branch snapshot was built: " + failures); + } + Map outcome = new LinkedHashMap<>(); + outcome.put("status", "completed"); + outcome.put("message", failures.isEmpty() + ? "Built RAG snapshots for " + String.join(", ", completed) + : "Built RAG snapshots for " + String.join(", ", completed) + + "; failed: " + String.join(", ", failures.keySet())); + outcome.put("branches", completed); + outcome.put("failedBranches", failures); + return outcome; + } + + private BranchBuildPlan prepareBuild(Project project, String branch) { + var binding = project.getVcsRepoBinding(); + if (binding == null || binding.getVcsConnection() == null) { + throw new IllegalStateException("Project has no VCS connection"); + } + VcsConnection connection = binding.getVcsConnection(); + String workspace = binding.getExternalNamespace(); + String repository = binding.getExternalRepoSlug(); + VcsClient client = vcsClientProvider.getClient(connection); + String revision; + try { + revision = client.getLatestCommitHash(workspace, repository, branch); + } catch (Exception failure) { + throw new IllegalStateException("Could not resolve the current revision for branch '" + branch + "'", failure); + } + if (revision == null || revision.isBlank()) { + throw new IllegalStateException("Branch '" + branch + "' has no resolvable revision"); + } + return new BranchBuildPlan(branch, connection, workspace, repository, revision); + } + + private void rebuildOne(Project project, BranchBuildPlan plan, Consumer> events) { + String branch = plan.branch(); + VcsConnection connection = plan.connection(); + String workspace = plan.workspace(); + String repository = plan.repository(); + String revision = plan.revision(); + + Optional lock = lockService.acquireLock( + project, branch, AnalysisLockType.RAG_INDEXING, revision, null); + if (lock.isEmpty()) { + throw new IllegalStateException("RAG indexing is already running for branch '" + branch + "'"); + } + + boolean primary = branch.equals(ragOperationsService.getBaseBranch(project)); + boolean primaryPreviouslyIndexed = primary && trackingService.isProjectIndexed(project); + Job job = jobService.createRagIndexJob( + project, + !primaryPreviouslyIndexed, + JobTriggerSource.UI, + branch, + revision); + jobService.startJob(job); + jobService.logToJob( + job, + JobLogLevel.INFO, + "branch_snapshot", + "Building exact RAG snapshot for branch: " + branch, + Map.of("branch", branch, "commit", revision)); + try { + if (primary) { + trackingService.markIndexingStarted(project, branch, revision); + } + var config = project.getConfiguration().ragConfig(); + Map result = generationBuildService.rebuild( + project, + connection, + workspace, + repository, + branch, + revision, + primary ? RagBranchIndexKind.PRIMARY : RagBranchIndexKind.DURABLE, + config.includePatterns(), + config.excludePatterns(), + job.getId(), event -> { + Map forwarded = new LinkedHashMap<>(event); + forwarded.put("type", "progress"); + forwarded.put("branch", branch); + String stage = String.valueOf( + forwarded.getOrDefault("stage", "indexing")); + String message = String.valueOf( + forwarded.getOrDefault("message", "Indexing branch '" + branch + "'")); + jobService.logToJob( + job, + JobLogLevel.INFO, + stage, + message, + forwarded); + events.accept(forwarded); + }); + if (primary) { + trackingService.markIndexingCompleted( + project, + branch, + revision, + number(result.get("document_count")), + number(result.get("chunk_count"))); + } + jobService.completeJob(job, Map.of("branch", branch, "revision", revision)); + events.accept(Map.of("type", "progress", "stage", "branch_complete", "branch", branch, + "message", "RAG snapshot is ready for branch '" + branch + "'")); + } catch (Throwable failure) { + String diagnostic = failure.getMessage() != null + ? failure.getMessage() : failure.getClass().getSimpleName(); + if (primary) { + if (primaryPreviouslyIndexed) { + trackingService.markIncrementalUpdateFailed(project, diagnostic); + } else { + trackingService.markIndexingFailed(project, diagnostic); + } + } + jobService.failJob(job, diagnostic); + if (failure instanceof Error error) { + throw error; + } + throw failure instanceof RuntimeException runtime ? runtime + : new IllegalStateException("Failed to build RAG snapshot for branch '" + branch + "'", failure); + } finally { + lockService.releaseLock(lock.get()); + } + } + + private List resolveBranches(Project project, String requestedBranch, boolean allConfiguredBranches) { + if (project.getConfiguration() == null || project.getConfiguration().ragConfig() == null + || !project.getConfiguration().ragConfig().enabled()) { + throw new IllegalStateException("RAG is not enabled for this project"); + } + var config = project.getConfiguration().ragConfig(); + String primary = ragOperationsService.getBaseBranch(project); + if (allConfiguredBranches) { + if (!config.isMultiBranchEnabled()) { + return List.of(primary); + } + LinkedHashSet branches = new LinkedHashSet<>(); + branches.add(primary); + branches.addAll(config.getEffectiveIndexedBranches()); + return List.copyOf(branches); + } + if (requestedBranch == null || requestedBranch.isBlank()) { + throw new IllegalArgumentException("A configured RAG branch must be selected"); + } + String branch = requestedBranch.trim(); + if (branch.equals(primary)) { + return List.of(branch); + } + if (config.isMultiBranchEnabled() && ragOperationsService.shouldHaveBranchIndex(project, branch)) { + return List.of(branch); + } + throw new IllegalArgumentException("Branch '" + branch + "' is not configured as a retained RAG branch"); + } + + private record BranchBuildPlan( + String branch, + VcsConnection connection, + String workspace, + String repository, + String revision) { + } + + private static int number(Object value) { + return value instanceof Number number ? number.intValue() : 0; + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java new file mode 100644 index 00000000..b58b5ea5 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java @@ -0,0 +1,74 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** + * Restores human-readable Qdrant aliases for already active branch generations. + * + * Normal generation activation publishes those aliases atomically alongside its + * immutable target. This independent, idempotent repair loop makes the feature + * safe for projects and Qdrant snapshots created before that contract existed. + * It never changes the registry or analysis binding, so a temporary Qdrant + * failure only affects operator discoverability and is retried later. + */ +@Service +public class RagBranchOperatorAliasReconciliationService { + private static final Logger log = LoggerFactory.getLogger( + RagBranchOperatorAliasReconciliationService.class); + + private final RagBranchIndexRepository branchIndexRepository; + private final RagPipelineClient pipelineClient; + + public RagBranchOperatorAliasReconciliationService( + RagBranchIndexRepository branchIndexRepository, + RagPipelineClient pipelineClient) { + this.branchIndexRepository = branchIndexRepository; + this.pipelineClient = pipelineClient; + } + + @Scheduled( + fixedDelayString = "${codecrow.rag.operator-alias.reconcile-interval-ms:300000}", + initialDelayString = "${codecrow.rag.operator-alias.reconcile-initial-delay-ms:15000}") + @Transactional(readOnly = true) + public void reconcileActiveGenerationAliases() { + for (RagBranchIndex branchIndex : branchIndexRepository.findAll()) { + if (!isPublishedOperatorBranch(branchIndex)) { + continue; + } + var generation = branchIndex.getActiveGeneration(); + var project = branchIndex.getProject(); + try { + pipelineClient.publishGenerationAliases( + project.getWorkspace().getName(), + project.getNamespace(), + branchIndex.getBranchName(), + generation.getRevision(), + generation.getCollectionName(), + true, + branchIndex.getIndexKind() == RagBranchIndexKind.PRIMARY); + } catch (Exception failure) { + log.warn( + "Could not reconcile readable RAG alias for project={} branch={}: {}", + project.getId(), + branchIndex.getBranchName(), + failure.getMessage()); + } + } + } + + private static boolean isPublishedOperatorBranch(RagBranchIndex branchIndex) { + if (branchIndex == null || branchIndex.getActiveGeneration() == null) { + return false; + } + return branchIndex.getIndexKind() == RagBranchIndexKind.PRIMARY + || branchIndex.getIndexKind() == RagBranchIndexKind.DURABLE; + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java new file mode 100644 index 00000000..7c194528 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java @@ -0,0 +1,52 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.OffsetDateTime; + +/** + * Terminates registry operations whose producer disappeared before publishing + * a generation. The prior active generation remains queryable; a later event + * retries the same idempotency key from a fresh snapshot or checkpoint. + */ +@Service +public class RagIndexOperationRecoveryService { + + private static final Logger log = LoggerFactory.getLogger( + RagIndexOperationRecoveryService.class); + + private final RagBranchIndexRegistryService registryService; + private final long staleAfterMinutes; + + public RagIndexOperationRecoveryService( + RagBranchIndexRegistryService registryService, + @Value("${codecrow.rag.generation.stale-after-minutes:30}") + long staleAfterMinutes) { + this.registryService = registryService; + this.staleAfterMinutes = Math.max(5, staleAfterMinutes); + } + + @Scheduled( + fixedDelayString = "${codecrow.rag.generation.recovery-interval-ms:300000}", + initialDelayString = "${codecrow.rag.generation.recovery-initial-delay-ms:60000}") + public void failAbandonedOperations() { + OffsetDateTime cutoff = OffsetDateTime.now().minusMinutes(staleAfterMinutes); + for (var operation : registryService.findRecoverableOperations(cutoff)) { + String diagnostic = "Exact RAG generation producer stopped heartbeating for " + + staleAfterMinutes + " minutes; the previous active generation was preserved"; + try { + registryService.fail(operation.getId(), diagnostic); + log.warn("Failed abandoned RAG generation operation {} for branch {}", + operation.getId(), operation.getBranchName()); + } catch (Exception failure) { + log.error("Could not terminalize abandoned RAG generation operation {}", + operation.getId(), failure); + } + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java new file mode 100644 index 00000000..2aa28b55 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java @@ -0,0 +1,77 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.OffsetDateTime; + +/** Removes only expired PR-target generations explicitly classified transient. */ +@Service +public class RagTransientBranchIndexCleanupService { + private static final Logger log = LoggerFactory.getLogger( + RagTransientBranchIndexCleanupService.class); + + private final RagBranchIndexRepository branchRepository; + private final RagBranchIndexGenerationRepository generationRepository; + private final RagPipelineClient pipelineClient; + + public RagTransientBranchIndexCleanupService( + RagBranchIndexRepository branchRepository, + RagBranchIndexGenerationRepository generationRepository, + RagPipelineClient pipelineClient) { + this.branchRepository = branchRepository; + this.generationRepository = generationRepository; + this.pipelineClient = pipelineClient; + } + + @Scheduled( + fixedDelayString = "${codecrow.rag.transient.cleanup-interval-ms:3600000}", + initialDelayString = "${codecrow.rag.transient.cleanup-initial-delay-ms:300000}") + @Transactional + public void cleanupExpired() { + OffsetDateTime now = OffsetDateTime.now(); + for (RagBranchIndex index : branchRepository.findByIndexKind( + RagBranchIndexKind.TRANSIENT)) { + var project = index.getProject(); + var config = project.getConfiguration() != null + ? project.getConfiguration().ragConfig() + : null; + int retentionDays = config != null + ? config.getEffectiveBranchRetentionDays() + : 90; + OffsetDateTime lastUse = index.getLastAccessedAt() != null + ? index.getLastAccessedAt() + : index.getUpdatedAt(); + if (lastUse == null || !lastUse.isBefore(now.minusDays(retentionDays))) { + continue; + } + + boolean removed = true; + for (var generation : generationRepository + .findByBranchIndexIdOrderByCreatedAtDesc(index.getId())) { + try { + removed &= pipelineClient.deleteBranch( + project.getWorkspace().getName(), project.getNamespace(), + index.getBranchName(), generation.getCollectionName()); + } catch (Exception failure) { + removed = false; + log.warn("Failed to clean transient RAG generation {}: {}", + generation.getId(), failure.getMessage()); + } + } + if (removed) { + branchRepository.delete(index); + log.info("Removed expired transient RAG branch index project={}, branch={}", + project.getId(), index.getBranchName()); + } + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java index 68398396..c6c0473d 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java @@ -4,13 +4,17 @@ import okhttp3.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.rostilos.codecrow.ragengine.source.RepositorySourceTreeIdentity; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.IOException; +import java.nio.file.Path; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.Consumer; @Service public class RagPipelineClient { @@ -70,6 +74,42 @@ public Map indexRepository( String commit, List includePatterns, List excludePatterns + ) throws IOException { + return indexRepository( + repoPath, projectWorkspace, projectNamespace, branch, commit, + includePatterns, excludePatterns, null); + } + + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget + ) throws IOException { + return indexRepository( + repoPath, projectWorkspace, projectNamespace, branch, commit, + includePatterns, excludePatterns, collectionTarget, false, false); + } + + /** + * Index an immutable generation and optionally publish its readable branch + * and legacy-project aliases in the same Qdrant transaction. + */ + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias ) throws IOException { if (!ragEnabled) { log.debug("RAG indexing disabled, skipping repository indexing"); @@ -82,6 +122,19 @@ public Map indexRepository( payload.put("project", projectNamespace); payload.put("branch", branch); payload.put("commit", commit); + if (collectionTarget != null && !collectionTarget.isBlank()) { + payload.put("collection_target", collectionTarget); + } + if (publishBranchAlias) { + payload.put("publish_branch_alias", true); + } + if (publishLegacyProjectAlias) { + payload.put("publish_legacy_project_alias", true); + } + payload.put( + "source_tree_sha256", + RepositorySourceTreeIdentity.sha256(Path.of(repoPath)) + ); if (includePatterns != null && !includePatterns.isEmpty()) { payload.put("include_patterns", includePatterns); } @@ -93,6 +146,74 @@ public Map indexRepository( return postLongRunning(url, payload); } + /** + * Build an index through the progress-streaming transport. The regular + * JSON method above remains available for legacy callers; this overload is + * used by explicit branch maintenance so detailed batch events can reach + * the operator without affecting index correctness. + */ + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget, + Consumer> progressConsumer + ) throws IOException { + return indexRepository( + repoPath, projectWorkspace, projectNamespace, branch, commit, + includePatterns, excludePatterns, collectionTarget, false, false, + progressConsumer); + } + + /** Streaming variant of exact generation indexing with alias publication. */ + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias, + Consumer> progressConsumer + ) throws IOException { + if (!ragEnabled) { + log.debug("RAG indexing disabled, skipping repository indexing"); + return Map.of("status", "skipped", "reason", "RAG disabled"); + } + + Map payload = new HashMap<>(); + payload.put("repo_path", repoPath); + payload.put("workspace", projectWorkspace); + payload.put("project", projectNamespace); + payload.put("branch", branch); + payload.put("commit", commit); + if (collectionTarget != null && !collectionTarget.isBlank()) { + payload.put("collection_target", collectionTarget); + } + if (publishBranchAlias) { + payload.put("publish_branch_alias", true); + } + if (publishLegacyProjectAlias) { + payload.put("publish_legacy_project_alias", true); + } + payload.put("source_tree_sha256", RepositorySourceTreeIdentity.sha256(Path.of(repoPath))); + if (includePatterns != null && !includePatterns.isEmpty()) { + payload.put("include_patterns", includePatterns); + } + if (excludePatterns != null && !excludePatterns.isEmpty()) { + payload.put("exclude_patterns", excludePatterns); + } + return postLongRunningSse( + ragApiUrl + "/index/repository/stream", payload, progressConsumer); + } + public Map updateFiles( List filePaths, String repoBase, @@ -179,6 +300,98 @@ public Map applyChanges( return postLongRunning(ragApiUrl + "/index/apply-changes", payload); } + public Map advanceGeneration( + List updatedFilePaths, + List deletedFilePaths, + String repoBase, + String workspace, + String project, + String branch, + String sourceCommit, + String commit, + String sourceTreeSha256, + String sourceCollectionTarget, + String collectionTarget + ) throws IOException { + return advanceGeneration( + updatedFilePaths, deletedFilePaths, repoBase, workspace, project, + branch, sourceCommit, commit, sourceTreeSha256, + sourceCollectionTarget, collectionTarget, false, false); + } + + /** Advance an exact generation and atomically update its readable aliases. */ + public Map advanceGeneration( + List updatedFilePaths, + List deletedFilePaths, + String repoBase, + String workspace, + String project, + String branch, + String sourceCommit, + String commit, + String sourceTreeSha256, + String sourceCollectionTarget, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias + ) throws IOException { + if (!ragEnabled) { + log.debug("RAG indexing disabled, skipping generation advance"); + return Map.of("status", "skipped", "reason", "RAG disabled"); + } + + Map payload = new HashMap<>(); + payload.put("updated_file_paths", updatedFilePaths); + payload.put("deleted_file_paths", deletedFilePaths); + if (repoBase != null && !repoBase.isBlank()) { + payload.put("repo_base", repoBase); + } + payload.put("workspace", workspace); + payload.put("project", project); + payload.put("branch", branch); + payload.put("source_commit", sourceCommit); + payload.put("commit", commit); + payload.put("source_tree_sha256", sourceTreeSha256); + payload.put("source_collection_target", sourceCollectionTarget); + payload.put("collection_target", collectionTarget); + if (publishBranchAlias) { + payload.put("publish_branch_alias", true); + } + if (publishLegacyProjectAlias) { + payload.put("publish_legacy_project_alias", true); + } + + return postLongRunning(ragApiUrl + "/index/advance-generation", payload); + } + + /** + * Idempotently repair human-readable aliases of one completed generation. + * Exact analysis never depends on this convenience mapping. + */ + public void publishGenerationAliases( + String workspace, + String project, + String branch, + String commit, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias) throws IOException { + if (!ragEnabled || !publishBranchAlias) { + return; + } + Map payload = new HashMap<>(); + payload.put("workspace", workspace); + payload.put("project", project); + payload.put("branch", branch); + payload.put("commit", commit); + payload.put("collection_target", collectionTarget); + payload.put("publish_branch_alias", true); + if (publishLegacyProjectAlias) { + payload.put("publish_legacy_project_alias", true); + } + post(ragApiUrl + "/index/generation-aliases", payload); + } + public Map getPRContext( String workspace, String project, @@ -341,16 +554,29 @@ public boolean deletePrFiles(String workspace, String project, int prNumber) { * Python endpoint: DELETE /index/{workspace}/{project}/branch/{branch} */ public boolean deleteBranch(String workspace, String project, String branch) throws IOException { + return deleteBranch(workspace, project, branch, null); + } + + public boolean deleteBranch( + String workspace, + String project, + String branch, + String collectionTarget + ) throws IOException { if (!ragEnabled) { return false; } // URL-encode branch name to handle slashes (e.g., feature/xyz -> feature%2Fxyz) String encodedBranch = java.net.URLEncoder.encode(branch, java.nio.charset.StandardCharsets.UTF_8); - String url = String.format("%s/index/%s/%s/branch/%s", ragApiUrl, workspace, project, encodedBranch); + HttpUrl.Builder urlBuilder = HttpUrl.get(String.format( + "%s/index/%s/%s/branch/%s", ragApiUrl, workspace, project, encodedBranch)).newBuilder(); + if (collectionTarget != null && !collectionTarget.isBlank()) { + urlBuilder.addQueryParameter("collection_target", collectionTarget); + } Request.Builder builder = new Request.Builder() - .url(url) + .url(urlBuilder.build()) .delete(); addAuthHeader(builder); Request request = builder.build(); @@ -531,6 +757,59 @@ private Map postLongRunning(String url, Map payl return doRequest(url, payload, longRunningHttpClient); } + @SuppressWarnings("unchecked") + private Map postLongRunningSse( + String url, + Map payload, + Consumer> progressConsumer + ) throws IOException { + RequestBody body = RequestBody.create(objectMapper.writeValueAsString(payload), JSON); + Request.Builder builder = new Request.Builder() + .url(url) + .header("Accept", "text/event-stream") + .post(body); + addAuthHeader(builder); + + try (Response response = longRunningHttpClient.newCall(builder.build()).execute()) { + if (!response.isSuccessful()) { + String detail = response.body() != null ? response.body().string() : "{}"; + throw new IOException("RAG API error: " + response.code() + " — " + detail); + } + if (response.body() == null) { + throw new IOException("RAG progress stream returned no body"); + } + String line; + while ((line = response.body().source().readUtf8Line()) != null) { + if (!line.startsWith("data:")) { + continue; + } + String json = line.substring(5).trim(); + if (json.isEmpty()) { + continue; + } + Map event = objectMapper.readValue(json, Map.class); + String type = String.valueOf(event.get("type")); + if ("progress".equals(type)) { + if (progressConsumer != null) { + progressConsumer.accept(new LinkedHashMap<>(event)); + } + continue; + } + if ("complete".equals(type)) { + Object result = event.get("result"); + if (result instanceof Map resultMap) { + return new LinkedHashMap<>((Map) resultMap); + } + throw new IOException("RAG progress stream completed without index result"); + } + if ("error".equals(type)) { + throw new IOException("RAG API error: " + event.getOrDefault("message", "unknown error")); + } + } + } + throw new IOException("RAG progress stream ended without a terminal result"); + } + /** * Adds the x-service-secret header to the request if a secret is configured. */ diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java index 631134a6..e9e0f52e 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java @@ -7,6 +7,7 @@ import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.rostilos.codecrow.ragengine.source.RepositorySourceTreeIdentity; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.slf4j.Logger; @@ -95,6 +96,51 @@ public Map performIncrementalUpdate( Set addedFiles, Set modifiedFiles, Set deletedFiles) throws IOException { + return performIncrementalUpdate( + project, vcsConnection, workspaceSlug, repoSlug, branch, + commitHash, addedFiles, modifiedFiles, deletedFiles, + null, null, null, false, false); + } + + public Map performIncrementalUpdate( + Project project, + VcsConnection vcsConnection, + String workspaceSlug, + String repoSlug, + String branch, + String commitHash, + Set addedFiles, + Set modifiedFiles, + Set deletedFiles, + String sourceRevision, + String sourceCollectionTarget, + String targetCollectionTarget) throws IOException { + return performIncrementalUpdate( + project, vcsConnection, workspaceSlug, repoSlug, branch, + commitHash, addedFiles, modifiedFiles, deletedFiles, + sourceRevision, sourceCollectionTarget, targetCollectionTarget, + false, false); + } + + /** + * Advances an immutable generation. The registry owner publishes readable + * aliases only after it has accepted this generation as the current head. + */ + public Map performIncrementalUpdate( + Project project, + VcsConnection vcsConnection, + String workspaceSlug, + String repoSlug, + String branch, + String commitHash, + Set addedFiles, + Set modifiedFiles, + Set deletedFiles, + String sourceRevision, + String sourceCollectionTarget, + String targetCollectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias) throws IOException { Set skippedNonTextFiles = new LinkedHashSet<>(); Set indexableAddedFiles = filterTextCandidates(addedFiles, skippedNonTextFiles); Set indexableModifiedFiles = filterTextCandidates(modifiedFiles, skippedNonTextFiles); @@ -119,7 +165,11 @@ public Map performIncrementalUpdate( List orderedAddedOrModifiedFiles = new ArrayList<>(sortedList(addedOrModifiedFiles)); List orderedDeletedFiles = sortedList(deletedFiles); - if (orderedAddedOrModifiedFiles.isEmpty() && orderedDeletedFiles.isEmpty()) { + boolean generationAdvanceRequested = sourceRevision != null + && sourceCollectionTarget != null + && targetCollectionTarget != null; + if (orderedAddedOrModifiedFiles.isEmpty() && orderedDeletedFiles.isEmpty() + && !generationAdvanceRequested) { result.put("status", "completed"); return result; } @@ -128,31 +178,37 @@ public Map performIncrementalUpdate( try { String revision = commitHash != null && !commitHash.isBlank() ? commitHash : branch; String repoBase = null; - if (!orderedAddedOrModifiedFiles.isEmpty()) { + String sourceTreeSha256 = null; + if (!orderedAddedOrModifiedFiles.isEmpty() || generationAdvanceRequested) { tempDir = Files.createTempDirectory("codecrow-rag-incremental-", PosixFilePermissions.asFileAttribute( PosixFilePermissions.fromString("rwxrwxrwx"))); int effectiveArchiveFileThreshold = fileRetrievalPolicy.archiveFileThreshold(); - boolean useArchive = - fileRetrievalPolicy.shouldUseArchive(orderedAddedOrModifiedFiles.size()); + boolean useArchive = generationAdvanceRequested + || fileRetrievalPolicy.shouldUseArchive(orderedAddedOrModifiedFiles.size()); Set fetchedFilePaths; Set presentFilePaths = Collections.emptySet(); String fileFetchMode; if (useArchive) { log.info("Using one repository archive at revision {} for {} incremental RAG files " - + "(threshold: {})", - revision, orderedAddedOrModifiedFiles.size(), effectiveArchiveFileThreshold); + + "(threshold: {}, exact generation: {})", + revision, orderedAddedOrModifiedFiles.size(), effectiveArchiveFileThreshold, + generationAdvanceRequested); BranchArchiveService.ArchiveDirectorySnapshot archiveSnapshot = branchArchiveService.downloadAndExtractSnapshotToDirectory( vcsConnection, workspaceSlug, repoSlug, revision, - new LinkedHashSet<>(orderedAddedOrModifiedFiles), + generationAdvanceRequested + ? null + : new LinkedHashSet<>(orderedAddedOrModifiedFiles), tempDir); fetchedFilePaths = archiveSnapshot.extractedFiles(); presentFilePaths = archiveSnapshot.presentFiles(); - fileFetchMode = "archive"; + fileFetchMode = generationAdvanceRequested + ? "exact-generation-archive" + : "archive"; } else { log.info("Using per-file VCS retrieval for {} incremental RAG files (threshold: {})", orderedAddedOrModifiedFiles.size(), effectiveArchiveFileThreshold); @@ -209,6 +265,10 @@ public Map performIncrementalUpdate( + revision + ": " + String.join(", ", missingFiles)); } repoBase = tempDir.toString(); + if (generationAdvanceRequested) { + sourceTreeSha256 = RepositorySourceTreeIdentity.sha256(tempDir); + result.put("sourceTreeSha256", sourceTreeSha256); + } result.put("updatedFiles", orderedAddedOrModifiedFiles.size()); int appliedAddedFiles = (int) orderedAddedOrModifiedFiles.stream() .filter(indexableAddedFiles::contains) @@ -219,17 +279,36 @@ public Map performIncrementalUpdate( } String changeSetRepoBase = repoBase; + String targetSourceTreeSha256 = sourceTreeSha256; List filesToUpdate = List.copyOf(orderedAddedOrModifiedFiles); + boolean generationAdvance = generationAdvanceRequested; Map updateResult = executeWithRetry( - "apply incremental RAG change set", - () -> ragPipelineClient.applyChanges( - filesToUpdate, - orderedDeletedFiles, - changeSetRepoBase, - projectWorkspace, - projectNamespace, - branch, - revision)); + generationAdvance + ? "advance immutable RAG generation" + : "apply incremental RAG change set", + () -> generationAdvance + ? ragPipelineClient.advanceGeneration( + filesToUpdate, + orderedDeletedFiles, + changeSetRepoBase, + projectWorkspace, + projectNamespace, + branch, + sourceRevision, + revision, + targetSourceTreeSha256, + sourceCollectionTarget, + targetCollectionTarget, + publishBranchAlias, + publishLegacyProjectAlias) + : ragPipelineClient.applyChanges( + filesToUpdate, + orderedDeletedFiles, + changeSetRepoBase, + projectWorkspace, + projectNamespace, + branch, + revision)); result.put("deletedFiles", orderedDeletedFiles.size()); result.putAll(updateResult); log.info( diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java new file mode 100644 index 00000000..5b2a3ab8 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java @@ -0,0 +1,288 @@ +package org.rostilos.codecrow.ragengine.service; + +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.*; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagIndexOperationRepository; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.OffsetDateTime; +import java.util.HexFormat; +import java.util.List; +import java.util.Optional; + +/** + * Owns durable branch-index identities and their immutable generations. It does + * not know how Qdrant or a VCS works; callers build and validate the physical + * generation, then atomically publish or fail it through this service. + */ +@Service +public class RagBranchIndexRegistryService { + + private final RagBranchIndexRepository branchIndexRepository; + private final RagBranchIndexGenerationRepository generationRepository; + private final RagIndexOperationRepository operationRepository; + + public RagBranchIndexRegistryService( + RagBranchIndexRepository branchIndexRepository, + RagBranchIndexGenerationRepository generationRepository, + RagIndexOperationRepository operationRepository) { + this.branchIndexRepository = branchIndexRepository; + this.generationRepository = generationRepository; + this.operationRepository = operationRepository; + } + + public record BuildRegistration( + RagBranchIndex branchIndex, + RagBranchIndexGeneration generation, + RagIndexOperation operation, + boolean existingOperation) { + } + + @Transactional + public BuildRegistration registerBuild( + Project project, + String branchName, + RagBranchIndexKind indexKind, + String fromRevision, + String toRevision, + String representationFingerprint) { + requireProjectIdentity(project); + String branch = requireText(branchName, "branchName"); + String targetRevision = requireText(toRevision, "toRevision"); + RagBranchIndexKind kind = indexKind != null ? indexKind : RagBranchIndexKind.DURABLE; + + String operationKey = operationKey( + project.getId(), branch, fromRevision, targetRevision, representationFingerprint); + Optional existing = operationRepository + .findByProjectIdAndOperationKey(project.getId(), operationKey); + if (existing.isPresent()) { + RagIndexOperation operation = existing.get(); + return new BuildRegistration( + operation.getGeneration().getBranchIndex(), + operation.getGeneration(), + operation, + true); + } + + RagBranchIndex branchIndex = branchIndexRepository + .findByProjectIdAndBranchNameForUpdate(project.getId(), branch) + .orElseGet(() -> new RagBranchIndex(project, branch, kind)); + if (branchIndex.getIndexKind() == RagBranchIndexKind.LEGACY + || kind == RagBranchIndexKind.PRIMARY + || (branchIndex.getIndexKind() == RagBranchIndexKind.TRANSIENT + && kind == RagBranchIndexKind.DURABLE)) { + branchIndex.setIndexKind(kind); + } + branchIndex.requestRevision(targetRevision); + branchIndex = branchIndexRepository.save(branchIndex); + + RagBranchIndexGeneration parent = branchIndex.getActiveGeneration(); + String collectionName = physicalCollectionName(project, branch, targetRevision, operationKey); + RagBranchIndexGeneration generation = new RagBranchIndexGeneration( + branchIndex, + targetRevision, + collectionName, + parent, + fromRevision, + representationFingerprint); + generation = generationRepository.save(generation); + + RagIndexOperation operation = new RagIndexOperation( + project, branch, fromRevision, targetRevision, operationKey); + operation.setGeneration(generation); + operation = operationRepository.save(operation); + + return new BuildRegistration(branchIndex, generation, operation, false); + } + + @Transactional + public void startBuild(long operationId, Long jobId) { + RagIndexOperation operation = requireOperation(operationId); + if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED) { + return; + } + if (operation.getStatus() == RagIndexOperationStatus.FAILED) { + operation.getGeneration().retry(); + generationRepository.save(operation.getGeneration()); + } + operation.setJobId(jobId); + operation.start(); + operationRepository.save(operation); + } + + @Transactional + public RagBranchIndexGeneration publish( + long operationId, + String manifestDigest, + int fileCount, + int chunkCount) { + RagIndexOperation operation = requireOperation(operationId); + RagBranchIndexGeneration generation = operation.getGeneration(); + if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED) { + return generation; + } + if (generation.getStatus() != RagBranchIndexGenerationStatus.BUILDING) { + throw new IllegalStateException("Only a building generation can be published"); + } + + Long branchIndexId = generation.getBranchIndex().getId(); + RagBranchIndex branchIndex = branchIndexRepository + .findByIdForPublication(branchIndexId) + .orElseThrow(() -> new IllegalStateException( + "RAG branch index not found: " + branchIndexId)); + generation.setBranchIndex(branchIndex); + + String digest = requireText(manifestDigest, "manifestDigest"); + if (!generation.getRevision().equals(branchIndex.getDesiredCommitHash())) { + // The physical generation is complete and remains useful for exact + // revision reads, but a newer request owns the branch head. Record + // this operation as successful without regressing the active head. + generation.activate(digest, fileCount, chunkCount); + generation.supersede(); + generationRepository.save(generation); + operation.succeed(generation); + operationRepository.save(operation); + return generation; + } + + RagBranchIndexGeneration previous = branchIndex.getActiveGeneration(); + generation.activate(digest, fileCount, chunkCount); + generationRepository.save(generation); + if (previous != null && previous.getId() != null && !previous.getId().equals(generation.getId())) { + previous.supersede(); + generationRepository.save(previous); + } + branchIndex.activate(generation); + branchIndexRepository.save(branchIndex); + operation.succeed(generation); + operationRepository.save(operation); + return generation; + } + + @Transactional + public void fail(long operationId, String errorMessage) { + RagIndexOperation operation = requireOperation(operationId); + if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED) { + return; + } + String failure = requireText(errorMessage, "errorMessage"); + RagBranchIndexGeneration generation = operation.getGeneration(); + Long branchIndexId = generation.getBranchIndex().getId(); + RagBranchIndex branchIndex = branchIndexRepository + .findByIdForPublication(branchIndexId) + .orElseThrow(() -> new IllegalStateException( + "RAG branch index not found: " + branchIndexId)); + generation.setBranchIndex(branchIndex); + generation.fail(failure); + generationRepository.save(generation); + if (generation.getRevision().equals(branchIndex.getDesiredCommitHash())) { + branchIndex.failUpdate(failure); + branchIndexRepository.save(branchIndex); + } + operation.fail(failure); + operationRepository.save(operation); + } + + @Transactional + public Optional findAvailableGeneration( + long projectId, + String branchName, + String revision) { + Optional branchIndex = branchIndexRepository + .findByProjectIdAndBranchName(projectId, requireText(branchName, "branchName")); + if (branchIndex.isEmpty()) { + return Optional.empty(); + } + RagBranchIndex index = branchIndex.get(); + index.markAccessed(); + branchIndexRepository.save(index); + if (index.getActiveGeneration() != null + && index.getActiveGeneration().getRevision().equals(revision)) { + return Optional.of(index.getActiveGeneration()); + } + return generationRepository + .findFirstByBranchIndexIdAndRevisionAndStatusInOrderByCreatedAtDesc( + index.getId(), revision, + List.of(RagBranchIndexGenerationStatus.ACTIVE, + RagBranchIndexGenerationStatus.SUPERSEDED)); + } + + @Transactional + public void heartbeatBuild(long operationId) { + RagIndexOperation operation = requireOperation(operationId); + operation.heartbeat(); + operationRepository.save(operation); + } + + public List findRecoverableOperations(OffsetDateTime updatedBefore) { + return operationRepository.findByStatusInAndUpdatedAtBefore( + List.of(RagIndexOperationStatus.PENDING, RagIndexOperationStatus.RUNNING), + updatedBefore); + } + + static String physicalCollectionName( + Project project, + String branchName, + String revision, + String operationKey) { + long workspaceId = project.getWorkspace() != null && project.getWorkspace().getId() != null + ? project.getWorkspace().getId() + : 0L; + return "cc_w" + workspaceId + + "_p" + project.getId() + + "_b" + digest(branchName).substring(0, 12) + + "_r" + digest(revision).substring(0, 12) + + "_g" + operationKey.substring(0, 12); + } + + static String operationKey( + long projectId, + String branchName, + String fromRevision, + String toRevision, + String representationFingerprint) { + return digest(projectId + "\n" + + branchName + "\n" + + nullToEmpty(fromRevision) + "\n" + + toRevision + "\n" + + nullToEmpty(representationFingerprint)); + } + + private RagIndexOperation requireOperation(long operationId) { + return operationRepository.findById(operationId) + .orElseThrow(() -> new IllegalArgumentException("RAG index operation not found: " + operationId)); + } + + private static void requireProjectIdentity(Project project) { + if (project == null || project.getId() == null) { + throw new IllegalArgumentException("A persisted project is required for branch indexing"); + } + } + + private static String requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " is required"); + } + return value.trim(); + } + + private static String nullToEmpty(String value) { + return value == null ? "" : value; + } + + private static String digest(String value) { + try { + return HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java index dbca6ad2..e37f2e84 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java @@ -7,24 +7,31 @@ import org.rostilos.codecrow.core.model.job.JobTriggerSource; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; import org.rostilos.codecrow.core.service.AnalysisJobService; import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; import java.io.IOException; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -32,11 +39,10 @@ import java.util.function.Consumer; /** - * Implementation of RagOperationsService using single-collection-per-project - * architecture. - * - * Each project has ONE Qdrant collection containing all branches. - * Branch is stored as metadata in each point, allowing multi-branch queries. + * Coordinates both legacy project collections and exact branch generations. + * Existing projects retain the shared collection path until exact branch + * indexing is configured; configured branches use immutable, branch-bound + * generation targets selected through the registry. */ @Service public class RagOperationsServiceImpl implements RagOperationsService { @@ -50,6 +56,11 @@ public class RagOperationsServiceImpl implements RagOperationsService { private final RagBranchIndexRepository ragBranchIndexRepository; private final VcsClientProvider vcsClientProvider; private final RagPipelineClient ragPipelineClient; + private final RagBranchIndexRegistryService branchIndexRegistryService; + private final BranchIndexGenerationBuildService branchGenerationBuildService; + + @Autowired(required = false) + private RagBranchIndexGenerationRepository branchGenerationRepository; @Value("${codecrow.rag.api.enabled:true}") private boolean ragApiEnabled; @@ -62,6 +73,38 @@ public RagOperationsServiceImpl( RagBranchIndexRepository ragBranchIndexRepository, VcsClientProvider vcsClientProvider, RagPipelineClient ragPipelineClient) { + this(ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, null, null); + } + + public RagOperationsServiceImpl( + RagIndexTrackingService ragIndexTrackingService, + IncrementalRagUpdateService incrementalRagUpdateService, + AnalysisLockService analysisLockService, + AnalysisJobService analysisJobService, + RagBranchIndexRepository ragBranchIndexRepository, + VcsClientProvider vcsClientProvider, + RagPipelineClient ragPipelineClient, + RagBranchIndexRegistryService branchIndexRegistryService) { + this(ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, branchIndexRegistryService, null); + } + + @Autowired + public RagOperationsServiceImpl( + RagIndexTrackingService ragIndexTrackingService, + IncrementalRagUpdateService incrementalRagUpdateService, + AnalysisLockService analysisLockService, + AnalysisJobService analysisJobService, + RagBranchIndexRepository ragBranchIndexRepository, + VcsClientProvider vcsClientProvider, + RagPipelineClient ragPipelineClient, + RagBranchIndexRegistryService branchIndexRegistryService, + BranchIndexGenerationBuildService branchGenerationBuildService) { this.ragIndexTrackingService = ragIndexTrackingService; this.incrementalRagUpdateService = incrementalRagUpdateService; this.analysisLockService = analysisLockService; @@ -69,6 +112,8 @@ public RagOperationsServiceImpl( this.ragBranchIndexRepository = ragBranchIndexRepository; this.vcsClientProvider = vcsClientProvider; this.ragPipelineClient = ragPipelineClient; + this.branchIndexRegistryService = branchIndexRegistryService; + this.branchGenerationBuildService = branchGenerationBuildService; } @Override @@ -132,8 +177,26 @@ public boolean triggerIncrementalUpdate( return false; } - String effectiveRawDiff = resolveDiffFromCompletedCheckpoint( - project, branchName, commitHash, rawDiff); + boolean exactGenerationMode = usesExactGenerations(project); + RagBranchIndexKind exactGenerationKind = exactGenerationMode + ? indexKind(project, branchName) : null; + boolean publishBranchAlias = exactGenerationKind == RagBranchIndexKind.PRIMARY + || exactGenerationKind == RagBranchIndexKind.DURABLE; + boolean publishLegacyProjectAlias = exactGenerationKind + == RagBranchIndexKind.PRIMARY; + RagBranchIndexGeneration initialSourceGeneration = exactGenerationMode + ? ragBranchIndexRepository + .findByProjectIdAndBranchName(project.getId(), branchName) + .map(RagBranchIndex::getActiveGeneration) + .orElse(null) + : null; + boolean fullExactSnapshotRequired = exactGenerationMode + && initialSourceGeneration == null; + + String effectiveRawDiff = fullExactSnapshotRequired + ? "" + : resolveDiffFromCompletedCheckpoint( + project, branchName, commitHash, rawDiff); log.info("RAG checkpoint reconciliation complete; parsing effective diff..."); // Parse the diff to find changed files @@ -147,7 +210,8 @@ public boolean triggerIncrementalUpdate( log.info("Diff parsed: added={}, modified={}, deleted={}", addedFiles, modifiedFiles, deletedFiles); - if (addedOrModifiedSize == 0 && deletedFiles.isEmpty()) { + if (addedOrModifiedSize == 0 && deletedFiles.isEmpty() + && !exactGenerationMode) { log.info("Skipping RAG incremental update - no files changed in diff"); return true; } @@ -198,17 +262,95 @@ public boolean triggerIncrementalUpdate( String workspaceSlug = vcsRepoBinding.getExternalNamespace(); String repoSlug = vcsRepoBinding.getExternalRepoSlug(); - // Perform the actual incremental update - Map result = incrementalRagUpdateService.performIncrementalUpdate( - project, - vcsConnection, - workspaceSlug, - repoSlug, - branchName, - commitHash, - addedFiles, - modifiedFiles, - deletedFiles); + RagBranchIndexGeneration sourceGeneration = null; + RagBranchIndexRegistryService.BuildRegistration branchBuild = null; + if (exactGenerationMode) { + sourceGeneration = initialSourceGeneration; + if (sourceGeneration != null) { + branchBuild = branchIndexRegistryService.registerBuild( + project, + branchName, + exactGenerationKind, + sourceGeneration.getRevision(), + commitHash, + sourceGeneration.getRepresentationFingerprint()); + branchIndexRegistryService.startBuild( + branchBuild.operation().getId(), + job != null ? job.getId() : null); + } + } + + Map result; + try { + if (fullExactSnapshotRequired) { + var ragConfig = project.getConfiguration().ragConfig(); + result = branchGenerationBuildService.build( + project, + vcsConnection, + workspaceSlug, + repoSlug, + branchName, + commitHash, + exactGenerationKind, + ragConfig.includePatterns(), + ragConfig.excludePatterns(), + job != null ? job.getId() : null); + } else if (exactGenerationMode) { + result = incrementalRagUpdateService.performIncrementalUpdate( + project, + vcsConnection, + workspaceSlug, + repoSlug, + branchName, + commitHash, + addedFiles, + modifiedFiles, + deletedFiles, + sourceGeneration.getRevision(), + sourceGeneration.getCollectionName(), + branchBuild.generation().getCollectionName(), + false, + false); + } else { + result = incrementalRagUpdateService.performIncrementalUpdate( + project, + vcsConnection, + workspaceSlug, + repoSlug, + branchName, + commitHash, + addedFiles, + modifiedFiles, + deletedFiles); + } + if (exactGenerationMode && !fullExactSnapshotRequired) { + Object digest = result.get("generation_manifest_sha256"); + if (!(digest instanceof String manifestDigest) + || manifestDigest.isBlank()) { + throw new IllegalStateException( + "Advanced RAG generation has no manifest digest"); + } + RagBranchIndexGeneration published = branchIndexRegistryService.publish( + branchBuild.operation().getId(), + manifestDigest, + ((Number) result.getOrDefault("document_count", 0)).intValue(), + ((Number) result.getOrDefault("chunk_count", 0)).intValue()); + publishReadableAliasesIfActive( + project, branchName, commitHash, + branchBuild.generation().getCollectionName(), + published, publishBranchAlias, + publishLegacyProjectAlias); + } + } catch (Exception generationFailure) { + if (branchBuild != null) { + branchIndexRegistryService.fail( + branchBuild.operation().getId(), + generationFailure.getMessage() != null + ? generationFailure.getMessage() + : generationFailure.getClass().getSimpleName()); + } + throw generationFailure; + } int filesUpdated = (Integer) result.getOrDefault("updatedFiles", 0); int filesDeleted = (Integer) result.getOrDefault("deletedFiles", 0); @@ -284,6 +426,30 @@ public boolean triggerIncrementalUpdate( } } + private void publishReadableAliasesIfActive( + Project project, + String branch, + String revision, + String collectionTarget, + RagBranchIndexGeneration published, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias) { + if (published == null + || published.getStatus() != RagBranchIndexGenerationStatus.ACTIVE + || !publishBranchAlias) { + return; + } + try { + ragPipelineClient.publishGenerationAliases( + project.getWorkspace().getName(), project.getNamespace(), + branch, revision, collectionTarget, + true, publishLegacyProjectAlias); + } catch (IOException aliasFailure) { + log.warn("Readable alias publication failed for active RAG generation {}: {}", + published.getId(), aliasFailure.getMessage()); + } + } + /** * Rebuilds the effective range from the last completed RAG checkpoint. * The caller's branch-analysis diff is used only when no completed @@ -411,9 +577,6 @@ public boolean updateBranchIndex( Project project, String targetBranch, Consumer> eventConsumer) { - // Update branch index - calculates diff between base branch and target branch - // Unlike ensureBranchIndexForPrTarget, this always recalculates the full diff - if (!isRagEnabled(project)) { log.debug("RAG not enabled for project={}", project.getId()); return false; @@ -424,6 +587,17 @@ public boolean updateBranchIndex( return false; } + if (!targetBranch.equals(getBaseBranch(project)) + && !shouldHaveBranchIndex(project, targetBranch)) { + log.info("Skipping branch index update for non-retained branch: project={}, branch={}", + project.getId(), targetBranch); + eventConsumer.accept(Map.of( + "type", "info", + "state", "rag_skipped", + "message", "Branch is not configured as a retained RAG branch")); + return false; + } + // Get VCS connection info VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); if (vcsRepoBinding == null) { @@ -445,7 +619,28 @@ public boolean updateBranchIndex( try { VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); - log.info("Updating branch index for project={}, branch={} (diff vs {})", + String targetCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, targetBranch); + Optional completedBranchIndex = ragBranchIndexRepository + .findByProjectIdAndBranchName(project.getId(), targetBranch) + .filter(index -> index.getCommitHash() != null && !index.getCommitHash().isBlank()); + + if (completedBranchIndex.isPresent()) { + String checkpoint = completedBranchIndex.get().getCommitHash(); + if (checkpoint.equals(targetCommit)) { + log.info("Branch index already represents project={}, branch={}, commit={}", + project.getId(), targetBranch, targetCommit); + return true; + } + + log.info("Updating branch index from completed checkpoint: project={}, branch={}, {}..{}", + project.getId(), targetBranch, checkpoint, targetCommit); + // triggerIncrementalUpdate resolves the exact checkpoint range. Passing an + // empty supplied diff avoids the former redundant base-to-target compare. + return triggerIncrementalUpdate( + project, targetBranch, targetCommit, "", eventConsumer); + } + + log.info("Seeding legacy branch index for project={}, branch={} (diff vs {})", project.getId(), targetBranch, baseBranch); eventConsumer.accept(Map.of( @@ -453,7 +648,8 @@ public boolean updateBranchIndex( "state", "branch_index", "message", String.format("Calculating diff between '%s' and '%s'", baseBranch, targetBranch))); - // Always get fresh diff between base branch and target branch + // Compatibility path for a branch that has no checkpoint yet. The exact + // generation builder replaces this with a complete verified snapshot. String rawDiff = vcsClient.getBranchDiff(workspaceSlug, repoSlug, baseBranch, targetBranch); if (rawDiff == null || rawDiff.isEmpty()) { @@ -461,11 +657,16 @@ public boolean updateBranchIndex( eventConsumer.accept(Map.of( "type", "info", "message", String.format("Branch '%s' has same content as '%s'", targetBranch, baseBranch))); + if (usesExactGenerations(project)) { + // No completed checkpoint means there is still no exact + // target-branch generation. An empty tree delta must seed + // the complete revision rather than report a false success. + return triggerIncrementalUpdate( + project, targetBranch, targetCommit, "", eventConsumer); + } return true; } - String targetCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, targetBranch); - log.info("Branch diff found: {} bytes, triggering incremental update for branch={}, commit={}", rawDiff.length(), targetBranch, targetCommit); @@ -519,7 +720,7 @@ public boolean ensureBranchIndexForPrTarget( String repoSlug = vcsRepoBinding.getExternalRepoSlug(); // Get base branch (main branch) - String baseBranch = getBaseBranch(project); + String baseBranch = getBaseBranch(project); // Same branch? Already indexed via main index if (targetBranch.equals(baseBranch)) { @@ -527,6 +728,27 @@ public boolean ensureBranchIndexForPrTarget( return true; } + // Exact-generation mode seeds an immutable snapshot of the target + // revision. It must not first compute the potentially enormous + // primary-to-target diff that motivated multi-branch indexing. + if (usesExactGenerations(project)) { + if (!shouldHaveBranchIndex(project, targetBranch) + && !shouldCreateTransientBranchIndex(project, targetBranch)) { + return false; + } + try { + VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); + String targetCommit = vcsClient.getLatestCommitHash( + workspaceSlug, repoSlug, targetBranch); + return triggerIncrementalUpdate( + project, targetBranch, targetCommit, "", eventConsumer); + } catch (Exception failure) { + log.warn("Failed to seed exact branch generation for project={}, branch={}: {}", + project.getId(), targetBranch, failure.getMessage()); + return false; + } + } + // Check if branch already has indexed data (RagBranchIndex exists) // Note: We still proceed with diff check to ensure any new changes are indexed boolean branchIndexExists = isBranchIndexReady(project, targetBranch); @@ -616,8 +838,26 @@ public boolean deleteBranchIndex( "state", "branch_delete", "message", String.format("Deleting RAG index for branch '%s'", branchName))); - // Delete from RAG pipeline - boolean success = ragPipelineClient.deleteBranch(workspaceSlug, projectSlug, branchName); + boolean success; + Optional trackedIndex = ragBranchIndexRepository + .findByProjectIdAndBranchName(project.getId(), branchName); + List generations = trackedIndex.isPresent() + && branchGenerationRepository != null + ? branchGenerationRepository.findByBranchIndexIdOrderByCreatedAtDesc( + trackedIndex.get().getId()) + : List.of(); + if (!generations.isEmpty()) { + success = true; + for (RagBranchIndexGeneration generation : generations) { + success &= ragPipelineClient.deleteBranch( + project.getWorkspace().getName(), project.getNamespace(), + branchName, generation.getCollectionName()); + } + } else { + // Backward-compatible cleanup for the legacy shared collection. + success = ragPipelineClient.deleteBranch( + workspaceSlug, projectSlug, branchName); + } if (success) { // Clean up database tracking @@ -663,8 +903,13 @@ public Map cleanupStaleBranches( String baseBranch = getBaseBranch(project); try { - // Get indexed branches - List indexedBranches = ragPipelineClient.getIndexedBranches(workspaceSlug, projectSlug); + // The durable registry is authoritative for exact-generation + // branches. Merge it with legacy shared-collection discovery so + // stale cleanup remains compatible with both storage models. + Set indexedBranches = new LinkedHashSet<>( + ragBranchIndexRepository.findBranchNamesByProjectId(project.getId())); + indexedBranches.addAll( + ragPipelineClient.getIndexedBranches(workspaceSlug, projectSlug)); // Determine branches to keep: base branch + active branches Set branchesToKeep = new HashSet<>(activeBranches); @@ -696,9 +941,8 @@ public Map cleanupStaleBranches( for (String branch : staleBranches) { try { - boolean success = ragPipelineClient.deleteBranch(workspaceSlug, projectSlug, branch); + boolean success = deleteBranchIndex(project, branch, eventConsumer); if (success) { - ragBranchIndexRepository.deleteByProjectIdAndBranchName(project.getId(), branch); deletedBranches.add(branch); } else { failedBranches.add(branch); @@ -769,6 +1013,17 @@ public boolean ensureRagIndexUpToDate( eventConsumer); } + if (!shouldHaveBranchIndex(project, targetBranch) + && !shouldCreateTransientBranchIndex(project, targetBranch)) { + log.info("Skipping RAG preparation for non-indexed PR target: project={}, branch={}", + project.getId(), targetBranch); + eventConsumer.accept(Map.of( + "type", "info", + "state", "rag_skipped", + "message", "PR target is not configured for retained or temporary RAG indexing")); + return false; + } + // Case 2: Different branch - ensure main index is ready, then ensure branch is // indexed log.info("Target branch '{}' differs from base branch '{}' - will ensure branch index", targetBranch, @@ -820,6 +1075,17 @@ private boolean ensureMainIndexUpToDate( String indexedCommit = indexStatus.get().getIndexedCommitHash(); + if (usesExactGenerations(project) + && ragBranchIndexRepository + .findByProjectIdAndBranchName(project.getId(), branchName) + .map(RagBranchIndex::getActiveGeneration) + .isEmpty()) { + log.info("Creating first exact primary generation for project={}, branch={}, commit={}", + project.getId(), branchName, currentCommit); + return triggerIncrementalUpdate( + project, branchName, currentCommit, "", eventConsumer); + } + // If commits match, index is up to date if (currentCommit.equals(indexedCommit)) { log.debug("Main RAG index is up-to-date for project={}, commit={}", project.getId(), currentCommit); @@ -873,7 +1139,12 @@ private boolean ensureBranchIndexUpToDate( .findByProjectIdAndBranchName(project.getId(), targetBranch); if (branchIndexOpt.isEmpty()) { - // No branch index exists - create it by getting full diff vs main + // The exact path seeds from a complete target snapshot; the legacy + // implementation below retains the former base-diff behavior. + if (usesExactGenerations(project)) { + return triggerIncrementalUpdate( + project, targetBranch, currentCommit, "", eventConsumer); + } log.info("No RagBranchIndex entry found for project={}, branch={} - will create with full diff vs {}", project.getId(), targetBranch, baseBranch); return ensureBranchIndexForPrTarget(project, targetBranch, eventConsumer); @@ -884,6 +1155,11 @@ private boolean ensureBranchIndexUpToDate( log.info("Existing RagBranchIndex for project={}, branch={}: indexedCommit={}", project.getId(), targetBranch, indexedCommit); + if (usesExactGenerations(project) && branchIndex.getActiveGeneration() == null) { + return triggerIncrementalUpdate( + project, targetBranch, currentCommit, "", eventConsumer); + } + // If commits match, index is up to date if (currentCommit.equals(indexedCommit)) { log.info("Branch index is up-to-date for project={}, branch={}, commit={}", @@ -926,4 +1202,22 @@ private boolean ensureBranchIndexUpToDate( project, targetBranch, currentCommit, rawDiff, eventConsumer); } + private boolean usesExactGenerations(Project project) { + return branchIndexRegistryService != null + && branchGenerationBuildService != null + && project != null + && project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); + } + + private RagBranchIndexKind indexKind(Project project, String branchName) { + if (branchName.equals(getBaseBranch(project))) { + return RagBranchIndexKind.PRIMARY; + } + return shouldHaveBranchIndex(project, branchName) + ? RagBranchIndexKind.DURABLE + : RagBranchIndexKind.TRANSIENT; + } + } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java index d43f40f0..92be1916 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java @@ -39,6 +39,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.rostilos.codecrow.queue.RedisQueueService; +import org.rostilos.codecrow.ragengine.source.RepositorySourceTreeIdentity; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.scheduling.annotation.Async; @@ -240,6 +241,8 @@ private Map performIndexing( // Extract the downloaded archive locally jobService.logToJob(job, JobLogLevel.INFO, "extraction", "Extracting repository archive..."); extractArchiveFileAndCleanup(tempArchiveFile, tempDir); + String sourceTreeSha256 = + RepositorySourceTreeIdentity.sha256(tempDir); if (includePatterns != null && !includePatterns.isEmpty()) { log.info("Using {} include patterns from project config", includePatterns.size()); @@ -262,6 +265,7 @@ private Map performIndexing( "project", project.getNamespace(), "branch", branch, "commit", commitHash, + "source_tree_sha256", sourceTreeSha256, "preserve_other_branches", config.ragConfig().isMultiBranchEnabled(), "cleanup_repo_path", true, "include_patterns", includePatterns != null ? includePatterns : java.util.List.of(), diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentity.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentity.java new file mode 100644 index 00000000..d6ee8914 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentity.java @@ -0,0 +1,183 @@ +package org.rostilos.codecrow.ragengine.source; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Set; + +/** + * Canonical content identity for a normalized repository source tree. + * + *

    The digest intentionally excludes Git's administrative directory. It + * includes every other regular file and symlink using repository-relative + * paths and exact bytes, matching the RAG consumer's verifier.

    + */ +public final class RepositorySourceTreeIdentity { + private static final byte[] SCHEMA = + "codecrow.repository-source-tree".getBytes(StandardCharsets.US_ASCII); + + private RepositorySourceTreeIdentity() { + } + + public static String sha256(Path repositoryRoot) throws IOException { + Path root = repositoryRoot.toAbsolutePath().normalize(); + if (!Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Repository source path is not a directory: " + root); + } + + List entries = new ArrayList<>(); + Files.walkFileTree(root, new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory( + Path directory, + BasicFileAttributes attributes + ) { + if (!directory.equals(root) + && ".git".equals(root.relativize(directory).getName(0).toString())) { + return FileVisitResult.SKIP_SUBTREE; + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) + throws IOException { + Path relative = root.relativize(file); + if (relative.getNameCount() > 0 + && ".git".equals(relative.getName(0).toString())) { + return FileVisitResult.CONTINUE; + } + if (attributes.isSymbolicLink()) { + entries.add(new Entry("symlink", relative, Files.readSymbolicLink(file))); + } else if (attributes.isRegularFile()) { + entries.add(new Entry("file", relative, file)); + } else { + throw new IOException( + "Repository source contains an unsupported filesystem entry: " + + relative.toString().replace('\\', '/')); + } + return FileVisitResult.CONTINUE; + } + }); + entries.sort(Comparator.comparing( + entry -> entry.relativePath().toString().replace('\\', '/') + .getBytes(StandardCharsets.UTF_8), + RepositorySourceTreeIdentity::compareUnsigned + )); + + MessageDigest digest = newDigest(); + feedFramed(digest, SCHEMA); + byte[] buffer = new byte[1024 * 1024]; + for (Entry entry : entries) { + feedFramed(digest, entry.kind().getBytes(StandardCharsets.US_ASCII)); + feedFramed( + digest, + entry.relativePath().toString().replace('\\', '/') + .getBytes(StandardCharsets.UTF_8) + ); + if ("symlink".equals(entry.kind())) { + feedFramed( + digest, + entry.value().toString().getBytes(StandardCharsets.UTF_8) + ); + continue; + } + + BasicFileAttributes attributes = Files.readAttributes( + entry.value(), + BasicFileAttributes.class, + LinkOption.NOFOLLOW_LINKS + ); + if (!attributes.isRegularFile()) { + throw new IOException( + "Repository source entry changed or is not a regular file: " + + entry.relativePath().toString().replace('\\', '/')); + } + long expectedSize = attributes.size(); + digest.update(longBytes(expectedSize)); + long observedSize = 0; + Set options = Set.of( + StandardOpenOption.READ, + LinkOption.NOFOLLOW_LINKS + ); + try (SeekableByteChannel channel = Files.newByteChannel( + entry.value(), + options + )) { + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer); + int count; + while ((count = channel.read(byteBuffer)) != -1) { + if (count > 0) { + digest.update(buffer, 0, count); + observedSize += count; + } + byteBuffer.clear(); + } + } + if (observedSize != expectedSize) { + throw new IOException( + "Repository source changed while it was being attested: " + + entry.relativePath().toString().replace('\\', '/')); + } + } + digest.update(longBytes(entries.size())); + return toHex(digest.digest()); + } + + private static int compareUnsigned(byte[] left, byte[] right) { + int length = Math.min(left.length, right.length); + for (int index = 0; index < length; index++) { + int comparison = Integer.compare( + Byte.toUnsignedInt(left[index]), + Byte.toUnsignedInt(right[index]) + ); + if (comparison != 0) { + return comparison; + } + } + return Integer.compare(left.length, right.length); + } + + private static void feedFramed(MessageDigest digest, byte[] value) { + digest.update(longBytes(value.length)); + digest.update(value); + } + + private static byte[] longBytes(long value) { + return ByteBuffer.allocate(Long.BYTES).putLong(value).array(); + } + + private static MessageDigest newDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } + } + + private static String toHex(byte[] bytes) { + StringBuilder result = new StringBuilder(bytes.length * 2); + for (byte value : bytes) { + result.append(Character.forDigit((value >>> 4) & 0xf, 16)); + result.append(Character.forDigit(value & 0xf, 16)); + } + return result.toString(); + } + + private record Entry(String kind, Path relativePath, Path value) { + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java new file mode 100644 index 00000000..e9e14f20 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java @@ -0,0 +1,221 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.rag.RagIndexOperation; +import org.rostilos.codecrow.core.model.rag.RagIndexOperationStatus; +import org.rostilos.codecrow.core.model.vcs.VcsConnection; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.springframework.test.util.ReflectionTestUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class BranchIndexGenerationBuildServiceTest { + @Mock private BranchArchiveService archiveService; + @Mock private RagPipelineClient pipelineClient; + @Mock private RagBranchIndexRegistryService registryService; + + private BranchIndexGenerationBuildService service; + private Project project; + private RagBranchIndexGeneration generation; + private RagIndexOperation operation; + + @BeforeEach + void setUp() { + service = new BranchIndexGenerationBuildService( + archiveService, pipelineClient, registryService); + project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + RagBranchIndex branchIndex = new RagBranchIndex( + project, "develop", RagBranchIndexKind.DURABLE); + branchIndex.setId(10L); + generation = new RagBranchIndexGeneration( + branchIndex, "develop-400", "opaque-generation-target", + null, null, null); + generation.setId(20L); + operation = new RagIndexOperation( + project, "develop", null, "develop-400", "operation-key"); + operation.setId(30L); + operation.setGeneration(generation); + } + + @Test + void buildsPinnedSnapshotPublishesManifestAndRemovesTemporaryTree() throws Exception { + when(registryService.registerBuild( + project, "develop", RagBranchIndexKind.DURABLE, + null, "develop-400", null)) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + when(pipelineClient.indexRepository( + anyString(), eq("workspace"), eq("namespace"), + eq("develop"), eq("develop-400"), eq(List.of("src/**")), + eq(List.of("vendor/**")), eq("opaque-generation-target"), + eq(false), eq(false))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-400", + "document_count", 231, + "chunk_count", 400)); + when(registryService.publish(30L, "manifest-400", 231, 400)) + .thenAnswer(invocation -> { + generation.activate("manifest-400", 231, 400); + return generation; + }); + ReflectionTestUtils.setField(project, "namespace", "namespace"); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + + Map result = service.build( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of("src/**"), List.of("vendor/**"), 77L); + + assertThat(result).containsEntry( + "generation_manifest_sha256", "manifest-400"); + verify(registryService).startBuild(30L, 77L); + verify(registryService).publish(30L, "manifest-400", 231, 400); + verify(pipelineClient).publishGenerationAliases( + "workspace", "namespace", "develop", "develop-400", + "opaque-generation-target", true, false); + ArgumentCaptor snapshot = ArgumentCaptor.forClass(Path.class); + verify(archiveService).downloadAndExtractSnapshotToDirectory( + any(), eq("provider-workspace"), eq("repo"), + eq("develop-400"), isNull(), snapshot.capture()); + assertThat(Files.exists(snapshot.getValue())).isFalse(); + } + + @Test + void missingManifestFailsOperationAndDoesNotPublish() throws Exception { + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), isNull())) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + when(pipelineClient.indexRepository( + anyString(), any(), any(), any(), any(), any(), any(), any(), + anyBoolean(), anyBoolean())) + .thenReturn(Map.of("document_count", 231)); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + + assertThatThrownBy(() -> service.build( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of())) + .isInstanceOf(IOException.class) + .hasMessageContaining("no manifest digest"); + + verify(registryService).fail(30L, + "RAG full branch generation has no manifest digest"); + verify(registryService, never()).publish(anyLong(), anyString(), anyInt(), anyInt()); + } + + @Test + void successfulIdempotentOperationIsReusedWithoutProviderOrVectorCalls() throws Exception { + operation.setStatus(RagIndexOperationStatus.SUCCEEDED); + generation.activate("manifest-400", 231, 400); + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), isNull())) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, true)); + + Map result = service.build( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of()); + + assertThat(result).containsEntry("status", "reused") + .containsEntry("collection_target", "opaque-generation-target") + .containsEntry("generation_manifest_sha256", "manifest-400"); + verifyNoInteractions(archiveService, pipelineClient); + } + + @Test + void explicitOperatorRefreshBuildsANewGenerationEvenForTheSameRevision() throws Exception { + when(registryService.registerBuild( + eq(project), eq("develop"), eq(RagBranchIndexKind.DURABLE), + isNull(), eq("develop-400"), eq("operator-refresh:77"))) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + when(pipelineClient.indexRepository( + anyString(), anyString(), anyString(), eq("develop"), eq("develop-400"), + anyList(), anyList(), eq("opaque-generation-target"), + eq(false), eq(false), any())) + .thenReturn(Map.of( + "generation_manifest_sha256", "fresh-manifest", + "document_count", 231, + "chunk_count", 400)); + when(registryService.publish(30L, "fresh-manifest", 231, 400)) + .thenAnswer(invocation -> { + generation.activate("fresh-manifest", 231, 400); + return generation; + }); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + + service.rebuild(project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of(), 77L, ignored -> { }); + + verify(archiveService).downloadAndExtractSnapshotToDirectory( + any(), eq("provider-workspace"), eq("repo"), eq("develop-400"), isNull(), any()); + verify(registryService).publish(30L, "fresh-manifest", 231, 400); + verify(pipelineClient).publishGenerationAliases( + "workspace", "namespace", "develop", "develop-400", + "opaque-generation-target", true, false); + } + + @Test + void staleCompletedGenerationDoesNotPublishReadableAliases() throws Exception { + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), isNull())) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + when(pipelineClient.indexRepository( + anyString(), anyString(), anyString(), anyString(), anyString(), + anyList(), anyList(), anyString(), eq(false), eq(false))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-400", + "document_count", 231, + "chunk_count", 400)); + generation.activate("manifest-400", 231, 400); + generation.supersede(); + when(registryService.publish(30L, "manifest-400", 231, 400)) + .thenReturn(generation); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + + service.build(project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of()); + + verify(pipelineClient, never()).publishGenerationAliases( + anyString(), anyString(), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java new file mode 100644 index 00000000..96896140 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java @@ -0,0 +1,59 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; + +import java.util.List; + +import static org.mockito.Mockito.*; + +class RagBranchOperatorAliasReconciliationServiceTest { + + @Test + void restoresReadableAliasesForDurableAndPrimaryGenerationsOnly() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + + RagBranchIndex primary = index("main", RagBranchIndexKind.PRIMARY, "main-target"); + RagBranchIndex durable = index("develop", RagBranchIndexKind.DURABLE, "develop-target"); + RagBranchIndex transientIndex = index("release", RagBranchIndexKind.TRANSIENT, "release-target"); + when(repository.findAll()).thenReturn(List.of(primary, durable, transientIndex)); + + service.reconcileActiveGenerationAliases(); + + verify(client).publishGenerationAliases( + "workspace", "project", "main", "revision", "main-target", true, true); + verify(client).publishGenerationAliases( + "workspace", "project", "develop", "revision", "develop-target", true, false); + verifyNoMoreInteractions(client); + } + + private static RagBranchIndex index( + String branch, + RagBranchIndexKind kind, + String target) { + Workspace workspace = mock(Workspace.class); + when(workspace.getName()).thenReturn("workspace"); + Project project = mock(Project.class); + when(project.getWorkspace()).thenReturn(workspace); + when(project.getNamespace()).thenReturn("project"); + when(project.getId()).thenReturn(1L); + RagBranchIndexGeneration generation = mock(RagBranchIndexGeneration.class); + when(generation.getRevision()).thenReturn("revision"); + when(generation.getCollectionName()).thenReturn(target); + RagBranchIndex index = mock(RagBranchIndex.class); + when(index.getProject()).thenReturn(project); + when(index.getBranchName()).thenReturn(branch); + when(index.getIndexKind()).thenReturn(kind); + when(index.getActiveGeneration()).thenReturn(generation); + return index; + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java new file mode 100644 index 00000000..404706b4 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java @@ -0,0 +1,31 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.rag.RagIndexOperation; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; + +import java.time.OffsetDateTime; +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +class RagIndexOperationRecoveryServiceTest { + + @Test + void abandonedPersistedOperationBecomesTerminalWithUsefulDiagnostic() { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + RagIndexOperation operation = new RagIndexOperation(); + operation.setId(81L); + operation.setBranchName("develop"); + when(registry.findRecoverableOperations(any(OffsetDateTime.class))) + .thenReturn(List.of(operation)); + + new RagIndexOperationRecoveryService(registry, 30) + .failAbandonedOperations(); + + verify(registry).fail(eq(81L), argThat(message -> + message.contains("stopped heartbeating") + && message.contains("previous active generation was preserved"))); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java new file mode 100644 index 00000000..072fe996 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java @@ -0,0 +1,59 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.project.config.ProjectConfig; +import org.rostilos.codecrow.core.model.project.config.RagConfig; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.OffsetDateTime; +import java.util.List; + +import static org.mockito.Mockito.*; + +class RagTransientBranchIndexCleanupServiceTest { + + @Test + void deletesExpiredTransientGenerationUsingProjectTenantCoordinates() throws Exception { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + Project project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + project.setNamespace("namespace"); + Workspace workspace = new Workspace(); + workspace.setName("workspace"); + project.setWorkspace(workspace); + project.setConfiguration(new ProjectConfig( + false, "master", null, + new RagConfig(true, "master", null, null, + true, 30, List.of("develop"), true))); + RagBranchIndex index = new RagBranchIndex( + project, "release/candidate", RagBranchIndexKind.TRANSIENT); + index.setId(10L); + index.setLastAccessedAt(OffsetDateTime.now().minusDays(31)); + RagBranchIndexGeneration generation = mock(RagBranchIndexGeneration.class); + when(generation.getCollectionName()).thenReturn("opaque-transient-target"); + when(branches.findByIndexKind(RagBranchIndexKind.TRANSIENT)) + .thenReturn(List.of(index)); + when(generations.findByBranchIndexIdOrderByCreatedAtDesc(10L)) + .thenReturn(List.of(generation)); + when(pipeline.deleteBranch( + "workspace", "namespace", "release/candidate", + "opaque-transient-target")) + .thenReturn(true); + + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + + verify(branches).delete(index); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java index acc891ec..46d092d5 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java @@ -7,8 +7,10 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -22,6 +24,9 @@ class RagPipelineClientTest { private RagPipelineClient client; private ObjectMapper objectMapper; + @TempDir + Path repositoryPath; + @BeforeEach void setUp() throws IOException { mockWebServer = new MockWebServer(); @@ -149,6 +154,57 @@ void testApplyChanges_DeleteOnlyOmitsRepositoryRoot() throws Exception { assertThat(payload).doesNotContainKey("repo_base"); } + @Test + @SuppressWarnings("unchecked") + void advanceGenerationBindsSourceAndTargetCollectionsAndRevisions() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("{\"generation_manifest_sha256\":\"digest\"}") + .addHeader("Content-Type", "application/json")); + + client.advanceGeneration( + List.of("src/Changed.java"), List.of("src/Deleted.java"), + "/tmp/repository", "workspace", "project", "develop", + "develop-400", "develop-401", + "a".repeat(64), + "opaque-generation-400", "opaque-generation-401"); + + RecordedRequest request = mockWebServer.takeRequest(); + assertThat(request.getPath()).isEqualTo("/index/advance-generation"); + Map payload = objectMapper.readValue( + request.getBody().readUtf8(), Map.class); + assertThat(payload) + .containsEntry("source_commit", "develop-400") + .containsEntry("commit", "develop-401") + .containsEntry("source_tree_sha256", "a".repeat(64)) + .containsEntry("source_collection_target", "opaque-generation-400") + .containsEntry("collection_target", "opaque-generation-401") + .containsEntry("branch", "develop"); + assertThat((List) payload.get("updated_file_paths")) + .containsExactly("src/Changed.java"); + assertThat((List) payload.get("deleted_file_paths")) + .containsExactly("src/Deleted.java"); + } + + @Test + @SuppressWarnings("unchecked") + void exactGenerationPublicationRequestsReadableAliases() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("{\"generation_manifest_sha256\":\"digest\"}") + .addHeader("Content-Type", "application/json")); + + client.advanceGeneration( + List.of(), List.of(), "/tmp/repository", "workspace", "project", "main", + "main-400", "main-401", "a".repeat(64), + "opaque-generation-400", "opaque-generation-401", true, true); + + RecordedRequest request = mockWebServer.takeRequest(); + Map payload = objectMapper.readValue( + request.getBody().readUtf8(), Map.class); + assertThat(payload) + .containsEntry("publish_branch_alias", true) + .containsEntry("publish_legacy_project_alias", true); + } + @Test void testSemanticSearch_Success() throws Exception { Map mockResponse = Map.of( @@ -365,7 +421,7 @@ void testIndexRepository_Success() throws Exception { .addHeader("Content-Type", "application/json")); Map result = client.indexRepository( - "/tmp/repo", "ws", "proj", "main", "abc123", null, null); + repositoryPath.toString(), "ws", "proj", "main", "abc123", null, null); assertThat(result).containsEntry("document_count", 42); RecordedRequest request = mockWebServer.takeRequest(); @@ -382,7 +438,7 @@ void testIndexRepository_WithExcludePatterns() throws Exception { List patterns = List.of("*.log", "vendor/**"); Map result = client.indexRepository( - "/tmp/repo", "ws", "proj", "main", "abc123", null, patterns); + repositoryPath.toString(), "ws", "proj", "main", "abc123", null, patterns); assertThat(result).containsEntry("document_count", 30); RecordedRequest request = mockWebServer.takeRequest(); @@ -402,6 +458,30 @@ void testIndexRepository_WhenDisabled() throws Exception { assertThat(mockWebServer.getRequestCount()).isEqualTo(0); } + @Test + void testIndexRepository_StreamForwardsProgressAndReturnsTerminalResult() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("data: {\"type\":\"progress\",\"stage\":\"indexing\",\"indexedChunks\":5,\"estimatedChunks\":150,\"estimatedRemainingMs\":9000}\n\n" + + "data: {\"type\":\"complete\",\"result\":{\"document_count\":2,\"chunk_count\":5,\"generation_manifest_sha256\":\"digest\"}}\n\n") + .addHeader("Content-Type", "text/event-stream")); + List> progress = new ArrayList<>(); + + Map result = client.indexRepository( + repositoryPath.toString(), "ws", "proj", "develop", "abc123", + List.of("src/**"), List.of("vendor/**"), "generation-target", + progress::add); + + assertThat(progress).hasSize(1); + assertThat(progress.get(0)).containsEntry("indexedChunks", 5) + .containsEntry("estimatedChunks", 150) + .containsEntry("estimatedRemainingMs", 9000); + assertThat(result).containsEntry("chunk_count", 5) + .containsEntry("generation_manifest_sha256", "digest"); + RecordedRequest request = mockWebServer.takeRequest(); + assertThat(request.getPath()).endsWith("/index/repository/stream"); + assertThat(request.getHeader("Accept")).isEqualTo("text/event-stream"); + } + // ── deleteBranch tests ─────────────────────────────────────────────────── @Test diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateServiceTest.java index 2d67645f..ecb19a5e 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateServiceTest.java @@ -647,6 +647,46 @@ void testPerformIncrementalUpdate_BothDeletesAndUpdates() throws Exception { anyString(), eq("test-ws"), eq("test-proj"), eq("main"), eq("abc123")); } + @Test + void exactEmptyDeltaStillAdvancesImmutableRevision() throws Exception { + setupProjectWithWorkspace(); + when(branchArchiveService.downloadAndExtractSnapshotToDirectory( + any(), eq("ws-slug"), eq("repo-slug"), eq("develop-401"), + isNull(), any())) + .thenReturn(archiveSnapshot(Set.of(), Set.of())); + when(ragPipelineClient.advanceGeneration( + eq(List.of()), eq(List.of()), anyString(), + eq("test-ws"), eq("test-proj"), eq("develop"), + eq("develop-400"), eq("develop-401"), + anyString(), + eq("generation-400"), eq("generation-401"), + eq(true), eq(false))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-401", + "document_count", 231, + "chunk_count", 400)); + + Map result = service.performIncrementalUpdate( + testProject, new VcsConnection(), "ws-slug", "repo-slug", + "develop", "develop-401", Set.of(), Set.of(), Set.of(), + "develop-400", "generation-400", "generation-401", + true, false); + + assertThat(result) + .containsEntry("status", "completed") + .containsEntry("generation_manifest_sha256", "manifest-401"); + verify(ragPipelineClient).advanceGeneration( + eq(List.of()), eq(List.of()), anyString(), + eq("test-ws"), eq("test-proj"), eq("develop"), + eq("develop-400"), eq("develop-401"), + anyString(), + eq("generation-400"), eq("generation-401"), + eq(true), eq(false)); + verify(ragPipelineClient, never()).applyChanges( + anyList(), anyList(), nullable(String.class), + anyString(), anyString(), anyString(), anyString()); + } + // ── Helpers ────────────────────────────────────────────────────────────── private void setupProjectWithWorkspace() { diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java new file mode 100644 index 00000000..7b5b3052 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java @@ -0,0 +1,253 @@ +package org.rostilos.codecrow.ragengine.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.*; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagIndexOperationRepository; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class RagBranchIndexRegistryServiceTest { + + @Mock + private RagBranchIndexRepository branchIndexRepository; + @Mock + private RagBranchIndexGenerationRepository generationRepository; + @Mock + private RagIndexOperationRepository operationRepository; + + private RagBranchIndexRegistryService service; + private Project project; + + @BeforeEach + void setUp() { + service = new RagBranchIndexRegistryService( + branchIndexRepository, generationRepository, operationRepository); + project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + lenient().when(branchIndexRepository.save(any())).thenAnswer(invocation -> { + RagBranchIndex value = invocation.getArgument(0); + if (value.getId() == null) { + value.setId(10L); + } + return value; + }); + when(generationRepository.save(any())).thenAnswer(invocation -> { + RagBranchIndexGeneration value = invocation.getArgument(0); + if (value.getId() == null) { + value.setId(20L); + } + return value; + }); + when(operationRepository.save(any())).thenAnswer(invocation -> { + RagIndexOperation value = invocation.getArgument(0); + if (value.getId() == null) { + value.setId(30L); + } + return value; + }); + } + + @Test + void registersIdempotentTenantScopedBuildWithoutLeakingBranchName() { + when(operationRepository.findByProjectIdAndOperationKey(eq(42L), anyString())) + .thenReturn(Optional.empty()); + when(branchIndexRepository.findByProjectIdAndBranchNameForUpdate(42L, "client/private-develop")) + .thenReturn(Optional.empty()); + + var registration = service.registerBuild( + project, + "client/private-develop", + RagBranchIndexKind.DURABLE, + "master-100", + "develop-400", + "representation"); + + assertThat(registration.existingOperation()).isFalse(); + assertThat(registration.branchIndex().getIndexKind()).isEqualTo(RagBranchIndexKind.DURABLE); + assertThat(registration.branchIndex().getDesiredCommitHash()).isEqualTo("develop-400"); + assertThat(registration.generation().getSeedRevision()).isEqualTo("master-100"); + assertThat(registration.generation().getCollectionName()) + .startsWith("cc_w0_p42_b") + .doesNotContain("client", "private", "develop"); + assertThat(registration.operation().getOperationKey()).hasSize(64); + } + + @Test + void publishesNewGenerationAndSupersedesPreviousOneAtomically() { + RagBranchIndex branchIndex = new RagBranchIndex(project, "develop", RagBranchIndexKind.DURABLE); + branchIndex.setId(10L); + RagBranchIndexGeneration previous = new RagBranchIndexGeneration( + branchIndex, "develop-400", "generation-400", null, + "master-100", "representation"); + previous.setId(19L); + previous.activate("manifest-400", 500, 1500); + branchIndex.activate(previous); + when(branchIndexRepository.findByProjectIdAndBranchNameForUpdate(42L, "develop")) + .thenReturn(Optional.of(branchIndex)); + when(operationRepository.findByProjectIdAndOperationKey(eq(42L), anyString())) + .thenReturn(Optional.empty()); + + var registration = service.registerBuild( + project, "develop", RagBranchIndexKind.DURABLE, + "develop-400", "develop-401", "representation"); + when(operationRepository.findById(30L)).thenReturn(Optional.of(registration.operation())); + when(branchIndexRepository.findByIdForPublication(10L)) + .thenReturn(Optional.of(branchIndex)); + + service.startBuild(30L, 99L); + RagBranchIndexGeneration published = service.publish(30L, "manifest-401", 501, 1504); + + assertThat(previous.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.SUPERSEDED); + assertThat(published.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.ACTIVE); + assertThat(branchIndex.getActiveGeneration()).isSameAs(published); + assertThat(branchIndex.getCommitHash()).isEqualTo("develop-401"); + assertThat(registration.operation().getStatus()).isEqualTo(RagIndexOperationStatus.SUCCEEDED); + assertThat(registration.operation().getAttemptCount()).isEqualTo(1); + assertThat(registration.operation().getJobId()).isEqualTo(99L); + } + + @Test + void completedOlderGenerationDoesNotRegressNewerDesiredRevision() { + RagBranchIndex branchIndex = new RagBranchIndex( + project, "develop", RagBranchIndexKind.DURABLE); + branchIndex.setId(10L); + RagBranchIndexGeneration active = new RagBranchIndexGeneration( + branchIndex, "develop-400", "generation-400", null, + "develop-399", "representation"); + active.setId(19L); + active.activate("manifest-400", 500, 1500); + branchIndex.activate(active); + branchIndex.requestRevision("develop-402"); + + RagBranchIndexGeneration late = new RagBranchIndexGeneration( + branchIndex, "develop-401", "generation-401", active, + "develop-400", "representation"); + late.setId(20L); + RagIndexOperation operation = new RagIndexOperation( + project, "develop", "develop-400", "develop-401", "late-key"); + operation.setId(30L); + operation.setGeneration(late); + operation.start(); + when(operationRepository.findById(30L)).thenReturn(Optional.of(operation)); + when(branchIndexRepository.findByIdForPublication(10L)) + .thenReturn(Optional.of(branchIndex)); + + RagBranchIndexGeneration published = service.publish( + 30L, "manifest-401", 501, 1504); + + assertThat(published.getStatus()) + .isEqualTo(RagBranchIndexGenerationStatus.SUPERSEDED); + assertThat(published.getManifestDigest()).isEqualTo("manifest-401"); + assertThat(branchIndex.getActiveGeneration()).isSameAs(active); + assertThat(branchIndex.getCommitHash()).isEqualTo("develop-400"); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("develop-402"); + assertThat(operation.getStatus()).isEqualTo(RagIndexOperationStatus.SUCCEEDED); + verify(branchIndexRepository, never()).save(branchIndex); + } + + @Test + void failedReplacementKeepsLastVerifiedGenerationAvailable() { + RagBranchIndex branchIndex = new RagBranchIndex(project, "master", RagBranchIndexKind.PRIMARY); + branchIndex.setId(10L); + RagBranchIndexGeneration active = new RagBranchIndexGeneration( + branchIndex, "master-100", "generation-100", null, + null, "representation"); + active.setId(19L); + active.activate("manifest-100", 500, 1500); + branchIndex.activate(active); + when(branchIndexRepository.findByProjectIdAndBranchNameForUpdate(42L, "master")) + .thenReturn(Optional.of(branchIndex)); + when(operationRepository.findByProjectIdAndOperationKey(eq(42L), anyString())) + .thenReturn(Optional.empty()); + + var registration = service.registerBuild( + project, "master", RagBranchIndexKind.PRIMARY, + "master-100", "master-101", "representation"); + when(operationRepository.findById(30L)).thenReturn(Optional.of(registration.operation())); + when(branchIndexRepository.findByIdForPublication(10L)) + .thenReturn(Optional.of(branchIndex)); + + service.fail(30L, "vector publication failed"); + + assertThat(branchIndex.getActiveGeneration()).isSameAs(active); + assertThat(branchIndex.getCommitHash()).isEqualTo("master-100"); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("master-101"); + assertThat(branchIndex.getLifecycleStatus()).isEqualTo(RagBranchIndexLifecycleStatus.READY); + assertThat(registration.generation().getStatus()).isEqualTo(RagBranchIndexGenerationStatus.FAILED); + assertThat(registration.operation().getStatus()).isEqualTo(RagIndexOperationStatus.FAILED); + } + + @Test + void failedIdempotentOperationCanRetryWithoutCreatingDuplicateGeneration() { + RagBranchIndex branchIndex = new RagBranchIndex( + project, "develop", RagBranchIndexKind.DURABLE); + branchIndex.setId(10L); + branchIndex.requestRevision("develop-401"); + RagBranchIndexGeneration generation = new RagBranchIndexGeneration( + branchIndex, "develop-401", "generation-401", null, + "develop-400", "representation"); + generation.setId(20L); + generation.fail("worker restarted"); + RagIndexOperation operation = new RagIndexOperation( + project, "develop", "develop-400", "develop-401", "key"); + operation.setId(30L); + operation.setGeneration(generation); + operation.fail("worker restarted"); + when(operationRepository.findById(30L)).thenReturn(Optional.of(operation)); + + service.startBuild(30L, 91L); + + assertThat(operation.getStatus()).isEqualTo(RagIndexOperationStatus.RUNNING); + assertThat(operation.getAttemptCount()).isEqualTo(1); + assertThat(operation.getJobId()).isEqualTo(91L); + assertThat(generation.getStatus()) + .isEqualTo(RagBranchIndexGenerationStatus.BUILDING); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("develop-401"); + verify(generationRepository).save(generation); + verify(branchIndexRepository, never()).save(branchIndex); + verify(operationRepository).save(operation); + } + + @Test + void lateFailureDoesNotOverwriteNewerDesiredRevisionState() { + RagBranchIndex branchIndex = new RagBranchIndex( + project, "develop", RagBranchIndexKind.DURABLE); + branchIndex.setId(10L); + branchIndex.requestRevision("develop-402"); + RagBranchIndexGeneration late = new RagBranchIndexGeneration( + branchIndex, "develop-401", "generation-401", null, + "develop-400", "representation"); + late.setId(20L); + RagIndexOperation operation = new RagIndexOperation( + project, "develop", "develop-400", "develop-401", "late-key"); + operation.setId(30L); + operation.setGeneration(late); + operation.start(); + when(operationRepository.findById(30L)).thenReturn(Optional.of(operation)); + when(branchIndexRepository.findByIdForPublication(10L)) + .thenReturn(Optional.of(branchIndex)); + + service.fail(30L, "late worker failed"); + + assertThat(late.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.FAILED); + assertThat(operation.getStatus()).isEqualTo(RagIndexOperationStatus.FAILED); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("develop-402"); + assertThat(branchIndex.getLifecycleStatus()) + .isEqualTo(RagBranchIndexLifecycleStatus.BUILDING); + assertThat(branchIndex.getErrorMessage()).isNull(); + verify(branchIndexRepository, never()).save(branchIndex); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java index 1c204505..cbec2923 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java @@ -13,12 +13,16 @@ import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.rostilos.codecrow.core.model.project.config.RagConfig; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; import org.rostilos.codecrow.core.model.workspace.Workspace; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; import org.rostilos.codecrow.core.service.AnalysisJobService; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.springframework.test.util.ReflectionTestUtils; @@ -52,6 +56,9 @@ class RagOperationsServiceImplTest { @Mock private RagBranchIndexRepository ragBranchIndexRepository; + @Mock + private RagBranchIndexGenerationRepository branchGenerationRepository; + @Mock private VcsClientProvider vcsClientProvider; @@ -71,6 +78,8 @@ void setUp() { ragBranchIndexRepository, vcsClientProvider, ragPipelineClient); + ReflectionTestUtils.setField( + service, "branchGenerationRepository", branchGenerationRepository); testProject = new Project(); ReflectionTestUtils.setField(testProject, "id", 100L); @@ -85,6 +94,62 @@ void testIsRagEnabled_ApiDisabled() { assertThat(result).isFalse(); } + @Test + void exactFirstUseBuildsTargetSnapshotWithoutPrimaryToDevelopDiff() throws Exception { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, registry, builder); + setupRagEnabled(); + setupVcsBinding(); + ReflectionTestUtils.setField(service, "ragApiEnabled", true); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(incrementalRagUpdateService.parseDiffForRag("")) + .thenReturn(new IncrementalRagUpdateService.DiffResult( + Set.of(), Set.of(), Set.of())); + Job job = mock(Job.class); + when(job.getId()).thenReturn(77L); + when(analysisJobService.createRagIndexJob(any(), eq(false), any())) + .thenReturn(job); + when(analysisLockService.acquireLock( + eq(testProject), eq("feature"), any(), eq("develop-400"), isNull())) + .thenReturn(Optional.of("exact-feature-lock")); + VcsClient vcs = mock(VcsClient.class); + when(vcsClientProvider.getClient(any())).thenReturn(vcs); + when(vcs.getLatestCommitHash("my-workspace", "my-repo", "feature")) + .thenReturn("develop-400"); + when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) + .thenReturn(Optional.empty()); + when(builder.build( + eq(testProject), any(), eq("my-workspace"), eq("my-repo"), + eq("feature"), eq("develop-400"), + eq(org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE), + any(), any(), eq(77L))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-400", + "document_count", 231, + "chunk_count", 400, + "updatedFiles", 231, + "deletedFiles", 0, + "skippedFiles", 0)); + + boolean ready = service.ensureBranchIndexForPrTarget( + testProject, "feature", ignored -> { }); + + assertThat(ready).isTrue(); + verify(vcs, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); + verify(builder).build( + eq(testProject), any(), eq("my-workspace"), eq("my-repo"), + eq("feature"), eq("develop-400"), + eq(org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE), + any(), any(), eq(77L)); + } + @Test void testIsRagEnabled_NullConfig() { ReflectionTestUtils.setField(service, "ragApiEnabled", true); @@ -428,6 +493,52 @@ void testCleanupStaleBranches_DeletesStaleBranch() throws Exception { assertThat(result).containsEntry("total_deleted", 1); } + @Test + void cleanupStaleBranchesDeletesEveryRegisteredExactGeneration() throws Exception { + setupRagEnabled(); + setupVcsBinding(); + setupProjectWithWorkspaceAndNamespace(); + when(ragBranchIndexRepository.findBranchNamesByProjectId(100L)) + .thenReturn(List.of("main", "stale-exact")); + when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) + .thenReturn(List.of("main")); + + RagBranchIndex branchIndex = new RagBranchIndex( + testProject, "stale-exact", RagBranchIndexKind.DURABLE); + branchIndex.setId(501L); + RagBranchIndexGeneration first = new RagBranchIndexGeneration(); + first.setCollectionName("cc_generation_1"); + RagBranchIndexGeneration second = new RagBranchIndexGeneration(); + second.setCollectionName("cc_generation_2"); + when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "stale-exact")) + .thenReturn(Optional.of(branchIndex)); + when(branchGenerationRepository.findByBranchIndexIdOrderByCreatedAtDesc(501L)) + .thenReturn(List.of(first, second)); + when(ragPipelineClient.deleteBranch( + "test-ws", "test-ns", "stale-exact", "cc_generation_1")) + .thenReturn(true); + when(ragPipelineClient.deleteBranch( + "test-ws", "test-ns", "stale-exact", "cc_generation_2")) + .thenReturn(true); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + Map result = service.cleanupStaleBranches( + testProject, Set.of(), eventConsumer); + + assertThat(result).containsEntry("status", "success"); + assertThat(result).containsEntry("total_deleted", 1); + assertThat(result.get("deleted_branches")).isEqualTo(List.of("stale-exact")); + verify(ragPipelineClient).deleteBranch( + "test-ws", "test-ns", "stale-exact", "cc_generation_1"); + verify(ragPipelineClient).deleteBranch( + "test-ws", "test-ns", "stale-exact", "cc_generation_2"); + verify(ragBranchIndexRepository) + .deleteByProjectIdAndBranchName(100L, "stale-exact"); + verify(ragPipelineClient, never()) + .deleteBranch("my-workspace", "my-repo", "stale-exact"); + } + @Test void testUpdateBranchIndex_WhenNotEnabled() { ReflectionTestUtils.setField(service, "ragApiEnabled", false); @@ -750,13 +861,76 @@ void testUpdateBranchIndex_SuccessWithDiff() throws Exception { } @Test - void testUpdateBranchIndex_EmptyDiff() throws Exception { + void updateBranchIndexUsesCompletedBranchCheckpointWithoutComparingMain() throws Exception { + setupRagEnabled(); + setupVcsBinding(); + setupProjectWithWorkspaceAndNamespace(); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); + when(incrementalRagUpdateService.parseDiffForRag("checkpoint diff")) + .thenReturn(new IncrementalRagUpdateService.DiffResult( + Set.of(), Set.of("src/Changed.java"), Set.of())); + when(analysisJobService.createRagIndexJob(eq(testProject), eq(false), any())).thenReturn(mock(Job.class)); + when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) + .thenReturn(Optional.of("lock")); + when(incrementalRagUpdateService.performIncrementalUpdate( + any(), any(), anyString(), anyString(), anyString(), anyString(), anySet(), anySet(), anySet())) + .thenReturn(Map.of("updatedFiles", 1, "deletedFiles", 0, "skippedFiles", 0)); + + VcsClient mockVcs = mock(VcsClient.class); + when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); + when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")).thenReturn("develop-401"); + when(mockVcs.getBranchDiff("my-workspace", "my-repo", "develop-400", "develop-401")) + .thenReturn("checkpoint diff"); + RagBranchIndex checkpoint = new RagBranchIndex(testProject, "feature"); + checkpoint.setCommitHash("develop-400"); + when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) + .thenReturn(Optional.of(checkpoint)); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); + + assertThat(result).isTrue(); + verify(mockVcs).getBranchDiff("my-workspace", "my-repo", "develop-400", "develop-401"); + verify(mockVcs, never()).getBranchDiff("my-workspace", "my-repo", "main", "feature"); + } + + @Test + void updateBranchIndexRejectsBranchOutsideRetainedConfiguration() { + setupRagEnabled(); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.updateBranchIndex(testProject, "release/preview", eventConsumer); + + assertThat(result).isFalse(); + verifyNoInteractions(vcsClientProvider); + verify(eventConsumer).accept(argThat(event -> "rag_skipped".equals(event.get("state")))); + } + + @Test + void updateBranchIndexEmptyDiffSeedsExactSnapshot() throws Exception { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, registry, builder); setupRagEnabled(); setupVcsBinding(); + service = spy(service); + doReturn(true).when(service).shouldHaveBranchIndex(testProject, "feature"); when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); VcsClient mockVcs = mock(VcsClient.class); when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); when(mockVcs.getBranchDiff("my-workspace", "my-repo", "main", "feature")).thenReturn(""); + when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")) + .thenReturn("feature-head"); + doReturn(true).when(service).triggerIncrementalUpdate( + eq(testProject), eq("feature"), eq("feature-head"), eq(""), any()); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -764,6 +938,8 @@ void testUpdateBranchIndex_EmptyDiff() throws Exception { assertThat(result).isTrue(); verify(eventConsumer).accept(argThat(m -> "info".equals(m.get("type")))); + verify(service).triggerIncrementalUpdate( + testProject, "feature", "feature-head", "", eventConsumer); } @Test @@ -1123,7 +1299,8 @@ void testCleanupStaleBranches_DeleteThrows() throws Exception { private void setupRagEnabled() { ReflectionTestUtils.setField(service, "ragApiEnabled", true); - RagConfig ragConfig = new RagConfig(true, "main"); + RagConfig ragConfig = new RagConfig( + true, "main", null, null, true, 30, List.of("feature"), true); ProjectConfig config = new ProjectConfig(false, "main", null, ragConfig); testProject.setConfiguration(config); } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentityTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentityTest.java new file mode 100644 index 00000000..12003942 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentityTest.java @@ -0,0 +1,50 @@ +package org.rostilos.codecrow.ragengine.source; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class RepositorySourceTreeIdentityTest { + + @TempDir + Path repository; + + @Test + void matchesTheRagConsumerCrossLanguageGoldenDigest() + throws Exception { + Files.createDirectories(repository.resolve("app")); + Files.writeString(repository.resolve("app/Module.php"), " + + 4.0.0 + + org.rostilos.codecrow + codecrow-parent + 1.0 + ../../pom.xml + + codecrow-scm-evidence + Provider-neutral commit evidence, promotion planning, and issue provenance + + + org.rostilos.codecrow + codecrow-vcs-client + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-autoconfigure + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-junit-jupiter + test + + + org.assertj + assertj-core + test + + + diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/module-info.java b/java-ecosystem/libs/scm-evidence/src/main/java/module-info.java new file mode 100644 index 00000000..9a0fbe01 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/module-info.java @@ -0,0 +1,25 @@ +module org.rostilos.codecrow.scmevidence { + requires jakarta.persistence; + requires spring.data.jpa; + requires spring.data.commons; + requires spring.context; + requires spring.boot.autoconfigure; + requires spring.tx; + requires org.slf4j; + requires org.rostilos.codecrow.vcs; + + exports org.rostilos.codecrow.scmevidence.api; + exports org.rostilos.codecrow.scmevidence.config; + exports org.rostilos.codecrow.scmevidence.model; + exports org.rostilos.codecrow.scmevidence.persistence; + exports org.rostilos.codecrow.scmevidence.service; + + opens org.rostilos.codecrow.scmevidence.model + to org.hibernate.orm.core, spring.core, spring.context; + opens org.rostilos.codecrow.scmevidence.persistence + to spring.core, spring.context; + opens org.rostilos.codecrow.scmevidence.service + to spring.core, spring.context; + opens org.rostilos.codecrow.scmevidence.config + to spring.core, spring.context; +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/AnalysisReceiptView.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/AnalysisReceiptView.java new file mode 100644 index 00000000..1a47e330 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/AnalysisReceiptView.java @@ -0,0 +1,11 @@ +package org.rostilos.codecrow.scmevidence.api; + +public record AnalysisReceiptView( + String commitHash, + String patchId, + String sourceBranch, + String targetBranch, + String targetBaseRevision, + Long analysisId, + String analysisType) { +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/CommitEvidenceView.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/CommitEvidenceView.java new file mode 100644 index 00000000..dae1dc4e --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/CommitEvidenceView.java @@ -0,0 +1,8 @@ +package org.rostilos.codecrow.scmevidence.api; + +public record CommitEvidenceView( + String commitHash, + String patchId, + String authorName, + String authorEmail) { +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/IssueProvenance.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/IssueProvenance.java new file mode 100644 index 00000000..7da75eb5 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/IssueProvenance.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.scmevidence.api; + +public record IssueProvenance( + String commitHash, + String authorName, + String authorEmail, + String filePath, + int lineNumber, + String confidence) { +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/PromotionPlan.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/PromotionPlan.java new file mode 100644 index 00000000..786fed27 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/PromotionPlan.java @@ -0,0 +1,16 @@ +package org.rostilos.codecrow.scmevidence.api; + +import java.util.List; + +public record PromotionPlan( + ReuseKind reuseKind, + List reusableCommits, + List commitsWithoutEvidence, + boolean requiresTargetContextAnalysis) { + + public enum ReuseKind { + EXACT_EVIDENCE_REUSE, + PARTIAL_EVIDENCE_REUSE, + NO_EVIDENCE_REUSE + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfiguration.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfiguration.java new file mode 100644 index 00000000..f66d35bc --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfiguration.java @@ -0,0 +1,18 @@ +package org.rostilos.codecrow.scmevidence.config; + +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +/** + * Self-contained Spring Boot integration for the provider-neutral SCM evidence + * package. Applications opt in by including the package artifact; they do not + * need to know its internal service, persistence, or entity package layout. + */ +@AutoConfiguration +@ComponentScan(basePackages = "org.rostilos.codecrow.scmevidence.service") +@EnableJpaRepositories(basePackages = "org.rostilos.codecrow.scmevidence.persistence") +@EntityScan(basePackages = "org.rostilos.codecrow.scmevidence.model") +public class ScmEvidenceAutoConfiguration { +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAddedLineEvidence.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAddedLineEvidence.java new file mode 100644 index 00000000..1e39c88b --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAddedLineEvidence.java @@ -0,0 +1,41 @@ +package org.rostilos.codecrow.scmevidence.model; + +import jakarta.persistence.*; + +@Entity +@Table(name = "scm_added_line_evidence", indexes = @Index( + name = "idx_scm_added_line_lookup", + columnList = "project_id,file_path,line_hash")) +public class ScmAddedLineEvidence { + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "commit_evidence_id", nullable = false) + private ScmCommitEvidence commitEvidence; + @Column(name = "project_id", nullable = false) + private Long projectId; + @Column(name = "file_path", nullable = false, length = 1024) + private String filePath; + @Column(name = "new_line_number", nullable = false) + private int newLineNumber; + @Column(name = "line_hash", nullable = false, length = 64) + private String lineHash; + + public ScmAddedLineEvidence() {} + + public ScmAddedLineEvidence(Long projectId, String filePath, + int newLineNumber, String lineHash) { + this.projectId = projectId; + this.filePath = filePath; + this.newLineNumber = newLineNumber; + this.lineHash = lineHash; + } + + public void setCommitEvidence(ScmCommitEvidence commitEvidence) { + this.commitEvidence = commitEvidence; + } + public ScmCommitEvidence getCommitEvidence() { return commitEvidence; } + public String getFilePath() { return filePath; } + public int getNewLineNumber() { return newLineNumber; } + public String getLineHash() { return lineHash; } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAnalysisReceipt.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAnalysisReceipt.java new file mode 100644 index 00000000..d1ae427b --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAnalysisReceipt.java @@ -0,0 +1,55 @@ +package org.rostilos.codecrow.scmevidence.model; + +import jakarta.persistence.*; + +import java.time.OffsetDateTime; + +@Entity +@Table(name = "scm_analysis_receipt", uniqueConstraints = @UniqueConstraint( + name = "uq_scm_analysis_receipt_context", + columnNames = {"project_id", "commit_evidence_id", "context_key"})) +public class ScmAnalysisReceipt { + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + @Column(name = "project_id", nullable = false) + private Long projectId; + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "commit_evidence_id", nullable = false) + private ScmCommitEvidence commitEvidence; + @Column(name = "source_branch", length = 256) + private String sourceBranch; + @Column(name = "target_branch", nullable = false, length = 256) + private String targetBranch; + @Column(name = "target_base_revision", length = 64) + private String targetBaseRevision; + @Column(name = "analysis_id") + private Long analysisId; + @Column(name = "analysis_type", nullable = false, length = 40) + private String analysisType; + @Column(name = "context_key", nullable = false, length = 64) + private String contextKey; + @Column(name = "analyzed_at", nullable = false) + private OffsetDateTime analyzedAt = OffsetDateTime.now(); + + public ScmAnalysisReceipt() {} + + public ScmAnalysisReceipt(Long projectId, ScmCommitEvidence commitEvidence, + String sourceBranch, String targetBranch, String targetBaseRevision, + Long analysisId, String analysisType, String contextKey) { + this.projectId = projectId; + this.commitEvidence = commitEvidence; + this.sourceBranch = sourceBranch; + this.targetBranch = targetBranch; + this.targetBaseRevision = targetBaseRevision; + this.analysisId = analysisId; + this.analysisType = analysisType; + this.contextKey = contextKey; + } + + public ScmCommitEvidence getCommitEvidence() { return commitEvidence; } + public String getSourceBranch() { return sourceBranch; } + public String getTargetBranch() { return targetBranch; } + public String getTargetBaseRevision() { return targetBaseRevision; } + public Long getAnalysisId() { return analysisId; } + public String getAnalysisType() { return analysisType; } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmCommitEvidence.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmCommitEvidence.java new file mode 100644 index 00000000..07741a25 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmCommitEvidence.java @@ -0,0 +1,56 @@ +package org.rostilos.codecrow.scmevidence.model; + +import jakarta.persistence.*; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; + +@Entity +@Table(name = "scm_commit_evidence", uniqueConstraints = @UniqueConstraint( + name = "uq_scm_commit_evidence_project_hash", + columnNames = {"project_id", "commit_hash"})) +public class ScmCommitEvidence { + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + @Column(name = "project_id", nullable = false) + private Long projectId; + @Column(name = "commit_hash", nullable = false, length = 64) + private String commitHash; + @Column(name = "patch_id", nullable = false, length = 64) + private String patchId; + @Column(name = "author_name", length = 200) + private String authorName; + @Column(name = "author_email", length = 320) + private String authorEmail; + @Column(name = "captured_at", nullable = false) + private OffsetDateTime capturedAt = OffsetDateTime.now(); + @OneToMany(mappedBy = "commitEvidence", cascade = CascadeType.ALL, + orphanRemoval = true) + private List addedLines = new ArrayList<>(); + + public ScmCommitEvidence() {} + + public ScmCommitEvidence(Long projectId, String commitHash, String patchId, + String authorName, String authorEmail) { + this.projectId = projectId; + this.commitHash = commitHash; + this.patchId = patchId; + this.authorName = authorName; + this.authorEmail = authorEmail; + } + + public void addLine(ScmAddedLineEvidence line) { + line.setCommitEvidence(this); + addedLines.add(line); + } + + public Long getId() { return id; } + public Long getProjectId() { return projectId; } + public String getCommitHash() { return commitHash; } + public String getPatchId() { return patchId; } + public String getAuthorName() { return authorName; } + public String getAuthorEmail() { return authorEmail; } + public OffsetDateTime getCapturedAt() { return capturedAt; } + public List getAddedLines() { return addedLines; } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAddedLineEvidenceRepository.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAddedLineEvidenceRepository.java new file mode 100644 index 00000000..e57645f3 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAddedLineEvidenceRepository.java @@ -0,0 +1,25 @@ +package org.rostilos.codecrow.scmevidence.persistence; + +import org.rostilos.codecrow.scmevidence.model.ScmAddedLineEvidence; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import java.util.List; + +public interface ScmAddedLineEvidenceRepository + extends JpaRepository { + @Query(""" + SELECT line FROM ScmAddedLineEvidence line + JOIN FETCH line.commitEvidence evidence + WHERE evidence.projectId = :projectId + AND evidence.commitHash IN :commitHashes + AND line.filePath = :filePath + AND line.lineHash = :lineHash + """) + List findMatchingLines( + @Param("projectId") Long projectId, + @Param("commitHashes") List commitHashes, + @Param("filePath") String filePath, + @Param("lineHash") String lineHash); +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAnalysisReceiptRepository.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAnalysisReceiptRepository.java new file mode 100644 index 00000000..66abee75 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAnalysisReceiptRepository.java @@ -0,0 +1,14 @@ +package org.rostilos.codecrow.scmevidence.persistence; + +import org.rostilos.codecrow.scmevidence.model.ScmAnalysisReceipt; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface ScmAnalysisReceiptRepository + extends JpaRepository { + boolean existsByProjectIdAndCommitEvidenceIdAndContextKey( + Long projectId, Long commitEvidenceId, String contextKey); + List findByProjectIdAndCommitEvidencePatchIdIn( + Long projectId, List patchIds); +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmCommitEvidenceRepository.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmCommitEvidenceRepository.java new file mode 100644 index 00000000..dcbe6cdb --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmCommitEvidenceRepository.java @@ -0,0 +1,17 @@ +package org.rostilos.codecrow.scmevidence.persistence; + +import org.rostilos.codecrow.scmevidence.model.ScmCommitEvidence; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface ScmCommitEvidenceRepository + extends JpaRepository { + Optional findByProjectIdAndCommitHash( + Long projectId, String commitHash); + List findByProjectIdAndCommitHashIn( + Long projectId, List commitHashes); + List findByProjectIdAndPatchIdIn( + Long projectId, List patchIds); +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/PatchIdentity.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/PatchIdentity.java new file mode 100644 index 00000000..b072ef36 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/PatchIdentity.java @@ -0,0 +1,51 @@ +package org.rostilos.codecrow.scmevidence.service; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +/** Stable identity for a patch across merge commits and cherry-picks. */ +public final class PatchIdentity { + private PatchIdentity() {} + + public static String sha256(String unifiedDiff) { + StringBuilder normalized = new StringBuilder(); + if (unifiedDiff != null) { + for (String raw : unifiedDiff.replace("\r\n", "\n").split("\n", -1)) { + String line = stripTrailingWhitespace(raw); + if (line.startsWith("diff --git ") + || line.startsWith("rename from ") + || line.startsWith("rename to ") + || line.startsWith("new file mode ") + || line.startsWith("deleted file mode ") + || (line.startsWith("+") && !line.startsWith("+++")) + || (line.startsWith("-") && !line.startsWith("---"))) { + normalized.append(line).append('\n'); + } + } + } + return digest(normalized.toString()); + } + + public static String lineSha256(String line) { + return digest(line == null ? "" : line.strip()); + } + + static String digest(String value) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 unavailable", impossible); + } + } + + private static String stripTrailingWhitespace(String line) { + int end = line.length(); + while (end > 0 && Character.isWhitespace(line.charAt(end - 1))) { + end--; + } + return line.substring(0, end); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceService.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceService.java new file mode 100644 index 00000000..8298761e --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceService.java @@ -0,0 +1,253 @@ +package org.rostilos.codecrow.scmevidence.service; + +import org.rostilos.codecrow.scmevidence.api.AnalysisReceiptView; +import org.rostilos.codecrow.scmevidence.api.CommitEvidenceView; +import org.rostilos.codecrow.scmevidence.api.IssueProvenance; +import org.rostilos.codecrow.scmevidence.api.PromotionPlan; +import org.rostilos.codecrow.scmevidence.model.ScmAddedLineEvidence; +import org.rostilos.codecrow.scmevidence.model.ScmAnalysisReceipt; +import org.rostilos.codecrow.scmevidence.model.ScmCommitEvidence; +import org.rostilos.codecrow.scmevidence.persistence.ScmAddedLineEvidenceRepository; +import org.rostilos.codecrow.scmevidence.persistence.ScmAnalysisReceiptRepository; +import org.rostilos.codecrow.scmevidence.persistence.ScmCommitEvidenceRepository; +import org.rostilos.codecrow.vcsclient.VcsClient; +import org.rostilos.codecrow.vcsclient.model.VcsCommit; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** Public boundary for provider-neutral SCM evidence. */ +@Service +public class ScmEvidenceService { + private final ScmCommitEvidenceRepository evidenceRepository; + private final ScmAnalysisReceiptRepository receiptRepository; + private final ScmAddedLineEvidenceRepository lineRepository; + private final ScmPromotionPlanner promotionPlanner; + private final UnifiedDiffAddedLineParser lineParser = + new UnifiedDiffAddedLineParser(); + + public ScmEvidenceService( + ScmCommitEvidenceRepository evidenceRepository, + ScmAnalysisReceiptRepository receiptRepository, + ScmAddedLineEvidenceRepository lineRepository, + ScmPromotionPlanner promotionPlanner) { + this.evidenceRepository = evidenceRepository; + this.receiptRepository = receiptRepository; + this.lineRepository = lineRepository; + this.promotionPlanner = promotionPlanner; + } + + @Transactional + public List capture( + Long projectId, + VcsClient client, + String workspace, + String repository, + List commits) throws IOException { + requireProject(projectId); + List result = new ArrayList<>(); + for (VcsCommit commit : commits == null ? List.of() : commits) { + Optional existing = evidenceRepository + .findByProjectIdAndCommitHash(projectId, commit.hash()); + ScmCommitEvidence evidence; + if (existing.isPresent()) { + evidence = existing.get(); + } else { + String diff = client.getCommitDiff( + workspace, repository, commit.hash()); + evidence = new ScmCommitEvidence( + projectId, + commit.hash(), + PatchIdentity.sha256(diff), + commit.authorName(), + commit.authorEmail()); + for (var line : lineParser.parse(diff)) { + evidence.addLine(new ScmAddedLineEvidence( + projectId, line.filePath(), line.lineNumber(), + line.lineHash())); + } + evidence = evidenceRepository.save(evidence); + } + result.add(view(evidence)); + } + return List.copyOf(result); + } + + @Transactional + public void recordAnalysisReceipts( + Long projectId, + List commitHashes, + String sourceBranch, + String targetBranch, + String targetBaseRevision, + Long analysisId, + String analysisType) { + requireProject(projectId); + if (commitHashes == null || commitHashes.isEmpty()) { + return; + } + String branch = requireText(targetBranch, "targetBranch"); + String type = requireText(analysisType, "analysisType"); + String contextKey = contextKey(branch, targetBaseRevision, type); + for (ScmCommitEvidence evidence : evidenceRepository + .findByProjectIdAndCommitHashIn(projectId, commitHashes)) { + if (!receiptRepository + .existsByProjectIdAndCommitEvidenceIdAndContextKey( + projectId, evidence.getId(), contextKey)) { + receiptRepository.save(new ScmAnalysisReceipt( + projectId, evidence, sourceBranch, branch, + targetBaseRevision, analysisId, type, contextKey)); + } + } + } + + @Transactional(readOnly = true) + public PromotionPlan planPromotion( + Long projectId, + List currentCommitHashes, + String targetBranch, + String targetBaseRevision) { + requireProject(projectId); + List current = evidenceRepository + .findByProjectIdAndCommitHashIn(projectId, currentCommitHashes); + Map byHash = new HashMap<>(); + current.forEach(evidence -> byHash.put(evidence.getCommitHash(), evidence)); + List ordered = currentCommitHashes.stream() + .map(hash -> byHash.containsKey(hash) + ? view(byHash.get(hash)) + : new CommitEvidenceView( + hash, "missing:" + hash, null, null)) + .toList(); + List patchIds = current.stream() + .map(ScmCommitEvidence::getPatchId) + .distinct() + .toList(); + List receipts = patchIds.isEmpty() + ? List.of() + : receiptRepository + .findByProjectIdAndCommitEvidencePatchIdIn( + projectId, patchIds) + .stream() + .map(ScmEvidenceService::view) + .toList(); + return promotionPlanner.plan( + ordered, receipts, requireText(targetBranch, "targetBranch"), + targetBaseRevision); + } + + @Transactional(readOnly = true) + public Optional resolveIssueProvenance( + Long projectId, + List commitHashesOldestFirst, + String filePath, + Integer lineNumber, + String codeSnippet) { + requireProject(projectId); + if (commitHashesOldestFirst == null + || commitHashesOldestFirst.isEmpty() + || filePath == null || codeSnippet == null + || codeSnippet.isBlank()) { + return Optional.empty(); + } + List snippetLines = snippetLines(codeSnippet, lineNumber); + List matches = new ArrayList<>(); + for (SnippetLine snippetLine : snippetLines) { + lineRepository.findMatchingLines( + projectId, commitHashesOldestFirst, filePath, + PatchIdentity.lineSha256(snippetLine.content())) + .forEach(line -> matches.add(new ProvenanceCandidate( + line, snippetLine.expectedLine()))); + } + Map order = new HashMap<>(); + for (int i = 0; i < commitHashesOldestFirst.size(); i++) { + order.put(commitHashesOldestFirst.get(i), i); + } + return matches.stream() + .min(Comparator + .comparingInt((ProvenanceCandidate candidate) -> + -order.getOrDefault( + candidate.line().getCommitEvidence() + .getCommitHash(), -1)) + .thenComparingInt(candidate -> candidate.expectedLine() == null + ? 0 + : Math.abs(candidate.line().getNewLineNumber() + - candidate.expectedLine()))) + .map(candidate -> new IssueProvenance( + candidate.line().getCommitEvidence().getCommitHash(), + candidate.line().getCommitEvidence().getAuthorName(), + candidate.line().getCommitEvidence().getAuthorEmail(), + candidate.line().getFilePath(), + candidate.line().getNewLineNumber(), + candidate.expectedLine() != null + && candidate.line().getNewLineNumber() + == candidate.expectedLine() + ? "EXACT_LINE_AND_CONTENT" + : "EXACT_CONTENT")); + } + + private static List snippetLines( + String codeSnippet, Integer firstLineNumber) { + String normalized = codeSnippet.replace("\r\n", "\n"); + String[] lines = normalized.split("\n", -1); + if (lines.length == 1) { + return List.of(new SnippetLine(lines[0], firstLineNumber)); + } + List result = new ArrayList<>(); + for (int index = 0; index < lines.length; index++) { + if (!lines[index].isBlank()) { + result.add(new SnippetLine( + lines[index], firstLineNumber == null + ? null : firstLineNumber + index)); + } + } + return result; + } + + private record SnippetLine(String content, Integer expectedLine) {} + + private record ProvenanceCandidate( + ScmAddedLineEvidence line, Integer expectedLine) {} + + private static CommitEvidenceView view(ScmCommitEvidence evidence) { + return new CommitEvidenceView( + evidence.getCommitHash(), evidence.getPatchId(), + evidence.getAuthorName(), evidence.getAuthorEmail()); + } + + private static AnalysisReceiptView view(ScmAnalysisReceipt receipt) { + ScmCommitEvidence evidence = receipt.getCommitEvidence(); + return new AnalysisReceiptView( + evidence.getCommitHash(), evidence.getPatchId(), + receipt.getSourceBranch(), receipt.getTargetBranch(), + receipt.getTargetBaseRevision(), receipt.getAnalysisId(), + receipt.getAnalysisType()); + } + + private static String contextKey( + String targetBranch, String targetBaseRevision, + String analysisType) { + return PatchIdentity.digest(targetBranch + "\n" + + (targetBaseRevision == null ? "" : targetBaseRevision) + + "\n" + analysisType); + } + + private static void requireProject(Long projectId) { + if (projectId == null) { + throw new IllegalArgumentException("projectId is required"); + } + } + + private static String requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " is required"); + } + return value.trim(); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlanner.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlanner.java new file mode 100644 index 00000000..ba45a2c4 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlanner.java @@ -0,0 +1,46 @@ +package org.rostilos.codecrow.scmevidence.service; + +import org.rostilos.codecrow.scmevidence.api.AnalysisReceiptView; +import org.rostilos.codecrow.scmevidence.api.CommitEvidenceView; +import org.rostilos.codecrow.scmevidence.api.PromotionPlan; +import org.springframework.stereotype.Service; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@Service +public class ScmPromotionPlanner { + public PromotionPlan plan( + List currentCommits, + List priorReceipts, + String targetBranch, + String targetBaseRevision) { + Set knownPatchIds = new HashSet<>(); + Set exactContextPatchIds = new HashSet<>(); + for (AnalysisReceiptView receipt : priorReceipts) { + knownPatchIds.add(receipt.patchId()); + if (targetBranch.equals(receipt.targetBranch()) + && java.util.Objects.equals( + targetBaseRevision, receipt.targetBaseRevision())) { + exactContextPatchIds.add(receipt.patchId()); + } + } + List reusable = currentCommits.stream() + .filter(commit -> knownPatchIds.contains(commit.patchId())) + .map(CommitEvidenceView::commitHash) + .toList(); + List unseen = currentCommits.stream() + .filter(commit -> !knownPatchIds.contains(commit.patchId())) + .map(CommitEvidenceView::commitHash) + .toList(); + PromotionPlan.ReuseKind kind = reusable.isEmpty() + ? PromotionPlan.ReuseKind.NO_EVIDENCE_REUSE + : unseen.isEmpty() + ? PromotionPlan.ReuseKind.EXACT_EVIDENCE_REUSE + : PromotionPlan.ReuseKind.PARTIAL_EVIDENCE_REUSE; + boolean contextAnalysisRequired = currentCommits.stream() + .anyMatch(commit -> !exactContextPatchIds.contains(commit.patchId())); + return new PromotionPlan(kind, reusable, unseen, contextAnalysisRequired); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParser.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParser.java new file mode 100644 index 00000000..95f865a2 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParser.java @@ -0,0 +1,43 @@ +package org.rostilos.codecrow.scmevidence.service; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +final class UnifiedDiffAddedLineParser { + private static final Pattern TARGET_FILE = Pattern.compile("^\\+\\+\\+ b/(.+)$"); + private static final Pattern HUNK = Pattern.compile( + "^@@ -\\d+(?:,\\d+)? \\+(\\d+)(?:,(\\d+))? @@.*$"); + + record AddedLine(String filePath, int lineNumber, String lineHash) {} + + List parse(String diff) { + List result = new ArrayList<>(); + String file = null; + int newLine = -1; + for (String line : (diff == null ? "" : diff.replace("\r\n", "\n")).split("\n", -1)) { + Matcher fileMatcher = TARGET_FILE.matcher(line); + if (fileMatcher.matches()) { + file = fileMatcher.group(1); + continue; + } + Matcher hunkMatcher = HUNK.matcher(line); + if (hunkMatcher.matches()) { + newLine = Integer.parseInt(hunkMatcher.group(1)); + continue; + } + if (file == null || newLine < 0 || line.startsWith("\\ No newline")) { + continue; + } + if (line.startsWith("+") && !line.startsWith("+++")) { + result.add(new AddedLine(file, newLine, + PatchIdentity.lineSha256(line.substring(1)))); + newLine++; + } else if (!line.startsWith("-")) { + newLine++; + } + } + return result; + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/java-ecosystem/libs/scm-evidence/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 00000000..69da5496 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +org.rostilos.codecrow.scmevidence.config.ScmEvidenceAutoConfiguration diff --git a/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfigurationTest.java b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfigurationTest.java new file mode 100644 index 00000000..7fbb3a61 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfigurationTest.java @@ -0,0 +1,33 @@ +package org.rostilos.codecrow.scmevidence.config; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +import static org.assertj.core.api.Assertions.assertThat; + +class ScmEvidenceAutoConfigurationTest { + + @Test + void ownsItsSpringIntegrationBoundaries() { + assertThat(ScmEvidenceAutoConfiguration.class) + .hasAnnotation(AutoConfiguration.class); + + ComponentScan componentScan = ScmEvidenceAutoConfiguration.class + .getAnnotation(ComponentScan.class); + assertThat(componentScan.basePackages()) + .containsExactly("org.rostilos.codecrow.scmevidence.service"); + + EnableJpaRepositories repositories = ScmEvidenceAutoConfiguration.class + .getAnnotation(EnableJpaRepositories.class); + assertThat(repositories.basePackages()) + .containsExactly("org.rostilos.codecrow.scmevidence.persistence"); + + EntityScan entities = ScmEvidenceAutoConfiguration.class + .getAnnotation(EntityScan.class); + assertThat(entities.basePackages()) + .containsExactly("org.rostilos.codecrow.scmevidence.model"); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceServiceTest.java b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceServiceTest.java new file mode 100644 index 00000000..61122c7a --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceServiceTest.java @@ -0,0 +1,142 @@ +package org.rostilos.codecrow.scmevidence.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.scmevidence.model.ScmAddedLineEvidence; +import org.rostilos.codecrow.scmevidence.model.ScmCommitEvidence; +import org.rostilos.codecrow.scmevidence.persistence.ScmAddedLineEvidenceRepository; +import org.rostilos.codecrow.scmevidence.persistence.ScmAnalysisReceiptRepository; +import org.rostilos.codecrow.scmevidence.persistence.ScmCommitEvidenceRepository; +import org.rostilos.codecrow.vcsclient.VcsClient; +import org.rostilos.codecrow.vcsclient.model.VcsCommit; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class ScmEvidenceServiceTest { + @Mock private ScmCommitEvidenceRepository evidenceRepository; + @Mock private ScmAnalysisReceiptRepository receiptRepository; + @Mock private ScmAddedLineEvidenceRepository lineRepository; + @Mock private VcsClient vcsClient; + + private ScmEvidenceService service; + + @BeforeEach + void setUp() { + service = new ScmEvidenceService( + evidenceRepository, receiptRepository, lineRepository, + new ScmPromotionPlanner()); + } + + @Test + void capturesProviderNeutralPatchAuthorAndAddedLineEvidence() throws Exception { + VcsCommit commit = new VcsCommit( + "commit-1", "message", "Actual Author", "author@example.test", + OffsetDateTime.now(), List.of("parent")); + when(evidenceRepository.findByProjectIdAndCommitHash(42L, "commit-1")) + .thenReturn(Optional.empty()); + when(vcsClient.getCommitDiff("workspace", "repo", "commit-1")) + .thenReturn(""" + diff --git a/src/A.java b/src/A.java + --- a/src/A.java + +++ b/src/A.java + @@ -9,0 +10,1 @@ + +return secure(value); + """); + when(evidenceRepository.save(any())).thenAnswer(invocation -> invocation.getArgument(0)); + + var views = service.capture( + 42L, vcsClient, "workspace", "repo", List.of(commit)); + + assertThat(views).singleElement().satisfies(view -> { + assertThat(view.commitHash()).isEqualTo("commit-1"); + assertThat(view.patchId()).hasSize(64); + assertThat(view.authorName()).isEqualTo("Actual Author"); + assertThat(view.authorEmail()).isEqualTo("author@example.test"); + }); + var evidence = org.mockito.ArgumentCaptor.forClass(ScmCommitEvidence.class); + verify(evidenceRepository).save(evidence.capture()); + assertThat(evidence.getValue().getAddedLines()).singleElement() + .satisfies(line -> { + assertThat(line.getFilePath()).isEqualTo("src/A.java"); + assertThat(line.getNewLineNumber()).isEqualTo(10); + assertThat(line.getLineHash()) + .isEqualTo(PatchIdentity.lineSha256("return secure(value);")); + }); + } + + @Test + void issueProvenanceReturnsCommitAuthorNotReviewCommentAuthor() { + ScmCommitEvidence evidence = new ScmCommitEvidence( + 42L, "introducing-commit", "a".repeat(64), + "Actual Commit Author", "author@example.test"); + ScmAddedLineEvidence line = new ScmAddedLineEvidence( + 42L, "src/A.java", 10, + PatchIdentity.lineSha256("return secure(value);")); + line.setCommitEvidence(evidence); + when(lineRepository.findMatchingLines( + 42L, List.of("introducing-commit"), "src/A.java", + PatchIdentity.lineSha256("return secure(value);"))) + .thenReturn(List.of(line)); + + var provenance = service.resolveIssueProvenance( + 42L, List.of("introducing-commit"), "src/A.java", 10, + "return secure(value);"); + + assertThat(provenance).isPresent(); + assertThat(provenance.orElseThrow().authorName()) + .isEqualTo("Actual Commit Author"); + assertThat(provenance.orElseThrow().confidence()) + .isEqualTo("EXACT_LINE_AND_CONTENT"); + } + + @Test + void issueProvenanceResolvesAnAddedLineInsideMultilineSnippet() { + ScmCommitEvidence evidence = new ScmCommitEvidence( + 42L, "introducing-commit", "a".repeat(64), + "Actual Commit Author", "author@example.test"); + ScmAddedLineEvidence line = new ScmAddedLineEvidence( + 42L, "src/A.java", 11, + PatchIdentity.lineSha256("return secure(value);")); + line.setCommitEvidence(evidence); + when(lineRepository.findMatchingLines( + 42L, List.of("introducing-commit"), "src/A.java", + PatchIdentity.lineSha256("if (allowed) {"))) + .thenReturn(List.of()); + when(lineRepository.findMatchingLines( + 42L, List.of("introducing-commit"), "src/A.java", + PatchIdentity.lineSha256("return secure(value);"))) + .thenReturn(List.of(line)); + when(lineRepository.findMatchingLines( + 42L, List.of("introducing-commit"), "src/A.java", + PatchIdentity.lineSha256("}"))) + .thenReturn(List.of()); + + var provenance = service.resolveIssueProvenance( + 42L, List.of("introducing-commit"), "src/A.java", 10, + "if (allowed) {\nreturn secure(value);\n}"); + + assertThat(provenance).isPresent(); + assertThat(provenance.orElseThrow().lineNumber()).isEqualTo(11); + assertThat(provenance.orElseThrow().confidence()) + .isEqualTo("EXACT_LINE_AND_CONTENT"); + } + + @Test + void tenantProjectIdIsMandatoryAtPublicBoundary() { + assertThat(org.assertj.core.api.Assertions.catchThrowable(() -> + service.capture(null, vcsClient, "workspace", "repo", List.of()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("projectId"); + verifyNoInteractions(evidenceRepository, receiptRepository, lineRepository); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlannerTest.java b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlannerTest.java new file mode 100644 index 00000000..2f5c363d --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlannerTest.java @@ -0,0 +1,109 @@ +package org.rostilos.codecrow.scmevidence.service; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.scmevidence.api.AnalysisReceiptView; +import org.rostilos.codecrow.scmevidence.api.CommitEvidenceView; +import org.rostilos.codecrow.scmevidence.api.PromotionPlan; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +class ScmPromotionPlannerTest { + private final ScmPromotionPlanner planner = new ScmPromotionPlanner(); + + @Test + void patchIdentitySurvivesCherryPickCoordinatesButNotContentChanges() { + String first = """ + diff --git a/src/A.java b/src/A.java + index 1111111..2222222 100644 + --- a/src/A.java + +++ b/src/A.java + @@ -10,2 +10,3 @@ + context + +return secure(value); + """; + String cherryPicked = """ + diff --git a/src/A.java b/src/A.java + index aaaaaaa..bbbbbbb 100644 + --- a/src/A.java + +++ b/src/A.java + @@ -410,2 +512,3 @@ + different surrounding context + +return secure(value); + """; + String changed = cherryPicked.replace("secure(value)", "unsafe(value)"); + + assertThat(PatchIdentity.sha256(cherryPicked)) + .isEqualTo(PatchIdentity.sha256(first)); + assertThat(PatchIdentity.sha256(changed)) + .isNotEqualTo(PatchIdentity.sha256(first)); + } + + @Test + void developToMasterReusesFourHundredCommitEvidenceButReviewsMasterContext() { + List commits = IntStream.range(0, 400) + .mapToObj(index -> commit(index, "patch-" + index)) + .toList(); + // The 400 commits deliberately rotate through the 231-file fixture. + assertThat(IntStream.range(0, 400) + .map(index -> index % 231).distinct().count()).isEqualTo(231); + List developReceipts = commits.stream() + .map(commit -> receipt(commit, "feature/1-x", "develop", "dev-base")) + .toList(); + + PromotionPlan master = planner.plan( + commits, developReceipts, "master", "master-base"); + + assertThat(master.reuseKind()) + .isEqualTo(PromotionPlan.ReuseKind.EXACT_EVIDENCE_REUSE); + assertThat(master.reusableCommits()).hasSize(400); + assertThat(master.commitsWithoutEvidence()).isEmpty(); + assertThat(master.requiresTargetContextAnalysis()).isTrue(); + + PromotionPlan exactDevelopRetry = planner.plan( + commits, developReceipts, "develop", "dev-base"); + assertThat(exactDevelopRetry.requiresTargetContextAnalysis()).isFalse(); + } + + @Test + void promotionWithAdditionalReleaseCommitsIsPartialAndRequiresAnalysis() { + List develop = IntStream.range(0, 12) + .mapToObj(index -> commit(index, "patch-" + index)) + .toList(); + List promoted = new ArrayList<>(develop); + promoted.add(commit(12, "release-hotfix")); + promoted.add(commit(13, "release-metadata")); + List receipts = develop.stream() + .map(commit -> receipt(commit, "feature", "develop", "d1")) + .toList(); + + PromotionPlan plan = planner.plan( + promoted, receipts, "master", "m1"); + + assertThat(plan.reuseKind()) + .isEqualTo(PromotionPlan.ReuseKind.PARTIAL_EVIDENCE_REUSE); + assertThat(plan.reusableCommits()).hasSize(12); + assertThat(plan.commitsWithoutEvidence()) + .containsExactly("commit-12", "commit-13"); + assertThat(plan.requiresTargetContextAnalysis()).isTrue(); + } + + private static CommitEvidenceView commit(int index, String patch) { + return new CommitEvidenceView( + "commit-" + index, patch, "author-" + index, + "author-" + index + "@example.test"); + } + + private static AnalysisReceiptView receipt( + CommitEvidenceView commit, + String source, + String target, + String base) { + return new AnalysisReceiptView( + commit.commitHash(), commit.patchId(), source, target, base, + 100L, "PR_REVIEW"); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParserTest.java b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParserTest.java new file mode 100644 index 00000000..a95aa34f --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParserTest.java @@ -0,0 +1,34 @@ +package org.rostilos.codecrow.scmevidence.service; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class UnifiedDiffAddedLineParserTest { + @Test + void capturesRenamedTargetPathAndExactNewLineCoordinates() { + String diff = """ + diff --git a/old/A.java b/new/A.java + similarity index 80% + rename from old/A.java + rename to new/A.java + --- a/old/A.java + +++ b/new/A.java + @@ -7,2 +20,4 @@ + keep(); + -old(); + +introducedByAlice(); + +secondLine(); + tail(); + """; + + var lines = new UnifiedDiffAddedLineParser().parse(diff); + + assertThat(lines).hasSize(2); + assertThat(lines.get(0).filePath()).isEqualTo("new/A.java"); + assertThat(lines.get(0).lineNumber()).isEqualTo(21); + assertThat(lines.get(0).lineHash()) + .isEqualTo(PatchIdentity.lineSha256("introducedByAlice();")); + assertThat(lines.get(1).lineNumber()).isEqualTo(22); + } +} diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java index 33fb79db..63210445 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java @@ -332,13 +332,32 @@ public int deletePreviousReviewComments( String repo, int pullRequestNumber, String markerText + ) throws IOException { + return deletePreviousReviewComments( + owner, repo, pullRequestNumber, markerText, null); + } + + /** + * Delete earlier generated review comments while preserving one newly + * submitted replacement review. + */ + public int deletePreviousReviewComments( + String owner, + String repo, + int pullRequestNumber, + String markerText, + Long preservedReviewId ) throws IOException { int deleted = 0; for (Map comment : listReviewComments( owner, repo, pullRequestNumber)) { String body = (String) comment.get("body"); Number id = (Number) comment.get("id"); - if (body != null && body.contains(markerText) && id != null) { + Number reviewId = (Number) comment.get("pull_request_review_id"); + boolean preserved = preservedReviewId != null + && reviewId != null + && preservedReviewId.longValue() == reviewId.longValue(); + if (!preserved && body != null && body.contains(markerText) && id != null) { deleteReviewComment(owner, repo, id.longValue()); deleted++; } @@ -357,12 +376,31 @@ public int clearPreviousReviewBodies( int pullRequestNumber, String markerText, String clearedBody + ) throws IOException { + return clearPreviousReviewBodies( + owner, repo, pullRequestNumber, markerText, clearedBody, null); + } + + /** + * Clear earlier generated review summaries while preserving one newly + * submitted replacement review. + */ + public int clearPreviousReviewBodies( + String owner, + String repo, + int pullRequestNumber, + String markerText, + String clearedBody, + Long preservedReviewId ) throws IOException { int cleared = 0; for (Map review : listReviews(owner, repo, pullRequestNumber)) { String body = (String) review.get("body"); Number id = (Number) review.get("id"); - if (body != null && body.contains(markerText) && id != null) { + boolean preserved = preservedReviewId != null + && id != null + && preservedReviewId.longValue() == id.longValue(); + if (!preserved && body != null && body.contains(markerText) && id != null) { updateReviewBody( owner, repo, diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffAction.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffAction.java index cc5f1dee..0750a4ba 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffAction.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffAction.java @@ -10,6 +10,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -57,6 +59,51 @@ public String getPullRequestDiff(String owner, String repo, int pullRequestNumbe */ private String getPullRequestDiffFromFiles(String owner, String repo, int pullRequestNumber) throws IOException { StringBuilder combinedDiff = new StringBuilder(); + for (PullRequestFilePatch file : getPullRequestFilePatches( + owner, repo, pullRequestNumber)) { + if (file.patch().isEmpty()) { + continue; + } + + // Build a unified diff header + String fromFile = "renamed".equals(file.status()) + && !file.previousFilename().isEmpty() + ? file.previousFilename() + : file.filename(); + combinedDiff.append("diff --git a/").append(fromFile) + .append(" b/").append(file.filename()).append("\n"); + + if ("added".equals(file.status())) { + combinedDiff.append("new file mode 100644\n"); + } else if ("removed".equals(file.status())) { + combinedDiff.append("deleted file mode 100644\n"); + } else if ("renamed".equals(file.status())) { + combinedDiff.append("rename from ").append(file.previousFilename()).append("\n"); + combinedDiff.append("rename to ").append(file.filename()).append("\n"); + } + + combinedDiff.append("--- a/").append(fromFile).append("\n"); + combinedDiff.append("+++ b/").append(file.filename()).append("\n"); + combinedDiff.append(file.patch()).append("\n"); + } + return combinedDiff.toString(); + } + + /** + * Return GitHub's structured per-file patches for a pull request. + * + *

    The file endpoint is used directly for inline-review planning because + * its destination path is authoritative and does not require parsing quoted + * {@code diff --git} headers. A missing patch (for example, a binary or very + * large file) is preserved as an empty value so callers can treat its lines + * as unavailable for inline comments.

    + */ + public List getPullRequestFilePatches( + String owner, + String repo, + int pullRequestNumber + ) throws IOException { + List patches = new ArrayList<>(); String nextUrl = String.format("%s/repos/%s/%s/pulls/%d/files?per_page=100", GitHubConfig.API_BASE, owner, repo, pullRequestNumber); @@ -84,25 +131,8 @@ private String getPullRequestDiffFromFiles(String owner, String repo, int pullRe String patch = file.has("patch") ? file.get("patch").asText() : ""; String status = file.has("status") ? file.get("status").asText() : ""; String previousFilename = file.has("previous_filename") ? file.get("previous_filename").asText() : ""; - - if (!patch.isEmpty()) { - // Build a unified diff header - String fromFile = "renamed".equals(status) && !previousFilename.isEmpty() ? previousFilename : filename; - combinedDiff.append("diff --git a/").append(fromFile).append(" b/").append(filename).append("\n"); - - if ("added".equals(status)) { - combinedDiff.append("new file mode 100644\n"); - } else if ("removed".equals(status)) { - combinedDiff.append("deleted file mode 100644\n"); - } else if ("renamed".equals(status)) { - combinedDiff.append("rename from ").append(previousFilename).append("\n"); - combinedDiff.append("rename to ").append(filename).append("\n"); - } - - combinedDiff.append("--- a/").append(fromFile).append("\n"); - combinedDiff.append("+++ b/").append(filename).append("\n"); - combinedDiff.append(patch).append("\n"); - } + patches.add(new PullRequestFilePatch( + filename, previousFilename, status, patch)); } // Check for next page in Link header @@ -117,6 +147,20 @@ private String getPullRequestDiffFromFiles(String owner, String repo, int pullRe } } - return combinedDiff.toString(); + return List.copyOf(patches); + } + + public record PullRequestFilePatch( + String filename, + String previousFilename, + String status, + String patch + ) { + public PullRequestFilePatch { + filename = filename != null ? filename : ""; + previousFilename = previousFilename != null ? previousFilename : ""; + status = status != null ? status : ""; + patch = patch != null ? patch : ""; + } } } diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestActionTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestActionTest.java index c4cbac60..44d4f559 100644 --- a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestActionTest.java +++ b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestActionTest.java @@ -208,6 +208,42 @@ void deletePreviousReviewComments_deletesOnlyMarkedInlineComments() throws IOExc assertThat(requests.get(0).url().queryParameter("page")).isEqualTo("1"); } + @Test + void deletePreviousReviewComments_preservesTheReplacementReview() throws IOException { + List requests = new ArrayList<>(); + when(okHttpClient.newCall(any(Request.class))).thenAnswer(invocation -> { + Request request = invocation.getArgument(0); + requests.add(request); + + String responseJson = request.method().equals("GET") + ? "[{\"id\":41,\"pull_request_review_id\":51," + + "\"body\":\"old \"}," + + "{\"id\":42,\"pull_request_review_id\":99," + + "\"body\":\"new \"}]" + : "{}"; + Response requestResponse = new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(ResponseBody.create(responseJson, MediaType.parse("application/json"))) + .build(); + Call requestCall = mock(Call.class); + when(requestCall.execute()).thenReturn(requestResponse); + return requestCall; + }); + + int deleted = action.deletePreviousReviewComments( + "owner", "repo", 123, "", 99L); + + assertThat(deleted).isEqualTo(1); + assertThat(requests).extracting(request -> request.method() + " " + request.url().encodedPath()) + .containsExactly( + "GET /repos/owner/repo/pulls/123/comments", + "DELETE /repos/owner/repo/pulls/comments/41" + ); + } + @Test void clearPreviousReviewBodies_clearsOnlyMarkedReviewSummaries() throws IOException { List requests = new ArrayList<>(); @@ -253,4 +289,43 @@ void clearPreviousReviewBodies_clearsOnlyMarkedReviewSummaries() throws IOExcept assertThat(new ObjectMapper().readTree(body.readUtf8()).path("body").asText()) .isEqualTo(""); } + + @Test + void clearPreviousReviewBodies_preservesTheReplacementReview() throws IOException { + List requests = new ArrayList<>(); + when(okHttpClient.newCall(any(Request.class))).thenAnswer(invocation -> { + Request request = invocation.getArgument(0); + requests.add(request); + + String responseJson = request.method().equals("GET") + ? "[{\"id\":51,\"body\":\"old \"}," + + "{\"id\":99,\"body\":\"new \"}]" + : "{}"; + Response requestResponse = new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(ResponseBody.create(responseJson, MediaType.parse("application/json"))) + .build(); + Call requestCall = mock(Call.class); + when(requestCall.execute()).thenReturn(requestResponse); + return requestCall; + }); + + int cleared = action.clearPreviousReviewBodies( + "owner", + "repo", + 123, + "", + "", + 99L); + + assertThat(cleared).isEqualTo(1); + assertThat(requests).extracting(request -> request.method() + " " + request.url().encodedPath()) + .containsExactly( + "GET /repos/owner/repo/pulls/123/reviews", + "PUT /repos/owner/repo/pulls/123/reviews/51" + ); + } } diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffActionTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffActionTest.java index 3a1b2df4..d2ece66a 100644 --- a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffActionTest.java +++ b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffActionTest.java @@ -8,6 +8,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -102,4 +103,42 @@ void testGetPullRequestDiff_UnsuccessfulResponse_ThrowsIOException() throws IOEx verify(response).close(); } + + @Test + void getPullRequestFilePatchesReturnsStructuredDestinationPaths() throws IOException { + String filesJson = """ + [ + { + "filename": "src/New Name.java", + "previous_filename": "src/Old Name.java", + "status": "renamed", + "patch": "@@ -1 +1 @@\\n-old\\n+new" + }, + { + "filename": "assets/logo.png", + "status": "modified" + } + ] + """; + + when(okHttpClient.newCall(any(Request.class))).thenReturn(call); + when(call.execute()).thenReturn(response); + when(response.isSuccessful()).thenReturn(true); + when(response.body()).thenReturn(responseBody); + when(responseBody.string()).thenReturn(filesJson); + when(response.header("Link")).thenReturn(null); + + List patches = + action.getPullRequestFilePatches("owner", "repo", 123); + + assertThat(patches).containsExactly( + new GetPullRequestDiffAction.PullRequestFilePatch( + "src/New Name.java", + "src/Old Name.java", + "renamed", + "@@ -1 +1 @@\n-old\n+new"), + new GetPullRequestDiffAction.PullRequestFilePatch( + "assets/logo.png", "", "modified", "")); + verify(response).close(); + } } diff --git a/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpStdioServer.java b/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpStdioServer.java index c04a8b34..764fc085 100644 --- a/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpStdioServer.java +++ b/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpStdioServer.java @@ -621,12 +621,26 @@ private static List defineTools() { "filePath": { "type": "string", "description": "File Path." + }, + "startLine": { + "type": "integer", + "minimum": 1, + "description": "Optional first source line for an anchor-centred verification window." + }, + "endLine": { + "type": "integer", + "minimum": 1, + "description": "Optional last source line for the verification window. The server returns at most 401 lines." } }, "required": ["workspace", "repoSlug", "branch", "filePath"] } """; - tools.add(new Tool("getBranchFileContent", "Get full file content from specified branch or commit.", getBranchFileContentSchema)); + tools.add(new Tool( + "getBranchFileContent", + "Get file content or an optional whole-line range from a branch or commit.", + getBranchFileContentSchema + )); String getRootDirectorySchema = """ diff --git a/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpTools.java b/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpTools.java index 9372a7d6..afd49a2c 100644 --- a/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpTools.java +++ b/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpTools.java @@ -1,6 +1,7 @@ package org.rostilos.codecrow.mcp; import java.io.IOException; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -15,6 +16,7 @@ import org.slf4j.LoggerFactory; public class McpTools { + private static final int MAX_SOURCE_WINDOW_LINES = 401; private final VcsMcpClientFactory vcsMcpClientFactory; private VcsMcpClient vcsClient = null; private final LargeContentFilter largeContentFilter; @@ -166,7 +168,9 @@ public Object execute(String toolName, Map arguments) throws IOE (String) arguments.get("workspace"), (String) arguments.get("repoSlug"), (String) arguments.get("branch"), - (String) arguments.get("filePath") + (String) arguments.get("filePath"), + integerArgument(arguments.get("startLine")), + integerArgument(arguments.get("endLine")) ); case "getRootDirectory": return getRootDirectory( @@ -319,16 +323,97 @@ public Map getPullRequestCommits(String workspace, String repoSl } public Map getBranchFileContent(String workspace, String repoSlug, String branch, String filePath) { + return getBranchFileContent(workspace, repoSlug, branch, filePath, null, null); + } + + public Map getBranchFileContent( + String workspace, + String repoSlug, + String branch, + String filePath, + Integer startLine, + Integer endLine + ) { try { String fileContent = getVcsClient(true).getBranchFileContent(workspace, repoSlug, branch, filePath); - // Filter large file content to reduce token usage + if (startLine != null && startLine > 0) { + return sourceWindow(fileContent, startLine, endLine); + } + + // Exploratory callers without an anchor retain the existing large-file + // safeguard. Review verification supplies an issue line and receives + // a bounded source window instead of this generic placeholder. String filteredContent = largeContentFilter.filterFileContent(fileContent, filePath); - return Map.of("fileContent", filteredContent); + boolean completeFile = filteredContent != null + && !filteredContent.contains(LargeContentFilter.FILTERED_PLACEHOLDER); + int totalLines = lineCount(fileContent); + Map response = new LinkedHashMap<>(); + response.put("fileContent", filteredContent != null ? filteredContent : ""); + response.put("startLine", completeFile && totalLines > 0 ? 1 : 0); + response.put("endLine", completeFile ? totalLines : 0); + response.put("totalLines", totalLines); + response.put("completeFile", completeFile); + return response; } catch (IOException e) { return Map.of("error", "Failed to get branch file content: " + e.getMessage()); } } + static Map sourceWindow( + String content, + int requestedStartLine, + Integer requestedEndLine + ) { + if (content == null) { + return Map.of("error", "File content is unavailable"); + } + String[] lines = content.split("\\R", -1); + int totalLines = lines.length; + int startLine = Math.max(1, requestedStartLine); + if (startLine > totalLines) { + return Map.of( + "error", "Requested source line is outside the file", + "totalLines", totalLines + ); + } + int requestedEnd = requestedEndLine != null && requestedEndLine >= startLine + ? requestedEndLine + : startLine; + int endLine = Math.min( + totalLines, + Math.min(requestedEnd, startLine + MAX_SOURCE_WINDOW_LINES - 1) + ); + String fileContent = String.join( + "\n", + java.util.Arrays.copyOfRange(lines, startLine - 1, endLine) + ); + Map response = new LinkedHashMap<>(); + response.put("fileContent", fileContent); + response.put("startLine", startLine); + response.put("endLine", endLine); + response.put("totalLines", totalLines); + response.put("completeFile", startLine == 1 && endLine == totalLines); + return response; + } + + private static int lineCount(String content) { + return content == null ? 0 : content.split("\\R", -1).length; + } + + private static Integer integerArgument(Object value) { + if (value instanceof Number number) { + return number.intValue(); + } + if (value instanceof String text) { + try { + return Integer.valueOf(text.trim()); + } catch (NumberFormatException ignored) { + return null; + } + } + return null; + } + public Map getRootDirectory(String workspace, String projectKey, String branch) { try { String rootDirectory = getVcsClient().getRootDirectory(workspace, projectKey, branch); diff --git a/java-ecosystem/mcp-servers/vcs-mcp/src/test/java/org/rostilos/codecrow/mcp/McpToolsSourceWindowTest.java b/java-ecosystem/mcp-servers/vcs-mcp/src/test/java/org/rostilos/codecrow/mcp/McpToolsSourceWindowTest.java new file mode 100644 index 00000000..61083c12 --- /dev/null +++ b/java-ecosystem/mcp-servers/vcs-mcp/src/test/java/org/rostilos/codecrow/mcp/McpToolsSourceWindowTest.java @@ -0,0 +1,67 @@ +package org.rostilos.codecrow.mcp; + +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +class McpToolsSourceWindowTest { + + @Test + void returnsAnchorCentredLinesWithCoverageMetadata() { + String content = numberedLines(1_000); + + Map result = McpTools.sourceWindow(content, 420, 580); + + assertThat(result.get("startLine")).isEqualTo(420); + assertThat(result.get("endLine")).isEqualTo(580); + assertThat(result.get("totalLines")).isEqualTo(1_000); + assertThat(result.get("completeFile")).isEqualTo(false); + assertThat((String) result.get("fileContent")) + .startsWith("line-420\n") + .endsWith("line-580"); + } + + @Test + void boundsAnOverwideRequestAtWholeLineBoundaries() { + Map result = McpTools.sourceWindow( + numberedLines(1_000), + 1, + 1_000 + ); + + assertThat(result.get("startLine")).isEqualTo(1); + assertThat(result.get("endLine")).isEqualTo(401); + assertThat(result.get("completeFile")).isEqualTo(false); + assertThat(((String) result.get("fileContent")).lines()).hasSize(401); + } + + @Test + void marksACompleteSmallFile() { + Map result = McpTools.sourceWindow( + "first\nsecond\nthird", + 1, + 3 + ); + + assertThat(result.get("fileContent")).isEqualTo("first\nsecond\nthird"); + assertThat(result.get("completeFile")).isEqualTo(true); + } + + @Test + void rejectsAnAnchorOutsideTheFile() { + Map result = McpTools.sourceWindow("one\ntwo", 3, 5); + + assertThat(result).containsKey("error"); + assertThat(result.get("totalLines")).isEqualTo(2); + } + + private String numberedLines(int count) { + return IntStream.rangeClosed(1, count) + .mapToObj(index -> "line-" + index) + .collect(Collectors.joining("\n")); + } +} diff --git a/java-ecosystem/pom.xml b/java-ecosystem/pom.xml index 5c827ea2..51123c1f 100644 --- a/java-ecosystem/pom.xml +++ b/java-ecosystem/pom.xml @@ -116,6 +116,12 @@ 1.0 + + org.rostilos.codecrow + codecrow-scm-evidence + 1.0 + + org.rostilos.codecrow codecrow-file-content @@ -582,6 +588,7 @@ libs/vcs-client libs/core + libs/scm-evidence libs/commit-graph libs/file-content libs/security diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java index 962db55e..a8b330a2 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java @@ -167,7 +167,8 @@ public WebhookResult handle(WebhookPayload payload, Project project, Consumer result = branchAnalysisProcessor.process(request, processorConsumer); + Map result = branchAnalysisProcessor.processAfterDependencyGate( + request, processorConsumer); return WebhookResult.success("Branch analysis completed", result); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java index a68281fe..5f6b4228 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java @@ -2,6 +2,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @@ -42,10 +43,12 @@ public Executor taskExecutor() { * Sized for concurrent webhook handling from VCS providers. */ @Bean(name = "webhookExecutor") - public Executor webhookExecutor() { + public Executor webhookExecutor( + @Value("${webhook.executor.core-pool-size:8}") int corePoolSize, + @Value("${webhook.executor.max-pool-size:20}") int maxPoolSize) { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(4); - executor.setMaxPoolSize(8); + executor.setCorePoolSize(corePoolSize); + executor.setMaxPoolSize(maxPoolSize); // Do not accept work into an ephemeral in-memory backlog. Saturated work // remains QUEUED in the database and is retried by the recovery scheduler. executor.setQueueCapacity(0); @@ -55,7 +58,7 @@ public Executor webhookExecutor() { executor.setRejectedExecutionHandler(new java.util.concurrent.ThreadPoolExecutor.AbortPolicy()); executor.initialize(); log.info("Webhook executor initialized with core={}, max={}, durable database backlog enabled", - 4, 8); + corePoolSize, maxPoolSize); return executor; } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/ProviderPipelineActionController.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/ProviderPipelineActionController.java index 94924d1c..1754dd4f 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/ProviderPipelineActionController.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/ProviderPipelineActionController.java @@ -76,7 +76,8 @@ public ResponseEntity handlePrWebhook( PipelineActionProcessor.EventConsumer dualConsumer = pipelineJobService.createDualConsumer(jobRef, consumer); try { - return pipelineActionProcessor.processPipelineActionWithConsumer(payload, dualConsumer); + return pipelineActionProcessor.processPipelineActionWithConsumer( + payload, dualConsumer, jobRef); } catch (org.rostilos.codecrow.analysisengine.exception.AnalysisLockedException e) { log.warn("Analysis locked: {}", e.getMessage()); dualConsumer.accept(Map.of( @@ -131,7 +132,8 @@ public ResponseEntity handleBranchWebhook( PipelineActionProcessor.EventConsumer dualConsumer = pipelineJobService.createDualConsumer(jobRef, consumer); try { - return pipelineActionProcessor.processPipelineActionWithConsumer(payload, dualConsumer); + return pipelineActionProcessor.processPipelineActionWithConsumer( + payload, dualConsumer, jobRef); } catch (org.rostilos.codecrow.analysisengine.exception.AnalysisLockedException e) { log.warn("Analysis locked: {}", e.getMessage()); dualConsumer.accept(Map.of( diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/RagIndexingController.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/RagIndexingController.java index 4a87510f..cb26ed21 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/RagIndexingController.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/RagIndexingController.java @@ -2,9 +2,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.rostilos.codecrow.core.dto.project.ProjectDTO; -import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; -import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; import org.rostilos.codecrow.ragengine.service.VcsRagIndexingService; +import org.rostilos.codecrow.ragengine.branch.BranchIndexMaintenanceService; +import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; @@ -32,19 +32,19 @@ public class RagIndexingController { private static final String EOF_MARKER = "__EOF__"; private final VcsRagIndexingService vcsRagIndexingService; - private final RagIndexTrackingService ragIndexTrackingService; - private final AnalysisLockService analysisLockService; + private final BranchIndexMaintenanceService branchIndexMaintenanceService; + private final ProjectRepository projectRepository; private final ObjectMapper objectMapper; public RagIndexingController( VcsRagIndexingService vcsRagIndexingService, - RagIndexTrackingService ragIndexTrackingService, - AnalysisLockService analysisLockService, + BranchIndexMaintenanceService branchIndexMaintenanceService, + ProjectRepository projectRepository, ObjectMapper objectMapper ) { this.vcsRagIndexingService = vcsRagIndexingService; - this.ragIndexTrackingService = ragIndexTrackingService; - this.analysisLockService = analysisLockService; + this.branchIndexMaintenanceService = branchIndexMaintenanceService; + this.projectRepository = projectRepository; this.objectMapper = objectMapper; } @@ -73,11 +73,14 @@ public ResponseEntity triggerIndexing( CompletableFuture> indexingFuture = CompletableFuture.supplyAsync(() -> { try { - return vcsRagIndexingService.indexProjectFromVcs( - authProject, - request.branch(), - messageConsumer - ); + if (request.allConfiguredBranches() + || (request.branch() != null && !request.branch().isBlank())) { + var project = projectRepository.findByIdWithFullDetails(authProject.id()) + .orElseThrow(() -> new IllegalStateException("Project not found")); + return branchIndexMaintenanceService.rebuild( + project, request.branch(), request.allConfiguredBranches(), messageConsumer); + } + return vcsRagIndexingService.indexProjectFromVcs(authProject, null, messageConsumer); } catch (Exception e) { log.error("RAG indexing failed", e); return Map.of( @@ -144,6 +147,7 @@ public ResponseEntity canStartIndexing(@PathVariable Long projectId) { } public record RagIndexRequest( - String branch // Optional: branch to index. If null, uses project's configured RAG branch or default + String branch, + boolean allConfiguredBranches ) {} } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessor.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessor.java index b725a5d2..06d9eb44 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessor.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessor.java @@ -9,6 +9,8 @@ import org.rostilos.codecrow.analysisengine.processor.analysis.BranchAnalysisProcessor; import org.rostilos.codecrow.analysisengine.processor.analysis.PullRequestAnalysisProcessor; import org.rostilos.codecrow.analysisengine.service.ProjectValidationService; +import org.rostilos.codecrow.analysisengine.service.branch.BranchAnalysisGateService; +import org.rostilos.codecrow.core.model.job.Job; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -30,15 +32,18 @@ public class PipelineActionProcessor { private final ProjectValidationService projectService; private final PullRequestAnalysisProcessor pullRequestAnalysisProcessor; private final BranchAnalysisProcessor branchAnalysisProcessor; + private final BranchAnalysisGateService branchAnalysisGateService; public PipelineActionProcessor( ProjectValidationService projectService, PullRequestAnalysisProcessor pullRequestAnalysisProcessor, - BranchAnalysisProcessor branchAnalysisProcessor + BranchAnalysisProcessor branchAnalysisProcessor, + BranchAnalysisGateService branchAnalysisGateService ) { this.projectService = projectService; this.pullRequestAnalysisProcessor = pullRequestAnalysisProcessor; this.branchAnalysisProcessor = branchAnalysisProcessor; + this.branchAnalysisGateService = branchAnalysisGateService; } public interface EventConsumer { @@ -56,12 +61,36 @@ public Map processPipelineActionWithConsumer( @Valid @RequestBody AnalysisProcessRequest request, EventConsumer consumer ) throws GeneralSecurityException { + return processPipelineActionWithConsumer(request, consumer, null); + } + + public Map processPipelineActionWithConsumer( + AnalysisProcessRequest request, + EventConsumer consumer, + Job job + ) throws GeneralSecurityException { try { Project project = projectService.getProjectWithConnections(request.getProjectId()); + boolean dependenciesGated = job != null; + if (dependenciesGated) { + BranchAnalysisGateService.GateResult gateResult = + branchAnalysisGateService.awaitDependencies( + project.getId(), job, consumer::accept); + if (gateResult == BranchAnalysisGateService.GateResult.SUPERSEDED) { + return Map.of( + "status", "ignored", + "message", "Superseded by a newer branch analysis job"); + } + } if(request.getAnalysisType() == AnalysisType.BRANCH_ANALYSIS) { - return branchAnalysisProcessor.process((BranchProcessRequest) request, consumer::accept); + if (dependenciesGated) { + return branchAnalysisProcessor.processAfterDependencyGate( + (BranchProcessRequest) request, consumer::accept); + } + return branchAnalysisProcessor.process( + (BranchProcessRequest) request, consumer::accept); } else { return pullRequestAnalysisProcessor.process( (PrProcessRequest) request, diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessor.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessor.java index 67f7733c..45f074ba 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessor.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessor.java @@ -144,13 +144,13 @@ public void processWebhookInTransaction( jobService.startJob(job); log.info("jobService.startJob completed for job {}", job.getExternalId()); - if (job.getJobType() == JobType.BRANCH_ANALYSIS) { - BranchAnalysisGateService.GateResult gateResult = branchAnalysisGateService.awaitTurn( - projectId, - job.getBranchName(), - job.getId(), - job.getPrNumber(), - event -> logHandlerEvent(job, event)); + if (job.getJobType() == JobType.BRANCH_ANALYSIS + || job.getJobType() == JobType.PR_ANALYSIS) { + BranchAnalysisGateService.GateResult gateResult = + branchAnalysisGateService.awaitDependencies( + projectId, + job, + event -> logHandlerEvent(job, event)); if (gateResult == BranchAnalysisGateService.GateResult.SUPERSEDED) { String reason = "Superseded by a newer branch analysis job for " + job.getBranchName(); log.info("Skipping branch job {}: {}", job.getExternalId(), reason); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java index 7119dcfc..89af6e8c 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java @@ -126,6 +126,26 @@ public Job createRagIndexJob(Project project, boolean isInitial, JobTriggerSourc ); } + @Override + public Job createRagIndexJob( + Project project, + boolean isInitial, + JobTriggerSource triggerSource, + String branchName, + String commitHash) { + log.info("Creating branch-bound RAG {} job for project: {}, branch: {} (trigger: {})", + isInitial ? "initial indexing" : "incremental update", + project.getName(), branchName, triggerSource); + return jobService.createRagIndexJob( + project, + isInitial, + triggerSource, + null, + branchName, + commitHash + ); + } + /** * Alias for createRagIndexJob for backward compatibility. * Used by pipeline processors for RAG jobs triggered by webhooks. diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingService.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingService.java index f73b7475..b93baad1 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingService.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingService.java @@ -13,6 +13,7 @@ import org.rostilos.codecrow.vcsclient.bitbucket.service.ReportGenerator; import org.rostilos.codecrow.vcsclient.github.actions.CheckRunAction; import org.rostilos.codecrow.vcsclient.github.actions.CommentOnPullRequestAction; +import org.rostilos.codecrow.vcsclient.github.actions.GetPullRequestDiffAction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -103,23 +104,29 @@ public void postAnalysisResults( pullRequestNumber, placeholderCommentId); AnalysisSummary summary = reportGenerator.createAnalysisSummary(codeAnalysis, platformPrEntityId); + VcsRepoInfo vcsRepoInfo = getVcsRepoInfo(project); + OkHttpClient httpClient = vcsClientProvider.getHttpClient( + vcsRepoInfo.getVcsConnection() + ); + GitHubReviewFormatter.ReviewPlan reviewPlan = createReviewPlan( + httpClient, vcsRepoInfo, pullRequestNumber, summary); + // Use GitHub-specific markdown with collapsible spoilers for suggested fixes String markdownSummary = reportGenerator.createMarkdownSummary(codeAnalysis, summary, true); + String nonInlineFindingsMarkdown = reviewFormatter.formatNonInlineFindings( + reviewPlan.nonInlineFindings()); String detailedIssuesMarkdown = reportGenerator.createDetailedIssuesMarkdown(summary, true); // GitHub doesn't support threaded replies for issue comments like Bitbucket does. // So we combine summary and detailed issues into ONE comment. String fullComment = markdownSummary; + if (!nonInlineFindingsMarkdown.isEmpty()) { + fullComment += "\n\n---\n\n" + nonInlineFindingsMarkdown; + } if (detailedIssuesMarkdown != null && !detailedIssuesMarkdown.isEmpty()) { - fullComment = markdownSummary + "\n\n---\n\n" + detailedIssuesMarkdown; + fullComment += "\n\n---\n\n" + detailedIssuesMarkdown; } - VcsRepoInfo vcsRepoInfo = getVcsRepoInfo(project); - - OkHttpClient httpClient = vcsClientProvider.getHttpClient( - vcsRepoInfo.getVcsConnection() - ); - // Post or update comment with full content (summary + issues) if (placeholderCommentId != null) { updatePlaceholderComment(httpClient, vcsRepoInfo, pullRequestNumber, fullComment, placeholderCommentId); @@ -129,13 +136,44 @@ public void postAnalysisResults( // Publish findings as a submitted COMMENT review so they appear inline in // "Files changed" and as grouped reviewer comments in the PR conversation. - postInlineReviewComments(httpClient, vcsRepoInfo, pullRequestNumber, codeAnalysis, summary); + postInlineReviewComments( + httpClient, vcsRepoInfo, pullRequestNumber, codeAnalysis, reviewPlan); // Create Check Run for the commit createCheckRun(httpClient, vcsRepoInfo, codeAnalysis, summary); log.info("Successfully posted analysis results to GitHub"); } + + private GitHubReviewFormatter.ReviewPlan createReviewPlan( + OkHttpClient httpClient, + VcsRepoInfo vcsRepoInfo, + Long pullRequestNumber, + AnalysisSummary summary + ) { + try { + List filePatches = + new GetPullRequestDiffAction(httpClient).getPullRequestFilePatches( + vcsRepoInfo.getRepoWorkspace(), + vcsRepoInfo.getRepoSlug(), + pullRequestNumber.intValue()); + GitHubReviewFormatter.ReviewPlan plan = reviewFormatter.planComments( + summary.getIssues(), CODECROW_REVIEW_MARKER, filePatches); + log.info("Prepared GitHub review plan for PR {}: {} inline, {} not inline", + pullRequestNumber, + plan.inlineComments().size(), + plan.nonInlineFindings().size()); + return plan; + } catch (Exception e) { + // Diff enrichment is supplemental. Publishing an unverified anchor can + // make GitHub reject the whole review, so retain every finding in the + // aggregate comment instead of guessing. + log.warn("Could not load GitHub PR diff for inline comment planning on PR {}: {}. " + + "Findings will remain in the aggregate comment.", + pullRequestNumber, e.getMessage()); + return reviewFormatter.planWithoutDiff(summary.getIssues()); + } + } /** * Post summary as a regular comment. @@ -201,15 +239,14 @@ private void postInlineReviewComments( VcsRepoInfo vcsRepoInfo, Long pullRequestNumber, CodeAnalysis codeAnalysis, - AnalysisSummary summary + GitHubReviewFormatter.ReviewPlan reviewPlan ) { CommentOnPullRequestAction commentAction = new CommentOnPullRequestAction(httpClient); - cleanupPreviousInlineReviewComments(commentAction, vcsRepoInfo, pullRequestNumber); - - List> comments = reviewFormatter.formatComments( - summary.getIssues(), CODECROW_REVIEW_MARKER); + List> comments = reviewPlan.inlineComments(); if (comments.isEmpty()) { - log.debug("No confidently anchored issues to post as GitHub review comments"); + cleanupPreviousInlineReviewComments( + commentAction, vcsRepoInfo, pullRequestNumber, null); + log.debug("No diff-verified issues to post as GitHub review comments"); return; } @@ -221,7 +258,7 @@ private void postInlineReviewComments( } try { - commentAction.createPullRequestReview( + String reviewId = commentAction.createPullRequestReview( vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(), pullRequestNumber.intValue(), @@ -232,27 +269,53 @@ private void postInlineReviewComments( ); log.info("Posted GitHub review with {} inline comment(s) on PR {}", comments.size(), pullRequestNumber); + Long preservedReviewId = parseReviewId(reviewId); + if (preservedReviewId == null) { + log.warn("GitHub created the replacement review for PR {} without a usable ID; " + + "leaving previous review artifacts intact", + pullRequestNumber); + } else { + cleanupPreviousInlineReviewComments( + commentAction, + vcsRepoInfo, + pullRequestNumber, + preservedReviewId); + } } catch (Exception e) { // Inline review publishing is supplemental. The aggregate comment - // still contains the complete result, and the Check Run still carries - // the quality status when GitHub rejects a stale or non-diff anchor. + // still contains the complete result. Cleanup deliberately happens + // only after a replacement succeeds, so a rejected review does not + // erase the last successfully published inline comments. log.warn("Failed to post inline GitHub review comments on PR {}: {}. " + "Issues remain available in the summary comment.", pullRequestNumber, e.getMessage()); } } + private Long parseReviewId(String reviewId) { + if (reviewId == null || reviewId.isBlank()) { + return null; + } + try { + return Long.parseLong(reviewId); + } catch (NumberFormatException e) { + return null; + } + } + private void cleanupPreviousInlineReviewComments( CommentOnPullRequestAction commentAction, VcsRepoInfo vcsRepoInfo, - Long pullRequestNumber + Long pullRequestNumber, + Long preservedReviewId ) { try { int deleted = commentAction.deletePreviousReviewComments( vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(), pullRequestNumber.intValue(), - CODECROW_REVIEW_MARKER + CODECROW_REVIEW_MARKER, + preservedReviewId ); if (deleted > 0) { log.info("Deleted {} previous CodeCrow inline review comment(s) from PR {}", @@ -271,7 +334,8 @@ private void cleanupPreviousInlineReviewComments( vcsRepoInfo.getRepoSlug(), pullRequestNumber.intValue(), CODECROW_REVIEW_MARKER, - CODECROW_CLEARED_REVIEW_MARKER + CODECROW_CLEARED_REVIEW_MARKER, + preservedReviewId ); if (cleared > 0) { log.info("Cleared {} previous CodeCrow review summary body/bodies from PR {}", diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatter.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatter.java index 17266831..af897d3b 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatter.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatter.java @@ -2,36 +2,60 @@ import org.rostilos.codecrow.core.util.tracking.DiffSanitizer; import org.rostilos.codecrow.vcsclient.bitbucket.model.report.AnalysisSummary; +import org.rostilos.codecrow.vcsclient.github.actions.GetPullRequestDiffAction.PullRequestFilePatch; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Formats CodeCrow issues for GitHub's pull-request review API. */ final class GitHubReviewFormatter { private static final int MAX_INLINE_COMMENTS = 20; + private static final Pattern HUNK_HEADER = Pattern.compile( + "^@@ -\\d+(?:,\\d+)? \\+(\\d+)(?:,\\d+)? @@.*$"); + private static final String OUTSIDE_DIFF_REASON = + "The reported line is outside the current pull-request diff."; + private static final String DIFF_UNAVAILABLE_REASON = + "The current pull-request diff could not be loaded, so CodeCrow did not guess an inline anchor."; - List> formatComments( + ReviewPlan planComments( List issues, - String marker + String marker, + List filePatches ) { if (issues == null || issues.isEmpty()) { - return List.of(); + return ReviewPlan.empty(); } + Map> rightSideLines = collectRightSideLines(filePatches); List> comments = new ArrayList<>(); + List nonInlineFindings = new ArrayList<>(); for (AnalysisSummary.IssueSummary issue : issues) { - if (comments.size() >= MAX_INLINE_COMMENTS) { - break; - } - String path = normalizePath(issue.getFilePath()); Integer line = issue.getLineNumber(); - if (path == null || line == null || line <= 0 || !hasConfidentAnchor(issue)) { + String ineligibleReason = ineligibleReason(issue, path, line); + if (ineligibleReason != null) { + nonInlineFindings.add(new NonInlineFinding(issue, ineligibleReason)); + continue; + } + if (!rightSideLines.getOrDefault(path, Set.of()).contains(line)) { + nonInlineFindings.add(new NonInlineFinding(issue, OUTSIDE_DIFF_REASON)); + continue; + } + if (comments.size() >= MAX_INLINE_COMMENTS) { + nonInlineFindings.add(new NonInlineFinding( + issue, + "GitHub's limit of " + MAX_INLINE_COMMENTS + + " CodeCrow inline comments was reached.")); continue; } @@ -42,7 +66,52 @@ List> formatComments( comment.put("body", formatBody(issue, marker)); comments.add(comment); } - return List.copyOf(comments); + return new ReviewPlan(comments, nonInlineFindings); + } + + ReviewPlan planWithoutDiff(List issues) { + if (issues == null || issues.isEmpty()) { + return ReviewPlan.empty(); + } + return new ReviewPlan( + List.of(), + issues.stream() + .map(issue -> new NonInlineFinding(issue, DIFF_UNAVAILABLE_REASON)) + .toList()); + } + + String formatNonInlineFindings(List findings) { + if (findings == null || findings.isEmpty()) { + return ""; + } + + StringBuilder markdown = new StringBuilder(); + markdown.append("
    \n📍 Findings not posted inline (") + .append(findings.size()) + .append(")\n\n") + .append("GitHub only accepts inline review comments on lines available in the current pull-request diff. ") + .append("These findings remain part of the complete review.\n\n"); + + for (NonInlineFinding finding : findings) { + AnalysisSummary.IssueSummary issue = finding.issue(); + String title = issue.getTitle() == null || issue.getTitle().isBlank() + ? "Untitled finding" + : issue.getTitle(); + markdown.append("- ").append(severityEmoji(issue)).append(" **") + .append(issue.getSeverity()).append("** — "); + if (issue.getIssueUrl() != null && !issue.getIssueUrl().isBlank()) { + markdown.append("[").append(title).append("](") + .append(issue.getIssueUrl()).append(")"); + } else { + markdown.append("**").append(title).append("**"); + } + markdown.append(" at `") + .append(issue.getLocationDescription().replace("`", "\\`")) + .append("`\n - ").append(finding.reason()).append("\n"); + } + + markdown.append("\n
    "); + return markdown.toString(); } String formatReviewBody(int commentCount, String marker) { @@ -58,6 +127,68 @@ private boolean hasConfidentAnchor(AnalysisSummary.IssueSummary issue) { || (issue.getCodeSnippet() != null && !issue.getCodeSnippet().isBlank()); } + private String ineligibleReason( + AnalysisSummary.IssueSummary issue, + String path, + Integer line + ) { + if (path == null) { + return "No repository file path was available for an inline anchor."; + } + if (line == null || line <= 0) { + return "No positive source line was available for an inline anchor."; + } + if (!hasConfidentAnchor(issue)) { + return "The finding does not have a confident source-line anchor."; + } + return null; + } + + private Map> collectRightSideLines( + List filePatches + ) { + Map> rightSideLines = new HashMap<>(); + if (filePatches == null) { + return rightSideLines; + } + + for (PullRequestFilePatch filePatch : filePatches) { + String path = normalizePath(filePatch.filename()); + if (path == null || filePatch.patch().isBlank()) { + continue; + } + Set lines = rightSideLines.computeIfAbsent( + path, ignored -> new HashSet<>()); + collectRightSideLines(filePatch.patch(), lines); + } + return rightSideLines; + } + + private void collectRightSideLines(String patch, Set lines) { + int nextRightLine = -1; + boolean inHunk = false; + for (String diffLine : patch.split("\\r?\\n", -1)) { + Matcher hunk = HUNK_HEADER.matcher(diffLine); + if (hunk.matches()) { + nextRightLine = Integer.parseInt(hunk.group(1)); + inHunk = true; + continue; + } + if (!inHunk || diffLine.isEmpty() || diffLine.startsWith("\\")) { + continue; + } + + char prefix = diffLine.charAt(0); + if (prefix == '+') { + lines.add(nextRightLine++); + } else if (prefix == ' ') { + lines.add(nextRightLine++); + } else if (prefix != '-') { + inHunk = false; + } + } + } + private String normalizePath(String path) { if (path == null || path.isBlank()) { return null; @@ -120,6 +251,9 @@ private void appendSuggestedFix( } private String severityEmoji(AnalysisSummary.IssueSummary issue) { + if (issue.getSeverity() == null) { + return "ℹ️"; + } return switch (issue.getSeverity()) { case HIGH -> "🔴"; case MEDIUM -> "🟡"; @@ -145,4 +279,23 @@ private String humanizeCategory(String category) { } return result.toString(); } + + record ReviewPlan( + List> inlineComments, + List nonInlineFindings + ) { + ReviewPlan { + inlineComments = inlineComments == null ? List.of() : List.copyOf(inlineComments); + nonInlineFindings = nonInlineFindings == null + ? List.of() + : List.copyOf(nonInlineFindings); + } + + static ReviewPlan empty() { + return new ReviewPlan(List.of(), List.of()); + } + } + + record NonInlineFinding(AnalysisSummary.IssueSummary issue, String reason) { + } } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubBranchWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubBranchWebhookHandler.java index 44a60c65..6366380f 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubBranchWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubBranchWebhookHandler.java @@ -100,7 +100,8 @@ public WebhookResult handle(WebhookPayload payload, Project project, Consumer result = branchAnalysisProcessor.process(request, processorConsumer); + Map result = branchAnalysisProcessor.processAfterDependencyGate( + request, processorConsumer); return WebhookResult.success("Branch analysis completed", result); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java index b4d70fff..80ad0679 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java @@ -370,7 +370,8 @@ private WebhookResult handlePrMergeEvent( } }; - Map result = branchAnalysisProcessor.process(request, processorConsumer); + Map result = branchAnalysisProcessor.processAfterDependencyGate( + request, processorConsumer); return WebhookResult.success("Branch reconciliation completed after PR #" + prNumber + " merge", result); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabBranchWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabBranchWebhookHandler.java index ad21e124..371643b3 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabBranchWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabBranchWebhookHandler.java @@ -109,7 +109,8 @@ private WebhookResult handlePushEvent( }; // Delegate to branch analysis processor - Map result = branchAnalysisProcessor.process(request, processorConsumer); + Map result = branchAnalysisProcessor.processAfterDependencyGate( + request, processorConsumer); // Check if analysis failed if ("error".equals(result.get("status"))) { diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java index 2363e385..51993f7d 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java @@ -161,7 +161,8 @@ private WebhookResult handleMrMergeEvent( } }; - Map result = branchAnalysisProcessor.process(request, processorConsumer); + Map result = branchAnalysisProcessor.processAfterDependencyGate( + request, processorConsumer); return WebhookResult.success("Branch reconciliation completed after MR !" + mrNumber + " merge", result); diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfigTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfigTest.java new file mode 100644 index 00000000..63942429 --- /dev/null +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfigTest.java @@ -0,0 +1,40 @@ +package org.rostilos.codecrow.pipelineagent.config; + +import org.junit.jupiter.api.Test; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +class AsyncConfigTest { + + @Test + void webhookExecutorRunsFiveAcceptedReviewsInParallel() throws Exception { + Executor configured = new AsyncConfig().webhookExecutor(2, 5); + ThreadPoolTaskExecutor executor = (ThreadPoolTaskExecutor) configured; + CountDownLatch started = new CountDownLatch(5); + CountDownLatch release = new CountDownLatch(1); + + try { + for (int i = 0; i < 5; i++) { + executor.execute(() -> { + started.countDown(); + try { + release.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + }); + } + + assertThat(started.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(executor.getMaxPoolSize()).isEqualTo(5); + } finally { + release.countDown(); + executor.shutdown(); + } + } +} diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessorDependencyGateTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessorDependencyGateTest.java new file mode 100644 index 00000000..e1cd64bd --- /dev/null +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessorDependencyGateTest.java @@ -0,0 +1,103 @@ +package org.rostilos.codecrow.pipelineagent.generic.processor; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest; +import org.rostilos.codecrow.analysisengine.dto.request.processor.PrProcessRequest; +import org.rostilos.codecrow.analysisengine.processor.analysis.BranchAnalysisProcessor; +import org.rostilos.codecrow.analysisengine.processor.analysis.PullRequestAnalysisProcessor; +import org.rostilos.codecrow.analysisengine.service.ProjectValidationService; +import org.rostilos.codecrow.analysisengine.service.branch.BranchAnalysisGateService; +import org.rostilos.codecrow.core.model.codeanalysis.AnalysisType; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobType; +import org.rostilos.codecrow.core.model.project.Project; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class PipelineActionProcessorDependencyGateTest { + + @Mock private ProjectValidationService projectService; + @Mock private PullRequestAnalysisProcessor pullRequestAnalysisProcessor; + @Mock private BranchAnalysisProcessor branchAnalysisProcessor; + @Mock private BranchAnalysisGateService branchAnalysisGateService; + @Mock private Project project; + + private PipelineActionProcessor processor; + + @BeforeEach + void setUp() throws Exception { + processor = new PipelineActionProcessor( + projectService, + pullRequestAnalysisProcessor, + branchAnalysisProcessor, + branchAnalysisGateService); + when(projectService.getProjectWithConnections(1L)).thenReturn(project); + when(project.getId()).thenReturn(1L); + } + + @Test + void pipelinePrResolvesDurableDependenciesBeforeAnalysis() throws Exception { + PrProcessRequest request = new PrProcessRequest(); + request.projectId = 1L; + request.analysisType = AnalysisType.PR_REVIEW; + Job job = new Job(); + job.setJobType(JobType.PR_ANALYSIS); + + when(branchAnalysisGateService.awaitDependencies( + org.mockito.ArgumentMatchers.eq(1L), + org.mockito.ArgumentMatchers.eq(job), + any())).thenReturn(BranchAnalysisGateService.GateResult.READY); + when(pullRequestAnalysisProcessor.process(any(), any(), any())) + .thenReturn(Map.of("status", "accepted")); + + Map result = processor.processPipelineActionWithConsumer( + request, event -> { }, job); + + assertThat(result).containsEntry("status", "accepted"); + InOrder ordered = inOrder(branchAnalysisGateService, pullRequestAnalysisProcessor); + ordered.verify(branchAnalysisGateService).awaitDependencies( + org.mockito.ArgumentMatchers.eq(1L), + org.mockito.ArgumentMatchers.eq(job), + any()); + ordered.verify(pullRequestAnalysisProcessor).process( + org.mockito.ArgumentMatchers.eq(request), any(), + org.mockito.ArgumentMatchers.eq(project)); + } + + @Test + void pipelineBranchSkipsTheProcessorFallbackAfterDurableGate() throws Exception { + BranchProcessRequest request = new BranchProcessRequest(); + request.projectId = 1L; + request.analysisType = AnalysisType.BRANCH_ANALYSIS; + Job job = new Job(); + job.setJobType(JobType.BRANCH_ANALYSIS); + + when(branchAnalysisGateService.awaitDependencies( + org.mockito.ArgumentMatchers.eq(1L), + org.mockito.ArgumentMatchers.eq(job), + any())).thenReturn(BranchAnalysisGateService.GateResult.READY); + when(branchAnalysisProcessor.processAfterDependencyGate(any(), any())) + .thenReturn(Map.of("status", "accepted")); + + Map result = processor.processPipelineActionWithConsumer( + request, event -> { }, job); + + assertThat(result).containsEntry("status", "accepted"); + verify(branchAnalysisProcessor).processAfterDependencyGate( + org.mockito.ArgumentMatchers.eq(request), any()); + verify(branchAnalysisProcessor, never()).process(any(), any()); + } +} diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessorBranchGateTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessorBranchGateTest.java index 13e561ec..f4737fca 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessorBranchGateTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessorBranchGateTest.java @@ -72,11 +72,9 @@ void supersededBranchJobNeverInvokesProviderHandler() { "41", "feature/one", "main", "merge-1", null); when(projectRepository.findById(1L)).thenReturn(Optional.of(project)); - when(branchAnalysisGateService.awaitTurn( + when(branchAnalysisGateService.awaitDependencies( org.mockito.ArgumentMatchers.eq(1L), - org.mockito.ArgumentMatchers.eq("main"), - org.mockito.ArgumentMatchers.eq(101L), - org.mockito.ArgumentMatchers.eq(41L), + org.mockito.ArgumentMatchers.eq(branchJob), any())) .thenReturn(BranchAnalysisGateService.GateResult.SUPERSEDED); @@ -92,6 +90,39 @@ void supersededBranchJobNeverInvokesProviderHandler() { verify(ragOperationsService).deletePrFiles(project, 41); } + @Test + void prJobPassesTargetBranchDependencyGateBeforeProviderHandler() { + Job prJob = new Job(); + ReflectionTestUtils.setField(prJob, "id", 102L); + prJob.setProject(project); + prJob.setJobType(JobType.PR_ANALYSIS); + prJob.setBranchName("main"); + prJob.setPrNumber(42L); + + WebhookPayload payload = new WebhookPayload( + EVcsProvider.GITHUB, "pull_request", "repo-id", "repo", "owner", + "42", "feature/two", "main", "head-2", null); + WebhookHandler.WebhookResult success = WebhookHandler.WebhookResult.ignored( + "test dependency ordering"); + + when(projectRepository.findById(1L)).thenReturn(Optional.of(project)); + when(branchAnalysisGateService.awaitDependencies( + org.mockito.ArgumentMatchers.eq(1L), + org.mockito.ArgumentMatchers.eq(prJob), + any())).thenReturn(BranchAnalysisGateService.GateResult.READY); + when(handler.handle(any(), any(), any())).thenReturn(success); + + processor.processWebhookInTransaction( + EVcsProvider.GITHUB, 1L, payload, handler, prJob); + + var ordered = org.mockito.Mockito.inOrder(branchAnalysisGateService, handler); + ordered.verify(branchAnalysisGateService).awaitDependencies( + org.mockito.ArgumentMatchers.eq(1L), + org.mockito.ArgumentMatchers.eq(prJob), + any()); + ordered.verify(handler).handle(any(), any(), any()); + } + @Test void promptDryRunSuppressesEveryWebhookVcsMutationPath() { System.setProperty(PromptDryRunMode.ENABLED_KEY, "true"); diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingServiceTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingServiceTest.java index 5d8c3386..8b7d524c 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingServiceTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingServiceTest.java @@ -50,6 +50,7 @@ class GitHubReportingServiceTest { private CodeAnalysis analysis; private Project project; private AnalysisSummary summary; + private AnalysisSummary.IssueSummary inlineIssue; @BeforeEach void setUp() { @@ -68,7 +69,7 @@ void setUp() { when(repoInfo.getVcsConnection()).thenReturn(connection); org.mockito.Mockito.lenient().when(analysis.getCommitHash()).thenReturn("head-sha"); - AnalysisSummary.IssueSummary issue = new AnalysisSummary.IssueSummary( + inlineIssue = new AnalysisSummary.IssueSummary( IssueSeverity.HIGH, "SECURITY", "src/App.java", @@ -81,7 +82,7 @@ void setUp() { 7L, "return repository.findById(id);" ); - org.mockito.Mockito.lenient().when(summary.getIssues()).thenReturn(List.of(issue)); + org.mockito.Mockito.lenient().when(summary.getIssues()).thenReturn(List.of(inlineIssue)); org.mockito.Mockito.lenient().when(summary.getTotalUnresolvedIssues()).thenReturn(1); org.mockito.Mockito.lenient().when(summary.getFileIssueCount()).thenReturn(Map.of()); org.mockito.Mockito.lenient().when(reportGenerator.createAnalysisSummary(analysis, 77L)).thenReturn(summary); @@ -125,7 +126,82 @@ void preservesAggregateCommentAndAddsSubmittedInlineReview() throws IOException } @Test - void removesPreviousGeneratedReviewArtifactsBeforePostingReplacement() throws IOException { + void postsValidCommentsAndAddsOutOfDiffFindingsToTheAggregateSpoiler() + throws IOException { + AnalysisSummary.IssueSummary outsideDiff = new AnalysisSummary.IssueSummary( + IssueSeverity.MEDIUM, + "TESTING", + "src/App.java", + 1, + "File-wide constructor mismatch", + "The reported location is not part of the pull-request patch.", + null, + null, + "https://codecrow.example/issues/8", + 8L, + "service = new AppService();" + ); + when(summary.getIssues()).thenReturn(List.of(inlineIssue, outsideDiff)); + + List requests = new ArrayList<>(); + when(vcsClientProvider.getHttpClient(org.mockito.ArgumentMatchers.any())) + .thenReturn(capturingClient(requests, false, false)); + + service.postAnalysisResults(analysis, project, 42L, 77L, "99"); + + CapturedRequest aggregate = requestAt(requests, "/repos/owner/repo/issues/comments/99"); + String aggregateBody = OBJECT_MAPPER.readTree(aggregate.body()).path("body").asText(); + assertThat(aggregateBody) + .contains("📍 Findings not posted inline (1)") + .contains("[File-wide constructor mismatch](https://codecrow.example/issues/8)") + .contains("outside the current pull-request diff"); + + CapturedRequest review = requestAt( + requests, "POST", "/repos/owner/repo/pulls/42/reviews"); + JsonNode reviewPayload = OBJECT_MAPPER.readTree(review.body()); + assertThat(reviewPayload.path("comments")).hasSize(1); + assertThat(reviewPayload.path("comments").get(0).path("line").asInt()).isEqualTo(12); + } + + @Test + void allOutOfDiffFindingsStayInTheAggregateAndRemoveStaleInlineArtifacts() + throws IOException { + AnalysisSummary.IssueSummary outsideDiff = new AnalysisSummary.IssueSummary( + IssueSeverity.MEDIUM, + "TESTING", + "src/App.java", + 1, + "File-wide constructor mismatch", + "The reported location is not part of the pull-request patch.", + null, + null, + "https://codecrow.example/issues/8", + 8L, + "service = new AppService();" + ); + when(summary.getIssues()).thenReturn(List.of(outsideDiff)); + + List requests = new ArrayList<>(); + when(vcsClientProvider.getHttpClient(org.mockito.ArgumentMatchers.any())) + .thenReturn(capturingClient(requests, false, true)); + + service.postAnalysisResults(analysis, project, 42L, 77L, "99"); + + assertThat(requests).noneMatch(request -> request.method().equals("POST") + && request.path().equals("/repos/owner/repo/pulls/42/reviews")); + assertThat(indexOf(requests, "DELETE", "/repos/owner/repo/pulls/comments/321")) + .isGreaterThanOrEqualTo(0); + assertThat(indexOf(requests, "PUT", "/repos/owner/repo/pulls/42/reviews/654")) + .isGreaterThanOrEqualTo(0); + + CapturedRequest aggregate = requestAt(requests, "/repos/owner/repo/issues/comments/99"); + assertThat(OBJECT_MAPPER.readTree(aggregate.body()).path("body").asText()) + .contains("Findings not posted inline (1)") + .contains("File-wide constructor mismatch"); + } + + @Test + void removesPreviousGeneratedReviewArtifactsAfterPostingReplacement() throws IOException { List requests = new ArrayList<>(); when(vcsClientProvider.getHttpClient(org.mockito.ArgumentMatchers.any())) .thenReturn(capturingClient(requests, false, true)); @@ -137,17 +213,17 @@ void removesPreviousGeneratedReviewArtifactsBeforePostingReplacement() throws IO int reviewIndex = indexOf(requests, "POST", "/repos/owner/repo/pulls/42/reviews"); assertThat(deleteIndex).isGreaterThanOrEqualTo(0); assertThat(clearIndex).isGreaterThan(deleteIndex); - assertThat(reviewIndex).isGreaterThan(deleteIndex); - assertThat(reviewIndex).isGreaterThan(clearIndex); + assertThat(reviewIndex).isLessThan(deleteIndex); + assertThat(reviewIndex).isLessThan(clearIndex); assertThat(OBJECT_MAPPER.readTree(requests.get(clearIndex).body()).path("body").asText()) .isEqualTo(""); } @Test - void reviewRejectionDoesNotBlockTheSummaryOrCheckRun() { + void reviewRejectionDoesNotBlockTheSummaryOrCheckRunOrDeletePreviousComments() { List requests = new ArrayList<>(); when(vcsClientProvider.getHttpClient(org.mockito.ArgumentMatchers.any())) - .thenReturn(capturingClient(requests, true, false)); + .thenReturn(capturingClient(requests, true, true)); assertThatCode(() -> service.postAnalysisResults(analysis, project, 42L, 77L, "99")) .doesNotThrowAnyException(); @@ -158,6 +234,9 @@ void reviewRejectionDoesNotBlockTheSummaryOrCheckRun() { "/repos/owner/repo/pulls/42/reviews", "/repos/owner/repo/check-runs" ); + assertThat(requests).noneMatch(request -> request.method().equals("DELETE")); + assertThat(requests).noneMatch(request -> request.method().equals("PUT") + && request.path().equals("/repos/owner/repo/pulls/42/reviews/654")); } @Test @@ -206,15 +285,22 @@ private OkHttpClient capturingClient( requests.add(new CapturedRequest(request.method(), path, buffer.readUtf8())); boolean reviewPost = request.method().equals("POST") && path.endsWith("/reviews"); + boolean pullRequestFiles = request.method().equals("GET") + && path.endsWith("/pulls/42/files"); boolean reviewCommentList = request.method().equals("GET") && path.endsWith("/pulls/42/comments"); boolean reviewList = request.method().equals("GET") && path.endsWith("/pulls/42/reviews"); boolean rejected = rejectReviews && reviewPost; String responseJson; - if (reviewCommentList) { + if (pullRequestFiles) { + responseJson = "[{\"filename\":\"src/App.java\"," + + "\"status\":\"modified\"," + + "\"patch\":\"@@ -12 +12 @@\\n-old\\n+new\"}]"; + } else if (reviewCommentList) { responseJson = includePreviousReviewComment - ? "[{\"id\":321,\"body\":\"old \"}]" + ? "[{\"id\":321,\"pull_request_review_id\":654," + + "\"body\":\"old \"}]" : "[]"; } else if (reviewList) { responseJson = includePreviousReviewComment diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatterTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatterTest.java index d64764ab..1024dfd3 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatterTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatterTest.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.Test; import org.rostilos.codecrow.core.model.codeanalysis.IssueSeverity; import org.rostilos.codecrow.vcsclient.bitbucket.model.report.AnalysisSummary; +import org.rostilos.codecrow.vcsclient.github.actions.GetPullRequestDiffAction.PullRequestFilePatch; import java.util.List; import java.util.Map; @@ -25,9 +26,13 @@ void formatsAnchoredIssuesAsGitHubReviewComments() { "--- a/src/App.java\n+++ b/src/App.java\n@@ -42 +42 @@\n-old\n+new" ); - List> comments = formatter.formatComments(List.of(issue), MARKER); + GitHubReviewFormatter.ReviewPlan plan = formatter.planComments( + List.of(issue), MARKER, List.of(patch( + "src/App.java", "@@ -42 +42 @@\n-old\n+new"))); + List> comments = plan.inlineComments(); assertThat(comments).hasSize(1); + assertThat(plan.nonInlineFindings()).isEmpty(); assertThat(comments.get(0)) .containsEntry("path", "src/App.java") .containsEntry("line", 42) @@ -49,14 +54,17 @@ void skipsIssuesWithoutAConfidentLineAnchor() { AnalysisSummary.IssueSummary syntheticLineOne = issue( "src/App.java", 1, "Synthetic anchor", "Reason", null, null); - List> comments = formatter.formatComments( - List.of(noPath, noLine, syntheticLineOne), MARKER); + GitHubReviewFormatter.ReviewPlan plan = formatter.planComments( + List.of(noPath, noLine, syntheticLineOne), + MARKER, + List.of(patch("src/App.java", "@@ -1,10 +1,10 @@\n context"))); - assertThat(comments).isEmpty(); + assertThat(plan.inlineComments()).isEmpty(); + assertThat(plan.nonInlineFindings()).hasSize(3); } @Test - void keepsLineOneWhenTheIssueIncludesItsSourceSnippet() { + void keepsLineOneWithASourceSnippetOnlyWhenItIsInTheDiff() { AnalysisSummary.IssueSummary issue = new AnalysisSummary.IssueSummary( IssueSeverity.LOW, "CODE_QUALITY", @@ -71,7 +79,47 @@ void keepsLineOneWhenTheIssueIncludesItsSourceSnippet() { "package example;" ); - assertThat(formatter.formatComments(List.of(issue), MARKER)).hasSize(1); + GitHubReviewFormatter.ReviewPlan outsideDiff = formatter.planComments( + List.of(issue), + MARKER, + List.of(patch("src/App.java", "@@ -20 +20 @@\n-old\n+new"))); + GitHubReviewFormatter.ReviewPlan newFile = formatter.planComments( + List.of(issue), + MARKER, + List.of(patch("src/App.java", "@@ -0,0 +1,2 @@\n+package example;\n+class App {}"))); + + assertThat(outsideDiff.inlineComments()).isEmpty(); + assertThat(outsideDiff.nonInlineFindings()).singleElement() + .satisfies(finding -> assertThat(finding.reason()) + .contains("outside the current pull-request diff")); + assertThat(newFile.inlineComments()).hasSize(1); + assertThat(newFile.nonInlineFindings()).isEmpty(); + } + + @Test + void separatesOutOfDiffFindingsWithoutDroppingValidComments() { + AnalysisSummary.IssueSummary valid = issue( + "src/App.java", 42, "Valid anchor", "Reason", null, null); + AnalysisSummary.IssueSummary outside = issue( + "src/App.java", 67, "File-wide finding", "Reason", null, null); + + GitHubReviewFormatter.ReviewPlan plan = formatter.planComments( + List.of(valid, outside), + MARKER, + List.of(patch("src/App.java", "@@ -42 +42 @@\n-old\n+new"))); + + assertThat(plan.inlineComments()).singleElement() + .satisfies(comment -> assertThat(comment).containsEntry("line", 42)); + assertThat(plan.nonInlineFindings()).singleElement() + .satisfies(finding -> { + assertThat(finding.issue().getTitle()).isEqualTo("File-wide finding"); + assertThat(finding.reason()).contains("outside the current pull-request diff"); + }); + assertThat(formatter.formatNonInlineFindings(plan.nonInlineFindings())) + .contains("📍 Findings not posted inline (1)") + .contains("[File-wide finding](https://codecrow.example/issues/1)") + .contains("`src/App.java:67`") + .contains("outside the current pull-request diff"); } @Test @@ -79,13 +127,32 @@ void limitsTheReviewToTwentyInlineComments() { AnalysisSummary.IssueSummary issue = issue( "src/App.java", 10, "Title", "Reason", null, null); - assertThat(formatter.formatComments(java.util.Collections.nCopies(25, issue), MARKER)) - .hasSize(20); + GitHubReviewFormatter.ReviewPlan plan = formatter.planComments( + java.util.Collections.nCopies(25, issue), + MARKER, + List.of(patch("src/App.java", "@@ -10 +10 @@\n-old\n+new"))); + + assertThat(plan.inlineComments()).hasSize(20); + assertThat(plan.nonInlineFindings()).hasSize(5) + .allSatisfy(finding -> assertThat(finding.reason()).contains("limit of 20")); assertThat(formatter.formatReviewBody(20, MARKER)) .contains("**Actionable comments posted: 20**") .endsWith(MARKER); } + @Test + void keepsAllFindingsInTheSummaryWhenTheDiffCannotBeLoaded() { + AnalysisSummary.IssueSummary issue = issue( + "src/App.java", 10, "Title", "Reason", null, null); + + GitHubReviewFormatter.ReviewPlan plan = formatter.planWithoutDiff(List.of(issue)); + + assertThat(plan.inlineComments()).isEmpty(); + assertThat(plan.nonInlineFindings()).singleElement() + .satisfies(finding -> assertThat(finding.reason()) + .contains("diff could not be loaded")); + } + private AnalysisSummary.IssueSummary issue( String path, Integer line, @@ -107,4 +174,8 @@ private AnalysisSummary.IssueSummary issue( 1L ); } + + private PullRequestFilePatch patch(String path, String patch) { + return new PullRequestFilePatch(path, "", "modified", patch); + } } diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/controller/ProjectController.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/controller/ProjectController.java index 9e17203a..4ced2b30 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/controller/ProjectController.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/controller/ProjectController.java @@ -29,11 +29,13 @@ import org.rostilos.codecrow.webserver.project.dto.request.ChangeVcsConnectionRequest; import org.rostilos.codecrow.webserver.project.dto.ProjectTokenDTO; import org.rostilos.codecrow.webserver.project.dto.response.RagIndexStatusDTO; +import org.rostilos.codecrow.webserver.project.dto.response.RagBranchIndexStatusDTO; import org.rostilos.codecrow.webserver.auth.service.TwoFactorAuthService; import org.rostilos.codecrow.webserver.project.service.ProjectService; import org.rostilos.codecrow.webserver.project.service.ProjectTokenService; import org.rostilos.codecrow.webserver.project.service.RagIndexStatusService; import org.rostilos.codecrow.webserver.project.service.RagIndexingTriggerService; +import org.rostilos.codecrow.webserver.project.service.RagBranchIndexStatusService; import org.rostilos.codecrow.webserver.project.service.VectorStorageService; import org.rostilos.codecrow.webserver.workspace.service.WorkspaceService; import org.springframework.http.HttpStatus; @@ -66,6 +68,7 @@ public class ProjectController { private final WorkspaceService workspaceService; private final RagIndexStatusService ragIndexStatusService; private final RagIndexingTriggerService ragIndexingTriggerService; + private final RagBranchIndexStatusService ragBranchIndexStatusService; private final VectorStorageService vectorStorageService; private final TwoFactorAuthService twoFactorAuthService; @@ -75,6 +78,7 @@ public ProjectController( WorkspaceService workspaceService, RagIndexStatusService ragIndexStatusService, RagIndexingTriggerService ragIndexingTriggerService, + RagBranchIndexStatusService ragBranchIndexStatusService, VectorStorageService vectorStorageService, TwoFactorAuthService twoFactorAuthService) { this.projectService = projectService; @@ -82,6 +86,7 @@ public ProjectController( this.workspaceService = workspaceService; this.ragIndexStatusService = ragIndexStatusService; this.ragIndexingTriggerService = ragIndexingTriggerService; + this.ragBranchIndexStatusService = ragBranchIndexStatusService; this.vectorStorageService = vectorStorageService; this.twoFactorAuthService = twoFactorAuthService; } @@ -416,6 +421,20 @@ public ResponseEntity getRagIndexStatus( ragIndexStatusService.canStartIndexing(project)), HttpStatus.OK); } + /** + * Returns a stable, tenant-scoped operational view of the configured + * primary and retained RAG branches. It never exposes PR-only transient + * snapshots or physical vector collection names. + */ + @GetMapping("/{projectNamespace}/rag/branches") + public ResponseEntity> getRagBranchIndexes( + @PathVariable String workspaceSlug, + @PathVariable String projectNamespace) { + Workspace workspace = workspaceService.getWorkspaceBySlug(workspaceSlug); + Project project = projectService.getProjectByWorkspaceAndNamespace(workspace.getId(), projectNamespace); + return ResponseEntity.ok(ragBranchIndexStatusService.getConfiguredBranches(project)); + } + /** * PUT /api/workspace/{workspaceSlug}/project/{projectNamespace}/rag/config * Updates the RAG configuration for the project (enable/disable, set branch, @@ -437,7 +456,9 @@ public ResponseEntity updateRagConfig( request.getIncludePatterns(), request.getExcludePatterns(), request.getMultiBranchEnabled(), - request.getBranchRetentionDays()); + request.getBranchRetentionDays(), + request.getIndexedBranches(), + request.getTransientBranchIndexesEnabled()); return new ResponseEntity<>(ProjectDTO.fromProject(updated), HttpStatus.OK); } @@ -457,6 +478,7 @@ public SseEmitter triggerRagIndexing( @PathVariable String workspaceSlug, @PathVariable String projectNamespace, @RequestParam(required = false) String branch, + @RequestParam(required = false, defaultValue = "false") boolean allConfiguredBranches, @AuthenticationPrincipal UserDetailsImpl userDetails) { Workspace workspace = workspaceService.getWorkspaceBySlug(workspaceSlug); Project project = projectService.getProjectByWorkspaceAndNamespace(workspace.getId(), projectNamespace); @@ -494,6 +516,7 @@ public SseEmitter triggerRagIndexing( project.getId(), userDetails.getId(), branch, + allConfiguredBranches, emitter); return emitter; diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/request/UpdateRagConfigRequest.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/request/UpdateRagConfigRequest.java index 84c56443..96410dd3 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/request/UpdateRagConfigRequest.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/request/UpdateRagConfigRequest.java @@ -18,6 +18,10 @@ public class UpdateRagConfigRequest { private Integer branchRetentionDays; + private List indexedBranches; + + private Boolean transientBranchIndexesEnabled; + public UpdateRagConfigRequest() { } @@ -34,13 +38,23 @@ public UpdateRagConfigRequest(Boolean enabled, String branch, List exclu public UpdateRagConfigRequest(Boolean enabled, String branch, List includePatterns, List excludePatterns, - Boolean multiBranchEnabled, Integer branchRetentionDays) { + Boolean multiBranchEnabled, Integer branchRetentionDays, + List indexedBranches, Boolean transientBranchIndexesEnabled) { this.enabled = enabled; this.branch = branch; this.includePatterns = includePatterns; this.excludePatterns = excludePatterns; this.multiBranchEnabled = multiBranchEnabled; this.branchRetentionDays = branchRetentionDays; + this.indexedBranches = indexedBranches; + this.transientBranchIndexesEnabled = transientBranchIndexesEnabled; + } + + public UpdateRagConfigRequest(Boolean enabled, String branch, List includePatterns, + List excludePatterns, + Boolean multiBranchEnabled, Integer branchRetentionDays) { + this(enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, + branchRetentionDays, null, null); } public Boolean getEnabled() { @@ -90,4 +104,20 @@ public Integer getBranchRetentionDays() { public void setBranchRetentionDays(Integer branchRetentionDays) { this.branchRetentionDays = branchRetentionDays; } + + public List getIndexedBranches() { + return indexedBranches; + } + + public void setIndexedBranches(List indexedBranches) { + this.indexedBranches = indexedBranches; + } + + public Boolean getTransientBranchIndexesEnabled() { + return transientBranchIndexesEnabled; + } + + public void setTransientBranchIndexesEnabled(Boolean transientBranchIndexesEnabled) { + this.transientBranchIndexesEnabled = transientBranchIndexesEnabled; + } } diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/response/RagBranchIndexStatusDTO.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/response/RagBranchIndexStatusDTO.java new file mode 100644 index 00000000..16e42ecd --- /dev/null +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/response/RagBranchIndexStatusDTO.java @@ -0,0 +1,19 @@ +package org.rostilos.codecrow.webserver.project.dto.response; + +import java.time.OffsetDateTime; + +/** + * Observable state of one configured RAG branch. Physical collection names are + * deliberately not exposed: they are internal, tenant-scoped storage details. + */ +public record RagBranchIndexStatusDTO( + String branchName, + String role, + String status, + String activeRevision, + String requestedRevision, + Integer fileCount, + Integer chunkCount, + OffsetDateTime lastUpdatedAt, + String errorMessage) { +} diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/IProjectService.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/IProjectService.java index b31f14f9..ff1ad28a 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/IProjectService.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/IProjectService.java @@ -76,6 +76,11 @@ void updateRepositorySettings(Long workspaceId, Long projectId, UpdateRepository Project updateBranchAnalysisConfig(Long workspaceId, Long projectId, List prTargetBranches, List branchPushPatterns); + Project updateRagConfig(Long workspaceId, Long projectId, boolean enabled, String branch, + List includePatterns, + List excludePatterns, Boolean multiBranchEnabled, Integer branchRetentionDays, + List indexedBranches, Boolean transientBranchIndexesEnabled); + Project updateRagConfig(Long workspaceId, Long projectId, boolean enabled, String branch, List includePatterns, List excludePatterns, Boolean multiBranchEnabled, Integer branchRetentionDays); diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java index eca688de..e6fa2c4d 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java @@ -740,10 +740,26 @@ public Project updateRagConfig( java.util.List includePatterns, java.util.List excludePatterns, Boolean multiBranchEnabled, - Integer branchRetentionDays) { + Integer branchRetentionDays, + java.util.List indexedBranches, + Boolean transientBranchIndexesEnabled) { Project project = projectRepository.findByWorkspaceIdAndId(workspaceId, projectId) .orElseThrow(() -> new NoSuchElementException("Project not found")); + return updateRagConfig(project, enabled, branch, includePatterns, excludePatterns, + multiBranchEnabled, branchRetentionDays, indexedBranches, + transientBranchIndexesEnabled); + } + private Project updateRagConfig( + Project project, + boolean enabled, + String branch, + java.util.List includePatterns, + java.util.List excludePatterns, + Boolean multiBranchEnabled, + Integer branchRetentionDays, + java.util.List indexedBranches, + Boolean transientBranchIndexesEnabled) { ProjectConfig currentConfig = project.getConfiguration(); boolean useLocalMcp = currentConfig != null && currentConfig.useLocalMcp(); boolean useMcpTools = currentConfig != null && currentConfig.useMcpTools(); @@ -758,7 +774,8 @@ public Project updateRagConfig( var commentCommands = currentConfig != null ? currentConfig.commentCommands() : null; RagConfig ragConfig = new RagConfig( - enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, branchRetentionDays); + enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, branchRetentionDays, + indexedBranches, transientBranchIndexesEnabled); ProjectConfig newConfig = new ProjectConfig(useLocalMcp, useMcpTools, mainBranch, branchAnalysis, ragConfig, prAnalysisEnabled, branchAnalysisEnabled, installationMethod, commentCommands, @@ -768,6 +785,25 @@ public Project updateRagConfig( return projectRepository.save(project); } + @Transactional + public Project updateRagConfig( + Long workspaceId, + Long projectId, + boolean enabled, + String branch, + java.util.List includePatterns, + java.util.List excludePatterns, + Boolean multiBranchEnabled, + Integer branchRetentionDays) { + Project project = projectRepository.findByWorkspaceIdAndId(workspaceId, projectId) + .orElseThrow(() -> new NoSuchElementException("Project not found")); + RagConfig currentRag = currentRagConfig(project); + return updateRagConfig(project, enabled, branch, includePatterns, excludePatterns, + multiBranchEnabled, branchRetentionDays, + currentRag != null ? currentRag.indexedBranches() : null, + currentRag != null ? currentRag.transientBranchIndexesEnabled() : null); + } + /** * Simplified RAG config update (backward compatible). */ @@ -779,7 +815,20 @@ public Project updateRagConfig( String branch, java.util.List includePatterns, java.util.List excludePatterns) { - return updateRagConfig(workspaceId, projectId, enabled, branch, includePatterns, excludePatterns, null, null); + Project project = projectRepository.findByWorkspaceIdAndId(workspaceId, projectId) + .orElseThrow(() -> new NoSuchElementException("Project not found")); + RagConfig currentRag = currentRagConfig(project); + return updateRagConfig(project, enabled, branch, includePatterns, excludePatterns, + currentRag != null ? currentRag.multiBranchEnabled() : null, + currentRag != null ? currentRag.branchRetentionDays() : null, + currentRag != null ? currentRag.indexedBranches() : null, + currentRag != null ? currentRag.transientBranchIndexesEnabled() : null); + } + + private RagConfig currentRagConfig(Project project) { + return project.getConfiguration() != null + ? project.getConfiguration().ragConfig() + : null; } @Transactional @@ -976,7 +1025,8 @@ public Project syncAnalysisScope(Long workspaceId, Long projectId, String direct RagConfig rag = config.ragConfig() != null ? config.ragConfig() : new RagConfig(); config.setRagConfig(new RagConfig( rag.enabled(), rag.branch(), scope.includePatterns(), scope.excludePatterns(), - rag.multiBranchEnabled(), rag.branchRetentionDays())); + rag.multiBranchEnabled(), rag.branchRetentionDays(), rag.indexedBranches(), + rag.transientBranchIndexesEnabled())); } else { throw new IllegalArgumentException("direction must be FROM_RAG or TO_RAG"); } diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusService.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusService.java new file mode 100644 index 00000000..dd61cc3d --- /dev/null +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusService.java @@ -0,0 +1,127 @@ +package org.rostilos.codecrow.webserver.project.service; + +import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; +import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.webserver.project.dto.response.RagBranchIndexStatusDTO; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Read-only projection of the primary and explicitly retained RAG branches. + * It intentionally excludes transient PR-only snapshots from the configuration + * view, while retaining a compatible primary status for pre-generation projects. + */ +@Service +public class RagBranchIndexStatusService { + private final RagBranchIndexRepository branchIndexRepository; + private final RagIndexStatusService projectStatusService; + + public RagBranchIndexStatusService( + RagBranchIndexRepository branchIndexRepository, + RagIndexStatusService projectStatusService) { + this.branchIndexRepository = branchIndexRepository; + this.projectStatusService = projectStatusService; + } + + @Transactional(readOnly = true) + public List getConfiguredBranches(Project project) { + if (project.getConfiguration() == null || project.getConfiguration().ragConfig() == null) { + return List.of(); + } + var config = project.getConfiguration().ragConfig(); + String primary = resolvePrimary(project, config.branch()); + if (primary == null) { + return List.of(); + } + + Map persisted = new LinkedHashMap<>(); + for (RagBranchIndex index : branchIndexRepository.findByProjectId(project.getId())) { + persisted.put(index.getBranchName(), index); + } + RagIndexStatus projectStatus = projectStatusService.getIndexStatus(project).orElse(null); + + List result = new ArrayList<>(); + result.add(toDto(primary, "PRIMARY", persisted.get(primary), projectStatus)); + for (String branch : config.getEffectiveIndexedBranches()) { + if (!primary.equals(branch)) { + result.add(toDto(branch, "RETAINED", persisted.get(branch), null)); + } + } + return result; + } + + private RagBranchIndexStatusDTO toDto( + String branch, + String role, + RagBranchIndex index, + RagIndexStatus legacyPrimaryStatus) { + if (index == null) { + return legacyPrimaryStatus == null + ? new RagBranchIndexStatusDTO(branch, role, "NOT_INDEXED", null, null, + null, null, null, null) + : legacyPrimaryDto(branch, role, legacyPrimaryStatus); + } + + var generation = index.getActiveGeneration(); + String status = switch (index.getLifecycleStatus()) { + case PENDING -> "PENDING"; + case BUILDING -> "BUILDING"; + case FAILED -> "FAILED"; + case READY -> generation == null ? "NOT_INDEXED" : "READY"; + }; + return new RagBranchIndexStatusDTO( + branch, + role, + status, + generation != null ? generation.getRevision() : index.getCommitHash(), + index.getDesiredCommitHash(), + generation != null ? generation.getFileCount() : null, + generation != null ? generation.getChunkCount() : index.getChunkCount(), + generation != null && generation.getActivatedAt() != null + ? generation.getActivatedAt() : index.getUpdatedAt(), + index.getErrorMessage()); + } + + private RagBranchIndexStatusDTO legacyPrimaryDto( + String branch, + String role, + RagIndexStatus status) { + String displayStatus = switch (status.getStatus()) { + case INDEXING -> "BUILDING"; + case UPDATING -> "BUILDING"; + case INDEXED -> "READY"; + case FAILED -> "FAILED"; + default -> "NOT_INDEXED"; + }; + OffsetDateTime updated = status.getLastIndexedAt() != null + ? status.getLastIndexedAt() : status.getUpdatedAt(); + return new RagBranchIndexStatusDTO( + branch, role, displayStatus, status.getIndexedCommitHash(), null, + status.getTotalFilesIndexed(), status.getChunkCount(), updated, + status.getErrorMessage()); + } + + private String resolvePrimary(Project project, String configuredBranch) { + if (configuredBranch != null && !configuredBranch.isBlank()) { + return configuredBranch.trim(); + } + if (project.getConfiguration().defaultBranch() != null + && !project.getConfiguration().defaultBranch().isBlank()) { + return project.getConfiguration().defaultBranch().trim(); + } + if (project.getDefaultBranch() != null && project.getDefaultBranch().getBranchName() != null) { + return project.getDefaultBranch().getBranchName().trim(); + } + return null; + } +} diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagIndexingTriggerService.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagIndexingTriggerService.java index 07ff7bd2..0608e18d 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagIndexingTriggerService.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagIndexingTriggerService.java @@ -117,6 +117,7 @@ public void triggerIndexing( Long projectId, Long userId, String branch, + boolean allConfiguredBranches, SseEmitter emitter ) { try { @@ -148,7 +149,7 @@ public void triggerIndexing( String projectJwt = generateShortLivedProjectJwt(projectId, userId); // Start indexing via pipeline-agent - proxyToPipelineAgent(projectJwt, branch, emitter); + proxyToPipelineAgent(projectJwt, branch, allConfiguredBranches, emitter); } catch (NoSuchElementException e) { sendError(emitter, e.getMessage()); @@ -169,16 +170,24 @@ private String generateShortLivedProjectJwt(Long projectId, Long userId) { ); } - private void proxyToPipelineAgent(String projectJwt, String branch, SseEmitter emitter) { + private void proxyToPipelineAgent( + String projectJwt, + String branch, + boolean allConfiguredBranches, + SseEmitter emitter) { String pipelineUrl = getPipelineAgentBaseUrl(); String indexUrl = pipelineUrl + "/api/rag/index"; Response response = null; try { // Build request body - Map requestBody = branch != null && !branch.isBlank() - ? Map.of("branch", branch) - : Map.of(); + Map requestBody = new java.util.LinkedHashMap<>(); + if (branch != null && !branch.isBlank()) { + requestBody.put("branch", branch.trim()); + } + if (allConfiguredBranches) { + requestBody.put("allConfiguredBranches", true); + } RequestBody body = RequestBody.create( objectMapper.writeValueAsString(requestBody), diff --git a/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/ProjectServiceRagConfigTest.java b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/ProjectServiceRagConfigTest.java new file mode 100644 index 00000000..e0458d8f --- /dev/null +++ b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/ProjectServiceRagConfigTest.java @@ -0,0 +1,79 @@ +package org.rostilos.codecrow.webserver.project.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.project.config.ProjectConfig; +import org.rostilos.codecrow.core.model.project.config.RagConfig; +import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; + +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ProjectServiceRagConfigTest { + + @Mock + private ProjectRepository projectRepository; + + @InjectMocks + private ProjectService projectService; + + private Project project; + + @BeforeEach + void setUp() { + project = new Project(); + project.setConfiguration(new ProjectConfig( + false, + "main", + null, + new RagConfig( + true, + "main", + List.of("src/**"), + List.of("target/**"), + true, + 30, + List.of("develop", "release"), + true))); + when(projectRepository.findByWorkspaceIdAndId(10L, 20L)) + .thenReturn(Optional.of(project)); + when(projectRepository.save(project)).thenReturn(project); + } + + @Test + void eightArgumentCompatibilityUpdatePreservesNewerBranchSettings() { + Project updated = projectService.updateRagConfig( + 10L, 20L, true, "main", + List.of("app/**"), List.of("build/**"), false, 14); + + RagConfig rag = updated.getConfiguration().ragConfig(); + assertThat(rag.multiBranchEnabled()).isFalse(); + assertThat(rag.branchRetentionDays()).isEqualTo(14); + assertThat(rag.indexedBranches()).containsExactly("develop", "release"); + assertThat(rag.transientBranchIndexesEnabled()).isTrue(); + } + + @Test + void sixArgumentCompatibilityUpdatePreservesAllMultiBranchSettings() { + Project updated = projectService.updateRagConfig( + 10L, 20L, false, "develop", + List.of("service/**"), List.of("generated/**")); + + RagConfig rag = updated.getConfiguration().ragConfig(); + assertThat(rag.enabled()).isFalse(); + assertThat(rag.branch()).isEqualTo("develop"); + assertThat(rag.multiBranchEnabled()).isTrue(); + assertThat(rag.branchRetentionDays()).isEqualTo(30); + assertThat(rag.indexedBranches()).containsExactly("develop", "release"); + assertThat(rag.transientBranchIndexesEnabled()).isTrue(); + } +} diff --git a/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusServiceTest.java b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusServiceTest.java new file mode 100644 index 00000000..0a07fc4d --- /dev/null +++ b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusServiceTest.java @@ -0,0 +1,56 @@ +package org.rostilos.codecrow.webserver.project.service; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.project.config.ProjectConfig; +import org.rostilos.codecrow.core.model.project.config.RagConfig; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexLifecycleStatus; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class RagBranchIndexStatusServiceTest { + + @Test + void reportsPrimaryAndEachExplicitRetainedBranchWithoutTransientIndexes() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagIndexStatusService projectStatus = mock(RagIndexStatusService.class); + RagBranchIndexStatusService service = new RagBranchIndexStatusService(branches, projectStatus); + + Project project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + ProjectConfig config = new ProjectConfig(); + config.setRagConfig(new RagConfig( + true, "master", null, null, true, 30, List.of("develop"), true)); + project.setConfiguration(config); + + RagBranchIndex develop = new RagBranchIndex(project, "develop", RagBranchIndexKind.DURABLE); + develop.setLifecycleStatus(RagBranchIndexLifecycleStatus.FAILED); + develop.setErrorMessage("archive unavailable"); + develop.setUpdatedAt(OffsetDateTime.parse("2026-08-07T00:00:00Z")); + + RagBranchIndex temporary = new RagBranchIndex(project, "release/candidate", RagBranchIndexKind.TRANSIENT); + temporary.setLifecycleStatus(RagBranchIndexLifecycleStatus.READY); + + when(branches.findByProjectId(42L)).thenReturn(List.of(develop, temporary)); + when(projectStatus.getIndexStatus(project)).thenReturn(Optional.empty()); + + var statuses = service.getConfiguredBranches(project); + + assertThat(statuses).extracting(value -> value.branchName()) + .containsExactly("master", "develop"); + assertThat(statuses.get(0).status()).isEqualTo("NOT_INDEXED"); + assertThat(statuses.get(1)) + .extracting(value -> value.role(), value -> value.status(), value -> value.errorMessage()) + .containsExactly("RETAINED", "FAILED", "archive unavailable"); + } +} diff --git a/python-ecosystem/inference-orchestrator/integration/test_qa_documentation.py b/python-ecosystem/inference-orchestrator/integration/test_qa_documentation.py index 24a3bc5a..af1353aa 100644 --- a/python-ecosystem/inference-orchestrator/integration/test_qa_documentation.py +++ b/python-ecosystem/inference-orchestrator/integration/test_qa_documentation.py @@ -32,6 +32,30 @@ async def test_qa_doc_success(client, auth_headers): assert data.get("documentation") is not None or data.get("error") is not None +@pytest.mark.asyncio(loop_scope="function") +async def test_qa_doc_accepts_empty_jira_description(client, auth_headers): + """A Jira task without a description must not block diff-based QA generation.""" + mock_svc = AsyncMock() + mock_svc.generate = AsyncMock(return_value={ + "documentation": "## QA Documentation\nTest changes", + "documentation_needed": True, + }) + payload = _minimal_qa_payload() + payload["task_context"] = { + "task_key": "PROJ-123", + "task_summary": "Implement feature", + "description": None, + } + + with patch("api.routers.qa_documentation.QaDocumentationService", return_value=mock_svc): + resp = await client.post("/qa-documentation", json=payload, headers=auth_headers) + + assert resp.status_code == 200 + generation_args = mock_svc.generate.await_args.kwargs + assert generation_args["task_context"]["description"] == "" + assert generation_args["diff"] == payload["diff"] + + @pytest.mark.asyncio(loop_scope="function") async def test_qa_doc_validation_error(client, auth_headers): """Missing required fields → 422.""" diff --git a/python-ecosystem/inference-orchestrator/src/README.MD b/python-ecosystem/inference-orchestrator/src/README.MD index 455fe76e..0547d2ce 100644 --- a/python-ecosystem/inference-orchestrator/src/README.MD +++ b/python-ecosystem/inference-orchestrator/src/README.MD @@ -508,9 +508,17 @@ finding. This applies to `FILE` scope as well as line/block/function scope, and missing, stale, or out-of-hunk anchors are removed before verifier calls. For unused-import-like claims, the gate also rejects the claim when the named symbol is visibly referenced elsewhere. The same publication gate runs again after Stage 2, and rejected -Stage 2 candidates are removed from Stage 3 report context. LLM tool verification -remains a secondary check, and its file-content cache is request-local for -concurrent reviews. +Stage 2 candidates are removed from Stage 3 report context. Optional Stage 3 MCP +verification remains a secondary check. Every active finding receives a stable +execution-local Verification ID, and every source read is host-pinned to the +immutable reviewed commit. For a concrete line, the host asks the VCS MCP server +for an anchor-centred source window instead of sending an unrelated full large +file or accepting a size-filter placeholder. A dismissal is accepted only when +the response metadata binds the same Verification ID, path, revision, and a +window covering the primary and every consolidated location. Empty content, +tool errors, filter placeholders, missing locations, wrong revisions, +comments-only evidence, or ambiguity keeps the finding. Fresh findings without +database IDs use the same object-identity path. RAG latency safeguards keep slow semantic search from blocking analysis while preserving deterministic context: @@ -530,17 +538,28 @@ Per-batch LLM reranking is disabled by default (`LLM_RERANK_ENABLED=false`) to avoid one extra model call per batch. The same retrieved chunks are preserved for the review model; set `LLM_RERANK_ENABLED=true` to opt into listwise reranking. -Final issue deduplication is also conservative and local by default -(`REVIEW_LLM_DEDUP_ENABLED=false`). It removes a candidate only when normalized -file, category, exact line or exact source snippet, and closely matching -root-cause text agree. Similar wording in different files or at different source -anchors is retained. Set `REVIEW_LLM_DEDUP_ENABLED=true` only when the deployment -explicitly accepts extra review-model calls and model-directed suppression. -The Java persistence boundary does not broaden that identity: it merges only -exact category-aware or category-agnostic anchored fingerprints. Sharing a file, -line, category, or FILE scope cannot by itself suppress another finding, and an -issue whose fingerprint lacks an actual line hash or exact snippet anchor passes -through unchanged. These checks add no model calls. +Final issue deduplication begins with exact local root identities and uses grouped +semantic dedup by default (`REVIEW_LLM_DEDUP_ENABLED=true`). The host sends only +ambiguous candidate components; unrelated singletons consume no dedup tokens. +The model returns explicit duplicate groups, and only `HIGH`-confidence decisions +inside one host-selected component can merge findings. Omitted, uncertain, +malformed, or failed decisions retain all ambiguous findings. Category or +severity drift does not split a proven root cause. Cross-file consolidation is +allowed only for one shared causal defect and retains every secondary `file:line` +in the finding's reason and `relatedLocations`. + +Candidate components are packed near `REVIEW_DEDUP_BATCH_CHAR_BUDGET=48000` +without field-level clipping; an oversized component is sent intact by itself. +`REVIEW_DEDUP_MAX_PARALLEL=4` limits candidate batches within one review and does +not serialize concurrently admitted reviews. Set `REVIEW_LLM_DEDUP_ENABLED=false` +to use exact local merging only. + +The Java persistence boundary adds an exact same-file substantive-title/reason +safety net before its exact category-aware and category-agnostic anchored +fingerprints. It retains the strongest source anchor, highest severity, best +valid fix, and additional concrete locations. Short boilerplate, sharing only a +file/line/category/FILE scope, and placeholder fingerprints without an actual +anchor cannot suppress another finding. These Java checks add no model calls. Full-pipeline prompt dry runs forward the normal Stage 0–3 progress events and persist a compact `review_evidence_completed` record with terminal hunk counts, diff --git a/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py b/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py index ea6c2c40..cbfdd63b 100644 --- a/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py +++ b/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py @@ -9,7 +9,7 @@ """ import logging from fastapi import APIRouter, Request -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from typing import Optional, Dict, Any, List, Literal from model.enrichment import PrEnrichmentDataDto @@ -71,6 +71,17 @@ class QaDocumentationRequest(BaseModel): oauth_secret: Optional[str] = None bearer_token: Optional[str] = None + @field_validator("task_context", mode="before") + @classmethod + def normalize_task_context_values(cls, value): + """Keep nullable provider fields compatible with the string prompt contract.""" + if value is None or not isinstance(value, dict): + return value + return { + key: "" if item is None else str(item) + for key, item in value.items() + } + class QaDocumentationResponse(BaseModel): """Response containing the generated QA documentation.""" diff --git a/python-ecosystem/inference-orchestrator/src/model/dtos.py b/python-ecosystem/inference-orchestrator/src/model/dtos.py index 78321e34..08a957a9 100644 --- a/python-ecosystem/inference-orchestrator/src/model/dtos.py +++ b/python-ecosystem/inference-orchestrator/src/model/dtos.py @@ -104,6 +104,46 @@ class ReviewRequestDto(BaseModel): previousCommitHash: Optional[str] = Field(default=None, description="Previously analyzed commit hash") currentCommitHash: Optional[str] = Field(default=None, description="Current commit hash being analyzed") baseCommitHash: Optional[str] = Field(default=None, description="Immutable pull-request base commit hash") + ragCollectionTarget: Optional[str] = Field( + default=None, + exclude=True, + description="Internal opaque collection target for the selected branch generation", + ) + ragBaseGenerationManifestSha256: Optional[str] = Field( + default=None, + exclude=True, + description="Internal sealed target-generation receipt returned by RAG indexing", + ) + ragPrGenerationFingerprint: Optional[str] = Field( + default=None, + exclude=True, + description="Internal exact PR-overlay generation receipt returned by RAG indexing", + ) + ragPrOverlayGenerationManifestSha256: Optional[str] = Field( + default=None, + exclude=True, + description="Internal content-addressed PR-overlay membership seal returned by RAG indexing", + ) + ragBasePluginFingerprint: Optional[str] = Field( + default=None, + exclude=True, + description="Internal plugin selection identity of the sealed target generation", + ) + ragBasePluginDescriptorFingerprint: Optional[str] = Field( + default=None, + exclude=True, + description="Internal plugin descriptor identity of the sealed target generation", + ) + ragBasePluginImplementationFingerprint: Optional[str] = Field( + default=None, + exclude=True, + description="Internal plugin implementation identity of the sealed target generation", + ) + ragBaseIndexRepresentationFingerprint: Optional[str] = Field( + default=None, + exclude=True, + description="Internal index representation identity of the sealed target generation", + ) # File enrichment data (full file contents + pre-computed dependency graph) enrichmentData: Optional[PrEnrichmentDataDto] = Field(default=None, description="Pre-computed file contents and dependency relationships from Java") projectCapabilities: Optional[ProjectCapabilitiesDto] = Field( diff --git a/python-ecosystem/inference-orchestrator/src/model/output_schemas.py b/python-ecosystem/inference-orchestrator/src/model/output_schemas.py index 441a0089..d94a4a49 100644 --- a/python-ecosystem/inference-orchestrator/src/model/output_schemas.py +++ b/python-ecosystem/inference-orchestrator/src/model/output_schemas.py @@ -88,6 +88,15 @@ def normalize_scope(cls, v) -> str: "Leave empty for a generic defect proved directly by changed source." ), ) + relatedLocations: List[str] = Field( + default_factory=list, + description=( + "Additional repository-relative file:line locations consolidated " + "under the same root-cause finding. The primary file and line remain " + "in file/line; this list prevents semantic deduplication from losing " + "other affected locations." + ), + ) @field_validator('codeSnippet', mode='before') @classmethod @@ -127,6 +136,41 @@ class DeduplicatedIssueList(BaseModel): ) +class SemanticDuplicateGroup(BaseModel): + """One explicit, high-confidence semantic duplicate decision.""" + + keeper_index: int = Field( + description="Batch-local index of the best representative to keep." + ) + duplicate_indices: List[int] = Field( + default_factory=list, + description=( + "Batch-local indices describing the same root cause as keeper_index." + ), + ) + confidence: str = Field( + description=( + "HIGH, MEDIUM, or LOW confidence. Only HIGH decisions are eligible " + "for host-side merging." + ), + ) + rationale: str = Field( + default="", + description="Short root-cause identity explanation, not chain-of-thought.", + ) + + +class SemanticDeduplicationDecision(BaseModel): + """Recall-safe semantic dedup output. + + Findings omitted from duplicate_groups are retained. This is intentionally + safer than a kept-index allowlist, where a malformed or incomplete response + can accidentally delete unrelated findings. + """ + + duplicate_groups: List[SemanticDuplicateGroup] = Field(default_factory=list) + + class AskOutput(BaseModel): """Schema for ask command output.""" answer: str = Field(description="Well-formatted markdown answer to the user's question") diff --git a/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py b/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py index 92cd73f8..63a4bb07 100644 --- a/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py +++ b/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py @@ -44,7 +44,7 @@ def __init__(self, review_service: ReviewService): int(self.consumer_heartbeat_seconds * 3), ) # Bound concurrent job processing to prevent memory pressure - max_concurrent = int(os.environ.get("MAX_CONCURRENT_REVIEWS", "4")) + max_concurrent = int(os.environ.get("MAX_CONCURRENT_REVIEWS", "20")) self._job_semaphore = asyncio.Semaphore(max_concurrent) self.heartbeat_seconds = max( 1.0, diff --git a/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py b/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py index 16937929..49895c8c 100644 --- a/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py +++ b/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py @@ -15,6 +15,10 @@ RAG_DEFAULT_TOP_K = int(os.environ.get("RAG_DEFAULT_TOP_K", "15")) +class RagRetrievalError(RuntimeError): + """A required revision-bound RAG retrieval could not be completed.""" + + def _env_int(name: str, default: int) -> int: value = os.environ.get(name) if value is None or not value.strip(): @@ -119,7 +123,13 @@ async def get_pr_context( base_branch: Optional[str] = None, deleted_files: Optional[List[str]] = None, pr_number: Optional[int] = None, - all_pr_changed_files: Optional[List[str]] = None + all_pr_changed_files: Optional[List[str]] = None, + source_revision: Optional[str] = None, + base_revision: Optional[str] = None, + base_generation_manifest_sha256: Optional[str] = None, + pr_generation_fingerprint: Optional[str] = None, + pr_overlay_generation_manifest_sha256: Optional[str] = None, + collection_target: Optional[str] = None, ) -> Dict[str, Any]: """ Get relevant context for PR review with multi-branch support. @@ -185,6 +195,24 @@ async def get_pr_context( payload["pr_number"] = pr_number if all_pr_changed_files: payload["all_pr_changed_files"] = all_pr_changed_files + if source_revision: + payload["source_revision"] = source_revision + if base_revision: + payload["base_revision"] = base_revision + if base_generation_manifest_sha256: + payload["base_generation_manifest_sha256"] = ( + base_generation_manifest_sha256 + ) + if pr_generation_fingerprint: + payload["pr_generation_fingerprint"] = ( + pr_generation_fingerprint + ) + if pr_overlay_generation_manifest_sha256: + payload["pr_overlay_generation_manifest_sha256"] = ( + pr_overlay_generation_manifest_sha256 + ) + if collection_target: + payload["collection_target"] = collection_target client = await self._get_client() response = await client.post( @@ -229,7 +257,10 @@ async def semantic_search( project: str, branch: str, top_k: int = 5, - filter_language: Optional[str] = None + filter_language: Optional[str] = None, + repository_revision: Optional[str] = None, + repository_generation_manifest_sha256: Optional[str] = None, + collection_target: Optional[str] = None, ) -> Dict[str, Any]: """ Perform semantic search in the repository. @@ -258,6 +289,14 @@ async def semantic_search( } if filter_language: payload["filter_language"] = filter_language + if repository_revision: + payload["repository_revision"] = repository_revision + if repository_generation_manifest_sha256: + payload["repository_generation_manifest_sha256"] = ( + repository_generation_manifest_sha256 + ) + if collection_target: + payload["collection_target"] = collection_target client = await self._get_client() response = await client.post( @@ -314,7 +353,10 @@ async def search_for_duplicates( branch: str, queries: List[str], top_k: int = 8, - base_branch: Optional[str] = None + base_branch: Optional[str] = None, + repository_revision: Optional[str] = None, + repository_generation_manifest_sha256: Optional[str] = None, + collection_target: Optional[str] = None, ) -> List[Dict[str, Any]]: """ Perform duplication-oriented semantic search to find existing implementations @@ -336,6 +378,16 @@ async def search_for_duplicates( """ if not self.enabled or not queries: return [] + exact_binding_values = ( + repository_revision, + repository_generation_manifest_sha256, + ) + exact_binding_active = any(exact_binding_values) + if exact_binding_active and not all(exact_binding_values): + raise RagRetrievalError( + "revision-bound duplication search requires both repository " + "revision and generation receipt" + ) max_queries = max(1, _env_int("REVIEW_DUPLICATION_RAG_MAX_QUERIES", 8)) query_timeout = max( @@ -368,6 +420,14 @@ async def _run_query(query_text: str) -> List[Dict[str, Any]]: } if base_branch: payload["base_branch"] = base_branch + if repository_revision: + payload["repository_revision"] = repository_revision + if repository_generation_manifest_sha256: + payload["repository_generation_manifest_sha256"] = ( + repository_generation_manifest_sha256 + ) + if collection_target: + payload["collection_target"] = collection_target async with semaphore: started_at = datetime.now() @@ -383,12 +443,20 @@ async def _run_query(query_text: str) -> List[Dict[str, Any]]: result = response.json() except asyncio.TimeoutError: elapsed_ms = (datetime.now() - started_at).total_seconds() * 1000 - logger.debug( - "Duplication search query timed out after %.0fms", - elapsed_ms, + message = ( + "revision-bound duplication search query timed out " + f"after {elapsed_ms:.0f}ms" ) + if exact_binding_active: + raise RagRetrievalError(message) + logger.debug(message) return [] except Exception as e: + if exact_binding_active: + raise RagRetrievalError( + "revision-bound duplication search query failed: " + f"{type(e).__name__}: {e}" + ) from e logger.debug(f"Duplication search query failed: {e}") return [] @@ -401,7 +469,7 @@ async def _run_query(query_text: str) -> List[Dict[str, Any]]: result_groups = await asyncio.gather( *(_run_query(query_text) for query_text in selected_queries), - return_exceptions=True, + return_exceptions=not exact_binding_active, ) all_results: List[Dict[str, Any]] = [] for group in result_groups: @@ -421,7 +489,14 @@ async def _run_query(query_text: str) -> List[Dict[str, Any]]: ) return all_results + except RagRetrievalError: + raise except Exception as e: + if exact_binding_active: + raise RagRetrievalError( + "revision-bound duplication search failed: " + f"{type(e).__name__}: {e}" + ) from e logger.warning(f"Failed duplication search: {e}") return [] @@ -434,7 +509,13 @@ async def get_deterministic_context( limit_per_file: int = 10, pr_number: Optional[int] = None, pr_changed_files: Optional[List[str]] = None, - additional_identifiers: Optional[List[str]] = None + additional_identifiers: Optional[List[str]] = None, + source_revision: Optional[str] = None, + base_revision: Optional[str] = None, + base_generation_manifest_sha256: Optional[str] = None, + pr_generation_fingerprint: Optional[str] = None, + pr_overlay_generation_manifest_sha256: Optional[str] = None, + collection_target: Optional[str] = None, ) -> Dict[str, Any]: """ Get context using DETERMINISTIC metadata-based retrieval. @@ -481,6 +562,24 @@ async def get_deterministic_context( payload["pr_changed_files"] = pr_changed_files if additional_identifiers: payload["additional_identifiers"] = additional_identifiers + if source_revision: + payload["source_revision"] = source_revision + if base_revision: + payload["base_revision"] = base_revision + if base_generation_manifest_sha256: + payload["base_generation_manifest_sha256"] = ( + base_generation_manifest_sha256 + ) + if pr_generation_fingerprint: + payload["pr_generation_fingerprint"] = ( + pr_generation_fingerprint + ) + if pr_overlay_generation_manifest_sha256: + payload["pr_overlay_generation_manifest_sha256"] = ( + pr_overlay_generation_manifest_sha256 + ) + if collection_target: + payload["collection_target"] = collection_target client = await self._get_client() response = await client.post( @@ -537,6 +636,8 @@ async def index_pr_files( plugin_detection_evidence: Optional[Dict[str, List[str]]] = None, plugin_fingerprint: str = "sha256:" + "0" * 64, plugin_descriptor_fingerprint: str = "sha256:" + "0" * 64, + base_generation_manifest_sha256: Optional[str] = None, + collection_target: Optional[str] = None, ) -> Dict[str, Any]: """ Index PR files into the main collection with PR-specific metadata. @@ -590,6 +691,12 @@ async def index_pr_files( "plugin_descriptor_fingerprint": plugin_descriptor_fingerprint, "files": files } + if base_generation_manifest_sha256: + payload["base_generation_manifest_sha256"] = ( + base_generation_manifest_sha256 + ) + if collection_target: + payload["collection_target"] = collection_target client = await self._get_client() response = await client.post( @@ -631,7 +738,8 @@ async def delete_pr_files( self, workspace: str, project: str, - pr_number: int + pr_number: int, + collection_target: Optional[str] = None, ) -> bool: """ Delete all indexed points for a specific PR. @@ -652,7 +760,11 @@ async def delete_pr_files( try: client = await self._get_client() response = await client.delete( - f"{self.base_url}/index/pr-files/{workspace}/{project}/{pr_number}" + f"{self.base_url}/index/pr-files/{workspace}/{project}/{pr_number}", + params=( + {"collection_target": collection_target} + if collection_target else None + ), ) response.raise_for_status() result = response.json() diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/inference_policy.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/inference_policy.py index 29ef0b5c..5109f00a 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/inference_policy.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/inference_policy.py @@ -40,7 +40,7 @@ def _env_int(name: str, default: int) -> int: OUTPUT_CAP_MODEL_KWARG = os.environ.get("REVIEW_OUTPUT_CAP_MODEL_KWARG", "").strip() FAST_CHECK_ENABLED = _env_bool("REVIEW_FAST_CHECK_ENABLED", True) STAGE_2_ENABLED = _env_bool("REVIEW_STAGE_2_ENABLED", True) -LLM_DEDUP_ENABLED = _env_bool("REVIEW_LLM_DEDUP_ENABLED", False) +LLM_DEDUP_ENABLED = _env_bool("REVIEW_LLM_DEDUP_ENABLED", True) FAST_CHECK_MAX_FILES = _env_int("REVIEW_FAST_CHECK_MAX_FILES", 4) FAST_CHECK_MAX_CHANGED_LINES = _env_int("REVIEW_FAST_CHECK_MAX_CHANGED_LINES", 800) @@ -188,12 +188,9 @@ def should_use_llm_dedup( profile: ReviewInferenceProfile, issue_count: int, ) -> bool: - """LLM dedup is an explicit cost/quality opt-in, never a default stage.""" - return ( - LLM_DEDUP_ENABLED - and issue_count > 1 - and not should_use_fast_dedup(profile, issue_count) - ) + """Enable grouped semantic dedup; singleton candidate sets skip internally.""" + del profile + return LLM_DEDUP_ENABLED and issue_count > 1 def _count_review_files( diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/mcp_tool_executor.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/mcp_tool_executor.py index a39fd3b2..a4800927 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/mcp_tool_executor.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/mcp_tool_executor.py @@ -5,11 +5,20 @@ Stage 3 (issue verification): getBranchFileContent, getPullRequestComments — max 5 calls total """ import asyncio +import json import logging +import re from typing import Any, Dict, List, Optional, Set logger = logging.getLogger(__name__) +_REVIEW_SOURCE_CONTEXT_LINES = 80 +_FILTERED_CONTENT_MARKER = "[CodeCrow Filter:" +_MCP_ERROR_PREFIXES = ("Error executing tool:", "Tool call failed:") +_RELATED_LOCATIONS_RE = re.compile( + r"(?im)^\s*(?:[*_]{1,2})?also affects\s*:(?:[*_]{1,2})?\s*(.+)$" +) + class McpToolExecutor: """ @@ -31,7 +40,14 @@ class McpToolExecutor: }, } - def __init__(self, mcp_client, request, stage: str): + def __init__( + self, + mcp_client, + request, + stage: str, + review_revision: Optional[str] = None, + verification_issues: Optional[Dict[str, Any]] = None, + ): if stage not in self.STAGE_CONFIG: raise ValueError(f"Unknown stage '{stage}'. Valid: {list(self.STAGE_CONFIG)}") @@ -43,6 +59,8 @@ def __init__(self, mcp_client, request, stage: str): self.max_calls: int = config["max_calls"] self.call_count: int = 0 self.call_log: List[Dict[str, Any]] = [] + self.review_revision = str(review_revision or "").strip() + self.verification_issues = dict(verification_issues or {}) self._lock = asyncio.Lock() # ------------------------------------------------------------------ @@ -66,8 +84,35 @@ async def execute_tool(self, tool_name: str, arguments: Dict[str, Any]) -> str: # Pre-fill workspace/repo from request context so the LLM doesn't # have to guess these values. + arguments = dict(arguments or {}) arguments.setdefault("workspace", self.request.projectVcsWorkspace) arguments.setdefault("repoSlug", self.request.projectVcsRepoSlug) + if ( + self.stage == "stage_3" + and tool_name == "getBranchFileContent" + and self.review_revision + ): + # Post-review evidence must come from the exact reviewed revision. + # Never let a model accidentally verify new PR code against target. + arguments["branch"] = self.review_revision + verification_id = str( + arguments.get("verificationId") or "" + ).strip() + source_line = self._verification_line_for_path( + verification_id, + str(arguments.get("filePath") or ""), + ) + if source_line > 0: + # Request an anchor-centred source window. This avoids replacing + # large files with a generic size placeholder or sending an + # unrelated full file merely to verify one concrete finding. + arguments["startLine"] = max( + 1, + source_line - _REVIEW_SOURCE_CONTEXT_LINES, + ) + arguments["endLine"] = ( + source_line + _REVIEW_SOURCE_CONTEXT_LINES + ) logger.info( f"[MCP {self.stage}] Calling {tool_name} " @@ -76,15 +121,48 @@ async def execute_tool(self, tool_name: str, arguments: Dict[str, Any]) -> str: try: result = await self.client.session.call_tool(tool_name, arguments) - self.call_log.append( - {"tool": tool_name, "args": arguments, "success": True} - ) # Extract text content from MCP result - if hasattr(result, "content") and result.content: - return "\n".join( - block.text for block in result.content if hasattr(block, "text") + if hasattr(result, "content"): + text = "\n".join( + block.text + for block in (result.content or []) + if hasattr(block, "text") ) - return str(result) + else: + text = str(result) + evidence = self._file_evidence_metadata(tool_name, text) + tool_reported_error = bool( + getattr(result, "isError", False) + or getattr(result, "is_error", False) + ) + if tool_reported_error: + evidence = self._file_evidence_metadata(tool_name, "") + if ( + self.stage == "stage_3" + and tool_name == "getBranchFileContent" + and evidence.get("evidence_valid") is True + and evidence.get("evidence_structured") is not True + and arguments.get("startLine") + ): + # Legacy adapters may return raw source rather than the VCS MCP + # metadata envelope. Since Stage 3 explicitly requested a + # window, bind that raw response to the requested range instead + # of incorrectly treating it as proof for the entire file. + evidence["evidence_complete_file"] = False + evidence["evidence_start_line"] = arguments["startLine"] + evidence["evidence_end_line"] = arguments.get( + "endLine", + arguments["startLine"], + ) + log_entry = { + "tool": tool_name, + "args": dict(arguments), + "success": not tool_reported_error, + "result_chars": len(text.strip()), + **evidence, + } + self.call_log.append(log_entry) + return text except Exception as e: logger.error(f"[MCP {self.stage}] Tool call failed: {e}") self.call_log.append( @@ -101,20 +179,40 @@ def get_tool_definitions(self) -> List[Dict[str, Any]]: "type": "function", "function": { "name": "getBranchFileContent", - "description": "Read a file's content from the target branch.", + "description": ( + "Read repository file content. For Stage 3, provide the " + "finding Verification ID; the host pins the reviewed " + "revision and requests an anchor-centred source window." + if self.stage == "stage_3" + else "Read a file's content from the target branch." + ), "parameters": { "type": "object", "properties": { - "branch": { - "type": "string", - "description": "Branch name (e.g. 'main', 'develop')" - }, + **({ + "branch": { + "type": "string", + "description": "Target branch name (for example main).", + }, + } if self.stage != "stage_3" else {}), "filePath": { "type": "string", "description": "Path to the file in the repository" }, + **({ + "verificationId": { + "type": "string", + "description": ( + "Verification ID from the Stage 3 finding record." + ), + }, + } if self.stage == "stage_3" else {}), }, - "required": ["branch", "filePath"], + "required": ( + ["filePath", "verificationId"] + if self.stage == "stage_3" + else ["branch", "filePath"] + ), }, }, }) @@ -153,3 +251,102 @@ def summary(self) -> str: f"{self.call_count}/{self.max_calls} calls used, " f"{len(self.call_log)} logged" ) + + @staticmethod + def _normalized_path(value: Any) -> str: + return str(value or "").strip().replace("\\", "/").lstrip("/") + + @staticmethod + def _location_parts(value: Any) -> tuple[str, int]: + normalized = McpToolExecutor._normalized_path(value) + path, separator, possible_line = normalized.rpartition(":") + if separator and possible_line.isdigit(): + return path, int(possible_line) + return normalized, 0 + + def _verification_line_for_path( + self, + verification_id: str, + file_path: str, + ) -> int: + issue = self.verification_issues.get(verification_id) + if issue is None: + return 0 + requested_path = self._normalized_path(file_path) + issue_path = self._normalized_path(getattr(issue, "file", "")) + if requested_path == issue_path: + try: + return max(0, int(getattr(issue, "line", 0) or 0)) + except (TypeError, ValueError): + return 0 + for location in self._related_locations(issue): + path, line = self._location_parts(location) + if path == requested_path: + return line + return 0 + + @staticmethod + def _related_locations(issue: Any) -> List[str]: + values = list(getattr(issue, "relatedLocations", None) or []) + reason = str(getattr(issue, "reason", "") or "") + for match in _RELATED_LOCATIONS_RE.finditer(reason): + values.extend(match.group(1).split(",")) + return sorted({ + str(value).strip() + for value in values + if str(value).strip() + }) + + @staticmethod + def _file_evidence_metadata( + tool_name: str, + text: str, + ) -> Dict[str, Any]: + if tool_name != "getBranchFileContent": + return {} + + stripped = str(text or "").strip() + metadata: Dict[str, Any] = { + "evidence_valid": False, + "evidence_structured": False, + "evidence_complete_file": False, + "evidence_start_line": 0, + "evidence_end_line": 0, + } + if ( + not stripped + or _FILTERED_CONTENT_MARKER in stripped + or stripped.startswith(_MCP_ERROR_PREFIXES) + ): + return metadata + + try: + payload = json.loads(stripped) + except (json.JSONDecodeError, TypeError): + # Some MCP adapters return raw source text instead of a JSON map. + metadata["evidence_valid"] = True + metadata["evidence_complete_file"] = True + return metadata + + if not isinstance(payload, dict) or payload.get("error"): + return metadata + metadata["evidence_structured"] = True + file_content = payload.get("fileContent") + if not isinstance(file_content, str) or not file_content.strip(): + return metadata + if _FILTERED_CONTENT_MARKER in file_content: + return metadata + + def integer(name: str) -> int: + try: + return max(0, int(payload.get(name) or 0)) + except (TypeError, ValueError): + return 0 + + metadata.update({ + "evidence_valid": True, + "evidence_complete_file": payload.get("completeFile") is True, + "evidence_start_line": integer("startLine"), + "evidence_end_line": integer("endLine"), + }) + return metadata diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py index f4a35d8e..888736fb 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py @@ -182,6 +182,9 @@ def _emit_review_evidence_completed( review_units: Optional[Stage1ReviewUnitState] = None, rag_state: Optional[Stage1RagState] = None, candidate_ledger: Optional[CandidateEvidenceLedger] = None, + *, + request: ReviewRequestDto, + pr_indexed: bool, ) -> None: """Expose compact host-owned completion evidence without prompt/source data.""" if callback is None: @@ -227,6 +230,43 @@ def _emit_review_evidence_completed( rag_state.exact_evidence_by_id if rag_state is not None else {} ), }, + "revisionBinding": { + "prIndexed": pr_indexed, + "pullRequestId": request.pullRequestId, + "targetBranch": request.targetBranchName, + "sourceRevision": ( + request.currentCommitHash or request.commitHash + ), + "baseRevision": request.baseCommitHash, + "baseGenerationManifestSha256": ( + request.ragBaseGenerationManifestSha256 + if pr_indexed else None + ), + "prGenerationFingerprint": ( + request.ragPrGenerationFingerprint + if pr_indexed else None + ), + "prOverlayGenerationManifestSha256": ( + request.ragPrOverlayGenerationManifestSha256 + if pr_indexed else None + ), + "basePluginFingerprint": ( + request.ragBasePluginFingerprint + if pr_indexed else None + ), + "basePluginDescriptorFingerprint": ( + request.ragBasePluginDescriptorFingerprint + if pr_indexed else None + ), + "basePluginImplementationFingerprint": ( + request.ragBasePluginImplementationFingerprint + if pr_indexed else None + ), + "baseIndexRepresentationFingerprint": ( + request.ragBaseIndexRepresentationFingerprint + if pr_indexed else None + ), + }, }) @@ -386,6 +426,10 @@ async def _index_pr_files( base_branch=identity.target_branch, source_revision=identity.head_revision, base_revision=identity.base_revision, + collection_target=request.ragCollectionTarget, + base_generation_manifest_sha256=( + request.ragBaseGenerationManifestSha256 + ), repository_plugins=( list(capabilities.repositoryPlugins) if capabilities else [] ), @@ -409,6 +453,32 @@ async def _index_pr_files( request, result.get("effective_project_capabilities"), ) + request.ragBaseGenerationManifestSha256 = ( + result.get("base_generation_manifest_sha256") + or request.ragBaseGenerationManifestSha256 + ) + request.ragPrGenerationFingerprint = result.get( + "generation_fingerprint" + ) + request.ragPrOverlayGenerationManifestSha256 = result.get( + "overlay_generation_manifest_sha256" + ) + request.ragBasePluginFingerprint = ( + result.get("plugin_fingerprint") + or request.ragBasePluginFingerprint + ) + request.ragBasePluginDescriptorFingerprint = ( + result.get("plugin_descriptor_fingerprint") + or request.ragBasePluginDescriptorFingerprint + ) + request.ragBasePluginImplementationFingerprint = ( + result.get("plugin_implementation_fingerprint") + or request.ragBasePluginImplementationFingerprint + ) + request.ragBaseIndexRepresentationFingerprint = ( + result.get("index_representation_fingerprint") + or request.ragBaseIndexRepresentationFingerprint + ) self._pr_indexed = True self._repository_review_groups = tuple( tuple( @@ -462,7 +532,8 @@ async def _cleanup_pr_files(self, request: ReviewRequestDto) -> None: await self.rag_client.delete_pr_files( workspace=request.projectWorkspace, project=request.projectNamespace, - pr_number=self._pr_number + pr_number=self._pr_number, + collection_target=request.ragCollectionTarget, ) logger.info(f"Cleaned up PR #{self._pr_number} indexed data") except Exception as e: @@ -921,6 +992,8 @@ async def orchestrate_review( self.event_callback, hunk_coverage, candidate_ledger=candidate_ledger, + request=request, + pr_indexed=self._pr_indexed, ) logger.info( "Review completed locally: every acquired hunk has a " @@ -1224,20 +1297,16 @@ async def orchestrate_review( # === FINAL DEDUP: after ALL issue-finding stages (1 + 1.5 + 2) === # Historical resolutions are lifecycle updates, not competing - # findings. Keep them out of both dedup implementations, which are - # intentionally content-based and could otherwise discard the update. + # findings. Active historical identities do participate so duplicate + # history and fresh recreations can be consolidated. The merge keeps + # one persisted identity and emits explicit close updates for any + # superseded historical IDs. active_issues, resolved_lifecycle_issues = _partition_issue_lifecycle( file_issues ) - fresh_active_issues, protected_active_issues = ( - _partition_protected_active_issues( - active_issues, - protected_open_issue_ids, - ) - ) - pre_dedup_count = len(fresh_active_issues) - if not fresh_active_issues: - deduplicated_fresh_issues = [] + pre_dedup_count = len(active_issues) + if not active_issues: + deduplicated_active_issues = [] elif should_use_llm_dedup( inference_profile, pre_dedup_count, @@ -1246,13 +1315,13 @@ async def orchestrate_review( self.event_callback, "final_dedup_started", ( - "Final dedup: opt-in semantic LLM dedup for " + "Final dedup: grouped recall-safe semantic dedup for " f"{pre_dedup_count} issue(s)" ), ) - deduplicated_fresh_issues = await deduplicate_final_issues_llm( + deduplicated_active_issues = await deduplicate_final_issues_llm( with_stage_output_cap(self.llm, "dedup", inference_profile), - fresh_active_issues, + active_issues, ) else: fast_dedup = should_use_fast_dedup( @@ -1276,37 +1345,50 @@ async def orchestrate_review( f"{pre_dedup_count} issue(s)" ), ) - deduplicated_fresh_issues = deduplicate_final_issues( - fresh_active_issues + deduplicated_active_issues = deduplicate_final_issues( + active_issues ) - before_final_dedup = list(fresh_active_issues) + before_final_dedup = list(active_issues) candidate_ledger.reject_removed( before_final_dedup, - deduplicated_fresh_issues, + deduplicated_active_issues, gate="deduplication", code="final_duplicate", ) - before_history_suppression = list(deduplicated_fresh_issues) - deduplicated_fresh_issues = _suppress_duplicates_of_protected_history( - deduplicated_fresh_issues, - protected_active_issues, - ) - candidate_ledger.reject_removed( - before_history_suppression, - deduplicated_fresh_issues, - gate="deduplication", - code="duplicate_of_open_history", - ) - if len(deduplicated_fresh_issues) != pre_dedup_count: + + retained_object_ids = { + id(issue) for issue in deduplicated_active_issues + } + consolidated_history: List[CodeReviewIssue] = [] + for removed_issue in before_final_dedup: + if id(removed_issue) in retained_object_ids: + continue + removed_id = str(getattr(removed_issue, "id", "") or "").strip() + if removed_id not in protected_open_issue_ids: + continue + resolved_copy = _resolved_historical_copy( + removed_issue, + protected_open_issue_ids, + ( + "Closed because final root-cause deduplication " + "consolidated this duplicate into the retained finding." + ), + ) + if resolved_copy is not None: + consolidated_history.append(resolved_copy) + + if len(deduplicated_active_issues) != pre_dedup_count: logger.info( - "Final dedup before Stage 3: %d → %d fresh active issues", + "Final dedup before Stage 3: %d → %d active root findings " + "(%d historical duplicate(s) closed)", pre_dedup_count, - len(deduplicated_fresh_issues), + len(deduplicated_active_issues), + len(consolidated_history), ) file_issues = ( - deduplicated_fresh_issues - + protected_active_issues + deduplicated_active_issues + resolved_lifecycle_issues + + consolidated_history ) # Stage 3 receives the structured Stage 2 result separately from the @@ -1341,25 +1423,34 @@ async def orchestrate_review( pr_evidence_ledger.task_implementation_evidence_payload(task_key) ) dismissed_ids = set(stage_3_result.get("dismissed_issue_ids", [])) + dismissed_object_ids = { + int(value) + for value in stage_3_result.get( + "dismissed_issue_object_ids", + [], + ) + } # A dismissed historical OPEN issue is a lifecycle update, not an # omission. Return it as resolved so the client can close the stored # record; only genuinely fresh candidates are removed outright. - if dismissed_ids: + if dismissed_ids or dismissed_object_ids: before_stage_3_dismissals = list(file_issues) file_issues, resolved_count, dropped_count = ( _apply_stage_3_dismissals( file_issues, dismissed_ids, protected_open_issue_ids, + dismissed_object_ids=dismissed_object_ids, ) ) logger.info( "Stage 3 dismissed %d fresh issue(s) and resolved %d " - "historical OPEN issue(s) (IDs: %s)", + "historical OPEN issue(s) after evidence validation " + "(verification keys: %s)", dropped_count, resolved_count, - dismissed_ids, + stage_3_result.get("dismissed_issue_keys", []), ) candidate_ledger.reject_removed( before_stage_3_dismissals, @@ -1379,6 +1470,8 @@ async def orchestrate_review( stage_1_review_unit_state, stage_1_rag_state, candidate_ledger, + request=request, + pr_indexed=self._pr_indexed, ) logger.info("Review hunk coverage complete: %s", hunk_coverage.summary()) @@ -1698,6 +1791,30 @@ def _normalized_issue_resolution(issue: CodeReviewIssue) -> Optional[str]: return None +def _resolved_historical_copy( + issue: CodeReviewIssue, + previous_open_ids: set[str], + explanation: str, +) -> Optional[CodeReviewIssue]: + """Create a lifecycle close update without re-publishing rejected provenance. + + Reconciled historical objects can be bound to a generated-candidate ledger + record. Dedup rejects the superseded candidate object, while this unbound copy + is returned solely so persistence can close the old database identity. + """ + if hasattr(issue, "model_copy"): + resolved = issue.model_copy(deep=True) + else: + resolved = CodeReviewIssue(**issue.model_dump()) + if not _resolve_historical_candidate( + resolved, + previous_open_ids, + explanation, + ): + return None + return resolved + + def _partition_protected_active_issues( active_issues: List[CodeReviewIssue], protected_ids: set[str], @@ -1742,18 +1859,38 @@ def _deduplicate_cross_batch_issues_preserving_lifecycle( issues: List[CodeReviewIssue], protected_ids: Optional[set[str]] = None, ) -> List[CodeReviewIssue]: - """Deduplicate fresh Stage 1 findings without losing historical identity.""" + """Deduplicate Stage 1 findings while retaining lifecycle close updates. + + Active history participates in exact merging so its persisted identity can + absorb a fresh, better source anchor. If two persisted OPEN records collapse + into one root finding, the superseded ID is returned as an explicit resolved + update instead of disappearing. + """ active, resolved = _partition_issue_lifecycle(issues) fresh, protected = _partition_protected_active_issues( active, protected_ids or set(), ) - deduplicated_fresh = deduplicate_cross_batch_issues(fresh) - deduplicated_fresh = _suppress_duplicates_of_protected_history( - deduplicated_fresh, - protected, - ) - return deduplicated_fresh + protected + resolved + # Preserve the established publication order (fresh, protected, resolved) + # while still allowing exact merging across the fresh/history boundary. + ordered_active = fresh + protected + deduplicated_active = deduplicate_cross_batch_issues(ordered_active) + retained_object_ids = {id(issue) for issue in deduplicated_active} + consolidated_history: List[CodeReviewIssue] = [] + for issue in ordered_active: + if id(issue) in retained_object_ids: + continue + resolved_copy = _resolved_historical_copy( + issue, + protected_ids or set(), + ( + "Closed because exact root-cause deduplication consolidated " + "this duplicate into the retained finding." + ), + ) + if resolved_copy is not None: + consolidated_history.append(resolved_copy) + return deduplicated_active + resolved + consolidated_history def _serialize_issue_for_client(issue: CodeReviewIssue) -> Dict[str, Any]: @@ -1783,8 +1920,15 @@ def _apply_stage_3_dismissals( issues: List[CodeReviewIssue], dismissed_ids: set[str], previous_open_ids: set[str], + *, + dismissed_object_ids: Optional[set[int]] = None, ) -> tuple[List[CodeReviewIssue], int, int]: - """Close dismissed OPEN history and drop only fresh false positives.""" + """Close verified OPEN history and drop only verified fresh false positives. + + Object identities are preferred because Stage 3 verification IDs cover fresh + findings that do not have a database ID and avoid touching resolved lifecycle + records that happen to share a persisted ID. + """ normalized_dismissed_ids = { str(issue_id).strip() for issue_id in dismissed_ids @@ -1793,10 +1937,16 @@ def _apply_stage_3_dismissals( retained: List[CodeReviewIssue] = [] resolved_count = 0 dropped_count = 0 + use_object_identity = bool(dismissed_object_ids) for issue in issues: issue_id = str(getattr(issue, "id", "") or "").strip() - if issue_id not in normalized_dismissed_ids: + is_dismissed = ( + id(issue) in (dismissed_object_ids or set()) + if use_object_identity + else issue_id in normalized_dismissed_ids + ) + if not is_dismissed: retained.append(issue) continue diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/reconciliation.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/reconciliation.py index 129fbea4..cf6a8a38 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/reconciliation.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/reconciliation.py @@ -4,11 +4,16 @@ import logging import difflib import asyncio +import json import os +import re from collections import defaultdict -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Sequence -from model.output_schemas import CodeReviewIssue, DeduplicatedIssueList +from model.output_schemas import ( + CodeReviewIssue, + SemanticDeduplicationDecision, +) from service.review.candidate_ledger import CandidateEvidenceLedger from utils.llm_response import extract_llm_response_text from service.review.orchestrator.json_utils import parse_llm_response, supports_structured_output @@ -159,6 +164,263 @@ def _issue_payload(issue: Any) -> Dict[str, Any]: return vars(issue) if hasattr(issue, "__dict__") else {} +_TEXT_TOKEN_RE = re.compile(r"[A-Za-z0-9_$]+") +_ROOT_STOP_WORDS = { + "a", "an", "and", "are", "as", "at", "be", "because", "by", "can", + "could", "for", "from", "has", "have", "if", "in", "into", "is", "it", + "may", "of", "on", "or", "that", "the", "their", "this", "to", "was", + "were", "will", "with", "would", +} +_SEVERITY_RANK = { + "CRITICAL": 5, + "HIGH": 4, + "MEDIUM": 3, + "LOW": 2, + "INFO": 1, +} + + +def _normalized_file(data: Dict[str, Any]) -> str: + value = str(data.get("file") or data.get("filePath") or "").strip() + normalized = value.replace("\\", "/") + while normalized.startswith("./"): + normalized = normalized[2:] + return normalized.lstrip("/") + + +def _split_reason_locations(value: Any) -> tuple[str, set[str]]: + """Separate root-cause prose from host-generated affected locations.""" + body: List[str] = [] + locations: set[str] = set() + for line in str(value or "").splitlines(): + stripped = line.strip() + if stripped.casefold().startswith("also affects:"): + raw_locations = stripped.split(":", 1)[1] + locations.update( + item.strip() + for item in raw_locations.split(",") + if item.strip() + ) + else: + body.append(line) + return "\n".join(body).strip(), locations + + +def _normalized_text(value: Any) -> str: + body, _ = _split_reason_locations(value) + return " ".join(token.casefold() for token in _TEXT_TOKEN_RE.findall(body)) + + +def _meaningful_tokens(value: Any) -> set[str]: + return { + token + for token in _normalized_text(value).split() + if token not in _ROOT_STOP_WORDS and len(token) > 1 + } + + +def _text_similarity(left: Any, right: Any) -> float: + normalized_left = _normalized_text(left) + normalized_right = _normalized_text(right) + if not normalized_left or not normalized_right: + return 0.0 + if normalized_left == normalized_right: + return 1.0 + return difflib.SequenceMatcher( + None, + normalized_left, + normalized_right, + ).ratio() + + +def _token_overlap(left: Any, right: Any) -> tuple[int, float, float]: + left_tokens = _meaningful_tokens(left) + right_tokens = _meaningful_tokens(right) + if not left_tokens or not right_tokens: + return 0, 0.0, 0.0 + overlap = len(left_tokens & right_tokens) + containment = overlap / min(len(left_tokens), len(right_tokens)) + union = len(left_tokens | right_tokens) + return overlap, containment, overlap / union if union else 0.0 + + +def _line_number(data: Dict[str, Any]) -> int: + try: + return int(data.get("line") or data.get("lineNumber") or 0) + except (TypeError, ValueError): + return 0 + + +def _issue_id(issue: Any) -> str: + return str(_issue_payload(issue).get("id") or "").strip() + + +def _issue_location(issue: Any) -> str: + data = _issue_payload(issue) + file_path = _normalized_file(data) + line = _line_number(data) + return f"{file_path}:{line}" if line > 0 else file_path + + +def _history_identity_rank(issue: Any) -> tuple[int, int, str]: + issue_id = _issue_id(issue) + if not issue_id: + return 0, 0, "" + try: + # Prefer the oldest numeric identity when historical duplicates already + # exist, preserving the longest-lived review/comment lineage. + return 1, -int(issue_id), issue_id + except ValueError: + return 1, 0, issue_id + + +def _representation_rank(issue: Any) -> tuple[int, int, int, int]: + data = _issue_payload(issue) + line = _line_number(data) + snippet = str(data.get("codeSnippet") or data.get("code_snippet") or "").strip() + reason, _ = _split_reason_locations(data.get("reason") or "") + fix = str(data.get("suggestedFixDescription") or "").strip() + return ( + 1 if snippet else 0, + 1 if line > 1 else 0, + len(reason), + len(fix), + ) + + +def _set_issue_field(issue: Any, name: str, value: Any) -> None: + if hasattr(issue, name): + setattr(issue, name, value) + + +def _merge_duplicate_issues( + left: Any, + right: Any, + *, + prefer_left: bool = False, +) -> Any: + """Merge two proven root-cause duplicates without losing locations. + + A persisted identity wins over a fresh object, while a fresh exact anchor + refreshes a stale historical location. The function mutates and returns one + input object so candidate-ledger provenance remains attached. + """ + left_history = _history_identity_rank(left) + right_history = _history_identity_rank(right) + if left_history != right_history: + canonical = left if left_history > right_history else right + elif prefer_left: + canonical = left + else: + canonical = ( + left + if _representation_rank(left) >= _representation_rank(right) + else right + ) + other = right if canonical is left else left + + canonical_data = _issue_payload(canonical) + other_data = _issue_payload(other) + left_location_before_merge = _issue_location(left) + right_location_before_merge = _issue_location(right) + same_file = _normalized_file(canonical_data) == _normalized_file(other_data) + + # A current fresh anchor is stronger than a carried historical hint. For two + # fresh findings, prefer the representation with the stronger concrete anchor. + if same_file: + if bool(_issue_id(canonical)) != bool(_issue_id(other)): + anchor_source = other if not _issue_id(other) else canonical + elif prefer_left: + anchor_source = canonical + else: + anchor_source = ( + canonical + if _representation_rank(canonical) >= _representation_rank(other) + else other + ) + anchor_data = _issue_payload(anchor_source) + for field in ("file", "line", "scope", "codeSnippet"): + value = anchor_data.get(field) + if value not in (None, ""): + _set_issue_field(canonical, field, value) + + canonical_reason, canonical_locations = _split_reason_locations( + canonical_data.get("reason") + ) + other_reason, other_locations = _split_reason_locations(other_data.get("reason")) + detail_source = canonical if len(canonical_reason) >= len(other_reason) else other + base_reason = canonical_reason if detail_source is canonical else other_reason + + locations = set(canonical_locations) | set(other_locations) + for value in canonical_data.get("relatedLocations") or []: + if str(value).strip(): + locations.add(str(value).strip()) + for value in other_data.get("relatedLocations") or []: + if str(value).strip(): + locations.add(str(value).strip()) + + primary_location = _issue_location(canonical) + if not same_file: + locations.update({ + left_location_before_merge, + right_location_before_merge, + }) + elif ( + (not _issue_id(left) and not _issue_id(right)) + or ( + _issue_id(left) + and _issue_id(right) + and _issue_id(left) != _issue_id(right) + ) + ): + # Distinct occurrences of one root cause remain visible after merging. + # A history/current pair is excluded because its line difference usually + # represents a refreshed anchor for the same persisted occurrence. + locations.update({ + left_location_before_merge, + right_location_before_merge, + }) + locations.discard("") + locations.discard(primary_location) + sorted_locations = sorted(locations) + merged_reason = base_reason + if sorted_locations: + merged_reason = ( + f"{base_reason}\n\nAlso affects: {', '.join(sorted_locations)}" + ).strip() + _set_issue_field(canonical, "reason", merged_reason) + _set_issue_field(canonical, "relatedLocations", sorted_locations) + + for field in ("title", "suggestedFixDescription", "suggestedFixDiff"): + current = canonical_data.get(field) + alternative = other_data.get(field) + best = max( + (current, alternative), + key=lambda value: len(str(value)) if value else 0, + ) + if best and best != current: + _set_issue_field(canonical, field, best) + + left_severity = str(_issue_payload(left).get("severity") or "").upper() + right_severity = str(_issue_payload(right).get("severity") or "").upper() + highest = max( + (left_severity, right_severity), + key=lambda value: _SEVERITY_RANK.get(value, 0), + ) + if highest: + _set_issue_field(canonical, "severity", highest) + + evidence_refs = sorted({ + str(value).strip() + for issue in (left, right) + for value in (_issue_payload(issue).get("evidenceRefs") or []) + if str(value).strip() + }) + if evidence_refs: + _set_issue_field(canonical, "evidenceRefs", evidence_refs) + return canonical + + def _normalized_anchor(value: Any) -> str: return " ".join(str(value or "").split()) @@ -191,7 +453,7 @@ def _issues_share_exact_plugin_proof( left: Dict[str, Any], right: Dict[str, Any], ) -> bool: - """Use plugin proof identity as a prose-independent root-cause key.""" + """Use shared plugin proof to nominate a semantic duplicate candidate.""" left_kind = str( left.get("claimKind") or left.get("claim_kind") or "" ).strip() @@ -222,45 +484,178 @@ def issues_are_conservative_duplicates( left: Any, right: Any, ) -> bool: - """Require location, category, anchor, and root-cause agreement. + """Return true only for deterministic, effectively exact root identities. - Similar prose alone is not an issue identity. Repeated guard/validation - defects in different files or at different lines remain separate findings. + Category and severity drift do not make a real duplicate independent. At + this tier, however, prose similarity alone is insufficient: uncertain pairs + are delegated to the grouped semantic pass and retained if that pass fails. """ left_data = _issue_payload(left) right_data = _issue_payload(right) - left_file = str( - left_data.get("file") or left_data.get("filePath") or "" - ).replace("\\", "/") - right_file = str( - right_data.get("file") or right_data.get("filePath") or "" - ).replace("\\", "/") + left_file = _normalized_file(left_data) + right_file = _normalized_file(right_data) if not left_file or left_file != right_file: return False - # Selected plugins supply stable proof identities. An exact match ties both - # candidates to the same framework/language relationship even when model - # prose, category, or the chosen line anchor differs across batches. - if _issues_share_exact_plugin_proof(left_data, right_data): + left_title = _normalized_text(left_data.get("title") or "") + right_title = _normalized_text(right_data.get("title") or "") + left_reason = _normalized_text( + left_data.get("reason") or left_data.get("description") or "" + ) + right_reason = _normalized_text( + right_data.get("reason") or right_data.get("description") or "" + ) + exact_narrative = bool( + left_title + and left_title == right_title + and left_reason + and left_reason == right_reason + ) + if exact_narrative: return True - left_category = str(left_data.get("category") or "").upper() - right_category = str(right_data.get("category") or "").upper() - if not left_category or left_category != right_category: - return False if not _issues_share_exact_anchor(left_data, right_data): return False + if left_reason and left_reason == right_reason: + return True + if ( + left_reason + and right_reason + and _text_similarity(left_reason, right_reason) >= 0.88 + ): + return True + return bool( + left_title + and left_title == right_title + and _text_similarity(left_reason, right_reason) >= 0.94 + ) + + +def issues_are_semantic_dedup_candidates(left: Any, right: Any) -> bool: + """Select plausible duplicate pairs for semantic comparison. + + This is a candidate-recall operation, not a deletion decision. It is broad + enough to include anchor/category wording drift, while cross-file grouping + requires a shared non-generic title and substantial technical-token overlap. + """ + if issues_are_conservative_duplicates(left, right): + return True + + left_data = _issue_payload(left) + right_data = _issue_payload(right) + left_file = _normalized_file(left_data) + right_file = _normalized_file(right_data) + if not left_file or not right_file: + return False + + # A stable plugin fact is strong enough to justify semantic comparison, but + # not deterministic deletion: one structural relationship can support more + # than one genuinely distinct defect claim. + if _issues_share_exact_plugin_proof(left_data, right_data): + return True + left_title = left_data.get("title") or "" + right_title = right_data.get("title") or "" left_reason = left_data.get("reason") or left_data.get("description") or "" - right_reason = ( - right_data.get("reason") or right_data.get("description") or "" + right_reason = right_data.get("reason") or right_data.get("description") or "" + title_similarity = _text_similarity(left_title, right_title) + reason_similarity = _text_similarity(left_reason, right_reason) + title_overlap, title_containment, title_jaccard = _token_overlap( + left_title, + right_title, ) - return is_semantically_similar( - str(left_reason), - str(right_reason), - threshold=0.82, + reason_overlap, reason_containment, reason_jaccard = _token_overlap( + left_reason, + right_reason, ) + if left_file != right_file: + exact_title = bool( + _normalized_text(left_title) + and _normalized_text(left_title) == _normalized_text(right_title) + ) + # Cross-file occurrences are candidates only for a distinctive shared + # root signature. Generic repeated warnings remain independent. + return bool( + exact_title + and len(_meaningful_tokens(left_title)) >= 4 + and reason_overlap >= 4 + and reason_containment >= 0.50 + and reason_jaccard >= 0.25 + ) + + left_line = _line_number(left_data) + right_line = _line_number(right_data) + near_line = bool( + left_line > 0 + and right_line > 0 + and abs(left_line - right_line) <= 3 + ) + exact_anchor = _issues_share_exact_anchor(left_data, right_data) + exact_title = bool( + _normalized_text(left_title) + and _normalized_text(left_title) == _normalized_text(right_title) + ) + + if exact_title and ( + reason_similarity >= 0.50 + or (reason_overlap >= 3 and reason_containment >= 0.45) + ): + return True + + anchor_related = exact_anchor or near_line + title_related = bool( + title_similarity >= 0.48 + or (title_overlap >= 3 and title_containment >= 0.55) + or title_jaccard >= 0.42 + ) + reason_related = bool( + reason_similarity >= 0.46 + or (reason_overlap >= 4 and reason_containment >= 0.50) + or reason_jaccard >= 0.36 + ) + if title_related and ( + reason_similarity >= 0.70 + or (reason_overlap >= 5 and reason_containment >= 0.65) + ): + # Strong same-file narrative identity can drift to a nearby helper or + # call site. Send it for semantic judgment without deleting locally. + return True + return anchor_related and title_related and reason_related + + +def _semantic_candidate_groups( + issues: Sequence[CodeReviewIssue], +) -> List[List[CodeReviewIssue]]: + """Build connected candidate components; singleton findings cost no tokens.""" + count = len(issues) + parents = list(range(count)) + + def find(index: int) -> int: + while parents[index] != index: + parents[index] = parents[parents[index]] + index = parents[index] + return index + + def union(left_index: int, right_index: int) -> None: + left_root = find(left_index) + right_root = find(right_index) + if left_root != right_root: + parents[right_root] = left_root + + for left_index in range(count): + for right_index in range(left_index + 1, count): + if issues_are_semantic_dedup_candidates( + issues[left_index], + issues[right_index], + ): + union(left_index, right_index) + + groups: Dict[int, List[CodeReviewIssue]] = defaultdict(list) + for index, issue in enumerate(issues): + groups[find(index)].append(issue) + return [group for group in groups.values() if len(group) > 1] + def deduplicate_issues(issues: List[Any]) -> List[dict]: """Deduplicate issues by fingerprint, keeping most recent version. @@ -369,28 +764,33 @@ def deduplicate_final_issues(issues: List[CodeReviewIssue]) -> List[CodeReviewIs Final deduplication pass after ALL issue-finding stages complete (Stage 1, Reconciliation, Verification, Stage 2 cross-file). - Conservative deterministic dedup requires the same normalized file, - category, exact line or exact current-source snippet, and closely matching - root-cause text. Similar prose by itself never suppresses another finding. + The deterministic tier merges only effectively exact root identities. It + preserves the best anchor, highest severity, historical identity, and all + additional affected locations. Ambiguous semantic pairs remain untouched + for the grouped LLM pass. """ if not issues: return [] deduped: List[CodeReviewIssue] = [] for issue in issues: - if any( - issues_are_conservative_duplicates(issue, existing) - for existing in deduped - ): - data = _issue_payload(issue) - logger.info( - "Final deterministic dedup: suppressed anchored duplicate at " - "%s:%s", - data.get("file", ""), - data.get("line", ""), - ) + duplicate_index = next(( + index + for index, existing in enumerate(deduped) + if issues_are_conservative_duplicates(issue, existing) + ), None) + if duplicate_index is None: + deduped.append(issue) continue - deduped.append(issue) + existing = deduped[duplicate_index] + merged = _merge_duplicate_issues(existing, issue) + deduped[duplicate_index] = merged + data = _issue_payload(issue) + logger.info( + "Final deterministic dedup: merged exact root identity at %s:%s", + data.get("file", ""), + data.get("line", ""), + ) original = len(issues) final = len(deduped) @@ -405,22 +805,27 @@ def deduplicate_final_issues(issues: List[CodeReviewIssue]) -> List[CodeReviewIs _DEDUP_BATCH_SIZE = 50 _DEDUP_MAX_PARALLEL = max(1, _env_int("REVIEW_DEDUP_MAX_PARALLEL", 4)) +_DEDUP_BATCH_CHAR_BUDGET = max( + 8_000, + _env_int("REVIEW_DEDUP_BATCH_CHAR_BUDGET", 48_000), +) _DEDUP_SYSTEM_PROMPT = ( - "You are a code review deduplication assistant. You will receive a list of " - "code-review issues (each with an index, file, line, severity, category, and " - "reason). Your task is to identify **semantic duplicates** — issues that " - "describe the same underlying problem even if they use different wording, " - "slightly different line numbers in the same file, or were found by different " - "analysis stages.\n\n" + "You are a code-review root-cause deduplication assistant. You receive only " + "host-selected candidate groups; findings outside these groups are never sent " + "and are automatically retained. Identify only HIGH-confidence duplicate " + "occurrences that describe one actionable root cause.\n\n" "Rules:\n" - "1. Two issues are duplicates if they point to the SAME root cause in the " - "SAME file (small line-number differences are OK).\n" - "2. When you find duplicates, KEEP the one with the most detailed/useful " - "reason text and DROP the rest.\n" - "3. Issues in DIFFERENT files are NEVER duplicates of each other.\n" - "4. Return ONLY the 0-based indices of the issues you decide to KEEP.\n" - "5. If there are no duplicates at all, return every index." + "1. Compare findings only inside the same candidate_group.\n" + "2. Duplicate means the same causal defect, not merely the same rule, pattern, " + "category, or suggested fix. Independent occurrences must remain separate.\n" + "3. Category, severity, stage, and small anchor differences do not make the " + "same root cause independent.\n" + "4. Cross-file findings may be duplicates only when one shared defect causes " + "the reported failures; repeated independent defects in different files are not.\n" + "5. Select the most current, concrete, and useful representative as keeper.\n" + "6. Emit a duplicate group only at HIGH confidence. Omit uncertain pairs; " + "omitted findings are retained. Never return a kept-index allowlist." ) @@ -443,6 +848,97 @@ def _format_issues_for_prompt(issues: List[CodeReviewIssue]) -> str: return "\n".join(lines) +def _semantic_issue_payload( + issue: CodeReviewIssue, + index: int, + group_id: str, +) -> Dict[str, Any]: + data = _issue_payload(issue) + reason, generated_locations = _split_reason_locations( + data.get("reason") or data.get("description") or "" + ) + related_locations = sorted({ + str(value).strip() + for value in ( + list(data.get("relatedLocations") or []) + + list(generated_locations) + ) + if str(value).strip() + }) + return { + "index": index, + "candidate_group": group_id, + "existing_issue_id": str(data.get("id") or ""), + "file": _normalized_file(data), + "line": _line_number(data), + "severity": str(data.get("severity") or ""), + "category": str(data.get("category") or ""), + "title": str(data.get("title") or ""), + # Candidate grouping keeps the request small enough to preserve complete + # root-cause prose. No field-level character clipping is performed here. + "reason": reason, + "suggested_fix": str(data.get("suggestedFixDescription") or ""), + "exact_source_anchor": str( + data.get("codeSnippet") or data.get("code_snippet") or "" + ), + "related_locations": related_locations, + } + + +def _format_semantic_batch( + issues: Sequence[CodeReviewIssue], + group_by_index: Dict[int, str], +) -> str: + return json.dumps( + [ + _semantic_issue_payload(issue, index, group_by_index[index]) + for index, issue in enumerate(issues) + ], + ensure_ascii=False, + separators=(",", ":"), + ) + + +def _build_semantic_dedup_batches( + groups: Sequence[Sequence[CodeReviewIssue]], +) -> List[tuple[List[CodeReviewIssue], Dict[int, str]]]: + """Pack whole candidate groups by rendered size without clipping content.""" + batches: List[tuple[List[CodeReviewIssue], Dict[int, str]]] = [] + current_groups: List[Sequence[CodeReviewIssue]] = [] + current_chars = 0 + + def group_size(group: Sequence[CodeReviewIssue], group_index: int) -> int: + mapping = {index: f"candidate_{group_index}" for index in range(len(group))} + return len(_format_semantic_batch(group, mapping)) + + def flush() -> None: + nonlocal current_groups, current_chars + if not current_groups: + return + issues: List[CodeReviewIssue] = [] + mapping: Dict[int, str] = {} + for local_group_index, group in enumerate(current_groups): + group_id = f"candidate_{local_group_index}" + for issue in group: + mapping[len(issues)] = group_id + issues.append(issue) + batches.append((issues, mapping)) + current_groups = [] + current_chars = 0 + + for group_index, group in enumerate(groups): + rendered_chars = group_size(group, group_index) + if current_groups and current_chars + rendered_chars > _DEDUP_BATCH_CHAR_BUDGET: + flush() + current_groups.append(group) + current_chars += rendered_chars + # An unusually detailed group is sent alone with its evidence intact. + if rendered_chars > _DEDUP_BATCH_CHAR_BUDGET: + flush() + flush() + return batches + + def _build_batches(issues: List[CodeReviewIssue], max_batch_size: int = _DEDUP_BATCH_SIZE, ) -> List[List[CodeReviewIssue]]: @@ -477,45 +973,111 @@ def _build_batches(issues: List[CodeReviewIssue], async def _dedup_batch_with_llm( llm, batch: List[CodeReviewIssue], + group_by_index: Optional[Dict[int, str]] = None, ) -> List[CodeReviewIssue]: - """Send one batch to the LLM and return the kept issues.""" - issues_text = _format_issues_for_prompt(batch) + """Merge only validated high-confidence duplicate groups from one batch.""" + groups = group_by_index or {index: "candidate_0" for index in range(len(batch))} + issues_text = _format_semantic_batch(batch, groups) prompt = ( f"{_DEDUP_SYSTEM_PROMPT}\n\n" - f"Here are the issues to deduplicate:\n\n{issues_text}\n\n" - "Return the kept_indices list." + f"Candidate findings JSON:\n{issues_text}\n\n" + "Return duplicate_groups only." ) try: if supports_structured_output(llm): - structured_llm = llm.with_structured_output(DeduplicatedIssueList) - result: DeduplicatedIssueList = await structured_llm.ainvoke(prompt) + structured_llm = llm.with_structured_output( + SemanticDeduplicationDecision + ) + result: SemanticDeduplicationDecision = await structured_llm.ainvoke( + prompt + ) else: logger.info("Structured output skipped for LLM dedup batch; using prompt JSON parsing") response = await llm.ainvoke(prompt) result = await parse_llm_response( extract_llm_response_text(response), - DeduplicatedIssueList, + SemanticDeduplicationDecision, llm, ) - kept_indices = set(result.kept_indices) - # Sanity-check: indices must be within range - valid = {i for i in kept_indices if 0 <= i < len(batch)} - if not valid: - logger.warning( - "LLM dedup returned no valid indices — keeping all issues in batch" - ) - return batch + removed_indices: set[int] = set() + replacements: Dict[int, CodeReviewIssue] = {} + accepted_groups = 0 + for decision in result.duplicate_groups: + if str(decision.confidence or "").strip().upper() != "HIGH": + continue + keeper_index = decision.keeper_index + duplicate_indices = list(dict.fromkeys(decision.duplicate_indices)) + all_indices = [keeper_index, *duplicate_indices] + if ( + keeper_index < 0 + or keeper_index >= len(batch) + or not duplicate_indices + or any(index < 0 or index >= len(batch) for index in all_indices) + or keeper_index in duplicate_indices + or any(index in removed_indices for index in all_indices) + or any( + index in replacements and index != keeper_index + for index in duplicate_indices + ) + or len({groups.get(index) for index in all_indices}) != 1 + ): + logger.warning( + "LLM dedup rejected malformed or cross-group decision: %s", + decision.model_dump(), + ) + continue + keeper = replacements.get(keeper_index, batch[keeper_index]) + if not all( + issues_are_semantic_dedup_candidates( + keeper, + batch[duplicate_index], + ) + for duplicate_index in duplicate_indices + ): + logger.warning( + "LLM dedup rejected decision without host candidate evidence: %s", + decision.model_dump(), + ) + continue + for duplicate_index in duplicate_indices: + keeper = _merge_duplicate_issues( + keeper, + batch[duplicate_index], + prefer_left=True, + ) + removed_indices.add(duplicate_index) + replacements[keeper_index] = keeper + accepted_groups += 1 - kept = [batch[i] for i in sorted(valid)] - dropped = len(batch) - len(kept) - if dropped: - logger.info(f"LLM dedup batch: kept {len(kept)}/{len(batch)} issues (dropped {dropped})") + if not removed_indices: + return batch + kept: List[CodeReviewIssue] = [] + emitted_replacements: set[int] = set() + for index, issue in enumerate(batch): + if index in removed_indices: + continue + replacement = replacements.get(index, issue) + replacement_id = id(replacement) + if replacement_id in emitted_replacements: + continue + emitted_replacements.add(replacement_id) + kept.append(replacement) + logger.info( + "LLM semantic dedup accepted %d group(s): %d → %d candidate issues", + accepted_groups, + len(batch), + len(kept), + ) return kept except Exception as exc: - logger.warning(f"LLM dedup batch failed ({exc}); falling back to algorithmic dedup") + logger.warning( + "LLM dedup batch failed (%s); retaining ambiguous findings after " + "exact deterministic dedup", + exc, + ) return deduplicate_final_issues(batch) @@ -523,52 +1085,86 @@ async def deduplicate_final_issues_llm( llm, issues: List[CodeReviewIssue], ) -> List[CodeReviewIssue]: - """Primary LLM-driven deduplication. + """Recall-safe semantic dedup over host-selected candidate components. - 1. Groups issues by filepath. - 2. Packs filepath-groups into batches of ≤ 50 issues. - 3. Sends each batch to the LLM to identify semantic duplicates. - 4. Returns the union of kept issues from all batches. - - Falls back to ``deduplicate_final_issues`` (algorithmic) for any batch - where the LLM call fails. + Exact identities merge locally first. Only ambiguous components with two or + more plausible duplicates consume model tokens, and a failed/incomplete LLM + decision retains every ambiguous finding. """ if not issues: return [] - if len(issues) <= 1: - return issues + exact_deduped = deduplicate_final_issues(issues) + if len(exact_deduped) <= 1: + return exact_deduped + + candidate_groups = _semantic_candidate_groups(exact_deduped) + if not candidate_groups: + logger.info( + "LLM semantic dedup skipped: %d findings produced no ambiguous " + "candidate group", + len(exact_deduped), + ) + return exact_deduped - batches = _build_batches(issues, max_batch_size=_DEDUP_BATCH_SIZE) + batches = _build_semantic_dedup_batches(candidate_groups) + rendered_chars = sum( + len(_format_semantic_batch(batch, mapping)) + for batch, mapping in batches + ) + candidate_count = sum(len(group) for group in candidate_groups) logger.info( - f"LLM dedup: {len(issues)} issues split into {len(batches)} batch(es) " - f"(sizes: {[len(b) for b in batches]}, concurrency={_DEDUP_MAX_PARALLEL})" + "LLM semantic dedup: %d/%d findings in %d candidate group(s), " + "%d batch(es), input≈%d tokens, concurrency=%d", + candidate_count, + len(exact_deduped), + len(candidate_groups), + len(batches), + rendered_chars // 4, + _DEDUP_MAX_PARALLEL, ) semaphore = asyncio.Semaphore(_DEDUP_MAX_PARALLEL) batch_results: Dict[int, List[CodeReviewIssue]] = {} - async def _run_batch(batch_idx: int, batch: List[CodeReviewIssue]) -> tuple[int, List[CodeReviewIssue]]: + async def _run_batch( + batch_idx: int, + batch: List[CodeReviewIssue], + mapping: Dict[int, str], + ) -> tuple[int, List[CodeReviewIssue]]: async with semaphore: logger.info( f"LLM dedup: processing batch {batch_idx + 1}/{len(batches)} " f"({len(batch)} issues)" ) - kept = await _dedup_batch_with_llm(llm, batch) + kept = await _dedup_batch_with_llm(llm, batch, mapping) return batch_idx, kept tasks = [ - asyncio.create_task(_run_batch(batch_idx, batch)) - for batch_idx, batch in enumerate(batches) + asyncio.create_task(_run_batch(batch_idx, batch, mapping)) + for batch_idx, (batch, mapping) in enumerate(batches) ] for completed_task in asyncio.as_completed(tasks): batch_idx, kept = await completed_task batch_results[batch_idx] = kept - kept_issues: List[CodeReviewIssue] = [] - for batch_idx in range(len(batches)): - kept_issues.extend(batch_results.get(batch_idx, [])) + candidate_object_ids = { + id(issue) + for group in candidate_groups + for issue in group + } + retained_candidate_ids = { + id(issue) + for kept in batch_results.values() + for issue in kept + } + removed_candidate_ids = candidate_object_ids - retained_candidate_ids + kept_issues = [ + issue + for issue in exact_deduped + if id(issue) not in removed_candidate_ids + ] original = len(issues) final = len(kept_issues) @@ -592,19 +1188,25 @@ def deduplicate_cross_batch_issues(issues: List[CodeReviewIssue]) -> List[CodeRe deduped = [] for issue in issues: - if any( - issues_are_conservative_duplicates(issue, existing) - for existing in deduped - ): - issue_data = _issue_payload(issue) - logger.info( - "Cross-batch dedup: suppressed anchored duplicate at %s:%s", - issue_data.get("file", ""), - issue_data.get("line", ""), - ) + duplicate_index = next(( + index + for index, existing in enumerate(deduped) + if issues_are_conservative_duplicates(issue, existing) + ), None) + if duplicate_index is None: + deduped.append(issue) continue - deduped.append(issue) - + deduped[duplicate_index] = _merge_duplicate_issues( + deduped[duplicate_index], + issue, + ) + issue_data = _issue_payload(issue) + logger.info( + "Cross-batch dedup: merged exact root identity at %s:%s", + issue_data.get("file", ""), + issue_data.get("line", ""), + ) + return deduped async def reconcile_previous_issues( diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py index ded1f4e3..f293e565 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py @@ -234,14 +234,19 @@ def _capture_deterministic_retrieval_state( if _rag_response_error(deterministic_response): rag_state.deterministic_retrieval_states.append("failed") return + rag_state.deterministic_retrieval_states.append( + _deterministic_retrieval_state(deterministic_response) + ) + + +def _deterministic_retrieval_state( + deterministic_response: Optional[Dict[str, Any]], +) -> str: context = _unwrap_rag_context(deterministic_response) metadata = context.get("_metadata") if isinstance(context, dict) else None - retrieval_state = ( - str(metadata.get("retrieval_state")) - if isinstance(metadata, dict) and metadata.get("retrieval_state") - else "unknown" - ) - rag_state.deterministic_retrieval_states.append(retrieval_state) + if isinstance(metadata, dict) and metadata.get("retrieval_state"): + return str(metadata["retrieval_state"]).strip().casefold() + return "unknown" def _rag_response_error(response: Optional[Dict[str, Any]]) -> Optional[str]: @@ -1212,6 +1217,25 @@ async def create_smart_batches_wrapper( "using local/enrichment grouping without a repository RAG lookup" ) batching_rag_client = None + exact_receipt_values = tuple( + value + for value in ( + getattr(request, "ragBaseGenerationManifestSha256", None), + getattr(request, "ragPrGenerationFingerprint", None), + getattr( + request, + "ragPrOverlayGenerationManifestSha256", + None, + ), + ) + if isinstance(value, str) and value + ) + if any(exact_receipt_values): + logger.info( + "Stage 1 smart-batching RAG discovery disabled while exact " + "base/overlay generation receipts are active" + ) + batching_rag_client = None enrichment_data = getattr(request, 'enrichmentData', None) @@ -1534,6 +1558,20 @@ def _expand_oversized_diff_batches( # ── RAG Context ─────────────────────────────────────────────── +def _is_exact_revision_bound( + request: ReviewRequestDto, + pr_indexed: bool, +) -> bool: + return bool( + pr_indexed + and (request.currentCommitHash or request.commitHash) + and request.baseCommitHash + and request.ragBaseGenerationManifestSha256 + and request.ragPrGenerationFingerprint + and request.ragPrOverlayGenerationManifestSha256 + ) + + async def fetch_batch_rag_context( rag_client, request: ReviewRequestDto, @@ -1547,9 +1585,16 @@ async def fetch_batch_rag_context( batch_raw_diffs: Optional[List[str]] = None, rag_state: Optional[Stage1RagState] = None, ) -> Optional[Dict[str, Any]]: + exact_revision_bound = _is_exact_revision_bound(request, pr_indexed) if not rag_client: + if exact_revision_bound: + raise RuntimeError( + "revision-bound Stage 1 retrieval requires a RAG client" + ) return None + duplication_task: Optional[asyncio.Task] = None + try: rag_branch = request.get_rag_branch() base_branch = request.get_rag_base_branch() @@ -1560,6 +1605,8 @@ async def fetch_batch_rag_context( {"status": "error", "error": message}, rag_state, ) + if exact_revision_bound: + raise RuntimeError(message) return None # Scale top_k based on batch priority to ensure adequate context @@ -1571,6 +1618,27 @@ async def fetch_batch_rag_context( pr_number = request.pullRequestId if pr_indexed else None all_pr_files = request.changedFiles if pr_indexed else None + source_revision = ( + request.currentCommitHash or request.commitHash + if pr_indexed + else None + ) + base_revision = request.baseCommitHash if pr_indexed else None + base_generation_receipt = ( + request.ragBaseGenerationManifestSha256 + if pr_indexed + else None + ) + pr_generation_fingerprint = ( + request.ragPrGenerationFingerprint + if pr_indexed + else None + ) + pr_overlay_generation_manifest_sha256 = ( + request.ragPrOverlayGenerationManifestSha256 + if pr_indexed + else None + ) context = None @@ -1579,12 +1647,26 @@ async def _fetch_deterministic_context() -> Optional[Dict[str, Any]]: return await rag_client.get_deterministic_context( workspace=request.projectWorkspace, project=request.projectNamespace, - branches=[branch for branch in [rag_branch, base_branch] if branch], + branches=list(dict.fromkeys( + branch + for branch in (rag_branch, base_branch) + if branch + )), file_paths=batch_file_paths, limit_per_file=5, pr_number=pr_number, pr_changed_files=all_pr_files, additional_identifiers=enrichment_identifiers, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=( + base_generation_receipt + ), + pr_generation_fingerprint=pr_generation_fingerprint, + pr_overlay_generation_manifest_sha256=( + pr_overlay_generation_manifest_sha256 + ), + collection_target=request.ragCollectionTarget, ) except Exception as det_err: logger.warning("Deterministic RAG lookup failed: %s", det_err) @@ -1613,6 +1695,14 @@ async def _fetch_semantic_context() -> Optional[Dict[str, Any]]: pr_number=pr_number, all_pr_changed_files=all_pr_files, deleted_files=request.deletedFiles or None, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_receipt, + pr_generation_fingerprint=pr_generation_fingerprint, + pr_overlay_generation_manifest_sha256=( + pr_overlay_generation_manifest_sha256 + ), + collection_target=request.ragCollectionTarget, ) async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: @@ -1649,8 +1739,15 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: queries=duplication_queries, top_k=8, base_branch=base_branch, + repository_revision=base_revision, + repository_generation_manifest_sha256=( + base_generation_receipt + ), + collection_target=request.ragCollectionTarget, ) except Exception as dup_err: + if base_revision or base_generation_receipt: + raise logger.debug(f"Duplication search skipped: {dup_err}") return None @@ -1670,6 +1767,22 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: deterministic_response, rag_state, ) + if deterministic_error and exact_revision_bound: + raise RuntimeError( + "revision-bound deterministic RAG retrieval failed: " + f"{deterministic_error}" + ) + deterministic_retrieval_state = _deterministic_retrieval_state( + deterministic_response + ) + if ( + exact_revision_bound + and deterministic_retrieval_state != "complete" + ): + raise RuntimeError( + "revision-bound deterministic RAG retrieval is not complete: " + f"{deterministic_retrieval_state}" + ) if deterministic_chunks: context = {"relevant_code": deterministic_chunks} logger.info( @@ -1682,9 +1795,22 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: semantic_fill = max(0, top_k - det_count) rag_response = None - if semantic_fill > 0 and rag_state and rag_state.semantic_disabled: + semantic_fill_enabled = ( + semantic_fill > 0 and SEMANTIC_RAG_FILLER_ENABLED + ) + if semantic_fill > 0 and not SEMANTIC_RAG_FILLER_ENABLED: + logger.info( + "Semantic RAG filler skipped by " + "REVIEW_SEMANTIC_RAG_FILLER_ENABLED" + ) + elif semantic_fill_enabled and rag_state and rag_state.semantic_disabled: logger.info("Semantic RAG filler skipped: %s", rag_state.semantic_disable_reason) - elif semantic_fill > 0: + if exact_revision_bound: + raise RuntimeError( + "revision-bound semantic RAG retrieval is disabled after " + f"a prior failure: {rag_state.semantic_disable_reason}" + ) + elif semantic_fill_enabled: try: rag_response = await asyncio.wait_for( _fetch_semantic_context(), @@ -1705,6 +1831,10 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: "Semantic RAG filler timed out after %ss; disabling for remaining Stage 1 batches", SEMANTIC_RAG_TIMEOUT_SECONDS, ) + if exact_revision_bound: + raise RuntimeError( + "revision-bound semantic RAG retrieval timed out" + ) except Exception as sem_err: rag_response = None if rag_state: @@ -1712,6 +1842,17 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: rag_state.semantic_disabled = True rag_state.semantic_disable_reason = str(sem_err) logger.warning("Semantic RAG filler failed; disabling for remaining Stage 1 batches: %s", sem_err) + if exact_revision_bound: + raise + + if ( + semantic_fill_enabled + and exact_revision_bound + and rag_response is None + ): + raise RuntimeError( + "revision-bound semantic RAG retrieval returned no response" + ) if semantic_fill > 0 and rag_response: sem_context = _unwrap_rag_context(rag_response) @@ -1811,7 +1952,13 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: return None except Exception as e: + if duplication_task is not None and not duplication_task.done(): + duplication_task.cancel() + if duplication_task is not None: + await asyncio.gather(duplication_task, return_exceptions=True) logger.warning(f"Failed to fetch per-batch RAG context: {e}") + if exact_revision_bound: + raise return None @@ -2254,7 +2401,7 @@ async def review_file_batch( str, tuple[Dict[str, Any], ...] ] = {} - if rag_client: + if rag_client or _is_exact_revision_bound(request, pr_indexed): batch_rag_context = await fetch_batch_rag_context( rag_client, request, batch_file_paths, batch_diff_snippets, pr_indexed, llm_reranker=llm_reranker, diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py index 47172ab1..880cb3e7 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py @@ -686,6 +686,37 @@ async def _fetch_cross_module_context( if not rag_client: return "" + base_revision_value = getattr(request, "baseCommitHash", None) + base_generation_receipt_value = getattr( + request, + "ragBaseGenerationManifestSha256", + None, + ) + base_revision = ( + base_revision_value + if isinstance(base_revision_value, str) and base_revision_value + else None + ) + base_generation_receipt = ( + base_generation_receipt_value + if ( + isinstance(base_generation_receipt_value, str) + and base_generation_receipt_value + ) + else None + ) + if not base_revision and not base_generation_receipt: + logger.info( + "Stage 2 cross-module RAG skipped: no exact target-generation lease" + ) + return "" + if not base_revision or not base_generation_receipt: + logger.warning( + "Stage 2 cross-module RAG requires both immutable target revision " + "and generation receipt" + ) + return "" + try: rag_branch = request.get_rag_branch() base_branch = request.get_rag_base_branch() @@ -733,6 +764,11 @@ async def _fetch_cross_module_context( queries=unique_queries, top_k=6, base_branch=base_branch, + repository_revision=base_revision, + repository_generation_manifest_sha256=( + base_generation_receipt + ), + collection_target=getattr(request, "ragCollectionTarget", None), ) if not dup_results: @@ -752,5 +788,9 @@ async def _fetch_cross_module_context( return formatted except Exception as e: - logger.warning(f"Failed to fetch cross-module context for Stage 2: {e}") + logger.warning( + "Revision-bound cross-module context unavailable for Stage 2: %s: %s", + type(e).__name__, + e, + ) return "" diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py index 446bda7f..cdb5576b 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py @@ -3,6 +3,7 @@ """ import json import logging +import re from typing import Any, Dict, List, Optional from model.dtos import ReviewRequestDto @@ -31,6 +32,7 @@ async def execute_stage_3_aggregation( fallback_llm=None, ) -> Dict[str, Any]: stage_1_json = _summarize_issues_for_stage_3(stage_1_issues) + verification_issues = _stage_3_verification_issue_map(stage_1_issues) stage_2_json = stage_2_results.model_dump_json(indent=2) plan_summary = _summarize_plan_for_stage_3(plan) @@ -49,7 +51,7 @@ async def execute_stage_3_aggregation( additions = processed_diff.total_additions if processed_diff else 0 deletions = processed_diff.total_deletions if processed_diff else 0 - target_branch = request.targetBranchName or "" + review_revision = _review_revision(request) prompt = PromptBuilder.build_stage_3_aggregation_prompt( repo_slug=request.projectVcsRepoSlug, @@ -69,22 +71,38 @@ async def execute_stage_3_aggregation( or "No task context available." ), use_mcp_tools=use_mcp_tools, - target_branch=target_branch, + review_revision=review_revision, ) - if use_mcp_tools and mcp_client and target_branch: + if use_mcp_tools and mcp_client and review_revision: return await _stage_3_with_mcp( llm, request, prompt, mcp_client, - target_branch, + review_revision, + verification_issues, fallback_llm=fallback_llm, ) + if use_mcp_tools and mcp_client and not review_revision: + logger.warning( + "[Stage 3] MCP verification skipped: no immutable reviewed commit " + "hash was supplied" + ) + return await _invoke_stage_3_report(llm, prompt, fallback_llm=fallback_llm) +def _review_revision(request: ReviewRequestDto) -> str: + """Return an immutable review revision; never substitute a moving branch.""" + for field in ("currentCommitHash", "commitHash"): + value = getattr(request, field, None) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + async def _invoke_stage_3_report(llm, prompt: str, fallback_llm=None) -> Dict[str, Any]: response = await llm.ainvoke(prompt) if _response_finished_by_length(response) and fallback_llm is not None and fallback_llm is not llm: @@ -133,26 +151,104 @@ def _summarize_issues_for_stage_3(issues: List[CodeReviewIssue]) -> str: "By category: " + ", ".join(f"{k}: {v}" for k, v in sorted(category_counts.items())), ] - priority_order = {'CRITICAL': 0, 'HIGH': 1, 'MEDIUM': 2, 'LOW': 3, 'INFO': 4} - ranked = sorted(issues, key=lambda i: priority_order.get(i.severity.upper(), 5)) - top_n = ranked[:10] - if top_n: - lines.append("\nTop findings (issue IDs are for internal reference):") - for i, issue in enumerate(top_n, 1): - issue_id = getattr(issue, 'id', '') or '' - title = getattr(issue, 'title', '') or '' - title_part = f" {title} —" if title else "" - lines.append(f" {i}. [id={issue_id}] [{issue.severity}] {issue.file}:{title_part} {issue.reason[:120]}") - - if issues: - all_ids = [getattr(i, 'id', '') or '' for i in issues] - all_ids = [i for i in all_ids if i] - if all_ids: - lines.append(f"\nAll issue IDs: {', '.join(all_ids)}") + # Stage 3 can verify any finding, including fresh ones without database IDs. + # Emit a semantically compact record for every active issue instead of + # clipping details to a top-ten list. + records = [ + _stage_3_verification_record(verification_id, issue) + for verification_id, issue in _stage_3_verification_issue_map(issues).items() + ] + lines.append("\nComplete verification records (JSON):") + lines.append(json.dumps(records, ensure_ascii=False, separators=(",", ":"))) return "\n".join(lines) +def _safe_issue_field(issue: CodeReviewIssue, name: str) -> Any: + value = getattr(issue, name, "") + if value is None: + return "" + if value.__class__.__module__.startswith("unittest.mock"): + return "" + return value + + +def _stage_3_verification_issue_map( + issues: List[CodeReviewIssue], +) -> Dict[str, CodeReviewIssue]: + active = [ + issue + for issue in issues + if getattr(issue, "isResolved", False) is not True + ] + return { + f"issue_{index}": issue + for index, issue in enumerate(active) + } + + +_RELATED_LOCATIONS_RE = re.compile( + r"(?im)^\s*(?:[*_]{1,2})?also affects\s*:(?:[*_]{1,2})?\s*(.+)$" +) + + +def _stage_3_reason_brief(issue: CodeReviewIssue) -> str: + """Remove exact repetition while preserving every substantive paragraph.""" + reason = str(_safe_issue_field(issue, "reason") or "").strip() + if not reason: + return "" + paragraphs = [ + paragraph.strip() + for paragraph in re.split(r"\n\s*\n", reason) + if paragraph.strip() + ] + title = str(_safe_issue_field(issue, "title") or "").strip().casefold() + selected: List[str] = [] + seen: set[str] = set() + for paragraph in paragraphs: + normalized = " ".join(paragraph.strip("*_# ").casefold().split()) + if normalized == title: + continue + if normalized in seen: + continue + selected.append(paragraph) + seen.add(normalized) + return "\n\n".join(selected) if selected else reason + + +def _normalized_related_locations(issue: CodeReviewIssue) -> List[str]: + values = _safe_issue_field(issue, "relatedLocations") or [] + locations = list(values) if isinstance(values, (list, tuple, set)) else [] + reason = str(_safe_issue_field(issue, "reason") or "") + for match in _RELATED_LOCATIONS_RE.finditer(reason): + locations.extend(match.group(1).split(",")) + return sorted({ + str(value).strip() + for value in locations + if str(value).strip() + }) + + +def _stage_3_verification_record( + verification_id: str, + issue: CodeReviewIssue, +) -> Dict[str, Any]: + return { + "verification_id": verification_id, + "original_id": str(_safe_issue_field(issue, "id") or ""), + "severity": str(_safe_issue_field(issue, "severity") or ""), + "category": str(_safe_issue_field(issue, "category") or ""), + "file": str(_safe_issue_field(issue, "file") or ""), + "line": _safe_issue_field(issue, "line") or 0, + "title": str(_safe_issue_field(issue, "title") or ""), + "reason": _stage_3_reason_brief(issue), + "exact_source_anchor": str( + _safe_issue_field(issue, "codeSnippet") or "" + ), + "related_locations": _normalized_related_locations(issue), + } + + def _summarize_plan_for_stage_3(plan: ReviewPlan) -> str: lines = [] total_files = sum(len(g.files) for g in plan.file_groups) @@ -194,26 +290,151 @@ def _extract_dismissed_issues(content: str) -> tuple: try: dismissed = json.loads(match.group(1)) if not isinstance(dismissed, list): - logger.warning(f"[Stage 3] DISMISSED_ISSUES was not a list: {match.group(1)}") + logger.warning( + "[Stage 3] DISMISSED_ISSUES was not a list: %s", + match.group(1), + ) return content, [] dismissed = [str(d) for d in dismissed if d] - logger.info(f"[Stage 3] MCP verification dismissed {len(dismissed)} issues: {dismissed}") + logger.info( + "[Stage 3] MCP verification requested dismissal of %d issues: %s", + len(dismissed), + dismissed, + ) clean_report = content[:match.start()].rstrip() + content[match.end():] return clean_report.strip(), dismissed - except (json.JSONDecodeError, TypeError) as e: - logger.warning(f"[Stage 3] Failed to parse DISMISSED_ISSUES: {e}") + except (json.JSONDecodeError, TypeError) as exc: + logger.warning("[Stage 3] Failed to parse DISMISSED_ISSUES: %s", exc) return content, [] +def _location_file_path(location: str) -> str: + normalized = str(location or "").strip().replace("\\", "/").lstrip("/") + if not normalized: + return "" + path, separator, possible_line = normalized.rpartition(":") + if separator and possible_line.isdigit(): + return path + return normalized + + +def _location_line(location: str) -> int: + normalized = str(location or "").strip().replace("\\", "/") + _, separator, possible_line = normalized.rpartition(":") + if separator and possible_line.isdigit(): + return int(possible_line) + return 0 + + +def _required_verification_locations( + issue: CodeReviewIssue, +) -> set[tuple[str, int]]: + primary_path = _location_file_path( + str(_safe_issue_field(issue, "file") or "") + ) + try: + primary_line = int(_safe_issue_field(issue, "line") or 0) + except (TypeError, ValueError): + primary_line = 0 + locations = {(primary_path, max(0, primary_line))} + locations.update( + (_location_file_path(location), _location_line(location)) + for location in _normalized_related_locations(issue) + ) + return {(path, line) for path, line in locations if path} + + +def _mcp_read_covers_location( + entry: Dict[str, Any], + verification_id: str, + file_path: str, + line: int, + review_revision: str, +) -> bool: + args = entry.get("args", {}) + if not ( + entry.get("tool") == "getBranchFileContent" + and entry.get("success") is True + and entry.get("evidence_valid") is True + and str(args.get("verificationId") or "") == verification_id + and _location_file_path(str(args.get("filePath") or "")) == file_path + and str(args.get("branch") or "") == review_revision + ): + return False + if entry.get("evidence_complete_file") is True: + return True + if line <= 0: + return False + try: + start_line = int(entry.get("evidence_start_line") or 0) + end_line = int(entry.get("evidence_end_line") or 0) + except (TypeError, ValueError): + return False + return start_line > 0 and start_line <= line <= end_line + + +def _validated_mcp_dismissals( + requested_ids: List[str], + issue_by_verification_id: Dict[str, CodeReviewIssue], + executor: McpToolExecutor, + review_revision: str, +) -> List[str]: + """Accept dismissals only when every affected anchor has bound evidence.""" + validated: List[str] = [] + for verification_id in requested_ids: + issue = issue_by_verification_id.get(verification_id) + if issue is None: + logger.warning( + "[Stage 3] Ignoring dismissal for unknown verification ID %s", + verification_id, + ) + continue + required_locations = _required_verification_locations(issue) + missing_locations = { + location + for location in required_locations + if not any( + _mcp_read_covers_location( + entry, + verification_id, + location[0], + location[1], + review_revision, + ) + for entry in executor.call_log + ) + } + if not required_locations or missing_locations: + logger.warning( + "[Stage 3] Keeping %s: dismissal lacks successful reviewed-revision " + "evidence for %s", + verification_id, + sorted( + f"{path}:{line}" if line > 0 else path + for path, line in missing_locations + ), + ) + continue + validated.append(verification_id) + return validated + + async def _stage_3_with_mcp( llm, request: ReviewRequestDto, prompt: str, mcp_client, - target_branch: str, + review_revision: str, + issue_by_verification_id: Dict[str, CodeReviewIssue], fallback_llm=None, ) -> Dict[str, Any]: - executor = McpToolExecutor(mcp_client, request, stage="stage_3") + executor = McpToolExecutor( + mcp_client, + request, + stage="stage_3", + review_revision=review_revision, + verification_issues=issue_by_verification_id, + ) tool_defs = executor.get_tool_definitions() max_iterations = 15 @@ -234,7 +455,8 @@ async def _stage_3_with_mcp( request, prompt, mcp_client, - target_branch, + review_revision, + issue_by_verification_id, ) content = extract_llm_response_text(response) logger.info( @@ -242,7 +464,24 @@ async def _stage_3_with_mcp( f"{executor.call_count} verification calls" ) report, dismissed = _extract_dismissed_issues(content) - return {"report": report, "dismissed_issue_ids": dismissed} + validated = _validated_mcp_dismissals( + dismissed, + issue_by_verification_id, + executor, + review_revision, + ) + return { + "report": report, + "dismissed_issue_ids": [ + str(_safe_issue_field(issue_by_verification_id[key], "id") or "") + for key in validated + if str(_safe_issue_field(issue_by_verification_id[key], "id") or "") + ], + "dismissed_issue_keys": validated, + "dismissed_issue_object_ids": [ + id(issue_by_verification_id[key]) for key in validated + ], + } for tc in tool_calls: tool_result = await executor.execute_tool(tc["name"], tc["args"]) diff --git a/python-ecosystem/inference-orchestrator/src/service/review/prompt_dry_run.py b/python-ecosystem/inference-orchestrator/src/service/review/prompt_dry_run.py index e749f0a5..229e7bdb 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/prompt_dry_run.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/prompt_dry_run.py @@ -713,6 +713,14 @@ async def search_for_duplicates(self, **_: Any) -> list[dict[str, Any]]: async def index_pr_files(self, **_: Any) -> dict[str, Any]: # Report the shape expected by the orchestrator so it follows the same # post-index prompt path, while keeping the operation entirely local. + # When deterministic retrieval is intentionally absent, do not mint + # synthetic exact-generation receipts: Stage 1 would correctly require + # a complete exact retrieval for receipts that claim such a generation. + if not self._enabled: + return { + "status": "skipped", + "reason": "deterministic retrieval disabled for prompt dry run", + } effective = None if self._project_capabilities is not None: from service.review.plugin_context import _plugin_host @@ -733,6 +741,13 @@ async def index_pr_files(self, **_: Any) -> dict[str, Any]: return { "status": "indexed", "chunks_indexed": 0, + "base_generation_manifest_sha256": "0" * 64, + "generation_fingerprint": "sha256:" + "0" * 64, + "overlay_generation_manifest_sha256": "0" * 64, + "plugin_fingerprint": "sha256:" + "0" * 64, + "plugin_descriptor_fingerprint": "sha256:" + "0" * 64, + "plugin_implementation_fingerprint": "sha256:" + "0" * 64, + "index_representation_fingerprint": "sha256:" + "0" * 64, "effective_project_capabilities": effective, } @@ -795,6 +810,8 @@ def capture_event(event: dict[str, Any]) -> None: include_deterministic_rag, safe_request.projectCapabilities, ) + if include_deterministic_rag + else None ) orchestrator = MultiStageReviewOrchestrator( llm=llm, @@ -836,7 +853,11 @@ def capture_event(event: dict[str, Any]) -> None: provider=request.aiProvider, model=request.aiModel, deterministic_rag_requests=( - None if full_pipeline_context else dry_rag.deterministic_requests + None + if full_pipeline_context + else dry_rag.deterministic_requests + if dry_rag is not None + else 0 ), deterministic_rag_enabled=( full_pipeline_rag_enabled diff --git a/python-ecosystem/inference-orchestrator/src/service/review/quality_capture.py b/python-ecosystem/inference-orchestrator/src/service/review/quality_capture.py index 560967ae..afd48752 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/quality_capture.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/quality_capture.py @@ -28,6 +28,9 @@ logger = logging.getLogger(__name__) _SAFE_FILENAME = re.compile(r"[^A-Za-z0-9_.-]+") +_IMMUTABLE_GIT_REVISION = re.compile(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})") +_SHA256_HEX = re.compile(r"[0-9a-f]{64}") +_SHA256_FINGERPRINT = re.compile(r"sha256:[0-9a-f]{64}") _SECRET_KEYS = { "access_token", "accesstoken", @@ -355,6 +358,43 @@ def _selected_project_ids() -> set[int]: return selected +def _provider_reported_models(value: Any) -> list[str]: + """Return model IDs explicitly reported inside a provider callback result.""" + + observed: set[str] = set() + + def inspect_metadata(current: Any) -> None: + if not isinstance(current, dict): + return + for key, child in current.items(): + normalized = str(key).strip().casefold().replace("-", "_") + if normalized in {"model", "model_id", "model_name"}: + if isinstance(child, str) and child.strip(): + observed.add(child.strip()) + elif normalized in { + "llm_output", + "metadata", + "response_metadata", + }: + inspect_metadata(child) + + if isinstance(value, dict): + inspect_metadata(value.get("llm_output")) + generations = value.get("generations") + if isinstance(generations, list): + for group in generations: + if not isinstance(group, list): + continue + for generation in group: + if not isinstance(generation, dict): + continue + inspect_metadata(generation.get("generation_info")) + message = generation.get("message") + if isinstance(message, dict): + inspect_metadata(message.get("response_metadata")) + return sorted(observed) + + class _ProviderBoundaryCallback: """Collect the underlying provider result before structured parsing.""" @@ -380,9 +420,11 @@ def on_llm_new_token(self, *_: Any, **__: Any) -> None: return None def on_llm_end(self, response: Any, **_: Any) -> None: + safe_response = _json_safe(response) self.events.append({ "status": "completed", - "response": _json_safe(response), + "providerReportedModels": _provider_reported_models(safe_response), + "response": safe_response, }) def on_llm_error(self, error: BaseException, **_: Any) -> None: @@ -827,6 +869,150 @@ def _terminal_pipeline_evidence(event: Any) -> Optional[dict[str, Any]]: "terminal pipeline evidence has invalid retrieval.semanticDisabled" ) + revision_binding = event.get("revisionBinding") + if not isinstance(revision_binding, dict): + raise ValueError("terminal pipeline evidence has no revision binding") + pr_indexed = revision_binding.get("prIndexed") + pull_request_id = revision_binding.get("pullRequestId") + target_branch = revision_binding.get("targetBranch") + source_revision = revision_binding.get("sourceRevision") + base_revision = revision_binding.get("baseRevision") + base_manifest = revision_binding.get("baseGenerationManifestSha256") + pr_fingerprint = revision_binding.get("prGenerationFingerprint") + overlay_manifest = revision_binding.get( + "prOverlayGenerationManifestSha256" + ) + base_plugin_fingerprint = revision_binding.get( + "basePluginFingerprint" + ) + base_plugin_descriptor_fingerprint = revision_binding.get( + "basePluginDescriptorFingerprint" + ) + base_plugin_implementation_fingerprint = revision_binding.get( + "basePluginImplementationFingerprint" + ) + base_index_representation_fingerprint = revision_binding.get( + "baseIndexRepresentationFingerprint" + ) + if not isinstance(pr_indexed, bool): + raise ValueError( + "terminal pipeline evidence has invalid revisionBinding.prIndexed" + ) + if ( + pull_request_id is not None + and ( + not isinstance(pull_request_id, int) + or isinstance(pull_request_id, bool) + or pull_request_id < 1 + ) + ): + raise ValueError( + "terminal pipeline evidence has invalid revisionBinding.pullRequestId" + ) + if not isinstance(target_branch, str) or not target_branch.strip(): + raise ValueError( + "terminal pipeline evidence has invalid revisionBinding.targetBranch" + ) + if ( + not isinstance(source_revision, str) + or _IMMUTABLE_GIT_REVISION.fullmatch(source_revision) is None + ): + raise ValueError( + "terminal pipeline evidence has invalid revisionBinding.sourceRevision" + ) + if ( + base_revision is not None + and ( + not isinstance(base_revision, str) + or _IMMUTABLE_GIT_REVISION.fullmatch(base_revision) is None + ) + ): + raise ValueError( + "terminal pipeline evidence has invalid revisionBinding.baseRevision" + ) + if ( + base_manifest is not None + and ( + not isinstance(base_manifest, str) + or _SHA256_HEX.fullmatch(base_manifest) is None + ) + ): + raise ValueError( + "terminal pipeline evidence has invalid " + "revisionBinding.baseGenerationManifestSha256" + ) + if ( + pr_fingerprint is not None + and ( + not isinstance(pr_fingerprint, str) + or _SHA256_FINGERPRINT.fullmatch(pr_fingerprint) is None + ) + ): + raise ValueError( + "terminal pipeline evidence has invalid " + "revisionBinding.prGenerationFingerprint" + ) + if ( + overlay_manifest is not None + and ( + not isinstance(overlay_manifest, str) + or _SHA256_HEX.fullmatch(overlay_manifest) is None + ) + ): + raise ValueError( + "terminal pipeline evidence has invalid " + "revisionBinding.prOverlayGenerationManifestSha256" + ) + for field, value in ( + ("basePluginFingerprint", base_plugin_fingerprint), + ( + "basePluginDescriptorFingerprint", + base_plugin_descriptor_fingerprint, + ), + ( + "basePluginImplementationFingerprint", + base_plugin_implementation_fingerprint, + ), + ( + "baseIndexRepresentationFingerprint", + base_index_representation_fingerprint, + ), + ): + if value is not None and ( + not isinstance(value, str) + or _SHA256_FINGERPRINT.fullmatch(value) is None + ): + raise ValueError( + "terminal pipeline evidence has invalid " + f"revisionBinding.{field}" + ) + if pr_indexed and ( + pull_request_id is None + or base_revision is None + or base_manifest is None + or pr_fingerprint is None + or overlay_manifest is None + or base_plugin_fingerprint is None + or base_plugin_descriptor_fingerprint is None + or base_plugin_implementation_fingerprint is None + or base_index_representation_fingerprint is None + ): + raise ValueError( + "terminal pipeline evidence has incomplete indexed revision binding" + ) + if not pr_indexed and ( + base_manifest is not None + or pr_fingerprint is not None + or overlay_manifest is not None + or base_plugin_fingerprint is not None + or base_plugin_descriptor_fingerprint is not None + or base_plugin_implementation_fingerprint is not None + or base_index_representation_fingerprint is not None + ): + raise ValueError( + "terminal pipeline evidence has receipts for an unindexed PR overlay" + ) + return { "state": "review_evidence_completed", "hunkCoverage": normalized_hunks, @@ -843,6 +1029,26 @@ def _terminal_pipeline_evidence(event: Any) -> Optional[dict[str, Any]]: "semanticDisabled": semantic_disabled, "exactEvidenceIds": exact_evidence_ids, }, + "revisionBinding": { + "prIndexed": pr_indexed, + "pullRequestId": pull_request_id, + "targetBranch": target_branch, + "sourceRevision": source_revision, + "baseRevision": base_revision, + "baseGenerationManifestSha256": base_manifest, + "prGenerationFingerprint": pr_fingerprint, + "prOverlayGenerationManifestSha256": overlay_manifest, + "basePluginFingerprint": base_plugin_fingerprint, + "basePluginDescriptorFingerprint": ( + base_plugin_descriptor_fingerprint + ), + "basePluginImplementationFingerprint": ( + base_plugin_implementation_fingerprint + ), + "baseIndexRepresentationFingerprint": ( + base_index_representation_fingerprint + ), + }, } @@ -1022,6 +1228,55 @@ def _prune(self) -> None: for stale in terminal[self._max_files:]: stale.unlink(missing_ok=True) + def receipt(self) -> dict[str, Any]: + """Return a source-free receipt for the completed capture artifact.""" + + if ( + self._artifact.get("status") not in {"completed", "failed"} + or not self._artifact.get("captureDigest") + ): + raise ValueError("quality capture receipt requires a terminal artifact") + call_receipts = [] + all_reported_models: set[str] = set() + model_evidence_complete = True + for call in self._artifact["calls"]: + reported = sorted({ + model + for event in call.get("providerEvents") or [] + if isinstance(event, dict) + for model in event.get("providerReportedModels") or [] + if isinstance(model, str) and model + }) + all_reported_models.update(reported) + if call.get("status") == "completed" and not reported: + model_evidence_complete = False + call_receipts.append({ + "sequence": call.get("sequence"), + "stage": call.get("stage"), + "status": call.get("status"), + "providerCallCount": call.get("providerCallCount"), + "providerReportedModels": reported, + "promptDigest": call.get("promptDigest"), + "responseDigest": call.get("responseDigest"), + }) + receipt = { + "kind": "review-quality-capture-receipt", + "status": self._artifact["status"], + "artifactContainerPath": self.container_path, + "captureDigest": self._artifact["captureDigest"], + "provider": self._artifact["provider"], + "requestedModel": self._artifact["model"], + "providerReportedModels": sorted(all_reported_models), + "providerModelEvidenceComplete": model_evidence_complete, + "modelBoundaryInvocations": self._artifact[ + "modelBoundaryInvocations" + ], + "providerCalls": self._artifact["providerCalls"], + "calls": call_receipts, + } + receipt["receiptDigest"] = _digest(receipt) + return receipt + async def invoke( self, delegate: Any, @@ -1280,10 +1535,16 @@ def with_structured_output( include_raw=include_raw, **kwargs, ) + bindings = dict(self._bindings) + bindings["structured_output"] = { + "include_raw": include_raw, + "options": dict(kwargs), + } return self._clone( delegate, schema=schema, include_raw=include_raw, + bindings=bindings, ) def bind_tools( @@ -1293,8 +1554,13 @@ def bind_tools( ) -> "ReviewQualityCaptureLLM": materialized = tuple(tools) delegate = self._delegate.bind_tools(materialized, **kwargs) + bindings = dict(self._bindings) + bindings["tool_binding"] = { + "options": dict(kwargs), + } return self._clone( delegate, + bindings=bindings, tools=tuple(_tool_descriptor(tool) for tool in materialized), ) diff --git a/python-ecosystem/inference-orchestrator/src/service/review/review_service.py b/python-ecosystem/inference-orchestrator/src/service/review/review_service.py index 338dab1d..6e384fe6 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/review_service.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/review_service.py @@ -33,6 +33,16 @@ logger = logging.getLogger(__name__) + +def allow_unbound_global_rag_fallback(request: ReviewRequestDto) -> bool: + """Legacy fallback is unsafe when the review carries an exact PR lease.""" + is_exact_pr = bool( + request.pullRequestId + and request.baseCommitHash + and (request.currentCommitHash or request.commitHash) + ) + return not is_exact_pr + class ReviewService: """Service class for handling code review requests with streaming support.""" @@ -40,7 +50,7 @@ class ReviewService: MAX_FIX_RETRIES = 2 # Maximum concurrent reviews (each spawns a JVM subprocess + LLM calls) - MAX_CONCURRENT_REVIEWS = int(os.environ.get("MAX_CONCURRENT_REVIEWS", "4")) + MAX_CONCURRENT_REVIEWS = int(os.environ.get("MAX_CONCURRENT_REVIEWS", "20")) # Hard timeout ceiling per review (seconds). Configurable via .env REVIEW_TIMEOUT_SECONDS = int(os.environ.get("REVIEW_TIMEOUT_SECONDS", "1500")) @@ -103,6 +113,12 @@ async def process_review_request( response, failed=review_response_indicates_failure(response), ) + self._emit_event(review_event_callback, { + "type": "status", + "state": "review_quality_capture_completed", + "message": "Review quality capture completed", + "qualityCapture": quality_capture.receipt(), + }) return response async def _process_prompt_dry_run( @@ -383,7 +399,11 @@ async def _process_review( # task is only awaited if a batch cannot obtain per-batch # context. Branch reconciliation does not need it. request_rag_client = self._rag_client_for_request(request) - if needs_multistage_review and request_rag_client is not None: + if ( + needs_multistage_review + and request_rag_client is not None + and allow_unbound_global_rag_fallback(request) + ): rag_context_task = asyncio.create_task( self._fetch_rag_context( request, diff --git a/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_mcp.py b/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_mcp.py index ccf63f00..51080a80 100644 --- a/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_mcp.py +++ b/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_mcp.py @@ -21,20 +21,29 @@ STAGE_3_MCP_VERIFICATION_SECTION = """ ## Issue Re-verification (Optional) Before producing the final report, you may verify HIGH/CRITICAL issues that seem uncertain -by reading actual file content from the repository. +by reading actual file content from the exact reviewed PR revision. Available tools: -- **getBranchFileContent(branch, filePath)** — Read a file to verify an issue's existence +- **getBranchFileContent(filePath, verificationId)** — Read an anchor-centred + source window; the host supplies the exact reviewed commit - **getPullRequestComments(pullRequestId)** — Read PR comments for additional context RULES: 1. You have a MAXIMUM of {max_calls} verification calls total. 2. Only verify issues you are UNCERTAIN about — do not verify every issue. -3. Focus on HIGH and CRITICAL severity issues. -4. If verification reveals a false positive, note its ID for dismissal. -5. After verification, produce the final executive summary. - -TARGET BRANCH: {target_branch} +3. Prioritize HIGH and CRITICAL severity, but you may verify a lower-severity + finding when its correctness materially affects the final report. +4. Use the Verification ID from the complete verification-record list, including + records whose persisted Original ID is empty. +5. Pass that same Verification ID in every file-content call. The host binds the + returned source window to the finding and verifies that it covers its line. +6. If a finding has related_locations, every affected location must be read with + the same Verification ID before dismissing the consolidated root finding. +7. If verification reveals a false positive, note its Verification ID for dismissal. +8. Missing, failed, partial, or ambiguous evidence means KEEP the finding. +9. After verification, produce the final executive summary. + +REVIEWED REVISION: {review_revision} PR ID: {pr_id} ## False Positive Dismissal @@ -42,11 +51,13 @@ positives, append an HTML comment at the very end of your response with the IDs of issues that should be removed from the issue list: - + RULES for dismissal: -- Only dismiss issues you VERIFIED as false positives via tool calls (read the actual code). +- Only dismiss issues you VERIFIED as false positives via successful file-content + tool calls against the reviewed revision. - Do NOT dismiss issues based on guessing — you must have read the relevant file. -- Architecture observations reported as HIGH severity bugs can be dismissed if they have no runtime impact. +- Do not dismiss a concrete architecture/maintainability defect merely because it + has no immediate runtime crash; verify the claim as written. - If no issues should be dismissed, omit the DISMISSED_ISSUES comment entirely. """ diff --git a/python-ecosystem/inference-orchestrator/src/utils/prompts/prompt_builder.py b/python-ecosystem/inference-orchestrator/src/utils/prompts/prompt_builder.py index bfb84e6b..18f0ca58 100644 --- a/python-ecosystem/inference-orchestrator/src/utils/prompts/prompt_builder.py +++ b/python-ecosystem/inference-orchestrator/src/utils/prompts/prompt_builder.py @@ -333,7 +333,7 @@ def build_stage_3_aggregation_prompt( incremental_context: str = "", task_context: str = "No task context available.", use_mcp_tools: bool = False, - target_branch: str = "", + review_revision: str = "", ) -> str: """ Build prompt for Stage 3: Aggregation & Final Report. @@ -356,12 +356,12 @@ def build_stage_3_aggregation_prompt( ) # Conditionally append MCP verification instructions - if use_mcp_tools and target_branch: + if use_mcp_tools and review_revision: from service.review.orchestrator.mcp_tool_executor import McpToolExecutor max_calls = McpToolExecutor.STAGE_CONFIG["stage_3"]["max_calls"] prompt += STAGE_3_MCP_VERIFICATION_SECTION.format( max_calls=max_calls, - target_branch=target_branch, + review_revision=review_revision, pr_id=pr_id ) diff --git a/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py b/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py index 8fd7d50f..82352e3f 100644 --- a/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py +++ b/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py @@ -9,6 +9,13 @@ SECRET_API_KEY = "dry-run-provider-key-must-never-be-used-or-returned" HEAD_REVISION = "1" * 40 BASE_REVISION = "2" * 40 +BASE_GENERATION_MANIFEST = "3" * 64 +PR_GENERATION_FINGERPRINT = "sha256:" + "4" * 64 +PR_OVERLAY_GENERATION_MANIFEST = "5" * 64 +BASE_PLUGIN_FINGERPRINT = "sha256:" + "6" * 64 +BASE_PLUGIN_DESCRIPTOR_FINGERPRINT = "sha256:" + "7" * 64 +BASE_PLUGIN_IMPLEMENTATION_FINGERPRINT = "sha256:" + "8" * 64 +BASE_INDEX_REPRESENTATION_FINGERPRINT = "sha256:" + "9" * 64 class DeterministicRagSpy: @@ -42,7 +49,25 @@ async def search_for_duplicates(self, **_kwargs): async def index_pr_files(self, **kwargs): self.index_requests.append(kwargs) - return {"status": "indexed", "chunks_indexed": 0} + return { + "status": "indexed", + "chunks_indexed": 0, + "base_generation_manifest_sha256": BASE_GENERATION_MANIFEST, + "generation_fingerprint": PR_GENERATION_FINGERPRINT, + "overlay_generation_manifest_sha256": ( + PR_OVERLAY_GENERATION_MANIFEST + ), + "plugin_fingerprint": BASE_PLUGIN_FINGERPRINT, + "plugin_descriptor_fingerprint": ( + BASE_PLUGIN_DESCRIPTOR_FINGERPRINT + ), + "plugin_implementation_fingerprint": ( + BASE_PLUGIN_IMPLEMENTATION_FINGERPRINT + ), + "index_representation_fingerprint": ( + BASE_INDEX_REPRESENTATION_FINGERPRINT + ), + } async def delete_pr_files(self, **kwargs): self.delete_requests.append(kwargs) diff --git a/python-ecosystem/inference-orchestrator/tests/test_candidate_ledger.py b/python-ecosystem/inference-orchestrator/tests/test_candidate_ledger.py index a9818bcf..c3c30208 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_candidate_ledger.py +++ b/python-ecosystem/inference-orchestrator/tests/test_candidate_ledger.py @@ -264,6 +264,20 @@ def test_terminal_capture_accepts_deterministic_candidate_ledger(): "semanticDisabled": False, "exactEvidenceIds": 0, }, + "revisionBinding": { + "prIndexed": True, + "pullRequestId": 12, + "targetBranch": "main", + "sourceRevision": "a" * 40, + "baseRevision": "b" * 40, + "baseGenerationManifestSha256": "c" * 64, + "prGenerationFingerprint": "sha256:" + "d" * 64, + "prOverlayGenerationManifestSha256": "e" * 64, + "basePluginFingerprint": "sha256:" + "1" * 64, + "basePluginDescriptorFingerprint": "sha256:" + "2" * 64, + "basePluginImplementationFingerprint": "sha256:" + "3" * 64, + "baseIndexRepresentationFingerprint": "sha256:" + "4" * 64, + }, }) assert evidence["candidates"]["generated"] == 1 diff --git a/python-ecosystem/inference-orchestrator/tests/test_inference_policy.py b/python-ecosystem/inference-orchestrator/tests/test_inference_policy.py index 806858a7..9dde882f 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_inference_policy.py +++ b/python-ecosystem/inference-orchestrator/tests/test_inference_policy.py @@ -53,21 +53,26 @@ def _full_profile(): ) -def test_llm_dedup_is_disabled_by_default(): - assert should_use_llm_dedup(_full_profile(), 20) is False +def test_grouped_llm_dedup_is_enabled_by_default(): + assert should_use_llm_dedup(_full_profile(), 20) is True + assert should_use_llm_dedup(_fast_profile(), 2) is True -def test_llm_dedup_requires_explicit_opt_in(monkeypatch): +def test_llm_dedup_can_be_disabled_explicitly(monkeypatch): monkeypatch.setattr( "service.review.orchestrator.inference_policy.LLM_DEDUP_ENABLED", - True, + False, ) - assert should_use_llm_dedup(_full_profile(), 20) is True + assert should_use_llm_dedup(_full_profile(), 20) is False assert should_use_llm_dedup(_full_profile(), 1) is False assert should_use_llm_dedup(_fast_profile(), 2) is False +def test_llm_dedup_skips_single_finding(): + assert should_use_llm_dedup(_full_profile(), 1) is False + + def test_task_context_forces_stage_2_in_fast_check(): request = _request(taskContext={"task_key": "PROJ-123"}) plan = ReviewPlan(analysis_summary="plan", file_groups=[], cross_file_concerns=[]) diff --git a/python-ecosystem/inference-orchestrator/tests/test_mcp_tool_executor.py b/python-ecosystem/inference-orchestrator/tests/test_mcp_tool_executor.py index 7b27c8d3..5810b538 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_mcp_tool_executor.py +++ b/python-ecosystem/inference-orchestrator/tests/test_mcp_tool_executor.py @@ -5,6 +5,7 @@ import pytest from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock +from model.output_schemas import CodeReviewIssue from service.review.orchestrator.mcp_tool_executor import McpToolExecutor @@ -60,6 +61,9 @@ async def test_successful_call(self): result = await e.execute_tool("getBranchFileContent", {"filePath": "a.py", "branch": "main"}) assert result == "file content here" assert e.call_count == 1 + assert e.call_log[0]["result_chars"] == len("file content here") + assert e.call_log[0]["evidence_valid"] is True + assert e.call_log[0]["evidence_complete_file"] is True @pytest.mark.asyncio(loop_scope="function") async def test_call_failure(self): @@ -71,6 +75,23 @@ async def test_call_failure(self): assert len(e.call_log) == 1 assert e.call_log[0]["success"] is False + @pytest.mark.asyncio(loop_scope="function") + async def test_mcp_error_result_is_not_valid_file_evidence(self): + mock_client = MagicMock() + mock_client.session.call_tool = AsyncMock(return_value=SimpleNamespace( + content=[SimpleNamespace(text="Error executing tool: unavailable")], + isError=True, + )) + e = McpToolExecutor(mock_client, _make_request(), "stage_1") + + await e.execute_tool( + "getBranchFileContent", + {"filePath": "a.py", "branch": "main"}, + ) + + assert e.call_log[0]["success"] is False + assert e.call_log[0]["evidence_valid"] is False + @pytest.mark.asyncio(loop_scope="function") async def test_prefills_workspace(self): mock_client = MagicMock() @@ -83,6 +104,133 @@ async def test_prefills_workspace(self): assert call_args[1]["workspace"] == "ws" assert call_args[1]["repoSlug"] == "repo" + @pytest.mark.asyncio(loop_scope="function") + async def test_stage_3_pins_file_reads_to_reviewed_revision(self): + mock_client = MagicMock() + mock_client.session.call_tool = AsyncMock( + return_value=SimpleNamespace(content=[SimpleNamespace(text="source")]) + ) + e = McpToolExecutor( + mock_client, + _make_request(), + "stage_3", + review_revision="commit-abc", + ) + + await e.execute_tool( + "getBranchFileContent", + {"filePath": "a.py", "branch": "main"}, + ) + + call_args = mock_client.session.call_tool.call_args[0][1] + assert call_args["branch"] == "commit-abc" + assert e.call_log[0]["args"]["branch"] == "commit-abc" + + @pytest.mark.asyncio(loop_scope="function") + async def test_stage_3_requests_window_around_bound_finding_anchor(self): + issue = CodeReviewIssue( + file="src/a.py", line=500, severity="HIGH", category="BUG_RISK", + reason="Concrete defect.", suggestedFixDescription="Fix it.", + ) + mock_client = MagicMock() + mock_client.session.call_tool = AsyncMock( + return_value=SimpleNamespace(content=[SimpleNamespace( + text=( + '{"fileContent":"source","startLine":420,' + '"endLine":580,"totalLines":1000,"completeFile":false}' + ) + )]) + ) + e = McpToolExecutor( + mock_client, + _make_request(), + "stage_3", + review_revision="commit-abc", + verification_issues={"issue_0": issue}, + ) + + await e.execute_tool("getBranchFileContent", { + "filePath": "src/a.py", + "branch": "main", + "verificationId": "issue_0", + }) + + call_args = mock_client.session.call_tool.await_args.args[1] + assert call_args["startLine"] == 420 + assert call_args["endLine"] == 580 + assert e.call_log[0]["evidence_valid"] is True + assert e.call_log[0]["evidence_structured"] is True + assert e.call_log[0]["evidence_start_line"] == 420 + assert e.call_log[0]["evidence_end_line"] == 580 + + @pytest.mark.asyncio(loop_scope="function") + async def test_stage_3_raw_adapter_response_is_bound_to_requested_window(self): + issue = CodeReviewIssue( + file="src/a.py", line=500, severity="HIGH", category="BUG_RISK", + reason="Concrete defect.", suggestedFixDescription="Fix it.", + ) + mock_client = MagicMock() + mock_client.session.call_tool = AsyncMock( + return_value=SimpleNamespace( + content=[SimpleNamespace(text="raw source window")] + ) + ) + e = McpToolExecutor( + mock_client, + _make_request(), + "stage_3", + review_revision="commit-abc", + verification_issues={"issue_0": issue}, + ) + + await e.execute_tool("getBranchFileContent", { + "filePath": "src/a.py", + "branch": "main", + "verificationId": "issue_0", + }) + + assert e.call_log[0]["evidence_valid"] is True + assert e.call_log[0]["evidence_structured"] is False + assert e.call_log[0]["evidence_complete_file"] is False + assert e.call_log[0]["evidence_start_line"] == 420 + assert e.call_log[0]["evidence_end_line"] == 580 + + @pytest.mark.asyncio(loop_scope="function") + @pytest.mark.parametrize("tool_text", [ + '{"error":"file not found"}', + 'Error executing tool: permission denied', + ( + '{"fileContent":"[CodeCrow Filter: file too large, omitted]",' + '"completeFile":false}' + ), + '{"fileContent":"","completeFile":true}', + ]) + async def test_error_empty_and_filtered_results_are_not_source_evidence( + self, + tool_text, + ): + mock_client = MagicMock() + mock_client.session.call_tool = AsyncMock( + return_value=SimpleNamespace( + content=[SimpleNamespace(text=tool_text)] + ) + ) + e = McpToolExecutor( + mock_client, + _make_request(), + "stage_3", + review_revision="commit-abc", + ) + + await e.execute_tool("getBranchFileContent", { + "filePath": "src/a.py", + "branch": "commit-abc", + "verificationId": "issue_0", + }) + + assert e.call_log[0]["success"] is True + assert e.call_log[0]["evidence_valid"] is False + # ── get_tool_definitions ───────────────────────────────────── @@ -99,6 +247,31 @@ def test_stage_3_definitions(self): names = {d["function"]["name"] for d in defs} assert "getBranchFileContent" in names assert "getPullRequestComments" in names + file_tool = next( + definition for definition in defs + if definition["function"]["name"] == "getBranchFileContent" + ) + assert "verificationId" in file_tool["function"]["parameters"]["required"] + assert "branch" not in file_tool["function"]["parameters"]["required"] + + def test_related_location_in_reason_drives_a_bound_source_window(self): + issue = CodeReviewIssue( + file="src/a.py", line=10, severity="HIGH", category="BUG_RISK", + reason=( + "One root cause.\n\n" + "Also affects: src/b.py:700, src/c.py:900" + ), + suggestedFixDescription="Fix it.", + ) + executor = McpToolExecutor( + MagicMock(), _make_request(), "stage_3", + review_revision="commit-abc", + verification_issues={"issue_0": issue}, + ) + + assert executor._verification_line_for_path( + "issue_0", "src/b.py" + ) == 700 # ── Properties ─────────────────────────────────────────────── diff --git a/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py index 496e87c3..5e0bf4e7 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py @@ -34,6 +34,7 @@ ) from model.output_schemas import CodeReviewIssue from model.dtos import ReviewRequestDto +from service.review.candidate_ledger import CandidateEvidenceLedger @pytest.fixture @@ -703,6 +704,52 @@ def test_protected_history_suppresses_exact_anchored_duplicate(self): assert retained_fresh == [] + def test_superseded_history_close_copy_does_not_republish_rejected_candidate(self): + retained_history = CodeReviewIssue( + id="100", file="a.py", line=10, severity="MEDIUM", + category="BUG_RISK", title="Missing workspace guard", + reason="The update path does not validate workspace ownership.", + suggestedFixDescription="Add the ownership guard.", + ) + superseded_history = CodeReviewIssue( + id="200", file="a.py", line=11, severity="HIGH", + category="SECURITY", title="Missing workspace guard", + reason="The update path does not validate workspace ownership.", + suggestedFixDescription="Add the ownership guard.", + ) + ledger = CandidateEvidenceLedger() + ledger.register( + superseded_history, + stage="stage_1", + source_key="batch-1", + review_unit_ids=["unit-1"], + prompt_hunk_ids=["hunk-1"], + prompt_digest="sha256:" + ("0" * 64), + ) + before = [retained_history, superseded_history] + + result = _deduplicate_cross_batch_issues_preserving_lifecycle( + before, + {"100", "200"}, + ) + + active, resolved = _partition_issue_lifecycle(result) + assert len(active) == 1 + assert active[0].id == "100" + assert len(resolved) == 1 + assert resolved[0].id == "200" + assert resolved[0] is not superseded_history + assert superseded_history.isResolved is False + + ledger.reject_removed( + before, + result, + gate="deduplication", + code="cross_batch_duplicate", + ) + ledger.publish(result) + ledger.assert_terminal() + class TestSerializeIssueForClient: def test_active_issue_does_not_serialize_resolution_metadata(self): @@ -795,3 +842,50 @@ def test_resolves_historical_open_issue_and_drops_fresh_candidate(self): assert historical.resolutionExplanation == historical.resolutionReason assert resolved_count == 1 assert dropped_count == 1 + + def test_object_identity_drops_verified_fresh_issue_without_database_id(self): + fresh = CodeReviewIssue( + file="a.py", line=10, severity="MEDIUM", category="BUG_RISK", + reason="Fresh false positive.", suggestedFixDescription="No fix.", + ) + unaffected = CodeReviewIssue( + file="a.py", line=20, severity="MEDIUM", category="BUG_RISK", + reason="Independent real issue.", suggestedFixDescription="Fix it.", + ) + + retained, resolved_count, dropped_count = _apply_stage_3_dismissals( + [fresh, unaffected], + set(), + set(), + dismissed_object_ids={id(fresh)}, + ) + + assert retained == [unaffected] + assert resolved_count == 0 + assert dropped_count == 1 + + def test_object_identity_does_not_touch_resolved_record_with_same_id(self): + active = CodeReviewIssue( + id="12524", file="a.py", line=10, severity="MEDIUM", + category="BUG_RISK", reason="Unsupported active finding.", + suggestedFixDescription="No fix.", + ) + prior_resolution = CodeReviewIssue( + id="12524", file="a.py", line=10, severity="INFO", + category="BUG_RISK", reason="Historical lifecycle update.", + suggestedFixDescription="Already fixed.", isResolved=True, + resolutionReason="Previously closed.", + ) + + retained, resolved_count, dropped_count = _apply_stage_3_dismissals( + [active, prior_resolution], + {"12524"}, + {"12524"}, + dismissed_object_ids={id(active)}, + ) + + assert retained == [active, prior_resolution] + assert active.isResolved is True + assert prior_resolution.resolutionReason == "Previously closed." + assert resolved_count == 1 + assert dropped_count == 0 diff --git a/python-ecosystem/inference-orchestrator/tests/test_prompt_builder.py b/python-ecosystem/inference-orchestrator/tests/test_prompt_builder.py index e6736ff9..310ae213 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_prompt_builder.py +++ b/python-ecosystem/inference-orchestrator/tests/test_prompt_builder.py @@ -320,3 +320,16 @@ def test_with_task_context(self): ) assert "PROJ-3" in result assert "task-coverage" in result + + def test_mcp_verification_uses_reviewed_revision_and_verification_ids(self): + result = PromptBuilder.build_stage_3_aggregation_prompt( + repo_slug="r", pr_id="7", author="d", pr_title="T", + total_files=1, additions=1, deletions=0, + stage_0_plan="p", stage_1_issues_json="[]", + stage_2_findings_json="[]", recommendation="APPROVE", + use_mcp_tools=True, review_revision="commit-abc", + ) + + assert "REVIEWED REVISION: commit-abc" in result + assert "Verification ID" in result + assert 'DISMISSED_ISSUES: ["issue_0", "issue_3"]' in result diff --git a/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py b/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py index 32bb748e..b834b212 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py +++ b/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py @@ -50,10 +50,12 @@ async def test_non_reviewable_hunk_manifest_completes_without_model_stage(): for hunk in generated.hunks ] rag = DeterministicRagSpy() + events = [] orchestrator = MultiStageReviewOrchestrator( llm=object(), mcp_client=None, rag_client=rag, + event_callback=events.append, ) result = await orchestrator.orchestrate_review( @@ -66,6 +68,16 @@ async def test_non_reviewable_hunk_manifest_completes_without_model_stage(): assert rag.index_requests == [] assert rag.requests == [] assert rag.semantic_requests == [] + terminal_event = next( + event + for event in events + if event.get("state") == "review_evidence_completed" + ) + assert terminal_event["revisionBinding"]["prIndexed"] is False + assert ( + terminal_event["revisionBinding"]["baseGenerationManifestSha256"] + is None + ) @pytest.mark.asyncio @@ -796,6 +808,25 @@ async def test_full_pipeline_capture_persists_real_context_artifact( event.get("state") == "review_evidence_completed" for event in forwarded_events ) + terminal_event = next( + event + for event in forwarded_events + if event.get("state") == "review_evidence_completed" + ) + assert terminal_event["revisionBinding"] == { + "prIndexed": True, + "pullRequestId": request.pullRequestId, + "targetBranch": "main", + "sourceRevision": HEAD_REVISION, + "baseRevision": BASE_REVISION, + "baseGenerationManifestSha256": "3" * 64, + "prGenerationFingerprint": "sha256:" + "4" * 64, + "prOverlayGenerationManifestSha256": "5" * 64, + "basePluginFingerprint": "sha256:" + "6" * 64, + "basePluginDescriptorFingerprint": "sha256:" + "7" * 64, + "basePluginImplementationFingerprint": "sha256:" + "8" * 64, + "baseIndexRepresentationFingerprint": "sha256:" + "9" * 64, + } assert report["reviewIdentity"]["targetBranch"] == "main" assert report["reviewIdentity"]["sourceBranch"] == "feature/dry-run" assert report["reviewIdentity"]["headRevision"] == HEAD_REVISION @@ -806,6 +837,17 @@ async def test_full_pipeline_capture_persists_real_context_artifact( assert rag.index_requests assert rag.index_requests[0]["source_revision"] == HEAD_REVISION assert rag.index_requests[0]["base_revision"] == BASE_REVISION + assert rag.requests + assert all( + request["source_revision"] == HEAD_REVISION + and request["base_revision"] == BASE_REVISION + and request["base_generation_manifest_sha256"] == "3" * 64 + and request["pr_generation_fingerprint"] + == "sha256:" + "4" * 64 + and request["pr_overlay_generation_manifest_sha256"] + == "5" * 64 + for request in rag.requests + ) assert { file_info["content_state"] for file_info in rag.index_requests[0]["files"] @@ -1013,6 +1055,13 @@ async def index_with_groups(**kwargs): return { "status": "indexed", "chunks_indexed": 4, + "base_generation_manifest_sha256": "3" * 64, + "generation_fingerprint": "sha256:" + "4" * 64, + "overlay_generation_manifest_sha256": "5" * 64, + "plugin_fingerprint": "sha256:" + "6" * 64, + "plugin_descriptor_fingerprint": "sha256:" + "7" * 64, + "plugin_implementation_fingerprint": "sha256:" + "8" * 64, + "index_representation_fingerprint": "sha256:" + "9" * 64, "review_groups": [ ["src/file_0.py", "src/file_1.py"], ["src/file_1.py", "src/file_2.py"], diff --git a/python-ecosystem/inference-orchestrator/tests/test_quality_capture.py b/python-ecosystem/inference-orchestrator/tests/test_quality_capture.py index 0f3431cc..f3d99f0b 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_quality_capture.py +++ b/python-ecosystem/inference-orchestrator/tests/test_quality_capture.py @@ -48,6 +48,23 @@ def _request(**updates): return ReviewRequestDto(**values) +def _revision_binding(): + return { + "prIndexed": True, + "pullRequestId": 12, + "targetBranch": "main", + "sourceRevision": "b" * 40, + "baseRevision": "a" * 40, + "baseGenerationManifestSha256": "c" * 64, + "prGenerationFingerprint": "sha256:" + "d" * 64, + "prOverlayGenerationManifestSha256": "e" * 64, + "basePluginFingerprint": "sha256:" + "1" * 64, + "basePluginDescriptorFingerprint": "sha256:" + "2" * 64, + "basePluginImplementationFingerprint": "sha256:" + "3" * 64, + "baseIndexRepresentationFingerprint": "sha256:" + "4" * 64, + } + + class _FakeDelegate: def __init__(self): self.calls = [] @@ -82,6 +99,9 @@ async def ainvoke(self, input_data, **kwargs): "output_tokens": 2, }, }]], + "llm_output": { + "model_name": "provider-resolved-review-model", + }, }) return { "answer": "captured", @@ -114,6 +134,26 @@ def test_non_allowlisted_project_is_not_captured(capture_environment): assert list(capture_environment.iterdir()) == [] +def test_provider_model_identity_uses_metadata_not_tool_arguments(): + response = { + "llm_output": {"model_name": "provider/resolved"}, + "generations": [[{ + "message": { + "tool_calls": [{ + "args": {"model": "domain-object-name"}, + }], + "response_metadata": { + "model": "provider/resolved", + }, + }, + }]], + } + + assert quality_capture._provider_reported_models(response) == [ + "provider/resolved" + ] + + @pytest.mark.asyncio async def test_capture_records_exact_model_boundary_and_redacts_credentials( capture_environment, @@ -125,7 +165,12 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( event_callback = session.wrap_event_callback(forwarded_events.append) capped = llm.model_copy(update={"max_tokens": 321}) - structured = capped.with_structured_output(dict, include_raw=True) + structured = capped.with_structured_output( + dict, + include_raw=True, + method="json_schema", + strict=True, + ) result = await structured.ainvoke("Review proprietary_source safely") event_callback({ "type": "status", @@ -161,6 +206,7 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( "semanticDisabled": False, "exactEvidenceIds": 2, }, + "revisionBinding": _revision_binding(), }) response = {"result": {"issues": [{"file": "private.py", "title": "Example"}]}} await session.complete(response) @@ -179,6 +225,9 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( "registered": 1, "completed": 1, } + assert artifact["pipelineEvidence"]["revisionBinding"] == ( + _revision_binding() + ) assert artifact["pipelineEvidenceDigest"] assert forwarded_events[0]["state"] == "review_evidence_completed" assert artifact["calls"][0]["status"] == "completed" @@ -186,9 +235,19 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( "Review proprietary_source safely" ) assert artifact["calls"][0]["modelBindings"]["max_tokens"] == 321 + assert artifact["calls"][0]["modelBindings"]["structured_output"] == { + "include_raw": True, + "options": { + "method": "json_schema", + "strict": True, + }, + } assert artifact["calls"][0]["response"]["answer"] == "captured" assert artifact["calls"][0]["providerCallCountSource"] == "callback" assert artifact["calls"][0]["providerEvents"][0]["response"]["generations"] + assert artifact["calls"][0]["providerEvents"][0][ + "providerReportedModels" + ] == ["provider-resolved-review-model"] assert artifact["request"]["rawDiff"].endswith("proprietary_source = True") assert artifact["request"]["aiCustomParameters"]["temperature"] == 0.2 assert artifact["request"]["aiCustomParameters"]["default_headers"]["X-Trace"] == ( @@ -209,6 +268,14 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( assert len(artifact["modeIdentity"]) == 64 assert artifact["captureDigest"] assert artifact["resultDigest"] + receipt = session.receipt() + assert receipt["requestedModel"] == "review-model" + assert receipt["providerReportedModels"] == [ + "provider-resolved-review-model" + ] + assert receipt["providerModelEvidenceComplete"] is True + assert receipt["calls"][0]["stage"] + assert receipt["receiptDigest"] assert "provider-secret" not in serialized assert "header-secret" not in serialized assert "oauth-secret" not in serialized @@ -216,6 +283,35 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( assert (os.stat(session.path).st_mode & 0o777) == 0o600 +@pytest.mark.asyncio +async def test_capture_records_tool_binding_options(capture_environment): + session = create_quality_capture_session(_request()) + delegate = _FakeDelegate() + llm = ReviewQualityCaptureLLM(delegate, session) + + bound = llm.bind_tools( + [{"name": "lookup", "description": "Read source evidence"}], + tool_choice="required", + strict=True, + ) + await bound.ainvoke("Use the declared evidence tool") + await session.complete({"result": {"issues": []}}) + + artifact = json.loads(session.path.read_text(encoding="utf-8")) + assert artifact["calls"][0]["modelBindings"]["tool_binding"] == { + "options": { + "strict": True, + "tool_choice": "required", + }, + } + assert artifact["calls"][0]["tools"] == [ + { + "description": "Read source evidence", + "name": "lookup", + }, + ] + + @pytest.mark.asyncio async def test_capture_marks_missing_or_invalid_terminal_pipeline_evidence( capture_environment, @@ -324,6 +420,12 @@ def wrapped(event): async def complete(self, response, error=None, *, failed=False): completed.append((response, error, failed)) + def receipt(self): + return { + "kind": "review-quality-capture-receipt", + "receiptDigest": "a" * 64, + } + capture = CaptureSpy() async def fake_review( @@ -352,8 +454,21 @@ async def fake_review( ) assert response == {"result": {"issues": []}} - assert observed == [{"state": "review_evidence_completed"}] - assert forwarded == observed + assert observed[0] == {"state": "review_evidence_completed"} + assert observed[1]["state"] == "review_quality_capture_completed" + assert observed[1]["qualityCapture"]["receiptDigest"] == "a" * 64 + assert forwarded == [ + {"state": "review_evidence_completed"}, + { + "type": "status", + "state": "review_quality_capture_completed", + "message": "Review quality capture completed", + "qualityCapture": { + "kind": "review-quality-capture-receipt", + "receiptDigest": "a" * 64, + }, + }, + ] assert completed == [(response, None, False)] diff --git a/python-ecosystem/inference-orchestrator/tests/test_rag_client.py b/python-ecosystem/inference-orchestrator/tests/test_rag_client.py index cbf2d7ae..510cac40 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_rag_client.py +++ b/python-ecosystem/inference-orchestrator/tests/test_rag_client.py @@ -169,6 +169,22 @@ async def test_delete_pr_files_ok(self): assert await c.delete_pr_files("ws", "proj", 1) is True await c.close() + @pytest.mark.asyncio(loop_scope="function") + @respx.mock + async def test_delete_pr_files_uses_exact_generation_target(self): + route = respx.delete("http://rag:8001/index/pr-files/ws/proj/1").mock( + return_value=httpx.Response(200, json={"status": "deleted"}) + ) + c = RagClient(base_url="http://rag:8001", enabled=True) + + assert await c.delete_pr_files( + "ws", "proj", 1, collection_target="cc_w1_p2_branch_generation" + ) is True + assert route.calls.last.request.url.params[ + "collection_target" + ] == "cc_w1_p2_branch_generation" + await c.close() + # ── Error handling ─────────────────────────────────────────── diff --git a/python-ecosystem/inference-orchestrator/tests/test_rag_client_duplication_unit.py b/python-ecosystem/inference-orchestrator/tests/test_rag_client_duplication_unit.py index 573161f0..997d0757 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_rag_client_duplication_unit.py +++ b/python-ecosystem/inference-orchestrator/tests/test_rag_client_duplication_unit.py @@ -3,7 +3,7 @@ import pytest -from service.rag.rag_client import RagClient +from service.rag.rag_client import RagClient, RagRetrievalError class _FakeResponse: @@ -59,6 +59,27 @@ async def get_client(): assert time.perf_counter() - started < 0.5 +@pytest.mark.asyncio(loop_scope="function") +async def test_revision_bound_duplication_timeout_is_propagated(monkeypatch): + monkeypatch.setenv("REVIEW_DUPLICATION_RAG_QUERY_TIMEOUT_SECONDS", "0.1") + client = RagClient(base_url="http://rag", enabled=True) + + async def get_client(): + return _SlowSearchClient() + + client._get_client = get_client + + with pytest.raises(RagRetrievalError, match="timed out"): + await client.search_for_duplicates( + workspace="ws", + project="proj", + branch="main", + queries=["find duplicate implementation"], + repository_revision="a" * 40, + repository_generation_manifest_sha256="b" * 64, + ) + + @pytest.mark.asyncio(loop_scope="function") async def test_duplication_search_runs_queries_concurrently(monkeypatch): monkeypatch.setenv("REVIEW_DUPLICATION_RAG_QUERY_TIMEOUT_SECONDS", "1") diff --git a/python-ecosystem/inference-orchestrator/tests/test_reconciliation.py b/python-ecosystem/inference-orchestrator/tests/test_reconciliation.py index 149a8d2a..f756a44f 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_reconciliation.py +++ b/python-ecosystem/inference-orchestrator/tests/test_reconciliation.py @@ -15,6 +15,7 @@ format_previous_issues_for_batch, deduplicate_final_issues, deduplicate_cross_batch_issues, + issues_are_semantic_dedup_candidates, _build_batches, reconcile_previous_issues, ) @@ -302,7 +303,54 @@ def test_same_source_snippet_survives_line_drift_and_deduplicates(self): assert len(deduplicate_final_issues(issues)) == 1 - def test_exact_plugin_proof_deduplicates_across_prose_and_anchors(self): + def test_exact_history_recreation_keeps_identity_and_refreshes_anchor(self): + historical = _make_issue( + id="3989", + file="src/service.py", + line=1, + title="Unbounded retry loop can exhaust workers", + category="CODE_QUALITY", + severity="MEDIUM", + reason="The retry loop has no terminal attempt limit.", + codeSnippet="class RetryService:", + ) + recreated = _make_issue( + file="src/service.py", + line=1233, + title="Unbounded retry loop can exhaust workers", + category="BUG_RISK", + severity="HIGH", + reason="The retry loop has no terminal attempt limit.", + codeSnippet="while should_retry(response):", + ) + + result = deduplicate_final_issues([historical, recreated]) + + assert len(result) == 1 + assert result[0].id == "3989" + assert result[0].line == 1233 + assert result[0].codeSnippet == "while should_retry(response):" + assert result[0].severity == "HIGH" + + def test_distinct_historical_duplicates_keep_both_concrete_locations(self): + first = _make_issue( + id="3989", file="src/service.py", line=10, + title="Unbounded retry loop can exhaust workers", + reason="The retry loop has no terminal attempt limit.", + ) + second = _make_issue( + id="3990", file="src/service.py", line=20, + title="Unbounded retry loop can exhaust workers", + reason="The retry loop has no terminal attempt limit.", + ) + + result = deduplicate_final_issues([second, first]) + + assert len(result) == 1 + assert result[0].id == "3989" + assert result[0].relatedLocations == ["src/service.py:20"] + + def test_exact_plugin_proof_is_a_semantic_candidate_not_a_delete_key(self): issues = [ _make_issue( file="app/code/Vendor/Module/etc/di.xml", @@ -324,7 +372,8 @@ def test_exact_plugin_proof_deduplicates_across_prose_and_anchors(self): ), ] - assert deduplicate_final_issues(issues) == [issues[0]] + assert deduplicate_final_issues(issues) == issues + assert issues_are_semantic_dedup_candidates(issues[0], issues[1]) def test_distinct_plugin_proofs_remain_distinct(self): issues = [ diff --git a/python-ecosystem/inference-orchestrator/tests/test_reconciliation_full.py b/python-ecosystem/inference-orchestrator/tests/test_reconciliation_full.py index e69c352d..991a1763 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_reconciliation_full.py +++ b/python-ecosystem/inference-orchestrator/tests/test_reconciliation_full.py @@ -1,12 +1,20 @@ """Extended tests for reconciliation: _format_issues_for_prompt, _build_batches, _dedup_batch_with_llm.""" import pytest from unittest.mock import MagicMock, AsyncMock, patch +from model.output_schemas import ( + CodeReviewIssue, + SemanticDeduplicationDecision, + SemanticDuplicateGroup, +) from service.review.orchestrator.reconciliation import ( _format_issues_for_prompt, _build_batches, + _build_semantic_dedup_batches, _dedup_batch_with_llm, + _semantic_candidate_groups, deduplicate_final_issues_llm, deduplicate_final_issues, + issues_are_semantic_dedup_candidates, ) @@ -30,6 +38,22 @@ def _make_issue(file="a.py", line=10, severity="HIGH", category="BUG_RISK", return issue +def _real_issue(file="a.py", line=10, severity="HIGH", category="BUG_RISK", + title="Issue", reason="Something wrong", issue_id=None, + code_snippet=""): + return CodeReviewIssue( + id=issue_id, + file=file, + line=line, + severity=severity, + category=category, + title=title, + reason=reason, + suggestedFixDescription="Fix the root cause.", + codeSnippet=code_snippet, + ) + + # ── _format_issues_for_prompt ───────────────────────────────── @@ -86,32 +110,91 @@ def test_empty(self): class TestDedupBatchWithLlm: @pytest.mark.asyncio(loop_scope="function") - async def test_keeps_selected_indices(self): - from model.output_schemas import DeduplicatedIssueList + async def test_merges_only_explicit_high_confidence_group(self): llm = MagicMock() structured = MagicMock() structured.ainvoke = AsyncMock( - return_value=DeduplicatedIssueList(kept_indices=[0, 2]) + return_value=SemanticDeduplicationDecision(duplicate_groups=[ + SemanticDuplicateGroup( + keeper_index=0, + duplicate_indices=[1], + confidence="HIGH", + rationale="One missing authorization guard.", + ) + ]) ) llm.with_structured_output.return_value = structured - issues = [_make_issue(file=f"f{i}.py") for i in range(3)] - result = await _dedup_batch_with_llm(llm, issues) - assert len(result) == 2 + issues = [ + _real_issue( + line=10, + title="Authorization guard missing from update path", + reason="The update path writes account data without checking the workspace role.", + ), + _real_issue( + line=40, + title="Authorization guard missing from update path", + reason="Account data is written by this update path before the workspace role is checked.", + ), + ] + result = await _dedup_batch_with_llm( + llm, + issues, + {0: "candidate_0", 1: "candidate_0"}, + ) + assert len(result) == 1 + assert result[0].relatedLocations == ["a.py:40"] @pytest.mark.asyncio(loop_scope="function") async def test_invalid_indices_keeps_all(self): - from model.output_schemas import DeduplicatedIssueList llm = MagicMock() structured = MagicMock() structured.ainvoke = AsyncMock( - return_value=DeduplicatedIssueList(kept_indices=[99]) + return_value=SemanticDeduplicationDecision(duplicate_groups=[ + SemanticDuplicateGroup( + keeper_index=0, + duplicate_indices=[99], + confidence="HIGH", + rationale="Malformed index.", + ) + ]) ) llm.with_structured_output.return_value = structured - issues = [_make_issue() for _ in range(2)] + issues = [ + _real_issue(line=10, reason="First independent problem."), + _real_issue(line=20, reason="Second independent problem."), + ] + result = await _dedup_batch_with_llm( + llm, + issues, + {0: "candidate_0", 1: "candidate_0"}, + ) + assert result == issues + + @pytest.mark.asyncio(loop_scope="function") + async def test_uncertain_decision_keeps_all(self): + llm = MagicMock() + structured = MagicMock() + structured.ainvoke = AsyncMock( + return_value=SemanticDeduplicationDecision(duplicate_groups=[ + SemanticDuplicateGroup( + keeper_index=0, + duplicate_indices=[1], + confidence="MEDIUM", + rationale="Possibly related.", + ) + ]) + ) + llm.with_structured_output.return_value = structured + issues = [ + _real_issue(line=10, reason="First problem."), + _real_issue(line=20, reason="Second problem."), + ] + result = await _dedup_batch_with_llm(llm, issues) - assert len(result) == 2 # All kept as fallback + + assert result == issues @pytest.mark.asyncio(loop_scope="function") async def test_exception_falls_back(self): @@ -120,10 +203,12 @@ async def test_exception_falls_back(self): structured.ainvoke = AsyncMock(side_effect=Exception("fail")) llm.with_structured_output.return_value = structured - issues = [_make_issue() for _ in range(2)] + issues = [ + _real_issue(line=10, reason="First independent problem."), + _real_issue(line=20, reason="Second independent problem."), + ] result = await _dedup_batch_with_llm(llm, issues) - # Falls back to algorithmic dedup - assert len(result) >= 1 + assert result == issues # ── deduplicate_final_issues ────────────────────────────────── @@ -152,12 +237,112 @@ def test_exact_duplicates(self): class TestDeduplicateFinalIssuesLlm: @pytest.mark.asyncio(loop_scope="function") - async def test_small_set_uses_algorithmic(self): - # Less than the batch threshold → should just use algorithmic - issues = [_make_issue(file=f"f{i}.py") for i in range(2)] + async def test_non_candidates_do_not_use_model_tokens(self): + issues = [ + _real_issue(file="a.py", title="SQL injection", reason="Raw SQL uses user input."), + _real_issue(file="b.py", title="Cache stampede", reason="Cache misses fan out."), + ] llm = MagicMock() result = await deduplicate_final_issues_llm(llm, issues) - assert len(result) >= 1 + assert result == issues + llm.with_structured_output.assert_not_called() + + @pytest.mark.asyncio(loop_scope="function") + async def test_sends_complete_candidate_evidence_but_not_singletons(self): + long_evidence = "Evidence marker " + ("complete-context " * 900) + candidate_a = _real_issue( + line=10, + title="Workspace authorization is missing from account update", + reason=long_evidence + "role check is absent", + ) + candidate_b = _real_issue( + line=50, + title="Workspace authorization is missing from account update", + reason=long_evidence + "role validation is absent", + ) + singleton = _real_issue( + file="different.py", + title="Independent resource leak", + reason="SINGLETON-MUST-NOT-BE-SENT", + ) + llm = MagicMock() + structured = MagicMock() + structured.ainvoke = AsyncMock( + return_value=SemanticDeduplicationDecision(duplicate_groups=[]) + ) + llm.with_structured_output.return_value = structured + + result = await deduplicate_final_issues_llm( + llm, + [candidate_a, singleton, candidate_b], + ) + + assert result == [candidate_a, singleton, candidate_b] + prompt = structured.ainvoke.await_args.args[0] + assert "complete-context" in prompt + assert "role validation is absent" in prompt + assert "SINGLETON-MUST-NOT-BE-SENT" not in prompt + + def test_same_anchor_independent_roots_are_not_candidates(self): + authorization = _real_issue( + line=25, + title="Request processing defect", + reason="The caller can update another workspace because ownership is never checked.", + code_snippet="service.update(request)", + ) + transaction = _real_issue( + line=25, + title="Request processing defect", + reason="The database transaction is committed before the downstream write completes.", + code_snippet="service.update(request)", + ) + + assert not issues_are_semantic_dedup_candidates( + authorization, + transaction, + ) + + @pytest.mark.asyncio(loop_scope="function") + async def test_cross_file_shared_root_keeps_both_locations(self): + first = _real_issue( + file="config/RagConfig.java", + line=30, + title="RAG constructor arity no longer matches shared configuration", + reason=( + "The shared RAG configuration now requires four constructor " + "arguments, but this caller supplies three and cannot compile." + ), + ) + second = _real_issue( + file="test/RagConfigTest.java", + line=70, + title="RAG constructor arity no longer matches shared configuration", + reason=( + "The shared RAG configuration requires four constructor arguments; " + "this test still supplies three and cannot compile." + ), + ) + assert issues_are_semantic_dedup_candidates(first, second) + llm = MagicMock() + structured = MagicMock() + structured.ainvoke = AsyncMock( + return_value=SemanticDeduplicationDecision(duplicate_groups=[ + SemanticDuplicateGroup( + keeper_index=0, + duplicate_indices=[1], + confidence="HIGH", + rationale="One constructor contract change breaks both callers.", + ) + ]) + ) + llm.with_structured_output.return_value = structured + + result = await deduplicate_final_issues_llm(llm, [first, second]) + + assert len(result) == 1 + assert result[0].file == "config/RagConfig.java" + assert result[0].relatedLocations == ["test/RagConfigTest.java:70"] + assert "Also affects: test/RagConfigTest.java:70" in result[0].reason @pytest.mark.asyncio(loop_scope="function") async def test_empty_returns_empty(self): diff --git a/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py index 7d7558c7..405c33c6 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py @@ -7,7 +7,10 @@ import pytest from unittest.mock import MagicMock, patch -from service.review.review_service import ReviewService +from service.review.review_service import ( + ReviewService, + allow_unbound_global_rag_fallback, +) @pytest.fixture @@ -143,6 +146,26 @@ def test_enabled_project_uses_enabled_shared_client(self, service): assert service._rag_client_for_request(request) is service.rag_client + def test_exact_pr_review_disables_unbound_global_fallback(self): + request = MagicMock( + pullRequestId=42, + baseCommitHash="a" * 40, + currentCommitHash="b" * 40, + commitHash=None, + ) + + assert allow_unbound_global_rag_fallback(request) is False + + def test_non_pr_request_can_use_global_fallback(self): + request = MagicMock( + pullRequestId=None, + baseCommitHash=None, + currentCommitHash="b" * 40, + commitHash=None, + ) + + assert allow_unbound_global_rag_fallback(request) is True + # ── _create_llm ────────────────────────────────────────────────── diff --git a/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py b/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py index 84f83633..461da71f 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py +++ b/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py @@ -1146,6 +1146,21 @@ def _request(self): request.enrichmentData = None return request + def _exact_request(self): + request = self._request() + request.currentCommitHash = "a" * 40 + request.commitHash = "a" * 40 + request.baseCommitHash = "b" * 40 + request.ragBaseGenerationManifestSha256 = "c" * 64 + request.ragPrGenerationFingerprint = "d" * 64 + request.ragPrOverlayGenerationManifestSha256 = "e" * 64 + request.rawDiff = "" + request.deltaDiff = None + request.taskContext = None + request.projectRules = [] + request.previousCodeAnalysisIssues = [] + return request + @pytest.mark.asyncio(loop_scope="function") async def test_missing_target_branch_does_not_query_an_invented_branch(self): request = self._request() @@ -1399,6 +1414,185 @@ async def search_for_duplicates(self, **kwargs): assert state.deterministic_retrieval_states == ["failed"] assert state.semantic_disabled is False + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_deterministic_failure_prevents_review_model_call(self): + class Rag: + async def get_deterministic_context(self, **kwargs): + return { + "status": "error", + "status_code": 500, + "error": "exact retrieval failed", + } + + async def get_pr_context(self, **kwargs): + raise AssertionError("semantic retrieval must not run") + + async def search_for_duplicates(self, **kwargs): + return [] + + batch = [{ + "file": ReviewFile( + path="src/a.py", + focus_areas=["general"], + risk_level="MEDIUM", + ), + "priority": "MEDIUM", + }] + + with patch( + "service.review.orchestrator.stage_1_file_review." + "_invoke_stage_1_batch_llm", + new_callable=AsyncMock, + ) as invoke: + with pytest.raises( + RuntimeError, + match="revision-bound deterministic RAG retrieval failed", + ): + await review_file_batch( + MagicMock(), + self._exact_request(), + batch, + rag_client=Rag(), + prepared_context=Stage1PreparedContext(), + pr_indexed=True, + rag_state=Stage1RagState(), + ) + + invoke.assert_not_awaited() + + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_missing_rag_client_prevents_review_model_call(self): + batch = [{ + "file": ReviewFile( + path="src/a.py", + focus_areas=["general"], + risk_level="MEDIUM", + ), + "priority": "MEDIUM", + }] + + with patch( + "service.review.orchestrator.stage_1_file_review." + "_invoke_stage_1_batch_llm", + new_callable=AsyncMock, + ) as invoke: + with pytest.raises( + RuntimeError, + match="requires a RAG client", + ): + await review_file_batch( + MagicMock(), + self._exact_request(), + batch, + rag_client=None, + prepared_context=Stage1PreparedContext(), + pr_indexed=True, + rag_state=Stage1RagState(), + ) + + invoke.assert_not_awaited() + + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_partial_deterministic_state_fails_closed(self): + class Rag: + async def get_deterministic_context(self, **kwargs): + return { + "context": { + "chunks": [{"text": "partial context"}], + "_metadata": {"retrieval_state": "partial"}, + } + } + + async def search_for_duplicates(self, **kwargs): + return [] + + with pytest.raises( + RuntimeError, + match="deterministic RAG retrieval is not complete: partial", + ): + await fetch_batch_rag_context( + Rag(), + self._exact_request(), + ["src/a.py"], + ["changed line"], + pr_indexed=True, + rag_state=Stage1RagState(), + ) + + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_semantic_transport_failure_fails_closed(self): + class Rag: + async def get_deterministic_context(self, **kwargs): + return { + "context": { + "chunks": [], + "_metadata": {"retrieval_state": "complete"}, + } + } + + async def get_pr_context(self, **kwargs): + return { + "status": "error", + "status_code": 503, + "error": "semantic backend unavailable", + } + + async def search_for_duplicates(self, **kwargs): + return [] + + with pytest.raises(RuntimeError, match="semantic backend unavailable"): + await fetch_batch_rag_context( + Rag(), + self._exact_request(), + ["src/a.py"], + ["changed line"], + pr_indexed=True, + rag_state=Stage1RagState(), + ) + + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_bound_success_allows_intentional_semantic_disable( + self, + monkeypatch, + ): + import service.review.orchestrator.stage_1_file_review as stage1 + + monkeypatch.setattr(stage1, "SEMANTIC_RAG_FILLER_ENABLED", False) + + class Rag: + async def get_deterministic_context(self, **kwargs): + return { + "context": { + "chunks": [{ + "text": "exact context", + "metadata": {"path": "src/dependency.py"}, + }], + "_metadata": {"retrieval_state": "complete"}, + } + } + + async def get_pr_context(self, **kwargs): + raise AssertionError("semantic retrieval is intentionally disabled") + + async def search_for_duplicates(self, **kwargs): + return [] + + state = Stage1RagState() + result = await fetch_batch_rag_context( + Rag(), + self._exact_request(), + ["src/a.py"], + ["changed line"], + pr_indexed=True, + rag_state=state, + ) + + assert [chunk["text"] for chunk in result["relevant_code"]] == [ + "exact context" + ] + assert state.deterministic_retrieval_states == ["complete"] + assert state.semantic_disabled is False + # ── _deduplicate_pr_stale_chunks ───────────────────────────────── @@ -1860,6 +2054,36 @@ async def test_missing_target_branch_uses_local_grouping_without_rag(self, mock_ assert mock_smart.call_args.kwargs["branches"] == [] assert mock_smart.call_args.kwargs["rag_client"] is None + @patch("service.review.orchestrator.stage_1_file_review.create_smart_batches_async") + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_receipts_disable_unbound_batching_rag(self, mock_smart): + groups = self._make_plan(["a.py"]) + mock_smart.return_value = [[{ + "file": groups[0].files[0], + "priority": "MEDIUM", + }]] + request = MagicMock( + enrichmentData=None, + maxAllowedTokens=200000, + projectWorkspace="ws", + projectNamespace="proj", + ragBaseGenerationManifestSha256="a" * 64, + ragPrGenerationFingerprint="sha256:" + "b" * 64, + ragPrOverlayGenerationManifestSha256="c" * 64, + ) + request.get_rag_branch.return_value = "main" + request.get_rag_base_branch.return_value = "main" + + result = await create_smart_batches_wrapper( + file_groups=groups, + processed_diff=MagicMock(), + request=request, + rag_client=MagicMock(), + ) + + assert result == mock_smart.return_value + assert mock_smart.call_args.kwargs["rag_client"] is None + class TestStage1Scheduling: @pytest.mark.asyncio(loop_scope="function") diff --git a/python-ecosystem/inference-orchestrator/tests/test_stage_2_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_stage_2_helpers.py index c5ce2621..a330e0f4 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_stage_2_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_stage_2_helpers.py @@ -171,6 +171,40 @@ async def test_cross_module_context_does_not_guess_main_without_target_branch(): rag.search_for_duplicates.assert_not_awaited() +@pytest.mark.asyncio(loop_scope="function") +async def test_revision_bound_cross_module_transport_failure_fails_open(): + request = SimpleNamespace( + baseCommitHash="a" * 40, + ragBaseGenerationManifestSha256="b" * 64, + changedFiles=["src/service.py"], + prTitle="Change service", + projectWorkspace="ws", + projectNamespace="project", + get_rag_branch=lambda: "main", + get_rag_base_branch=lambda: "main", + ) + rag = SimpleNamespace( + search_for_duplicates=AsyncMock( + side_effect=ConnectionError("transport unavailable") + ), + ) + + assert await _fetch_cross_module_context(rag, request) == "" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_cross_module_context_skips_partial_generation_lease(): + request = SimpleNamespace( + baseCommitHash="a" * 40, + ragBaseGenerationManifestSha256=None, + ) + rag = SimpleNamespace(search_for_duplicates=AsyncMock()) + + assert await _fetch_cross_module_context(rag, request) == "" + + rag.search_for_duplicates.assert_not_awaited() + + # ── _slim_issues_for_stage_2 ──────────────────────────────── class TestSlimIssues: diff --git a/python-ecosystem/inference-orchestrator/tests/test_stage_3_full.py b/python-ecosystem/inference-orchestrator/tests/test_stage_3_full.py index 903e9463..2c9afb15 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_stage_3_full.py +++ b/python-ecosystem/inference-orchestrator/tests/test_stage_3_full.py @@ -1,12 +1,15 @@ """Tests for stage_3_aggregation: summarizers, dismissed issues, MCP stage 3.""" import json import pytest +from types import SimpleNamespace from unittest.mock import MagicMock, AsyncMock, patch +from model.output_schemas import CodeReviewIssue from service.review.orchestrator.stage_3_aggregation import ( execute_stage_3_aggregation, _summarize_issues_for_stage_3, _summarize_plan_for_stage_3, _extract_dismissed_issues, + _stage_3_with_mcp, ) @@ -34,7 +37,7 @@ def test_severity_counts(self): assert "HIGH: 2" in result assert "MEDIUM: 1" in result - def test_top_findings_priority_order(self): + def test_complete_records_keep_stable_verification_identity(self): critical = MagicMock() critical.severity = "CRITICAL" critical.category = "SECURITY" @@ -52,12 +55,11 @@ def test_top_findings_priority_order(self): low.reason = "Naming convention" result = _summarize_issues_for_stage_3([low, critical]) - lines = result.split("\n") - # Critical should appear before LOW in the top findings section - top_lines = [l for l in lines if "[CRITICAL]" in l or "[LOW]" in l] - assert len(top_lines) == 2 - assert "CRITICAL" in top_lines[0] - assert "LOW" in top_lines[1] + records = json.loads(result.split("Complete verification records (JSON):\n", 1)[1]) + assert records[0]["verification_id"] == "issue_0" + assert records[0]["original_id"] == "l1" + assert records[1]["verification_id"] == "issue_1" + assert records[1]["original_id"] == "c1" def test_all_issue_ids_listed(self): issues = [] @@ -190,6 +192,9 @@ async def test_basic_no_mcp(self): request.changedFiles = ["a.py"] request.targetBranchName = "main" request.previousCodeAnalysisIssues = [] + request.currentCommitHash = None + request.commitHash = None + request.taskContext = None plan = MagicMock() plan.file_groups = [] @@ -220,6 +225,9 @@ async def test_incremental_review_context(self): request.changedFiles = [] request.targetBranchName = "" request.previousCodeAnalysisIssues = ["prev1", "prev2"] + request.currentCommitHash = None + request.commitHash = None + request.taskContext = None plan = MagicMock() plan.file_groups = [] @@ -237,7 +245,7 @@ async def test_incremental_review_context(self): @pytest.mark.asyncio(loop_scope="function") async def test_mcp_stage_dispatches(self): - """When use_mcp_tools=True and target_branch given, dispatches to MCP.""" + """An immutable reviewed commit enables the MCP verification loop.""" llm = MagicMock() mcp_client = MagicMock() @@ -249,6 +257,9 @@ async def test_mcp_stage_dispatches(self): request.changedFiles = [] request.targetBranchName = "main" request.previousCodeAnalysisIssues = [] + request.currentCommitHash = "abc123" + request.commitHash = None + request.taskContext = None plan = MagicMock() plan.file_groups = [] @@ -266,3 +277,64 @@ async def test_mcp_stage_dispatches(self): ) mock_mcp.assert_called_once() assert result["dismissed_issue_ids"] == ["x"] + + +class TestStage3McpVerification: + @pytest.mark.asyncio(loop_scope="function") + async def test_successful_revision_read_validates_fresh_issue_dismissal(self): + issue = CodeReviewIssue( + file="src/a.py", line=10, severity="HIGH", category="BUG_RISK", + reason="Claim to verify.", suggestedFixDescription="Fix it.", + ) + request = SimpleNamespace( + projectVcsWorkspace="workspace", + projectVcsRepoSlug="repo", + ) + tool_response = SimpleNamespace( + content="", + tool_calls=[{ + "id": "call-1", + "name": "getBranchFileContent", + "args": { + "filePath": "src/a.py", + "verificationId": "issue_0", + }, + }], + response_metadata={}, + ) + final_response = SimpleNamespace( + content=( + "Verified report\n" + '' + ), + tool_calls=[], + response_metadata={}, + ) + bound_llm = MagicMock() + bound_llm.ainvoke = AsyncMock( + side_effect=[tool_response, final_response] + ) + llm = MagicMock() + llm.bind_tools.return_value = bound_llm + mcp_client = MagicMock() + mcp_client.session.call_tool = AsyncMock(return_value=SimpleNamespace( + content=[SimpleNamespace(text="current source")] + )) + + result = await _stage_3_with_mcp( + llm, + request, + "prompt", + mcp_client, + "commit-abc", + {"issue_0": issue}, + ) + + assert result["report"] == "Verified report" + assert result["dismissed_issue_ids"] == [] + assert result["dismissed_issue_keys"] == ["issue_0"] + assert result["dismissed_issue_object_ids"] == [id(issue)] + call_args = mcp_client.session.call_tool.await_args.args[1] + assert call_args["branch"] == "commit-abc" + assert call_args["startLine"] == 1 + assert call_args["endLine"] == 90 diff --git a/python-ecosystem/inference-orchestrator/tests/test_stage_3_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_stage_3_helpers.py index fece667b..6424bc2d 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_stage_3_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_stage_3_helpers.py @@ -10,6 +10,8 @@ _summarize_issues_for_stage_3, _summarize_plan_for_stage_3, _extract_dismissed_issues, + _stage_3_verification_issue_map, + _validated_mcp_dismissals, ) @@ -38,16 +40,17 @@ def test_with_issues(self): assert "HIGH" in result assert "ISS-1" in result - def test_sorting_by_severity(self): - issues = [ - CodeReviewIssue(id="L1", file="a.py", line=1, severity="LOW", category="CODE_QUALITY", reason="x", suggestedFixDescription="f"), - CodeReviewIssue(id="C1", file="b.py", line=2, severity="CRITICAL", category="BUG_RISK", reason="y", suggestedFixDescription="f"), - ] + def test_lists_every_issue_with_stable_verification_ids(self): + issues = [CodeReviewIssue( + id=f"I-{index}", file=f"src/f{index}.py", line=index + 2, + severity="LOW", category="CODE_QUALITY", reason=f"reason {index}", + suggestedFixDescription="fix", + ) for index in range(14)] result = _summarize_issues_for_stage_3(issues) - # CRITICAL should appear before LOW in top findings - crit_pos = result.find("C1") - low_pos = result.find("L1") - assert crit_pos < low_pos + + assert '"verification_id":"issue_0"' in result + assert '"verification_id":"issue_13"' in result + assert '"original_id":"I-13"' in result def test_excludes_resolved_history_records(self): resolved = CodeReviewIssue( @@ -61,6 +64,44 @@ def test_excludes_resolved_history_records(self): assert "No issues" in result assert "OLD-1" not in result + def test_reason_compaction_only_removes_exact_repetition(self): + issue = CodeReviewIssue( + file="a.py", line=1, severity="HIGH", category="BUG_RISK", + title="Repeated title", + reason=( + "Repeated title\n\nRoot cause.\n\nMore proof.\n\n" + "More proof.\n\nImpact: request fails." + ), + suggestedFixDescription="Fix it.", + ) + + records = json.loads( + _summarize_issues_for_stage_3([issue]).split( + "Complete verification records (JSON):\n", 1 + )[1] + ) + + reason = records[0]["reason"] + assert "Repeated title" not in reason + assert "Root cause." in reason + assert reason.count("More proof.") == 1 + assert "Impact: request fails." in reason + + def test_related_locations_are_recovered_from_persisted_reason(self): + issue = CodeReviewIssue( + file="a.py", line=1, severity="HIGH", category="BUG_RISK", + reason="Root cause.\n\nAlso affects: b.py:20, c.py:30", + suggestedFixDescription="Fix it.", + ) + + records = json.loads( + _summarize_issues_for_stage_3([issue]).split( + "Complete verification records (JSON):\n", 1 + )[1] + ) + + assert records[0]["related_locations"] == ["b.py:20", "c.py:30"] + # ── _summarize_plan_for_stage_3 ────────────────────────────── @@ -109,3 +150,107 @@ def test_non_list_marker(self): content = '' clean, dismissed = _extract_dismissed_issues(content) assert dismissed == [] + + +class TestValidatedMcpDismissals: + def test_requires_successful_read_at_exact_revision_for_every_location(self): + issue = CodeReviewIssue( + file="src/a.py", line=10, severity="HIGH", category="BUG_RISK", + title="Shared defect", reason="One root cause.", + suggestedFixDescription="Fix it.", + relatedLocations=["src/b.py:20"], + ) + issue_map = _stage_3_verification_issue_map([issue]) + executor = SimpleNamespace(call_log=[ + { + "tool": "getBranchFileContent", + "args": { + "filePath": "src/a.py", "branch": "abc123", + "verificationId": "issue_0", + }, + "success": True, + "evidence_valid": True, + "evidence_complete_file": True, + }, + ]) + + assert _validated_mcp_dismissals( + ["issue_0"], issue_map, executor, "abc123" + ) == [] + + executor.call_log.append({ + "tool": "getBranchFileContent", + "args": { + "filePath": "src/b.py", "branch": "abc123", + "verificationId": "issue_0", + }, + "success": True, + "evidence_valid": True, + "evidence_complete_file": False, + "evidence_start_line": 1, + "evidence_end_line": 100, + }) + assert _validated_mcp_dismissals( + ["issue_0"], issue_map, executor, "abc123" + ) == ["issue_0"] + + def test_wrong_revision_failed_or_unknown_dismissal_fails_open(self): + issue = CodeReviewIssue( + file="src/a.py", line=10, severity="HIGH", category="BUG_RISK", + reason="Concrete defect.", suggestedFixDescription="Fix it.", + ) + issue_map = _stage_3_verification_issue_map([issue]) + executor = SimpleNamespace(call_log=[ + { + "tool": "getBranchFileContent", + "args": { + "filePath": "src/a.py", "branch": "main", + "verificationId": "issue_0", + }, + "success": True, + "evidence_valid": True, + "evidence_complete_file": True, + }, + { + "tool": "getBranchFileContent", + "args": { + "filePath": "src/a.py", "branch": "abc123", + "verificationId": "issue_0", + }, + "success": False, + "evidence_valid": False, + }, + ]) + + assert _validated_mcp_dismissals( + ["issue_0", "issue_99"], issue_map, executor, "abc123" + ) == [] + + def test_window_must_cover_the_bound_issue_line(self): + issue = CodeReviewIssue( + file="src/a.py", line=500, severity="HIGH", category="BUG_RISK", + reason="Concrete defect.", suggestedFixDescription="Fix it.", + ) + issue_map = _stage_3_verification_issue_map([issue]) + executor = SimpleNamespace(call_log=[{ + "tool": "getBranchFileContent", + "args": { + "filePath": "src/a.py", "branch": "abc123", + "verificationId": "issue_0", + }, + "success": True, + "evidence_valid": True, + "evidence_complete_file": False, + "evidence_start_line": 1, + "evidence_end_line": 100, + }]) + + assert _validated_mcp_dismissals( + ["issue_0"], issue_map, executor, "abc123" + ) == [] + + executor.call_log[0]["evidence_start_line"] = 420 + executor.call_log[0]["evidence_end_line"] = 580 + assert _validated_mcp_dismissals( + ["issue_0"], issue_map, executor, "abc123" + ) == ["issue_0"] diff --git a/python-ecosystem/rag-pipeline/Dockerfile b/python-ecosystem/rag-pipeline/Dockerfile index 0a35959d..3bbb380d 100644 --- a/python-ecosystem/rag-pipeline/Dockerfile +++ b/python-ecosystem/rag-pipeline/Dockerfile @@ -51,7 +51,7 @@ ENV TRANSFORMERS_CACHE=/tmp/.transformers_cache ENV HF_HOME=/tmp/.huggingface ENV LLAMA_INDEX_CACHE_DIR=/tmp/.llama_index # Allow concurrent indexing by running multiple Uvicorn workers -ENV UVICORN_WORKERS=4 +ENV UVICORN_WORKERS=1 # CPU Threading Optimization (can be overridden via .env) # These control parallelism for numerical operations diff --git a/python-ecosystem/rag-pipeline/Dockerfile.observable b/python-ecosystem/rag-pipeline/Dockerfile.observable index b828eb65..c2e30204 100644 --- a/python-ecosystem/rag-pipeline/Dockerfile.observable +++ b/python-ecosystem/rag-pipeline/Dockerfile.observable @@ -51,7 +51,7 @@ ENV TRANSFORMERS_CACHE=/tmp/.transformers_cache ENV HF_HOME=/tmp/.huggingface ENV LLAMA_INDEX_CACHE_DIR=/tmp/.llama_index # Allow concurrent indexing by running multiple Uvicorn workers -ENV UVICORN_WORKERS=4 +ENV UVICORN_WORKERS=1 # CPU Threading Optimization (can be overridden via .env) # These control parallelism for numerical operations diff --git a/python-ecosystem/rag-pipeline/integration/conftest.py b/python-ecosystem/rag-pipeline/integration/conftest.py index 1a2d0289..a753d0f5 100644 --- a/python-ecosystem/rag-pipeline/integration/conftest.py +++ b/python-ecosystem/rag-pipeline/integration/conftest.py @@ -91,6 +91,7 @@ def rag_app(_mock_qdrant, _mock_embedding): assert_owned=MagicMock() ) mock_im.project_mutation.return_value = mutation_context + mock_im.pr_overlay_mutation.return_value = mutation_context mock_im.embed_model = _mock_embedding mock_im.qdrant_client = _mock_qdrant mock_im.splitter.split_documents_resilient.side_effect = ( diff --git a/python-ecosystem/rag-pipeline/main.py b/python-ecosystem/rag-pipeline/main.py index 38226ecd..6e3df0f8 100644 --- a/python-ecosystem/rag-pipeline/main.py +++ b/python-ecosystem/rag-pipeline/main.py @@ -125,10 +125,12 @@ def validate_environment(): from rag_pipeline.api.api import app if __name__ == "__main__": - # Use multiple workers to allow concurrent indexing requests - # Each worker can handle one long-running indexing task - workers = int(os.environ.get("UVICORN_WORKERS", "4")) - logger.info(f"Starting Uvicorn with {workers} workers for concurrent request handling") + # Keep one process by default. Each worker initializes its own embedding and + # indexing state, so multiplying Uvicorn workers multiplies the memory cost + # of concurrent branch snapshots. Branch-level parallelism is bounded by + # the Java maintenance executor instead. + workers = int(os.environ.get("UVICORN_WORKERS", "1")) + logger.info(f"Starting Uvicorn with {workers} worker process(es)") uvicorn.run( "rag_pipeline.api.api:app", host="0.0.0.0", diff --git a/python-ecosystem/rag-pipeline/requirements.local.txt b/python-ecosystem/rag-pipeline/requirements.local.txt index 7e48bdc2..0ea464a5 100644 --- a/python-ecosystem/rag-pipeline/requirements.local.txt +++ b/python-ecosystem/rag-pipeline/requirements.local.txt @@ -10,7 +10,9 @@ llama-index-core==0.13.0 # LlamaIndex extensions llama-index-embeddings-openai>=0.3.0 -llama-index-vector-stores-qdrant>=0.5.0 +# Keep the adapter and client as one tested compatibility pair. qdrant-client +# 1.19 removed IDF_EMBEDDING_MODELS while this adapter still imports it. +llama-index-vector-stores-qdrant==0.10.2 llama-index-llms-openai>=0.3.0 # LangChain text splitters (language-aware code splitting) @@ -36,7 +38,7 @@ tree-sitter-ruby==0.23.1 tree-sitter-php==0.24.1 # Qdrant for vector storage -qdrant-client>=1.7.0,<2.0.0 +qdrant-client==1.16.2 # Redis for async queue redis>=5.0.0,<6.0.0 diff --git a/python-ecosystem/rag-pipeline/requirements.txt b/python-ecosystem/rag-pipeline/requirements.txt index 0ad2b000..b11a15ea 100644 --- a/python-ecosystem/rag-pipeline/requirements.txt +++ b/python-ecosystem/rag-pipeline/requirements.txt @@ -10,7 +10,9 @@ llama-index-core==0.13.0 # LlamaIndex extensions llama-index-embeddings-openai>=0.3.0 -llama-index-vector-stores-qdrant>=0.5.0 +# Keep the adapter and client as one tested compatibility pair. qdrant-client +# 1.19 removed IDF_EMBEDDING_MODELS while this adapter still imports it. +llama-index-vector-stores-qdrant==0.10.2 llama-index-llms-openai>=0.3.0 # LangChain text splitters (language-aware code splitting) @@ -36,7 +38,7 @@ tree-sitter-ruby==0.23.1 tree-sitter-php==0.24.1 # Qdrant for vector storage -qdrant-client>=1.7.0,<2.0.0 +qdrant-client==1.16.2 # Redis for async queue redis>=5.0.0,<6.0.0 diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py index 896ef595..21922f54 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py @@ -39,6 +39,13 @@ class IndexRequest(BaseModel): project: str branch: str commit: str + source_tree_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + collection_target: Optional[str] = Field(default=None, min_length=1) + publish_branch_alias: bool = False + publish_legacy_project_alias: bool = False preserve_other_branches: bool = False cleanup_repo_path: bool = False include_patterns: Optional[List[str]] = None @@ -102,6 +109,43 @@ def validate_file_paths(cls, v: List[str]) -> List[str]: return _validate_file_paths(v) +class AdvanceGenerationRequest(BaseModel): + updated_file_paths: List[str] = Field(default_factory=list) + deleted_file_paths: List[str] = Field(default_factory=list) + repo_base: Optional[str] = None + workspace: str + project: str + branch: str + source_commit: str + commit: str + source_tree_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + source_collection_target: str = Field(min_length=1) + collection_target: str = Field(min_length=1) + publish_branch_alias: bool = False + publish_legacy_project_alias: bool = False + + @field_validator("repo_base") + @classmethod + def validate_repo_base(cls, v: Optional[str]) -> Optional[str]: + return _validate_repo_path(v) if v is not None else None + + @field_validator("updated_file_paths", "deleted_file_paths") + @classmethod + def validate_file_paths(cls, v: List[str]) -> List[str]: + return _validate_file_paths(v) + + +class GenerationAliasPublicationRequest(BaseModel): + """Repair the readable aliases of one already sealed generation.""" + workspace: str + project: str + branch: str + commit: str + collection_target: str = Field(min_length=1) + publish_branch_alias: bool = True + publish_legacy_project_alias: bool = False + + class DeleteBranchRequest(BaseModel): workspace: str project: str @@ -126,6 +170,29 @@ def validate_branch_names(cls, value: Optional[List[str]]) -> Optional[List[str] return value +class RevisionPreflightResponse(BaseModel): + workspace: str + project: str + branch: str + commit: str + point_count: int = Field(gt=0) + repository_revision: str + repository_facts_sha256: str + plugin_ids: List[str] + plugin_fingerprint: str + plugin_descriptor_fingerprint: str + plugin_implementation_fingerprint: str + index_representation_fingerprint: str + generation_schema: str + generation_member_count: int = Field(gt=0) + generation_members_sha256: str + generation_manifest_sha256: str + source_tree_sha256: str + index_include_patterns: List[str] + index_exclude_patterns: List[str] + index_selection_policy_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + class EstimateRequest(BaseModel): repo_path: str include_patterns: Optional[List[str]] = None @@ -155,6 +222,16 @@ class QueryRequest(BaseModel): branch: str top_k: Optional[int] = 10 filter_language: Optional[str] = None + repository_revision: Optional[str] = Field( + default=None, + min_length=1, + max_length=200, + ) + repository_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + collection_target: Optional[str] = Field(default=None, min_length=1) class PRContextRequest(BaseModel): @@ -172,6 +249,29 @@ class PRContextRequest(BaseModel): deleted_files: Optional[List[str]] = Field(default_factory=list) pr_number: Optional[int] = None all_pr_changed_files: Optional[List[str]] = Field(default_factory=list) + source_revision: Optional[str] = Field( + default=None, + min_length=1, + max_length=200, + ) + base_revision: Optional[str] = Field( + default=None, + min_length=1, + max_length=200, + ) + base_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + pr_generation_fingerprint: Optional[str] = Field( + default=None, + pattern=r"^sha256:[0-9a-f]{64}$", + ) + pr_overlay_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + collection_target: Optional[str] = Field(default=None, min_length=1) @field_validator('changed_files') @classmethod @@ -205,6 +305,29 @@ class DeterministicContextRequest(BaseModel): description="Extra type/function names to look up (from AST enrichment: extends, implements, calls). " "Injected directly into Step 2 definition lookup alongside Qdrant-extracted identifiers." ) + source_revision: Optional[str] = Field( + default=None, + min_length=1, + max_length=200, + ) + base_revision: Optional[str] = Field( + default=None, + min_length=1, + max_length=200, + ) + base_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + pr_generation_fingerprint: Optional[str] = Field( + default=None, + pattern=r"^sha256:[0-9a-f]{64}$", + ) + pr_overlay_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + collection_target: Optional[str] = Field(default=None, min_length=1) # ── Parse models ── @@ -278,6 +401,11 @@ class PRIndexRequest(BaseModel): plugin_fingerprint: str = "sha256:" + "0" * 64 plugin_descriptor_fingerprint: str = "sha256:" + "0" * 64 files: List[PRFileInfo] + base_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + collection_target: Optional[str] = Field(default=None, min_length=1) # ── Vector storage inspection models ── diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py index 57a3f9e4..aeb8f473 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py @@ -1,11 +1,18 @@ """Index and branch management endpoints.""" +import asyncio +import json import logging +from queue import Empty, Queue +from threading import Thread from typing import List -from fastapi import APIRouter, HTTPException, BackgroundTasks +from fastapi import APIRouter, HTTPException, BackgroundTasks, Query +from fastapi.responses import StreamingResponse from ...models.config import IndexStats from ..models import ( IndexRequest, UpdateFilesRequest, DeleteFilesRequest, ApplyChangesRequest, + AdvanceGenerationRequest, + GenerationAliasPublicationRequest, DeleteBranchRequest, CleanupStaleBranchesRequest, EstimateRequest, EstimateResponse, ) @@ -46,7 +53,7 @@ def estimate_repository(request: EstimateRequest): file_count, estimated_chunks = index_manager.estimate_repository_size( repo_path=request.repo_path, include_patterns=request.include_patterns, - exclude_patterns=request.exclude_patterns + exclude_patterns=request.exclude_patterns, ) within_limits = True @@ -87,6 +94,17 @@ def index_repository(request: IndexRequest, background_tasks: BackgroundTasks): """Index entire repository.""" _, index_manager = _get_singletons() try: + optional_generation_args = {} + source_tree_sha256 = getattr(request, "source_tree_sha256", None) + collection_target = getattr(request, "collection_target", None) + if isinstance(source_tree_sha256, str) and source_tree_sha256: + optional_generation_args["source_tree_sha256"] = source_tree_sha256 + if isinstance(collection_target, str) and collection_target: + optional_generation_args["collection_target"] = collection_target + if getattr(request, "publish_branch_alias", False) is True: + optional_generation_args["publish_branch_alias"] = True + if getattr(request, "publish_legacy_project_alias", False) is True: + optional_generation_args["publish_legacy_project_alias"] = True stats = index_manager.index_repository( repo_path=request.repo_path, workspace=request.workspace, @@ -95,7 +113,8 @@ def index_repository(request: IndexRequest, background_tasks: BackgroundTasks): commit=request.commit, preserve_other_branches=request.preserve_other_branches, include_patterns=request.include_patterns, - exclude_patterns=request.exclude_patterns + exclude_patterns=request.exclude_patterns, + **optional_generation_args, ) return stats except ValueError as e: @@ -110,6 +129,87 @@ def index_repository(request: IndexRequest, background_tasks: BackgroundTasks): raise HTTPException(status_code=500, detail=str(e)) +@router.post("/index/repository/stream") +def index_repository_stream(request: IndexRequest): + """Index one repository and stream observable batch progress as SSE. + + The ordinary endpoint remains the stable JSON contract. This endpoint is + intentionally only an observability transport: it runs the same index + operation and forwards optional progress events without making progress + delivery a prerequisite for a successful snapshot. + """ + _, index_manager = _get_singletons() + + async def event_stream(): + events: Queue[tuple[str, object]] = Queue() + + def progress(event: dict) -> None: + events.put(("progress", event)) + + def run_index() -> None: + try: + optional_generation_args = {} + if request.source_tree_sha256: + optional_generation_args["source_tree_sha256"] = ( + request.source_tree_sha256 + ) + if request.collection_target: + optional_generation_args["collection_target"] = ( + request.collection_target + ) + if getattr(request, "publish_branch_alias", False) is True: + optional_generation_args["publish_branch_alias"] = True + if getattr(request, "publish_legacy_project_alias", False) is True: + optional_generation_args["publish_legacy_project_alias"] = True + stats = index_manager.index_repository( + repo_path=request.repo_path, + workspace=request.workspace, + project=request.project, + branch=request.branch, + commit=request.commit, + preserve_other_branches=request.preserve_other_branches, + include_patterns=request.include_patterns, + exclude_patterns=request.exclude_patterns, + progress_callback=progress, + **optional_generation_args, + ) + events.put(("complete", stats.model_dump(mode="json"))) + except Exception as exception: + logger.error("Error indexing repository with progress: %s", exception) + events.put(("error", {"message": str(exception)})) + + worker = Thread( + target=run_index, + name="rag-index-progress", + daemon=True, + ) + worker.start() + while True: + try: + event_type, payload = events.get_nowait() + except Empty: + # Polling a thread-safe queue avoids nesting a blocking queue + # consumer inside Starlette's thread pool. The short wait keeps + # the event loop responsive and progress delivery prompt. + await asyncio.sleep(0.05) + continue + if event_type == "progress": + event = {"type": "progress", **payload} + elif event_type == "complete": + event = {"type": "complete", "result": payload} + else: + event = {"type": "error", **payload} + yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n" + if event_type in {"complete", "error"}: + # The terminal event is scheduled just before the producer + # returns. Join that final unwind so a short-lived consumer + # cannot finish while its producer thread is still active. + worker.join(timeout=1.0) + break + + return StreamingResponse(event_stream(), media_type="text/event-stream") + + @router.post("/index/update-files", response_model=IndexStats) def update_files(request: UpdateFilesRequest): """Update specific files in index.""" @@ -195,6 +295,74 @@ def apply_changes(request: ApplyChangesRequest): raise HTTPException(status_code=500, detail=str(e)) +@router.post("/index/advance-generation", response_model=IndexStats) +def advance_generation(request: AdvanceGenerationRequest): + """Build one immutable target generation from an exact sealed source.""" + _, index_manager = _get_singletons() + if request.repo_base is None: + raise HTTPException( + status_code=422, + detail="repo_base is required to attest the target source tree", + ) + try: + return index_manager.advance_generation( + source_collection_target=request.source_collection_target, + target_collection_target=request.collection_target, + source_commit=request.source_commit, + source_tree_sha256=request.source_tree_sha256, + updated_file_paths=request.updated_file_paths, + deleted_file_paths=request.deleted_file_paths, + repo_base=request.repo_base, + workspace=request.workspace, + project=request.project, + branch=request.branch, + commit=request.commit, + publish_branch_alias=request.publish_branch_alias, + publish_legacy_project_alias=request.publish_legacy_project_alias, + ) + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=422, detail=str(e)) + except MutationLeaseUnavailable as e: + raise HTTPException(status_code=409, detail=str(e)) + except MutationCoordinationUnavailable as e: + raise HTTPException(status_code=503, detail=str(e)) + except Exception as e: + logger.error(f"Error advancing repository generation: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/index/generation-aliases") +def publish_generation_aliases(request: GenerationAliasPublicationRequest): + """Publish or repair readable aliases for an accepted immutable generation. + + This is deliberately separate from indexing so the Java registry can reject + a stale completed build before any mutable branch-head alias is moved. + """ + _, index_manager = _get_singletons() + try: + aliases = index_manager.publish_generation_aliases( + workspace=request.workspace, + project=request.project, + branch=request.branch, + commit=request.commit, + collection_target=request.collection_target, + publish_branch_alias=request.publish_branch_alias, + publish_legacy_project_alias=request.publish_legacy_project_alias, + ) + return {"status": "published", "aliases": aliases} + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) + except MutationLeaseUnavailable as e: + raise HTTPException(status_code=409, detail=str(e)) + except MutationCoordinationUnavailable as e: + raise HTTPException(status_code=503, detail=str(e)) + except Exception as e: + logger.error("Error publishing readable generation aliases: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + @router.delete("/index/{workspace}/{project}/{branch}") def delete_index(workspace: str, project: str, branch: str): """Delete entire index.""" @@ -214,11 +382,18 @@ def delete_index(workspace: str, project: str, branch: str): # ── Branch management ── @router.delete("/index/{workspace}/{project}/branch/{branch}") -def delete_branch(workspace: str, project: str, branch: str): +def delete_branch( + workspace: str, + project: str, + branch: str, + collection_target: str | None = Query(default=None), +): """Delete all points for a specific branch from the project collection.""" _, index_manager = _get_singletons() try: - success = index_manager.delete_branch(workspace, project, branch) + success = index_manager.delete_branch( + workspace, project, branch, collection_target=collection_target + ) if success: return { "status": "success", @@ -229,6 +404,8 @@ def delete_branch(workspace: str, project: str, branch: str): "status": "not_found", "message": f"Branch '{branch}' not found or collection doesn't exist" } + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) except MutationLeaseUnavailable as e: raise HTTPException(status_code=409, detail=str(e)) except MutationCoordinationUnavailable as e: diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/inspect.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/inspect.py index 1d9a66ab..df9a0de2 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/inspect.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/inspect.py @@ -34,6 +34,9 @@ "repository_facts_state", "facts_part", "facts_parts", "facts_content_sha256", "plugin_ids", "plugin_fingerprint", "plugin_descriptor_fingerprint", "plugin_implementation_fingerprint", + "repository_generation_manifest", "generation_schema", + "generation_member_count", "generation_members_sha256", + "generation_manifest_sha256", "generation_member_sha256", "indexed_at", "fragment_of", "text", "_node_content", ] GRAPH_TEXT_LIMIT = 280 @@ -191,6 +194,8 @@ def _node_title(payload: Dict[str, Any]) -> str: return f"{plugin}: {kind}" + (f" — {source}" if source else "") if payload.get("repository_facts_state"): return "Repository detection facts" + if payload.get("repository_generation_manifest"): + return "Repository generation manifest" if payload.get("repository_snapshot"): return ( f"{payload.get('snapshot_plugin') or 'plugin'}: " @@ -218,6 +223,8 @@ def _node_kind(payload: Dict[str, Any]) -> str: return "repository_snapshot" if payload.get("repository_facts_state"): return "repository_facts" + if payload.get("repository_generation_manifest"): + return "repository_generation_manifest" if payload.get("node_type"): return str(payload["node_type"]) if payload.get("content_type"): diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py index cba269d3..b86625e4 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py @@ -1,7 +1,7 @@ """PR file indexing endpoints.""" import logging from datetime import datetime, timezone -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Query from llama_index.core import Document as LlamaDocument from qdrant_client.models import Filter, FieldCondition, MatchValue @@ -28,6 +28,10 @@ from ...core.pr_overlay_representation import ( PR_OVERLAY_REPRESENTATION_PAYLOAD_KEY, ) +from ...core.pr_overlay_manifest import ( + read_pr_overlay_generation, +) +from ...core.revision_binding import require_repository_generation logger = logging.getLogger(__name__) router = APIRouter(tags=["pr"]) @@ -106,16 +110,76 @@ def index_pr_files(request: PRIndexRequest): completely and then replaces the previous points with rollback protection. """ index_manager = _get_index_manager() - mutation_context = index_manager.project_mutation( + mutation_context = index_manager.pr_overlay_mutation( request.workspace, request.project, + request.pr_number, "index-pr-overlay", ) mutation_lease = None try: mutation_lease = mutation_context.__enter__() - collection_name = index_manager._get_project_collection_name( - request.workspace, request.project + base_receipt = None + target_branch = request.base_branch or request.branch + requested_collection_target = getattr( + request, "collection_target", None + ) + requested_base_manifest = getattr( + request, "base_generation_manifest_sha256", None + ) + if not isinstance(requested_collection_target, str): + requested_collection_target = None + if not isinstance(requested_base_manifest, str): + requested_base_manifest = None + if requested_collection_target and not request.base_revision: + raise IncrementalIndexPreconditionError( + "PR overlay collection target requires an exact base revision" + ) + if requested_base_manifest and not request.base_revision: + raise IncrementalIndexPreconditionError( + "PR overlay base generation receipt requires an exact base revision" + ) + exact_binding = bool( + request.base_revision + and (requested_collection_target or requested_base_manifest) + ) + if exact_binding: + base_receipt = require_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=target_branch, + revision=request.base_revision, + generation_manifest_sha256=( + requested_base_manifest + ), + collection_target=requested_collection_target, + ) + collection_name = ( + base_receipt["_collection_target"] + if base_receipt + else requested_collection_target + or index_manager._get_project_collection_name( + request.workspace, request.project + ) + ) + base_generation_receipt = ( + { + "base_generation_manifest_sha256": base_receipt[ + "generation_manifest_sha256" + ], + "plugin_fingerprint": base_receipt["plugin_fingerprint"], + "plugin_descriptor_fingerprint": base_receipt[ + "plugin_descriptor_fingerprint" + ], + "plugin_implementation_fingerprint": base_receipt[ + "plugin_implementation_fingerprint" + ], + "index_representation_fingerprint": base_receipt[ + "index_representation_fingerprint" + ], + } + if base_receipt else {} ) index_manager._ensure_collection_exists(collection_name) @@ -147,7 +211,6 @@ def index_pr_files(request: PRIndexRequest): # Recover the exact plugin-owned repository state from the PR target # branch. The host never interprets the snapshots; it only validates, # overlays changed artifacts, and asks the selected plugins to rebuild. - target_branch = request.base_branch or request.branch representation_fingerprint = ( index_manager.index_representation_fingerprint ) @@ -322,6 +385,10 @@ def index_pr_files(request: PRIndexRequest): base_branch=target_branch, source_revision=request.source_revision, base_revision=request.base_revision, + base_generation_manifest_sha256=( + base_receipt["generation_manifest_sha256"] + if base_receipt else "" + ), files=request.files, requested_plugin_ids=requested_plugin_ids, repository_plugin_ids=repository_plugins, @@ -344,9 +411,32 @@ def index_pr_files(request: PRIndexRequest): ), snapshots=snapshots, ) - if is_complete_reusable_generation( - old_pr_points, - generation_fingerprint, + reusable_receipt = ( + read_pr_overlay_generation( + index_manager.qdrant_client, + collection_name, + workspace=request.workspace, + project=request.project, + pr_number=request.pr_number, + branch=request.branch, + base_branch=target_branch, + source_revision=request.source_revision, + base_revision=request.base_revision, + base_generation_manifest_sha256=base_receipt[ + "generation_manifest_sha256" + ], + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + ) + if base_receipt else None + ) + if reusable_receipt is not None or ( + base_receipt is None + and is_complete_reusable_generation( + old_pr_points, generation_fingerprint + ) ): architecture_points = sum( 1 @@ -364,12 +454,14 @@ def index_pr_files(request: PRIndexRequest): ) return { "status": "reused", + **base_generation_receipt, "pr_number": request.pr_number, "files_processed": len(request.files), "chunks_indexed": len(old_pr_points), "chunks_failed": 0, "architecture_packets_indexed": architecture_points, "generation_fingerprint": generation_fingerprint, + **(reusable_receipt or {}), "overlay_representation_fingerprint": ( overlay_representation_fingerprint ), @@ -497,6 +589,11 @@ def index_pr_files(request: PRIndexRequest): ) chunk.metadata["pr_source_revision"] = request.source_revision chunk.metadata["pr_base_revision"] = request.base_revision + if base_receipt: + chunk.metadata["pr_base_generation_manifest_sha256"] = ( + base_receipt["generation_manifest_sha256"] + ) + chunk.metadata["pr_overlay_base_branch"] = target_branch chunk.metadata["indexed_at"] = datetime.now(timezone.utc).isoformat() point_id_branch = f"__pr__/{request.pr_number}/{request.branch}" @@ -593,16 +690,71 @@ def index_pr_files(request: PRIndexRequest): ) node.metadata["pr_source_revision"] = request.source_revision node.metadata["pr_base_revision"] = request.base_revision + if base_receipt: + node.metadata["pr_base_generation_manifest_sha256"] = ( + base_receipt["generation_manifest_sha256"] + ) + node.metadata["pr_overlay_base_branch"] = target_branch node.metadata["indexed_at"] = datetime.now(timezone.utc).isoformat() - successful = index_manager._file_ops._replace_points( - [*chunks, *architecture_nodes], - old_pr_points, - collection_name, - request.workspace, - request.project, - point_id_branch, - mutation_lease.assert_owned, - ) + overlay_receipt = {} + if generation_fingerprint and base_receipt: + identity_metadata = { + "plugin_ids": list( + capabilities.repository_plugins + if capabilities is not None else stored_plugin_ids + ), + "plugin_fingerprint": ( + capabilities.fingerprint + if capabilities is not None else stored_fingerprint + ) or ZERO_FINGERPRINT, + "plugin_descriptor_fingerprint": ( + capabilities.descriptor_fingerprint + if capabilities is not None + else _stored_descriptor_fingerprint + ) or ZERO_FINGERPRINT, + "plugin_implementation_fingerprint": ( + implementation_fingerprint or ZERO_FINGERPRINT + ), + INDEX_REPRESENTATION_PAYLOAD_KEY: representation_fingerprint, + PR_OVERLAY_REPRESENTATION_PAYLOAD_KEY: ( + overlay_representation_fingerprint + ), + } + + successful, overlay_receipt = ( + index_manager._file_ops.replace_pr_overlay_generation( + [*chunks, *architecture_nodes], + old_pr_points, + collection_name, + request.workspace, + request.project, + point_id_branch, + mutation_lease.assert_owned, + pr_number=request.pr_number, + branch=request.branch, + base_branch=target_branch, + source_revision=request.source_revision, + base_revision=request.base_revision, + base_generation_manifest_sha256=base_receipt[ + "generation_manifest_sha256" + ], + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + identity_metadata=identity_metadata, + ) + ) + else: + successful = index_manager._file_ops._replace_points( + [*chunks, *architecture_nodes], + old_pr_points, + collection_name, + request.workspace, + request.project, + point_id_branch, + mutation_lease.assert_owned, + ) skipped_points = ( len(chunks) + len(architecture_nodes) - successful ) @@ -617,6 +769,7 @@ def index_pr_files(request: PRIndexRequest): return { "status": "indexed", + **base_generation_receipt, "pr_number": request.pr_number, "files_processed": len(request.files), "chunks_indexed": successful, @@ -625,6 +778,7 @@ def index_pr_files(request: PRIndexRequest): "skipped_files": list(split_skipped_paths), "architecture_packets_indexed": len(architecture_nodes), "generation_fingerprint": generation_fingerprint, + **overlay_receipt, "overlay_representation_fingerprint": ( overlay_representation_fingerprint ), @@ -664,16 +818,25 @@ def index_pr_files(request: PRIndexRequest): @router.delete("/index/pr-files/{workspace}/{project}/{pr_number}") -def delete_pr_files(workspace: str, project: str, pr_number: int): +def delete_pr_files( + workspace: str, + project: str, + pr_number: int, + collection_target: str | None = Query(default=None), +): """Delete all indexed points for a specific PR.""" index_manager = _get_index_manager() try: - with index_manager.project_mutation( + with index_manager.pr_overlay_mutation( workspace, project, + pr_number, "delete-pr-overlay", ) as lease: - collection_name = index_manager._get_project_collection_name(workspace, project) + collection_name = ( + collection_target + or index_manager._get_project_collection_name(workspace, project) + ) if not index_manager._collection_manager.collection_exists(collection_name): return {"status": "skipped", "message": "Collection does not exist"} @@ -683,7 +846,10 @@ def delete_pr_files(workspace: str, project: str, pr_number: int): collection_name=collection_name, points_selector=Filter( must=[ - FieldCondition(key="pr_number", match=MatchValue(value=pr_number)) + FieldCondition(key="workspace", match=MatchValue(value=workspace)), + FieldCondition(key="project", match=MatchValue(value=project)), + FieldCondition(key="pr", match=MatchValue(value=True)), + FieldCondition(key="pr_number", match=MatchValue(value=pr_number)), ] ) ) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py index 9e9c0894..52eaef43 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py @@ -5,6 +5,15 @@ from qdrant_client.models import Filter, FieldCondition, MatchAny, MatchValue from ..models import QueryRequest, PRContextRequest, DeterministicContextRequest +from ...core.revision_binding import ( + require_repository_generation, + require_same_repository_generation, +) +from ...core.repository_overlay import IncrementalIndexPreconditionError +from ...core.pr_overlay_manifest import ( + PR_OVERLAY_MANIFEST_PAYLOAD_KEY, + read_pr_overlay_generation, +) logger = logging.getLogger(__name__) router = APIRouter(tags=["query"]) @@ -15,6 +24,11 @@ def _get_singletons(): return index_manager, query_service +def _optional_string(value) -> Optional[str]: + """Ignore absent/loose legacy DTO attributes instead of binding them.""" + return value if isinstance(value, str) and value else None + + def _authoritative_pr_branch(request: PRContextRequest) -> Optional[str]: """Return target-branch truth for hybrid PR retrieval. @@ -28,20 +42,85 @@ def _authoritative_pr_branch(request: PRContextRequest) -> Optional[str]: return request.branch +def _require_complete_pr_overlay_binding( + *, + pr_number: Optional[int], + target_branch: Optional[str], + source_revision: Optional[str], + base_revision: Optional[str], + base_generation_manifest: Optional[str], + pr_generation_fingerprint: Optional[str], + pr_overlay_manifest: Optional[str], +) -> bool: + """Reject a claimed exact overlay whose identity is incomplete.""" + overlay_binding_requested = bool( + pr_generation_fingerprint or pr_overlay_manifest + ) + if not overlay_binding_requested: + return False + if not all(( + pr_number, + target_branch, + source_revision, + base_revision, + base_generation_manifest, + pr_generation_fingerprint, + pr_overlay_manifest, + )): + raise IncrementalIndexPreconditionError( + "revision-bound PR overlay requires PR number, one authoritative " + "branch, source/base revisions, and both generation receipts" + ) + return True + + @router.post("/query/search") def semantic_search(request: QueryRequest): """Perform semantic search.""" - _, query_service = _get_singletons() + index_manager, query_service = _get_singletons() try: + repository_revision = _optional_string( + request.repository_revision + ) + generation_manifest = _optional_string( + request.repository_generation_manifest_sha256 + ) + collection_target = _optional_string(request.collection_target) + receipt = None + if repository_revision: + receipt = require_repository_generation( + index_manager=index_manager, + workspace=request.workspace, + project=request.project, + branch=request.branch, + revision=repository_revision, + generation_manifest_sha256=generation_manifest, + collection_target=collection_target, + ) results = query_service.semantic_search( query=request.query, workspace=request.workspace, project=request.project, branch=request.branch, top_k=request.top_k, - filter_language=request.filter_language + filter_language=request.filter_language, + expected_revision=repository_revision, + collection_target=( + receipt["_collection_target"] if receipt else collection_target + ), ) + if receipt: + require_same_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=request.branch, + revision=repository_revision, + receipt=receipt, + ) return {"results": results} + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) except Exception as e: logger.error(f"Error performing search: {e}") raise HTTPException(status_code=500, detail=str(e)) @@ -75,10 +154,48 @@ def get_pr_context(request: PRContextRequest): } } + source_revision = _optional_string(request.source_revision) + base_revision = _optional_string(request.base_revision) + base_generation_manifest = _optional_string( + request.base_generation_manifest_sha256 + ) + pr_generation_fingerprint = _optional_string( + request.pr_generation_fingerprint + ) + pr_overlay_manifest = _optional_string( + request.pr_overlay_generation_manifest_sha256 + ) + collection_target = _optional_string(request.collection_target) + exact_overlay_binding = _require_complete_pr_overlay_binding( + pr_number=request.pr_number, + target_branch=authoritative_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest=base_generation_manifest, + pr_generation_fingerprint=pr_generation_fingerprint, + pr_overlay_manifest=pr_overlay_manifest, + ) + receipt = None + overlay_receipt = None + if base_revision: + receipt = require_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=authoritative_branch, + revision=base_revision, + generation_manifest_sha256=base_generation_manifest, + collection_target=collection_target, + ) pr_results = [] - collection_name = index_manager._get_project_collection_name( - request.workspace, - request.project, + collection_name = ( + receipt["_collection_target"] + if receipt + else collection_target + or index_manager._get_project_collection_name( + request.workspace, + request.project, + ) ) preflight_branches = [authoritative_branch] if query_service._collection_or_alias_exists(collection_name): @@ -89,6 +206,30 @@ def get_pr_context(request: PRContextRequest): # HYBRID MODE: Query PR-indexed data first if pr_number is provided if request.pr_number: + if exact_overlay_binding: + overlay_receipt = read_pr_overlay_generation( + index_manager.qdrant_client, + collection_name, + workspace=request.workspace, + project=request.project, + pr_number=request.pr_number, + branch=request.branch or authoritative_branch, + base_branch=authoritative_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_manifest, + generation_fingerprint=pr_generation_fingerprint, + overlay_representation_fingerprint=( + index_manager.pr_overlay_representation_fingerprint + ), + expected_manifest_sha256=( + pr_overlay_manifest + ), + ) + if overlay_receipt is None: + raise IncrementalIndexPreconditionError( + "requested PR overlay generation is unavailable" + ) pr_results = _query_pr_indexed_data( index_manager=index_manager, query_service=query_service, @@ -98,7 +239,12 @@ def get_pr_context(request: PRContextRequest): changed_files=request.changed_files, query_texts=request.diff_snippets or [], pr_title=request.pr_title, - top_k=request.top_k or 15 + top_k=request.top_k or 15, + collection_target=collection_name, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_manifest, + pr_generation_fingerprint=pr_generation_fingerprint, ) logger.info(f"Hybrid mode: Found {len(pr_results)} PR-specific chunks for PR #{request.pr_number}") @@ -118,7 +264,12 @@ def get_pr_context(request: PRContextRequest): # branch must not enter the repository branch query a second time. base_branch=None if request.pr_number else request.base_branch, deleted_files=request.deleted_files or [], - exclude_pr_files=(request.all_pr_changed_files or []) if request.pr_number else [] + exclude_pr_files=(request.all_pr_changed_files or []) if request.pr_number else [], + expected_revisions=( + {authoritative_branch: base_revision} + if base_revision else None + ), + collection_target=collection_name, ) # Merge PR results with branch results (PR first, then branch) @@ -155,7 +306,41 @@ def get_pr_context(request: PRContextRequest): "pr_number": request.pr_number } + if receipt: + require_same_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=authoritative_branch, + revision=base_revision, + receipt=receipt, + ) + if request.pr_number and overlay_receipt: + current_overlay = read_pr_overlay_generation( + index_manager.qdrant_client, + collection_name, + workspace=request.workspace, + project=request.project, + pr_number=request.pr_number, + branch=request.branch or authoritative_branch, + base_branch=authoritative_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_manifest, + generation_fingerprint=pr_generation_fingerprint, + overlay_representation_fingerprint=( + index_manager.pr_overlay_representation_fingerprint + ), + expected_manifest_sha256=None, + ) + if current_overlay != overlay_receipt: + raise IncrementalIndexPreconditionError( + "PR overlay generation changed while context was retrieved" + ) + return {"context": context} + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) except Exception as e: logger.error(f"Error getting PR context: {e}") raise HTTPException(status_code=500, detail=str(e)) @@ -170,7 +355,12 @@ def _query_pr_indexed_data( changed_files: List[str], query_texts: List[str], pr_title: Optional[str], - top_k: int = 15 + top_k: int = 15, + collection_target: Optional[str] = None, + source_revision: Optional[str] = None, + base_revision: Optional[str] = None, + base_generation_manifest_sha256: Optional[str] = None, + pr_generation_fingerprint: Optional[str] = None, ) -> List[Dict]: """ Query PR-indexed chunks from the main collection. @@ -180,9 +370,22 @@ def _query_pr_indexed_data( wasting an embedding call on a fabricated query. """ try: - collection_name = index_manager._get_project_collection_name(workspace, project) + collection_name = ( + collection_target + or index_manager._get_project_collection_name(workspace, project) + ) + exact_binding = all(( + source_revision, + base_revision, + base_generation_manifest_sha256, + pr_generation_fingerprint, + )) if not index_manager._collection_manager.collection_exists(collection_name): + if exact_binding: + raise IncrementalIndexPreconditionError( + "revision-bound PR overlay collection is unavailable" + ) return [] query_parts = [] @@ -194,8 +397,21 @@ def _query_pr_indexed_data( pr_filter = Filter( must=[ FieldCondition(key="pr", match=MatchValue(value=True)), - FieldCondition(key="pr_number", match=MatchValue(value=pr_number)) - ] + FieldCondition(key="pr_number", match=MatchValue(value=pr_number)), + *( + [ + FieldCondition(key="pr_source_revision", match=MatchValue(value=source_revision)), + FieldCondition(key="pr_base_revision", match=MatchValue(value=base_revision)), + FieldCondition(key="pr_base_generation_manifest_sha256", match=MatchValue(value=base_generation_manifest_sha256)), + FieldCondition(key="pr_generation_fingerprint", match=MatchValue(value=pr_generation_fingerprint)), + ] + if exact_binding else [] + ), + ], + must_not=[FieldCondition( + key=PR_OVERLAY_MANIFEST_PAYLOAD_KEY, + match=MatchValue(value=True), + )], ) direct_file_results = _fetch_direct_pr_file_chunks( @@ -214,7 +430,21 @@ def _query_pr_indexed_data( with_payload=True, with_vectors=False ) + if exact_binding: + for point in results: + payload = point.payload or {} + if any(payload.get(key) != value for key, value in ( + ("pr_source_revision", source_revision), + ("pr_base_revision", base_revision), + ("pr_base_generation_manifest_sha256", base_generation_manifest_sha256), + ("pr_generation_fingerprint", pr_generation_fingerprint), + )): + raise IncrementalIndexPreconditionError( + "PR point is outside the requested overlay generation" + ) accepted = query_service._accept_stored_points(results) + if not isinstance(accepted, list): + accepted = results formatted = _format_pr_results(accepted[:top_k]) return _merge_pr_results(direct_file_results, formatted) @@ -238,6 +468,8 @@ def _query_pr_indexed_data( return _merge_pr_results(direct_file_results, formatted) except Exception as e: + if exact_binding: + raise logger.warning(f"Error querying PR-indexed data: {e}") return [] @@ -350,8 +582,70 @@ def get_deterministic_context(request: DeterministicContextRequest): No language-specific parsing needed - tree-sitter already did it during indexing. Predictable: same input = same output. """ - _, query_service = _get_singletons() + index_manager, query_service = _get_singletons() try: + target_branch = request.branches[0] if request.branches else None + source_revision = _optional_string(request.source_revision) + base_revision = _optional_string(request.base_revision) + base_generation_manifest = _optional_string( + request.base_generation_manifest_sha256 + ) + pr_generation_fingerprint = _optional_string( + request.pr_generation_fingerprint + ) + pr_overlay_manifest = _optional_string( + request.pr_overlay_generation_manifest_sha256 + ) + collection_target = _optional_string(request.collection_target) + receipt = None + overlay_receipt = None + exact_overlay_binding = _require_complete_pr_overlay_binding( + pr_number=request.pr_number, + target_branch=target_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest=base_generation_manifest, + pr_generation_fingerprint=pr_generation_fingerprint, + pr_overlay_manifest=pr_overlay_manifest, + ) + if base_revision and target_branch: + if len(request.branches) != 1: + raise IncrementalIndexPreconditionError( + "revision-bound deterministic context requires exactly one authoritative branch" + ) + receipt = require_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=target_branch, + revision=base_revision, + generation_manifest_sha256=base_generation_manifest, + collection_target=collection_target, + ) + if exact_overlay_binding: + overlay_receipt = read_pr_overlay_generation( + index_manager.qdrant_client, + receipt["_collection_target"], + workspace=request.workspace, + project=request.project, + pr_number=request.pr_number, + branch=target_branch, + base_branch=target_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_manifest, + generation_fingerprint=pr_generation_fingerprint, + overlay_representation_fingerprint=( + index_manager.pr_overlay_representation_fingerprint + ), + expected_manifest_sha256=( + pr_overlay_manifest + ), + ) + if overlay_receipt is None: + raise IncrementalIndexPreconditionError( + "requested PR overlay generation is unavailable" + ) context = query_service.get_deterministic_context( workspace=request.workspace, project=request.project, @@ -360,9 +654,52 @@ def get_deterministic_context(request: DeterministicContextRequest): limit_per_file=request.limit_per_file or 10, pr_number=request.pr_number, pr_changed_files=request.pr_changed_files, - additional_identifiers=request.additional_identifiers + additional_identifiers=request.additional_identifiers, + expected_revisions=( + {target_branch: base_revision} + if base_revision and target_branch else None + ), + pr_source_revision=source_revision, + pr_base_revision=base_revision, + pr_base_generation_manifest_sha256=base_generation_manifest, + pr_generation_fingerprint=pr_generation_fingerprint, + collection_target=( + receipt["_collection_target"] if receipt else collection_target + ), ) + if receipt: + require_same_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=target_branch, + revision=base_revision, + receipt=receipt, + ) + if overlay_receipt: + second_overlay = read_pr_overlay_generation( + index_manager.qdrant_client, + receipt["_collection_target"], + workspace=request.workspace, + project=request.project, + pr_number=request.pr_number, + branch=target_branch, + base_branch=target_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_manifest, + generation_fingerprint=pr_generation_fingerprint, + overlay_representation_fingerprint=( + index_manager.pr_overlay_representation_fingerprint + ), + ) + if second_overlay != overlay_receipt: + raise IncrementalIndexPreconditionError( + "PR overlay generation changed while context was retrieved" + ) return {"context": context} + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) except Exception as e: logger.error(f"Error getting deterministic context: {e}") raise HTTPException(status_code=500, detail=str(e)) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py index 8e430550..562d5b9e 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py @@ -140,7 +140,14 @@ def close(self) -> None: class ProjectMutationCoordinator: - """Serialize collection mutations for one workspace/project across workers.""" + """Serialize mutations that share one RAG publication resource. + + Legacy indexes share a project-wide collection. Exact generations have an + immutable collection target, while a published branch also has one mutable + human-readable head alias. ``publication_scope`` serializes only that + branch head, so main and develop can build concurrently without allowing + two generations of the same branch to race its current alias. + """ def __init__( self, @@ -166,9 +173,23 @@ def __init__( ) @staticmethod - def _project_key(workspace: str, project: str) -> str: + def _resource_key( + workspace: str, + project: str, + collection_target: Optional[str] = None, + publication_scope: Optional[str] = None, + ) -> str: + # Legacy indexing keeps the historical project-wide key because its + # branches share one physical alias. Exact branch generations supply + # their distinct collection target, so independent branch snapshots + # may proceed concurrently without weakening same-collection safety. + resource = ( + publication_scope + or collection_target + or "project-shared-collection" + ) digest = hashlib.sha256( - f"{workspace}\0{project}".encode("utf-8") + f"{workspace}\0{project}\0{resource}".encode("utf-8") ).hexdigest() return f"codecrow:rag:mutation:{digest}" @@ -178,6 +199,9 @@ def acquire( workspace: str, project: str, operation: str, + *, + collection_target: Optional[str] = None, + publication_scope: Optional[str] = None, ) -> Iterator[MutationLease]: token = uuid.uuid4().hex if not self.enabled or self._client is None: @@ -185,7 +209,12 @@ def acquire( yield lease return - key = self._project_key(workspace, project) + key = self._resource_key( + workspace, + project, + collection_target, + publication_scope, + ) operation_key = f"codecrow:rag:operation:{token}" deadline = time.monotonic() + self.acquire_timeout_seconds while True: @@ -221,7 +250,16 @@ def acquire( ) from exception if time.monotonic() >= deadline: raise MutationLeaseUnavailable( - f"another RAG mutation is active for {workspace}/{project}" + "another RAG mutation is active for " + f"{workspace}/{project}" + + ( + f" publication {publication_scope}" + if publication_scope + else ( + f" collection {collection_target}" + if collection_target else "" + ) + ) ) time.sleep(0.1) @@ -234,10 +272,11 @@ def acquire( ) lease.start_renewal() logger.info( - "Acquired RAG mutation lease operation=%s workspace=%s project=%s operation_id=%s", + "Acquired RAG mutation lease operation=%s workspace=%s project=%s collection=%s operation_id=%s", operation, workspace, project, + publication_scope or collection_target or "project-shared-collection", token, ) try: diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py new file mode 100644 index 00000000..75c5977b --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py @@ -0,0 +1,379 @@ +"""Content-addressed membership manifests for repository index generations.""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Iterable, Mapping, Sequence +from typing import Any + +from llama_index.core.schema import TextNode +from qdrant_client.models import FieldCondition, Filter, MatchValue, PointStruct + + +GENERATION_MANIFEST_PAYLOAD_KEY = "repository_generation_manifest" +GENERATION_MEMBER_DIGEST_PAYLOAD_KEY = "generation_member_sha256" +GENERATION_SCHEMA = "codecrow.repository-index-generation" +INDEX_SELECTION_POLICY_SCHEMA = "codecrow.repository-index-selection" +GENERATION_MANIFEST_PATH = ( + "__analysis_state__/repository-generation-manifest/000000.state" +) + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + + +class GenerationManifestError(RuntimeError): + """A repository generation is incomplete or its seal is inconsistent.""" + + +def _canonical_json(value: Any) -> str: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +def _canonical_vector(value: Any) -> Any: + """Canonicalize cosine vectors as Qdrant stores them. + + Qdrant normalizes cosine vectors and persists float32 values. Rounding the + normalized representation avoids treating harmless transport precision as + a generation-integrity failure while still binding every vector value. + """ + if isinstance(value, Mapping): + return { + str(key): _canonical_vector(item) + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) + } + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + values = list(value) + if values and all(isinstance(item, (int, float)) for item in values): + norm = math.sqrt(sum(float(item) ** 2 for item in values)) + if norm: + return [round(float(item) / norm, 7) for item in values] + return [0.0 for _ in values] + return [_canonical_vector(item) for item in values] + return value + + +def canonical_index_selection_policy( + include_patterns: Sequence[str] | None, + exclude_patterns: Sequence[str] | None, +) -> dict[str, Any]: + """Return the order-independent effective repository selection policy.""" + + for label, patterns in ( + ("include", include_patterns), + ("exclude", exclude_patterns), + ): + if patterns is not None and ( + isinstance(patterns, (str, bytes)) + or not all(isinstance(pattern, str) for pattern in patterns) + ): + raise GenerationManifestError( + f"repository index {label} patterns are invalid" + ) + return { + "schema": INDEX_SELECTION_POLICY_SCHEMA, + "includePatterns": sorted(set(include_patterns or ())), + "excludePatterns": sorted(set(exclude_patterns or ())), + } + + +def compute_index_selection_policy_sha256( + include_patterns: Sequence[str] | None, + exclude_patterns: Sequence[str] | None, +) -> str: + """Digest the canonical effective repository selection policy.""" + + policy = canonical_index_selection_policy( + include_patterns, + exclude_patterns, + ) + return hashlib.sha256(_canonical_json(policy).encode("utf-8")).hexdigest() + + +def compute_generation_member_digest( + point_id: object, + payload: Mapping[str, Any], + vector: object, +) -> str: + """Bind one persisted point's deterministic identity, payload and vector. + + ``indexed_at`` is operational metadata, not representation content. It is + excluded so rebuilding identical repository content can produce the same + generation identity. The digest field itself is also excluded to avoid a + recursive projection. + """ + content_payload = { + key: value + for key, value in payload.items() + if key not in { + GENERATION_MEMBER_DIGEST_PAYLOAD_KEY, + "indexed_at", + } + } + encoded = _canonical_json({ + "id": str(point_id), + "payload": content_payload, + "vector": _canonical_vector(vector), + }).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def compute_generation_members_digest( + members: Iterable[tuple[object, str]], +) -> str: + """Return the order-independent aggregate identity for generation members.""" + normalized = [] + seen_ids = set() + for point_id, member_digest in members: + normalized_id = str(point_id) + if normalized_id in seen_ids: + raise GenerationManifestError( + "repository generation contains duplicate point identities" + ) + if not isinstance(member_digest, str) or not _SHA256_RE.fullmatch( + member_digest + ): + raise GenerationManifestError( + "repository generation member is missing a valid content digest" + ) + seen_ids.add(normalized_id) + normalized.append((normalized_id, member_digest)) + + hasher = hashlib.sha256() + for point_id, member_digest in sorted(normalized): + encoded_id = point_id.encode("utf-8") + hasher.update(len(encoded_id).to_bytes(8, "big")) + hasher.update(encoded_id) + hasher.update(bytes.fromhex(member_digest)) + return hasher.hexdigest() + + +def verified_generation_member(point) -> tuple[object, str]: + """Recompute and verify one persisted generation member's full content.""" + payload = point.payload or {} + stored_digest = payload.get(GENERATION_MEMBER_DIGEST_PAYLOAD_KEY) + if not is_sha256_hex(stored_digest): + raise GenerationManifestError( + "repository generation member is missing a valid content digest" + ) + computed_digest = compute_generation_member_digest( + point.id, + payload, + point.vector, + ) + if computed_digest != stored_digest: + raise GenerationManifestError( + "repository generation member content digest does not match its " + "persisted payload and vector" + ) + return point.id, computed_digest + + +def _generation_filter(branch: str, commit: str) -> Filter: + return Filter( + must=[ + FieldCondition(key="branch", match=MatchValue(value=branch)), + FieldCondition(key="commit", match=MatchValue(value=commit)), + ], + must_not=[ + FieldCondition(key="pr", match=MatchValue(value=True)), + ], + ) + + +def collect_generation_members( + client, + collection_name: str, + branch: str, + commit: str, +) -> list[tuple[object, str]]: + """Read all unsealed members of one exact pending generation.""" + members = [] + offset = None + while True: + points, offset = client.scroll( + collection_name=collection_name, + scroll_filter=_generation_filter(branch, commit), + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + for point in points: + payload = point.payload or {} + if payload.get(GENERATION_MANIFEST_PAYLOAD_KEY) is True: + raise GenerationManifestError( + "pending repository generation already contains a manifest" + ) + members.append(verified_generation_member(point)) + if offset is None: + break + return members + + +def seal_generation_members( + client, + collection_name: str, + branch: str, + commit: str, +) -> int: + """Persist member digests against Qdrant's actual stored vectors. + + Qdrant may normalize cosine vectors and reduce their precision. A digest + made from the embedding request is therefore not necessarily a digest of + the representation which will later be verified. Seal after every member + is stored, then collect/verify the same representation before publishing a + generation manifest. + """ + offset = None + sealed = 0 + while True: + points, offset = client.scroll( + collection_name=collection_name, + scroll_filter=_generation_filter(branch, commit), + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + replacements = [] + for point in points: + payload = dict(point.payload or {}) + if payload.get(GENERATION_MANIFEST_PAYLOAD_KEY) is True: + raise GenerationManifestError( + "pending repository generation already contains a manifest" + ) + payload[GENERATION_MEMBER_DIGEST_PAYLOAD_KEY] = ( + compute_generation_member_digest(point.id, payload, point.vector) + ) + replacements.append(PointStruct( + id=point.id, + vector=point.vector, + payload=payload, + )) + if replacements: + client.upsert( + collection_name=collection_name, + points=replacements, + wait=True, + ) + sealed += len(replacements) + if offset is None: + break + return sealed + + +def generation_manifest_content( + *, + workspace: str, + project: str, + branch: str, + commit: str, + member_count: int, + members_sha256: str, + source_tree_sha256: str, + index_include_patterns: Sequence[str], + index_exclude_patterns: Sequence[str], + index_selection_policy_sha256: str, +) -> str: + """Serialize the immutable generation seal content.""" + selection_policy = canonical_index_selection_policy( + index_include_patterns, + index_exclude_patterns, + ) + return _canonical_json({ + "branch": branch, + "commit": commit, + "indexSelectionPolicy": selection_policy, + "indexSelectionPolicySha256": index_selection_policy_sha256, + "memberCount": member_count, + "membersSha256": members_sha256, + "project": project, + "schema": GENERATION_SCHEMA, + "sourceTreeSha256": source_tree_sha256, + "workspace": workspace, + }) + + +def build_generation_manifest_node( + *, + workspace: str, + project: str, + branch: str, + commit: str, + member_count: int, + members_sha256: str, + source_tree_sha256: str, + index_include_patterns: Sequence[str], + index_exclude_patterns: Sequence[str], + identity_metadata: Mapping[str, Any], +) -> TextNode: + """Build the single zero-vector state node that seals a generation.""" + if member_count < 1: + raise GenerationManifestError( + "repository generation cannot be sealed without members" + ) + if not _SHA256_RE.fullmatch(members_sha256): + raise GenerationManifestError( + "repository generation aggregate digest is invalid" + ) + if not _SHA256_RE.fullmatch(source_tree_sha256): + raise GenerationManifestError( + "repository generation source-tree digest is invalid" + ) + selection_policy = canonical_index_selection_policy( + index_include_patterns, + index_exclude_patterns, + ) + selection_policy_sha256 = compute_index_selection_policy_sha256( + selection_policy["includePatterns"], + selection_policy["excludePatterns"], + ) + content = generation_manifest_content( + workspace=workspace, + project=project, + branch=branch, + commit=commit, + member_count=member_count, + members_sha256=members_sha256, + source_tree_sha256=source_tree_sha256, + index_include_patterns=selection_policy["includePatterns"], + index_exclude_patterns=selection_policy["excludePatterns"], + index_selection_policy_sha256=selection_policy_sha256, + ) + content_sha256 = hashlib.sha256(content.encode("utf-8")).hexdigest() + return TextNode( + text=content, + metadata={ + "workspace": workspace, + "project": project, + "branch": branch, + "commit": commit, + "path": GENERATION_MANIFEST_PATH, + "language": "repository-state", + "filetype": "state", + GENERATION_MANIFEST_PAYLOAD_KEY: True, + "generation_schema": GENERATION_SCHEMA, + "generation_member_count": member_count, + "generation_members_sha256": members_sha256, + "generation_manifest_sha256": content_sha256, + "source_tree_sha256": source_tree_sha256, + "index_include_patterns": selection_policy["includePatterns"], + "index_exclude_patterns": selection_policy["excludePatterns"], + "index_selection_policy_sha256": selection_policy_sha256, + **dict(identity_metadata), + }, + ) + + +def is_sha256_hex(value: object) -> bool: + """Return whether ``value`` is one canonical lower-case SHA-256 digest.""" + return isinstance(value, str) and _SHA256_RE.fullmatch(value) is not None diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/branch_manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/branch_manager.py index abd33ed7..f984a8be 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/branch_manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/branch_manager.py @@ -114,10 +114,10 @@ def preserve_other_branch_points( offset = None total_yielded = 0 - + try: while True: - results = self.client.scroll( + points, next_offset = self.client.scroll( collection_name=collection_name, limit=batch_size, offset=offset, @@ -132,16 +132,15 @@ def preserve_other_branch_points( with_payload=True, with_vectors=True ) - points, next_offset = results - + if points: total_yielded += len(points) yield points - - if next_offset is None or len(points) < batch_size: + + if next_offset is None: break offset = next_offset - + logger.info(f"Streamed {total_yielded} points from other branches") except Exception as e: logger.warning(f"Could not read existing points: {e}") @@ -189,10 +188,10 @@ def stream_copy_points_to_collection( f"Re-embedding required for all branches." ) return 0 - except Exception as e: logger.warning(f"Could not verify collection dimensions: {e}") - # Continue anyway - will fail at upsert if dimensions don't match + # Optional preservation remains best effort. The target branch can + # still be published without copying unrelated legacy branches. total_copied = 0 for batch in self.preserve_other_branch_points(source_collection, exclude_branch, batch_size): diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py index 873ed0ff..f2a43d84 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py @@ -9,7 +9,7 @@ import re import time import uuid -from typing import Callable, Optional, List +from typing import Callable, Mapping, Optional, List from qdrant_client import QdrantClient from qdrant_client.http.exceptions import UnexpectedResponse @@ -187,6 +187,30 @@ def resolve_alias(self, alias_name: str) -> Optional[str]: except Exception as e: logger.debug(f"Error resolving alias {alias_name}: {e}") return None + + def resolve_collection_target(self, collection_name: str) -> Optional[str]: + """Resolve an alias or direct collection without hiding backend errors. + + Mutation leases use this strict resolver so a transient alias lookup + failure cannot be mistaken for a direct collection. + """ + aliases = self.client.get_aliases().aliases + matching_aliases = [ + alias.collection_name + for alias in aliases + if alias.alias_name == collection_name + ] + if len(matching_aliases) > 1: + raise RuntimeError( + f"collection alias '{collection_name}' has multiple targets" + ) + if matching_aliases: + return matching_aliases[0] + + collections = self.client.get_collections().collections + if collection_name in {collection.name for collection in collections}: + return collection_name + return None def atomic_alias_swap( self, @@ -213,6 +237,63 @@ def atomic_alias_swap( change_aliases_operations=alias_operations ) logger.info(f"Alias swap completed: {alias_name} -> {new_collection}") + + def read_alias_targets(self, alias_names: List[str]) -> dict[str, Optional[str]]: + """Read several alias targets from one consistent Qdrant response.""" + requested = list(dict.fromkeys(name for name in alias_names if name)) + aliases = { + alias.alias_name: alias.collection_name + for alias in self.client.get_aliases().aliases + } + return {name: aliases.get(name) for name in requested} + + def atomic_assign_aliases( + self, + assignments: Mapping[str, Optional[str]], + ) -> None: + """Atomically point a set of aliases at already-validated collections. + + An immutable generation alias and its human-facing aliases must move in + the same Qdrant transaction. Reads that bind analysis to a historical + generation keep using the immutable alias; readable aliases are solely + the current branch and legacy-project pointers. + """ + desired = { + alias_name: collection_name + for alias_name, collection_name in assignments.items() + if alias_name + } + if not desired: + return + current = self.read_alias_targets(list(desired)) + operations = [] + for alias_name, collection_name in desired.items(): + if current.get(alias_name) == collection_name: + continue + if current.get(alias_name) is not None: + operations.append( + DeleteAliasOperation( + delete_alias=DeleteAlias(alias_name=alias_name) + ) + ) + if collection_name is not None: + operations.append( + CreateAliasOperation( + create_alias=CreateAlias( + alias_name=alias_name, + collection_name=collection_name, + ) + ) + ) + if not operations: + return + self.client.update_collection_aliases( + change_aliases_operations=operations + ) + logger.info( + "Atomically assigned Qdrant aliases: %s", + ", ".join(sorted(desired)), + ) def delete_alias(self, alias_name: str) -> bool: """Delete an alias.""" diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py index 856bad48..da808628 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py @@ -30,6 +30,13 @@ index_representation_fingerprint, observe_branch_representation, ) +from ..generation_manifest import ( + build_generation_manifest_node, + collect_generation_members, + compute_generation_members_digest, + seal_generation_members, +) +from ..source_tree import require_repository_source_tree_unchanged from .collection_manager import CollectionManager from .branch_manager import BranchManager from .point_operations import PointOperations @@ -48,6 +55,10 @@ def _plugin_identity_metadata( representation_fingerprint: Optional[str] = None, ): metadata = { + "plugin_ids": [], + "plugin_fingerprint": "sha256:" + "0" * 64, + "plugin_descriptor_fingerprint": "sha256:" + "0" * 64, + "plugin_implementation_fingerprint": "sha256:" + "0" * 64, INDEX_REPRESENTATION_PAYLOAD_KEY: ( representation_fingerprint or index_representation_fingerprint() @@ -482,6 +493,10 @@ def index_repository( preserve_other_branches: bool = False, include_patterns: Optional[List[str]] = None, exclude_patterns: Optional[List[str]] = None, + source_tree_sha256: Optional[str] = None, + source_tree=None, + seal_generation: bool = False, + publication_aliases: Optional[List[str]] = None, operation_id: Optional[str] = None, activation_guard: Optional[Callable[[], None]] = None, progress_callback: Optional[Callable[[dict], None]] = None, @@ -492,6 +507,7 @@ def report_progress( message: str, progress: Optional[int] = None, total: Optional[int] = None, + **details, ) -> None: if progress_callback is None: return @@ -500,6 +516,8 @@ def report_progress( event["progress"] = max(0, min(100, progress)) if total is not None: event["total"] = total + event.update({key: value for key, value in details.items() + if value is not None}) try: progress_callback(event) except Exception as exception: @@ -532,6 +550,13 @@ def report_progress( operation_id=operation_id, ) + activation_aliases = list(dict.fromkeys( + [alias_name, *(publication_aliases or [])] + )) + activation_alias_targets = self.collection_manager.read_alias_targets( + activation_aliases + ) + # Check existing collection and preserve other branch data using streaming old_alias_exists = self.collection_manager.alias_exists(alias_name) old_collection_exists = old_alias_exists or self.collection_manager.collection_exists(alias_name) @@ -549,7 +574,14 @@ def report_progress( # Get file list repository_file_list = list( - self.loader.iter_repository_files(repo_path_obj, include_patterns, exclude_patterns) + self.loader.iter_repository_files( + repo_path_obj, + include_patterns, + exclude_patterns, + expected_file_sha256=( + source_tree.file_sha256_by_path if source_tree else None + ), + ) ) logger.info( "Found %s repository files before plugin file policy for branch '%s'", @@ -673,6 +705,23 @@ def report_progress( skipped_file_paths: set[str] = set() preserved_point_count = 0 embedding_metrics = {"reused": 0, "embedded": 0} + estimated_chunks = None + + # Chunk totals are an estimate rather than an expensive second exact + # indexing pass. Progress remains useful even when estimation fails. + if progress_callback is not None: + try: + _, estimated_chunks = self.estimate_repository_size( + repo_path, include_patterns, exclude_patterns + ) + report_progress( + "estimating", + f"Estimated approximately {estimated_chunks} chunks", + 14, + estimatedChunks=estimated_chunks, + ) + except Exception as exception: + logger.warning("Could not estimate RAG progress chunks: %s", exception) try: # A main-only project must not carry stale non-target branches into @@ -700,6 +749,9 @@ def report_progress( f"Starting {total_batches} indexing batches", 18, total_batches, + totalBatches=total_batches, + indexedChunks=0, + estimatedChunks=estimated_chunks, ) # Architecture-only files still have to reach the repository @@ -715,6 +767,9 @@ def report_progress( documents = self.loader.load_file_batch( file_batch, repo_path_obj, workspace, project, branch, commit, strict=False, + expected_file_sha256=( + source_tree.file_sha256_by_path if source_tree else None + ), ) load_duration_ms = round( (time.perf_counter() - load_started) * 1000 @@ -829,6 +884,11 @@ def report_progress( round((time.perf_counter() - batch_started) * 1000), ) batch_progress = 18 + round(67 * batch_num / max(total_batches, 1)) + elapsed_ms = round((time.perf_counter() - operation_started) * 1000) + average_batch_ms = elapsed_ms / batch_num + estimated_remaining_ms = round( + average_batch_ms * max(total_batches - batch_num, 0) + ) report_progress( "indexing", ( @@ -838,6 +898,14 @@ def report_progress( ), batch_progress, total_batches, + indexedChunks=successful_chunks, + estimatedChunks=estimated_chunks, + completedBatches=batch_num, + totalBatches=total_batches, + batchDurationMs=round( + (time.perf_counter() - batch_started) * 1000 + ), + estimatedRemainingMs=estimated_remaining_ms, ) del documents @@ -949,6 +1017,73 @@ def report_progress( len(facts_nodes), ) + generation_manifest_sha256 = None + generation_manifest_points = 0 + if seal_generation: + report_progress( + "sealing", + "Sealing persisted vectors for generation integrity", + 91, + indexedChunks=successful_chunks, + estimatedChunks=estimated_chunks, + ) + seal_generation_members( + self.point_ops.client, + pending_collection_name, + branch, + commit, + ) + identity_metadata = _plugin_identity_metadata( + capabilities, + implementation_fingerprint, + self.representation_fingerprint, + ) + members = collect_generation_members( + self.point_ops.client, + pending_collection_name, + branch, + commit, + ) + if source_tree is not None: + require_repository_source_tree_unchanged( + repo_path_obj, + source_tree, + ) + if not source_tree_sha256: + raise RuntimeError( + "repository source-tree identity is required to seal an index generation" + ) + manifest = build_generation_manifest_node( + workspace=workspace, + project=project, + branch=branch, + commit=commit, + member_count=len(members), + members_sha256=compute_generation_members_digest(members), + source_tree_sha256=source_tree_sha256, + index_include_patterns=include_patterns or (), + index_exclude_patterns=exclude_patterns or (), + identity_metadata=identity_metadata, + ) + manifest_success, manifest_failed = ( + self.point_ops.process_and_upsert_chunks( + [manifest], + pending_collection_name, + workspace, + project, + branch, + operation_id=operation_id, + ) + ) + if manifest_success != 1 or manifest_failed: + raise RuntimeError( + "repository generation manifest could not be persisted" + ) + generation_manifest_points = 1 + generation_manifest_sha256 = manifest.metadata[ + "generation_manifest_sha256" + ] + logger.info( f"Streaming indexing complete: {document_count} files, " f"{successful_chunks}/{chunk_count} chunks indexed " @@ -960,7 +1095,11 @@ def report_progress( report_progress("verifying", "Verifying the pending vector collection", 94) pending_info = self.point_ops.client.get_collection(pending_collection_name) actual_point_count = int(pending_info.points_count or 0) - expected_point_count = preserved_point_count + successful_chunks + expected_point_count = ( + preserved_point_count + + successful_chunks + + generation_manifest_points + ) if actual_point_count != expected_point_count: raise RuntimeError( "Pending collection point count is incomplete: " @@ -973,29 +1112,32 @@ def report_progress( branch, ) ) - if target_branch_point_count != successful_chunks: + expected_target_branch_points = ( + successful_chunks + generation_manifest_points + ) + if target_branch_point_count != expected_target_branch_points: raise RuntimeError( "Pending target-branch point count is incomplete: " - f"branch={branch}, expected={successful_chunks}, " + f"branch={branch}, expected={expected_target_branch_points}, " f"actual={target_branch_point_count}" ) if activation_guard is not None: activation_guard() - observed_target = self.collection_manager.resolve_alias(alias_name) - if old_alias_exists and observed_target != actual_old_collection: - raise RuntimeError( - "Active RAG collection changed before pending activation" - ) - if not old_alias_exists and observed_target is not None: + observed_targets = self.collection_manager.read_alias_targets( + activation_aliases + ) + if observed_targets != activation_alias_targets: raise RuntimeError( - "RAG alias was created concurrently before pending activation" + "Active RAG alias changed before pending activation" ) activation_started = time.perf_counter() report_progress("activating", "Activating the completed vector collection", 97) - old_target = self._perform_atomic_swap( - alias_name, pending_collection_name, old_alias_exists + old_targets = self._perform_atomic_swap( + alias_name, + pending_collection_name, + activation_aliases, ) logger.info( "RAG pending collection activated operation_id=%s collection=%s " @@ -1015,10 +1157,18 @@ def report_progress( successful_chunks, ) except Exception: - self._rollback_atomic_swap(alias_name, old_target) + self._rollback_atomic_swap(old_targets) raise - if old_target and old_target != pending_collection_name: + old_target = old_targets.get(alias_name) + # Exact aliases identify prior sealed revisions that can still be + # selected by PR analysis. Their registry retention owns physical + # cleanup; a replacement build must not delete them here. + if ( + not seal_generation + and old_target + and old_target != pending_collection_name + ): self.collection_manager.delete_collection(old_target) except Exception as e: @@ -1048,6 +1198,11 @@ def report_progress( f"Indexed {document_count} files into {successful_chunks} chunks", 100, document_count, + indexedChunks=successful_chunks, + estimatedChunks=successful_chunks, + completedBatches=total_batches, + totalBatches=total_batches, + estimatedRemainingMs=0, ) return IndexStats( namespace=namespace, @@ -1058,33 +1213,29 @@ def report_progress( last_updated=datetime.now(timezone.utc).isoformat(), workspace=workspace, project=project, - branch=branch + branch=branch, + generation_manifest_sha256=generation_manifest_sha256, + source_tree_sha256=source_tree_sha256, + collection_target=alias_name, ) def _perform_atomic_swap( - self, - alias_name: str, - pending_collection_name: str, - old_alias_exists: bool - ) -> Optional[str]: - """Activate a complete pending collection and retain its rollback target.""" + self, + alias_name: str, + pending_collection_name: str, + activation_aliases: List[str], + ) -> dict[str, Optional[str]]: + """Activate a complete generation and its readable aliases together.""" logger.info("Performing atomic alias swap...") - - old_target = self.collection_manager.resolve_alias(alias_name) if old_alias_exists else None - self.collection_manager.atomic_alias_swap( - alias_name, - pending_collection_name, - old_alias_exists, + old_targets = self.collection_manager.read_alias_targets(activation_aliases) + self.collection_manager.atomic_assign_aliases( + {alias: pending_collection_name for alias in activation_aliases} ) - return old_target + return old_targets - def _rollback_atomic_swap(self, alias_name: str, old_target: Optional[str]) -> None: - """Restore the previously active collection after metadata publication fails.""" - if old_target: - self.collection_manager.atomic_alias_swap(alias_name, old_target, True) - return - if not self.collection_manager.delete_alias(alias_name): - logger.critical("Failed to remove newly activated alias %s during rollback", alias_name) + def _rollback_atomic_swap(self, old_targets: dict[str, Optional[str]]) -> None: + """Restore every alias changed during a failed metadata publication.""" + self.collection_manager.atomic_assign_aliases(old_targets) class FileOperations: @@ -1217,6 +1368,7 @@ def _replace_points( mutation_guard: Optional[Callable[[], None]] = None, ) -> int: """Upsert a prepared generation, delete stale IDs, and roll back on error.""" + old_records = list(old_records) old_points = {str(record.id): record for record in old_records} chunk_data = self.point_ops.prepare_chunks_for_embedding( nodes, @@ -1239,6 +1391,25 @@ def _replace_points( embedding_metrics["reused"], embedding_metrics["embedded"], ) + return self._replace_prepared_points( + new_points, + old_records, + collection_name, + mutation_guard, + allow_skipped=True, + ) + + def _replace_prepared_points( + self, + new_points, + old_records, + collection_name: str, + mutation_guard: Optional[Callable[[], None]] = None, + *, + allow_skipped: bool, + ) -> int: + """Publish prepared points with rollback and explicit skip semantics.""" + old_points = {str(record.id): record for record in old_records} new_ids = {str(point.id) for point in new_points} old_ids = set(old_points) new_only_ids = [ @@ -1264,6 +1435,15 @@ def _replace_points( str(point.id) for point in write_result.skipped_points } if skipped_ids: + if not allow_skipped: + self._restore_old_points( + collection_name, + old_points, + new_only_ids, + ) + raise RuntimeError( + "an exact index generation contains rejected points" + ) logger.warning( "Quarantining %s rejected points during incremental replacement", len(skipped_ids), @@ -1287,6 +1467,105 @@ def _replace_points( raise return write_result.successful + def replace_pr_overlay_generation( + self, + nodes, + old_records, + collection_name: str, + workspace: str, + project: str, + point_id_branch: str, + mutation_guard: Optional[Callable[[], None]] = None, + *, + pr_number: int, + branch: str, + base_branch: str, + source_revision: str, + base_revision: str, + base_generation_manifest_sha256: str, + generation_fingerprint: str, + overlay_representation_fingerprint: str, + identity_metadata, + ) -> tuple[int, dict]: + """Embed all PR members, seal them, and publish one complete set.""" + from ..pr_overlay_manifest import build_pr_overlay_manifest_node + + old_records = list(old_records) + old_points = {str(record.id): record for record in old_records} + chunk_data = self.point_ops.prepare_chunks_for_embedding( + nodes, workspace, project, point_id_branch + ) + new_points = self.point_ops.embed_and_create_points( + chunk_data, + reuse_records=old_points.values(), + ) + new_ids = [point.id for point in new_points] + if mutation_guard is not None: + mutation_guard() + successful, failed = self.point_ops.upsert_points( + collection_name, new_points + ) + if failed or successful != len(new_points): + self._delete_point_ids(collection_name, new_ids) + raise RuntimeError("PR overlay member write was incomplete") + try: + members = self.point_ops._seal_persisted_point_digests( + collection_name, new_points + ) + manifest_node, receipt = build_pr_overlay_manifest_node( + workspace=workspace, + project=project, + pr_number=pr_number, + branch=branch, + base_branch=base_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=( + base_generation_manifest_sha256 + ), + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + members=members, + identity_metadata=identity_metadata, + ) + manifest_data = self.point_ops.prepare_chunks_for_embedding( + [manifest_node], workspace, project, point_id_branch + ) + manifest_points = self.point_ops.embed_and_create_points( + manifest_data + ) + manifest_success, manifest_failed = self.point_ops.upsert_points( + collection_name, manifest_points + ) + if manifest_success != 1 or manifest_failed: + raise RuntimeError("PR overlay manifest write was incomplete") + except Exception: + self._delete_point_ids( + collection_name, + [*new_ids, *(point.id for point in locals().get("manifest_points", []))], + ) + raise + + active_ids = {str(point.id) for point in (*new_points, *manifest_points)} + stale_ids = [ + record.id for point_id, record in old_points.items() + if point_id not in active_ids + ] + try: + if mutation_guard is not None: + mutation_guard() + self._delete_point_ids(collection_name, stale_ids) + except Exception: + self._restore_old_points( + collection_name, + old_points, + [*new_ids, *(point.id for point in manifest_points)], + ) + raise + return successful, receipt + def _apply_change_set( self, updated_file_paths: List[str], diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py index 0a5ae6de..ffce9bdc 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py @@ -6,10 +6,14 @@ import logging import os +import hashlib +import json +import re from typing import Callable, Optional, List from llama_index.core import Settings from qdrant_client import QdrantClient +from qdrant_client.models import FieldCondition, Filter, MatchValue, PointStruct from ...models.config import RAGConfig, IndexStats from ...utils.utils import make_namespace, make_project_namespace @@ -24,6 +28,24 @@ from ..pr_overlay_representation import ( pr_overlay_representation_fingerprint, ) +from ..generation_manifest import ( + GENERATION_MANIFEST_PAYLOAD_KEY, + GENERATION_MEMBER_DIGEST_PAYLOAD_KEY, + build_generation_manifest_node, + collect_generation_members, + compute_generation_member_digest, + compute_generation_members_digest, +) +from .. import revision_preflight +from ..repository_overlay import IncrementalIndexPreconditionError +from ..revision_preflight_cache import ( + RevisionPreflightCache, + RevisionPreflightKey, +) +from ..source_tree import ( + compute_repository_source_tree_sha256, + verify_repository_source_tree, +) from .collection_manager import CollectionManager from .branch_manager import BranchManager @@ -34,6 +56,13 @@ logger = logging.getLogger(__name__) +def read_repository_revision_preflight(*args, **kwargs): + """Patchable module boundary around strict generation verification.""" + return revision_preflight.read_repository_revision_preflight( + *args, **kwargs + ) + + def _config_int(config, name: str, default: int) -> int: value = getattr(config, name, default) if isinstance(value, bool) or not isinstance(value, (int, str)): @@ -44,6 +73,16 @@ def _config_int(config, name: str, default: int) -> int: return default +def _config_nonnegative_int(config, name: str, default: int) -> int: + value = getattr(config, name, default) + if isinstance(value, bool) or not isinstance(value, (int, str)): + return default + try: + return max(0, int(value)) + except (TypeError, ValueError): + return default + + def _config_float(config, name: str, default: float) -> float: value = getattr(config, name, default) if not isinstance(value, (int, float, str)): @@ -75,6 +114,23 @@ def __init__(self, config: RAGConfig): 5.0, ), ) + self._revision_preflight_cache = RevisionPreflightCache( + max_entries=_config_int( + config, + "revision_preflight_cache_entries", + 512, + ), + ttl_seconds=_config_nonnegative_int( + config, + "revision_preflight_cache_ttl_seconds", + 0, + ), + max_concurrent_loads=_config_int( + config, + "revision_preflight_max_concurrency", + 2, + ), + ) self.index_representation_fingerprint = ( index_representation_fingerprint(config) ) @@ -189,6 +245,63 @@ def _get_project_collection_name(self, workspace: str, project: str) -> str: namespace = make_project_namespace(workspace, project) return f"{self.config.qdrant_collection_prefix}_{namespace}" + def _get_branch_operator_alias( + self, + workspace: str, + project: str, + branch: str, + ) -> str: + """Return a readable, collision-safe alias for one branch's head. + + Exact review context always uses the immutable generation target from + the Java registry. This alias is an operator and legacy-integration + pointer to the current published head of one branch. + """ + legacy_alias = self._get_project_collection_name(workspace, project) + raw_branch = branch.strip() + readable = re.sub(r"[^a-zA-Z0-9_-]", "_", raw_branch).lower() + if not readable: + readable = "branch" + # Branch identities are case-sensitive even though readable aliases are + # normalized to lowercase. Hash whenever normalization changes the + # original identity so case-only branches cannot share an alias. + if readable != raw_branch or len(readable) > 64: + readable = ( + f"{readable[:64]}_" + f"{hashlib.sha256(raw_branch.encode('utf-8')).hexdigest()[:10]}" + ) + return f"{legacy_alias}__{readable}" + + def _publication_aliases( + self, + workspace: str, + project: str, + branch: str, + publish_branch_alias: bool, + publish_legacy_project_alias: bool, + ) -> List[str]: + aliases: List[str] = [] + if publish_branch_alias: + aliases.append(self._get_branch_operator_alias(workspace, project, branch)) + if publish_legacy_project_alias: + aliases.append(self._get_project_collection_name(workspace, project)) + return list(dict.fromkeys(aliases)) + + @staticmethod + def _publication_scope( + branch: str, + publication_aliases: List[str], + ) -> Optional[str]: + """Return the mutable resource shared by one branch head. + + Exact target aliases are immutable and may be built independently. + Once a build publishes a readable branch pointer, later builds of the + same branch must serialize their final alias activation. Other branch + scopes remain independent, so retained branches still index in + parallel. + """ + return f"branch-head:{branch}" if publication_aliases else None + # Repository indexing def estimate_repository_size( @@ -210,14 +323,44 @@ def index_repository( preserve_other_branches: bool = False, include_patterns: Optional[List[str]] = None, exclude_patterns: Optional[List[str]] = None, + source_tree_sha256: Optional[str] = None, + collection_target: Optional[str] = None, + publish_branch_alias: bool = False, + publish_legacy_project_alias: bool = False, progress_callback: Optional[Callable[[dict], None]] = None, ) -> IndexStats: """Index entire repository for a branch using atomic swap strategy.""" - alias_name = self._get_project_collection_name(workspace, project) + if collection_target is None and ( + publish_branch_alias or publish_legacy_project_alias + ): + raise ValueError( + "readable generation aliases require an immutable collection target" + ) + alias_name = collection_target or self._get_project_collection_name( + workspace, project + ) + expected_source_tree = ( + source_tree_sha256 + or compute_repository_source_tree_sha256(repo_path) + ) + source_tree = verify_repository_source_tree( + repo_path, + commit, + expected_source_tree, + ) + publication_aliases = self._publication_aliases( + workspace, + project, + branch, + publish_branch_alias, + publish_legacy_project_alias, + ) with self._mutation_coordinator.acquire( workspace, project, "full-index", + collection_target=collection_target, + publication_scope=self._publication_scope(branch, publication_aliases), ) as lease: return self._indexer.index_repository( repo_path=repo_path, @@ -229,11 +372,141 @@ def index_repository( preserve_other_branches=preserve_other_branches, include_patterns=include_patterns, exclude_patterns=exclude_patterns, + source_tree_sha256=source_tree.tree_sha256, + source_tree=source_tree, + seal_generation=collection_target is not None, + publication_aliases=publication_aliases, operation_id=lease.token, activation_guard=lease.assert_owned, progress_callback=progress_callback, ) + def get_revision_preflight( + self, + workspace: str, + project: str, + branch: str, + commit: str, + *, + collection_target: Optional[str] = None, + ): + """Verify one immutable revision in an explicitly selected target.""" + target = collection_target or self._get_project_collection_name( + workspace, project + ) + physical = self._collection_manager.resolve_collection_target(target) + if physical is None: + return None + cache = getattr(self, "_revision_preflight_cache", None) + if cache is None: + # Preserve lightweight construction used by tooling and tests. + cache = RevisionPreflightCache( + max_entries=_config_int( + self.config, + "revision_preflight_cache_entries", + 512, + ), + ttl_seconds=_config_nonnegative_int( + self.config, + "revision_preflight_cache_ttl_seconds", + 0, + ), + max_concurrent_loads=_config_int( + self.config, + "revision_preflight_max_concurrency", + 2, + ), + ) + self._revision_preflight_cache = cache + + def verify(): + result = read_repository_revision_preflight( + self.qdrant_client, + physical, + branch, + commit, + ) + if result is None: + return None + if ( + result.get("workspace") != workspace + or result.get("project") != project + ): + raise IncrementalIndexPreconditionError( + "repository generation coordinates do not match the requested tenant" + ) + return result + + return cache.get_or_load( + RevisionPreflightKey( + collection=physical, + workspace=workspace, + project=project, + branch=branch, + commit=commit, + ), + verify, + ) + + def publish_generation_aliases( + self, + workspace: str, + project: str, + branch: str, + commit: str, + collection_target: str, + publish_branch_alias: bool = True, + publish_legacy_project_alias: bool = False, + ) -> List[str]: + """Publish operator aliases after proving one sealed generation identity. + + The Java registry calls this only after accepting the generation as the + active branch head. It is also an idempotent repair path after restore. + """ + aliases = self._publication_aliases( + workspace, + project, + branch, + publish_branch_alias, + publish_legacy_project_alias, + ) + if not aliases: + return [] + with self._mutation_coordinator.acquire( + workspace, + project, + "publish-generation-aliases", + collection_target=collection_target, + publication_scope=self._publication_scope(branch, aliases), + ) as lease: + physical = self._collection_manager.resolve_collection_target( + collection_target + ) + if physical is None: + raise IncrementalIndexPreconditionError( + "repository generation is unavailable for alias publication" + ) + receipt = read_repository_revision_preflight( + self.qdrant_client, + physical, + branch, + commit, + ) + if receipt is None or any(receipt.get(key) != value for key, value in ( + ("workspace", workspace), + ("project", project), + ("branch", branch), + ("commit", commit), + )): + raise IncrementalIndexPreconditionError( + "repository generation does not match the requested alias coordinates" + ) + lease.assert_owned() + self._collection_manager.atomic_assign_aliases( + {alias: physical for alias in aliases} + ) + return aliases + # File operations def update_files( @@ -317,10 +590,329 @@ def apply_changes( mutation_guard=lease.assert_owned, ) + def _rebind_repository_facts_revision( + self, + collection_name: str, + branch: str, + commit: str, + ) -> None: + """Rebind the copied neutral inventory before applying its delta.""" + points = [] + offset = None + while True: + batch, offset = self.qdrant_client.scroll( + collection_name=collection_name, + scroll_filter=Filter(must=[ + FieldCondition(key="branch", match=MatchValue(value=branch)), + FieldCondition( + key="repository_facts_state", + match=MatchValue(value=True), + ), + ]), + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + points.extend(batch) + if offset is None: + break + if not points: + raise IncrementalIndexPreconditionError( + "source generation has no repository detection facts" + ) + ordered = sorted( + points, key=lambda point: (point.payload or {}).get("facts_part", -1) + ) + content = "".join((point.payload or {}).get("text", "") for point in ordered) + decoded = json.loads(content) + decoded["revision"] = commit + rebound = json.dumps( + decoded, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + parts = [ + rebound[index:index + 400_000] + for index in range(0, len(rebound), 400_000) + ] + if len(parts) != len(ordered): + raise IncrementalIndexPreconditionError( + "repository facts cannot be rebound incrementally; fully reindex" + ) + digest = hashlib.sha256(rebound.encode("utf-8")).hexdigest() + replacements = [] + for index, (point, text_part) in enumerate(zip(ordered, parts)): + payload = dict(point.payload or {}) + payload.update({ + "commit": commit, + "text": text_part, + "facts_part": index, + "facts_parts": len(parts), + "facts_content_sha256": digest, + }) + payload[GENERATION_MEMBER_DIGEST_PAYLOAD_KEY] = ( + compute_generation_member_digest(point.id, payload, point.vector) + ) + replacements.append(PointStruct( + id=point.id, vector=point.vector, payload=payload + )) + self.qdrant_client.upsert( + collection_name=collection_name, + points=replacements, + wait=True, + ) + + def advance_generation( + self, + source_collection_target: str, + target_collection_target: str, + source_commit: str, + source_tree_sha256: str, + updated_file_paths: List[str], + deleted_file_paths: List[str], + repo_base: Optional[str], + workspace: str, + project: str, + branch: str, + commit: str, + publish_branch_alias: bool = False, + publish_legacy_project_alias: bool = False, + ) -> IndexStats: + """Copy, mutate, seal, and atomically publish one exact generation.""" + publication_aliases = self._publication_aliases( + workspace, + project, + branch, + publish_branch_alias, + publish_legacy_project_alias, + ) + activation_aliases = list(dict.fromkeys( + [target_collection_target, *publication_aliases] + )) + with self._mutation_coordinator.acquire( + workspace, + project, + "advance-generation", + collection_target=target_collection_target, + publication_scope=self._publication_scope( + branch, + publication_aliases, + ), + ) as lease: + source_physical = self._collection_manager.resolve_collection_target( + source_collection_target + ) + if source_physical is None: + raise IncrementalIndexPreconditionError( + "source repository generation is unavailable" + ) + source_receipt = self.get_revision_preflight( + workspace, + project, + branch, + source_commit, + collection_target=source_physical, + ) + if source_receipt is None: + raise IncrementalIndexPreconditionError( + "source repository generation is not sealed" + ) + + target_physical = self._collection_manager.resolve_collection_target( + target_collection_target + ) + if target_physical is not None: + existing = self.get_revision_preflight( + workspace, + project, + branch, + commit, + collection_target=target_physical, + ) + if existing is None: + raise IncrementalIndexPreconditionError( + "target collection exists without the requested sealed generation" + ) + stats = self._stats_manager.get_branch_stats( + workspace, project, branch, target_physical + ) + if publication_aliases: + self._collection_manager.atomic_assign_aliases( + {alias: target_physical for alias in publication_aliases} + ) + if hasattr(stats, "model_copy"): + return stats.model_copy(update={ + "generation_manifest_sha256": existing[ + "generation_manifest_sha256" + ], + "source_tree_sha256": existing["source_tree_sha256"], + "collection_target": target_collection_target, + }) + return stats + + if repo_base is None: + raise IncrementalIndexPreconditionError( + "target repository snapshot is required for generation advance" + ) + observed_source_tree_sha256 = ( + compute_repository_source_tree_sha256(repo_base) + ) + if observed_source_tree_sha256 != source_tree_sha256: + raise IncrementalIndexPreconditionError( + "target repository source tree does not match its attestation" + ) + + pending = self._collection_manager.create_pending_collection( + target_collection_target, + operation_id=lease.token, + ) + activation_alias_targets = self._collection_manager.read_alias_targets( + activation_aliases + ) + activated = False + try: + offset = None + copied = 0 + while True: + points, offset = self.qdrant_client.scroll( + collection_name=source_physical, + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + batch = [] + for point in points: + payload = dict(point.payload or {}) + if payload.get(GENERATION_MANIFEST_PAYLOAD_KEY) is True: + continue + if any(payload.get(key) != value for key, value in ( + ("workspace", workspace), + ("project", project), + ("branch", branch), + ("commit", source_commit), + )): + raise IncrementalIndexPreconditionError( + "source generation contains points outside its tenant or revision" + ) + payload["commit"] = commit + payload[GENERATION_MEMBER_DIGEST_PAYLOAD_KEY] = ( + compute_generation_member_digest( + point.id, payload, point.vector + ) + ) + batch.append(PointStruct( + id=point.id, + vector=point.vector, + payload=payload, + )) + if batch: + self.qdrant_client.upsert( + collection_name=pending, + points=batch, + wait=True, + ) + copied += len(batch) + if offset is None: + break + if copied < 1: + raise IncrementalIndexPreconditionError( + "source generation has no repository members" + ) + + self._rebind_repository_facts_revision( + pending, branch, commit + ) + + self._file_ops.apply_changes( + updated_file_paths=updated_file_paths, + deleted_file_paths=deleted_file_paths, + repo_base=repo_base, + workspace=workspace, + project=project, + branch=branch, + commit=commit, + collection_name=pending, + mutation_guard=lease.assert_owned, + ) + + members = collect_generation_members( + self.qdrant_client, pending, branch, commit + ) + identity = { + "plugin_ids": source_receipt["plugin_ids"], + "plugin_fingerprint": source_receipt["plugin_fingerprint"], + "plugin_descriptor_fingerprint": source_receipt[ + "plugin_descriptor_fingerprint" + ], + "plugin_implementation_fingerprint": source_receipt[ + "plugin_implementation_fingerprint" + ], + "index_representation_fingerprint": source_receipt[ + "index_representation_fingerprint" + ], + } + manifest = build_generation_manifest_node( + workspace=workspace, + project=project, + branch=branch, + commit=commit, + member_count=len(members), + members_sha256=compute_generation_members_digest(members), + source_tree_sha256=source_tree_sha256, + index_include_patterns=source_receipt[ + "index_include_patterns" + ], + index_exclude_patterns=source_receipt[ + "index_exclude_patterns" + ], + identity_metadata=identity, + ) + success, failed = self._point_ops.process_and_upsert_chunks( + [manifest], pending, workspace, project, branch + ) + if success != 1 or failed: + raise RuntimeError("target generation manifest was not persisted") + lease.assert_owned() + if self._collection_manager.read_alias_targets( + activation_aliases) != activation_alias_targets: + raise RuntimeError( + "Active RAG alias changed before pending activation" + ) + self._collection_manager.atomic_assign_aliases( + {alias: pending for alias in activation_aliases} + ) + activated = True + stats = self._stats_manager.get_branch_stats( + workspace, project, branch, pending + ) + return stats.model_copy(update={ + "generation_manifest_sha256": manifest.metadata[ + "generation_manifest_sha256" + ], + "source_tree_sha256": source_tree_sha256, + "collection_target": target_collection_target, + }) + finally: + if not activated: + self._collection_manager.delete_collection(pending) + # Branch operations - def delete_branch(self, workspace: str, project: str, branch: str) -> bool: + def delete_branch( + self, + workspace: str, + project: str, + branch: str, + collection_target: Optional[str] = None, + ) -> bool: """Delete all points for a specific branch from the project collection.""" + if collection_target: + return self.delete_collection_target( + workspace, project, branch, collection_target + ) with self._mutation_coordinator.acquire( workspace, project, @@ -335,6 +927,55 @@ def delete_branch(self, workspace: str, project: str, branch: str) -> bool: lease.assert_owned() return self._branch_manager.delete_branch_points(collection_name, branch) + def delete_collection_target( + self, + workspace: str, + project: str, + branch: str, + collection_target: str, + ) -> bool: + """Delete one exact generation after proving its tenant ownership.""" + with self._mutation_coordinator.acquire( + workspace, project, "delete-generation" + ) as lease: + physical = self._collection_manager.resolve_collection_target( + collection_target + ) + if physical is None: + return False + offset = None + point_count = 0 + while True: + points, offset = self.qdrant_client.scroll( + collection_name=physical, + limit=256, + offset=offset, + with_payload=["workspace", "project", "branch"], + with_vectors=False, + ) + point_count += len(points) + for point in points: + payload = point.payload or {} + if any(payload.get(key) != value for key, value in ( + ("workspace", workspace), + ("project", project), + ("branch", branch), + )): + raise IncrementalIndexPreconditionError( + "collection target does not belong to the requested tenant branch" + ) + if offset is None: + break + if point_count == 0: + raise IncrementalIndexPreconditionError( + "empty collection target cannot be ownership-verified" + ) + lease.assert_owned() + if self._collection_manager.alias_exists(collection_target): + if not self._collection_manager.delete_alias(collection_target): + return False + return self._collection_manager.delete_collection(physical) + def get_branch_point_count(self, workspace: str, project: str, branch: str) -> int: """Get the number of points for a specific branch.""" collection_name = self._get_project_collection_name(workspace, project) @@ -402,6 +1043,26 @@ def project_mutation(self, workspace: str, project: str, operation: str): """Return the shared mutation boundary for auxiliary index endpoints.""" return self._mutation_coordinator.acquire(workspace, project, operation) + def pr_overlay_mutation( + self, + workspace: str, + project: str, + pr_number: int, + operation: str, + ): + """Serialize one PR overlay without blocking unrelated PRs. + + Index replacement and deletion for the same PR must not overlap, even + when the target generation changes between attempts. Different PR + numbers have disjoint point identities and can mutate concurrently. + """ + return self._mutation_coordinator.acquire( + workspace, + project, + operation, + publication_scope=f"pr-overlay:{pr_number}", + ) + def close(self) -> None: self._mutation_coordinator.close() diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py index 547807b4..e7640a49 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py @@ -19,6 +19,11 @@ from qdrant_client import QdrantClient from qdrant_client.models import PointStruct +from ..generation_manifest import ( + GENERATION_MEMBER_DIGEST_PAYLOAD_KEY, + compute_generation_member_digest, +) + logger = logging.getLogger(__name__) EMBEDDING_INPUT_HASH_PAYLOAD_KEY = "embedding_input_sha256" @@ -108,6 +113,8 @@ def _is_architecture_chunk(chunk: TextNode) -> bool: or chunk.metadata.get("architecture_source") or chunk.metadata.get("repository_snapshot") or chunk.metadata.get("repository_facts_state") + or chunk.metadata.get("repository_generation_manifest") + or chunk.metadata.get("pr_overlay_generation_manifest") ) @staticmethod @@ -196,7 +203,7 @@ def _reusable_vectors( continue reusable[point_id] = vector return reusable - + @staticmethod def generate_point_id( workspace: str, @@ -297,7 +304,7 @@ def embed_and_create_points( metrics["embedded"] = metrics.get("embedded", 0) + len( embedded_by_id ) - + # Build points with embeddings points = [] for point_id, chunk in chunk_data: @@ -330,6 +337,9 @@ def embed_and_create_points( or chunk.metadata.get("repository_facts_state") ): payload["_node_content"] = chunk.text + payload[GENERATION_MEMBER_DIGEST_PAYLOAD_KEY] = ( + compute_generation_member_digest(point_id, payload, embedding) + ) points.append(PointStruct( id=point_id, vector=embedding, @@ -351,6 +361,44 @@ def upsert_points( result = self.upsert_points_detailed(collection_name, points) return result.successful, result.failed + def _seal_persisted_point_digests( + self, + collection_name: str, + points: List[PointStruct], + ) -> list[tuple[object, str]]: + """Bind digests to the vectors exactly as persisted by Qdrant.""" + if not points: + return [] + expected = {str(point.id): point for point in points} + records = self.client.retrieve( + collection_name=collection_name, + ids=[point.id for point in points], + with_payload=True, + with_vectors=True, + ) + if {str(record.id) for record in records} != set(expected): + raise RuntimeError("persisted point set is incomplete before sealing") + sealed = [] + replacements = [] + for record in records: + payload = dict(record.payload or {}) + digest = compute_generation_member_digest( + record.id, payload, record.vector + ) + payload[GENERATION_MEMBER_DIGEST_PAYLOAD_KEY] = digest + replacements.append(PointStruct( + id=record.id, + vector=record.vector, + payload=payload, + )) + sealed.append((record.id, digest)) + self.client.upsert( + collection_name=collection_name, + points=replacements, + wait=True, + ) + return sealed + def upsert_points_detailed( self, collection_name: str, diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_representation.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_representation.py index 45205180..612915cd 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_representation.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_representation.py @@ -16,6 +16,10 @@ INDEX_REPRESENTATION_PAYLOAD_KEY = "index_representation_fingerprint" + +class IndexCompatibilityError(RuntimeError): + """Deprecated compatibility name retained for older internal callers.""" + # These inputs can change persistent target-branch point text, metadata, or # vectors. PR-only request/overlay code is intentionally excluded so a PR # orchestration fix cannot force every repository embedding to be rebuilt. diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/loader.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/loader.py index d781bc0b..9075d354 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/loader.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/loader.py @@ -1,11 +1,16 @@ from pathlib import Path -from typing import List, Optional, Generator +from typing import List, Optional, Generator, Mapping import logging import re from llama_index.core.schema import Document -from ..utils.utils import detect_language_from_path, should_exclude_file, should_include_file, is_binary_file, clean_archive_path +from ..utils.utils import detect_language_from_path, should_exclude_file, should_include_file, clean_archive_path from ..models.config import RAGConfig +from .source_tree import ( + RepositorySourceTreeError, + iter_repository_regular_file_paths, + read_repository_file_bytes, +) logger = logging.getLogger(__name__) @@ -36,6 +41,15 @@ def _is_generated_asset(filename: str) -> bool: return has_letter and has_digit +def _decode_text(content: bytes) -> str | None: + if b"\0" in content: + return None + try: + return content.decode("utf-8") + except UnicodeDecodeError: + return None + + class DocumentLoader: """Load repository files as documents""" @@ -46,7 +60,8 @@ def iter_repository_files( self, repo_path: Path, extra_include_patterns: Optional[List[str]] = None, - extra_exclude_patterns: Optional[List[str]] = None + extra_exclude_patterns: Optional[List[str]] = None, + expected_file_sha256: Optional[Mapping[str, str]] = None, ) -> Generator[Path, None, None]: """Iterate over repository files without loading them into memory. @@ -78,15 +93,20 @@ def iter_repository_files( # Include patterns (project-specific only, no defaults) include_patterns = extra_include_patterns if extra_include_patterns else [] + candidates = ( + (repo_path / Path(path) for path in sorted(expected_file_sha256)) + if expected_file_sha256 is not None + else ( + repo_path / path + for path in iter_repository_regular_file_paths(repo_path) + ) + ) total_entries = 0 yielded_count = 0 - for file_path in repo_path.rglob("*"): + for file_path in candidates: total_entries += 1 - if not file_path.is_file(): - continue - relative_path = file_path.relative_to(repo_path) - relative_path_str = str(relative_path) + relative_path_str = relative_path.as_posix() # Step 1: Apply inclusion filter first # If include patterns are specified, only files matching at least one pattern pass @@ -97,10 +117,30 @@ def iter_repository_files( if should_exclude_file(relative_path_str, exclude_patterns): continue - if file_path.stat().st_size > self.config.max_file_size_bytes: + expected_digest = ( + expected_file_sha256.get(relative_path_str) + if expected_file_sha256 is not None + else None + ) + try: + content = read_repository_file_bytes( + repo_path, + relative_path, + expected_sha256=expected_digest, + ) + except RepositorySourceTreeError: + if expected_file_sha256 is not None: + raise + logger.warning( + "Cannot safely inspect repository file, skipping: %s", + relative_path_str, + ) continue - if is_binary_file(file_path): + if len(content) > self.config.max_file_size_bytes: + continue + + if _decode_text(content) is None: continue # Skip build-tool-generated assets with content hashes @@ -121,6 +161,7 @@ def load_file_batch( branch: str, commit: str, strict: bool = False, + expected_file_sha256: Optional[Mapping[str, str]] = None, ) -> List[Document]: """Load a batch of files as Documents. @@ -149,7 +190,22 @@ def load_file_batch( continue try: - text = full_path.read_text(encoding="utf-8") + expected_digest = None + if expected_file_sha256 is not None: + expected_digest = expected_file_sha256.get( + Path(relative_path).as_posix() + ) + if expected_digest is None: + raise RepositorySourceTreeError( + "repository source file was not present in the " + f"attested tree: {relative_path_str}" + ) + content = read_repository_file_bytes( + repo_base, + relative_path, + expected_sha256=expected_digest, + ) + text = content.decode("utf-8") if not text or not text.strip(): continue @@ -209,85 +265,20 @@ def load_from_directory( commit: Commit hash extra_exclude_patterns: Additional patterns to exclude (from project config) """ - documents = [] - - if not repo_path.exists(): - logger.error(f"Repository path does not exist: {repo_path}") - return documents - - # Combine default exclude patterns with project-specific ones - exclude_patterns = list(self.config.excluded_patterns) - if extra_exclude_patterns: - exclude_patterns.extend(extra_exclude_patterns) - logger.info(f"Using {len(extra_exclude_patterns)} additional exclude patterns from project config: {extra_exclude_patterns}") - - excluded_count = 0 - for file_path in repo_path.rglob("*"): - if not file_path.is_file(): - continue - - relative_path = str(file_path.relative_to(repo_path)) - - if should_exclude_file(relative_path, exclude_patterns): - logger.debug(f"Excluding file: {relative_path}") - excluded_count += 1 - continue - - if file_path.stat().st_size > self.config.max_file_size_bytes: - logger.warning(f"File too large, skipping: {relative_path}") - continue - - if is_binary_file(file_path): - logger.debug(f"Binary file, skipping: {relative_path}") - continue - - if _is_generated_asset(file_path.name): - logger.debug(f"Generated asset, skipping: {relative_path}") - excluded_count += 1 - continue - - try: - text = file_path.read_text(encoding="utf-8") - - # Skip empty files - if not text or not text.strip(): - logger.debug(f"Empty file, skipping: {relative_path}") - continue - - except UnicodeDecodeError: - logger.warning(f"Cannot decode file, skipping: {relative_path}") - continue - except Exception as e: - logger.error(f"Error reading file {relative_path}: {e}") - continue - - language = detect_language_from_path(str(file_path)) - filetype = file_path.suffix.lstrip('.') - - # Clean archive root prefix from path - clean_path = clean_archive_path(relative_path) - - metadata = { - "workspace": workspace, - "project": project, - "branch": branch, - "path": clean_path, - "commit": commit, - "language": language, - "filetype": filetype, - } - - doc = Document( - text=text, - metadata=metadata - # Don't set id_ - let LlamaIndex/Qdrant generate it automatically + file_paths = list( + self.iter_repository_files( + repo_path, + extra_exclude_patterns=extra_exclude_patterns, ) - - documents.append(doc) - logger.debug(f"Loaded document: {clean_path} ({language})") - - logger.info(f"Loaded {len(documents)} documents from {repo_path} (excluded {excluded_count} files by patterns)") - return documents + ) + return self.load_file_batch( + file_paths, + repo_path, + workspace, + project, + branch, + commit, + ) def load_specific_files( self, @@ -306,31 +297,26 @@ def load_specific_files( full_path = repo_base / relative_file_path relative_path = str(relative_file_path) - if not full_path.exists(): - logger.warning(f"File does not exist: {full_path} (relative: {relative_path})") - continue - - if not full_path.is_file(): - continue - if should_exclude_file(relative_path, self.config.excluded_patterns): logger.debug(f"Excluding file: {relative_path}") continue - if full_path.stat().st_size > self.config.max_file_size_bytes: - logger.warning(f"File too large, skipping: {relative_path}") - continue - - if is_binary_file(full_path): - logger.debug(f"Binary file, skipping: {relative_path}") - continue - if _is_generated_asset(full_path.name): logger.debug(f"Generated asset, skipping: {relative_path}") continue try: - text = full_path.read_text(encoding="utf-8") + content = read_repository_file_bytes( + repo_base, + relative_file_path, + ) + if len(content) > self.config.max_file_size_bytes: + logger.warning(f"File too large, skipping: {relative_path}") + continue + text = _decode_text(content) + if text is None: + logger.debug(f"Binary file, skipping: {relative_path}") + continue except Exception as e: logger.error(f"Error reading file {relative_path}: {e}") continue diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_identity.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_identity.py index a6f36074..6108bb9c 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_identity.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_identity.py @@ -47,6 +47,7 @@ def pr_overlay_generation_fingerprint( index_representation_fingerprint: str, pr_overlay_representation_fingerprint: str, snapshots: Iterable[object], + base_generation_manifest_sha256: str = "", ) -> str: """Hash every input that can change persisted semantic or plugin context.""" digest = hashlib.sha256() @@ -59,6 +60,10 @@ def pr_overlay_generation_fingerprint( ("base_branch", base_branch), ("source_revision", source_revision), ("base_revision", base_revision), + ( + "base_generation_manifest_sha256", + base_generation_manifest_sha256, + ), ("request_plugin_fingerprint", request_plugin_fingerprint), ("target_plugin_fingerprint", target_plugin_fingerprint), ("capability_fingerprint", capability_fingerprint), diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_manifest.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_manifest.py new file mode 100644 index 00000000..8d34e1b6 --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_manifest.py @@ -0,0 +1,374 @@ +"""Content-addressed completeness seals for persisted pull-request overlays.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from typing import Any + +from llama_index.core.schema import TextNode +from qdrant_client.models import FieldCondition, Filter, MatchValue + +from .generation_manifest import ( + GenerationManifestError, + compute_generation_members_digest, + verified_generation_member, +) +from .repository_overlay import IncrementalIndexPreconditionError + + +PR_OVERLAY_MANIFEST_PAYLOAD_KEY = "pr_overlay_generation_manifest" +PR_OVERLAY_SCHEMA = "codecrow.pr-overlay-generation" +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_FINGERPRINT_RE = re.compile(r"^sha256:[0-9a-f]{64}$") + + +def _canonical_json(value: Any) -> str: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +def pr_overlay_manifest_path(pr_number: int) -> str: + return ( + "__analysis_state__/pr-overlay-generation/" + f"{pr_number}/000000.state" + ) + + +def pr_overlay_manifest_content( + *, + workspace: str, + project: str, + pr_number: int, + branch: str, + base_branch: str, + source_revision: str, + base_revision: str, + base_generation_manifest_sha256: str, + generation_fingerprint: str, + overlay_representation_fingerprint: str, + member_count: int, + members_sha256: str, +) -> str: + return _canonical_json({ + "baseBranch": base_branch, + "baseGenerationManifestSha256": base_generation_manifest_sha256, + "baseRevision": base_revision, + "branch": branch, + "generationFingerprint": generation_fingerprint, + "memberCount": member_count, + "membersSha256": members_sha256, + "overlayRepresentationFingerprint": ( + overlay_representation_fingerprint + ), + "prNumber": pr_number, + "project": project, + "schema": PR_OVERLAY_SCHEMA, + "sourceRevision": source_revision, + "workspace": workspace, + }) + + +def build_pr_overlay_manifest_node( + *, + workspace: str, + project: str, + pr_number: int, + branch: str, + base_branch: str, + source_revision: str, + base_revision: str, + base_generation_manifest_sha256: str, + generation_fingerprint: str, + overlay_representation_fingerprint: str, + members: Sequence[tuple[object, str]], + identity_metadata: Mapping[str, Any], +) -> tuple[TextNode, dict[str, Any]]: + members_sha256 = compute_generation_members_digest(members) + content = pr_overlay_manifest_content( + workspace=workspace, + project=project, + pr_number=pr_number, + branch=branch, + base_branch=base_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=( + base_generation_manifest_sha256 + ), + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + member_count=len(members), + members_sha256=members_sha256, + ) + manifest_sha256 = hashlib.sha256(content.encode("utf-8")).hexdigest() + metadata = { + "workspace": workspace, + "project": project, + "branch": branch, + "path": pr_overlay_manifest_path(pr_number), + "language": "repository-state", + "filetype": "state", + "pr": True, + "pr_number": pr_number, + "pr_branch": branch, + "pr_source_revision": source_revision, + "pr_base_revision": base_revision, + "pr_base_generation_manifest_sha256": ( + base_generation_manifest_sha256 + ), + "pr_generation_fingerprint": generation_fingerprint, + PR_OVERLAY_MANIFEST_PAYLOAD_KEY: True, + "pr_overlay_generation_schema": PR_OVERLAY_SCHEMA, + "pr_overlay_generation_member_count": len(members), + "pr_overlay_generation_members_sha256": members_sha256, + "pr_overlay_generation_manifest_sha256": manifest_sha256, + "pr_overlay_base_branch": base_branch, + "pr_overlay_representation_fingerprint": ( + overlay_representation_fingerprint + ), + **dict(identity_metadata), + } + return TextNode(text=content, metadata=metadata), { + "overlay_generation_member_count": len(members), + "overlay_generation_members_sha256": members_sha256, + "overlay_generation_manifest_sha256": manifest_sha256, + } + + +def _pr_filter( + *, + workspace: str, + project: str, + pr_number: int, + branch: str, + source_revision: str, + base_revision: str, + base_generation_manifest_sha256: str, + generation_fingerprint: str, + overlay_representation_fingerprint: str, +) -> Filter: + return Filter(must=[ + FieldCondition(key="pr", match=MatchValue(value=True)), + FieldCondition( + key="workspace", + match=MatchValue(value=workspace), + ), + FieldCondition( + key="project", + match=MatchValue(value=project), + ), + FieldCondition( + key="pr_number", + match=MatchValue(value=pr_number), + ), + FieldCondition( + key="branch", + match=MatchValue(value=branch), + ), + FieldCondition( + key="pr_source_revision", + match=MatchValue(value=source_revision), + ), + FieldCondition( + key="pr_base_revision", + match=MatchValue(value=base_revision), + ), + FieldCondition( + key="pr_base_generation_manifest_sha256", + match=MatchValue(value=base_generation_manifest_sha256), + ), + FieldCondition( + key="pr_generation_fingerprint", + match=MatchValue(value=generation_fingerprint), + ), + FieldCondition( + key="pr_overlay_representation_fingerprint", + match=MatchValue(value=overlay_representation_fingerprint), + ), + ]) + + +def read_pr_overlay_generation( + client, + collection_name: str, + *, + workspace: str, + project: str, + pr_number: int, + branch: str, + base_branch: str, + source_revision: str, + base_revision: str, + base_generation_manifest_sha256: str, + generation_fingerprint: str, + overlay_representation_fingerprint: str, + expected_manifest_sha256: str | None = None, +) -> dict[str, Any] | None: + """Read and verify every member of one exact PR overlay generation.""" + points = [] + offset = None + while True: + batch, offset = client.scroll( + collection_name=collection_name, + scroll_filter=_pr_filter( + workspace=workspace, + project=project, + pr_number=pr_number, + branch=branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=( + base_generation_manifest_sha256 + ), + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + ), + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + points.extend(batch) + if offset is None: + break + if not points: + return None + + manifest_points = [ + point + for point in points + if (point.payload or {}).get(PR_OVERLAY_MANIFEST_PAYLOAD_KEY) is True + ] + if len(manifest_points) != 1: + raise IncrementalIndexPreconditionError( + "PR overlay generation manifest is missing or not unique" + ) + manifest = manifest_points[0].payload or {} + members = [ + point + for point in points + if (point.payload or {}).get(PR_OVERLAY_MANIFEST_PAYLOAD_KEY) is not True + ] + expected_identity = { + "workspace": workspace, + "project": project, + "branch": branch, + "pr_number": pr_number, + "pr_source_revision": source_revision, + "pr_base_revision": base_revision, + "pr_base_generation_manifest_sha256": ( + base_generation_manifest_sha256 + ), + "pr_generation_fingerprint": generation_fingerprint, + "pr_overlay_base_branch": base_branch, + "pr_overlay_representation_fingerprint": ( + overlay_representation_fingerprint + ), + } + if any( + manifest.get(field) != expected + for field, expected in expected_identity.items() + ): + raise IncrementalIndexPreconditionError( + "PR overlay generation manifest identity does not match the request" + ) + manifest_sha256 = manifest.get( + "pr_overlay_generation_manifest_sha256" + ) + member_count = manifest.get("pr_overlay_generation_member_count") + members_sha256 = manifest.get("pr_overlay_generation_members_sha256") + if ( + manifest.get("pr_overlay_generation_schema") != PR_OVERLAY_SCHEMA + or manifest.get("path") != pr_overlay_manifest_path(pr_number) + or type(member_count) is not int + or member_count < 0 + or not isinstance(members_sha256, str) + or _SHA256_RE.fullmatch(members_sha256) is None + or not isinstance(manifest_sha256, str) + or _SHA256_RE.fullmatch(manifest_sha256) is None + or ( + expected_manifest_sha256 is not None + and manifest_sha256 != expected_manifest_sha256 + ) + ): + raise IncrementalIndexPreconditionError( + "PR overlay generation manifest is invalid" + ) + if len(members) != member_count: + raise IncrementalIndexPreconditionError( + "PR overlay generation membership is incomplete" + ) + for point in members: + payload = point.payload or {} + if any( + payload.get(field) != expected + for field, expected in expected_identity.items() + if field not in {"pr_overlay_base_branch"} + ): + raise IncrementalIndexPreconditionError( + "PR overlay contains a member outside the sealed generation" + ) + if payload.get(PR_OVERLAY_MANIFEST_PAYLOAD_KEY) is True: + raise IncrementalIndexPreconditionError( + "PR overlay manifest was included as an ordinary member" + ) + try: + observed_members = [ + verified_generation_member(point) for point in members + ] + observed_members_sha256 = compute_generation_members_digest( + observed_members + ) + except GenerationManifestError as exception: + raise IncrementalIndexPreconditionError( + "PR overlay generation member integrity failed" + ) from exception + if observed_members_sha256 != members_sha256: + raise IncrementalIndexPreconditionError( + "PR overlay generation membership digest does not match its seal" + ) + expected_content = pr_overlay_manifest_content( + workspace=workspace, + project=project, + pr_number=pr_number, + branch=branch, + base_branch=base_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=( + base_generation_manifest_sha256 + ), + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + member_count=member_count, + members_sha256=members_sha256, + ) + if hashlib.sha256(expected_content.encode("utf-8")).hexdigest() != ( + manifest_sha256 + ): + raise IncrementalIndexPreconditionError( + "PR overlay generation manifest content failed integrity validation" + ) + return { + "overlay_generation_member_count": member_count, + "overlay_generation_members_sha256": members_sha256, + "overlay_generation_manifest_sha256": manifest_sha256, + } + + +def is_pr_overlay_fingerprint(value: object) -> bool: + return isinstance(value, str) and _FINGERPRINT_RE.fullmatch(value) is not None diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_representation.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_representation.py index 57cf5fcc..94e13d0f 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_representation.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_representation.py @@ -18,10 +18,19 @@ _PR_OVERLAY_SOURCE_PATHS = ( "api/models.py", "api/routers/pr.py", + "api/routers/query.py", "core/index_manager/manager.py", "core/pr_overlay_identity.py", + "core/pr_overlay_manifest.py", "core/pr_overlay_representation.py", + "core/revision_binding.py", + "core/revision_preflight.py", "core/review_grouping.py", + "services/base.py", + "services/deterministic_context.py", + "services/pr_context.py", + "services/query_service.py", + "services/semantic_search.py", ) _PR_OVERLAY_DEPENDENCIES = ( diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_binding.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_binding.py new file mode 100644 index 00000000..d089a479 --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_binding.py @@ -0,0 +1,78 @@ +"""Fail-closed repository-generation leases for review retrieval.""" + +from __future__ import annotations + +from .repository_overlay import IncrementalIndexPreconditionError + + +def require_repository_generation( + index_manager, + *, + workspace: str, + project: str, + branch: str, + revision: str, + generation_manifest_sha256: str | None = None, + collection_target: str | None = None, +): + """Load one exact sealed generation and optionally match its receipt.""" + logical_collection = index_manager._get_project_collection_name( + workspace, + project, + ) + requested_target = collection_target or logical_collection + active_target = index_manager._collection_manager.resolve_collection_target( + requested_target + ) + if active_target is None: + raise IncrementalIndexPreconditionError( + "requested repository collection is unavailable" + ) + bound_target = active_target + result = index_manager.get_revision_preflight( + workspace, + project, + branch, + revision, + collection_target=bound_target, + ) + if result is None: + raise IncrementalIndexPreconditionError( + "requested repository revision is not available as one complete " + f"sealed generation: {branch}@{revision}" + ) + if ( + generation_manifest_sha256 is not None + and result["generation_manifest_sha256"] + != generation_manifest_sha256 + ): + raise IncrementalIndexPreconditionError( + "requested repository generation changed or does not match its " + f"receipt: {branch}@{revision}" + ) + return { + **result, + "_collection_target": bound_target, + "_lease_target": requested_target, + } + + +def require_same_repository_generation( + index_manager, + *, + workspace: str, + project: str, + branch: str, + revision: str, + receipt, +): + """Recheck a generation lease after a non-transactional retrieval/build.""" + return require_repository_generation( + index_manager, + workspace=workspace, + project=project, + branch=branch, + revision=revision, + generation_manifest_sha256=receipt["generation_manifest_sha256"], + collection_target=receipt["_lease_target"], + ) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py new file mode 100644 index 00000000..96825c99 --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py @@ -0,0 +1,492 @@ +"""Read-only verification for immutable repository index revisions.""" + +from __future__ import annotations + +import hashlib +import json +from types import SimpleNamespace + +from qdrant_client.models import FieldCondition, Filter, MatchValue + +from .generation_manifest import ( + GENERATION_MANIFEST_PATH, + GENERATION_MANIFEST_PAYLOAD_KEY, + GENERATION_SCHEMA, + GenerationManifestError, + canonical_index_selection_policy, + compute_index_selection_policy_sha256, + compute_generation_members_digest, + generation_manifest_content, + is_sha256_hex, + verified_generation_member, +) +from .index_representation import INDEX_REPRESENTATION_PAYLOAD_KEY +from .repository_overlay import ( + IncrementalIndexPreconditionError, + scroll_branch_points, +) + + +_IDENTITY_PAYLOAD_FIELDS = ( + "plugin_ids", + "plugin_fingerprint", + "plugin_descriptor_fingerprint", + "plugin_implementation_fingerprint", + INDEX_REPRESENTATION_PAYLOAD_KEY, +) + + +def _payload_identity(payload): + plugin_ids = payload.get("plugin_ids") + fingerprints = tuple( + payload.get(field) for field in _IDENTITY_PAYLOAD_FIELDS[1:] + ) + if ( + not isinstance(plugin_ids, (list, tuple)) + or not all( + isinstance(plugin_id, str) and plugin_id + for plugin_id in plugin_ids + ) + or not all( + isinstance(fingerprint, str) and fingerprint + for fingerprint in fingerprints + ) + ): + raise IncrementalIndexPreconditionError( + "exact revision point is missing repository build identity; " + "fully reindex the revision" + ) + return (tuple(plugin_ids), *fingerprints) + + +def _load_exact_repository_facts( + client, + collection_name: str, + branch: str, + commit: str, +): + """Load and integrity-check the facts sentinel for one exact revision.""" + from codecrow_plugins import RepositoryFacts + + points = scroll_branch_points( + client, + collection_name, + branch, + ( + FieldCondition(key="commit", match=MatchValue(value=commit)), + FieldCondition( + key="repository_facts_state", + match=MatchValue(value=True), + ), + ), + ) + if not points: + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts are missing; " + "fully reindex the revision" + ) + + payloads = [point.payload or {} for point in points] + for payload in payloads: + if ( + type(payload.get("facts_part")) is not int + or payload["facts_part"] < 0 + or type(payload.get("facts_parts")) is not int + or payload["facts_parts"] < 1 + ): + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts have invalid part " + "metadata; fully reindex the revision" + ) + ordered = sorted(payloads, key=lambda payload: payload.get("facts_part", -1)) + expected_parts = ordered[0].get("facts_parts") + actual_parts = [payload.get("facts_part") for payload in ordered] + if ( + actual_parts != list(range(expected_parts)) + or any(payload.get("facts_parts") != expected_parts for payload in ordered) + ): + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts are incomplete; " + "fully reindex the revision" + ) + + expected_digest = ordered[0].get("facts_content_sha256") + if ( + not isinstance(expected_digest, str) + or len(expected_digest) != 64 + or any( + character not in "0123456789abcdef" + for character in expected_digest + ) + or any( + payload.get("facts_content_sha256") != expected_digest + for payload in ordered + ) + ): + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts have invalid integrity " + "metadata; fully reindex the revision" + ) + + content_parts = [ + payload.get("text", payload.get("_node_content", "")) + for payload in ordered + ] + if not all(isinstance(part, str) for part in content_parts): + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts have invalid content; " + "fully reindex the revision" + ) + content = "".join(content_parts) + if hashlib.sha256(content.encode("utf-8")).hexdigest() != expected_digest: + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts failed integrity " + "validation; fully reindex the revision" + ) + + state_identity = None + for payload in ordered: + candidate = _payload_identity(payload) + if state_identity is None: + state_identity = candidate + elif state_identity != candidate: + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts have inconsistent " + "build identity; fully reindex the revision" + ) + + try: + decoded = json.loads(content) + repository_facts = RepositoryFacts( + revision=decoded["revision"], + paths=tuple(decoded["paths"]), + marker_contents=decoded.get("markerContents", {}), + ) + except Exception as exception: + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts are invalid; " + "fully reindex the revision" + ) from exception + if repository_facts.revision != commit: + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts do not match the " + "requested commit; fully reindex the revision" + ) + return repository_facts, expected_digest, state_identity + + +def _validate_generation_manifest( + manifest_points, + members, + branch: str, + commit: str, +): + """Validate the single seal against every observed non-manifest point.""" + if len(manifest_points) != 1: + reason = "missing" if not manifest_points else "not unique" + raise IncrementalIndexPreconditionError( + f"exact revision repository generation manifest is {reason}; " + "fully reindex the revision" + ) + + manifest_payload = manifest_points[0].payload or {} + expected_count = manifest_payload.get("generation_member_count") + expected_members_digest = manifest_payload.get( + "generation_members_sha256" + ) + expected_manifest_digest = manifest_payload.get( + "generation_manifest_sha256" + ) + source_tree_sha256 = manifest_payload.get("source_tree_sha256") + index_include_patterns = manifest_payload.get("index_include_patterns") + index_exclude_patterns = manifest_payload.get("index_exclude_patterns") + index_selection_policy_sha256 = manifest_payload.get( + "index_selection_policy_sha256" + ) + workspace = manifest_payload.get("workspace") + project = manifest_payload.get("project") + if ( + manifest_payload.get("generation_schema") != GENERATION_SCHEMA + or manifest_payload.get("path") != GENERATION_MANIFEST_PATH + or not isinstance(workspace, str) + or not workspace + or not isinstance(project, str) + or not project + or type(expected_count) is not int + or expected_count < 1 + or not is_sha256_hex(expected_members_digest) + or not is_sha256_hex(expected_manifest_digest) + or not is_sha256_hex(source_tree_sha256) + or not isinstance(index_include_patterns, list) + or not isinstance(index_exclude_patterns, list) + or not is_sha256_hex(index_selection_policy_sha256) + ): + raise IncrementalIndexPreconditionError( + "exact revision repository generation manifest is invalid; " + "fully reindex the revision" + ) + try: + selection_policy = canonical_index_selection_policy( + index_include_patterns, + index_exclude_patterns, + ) + observed_selection_policy_sha256 = ( + compute_index_selection_policy_sha256( + selection_policy["includePatterns"], + selection_policy["excludePatterns"], + ) + ) + except GenerationManifestError as exception: + raise IncrementalIndexPreconditionError( + "exact revision repository index selection policy is invalid; " + "fully reindex the revision" + ) from exception + if ( + index_include_patterns != selection_policy["includePatterns"] + or index_exclude_patterns != selection_policy["excludePatterns"] + or index_selection_policy_sha256 + != observed_selection_policy_sha256 + ): + raise IncrementalIndexPreconditionError( + "exact revision repository index selection policy failed " + "integrity validation; fully reindex the revision" + ) + if expected_count != len(members): + raise IncrementalIndexPreconditionError( + "exact revision repository generation is incomplete: " + f"expected={expected_count}, actual={len(members)}; " + "fully reindex the revision" + ) + + try: + observed_members_digest = compute_generation_members_digest(members) + except GenerationManifestError as exception: + raise IncrementalIndexPreconditionError( + "exact revision repository generation members are invalid; " + "fully reindex the revision" + ) from exception + if observed_members_digest != expected_members_digest: + raise IncrementalIndexPreconditionError( + "exact revision repository generation membership failed integrity " + "validation; fully reindex the revision" + ) + + expected_content = generation_manifest_content( + workspace=workspace, + project=project, + branch=branch, + commit=commit, + member_count=expected_count, + members_sha256=expected_members_digest, + source_tree_sha256=source_tree_sha256, + index_include_patterns=index_include_patterns, + index_exclude_patterns=index_exclude_patterns, + index_selection_policy_sha256=index_selection_policy_sha256, + ) + if ( + hashlib.sha256(expected_content.encode("utf-8")).hexdigest() + != expected_manifest_digest + ): + raise IncrementalIndexPreconditionError( + "exact revision repository generation manifest failed integrity " + "validation; fully reindex the revision" + ) + return { + "workspace": workspace, + "project": project, + "generation_schema": GENERATION_SCHEMA, + "generation_member_count": expected_count, + "generation_members_sha256": expected_members_digest, + "generation_manifest_sha256": expected_manifest_digest, + "source_tree_sha256": source_tree_sha256, + "index_include_patterns": index_include_patterns, + "index_exclude_patterns": index_exclude_patterns, + "index_selection_policy_sha256": index_selection_policy_sha256, + } + + +def _require_unmixed_branch_revision( + client, + collection_name: str, + branch: str, + commit: str, +) -> None: + """Reject a branch whose non-PR retrieval population spans revisions.""" + branch_filter = Filter( + must=[ + FieldCondition(key="branch", match=MatchValue(value=branch)), + ], + must_not=[ + FieldCondition(key="pr", match=MatchValue(value=True)), + ], + ) + offset = None + while True: + points, offset = client.scroll( + collection_name=collection_name, + scroll_filter=branch_filter, + limit=256, + offset=offset, + with_payload=["branch", "commit", "pr"], + with_vectors=False, + ) + for point in points: + payload = point.payload or {} + if ( + payload.get("branch") != branch + or payload.get("pr") is True + or payload.get("commit") != commit + ): + raise IncrementalIndexPreconditionError( + "exact revision branch contains mixed repository revisions; " + "fully reindex the revision" + ) + if offset is None: + break + + +def read_repository_revision_preflight( + client, + collection_name: str, + branch: str, + commit: str, +): + """Return a verified exact repository revision or ``None`` when absent. + + A revision is reusable only when every matching repository point carries + one consistent representation/plugin identity, its content-addressed + generation membership matches the atomic full-index seal, and its + repository-facts sentinel is complete, digest-valid, and bound to the + requested commit. Qdrant failures intentionally propagate so callers + cannot mistake an unavailable store for a missing revision. + """ + revision_filter = Filter( + must=[ + FieldCondition(key="branch", match=MatchValue(value=branch)), + FieldCondition(key="commit", match=MatchValue(value=commit)), + ], + must_not=[ + FieldCondition(key="pr", match=MatchValue(value=True)), + ], + ) + point_count = 0 + identity = None + manifest_points = [] + members = [] + member_validation_error = None + offset = None + + while True: + points, offset = client.scroll( + collection_name=collection_name, + scroll_filter=revision_filter, + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + point_count += len(points) + for point in points: + payload = point.payload or {} + if ( + payload.get("branch") != branch + or payload.get("commit") != commit + or payload.get("pr") is True + ): + raise IncrementalIndexPreconditionError( + "exact revision query returned a point outside the requested " + "repository snapshot" + ) + candidate = _payload_identity(payload) + if identity is None: + identity = candidate + elif identity != candidate: + raise IncrementalIndexPreconditionError( + "exact revision has inconsistent repository build identity; " + "fully reindex the revision" + ) + if payload.get(GENERATION_MANIFEST_PAYLOAD_KEY) is True: + # Retain only the small manifest payload. Keeping its 4096-d + # vector would otherwise pin one full Qdrant point until the + # complete generation scan finishes. + manifest_points.append(SimpleNamespace(payload=dict(payload))) + else: + try: + # Verify while this page is live and retain only the + # compact (point id, digest) receipt. Never accumulate + # complete payloads and vectors for the whole repository. + members.append(verified_generation_member(point)) + except GenerationManifestError as exception: + # Preserve validation priority from the original preflight: + # mixed identity/revision and missing-manifest diagnostics + # are established before member-content failure. Keep only + # the error text so its traceback cannot pin this page. + if member_validation_error is None: + member_validation_error = str(exception) + point = None + payload = None + del points + if offset is None: + break + + if point_count == 0: + return None + + _require_unmixed_branch_revision( + client, + collection_name, + branch, + commit, + ) + if len(manifest_points) != 1: + _validate_generation_manifest( + manifest_points, + [], + branch, + commit, + ) + if member_validation_error is not None: + raise IncrementalIndexPreconditionError( + "exact revision repository generation member content failed " + "integrity validation; fully reindex the revision" + ) + generation_identity = _validate_generation_manifest( + manifest_points, + members, + branch, + commit, + ) + repository_facts, facts_digest, state_identity = ( + _load_exact_repository_facts( + client, + collection_name, + branch, + commit, + ) + ) + if identity != state_identity: + raise IncrementalIndexPreconditionError( + "exact revision points do not match repository detection build " + "identity; fully reindex the revision" + ) + + ( + plugin_ids, + plugin_fingerprint, + descriptor_fingerprint, + implementation_fingerprint, + representation_fingerprint, + ) = identity + return { + "branch": branch, + "commit": commit, + "point_count": point_count, + "repository_revision": repository_facts.revision, + "repository_facts_sha256": facts_digest, + "plugin_ids": list(plugin_ids), + "plugin_fingerprint": plugin_fingerprint, + "plugin_descriptor_fingerprint": descriptor_fingerprint, + "plugin_implementation_fingerprint": implementation_fingerprint, + "index_representation_fingerprint": representation_fingerprint, + **generation_identity, + } diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight_cache.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight_cache.py new file mode 100644 index 00000000..cf8b72b0 --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight_cache.py @@ -0,0 +1,144 @@ +"""Bounded, single-flight cache for immutable revision verification receipts.""" + +from __future__ import annotations + +import copy +import threading +import time +from collections import OrderedDict +from dataclasses import dataclass +from typing import Callable, Hashable, Optional + + +@dataclass(frozen=True) +class RevisionPreflightKey: + """Identity of one tenant-bound immutable physical generation.""" + + collection: str + workspace: str + project: str + branch: str + commit: str + + +@dataclass +class _CacheEntry: + expires_at: float | None + value: Optional[dict] + + +@dataclass +class _Flight: + event: threading.Event + value: Optional[dict] = None + error: BaseException | None = None + + +class RevisionPreflightCache: + """Cache expensive immutable-generation verification without stampedes. + + Only the compact verification receipt is retained. A bounded semaphore + limits cold verification across different generations, while callers for + the same generation share one in-flight load. Loader failures are shared + with current waiters but are never cached, so a transient Qdrant failure + remains retryable. + """ + + def __init__( + self, + *, + max_entries: int, + ttl_seconds: float, + max_concurrent_loads: int, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if max_entries < 1: + raise ValueError("max_entries must be positive") + if ttl_seconds < 0: + raise ValueError("ttl_seconds must not be negative") + if max_concurrent_loads < 1: + raise ValueError("max_concurrent_loads must be positive") + self._max_entries = max_entries + self._ttl_seconds = ttl_seconds + self._clock = clock + self._lock = threading.Lock() + self._values: OrderedDict[Hashable, _CacheEntry] = OrderedDict() + self._flights: dict[Hashable, _Flight] = {} + self._load_slots = threading.BoundedSemaphore(max_concurrent_loads) + + def get_or_load( + self, + key: Hashable, + loader: Callable[[], Optional[dict]], + ) -> Optional[dict]: + """Return a cached receipt or execute one bounded single-flight load.""" + now = self._clock() + with self._lock: + entry = self._values.get(key) + if entry is not None and ( + entry.expires_at is None or entry.expires_at > now + ): + self._values.move_to_end(key) + return copy.deepcopy(entry.value) + if entry is not None: + self._values.pop(key, None) + + flight = self._flights.get(key) + owner = flight is None + if owner: + flight = _Flight(event=threading.Event()) + self._flights[key] = flight + + if not owner: + flight.event.wait() + if flight.error is not None: + raise flight.error + return copy.deepcopy(flight.value) + + try: + with self._load_slots: + value = loader() + except BaseException as exception: + with self._lock: + self._flights.pop(key, None) + flight.error = exception + flight.event.set() + raise + + stored_value = copy.deepcopy(value) + with self._lock: + # An absent revision can later appear in a legacy mutable target. + # Positive receipts belong to sealed immutable generations; cache + # those, but keep absence immediately observable and retryable. + if stored_value is not None: + self._values[key] = _CacheEntry( + expires_at=( + None + if self._ttl_seconds == 0 + else self._clock() + self._ttl_seconds + ), + value=stored_value, + ) + self._values.move_to_end(key) + while len(self._values) > self._max_entries: + self._values.popitem(last=False) + self._flights.pop(key, None) + flight.value = stored_value + flight.event.set() + return copy.deepcopy(stored_value) + + def invalidate_collection(self, collection: str) -> None: + """Discard cached receipts for one physical collection.""" + with self._lock: + keys = [ + key + for key in self._values + if getattr(key, "collection", None) == collection + ] + for key in keys: + self._values.pop(key, None) + + def clear(self) -> None: + """Discard all completed cache entries without disturbing loaders.""" + with self._lock: + self._values.clear() diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/source_tree.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/source_tree.py new file mode 100644 index 00000000..684c1c56 --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/source_tree.py @@ -0,0 +1,382 @@ +"""Immutable source-tree verification for repository indexing.""" + +from __future__ import annotations + +import hashlib +import os +import re +import stat +import subprocess +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import BinaryIO, Mapping + + +SOURCE_TREE_SCHEMA = "codecrow.repository-source-tree" +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + + +class RepositorySourceTreeError(RuntimeError): + """The indexing source is not the attested immutable repository tree.""" + + +@dataclass(frozen=True) +class RepositorySourceTree: + """Verified source identity retained across the indexing operation.""" + + commit: str + tree_sha256: str + git_commit_verified: bool + file_sha256_by_path: Mapping[str, str] + + +def _feed_framed(hasher, value: bytes) -> None: + hasher.update(len(value).to_bytes(8, "big")) + hasher.update(value) + + +def _relative_parts(relative_path: str | Path) -> tuple[str, ...]: + path = Path(relative_path) + parts = path.parts + if ( + path.is_absolute() + or not parts + or any(part in {"", ".", ".."} for part in parts) + ): + raise RepositorySourceTreeError( + f"invalid repository-relative source path: {relative_path}" + ) + return tuple(parts) + + +@contextmanager +def open_repository_file_no_follow( + repo_path: str | Path, + relative_path: str | Path, +) -> BinaryIO: + """Open one regular repository file through pinned, no-follow descriptors.""" + parts = _relative_parts(relative_path) + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + file_flags = os.O_RDONLY + no_follow = getattr(os, "O_NOFOLLOW", 0) + if not no_follow: + raise RepositorySourceTreeError( + "repository source verification requires O_NOFOLLOW support" + ) + directory_flags |= no_follow + file_flags |= no_follow + close_on_exec = getattr(os, "O_CLOEXEC", 0) + directory_flags |= close_on_exec + file_flags |= close_on_exec + + directory_fd = None + file_fd = None + try: + directory_fd = os.open(os.fspath(repo_path), directory_flags) + for component in parts[:-1]: + child_fd = os.open( + component, + directory_flags, + dir_fd=directory_fd, + ) + os.close(directory_fd) + directory_fd = child_fd + file_fd = os.open(parts[-1], file_flags, dir_fd=directory_fd) + file_stat = os.fstat(file_fd) + if not stat.S_ISREG(file_stat.st_mode): + raise RepositorySourceTreeError( + "repository source entry is not a regular file: " + + Path(*parts).as_posix() + ) + with os.fdopen(file_fd, "rb", closefd=True) as source: + file_fd = None + yield source + except OSError as exception: + raise RepositorySourceTreeError( + "cannot safely open repository source file: " + + Path(*parts).as_posix() + ) from exception + finally: + if file_fd is not None: + os.close(file_fd) + if directory_fd is not None: + os.close(directory_fd) + + +def read_repository_file_bytes( + repo_path: str | Path, + relative_path: str | Path, + *, + expected_sha256: str | None = None, +) -> bytes: + """Read one regular file without symlink traversal and verify its identity.""" + with open_repository_file_no_follow(repo_path, relative_path) as source: + content = source.read() + if ( + expected_sha256 is not None + and hashlib.sha256(content).hexdigest() != expected_sha256 + ): + raise RepositorySourceTreeError( + "repository source file changed after attestation: " + + Path(relative_path).as_posix() + ) + return content + + +def _repository_entries(root: Path): + """Yield non-Git source entries without following repository symlinks.""" + + collected_entries = [] + directory_flags = ( + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + if not getattr(os, "O_NOFOLLOW", 0): + raise RepositorySourceTreeError( + "repository source verification requires O_NOFOLLOW support" + ) + + def visit(directory_fd: int, relative_directory: Path): + try: + with os.scandir(directory_fd) as scanner: + entries = list(scanner) + except OSError as exception: + raise RepositorySourceTreeError( + f"cannot enumerate repository source directory: {relative_directory}" + ) from exception + + for entry in entries: + relative_path = relative_directory / entry.name + if relative_directory == Path() and entry.name == ".git": + continue + try: + entry_stat = os.stat( + entry.name, + dir_fd=directory_fd, + follow_symlinks=False, + ) + if stat.S_ISLNK(entry_stat.st_mode): + collected_entries.append( + ( + "symlink", + relative_path, + os.readlink(entry.name, dir_fd=directory_fd), + ) + ) + elif stat.S_ISDIR(entry_stat.st_mode): + child_fd = os.open( + entry.name, + directory_flags, + dir_fd=directory_fd, + ) + try: + visit(child_fd, relative_path) + finally: + os.close(child_fd) + elif stat.S_ISREG(entry_stat.st_mode): + collected_entries.append( + ("file", relative_path, None) + ) + else: + raise RepositorySourceTreeError( + "repository source contains an unsupported filesystem " + f"entry: {relative_path.as_posix()}" + ) + except OSError as exception: + raise RepositorySourceTreeError( + "cannot inspect repository source entry: " + f"{relative_path.as_posix()}" + ) from exception + + root_fd = None + try: + root_fd = os.open(os.fspath(root), directory_flags) + visit(root_fd, Path()) + except OSError as exception: + raise RepositorySourceTreeError( + f"cannot safely enumerate repository source directory: {root}" + ) from exception + finally: + if root_fd is not None: + os.close(root_fd) + collected_entries.sort( + key=lambda item: item[1] + .as_posix() + .encode("utf-8", "surrogateescape") + ) + yield from collected_entries + + +def iter_repository_regular_file_paths( + repo_path: str | Path, +): + """Yield regular-file paths from a no-follow repository traversal.""" + for kind, relative_path, _ in _repository_entries(Path(repo_path)): + if kind == "file": + yield relative_path + + +def _compute_repository_source_tree( + repo_path: str | Path, +) -> tuple[str, Mapping[str, str]]: + """Hash exact repository-relative paths and retain regular-file identities.""" + root = Path(repo_path) + try: + root_stat = root.lstat() + except OSError as exception: + raise RepositorySourceTreeError( + f"repository source path is not a directory: {root}" + ) from exception + if not stat.S_ISDIR(root_stat.st_mode) or stat.S_ISLNK(root_stat.st_mode): + raise RepositorySourceTreeError( + f"repository source path is not a directory: {root}" + ) + + hasher = hashlib.sha256() + _feed_framed(hasher, SOURCE_TREE_SCHEMA.encode("ascii")) + entry_count = 0 + file_sha256_by_path: dict[str, str] = {} + for kind, relative_path, value in _repository_entries(root): + entry_count += 1 + _feed_framed(hasher, kind.encode("ascii")) + _feed_framed( + hasher, + relative_path.as_posix().encode("utf-8", "surrogateescape"), + ) + if kind == "symlink": + _feed_framed( + hasher, + value.encode("utf-8", "surrogateescape"), + ) + continue + + try: + observed_size = 0 + file_hasher = hashlib.sha256() + with open_repository_file_no_follow(root, relative_path) as source: + expected_size = os.fstat(source.fileno()).st_size + hasher.update(expected_size.to_bytes(8, "big")) + while True: + chunk = source.read(1024 * 1024) + if not chunk: + break + observed_size += len(chunk) + hasher.update(chunk) + file_hasher.update(chunk) + except (OSError, RepositorySourceTreeError) as exception: + raise RepositorySourceTreeError( + "cannot read repository source file: " + f"{relative_path.as_posix()}" + ) from exception + if observed_size != expected_size: + raise RepositorySourceTreeError( + "repository source changed while it was being attested: " + f"{relative_path.as_posix()}" + ) + file_sha256_by_path[relative_path.as_posix()] = file_hasher.hexdigest() + + hasher.update(entry_count.to_bytes(8, "big")) + return hasher.hexdigest(), MappingProxyType(file_sha256_by_path) + + +def compute_repository_source_tree_sha256(repo_path: str | Path) -> str: + """Hash exact repository-relative paths and bytes deterministically.""" + return _compute_repository_source_tree(repo_path)[0] + + +def _git_output(root: Path, *arguments: str) -> str: + environment = dict(os.environ) + environment["GIT_OPTIONAL_LOCKS"] = "0" + try: + result = subprocess.run( + ["git", "-C", str(root), *arguments], + check=True, + capture_output=True, + text=True, + timeout=30, + env=environment, + ) + except (OSError, subprocess.SubprocessError) as exception: + raise RepositorySourceTreeError( + "cannot verify repository Git revision" + ) from exception + return result.stdout + + +def _verify_git_checkout(root: Path, commit: str) -> bool: + """Require exact HEAD and no tracked or untracked working-tree changes.""" + git_marker = root / ".git" + if not git_marker.exists(): + return False + + observed_commit = _git_output(root, "rev-parse", "--verify", "HEAD^{commit}").strip() + if observed_commit != commit: + raise RepositorySourceTreeError( + "repository Git HEAD does not match the supplied commit: " + f"expected={commit}, actual={observed_commit}" + ) + status = _git_output( + root, + "status", + "--porcelain=v1", + "--untracked-files=all", + ) + if status: + raise RepositorySourceTreeError( + "repository Git working tree is not clean for the supplied commit" + ) + return True + + +def verify_repository_source_tree( + repo_path: str | Path, + commit: str, + expected_tree_sha256: str, +) -> RepositorySourceTree: + """Verify the caller-attested tree and, for Git worktrees, exact HEAD.""" + if not isinstance(commit, str) or not commit: + raise RepositorySourceTreeError("repository source commit is required") + if ( + not isinstance(expected_tree_sha256, str) + or not _SHA256_RE.fullmatch(expected_tree_sha256) + ): + raise RepositorySourceTreeError( + "repository source tree requires a canonical SHA-256 attestation" + ) + + root = Path(repo_path) + git_commit_verified = _verify_git_checkout(root, commit) + observed_tree_sha256, file_sha256_by_path = _compute_repository_source_tree( + root + ) + if observed_tree_sha256 != expected_tree_sha256: + raise RepositorySourceTreeError( + "repository source tree does not match its acquisition attestation: " + f"expected={expected_tree_sha256}, actual={observed_tree_sha256}" + ) + return RepositorySourceTree( + commit=commit, + tree_sha256=observed_tree_sha256, + git_commit_verified=git_commit_verified, + file_sha256_by_path=file_sha256_by_path, + ) + + +def require_repository_source_tree_unchanged( + repo_path: str | Path, + source_tree: RepositorySourceTree, +) -> None: + """Recheck the exact source just before its generation is sealed.""" + verified = verify_repository_source_tree( + repo_path, + source_tree.commit, + source_tree.tree_sha256, + ) + if verified.git_commit_verified != source_tree.git_commit_verified: + raise RepositorySourceTreeError( + "repository source verification mode changed during indexing" + ) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py index ca46d9c4..163b39aa 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py @@ -113,6 +113,24 @@ class RAGConfig(BaseModel): ), ge=0, ) + revision_preflight_cache_entries: int = Field( + default_factory=lambda: int( + os.getenv("RAG_REVISION_PREFLIGHT_CACHE_ENTRIES", "512") + ), + ge=1, + ) + revision_preflight_cache_ttl_seconds: int = Field( + default_factory=lambda: int( + os.getenv("RAG_REVISION_PREFLIGHT_CACHE_TTL_SECONDS", "0") + ), + ge=0, + ) + revision_preflight_max_concurrency: int = Field( + default_factory=lambda: int( + os.getenv("RAG_REVISION_PREFLIGHT_MAX_CONCURRENCY", "2") + ), + ge=1, + ) # Embedding dimensions - auto-detected from model or set via env var embedding_dim: int = Field(default_factory=lambda: int(os.getenv("EMBEDDING_DIM", "0"))) @@ -254,3 +272,6 @@ class IndexStats(BaseModel): workspace: str project: str branch: str + generation_manifest_sha256: Optional[str] = None + source_tree_sha256: Optional[str] = None + collection_target: Optional[str] = None diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py index cee27af4..9926393a 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py @@ -158,6 +158,7 @@ async def _handle_job(self, payload_str: str): # index_manager.index_repository is synchronous, so we run it in an executor loop = asyncio.get_running_loop() progress_delivery_available = True + progress_futures = [] def publish_progress(event: Dict[str, Any]) -> None: nonlocal progress_delivery_available @@ -168,17 +169,10 @@ def publish_progress(event: Dict[str, Any]) -> None: self._publish_event(event_queue_key, payload), loop, ) - # Surface Redis publication failures promptly without coupling - # indexing correctness to progress delivery. - try: - future.result(timeout=5) - except Exception as exception: - progress_delivery_available = False - logger.warning( - "Could not publish RAG progress for job %s: %s", - job_id, - exception, - ) + # Never block the indexing executor waiting for its own event + # loop. Drain these publications before the terminal event so + # ordering is retained and delivery remains fail-open. + progress_futures.append(future) indexing_future = loop.run_in_executor( None, @@ -188,9 +182,11 @@ def publish_progress(event: Dict[str, Any]) -> None: project=request_dto.project, branch=request_dto.branch, commit=request_dto.commit, + source_tree_sha256=request_dto.source_tree_sha256, preserve_other_branches=request_dto.preserve_other_branches, include_patterns=request_dto.include_patterns, exclude_patterns=request_dto.exclude_patterns, + collection_target=request_dto.collection_target, progress_callback=publish_progress, ) ) @@ -209,6 +205,16 @@ def publish_progress(event: Dict[str, Any]) -> None: }) result_obj = await indexing_future + for progress_future in progress_futures: + try: + await asyncio.wrap_future(progress_future) + except Exception as exception: + progress_delivery_available = False + logger.warning( + "Could not publish RAG progress for job %s: %s", + job_id, + exception, + ) # Serialize the IndexStats result to a dictionary result = result_obj.dict() if hasattr(result_obj, "dict") else result_obj.model_dump() diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py index 5635e918..a2c8b192 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py @@ -96,7 +96,7 @@ def _collection_or_alias_exists(self, name: str) -> bool: return False def _get_project_collection_name(self, workspace: str, project: str) -> str: - """Generate collection name for a project (single collection for all branches).""" + """Generate the legacy shared collection name for a project.""" namespace = make_project_namespace(workspace, project) return f"{self.config.qdrant_collection_prefix}_{namespace}" diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py index ca0bb9a1..65b335ff 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py @@ -12,6 +12,8 @@ from qdrant_client.http.models import Filter, FieldCondition, MatchValue, MatchAny from .base import RAGQueryBase +from ..core.pr_overlay_manifest import PR_OVERLAY_MANIFEST_PAYLOAD_KEY +from ..core.repository_overlay import IncrementalIndexPreconditionError from rag_pipeline.utils.path_identity import ( normalize_repository_path, repository_path_suffix_candidates, @@ -235,7 +237,13 @@ def get_deterministic_context( limit_per_file: int = 10, pr_number: Optional[int] = None, pr_changed_files: Optional[List[str]] = None, - additional_identifiers: Optional[List[str]] = None + additional_identifiers: Optional[List[str]] = None, + expected_revisions: Optional[Dict[str, str]] = None, + pr_source_revision: Optional[str] = None, + pr_base_revision: Optional[str] = None, + pr_base_generation_manifest_sha256: Optional[str] = None, + pr_generation_fingerprint: Optional[str] = None, + collection_target: Optional[str] = None, ) -> Dict: """ Get context using DETERMINISTIC metadata-based retrieval. @@ -271,7 +279,28 @@ def get_deterministic_context( Returns: Dict with chunks grouped by retrieval type and rich metadata """ - collection_name = self._get_project_collection_name(workspace, project) + if pr_generation_fingerprint and not all(( + pr_number, + pr_source_revision, + pr_base_revision, + pr_base_generation_manifest_sha256, + )): + raise IncrementalIndexPreconditionError( + "PR generation fingerprint requires PR number and complete " + "source/base generation identity" + ) + if pr_generation_fingerprint and len([ + branch for branch in branches if branch + ]) != 1: + raise IncrementalIndexPreconditionError( + "revision-bound deterministic context requires exactly one " + "authoritative branch" + ) + + collection_name = ( + collection_target + or self._get_project_collection_name(workspace, project) + ) if not self._collection_or_alias_exists(collection_name): logger.warning(f"Collection {collection_name} does not exist") @@ -288,6 +317,11 @@ def get_deterministic_context( }} file_paths = sorted({path.lstrip("/") for path in file_paths if path}) + pr_changed_path_set = { + normalize_repository_path(path) + for path in (pr_changed_files or []) + if normalize_repository_path(path) + } branches = list(dict.fromkeys(branch for branch in branches if branch)) self._observe_branches(collection_name, branches) logger.info(f"Deterministic context: files={file_paths[:5]}, branches={branches}") @@ -295,19 +329,80 @@ def get_deterministic_context( # ── Build branch filter ── target_branch = branches[0] if branches else None + repository_branch_filters = [] + for branch in branches: + conditions = [ + FieldCondition( + key="branch", + match=MatchValue(value=branch), + ), + ] + if expected_revisions and branch in expected_revisions: + conditions.append(FieldCondition( + key="commit", + match=MatchValue(value=expected_revisions[branch]), + )) + repository_branch_filters.append(Filter(must=conditions)) base_branch_condition = ( - FieldCondition(key="branch", match=MatchValue(value=branches[0])) - if len(branches) == 1 - else FieldCondition(key="branch", match=MatchAny(any=branches)) + repository_branch_filters[0] + if len(repository_branch_filters) == 1 + else Filter(should=repository_branch_filters) ) if pr_number: - branch_filter = Filter(should=[ - Filter(must=[base_branch_condition]), - Filter(must=[ - FieldCondition(key="pr", match=MatchValue(value=True)), - FieldCondition(key="pr_number", match=MatchValue(value=pr_number)) + pr_conditions = [ + FieldCondition(key="pr", match=MatchValue(value=True)), + FieldCondition( + key="pr_number", + match=MatchValue(value=pr_number), + ), + ] + if pr_generation_fingerprint: + pr_conditions.extend([ + FieldCondition( + key="pr_source_revision", + match=MatchValue(value=pr_source_revision), + ), + FieldCondition( + key="pr_base_revision", + match=MatchValue(value=pr_base_revision), + ), + FieldCondition( + key="pr_base_generation_manifest_sha256", + match=MatchValue( + value=pr_base_generation_manifest_sha256 + ), + ), + FieldCondition( + key="pr_generation_fingerprint", + match=MatchValue(value=pr_generation_fingerprint), + ), ]) + base_conditions = [base_branch_condition] + base_exclusions = ( + [ + FieldCondition( + key="path", + match=MatchAny(any=sorted(pr_changed_path_set)), + ), + ] + if pr_changed_path_set + else [] + ) + branch_filter = Filter(should=[ + Filter( + must=base_conditions, + must_not=base_exclusions, + ), + Filter( + must=pr_conditions, + must_not=[ + FieldCondition( + key=PR_OVERLAY_MANIFEST_PAYLOAD_KEY, + match=MatchValue(value=True), + ), + ], + ), ]) logger.info(f"Deterministic hybrid mode: also searching PR-indexed data (pr_number={pr_number})") else: @@ -333,9 +428,7 @@ def get_deterministic_context( # The request is the authority for invalidating materialized branch # context. Do not rely on finding a PR-indexed chunk: deleted files and # architecture-only files legitimately have no replacement code chunk. - changed_file_paths = { - path.lstrip("/") for path in (pr_changed_files or []) if path - } + changed_file_paths = set(pr_changed_path_set) seen_texts = set() target_branch_paths = set() @@ -476,6 +569,38 @@ def get_deterministic_context( f"class_ctx: {sum(len(v) for v in class_context.values())}, " f"ns_ctx: {sum(len(v) for v in namespace_context.values())})") + for chunk in all_chunks: + metadata = chunk.get("metadata") or {} + if metadata.get("pr") is True: + if pr_generation_fingerprint and ( + metadata.get("pr_generation_fingerprint") + != pr_generation_fingerprint + or metadata.get("pr_source_revision") + != pr_source_revision + or metadata.get("pr_base_revision") != pr_base_revision + or metadata.get( + "pr_base_generation_manifest_sha256" + ) != pr_base_generation_manifest_sha256 + ): + raise IncrementalIndexPreconditionError( + "deterministic retrieval returned a PR point outside " + "the requested overlay generation" + ) + continue + expected_revision = ( + expected_revisions.get(metadata.get("branch")) + if expected_revisions + else None + ) + if ( + expected_revision is not None + and metadata.get("commit") != expected_revision + ): + raise IncrementalIndexPreconditionError( + "deterministic retrieval returned a repository point " + "outside the requested immutable revision" + ) + return { "chunks": all_chunks, "changed_files": changed_files_chunks, @@ -839,6 +964,20 @@ def _query_changed_file( key=_point_sort_key, ) + # A revision-bound PR request's changed-path manifest is authoritative. + # Modified paths may have an exact overlay member; deleted and + # architecture-only paths legitimately may not. In either case, never + # fall back to the pre-PR target-branch source for that path. + if any( + repository_paths_match(normalized_path, changed_path) + for changed_path in changed_file_paths + ): + results = [ + point + for point in results + if (point.payload or {}).get("pr") is True + ] + # Apply branch priority if target_branch and len(branches) > 1: has_target = any(p.payload.get("branch") == target_branch for p in results) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/pr_context.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/pr_context.py index fca4935b..c8555bdc 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/pr_context.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/pr_context.py @@ -12,6 +12,7 @@ from .base import RAGQueryBase from .duplication import generate_duplication_queries +from ..core.repository_overlay import IncrementalIndexPreconditionError from ..models.instructions import InstructionType from ..models.scoring_config import get_scoring_config from ..utils.utils import detect_language_from_path @@ -93,7 +94,9 @@ def get_context_for_pr( min_relevance_score: float = 0.7, base_branch: Optional[str] = None, deleted_files: Optional[List[str]] = None, - exclude_pr_files: Optional[List[str]] = None + exclude_pr_files: Optional[List[str]] = None, + expected_revisions: Optional[Dict[str, str]] = None, + collection_target: Optional[str] = None, ) -> Dict: """ Get relevant context for review using Smart RAG. @@ -117,10 +120,17 @@ def get_context_for_pr( # Determine branches to search branches_to_search = [branch] - collection_name = self._get_project_collection_name(workspace, project) + collection_name = ( + collection_target + or self._get_project_collection_name(workspace, project) + ) if not self._collection_or_alias_exists(collection_name): logger.warning(f"Collection {collection_name} does not exist") + if expected_revisions is not None: + raise IncrementalIndexPreconditionError( + "revision-bound PR-context collection is unavailable" + ) return { "relevant_code": [], "related_files": [], @@ -167,7 +177,9 @@ def get_context_for_pr( branches=branches_to_search, top_k=q_top_k, instruction_type=q_instruction_type, - excluded_paths=all_excluded_paths + excluded_paths=all_excluded_paths, + expected_revisions=expected_revisions, + collection_target=collection_name, ) logger.info(f"Query {i+1}/{len(queries)} returned {len(results)} results") diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/semantic_search.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/semantic_search.py index ce7daaf1..bbf28853 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/semantic_search.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/semantic_search.py @@ -12,6 +12,7 @@ from qdrant_client.http.models import FieldCondition, MatchValue, MatchAny from .base import RAGQueryBase +from ..core.repository_overlay import IncrementalIndexPreconditionError from ..models.instructions import InstructionType, format_query logger = logging.getLogger(__name__) @@ -36,7 +37,9 @@ def semantic_search( branch: str, top_k: int = 10, filter_language: Optional[str] = None, - instruction_type: InstructionType = InstructionType.GENERAL + instruction_type: InstructionType = InstructionType.GENERAL, + expected_revision: Optional[str] = None, + collection_target: Optional[str] = None, ) -> List[Dict]: """Perform semantic search in the repository for a single branch.""" return self.semantic_search_multi_branch( @@ -46,7 +49,11 @@ def semantic_search( branches=[branch], top_k=top_k, filter_language=filter_language, - instruction_type=instruction_type + instruction_type=instruction_type, + expected_revisions=( + {branch: expected_revision} if expected_revision else None + ), + collection_target=collection_target, ) def semantic_search_multi_branch( @@ -58,7 +65,9 @@ def semantic_search_multi_branch( top_k: int = 10, filter_language: Optional[str] = None, instruction_type: InstructionType = InstructionType.GENERAL, - excluded_paths: Optional[List[str]] = None + excluded_paths: Optional[List[str]] = None, + expected_revisions: Optional[Dict[str, str]] = None, + collection_target: Optional[str] = None, ) -> List[Dict]: """Perform semantic search across multiple branches with filtering. @@ -66,7 +75,10 @@ def semantic_search_multi_branch( branches: List of branches to search (e.g., ['feature/xyz', 'main']) excluded_paths: Files to exclude from results (e.g., deleted files) """ - collection_name = self._get_project_collection_name(workspace, project) + collection_name = ( + collection_target + or self._get_project_collection_name(workspace, project) + ) excluded_paths = excluded_paths or [] logger.info(f"Multi-branch search in {collection_name} branches={branches} for: {query[:50]}...") @@ -74,21 +86,42 @@ def semantic_search_multi_branch( try: if not self._collection_or_alias_exists(collection_name): logger.warning(f"Collection {collection_name} does not exist") + if expected_revisions is not None: + raise IncrementalIndexPreconditionError( + "revision-bound semantic-search collection is unavailable" + ) return [] - self._observe_branches(collection_name, branches) + observer = getattr(self, "_observe_branches", None) + if callable(observer): + observer(collection_name, branches) + else: + self._require_compatible_branches(collection_name, branches) # Get or create cached VectorStoreIndex index = self._get_or_create_index(collection_name) # Create retriever with branch filter + if expected_revisions and len(branches) != 1: + raise ValueError( + "revision-bound semantic search requires one authoritative branch" + ) filters = [] for branch in branches: filters.append(MetadataFilter(key="branch", value=branch, operator=FilterOperator.EQ)) + if expected_revisions and branch in expected_revisions: + filters.append(MetadataFilter( + key="commit", + value=expected_revisions[branch], + operator=FilterOperator.EQ, + )) branch_filters = MetadataFilters( filters=filters, - condition="or" if len(filters) > 1 else "and" + condition=( + "and" if expected_revisions + else ("or" if len(filters) > 1 else "and") + ) ) # LlamaIndex's strict MetadataFilter value model does not accept # booleans even though Qdrant payloads do. Retrieve a bounded @@ -118,6 +151,7 @@ def semantic_search_multi_branch( "repository_snapshot", "repository_facts_state", "architecture_source", + "repository_generation_manifest", ) ): continue @@ -143,6 +177,8 @@ def semantic_search_multi_branch( except Exception as e: logger.error(f"Error during multi-branch semantic search: {e}") + if expected_revisions is not None: + raise return [] def _dedupe_by_branch_priority( diff --git a/python-ecosystem/rag-pipeline/tests/test_coordination.py b/python-ecosystem/rag-pipeline/tests/test_coordination.py index c8c542a4..2d560346 100644 --- a/python-ecosystem/rag-pipeline/tests/test_coordination.py +++ b/python-ecosystem/rag-pipeline/tests/test_coordination.py @@ -11,6 +11,7 @@ RedisPermitPool, ) from rag_pipeline.core.index_manager.collection_manager import CollectionManager +from rag_pipeline.core.index_manager.manager import RAGIndexManager def _coordinator(timeout=0): @@ -46,6 +47,69 @@ def test_project_mutation_lease_rejects_an_overlapping_job(): pass +def test_exact_generation_targets_have_independent_mutation_resources(): + coordinator = _coordinator() + + main = coordinator._resource_key("workspace", "project", "main-target") + develop = coordinator._resource_key("workspace", "project", "develop-target") + + assert main != develop + assert main == coordinator._resource_key("workspace", "project", "main-target") + + +def test_branch_publication_scope_serializes_only_the_same_branch_head(): + coordinator = _coordinator() + + main = coordinator._resource_key( + "workspace", "project", "main-target", "branch-head:main" + ) + main_next = coordinator._resource_key( + "workspace", "project", "next-main-target", "branch-head:main" + ) + develop = coordinator._resource_key( + "workspace", "project", "develop-target", "branch-head:develop" + ) + + assert main == main_next + assert main != develop + + +def test_pr_overlay_scope_serializes_only_the_same_pr(): + coordinator = _coordinator() + + pr_41_index = coordinator._resource_key( + "workspace", "project", publication_scope="pr-overlay:41" + ) + pr_41_delete = coordinator._resource_key( + "workspace", "project", publication_scope="pr-overlay:41" + ) + pr_42_index = coordinator._resource_key( + "workspace", "project", publication_scope="pr-overlay:42" + ) + + assert pr_41_index == pr_41_delete + assert pr_41_index != pr_42_index + + +def test_index_manager_binds_overlay_mutations_to_pr_scope(): + coordinator = MagicMock() + manager = SimpleNamespace(_mutation_coordinator=coordinator) + lease = object() + coordinator.acquire.return_value = lease + + result = RAGIndexManager.pr_overlay_mutation( + manager, "workspace", "project", 42, "index-pr-overlay" + ) + + assert result is lease + coordinator.acquire.assert_called_once_with( + "workspace", + "project", + "index-pr-overlay", + publication_scope="pr-overlay:42", + ) + + def test_project_mutation_coordination_fails_closed_when_redis_is_unavailable(): coordinator = _coordinator() coordinator._client.set.side_effect = RuntimeError("redis unavailable") diff --git a/python-ecosystem/rag-pipeline/tests/test_generation_advance.py b/python-ecosystem/rag-pipeline/tests/test_generation_advance.py new file mode 100644 index 00000000..f7c22c9e --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_generation_advance.py @@ -0,0 +1,284 @@ +from types import SimpleNamespace +from unittest.mock import MagicMock +import hashlib +import json +import pytest + +from llama_index.core.schema import TextNode +from qdrant_client import QdrantClient +from qdrant_client.models import ( + CreateAlias, + CreateAliasOperation, + Distance, + VectorParams, +) + +from rag_pipeline.core.generation_manifest import ( + build_generation_manifest_node, + collect_generation_members, + compute_generation_members_digest, +) +from rag_pipeline.core.index_manager.collection_manager import CollectionManager +from rag_pipeline.core.index_manager.manager import RAGIndexManager +from rag_pipeline.core.index_manager.point_operations import PointOperations +from rag_pipeline.core.index_manager.stats_manager import StatsManager +from rag_pipeline.core.revision_preflight import read_repository_revision_preflight +from rag_pipeline.core.repository_overlay import IncrementalIndexPreconditionError +from rag_pipeline.core.source_tree import compute_repository_source_tree_sha256 + + +SOURCE_COMMIT = "a" * 40 +TARGET_COMMIT = "b" * 40 +SOURCE_TREE = "c" * 64 + + +class _Embedding: + def get_text_embedding_batch(self, texts): + return [[1.0, 0.0, 0.0, 0.0] for _ in texts] + + +class _Lease: + token = "d" * 32 + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def assert_owned(self): + return None + + +class _Coordinator: + def __init__(self): + self.calls = [] + + def acquire(self, *_args, **kwargs): + self.calls.append((tuple(_args), kwargs)) + return _Lease() + + +def _sealed_source(client, point_ops, collection): + identity = { + "plugin_ids": [], + "plugin_fingerprint": "sha256:" + "1" * 64, + "plugin_descriptor_fingerprint": "sha256:" + "2" * 64, + "plugin_implementation_fingerprint": "sha256:" + "3" * 64, + "index_representation_fingerprint": "sha256:" + "4" * 64, + } + facts = json.dumps( + { + "revision": SOURCE_COMMIT, + "paths": ["src/A.java", "src/B.java"], + "markerContents": {}, + }, + sort_keys=True, + separators=(",", ":"), + ) + nodes = [ + TextNode(text="alpha", metadata={ + "workspace": "ws", "project": "project", + "path": "src/A.java", "branch": "develop", + "commit": SOURCE_COMMIT, **identity, + }), + TextNode(text="beta", metadata={ + "workspace": "ws", "project": "project", + "path": "src/B.java", "branch": "develop", + "commit": SOURCE_COMMIT, **identity, + }), + TextNode(text=facts, metadata={ + "workspace": "ws", "project": "project", + "path": "__analysis_state__/repository-facts/000000.state", + "branch": "develop", "commit": SOURCE_COMMIT, + "repository_facts_state": True, + "facts_part": 0, + "facts_parts": 1, + "facts_content_sha256": hashlib.sha256( + facts.encode("utf-8") + ).hexdigest(), + **identity, + }), + ] + assert point_ops.process_and_upsert_chunks( + nodes, collection, "ws", "project", "develop" + ) == (3, 0) + members = collect_generation_members( + client, collection, "develop", SOURCE_COMMIT + ) + manifest = build_generation_manifest_node( + workspace="ws", + project="project", + branch="develop", + commit=SOURCE_COMMIT, + member_count=len(members), + members_sha256=compute_generation_members_digest(members), + source_tree_sha256=SOURCE_TREE, + index_include_patterns=(), + index_exclude_patterns=(), + identity_metadata=identity, + ) + assert point_ops.process_and_upsert_chunks( + [manifest], collection, "ws", "project", "develop" + ) == (1, 0) + + +def test_copy_on_write_advance_keeps_source_and_seals_target_revision(tmp_path): + client = QdrantClient(":memory:") + source_physical = "develop_source_physical" + source_alias = "develop_source" + target_alias = "develop_target" + client.create_collection( + collection_name=source_physical, + vectors_config=VectorParams(size=4, distance=Distance.COSINE), + ) + client.update_collection_aliases(change_aliases_operations=[ + CreateAliasOperation(create_alias=CreateAlias( + alias_name=source_alias, + collection_name=source_physical, + )) + ]) + point_ops = PointOperations( + client, _Embedding(), batch_size=50, embedding_dim=4 + ) + _sealed_source(client, point_ops, source_physical) + + manager = object.__new__(RAGIndexManager) + manager.qdrant_client = client + manager._collection_manager = CollectionManager(client, 4) + manager._mutation_coordinator = _Coordinator() + manager._file_ops = MagicMock() + manager._point_ops = point_ops + manager._stats_manager = StatsManager(client, "rag") + manager.config = SimpleNamespace(qdrant_collection_prefix="rag") + target_tree = compute_repository_source_tree_sha256(tmp_path) + + result = manager.advance_generation( + source_collection_target=source_alias, + target_collection_target=target_alias, + source_commit=SOURCE_COMMIT, + source_tree_sha256=target_tree, + updated_file_paths=[], + deleted_file_paths=[], + repo_base=str(tmp_path), + workspace="ws", + project="project", + branch="develop", + commit=TARGET_COMMIT, + publish_branch_alias=True, + ) + + source = read_repository_revision_preflight( + client, source_alias, "develop", SOURCE_COMMIT + ) + target = read_repository_revision_preflight( + client, target_alias, "develop", TARGET_COMMIT + ) + assert source is not None + assert target is not None + assert target["generation_manifest_sha256"] == ( + result.generation_manifest_sha256 + ) + assert target["generation_manifest_sha256"] != ( + source["generation_manifest_sha256"] + ) + assert result.collection_target == target_alias + aliases = { + alias.alias_name: alias.collection_name + for alias in client.get_aliases().aliases + } + assert aliases["rag_ws__project__develop"] == aliases[target_alias] + manager._file_ops.apply_changes.assert_called_once() + assert manager._mutation_coordinator.calls[-1][1]["publication_scope"] == ( + "branch-head:develop" + ) + + +def test_advance_is_idempotent_when_exact_target_already_exists(): + manager = object.__new__(RAGIndexManager) + manager._mutation_coordinator = _Coordinator() + manager._collection_manager = MagicMock() + manager._collection_manager.resolve_collection_target.side_effect = [ + "source-physical", "target-physical" + ] + manager.qdrant_client = MagicMock() + manager._stats_manager = MagicMock() + expected = SimpleNamespace(generation_manifest_sha256="e" * 64) + manager._stats_manager.get_branch_stats.return_value = expected + manager.get_revision_preflight = MagicMock(return_value={ + "generation_manifest_sha256": "e" * 64, + "source_tree_sha256": SOURCE_TREE, + }) + + result = manager.advance_generation( + "source", "target", SOURCE_COMMIT, SOURCE_TREE, [], [], None, + "ws", "project", "develop", TARGET_COMMIT, + ) + + assert result is expected + manager._collection_manager.create_pending_collection.assert_not_called() + + +def test_existing_target_is_not_published_when_tenant_preflight_rejects_it(): + manager = object.__new__(RAGIndexManager) + manager._mutation_coordinator = _Coordinator() + manager._collection_manager = MagicMock() + manager._collection_manager.resolve_collection_target.side_effect = [ + "source-physical", "foreign-target-physical" + ] + manager.qdrant_client = MagicMock() + manager._stats_manager = MagicMock() + manager.config = SimpleNamespace(qdrant_collection_prefix="rag") + manager.get_revision_preflight = MagicMock(side_effect=[ + {"generation_manifest_sha256": "1" * 64}, + IncrementalIndexPreconditionError( + "repository generation coordinates do not match the requested tenant" + ), + ]) + + with pytest.raises( + IncrementalIndexPreconditionError, + match="coordinates do not match", + ): + manager.advance_generation( + "source", "foreign-target", SOURCE_COMMIT, SOURCE_TREE, + [], [], None, "ws", "project", "develop", TARGET_COMMIT, + publish_branch_alias=True, + ) + + manager._collection_manager.atomic_assign_aliases.assert_not_called() + + +def test_exact_generation_delete_verifies_tenant_coordinates(): + client = QdrantClient(":memory:") + client.create_collection( + collection_name="foreign_generation", + vectors_config=VectorParams(size=4, distance=Distance.COSINE), + ) + point_ops = PointOperations(client, _Embedding(), embedding_dim=4) + point_ops.process_and_upsert_chunks( + [TextNode(text="secret", metadata={ + "workspace": "other-ws", + "project": "other-project", + "branch": "develop", + "path": "Secret.java", + })], + "foreign_generation", + "other-ws", + "other-project", + "develop", + ) + manager = object.__new__(RAGIndexManager) + manager.qdrant_client = client + manager._collection_manager = CollectionManager(client, 4) + manager._mutation_coordinator = _Coordinator() + + with pytest.raises( + IncrementalIndexPreconditionError, + match="does not belong", + ): + manager.delete_collection_target( + "ws", "project", "develop", "foreign_generation" + ) + + assert manager._collection_manager.collection_exists("foreign_generation") diff --git a/python-ecosystem/rag-pipeline/tests/test_index_manager.py b/python-ecosystem/rag-pipeline/tests/test_index_manager.py index d0598a24..aba24d6d 100644 --- a/python-ecosystem/rag-pipeline/tests/test_index_manager.py +++ b/python-ecosystem/rag-pipeline/tests/test_index_manager.py @@ -108,6 +108,24 @@ def test_create_pending_collection_uses_unique_names(self): assert first != second + def test_atomic_assign_aliases_replaces_all_requested_aliases_in_one_call(self): + cm = self._make() + old = MagicMock() + old.alias_name = "codecrow_ws__project" + old.collection_name = "old-primary" + cm.client.get_aliases.return_value.aliases = [old] + + cm.atomic_assign_aliases({ + "codecrow_ws__project": "new-generation", + "codecrow_ws__project__develop": "new-generation", + }) + + cm.client.update_collection_aliases.assert_called_once() + operations = cm.client.update_collection_aliases.call_args.kwargs[ + "change_aliases_operations" + ] + assert len(operations) == 3 + def test_payload_index_failure_does_not_skip_remaining_indexes(self): cm = self._make() cm.client.create_payload_index.side_effect = [ @@ -274,7 +292,7 @@ def test_embed_and_create_points(self): points = po.embed_and_create_points([("point-id-1", mock_chunk)]) assert len(points) == 1 assert points[0].id == "point-id-1" - assert points[0].vector == [0.1, 0.2, 0.3] + assert points[0].vector == pytest.approx([0.1, 0.2, 0.3]) # ───────────────────────────────────────────────────────────── @@ -342,6 +360,16 @@ def test_list_all_indices(self): # ───────────────────────────────────────────────────────────── class TestRAGIndexManager: + @pytest.fixture(autouse=True) + def avoid_network_tokenizer_download(self, monkeypatch): + # Constructing LlamaIndex's default SentenceSplitter may lazily fetch + # the tiktoken vocabulary. These manager unit tests mock embeddings and + # Qdrant, so they must remain hermetic as well. + from rag_pipeline.core.index_manager.manager import Settings + + monkeypatch.setattr(Settings, "_node_parser", MagicMock(), raising=False) + + def _mock_config(self): mock_config = MagicMock() mock_config.qdrant_url = "http://localhost:6333" @@ -386,6 +414,56 @@ def test_get_project_collection_name(self, MockQdrant, mock_info, mock_create): assert name.startswith("rag_") assert "workspace" in name + def test_branch_operator_alias_is_readable_and_branch_specific(self): + from rag_pipeline.core.index_manager.manager import RAGIndexManager + + manager = object.__new__(RAGIndexManager) + manager.config = MagicMock(qdrant_collection_prefix="rag") + manager.config.qdrant_collection_prefix = "rag" + + assert manager._get_branch_operator_alias( + "Workspace", "Project", "develop" + ) == "rag_workspace__project__develop" + assert manager._get_branch_operator_alias( + "Workspace", "Project", "release/1.2" + ).startswith("rag_workspace__project__release_1_2_") + + def test_branch_operator_alias_preserves_case_sensitive_identity(self): + from rag_pipeline.core.index_manager.manager import RAGIndexManager + + manager = object.__new__(RAGIndexManager) + manager.config = MagicMock(qdrant_collection_prefix="rag") + manager.config.qdrant_collection_prefix = "rag" + + lowercase = manager._get_branch_operator_alias( + "Workspace", "Project", "feature" + ) + uppercase = manager._get_branch_operator_alias( + "Workspace", "Project", "Feature" + ) + + assert lowercase != uppercase + assert lowercase == "rag_workspace__project__feature" + assert uppercase.startswith("rag_workspace__project__feature_") + + def test_readable_alias_publication_requires_immutable_generation_target(self): + from rag_pipeline.core.index_manager.manager import RAGIndexManager + + manager = object.__new__(RAGIndexManager) + + with pytest.raises( + ValueError, + match="require an immutable collection target", + ): + manager.index_repository( + repo_path="/tmp/repository", + workspace="workspace", + project="project", + branch="main", + commit="abc123", + publish_branch_alias=True, + ) + @patch("rag_pipeline.core.index_manager.manager.create_embedding_model") @patch("rag_pipeline.core.index_manager.manager.get_embedding_model_info") @patch("rag_pipeline.core.index_manager.manager.QdrantClient") diff --git a/python-ecosystem/rag-pipeline/tests/test_indexer.py b/python-ecosystem/rag-pipeline/tests/test_indexer.py index f549eb8a..3918d0e9 100644 --- a/python-ecosystem/rag-pipeline/tests/test_indexer.py +++ b/python-ecosystem/rag-pipeline/tests/test_indexer.py @@ -475,7 +475,7 @@ def test_rejected_vector_point_is_skipped_and_valid_index_is_published(self): assert result.chunk_count == 1 assert result.skipped_chunk_count == 1 - coll_mgr.atomic_alias_swap.assert_called_once() + coll_mgr.atomic_assign_aliases.assert_called_once_with({"alias1": "pending"}) stats_mgr.store_metadata.assert_called_once() def test_repository_architecture_is_streamed_and_indexed_as_context(self, tmp_path): @@ -691,28 +691,30 @@ def test_normal_swap(self): config = _mock_config() coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - coll_mgr.collection_exists.return_value = False - coll_mgr.alias_exists.return_value = True - coll_mgr.resolve_alias.return_value = "active" + coll_mgr.read_alias_targets.return_value = {"alias1": "active"} indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - old_target = indexer._perform_atomic_swap("alias1", "pending", old_alias_exists=True) + old_targets = indexer._perform_atomic_swap( + "alias1", "pending", ["alias1"] + ) - coll_mgr.atomic_alias_swap.assert_called_once() - assert old_target == "active" + coll_mgr.atomic_assign_aliases.assert_called_once_with({"alias1": "pending"}) + assert old_targets == {"alias1": "active"} coll_mgr.delete_collection.assert_not_called() def test_first_activation_has_no_rollback_target(self): config = _mock_config() coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - coll_mgr.resolve_alias.return_value = None + coll_mgr.read_alias_targets.return_value = {"alias1": None} indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - old_target = indexer._perform_atomic_swap("alias1", "pending", old_alias_exists=False) + old_targets = indexer._perform_atomic_swap( + "alias1", "pending", ["alias1"] + ) - assert old_target is None - coll_mgr.atomic_alias_swap.assert_called_once_with("alias1", "pending", False) + assert old_targets == {"alias1": None} + coll_mgr.atomic_assign_aliases.assert_called_once_with({"alias1": "pending"}) def test_metadata_failure_rolls_back_before_pending_collection_is_deleted(self): config = _mock_config() @@ -725,6 +727,7 @@ def test_metadata_failure_rolls_back_before_pending_collection_is_deleted(self): coll_mgr.create_pending_collection.return_value = "pending" coll_mgr.alias_exists.return_value = True coll_mgr.resolve_alias.return_value = "active" + coll_mgr.read_alias_targets.return_value = {"alias1": "active"} point_ops.client.get_collection.return_value = SimpleNamespace(points_count=1) stats_mgr.store_metadata.side_effect = RuntimeError("metadata unavailable") @@ -732,9 +735,9 @@ def test_metadata_failure_rolls_back_before_pending_collection_is_deleted(self): with pytest.raises(RuntimeError, match="metadata unavailable"): indexer.index_repository("/repo", "ws", "proj", "main", "abc123", "alias1") - assert coll_mgr.atomic_alias_swap.call_args_list == [ - call("alias1", "pending", True), - call("alias1", "active", True), + assert coll_mgr.atomic_assign_aliases.call_args_list == [ + call({"alias1": "pending"}), + call({"alias1": "active"}), ] coll_mgr.delete_collection.assert_called_with("pending") diff --git a/python-ecosystem/rag-pipeline/tests/test_loader_extended.py b/python-ecosystem/rag-pipeline/tests/test_loader_extended.py index 9dcf05d1..3574e114 100644 --- a/python-ecosystem/rag-pipeline/tests/test_loader_extended.py +++ b/python-ecosystem/rag-pipeline/tests/test_loader_extended.py @@ -301,7 +301,10 @@ def test_skips_generated_assets(self, loader, tmp_path): def test_read_error_skipped(self, loader, tmp_path): (tmp_path / "a.py").write_text("code") - with patch("pathlib.Path.read_text", side_effect=PermissionError("no")): + with patch( + "rag_pipeline.core.loader.read_repository_file_bytes", + side_effect=PermissionError("no"), + ): docs = loader.load_specific_files( [Path("a.py")], tmp_path, "ws", "proj", "main", "abc" ) diff --git a/python-ecosystem/rag-pipeline/tests/test_pr_overlay_manifest.py b/python-ecosystem/rag-pipeline/tests/test_pr_overlay_manifest.py new file mode 100644 index 00000000..d91ae1cf --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_pr_overlay_manifest.py @@ -0,0 +1,274 @@ +from types import SimpleNamespace + +import pytest +from llama_index.core.schema import TextNode +from qdrant_client import QdrantClient +from qdrant_client.models import Distance, VectorParams + +from rag_pipeline.core.index_manager.indexer import FileOperations +from rag_pipeline.core.index_manager.point_operations import PointOperations +from rag_pipeline.core.pr_overlay_manifest import read_pr_overlay_generation +from rag_pipeline.core.repository_overlay import ( + IncrementalIndexPreconditionError, +) + + +SOURCE_REVISION = "a" * 40 +BASE_REVISION = "b" * 40 +BASE_MANIFEST = "c" * 64 +GENERATION_FINGERPRINT = "sha256:" + "d" * 64 +OVERLAY_REPRESENTATION = "sha256:" + "e" * 64 +NEXT_SOURCE_REVISION = "1" * 40 +NEXT_GENERATION_FINGERPRINT = "sha256:" + "2" * 64 + + +def _operations(): + client = QdrantClient(":memory:") + client.create_collection( + collection_name="overlay", + vectors_config=VectorParams(size=2, distance=Distance.COSINE), + ) + embed_model = SimpleNamespace( + get_text_embedding_batch=lambda texts: [ + [0.1, 0.2] for _ in texts + ], + ) + point_operations = PointOperations( + client, + embed_model, + embedding_dim=2, + ) + file_operations = FileOperations( + client, + point_operations, + SimpleNamespace(), + SimpleNamespace(), + SimpleNamespace(), + SimpleNamespace(), + ) + return client, file_operations + + +def _node( + text="final class Service {}", + *, + source_revision=SOURCE_REVISION, + generation_fingerprint=GENERATION_FINGERPRINT, +): + return TextNode( + text=text, + metadata={ + "workspace": "workspace", + "project": "project", + "branch": "main", + "path": "app/code/Service.php", + "pr": True, + "pr_number": 42, + "pr_branch": "main", + "pr_source_revision": source_revision, + "pr_base_revision": BASE_REVISION, + "pr_base_generation_manifest_sha256": BASE_MANIFEST, + "pr_generation_fingerprint": generation_fingerprint, + "pr_overlay_representation_fingerprint": ( + OVERLAY_REPRESENTATION + ), + }, + ) + + +def _read( + client, + manifest_sha256=None, + *, + source_revision=SOURCE_REVISION, + generation_fingerprint=GENERATION_FINGERPRINT, +): + return read_pr_overlay_generation( + client, + "overlay", + workspace="workspace", + project="project", + pr_number=42, + branch="main", + base_branch="main", + source_revision=source_revision, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_MANIFEST, + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=OVERLAY_REPRESENTATION, + expected_manifest_sha256=manifest_sha256, + ) + + +def test_pr_overlay_seal_binds_exact_persisted_membership(): + client, operations = _operations() + + count, receipt = operations.replace_pr_overlay_generation( + [_node()], + [], + "overlay", + "workspace", + "project", + "__pr__/42/main", + pr_number=42, + branch="main", + base_branch="main", + source_revision=SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_MANIFEST, + generation_fingerprint=GENERATION_FINGERPRINT, + overlay_representation_fingerprint=OVERLAY_REPRESENTATION, + identity_metadata={ + "index_representation_fingerprint": "sha256:" + "f" * 64, + }, + ) + + assert count == 1 + assert receipt["overlay_generation_member_count"] == 1 + assert len(receipt["overlay_generation_manifest_sha256"]) == 64 + assert _read( + client, + receipt["overlay_generation_manifest_sha256"], + ) == receipt + + +def test_pr_overlay_rejects_tampered_member_with_same_count(): + client, operations = _operations() + _, receipt = operations.replace_pr_overlay_generation( + [_node()], + [], + "overlay", + "workspace", + "project", + "__pr__/42/main", + pr_number=42, + branch="main", + base_branch="main", + source_revision=SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_MANIFEST, + generation_fingerprint=GENERATION_FINGERPRINT, + overlay_representation_fingerprint=OVERLAY_REPRESENTATION, + identity_metadata={}, + ) + points, _ = client.scroll( + collection_name="overlay", + limit=10, + with_payload=True, + with_vectors=False, + ) + member = next( + point + for point in points + if not (point.payload or {}).get("pr_overlay_generation_manifest") + ) + client.set_payload( + collection_name="overlay", + payload={"text": "tampered"}, + points=[member.id], + ) + + with pytest.raises( + IncrementalIndexPreconditionError, + match="member integrity", + ): + _read(client, receipt["overlay_generation_manifest_sha256"]) + + +def test_pr_overlay_without_unique_manifest_is_not_reusable(): + client, operations = _operations() + chunk_data = operations.point_ops.prepare_chunks_for_embedding( + [_node()], + "workspace", + "project", + "__pr__/42/main", + ) + points = operations.point_ops.embed_and_create_points(chunk_data) + operations.point_ops.upsert_points("overlay", points) + operations.point_ops._seal_persisted_point_digests("overlay", points) + + with pytest.raises( + IncrementalIndexPreconditionError, + match="manifest is missing", + ): + _read(client) + + +def test_failed_new_generation_never_overwrites_or_hides_prior_lease(): + client, operations = _operations() + _, prior_receipt = operations.replace_pr_overlay_generation( + [_node()], + [], + "overlay", + "workspace", + "project", + f"__pr__/42/main/{GENERATION_FINGERPRINT}", + pr_number=42, + branch="main", + base_branch="main", + source_revision=SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_MANIFEST, + generation_fingerprint=GENERATION_FINGERPRINT, + overlay_representation_fingerprint=OVERLAY_REPRESENTATION, + identity_metadata={}, + ) + prior_points, _ = client.scroll( + collection_name="overlay", + limit=20, + with_payload=True, + with_vectors=True, + ) + + original_upsert = operations.point_ops.upsert_points + observed_prior_receipts = [] + + def fail_new_manifest(collection_name, points): + if any( + (point.payload or {}).get("pr_overlay_generation_manifest") + for point in points + ): + # The new generation's members are already persisted here. An + # exact reader of the prior lease must still see only prior IDs. + observed_prior_receipts.append( + _read( + client, + prior_receipt[ + "overlay_generation_manifest_sha256" + ], + ) + ) + return 0, len(points) + return original_upsert(collection_name, points) + + operations.point_ops.upsert_points = fail_new_manifest + with pytest.raises(RuntimeError, match="manifest write was incomplete"): + operations.replace_pr_overlay_generation( + [ + _node( + "final class ServiceV2 {}", + source_revision=NEXT_SOURCE_REVISION, + generation_fingerprint=NEXT_GENERATION_FINGERPRINT, + ), + ], + prior_points, + "overlay", + "workspace", + "project", + f"__pr__/42/main/{NEXT_GENERATION_FINGERPRINT}", + pr_number=42, + branch="main", + base_branch="main", + source_revision=NEXT_SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_MANIFEST, + generation_fingerprint=NEXT_GENERATION_FINGERPRINT, + overlay_representation_fingerprint=OVERLAY_REPRESENTATION, + identity_metadata={}, + ) + + assert observed_prior_receipts == [prior_receipt] + assert _read( + client, + prior_receipt["overlay_generation_manifest_sha256"], + ) == prior_receipt diff --git a/python-ecosystem/rag-pipeline/tests/test_revision_bound_queries.py b/python-ecosystem/rag-pipeline/tests/test_revision_bound_queries.py new file mode 100644 index 00000000..60f0c65f --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_revision_bound_queries.py @@ -0,0 +1,597 @@ +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from fastapi import HTTPException + +from rag_pipeline.api.models import ( + DeterministicContextRequest, + PRContextRequest, + QueryRequest, +) +from rag_pipeline.api.routers.query import ( + _query_pr_indexed_data, + get_deterministic_context, + get_pr_context, + semantic_search, +) +from rag_pipeline.core.repository_overlay import ( + IncrementalIndexPreconditionError, +) +from rag_pipeline.services.base import RAGQueryBase +from rag_pipeline.services.pr_context import PRContextMixin +from rag_pipeline.services.semantic_search import SemanticSearchMixin +from rag_pipeline.services.deterministic_context import DeterministicContextMixin + + +BASE_REVISION = "a" * 40 +SOURCE_REVISION = "b" * 40 +BASE_GENERATION = "c" * 64 +PR_GENERATION = "sha256:" + "d" * 64 +PR_OVERLAY_MANIFEST = "e" * 64 +OVERLAY_REPRESENTATION = "sha256:" + "f" * 64 + + +def _overlay_receipt(manifest=PR_OVERLAY_MANIFEST): + return { + "overlay_generation_member_count": 1, + "overlay_generation_members_sha256": "1" * 64, + "overlay_generation_manifest_sha256": manifest, + } + + +def _manager(): + manager = MagicMock() + manager._get_project_collection_name.return_value = "rag_ws__project" + manager._collection_manager.collection_exists.return_value = True + manager._collection_manager.resolve_collection_target.return_value = ( + "rag_ws__project_active" + ) + manager.pr_overlay_representation_fingerprint = OVERLAY_REPRESENTATION + return manager + + +def test_semantic_query_detects_generation_swap_during_request(): + manager = _manager() + manager.get_revision_preflight.side_effect = [ + {"generation_manifest_sha256": BASE_GENERATION}, + {"generation_manifest_sha256": "e" * 64}, + ] + service = MagicMock() + service.semantic_search.return_value = [] + request = QueryRequest( + query="dependency lookup", + workspace="ws", + project="project", + branch="main", + repository_revision=BASE_REVISION, + repository_generation_manifest_sha256=BASE_GENERATION, + ) + + with patch( + "rag_pipeline.api.routers.query._get_singletons", + return_value=(manager, service), + ): + with pytest.raises(HTTPException) as exception: + semantic_search(request) + + assert exception.value.status_code == 409 + assert "generation changed" in exception.value.detail + service.semantic_search.assert_called_once_with( + query="dependency lookup", + workspace="ws", + project="project", + branch="main", + top_k=10, + filter_language=None, + expected_revision=BASE_REVISION, + collection_target="rag_ws__project_active", + ) + + +def test_pr_context_detects_base_generation_swap_after_retrieval(): + manager = _manager() + manager.get_revision_preflight.side_effect = [ + {"generation_manifest_sha256": BASE_GENERATION}, + {"generation_manifest_sha256": "e" * 64}, + ] + service = MagicMock() + service._collection_or_alias_exists.return_value = True + service.get_context_for_pr.return_value = { + "relevant_code": [], + "related_files": [], + "changed_files": ["src/Foo.php"], + } + request = PRContextRequest( + workspace="ws", + project="project", + branch="main", + base_branch="main", + changed_files=["src/Foo.php"], + pr_number=42, + source_revision=SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_GENERATION, + pr_generation_fingerprint=PR_GENERATION, + pr_overlay_generation_manifest_sha256=PR_OVERLAY_MANIFEST, + ) + + with ( + patch( + "rag_pipeline.api.routers.query._get_singletons", + return_value=(manager, service), + ), + patch( + "rag_pipeline.api.routers.query._query_pr_indexed_data", + return_value=[], + ) as query_overlay, + patch( + "rag_pipeline.api.routers.query.read_pr_overlay_generation", + return_value=_overlay_receipt(), + ), + ): + with pytest.raises(HTTPException) as exception: + get_pr_context(request) + + assert exception.value.status_code == 409 + assert "generation changed" in exception.value.detail + assert query_overlay.call_args.kwargs["collection_target"] == ( + "rag_ws__project_active" + ) + assert service.get_context_for_pr.call_args.kwargs[ + "collection_target" + ] == "rag_ws__project_active" + + +@pytest.mark.parametrize( + "probe_error", + [None, RuntimeError("qdrant unavailable")], + ids=["missing-collection", "backend-error"], +) +def test_revision_bound_pr_context_probe_failure_returns_http_conflict( + probe_error, +): + manager = _manager() + manager.get_revision_preflight.return_value = { + "generation_manifest_sha256": BASE_GENERATION, + } + + class ProbeFailingService(PRContextMixin, RAGQueryBase): + pass + + service = object.__new__(ProbeFailingService) + service.qdrant_client = MagicMock() + if probe_error is None: + service.qdrant_client.get_collections.return_value.collections = [] + service.qdrant_client.get_aliases.return_value.aliases = [] + else: + service.qdrant_client.get_collections.side_effect = probe_error + request = PRContextRequest( + workspace="ws", + project="project", + branch="main", + base_branch="main", + changed_files=["src/Foo.php"], + pr_number=42, + source_revision=SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_GENERATION, + pr_generation_fingerprint=PR_GENERATION, + pr_overlay_generation_manifest_sha256=PR_OVERLAY_MANIFEST, + ) + + with ( + patch( + "rag_pipeline.api.routers.query._get_singletons", + return_value=(manager, service), + ), + patch( + "rag_pipeline.api.routers.query._query_pr_indexed_data", + return_value=[], + ), + patch( + "rag_pipeline.api.routers.query.read_pr_overlay_generation", + return_value=_overlay_receipt(), + ), + ): + with pytest.raises(HTTPException) as exception: + get_pr_context(request) + + assert exception.value.status_code == 409 + assert ( + exception.value.detail + == "revision-bound PR-context collection is unavailable" + ) + manager.get_revision_preflight.assert_called_once() + + +def test_pr_overlay_query_filters_and_validates_exact_generation(): + manager = _manager() + point = SimpleNamespace( + payload={ + "path": "src/Foo.php", + "text": "