diff --git a/reviewer-panel-consensus-monitor/README.md b/reviewer-panel-consensus-monitor/README.md new file mode 100644 index 0000000..33d6bdc --- /dev/null +++ b/reviewer-panel-consensus-monitor/README.md @@ -0,0 +1,58 @@ +# Reviewer Panel Consensus Monitor + +This module adds a focused Scientific Bounty System slice for third-party reviewer panel quality control. It helps sponsors and platform arbitrators decide whether a challenge submission is ready for phase advancement or reward action. + +It evaluates: + +- declared reviewer conflicts of interest +- independent reviewer coverage and expertise tags +- locked submission manifests and artifact hashes +- reviewer score variance against the rubric threshold +- missing reviewer feedback +- missing decision explanations or response deadlines + +The output is an arbitration-ready packet with a deterministic digest, required actions, reviewer counts, the review phase, and a clear panel decision. + +## Usage + +```bash +node reviewer-panel-consensus-monitor/demo.js +``` + +```js +const { evaluateReviewerPanel } = require("./index.js"); + +const result = evaluateReviewerPanel({ + challengeTitle: "Climate Materials Prize", + phase: "final award review", + submission: { + id: "sub-2026-088", + teamId: "team-carbon", + artifactHashes: ["sha256:model", "sha256:dataset", "sha256:report"], + manifestLockedAt: "2026-05-18T16:40:00Z" + }, + rubric: { + version: "v3.0", + criteria: ["scientific rigor", "reproducibility", "field impact"], + decisionThreshold: 82, + maxScoreVariance: 12 + }, + reviewers: [], + decision: {} +}); +``` + +## Verification + +```bash +node reviewer-panel-consensus-monitor/test.js +node reviewer-panel-consensus-monitor/demo.js +``` + +The tests cover a blocked panel with conflicts and missing review controls, an approved panel with independent consensus, and arbitration-ready evidence/remediation fields for every finding. + +## Demo + +Short demo artifact: `demo.gif` + +The demo uses synthetic data only. It does not call external services, use credentials, or move funds. diff --git a/reviewer-panel-consensus-monitor/demo.gif b/reviewer-panel-consensus-monitor/demo.gif new file mode 100644 index 0000000..ef3d7f4 Binary files /dev/null and b/reviewer-panel-consensus-monitor/demo.gif differ diff --git a/reviewer-panel-consensus-monitor/demo.js b/reviewer-panel-consensus-monitor/demo.js new file mode 100644 index 0000000..ac15600 --- /dev/null +++ b/reviewer-panel-consensus-monitor/demo.js @@ -0,0 +1,65 @@ +const { evaluateReviewerPanel } = require("./index.js"); + +const panel = { + challengeTitle: "Open Ocean Carbon Prize", + phase: "milestone award review", + submission: { + id: "sub-2026-112", + teamId: "team-kelp", + artifactHashes: ["sha256:field-data", "sha256:model-card", "sha256:analysis-notebook"], + manifestLockedAt: "2026-05-18T17:05:00Z" + }, + rubric: { + version: "v1.4", + criteria: ["scientific rigor", "reproducibility", "field impact"], + decisionThreshold: 80, + maxScoreVariance: 10 + }, + reviewers: [ + { + id: "rev-ocean-1", + type: "independent", + expertise: ["oceanography", "field-methods"], + conflicts: [], + scores: { "scientific rigor": 88, reproducibility: 84, "field impact": 91 }, + feedback: "Field protocol and model card support the milestone claim." + }, + { + id: "rev-stat-2", + type: "independent", + expertise: ["statistics", "carbon-accounting"], + conflicts: [], + scores: { "scientific rigor": 85, reproducibility: 86, "field impact": 89 }, + feedback: "Uncertainty intervals are reproducible from the locked notebook." + }, + { + id: "rev-sponsor-observer", + type: "sponsor", + expertise: ["deployment"], + conflicts: [], + scores: { "scientific rigor": 86, reproducibility: 83, "field impact": 90 }, + feedback: "Milestone deliverables match the published challenge terms." + } + ], + decision: { + proposedOutcome: "award", + explanation: "Independent reviewers reached a low-variance consensus above threshold.", + reviewerResponsesDueAt: "2026-05-24T12:00:00Z" + } +}; + +const result = evaluateReviewerPanel(panel); + +console.log("Reviewer panel consensus monitor demo"); +console.log("-------------------------------------"); +console.log(`Challenge: ${result.summary.challengeTitle}`); +console.log(`Phase: ${result.summary.phase}`); +console.log(`Panel decision: ${result.summary.panelDecision}`); +console.log(`Consensus quality: ${result.summary.consensusQuality}`); +console.log(`Average score: ${result.summary.averageScore}`); +console.log(`Findings: ${result.summary.findingCount}`); +console.log(`Digest: ${result.arbitrationPacket.digest}`); +console.log("Required actions:"); +for (const action of result.arbitrationPacket.requiredActions) { + console.log(`- ${action}`); +} diff --git a/reviewer-panel-consensus-monitor/index.js b/reviewer-panel-consensus-monitor/index.js new file mode 100644 index 0000000..c3903fd --- /dev/null +++ b/reviewer-panel-consensus-monitor/index.js @@ -0,0 +1,294 @@ +const crypto = require("node:crypto"); + +function evaluateReviewerPanel(panel) { + const normalized = normalizePanel(panel); + const findings = [ + ...findReviewerConflicts(normalized), + ...findIndependenceGaps(normalized), + ...findUnlockedSubmissionManifest(normalized), + ...findScoreVariance(normalized), + ...findFeedbackGaps(normalized), + ...findDecisionExplanationGaps(normalized) + ]; + + const severityCounts = countSeverities(findings); + const averageScore = scorePanelAverage(normalized); + const summary = { + challengeTitle: normalized.challengeTitle, + phase: normalized.phase, + findingCount: findings.length, + severityCounts, + averageScore, + consensusQuality: scoreConsensusQuality(findings, averageScore, normalized.rubric), + panelDecision: decidePanel(severityCounts, averageScore, normalized.rubric) + }; + + return { + summary, + findings, + arbitrationPacket: buildArbitrationPacket(normalized, summary, findings) + }; +} + +function normalizePanel(panel) { + if (!panel || typeof panel !== "object") { + throw new TypeError("evaluateReviewerPanel expects a panel object"); + } + + return { + challengeTitle: panel.challengeTitle || "Untitled scientific bounty challenge", + phase: panel.phase || "unlabeled review phase", + submission: panel.submission || {}, + rubric: { + version: panel.rubric?.version || "unversioned", + criteria: Array.isArray(panel.rubric?.criteria) ? panel.rubric.criteria : [], + decisionThreshold: Number(panel.rubric?.decisionThreshold || 0), + maxScoreVariance: Number(panel.rubric?.maxScoreVariance || 15) + }, + reviewers: Array.isArray(panel.reviewers) ? panel.reviewers : [], + decision: panel.decision || {} + }; +} + +function findReviewerConflicts(panel) { + const conflicted = panel.reviewers + .filter((reviewer) => safeArray(reviewer.conflicts).length > 0) + .map( + (reviewer) => + `${reviewer.id || "unknown reviewer"} has conflicts: ${safeArray(reviewer.conflicts).join(", ")}` + ); + + if (conflicted.length === 0) return []; + + return [ + finding({ + type: "reviewer_conflict", + severity: "critical", + title: "Reviewer panel includes declared conflicts of interest", + evidence: conflicted, + remediation: + "Replace conflicted reviewers or quarantine their scores before phase advancement or payout recommendation." + }) + ]; +} + +function findIndependenceGaps(panel) { + const independentReviewers = panel.reviewers.filter( + (reviewer) => reviewer.type === "independent" && safeArray(reviewer.conflicts).length === 0 + ); + const missingExpertise = panel.reviewers + .filter((reviewer) => safeArray(reviewer.expertise).length === 0) + .map((reviewer) => reviewer.id || "unknown reviewer"); + const evidence = []; + + if (independentReviewers.length < 2) { + evidence.push(`only ${independentReviewers.length} unconflicted independent reviewer(s) are assigned`); + } + if (missingExpertise.length > 0) { + evidence.push(`reviewers missing expertise tags: ${missingExpertise.join(", ")}`); + } + + if (evidence.length === 0) return []; + + return [ + finding({ + type: "independence_gap", + severity: "high", + title: "Reviewer panel lacks enough independent, expertise-tagged review capacity", + evidence, + remediation: + "Add at least two unconflicted independent reviewers with domain expertise before issuing an award decision." + }) + ]; +} + +function findUnlockedSubmissionManifest(panel) { + const hashes = safeArray(panel.submission.artifactHashes); + const evidence = []; + + if (hashes.length === 0) evidence.push("submission package does not list artifact hashes"); + if (!panel.submission.manifestLockedAt) { + evidence.push("submission manifest lock timestamp is missing"); + } + + if (evidence.length === 0) return []; + + return [ + finding({ + type: "unlocked_submission_manifest", + severity: "high", + title: "Submission evidence is not locked before review", + evidence, + remediation: + "Lock the submission manifest with artifact hashes before reviewers score the package." + }) + ]; +} + +function findScoreVariance(panel) { + const reviewerAverages = panel.reviewers.map((reviewer) => averageReviewerScore(reviewer, panel.rubric.criteria)); + const validAverages = reviewerAverages.filter(Number.isFinite); + + if (validAverages.length < 2) return []; + + const spread = Math.max(...validAverages) - Math.min(...validAverages); + if (spread <= panel.rubric.maxScoreVariance) return []; + + return [ + finding({ + type: "score_variance", + severity: "medium", + title: "Reviewer scores are too far apart for consensus", + evidence: [ + `score spread ${round(spread)} exceeds allowed variance ${panel.rubric.maxScoreVariance}`, + `reviewer averages: ${validAverages.map(round).join(", ")}` + ], + remediation: + "Trigger a calibration discussion or arbitration review before advancing the submission." + }) + ]; +} + +function findFeedbackGaps(panel) { + const reviewersWithoutFeedback = panel.reviewers + .filter((reviewer) => !String(reviewer.feedback || "").trim()) + .map((reviewer) => reviewer.id || "unknown reviewer"); + + if (reviewersWithoutFeedback.length === 0) return []; + + return [ + finding({ + type: "feedback_gap", + severity: "medium", + title: "Reviewer feedback is missing for one or more scores", + evidence: [`missing feedback from: ${reviewersWithoutFeedback.join(", ")}`], + remediation: + "Require reviewer-facing rationale so solvers and arbitrators can understand the score basis." + }) + ]; +} + +function findDecisionExplanationGaps(panel) { + const evidence = []; + if (!String(panel.decision.proposedOutcome || "").trim()) { + evidence.push("proposed outcome is missing"); + } + if (!String(panel.decision.explanation || "").trim()) { + evidence.push("decision explanation is missing"); + } + if (!String(panel.decision.reviewerResponsesDueAt || "").trim()) { + evidence.push("reviewer response deadline is missing"); + } + + if (evidence.length === 0) return []; + + return [ + finding({ + type: "decision_explanation_gap", + severity: "medium", + title: "Panel decision lacks arbitration-ready explanation metadata", + evidence, + remediation: + "Record the proposed outcome, reasoning, and reviewer response deadline before sponsor payout action." + }) + ]; +} + +function buildArbitrationPacket(panel, summary, findings) { + return { + challengeTitle: panel.challengeTitle, + submissionId: panel.submission.id || "unknown-submission", + teamId: panel.submission.teamId || "unknown-team", + phase: panel.phase, + rubricVersion: panel.rubric.version, + digest: digestPanel(panel, findings), + reviewerCount: panel.reviewers.length, + independentReviewerCount: panel.reviewers.filter((reviewer) => reviewer.type === "independent").length, + requiredActions: + findings.length === 0 + ? ["Proceed with sponsor award or phase advancement."] + : findings.map((finding) => finding.remediation), + decisionSummary: `${summary.panelDecision} with ${summary.consensusQuality} consensus quality` + }; +} + +function scorePanelAverage(panel) { + const scores = panel.reviewers + .map((reviewer) => averageReviewerScore(reviewer, panel.rubric.criteria)) + .filter(Number.isFinite); + + if (scores.length === 0) return 0; + return round(scores.reduce((sum, score) => sum + score, 0) / scores.length); +} + +function averageReviewerScore(reviewer, criteria) { + const scores = criteria + .map((criterion) => Number(reviewer.scores?.[criterion])) + .filter(Number.isFinite); + + if (scores.length === 0) return NaN; + return scores.reduce((sum, score) => sum + score, 0) / scores.length; +} + +function scoreConsensusQuality(findings, averageScore, rubric) { + if (findings.some((finding) => finding.severity === "critical" || finding.severity === "high")) { + return "low"; + } + if (findings.length > 0 || averageScore < rubric.decisionThreshold) return "medium"; + return "high"; +} + +function decidePanel(severityCounts, averageScore, rubric) { + if ((severityCounts.critical || 0) > 0 || (severityCounts.high || 0) > 0) return "block"; + if ((severityCounts.medium || 0) > 0 || averageScore < rubric.decisionThreshold) { + return "needs-arbitration"; + } + return "approve"; +} + +function countSeverities(findings) { + return findings.reduce((counts, finding) => { + counts[finding.severity] = (counts[finding.severity] || 0) + 1; + return counts; + }, {}); +} + +function finding({ type, severity, title, evidence, remediation }) { + return { + id: `${type}:${crypto.createHash("sha1").update(evidence.join("|")).digest("hex").slice(0, 10)}`, + type, + severity, + title, + evidence, + remediation + }; +} + +function digestPanel(panel, findings) { + const payload = { + submission: panel.submission, + rubric: panel.rubric, + reviewers: panel.reviewers.map((reviewer) => ({ + id: reviewer.id, + type: reviewer.type, + expertise: reviewer.expertise, + conflicts: reviewer.conflicts, + scores: reviewer.scores + })), + findingTypes: findings.map((finding) => finding.type) + }; + + return `sha256:${crypto.createHash("sha256").update(JSON.stringify(payload)).digest("hex")}`; +} + +function safeArray(value) { + return Array.isArray(value) ? value : []; +} + +function round(value) { + return Math.round(value * 100) / 100; +} + +module.exports = { + evaluateReviewerPanel +}; diff --git a/reviewer-panel-consensus-monitor/requirement-map.md b/reviewer-panel-consensus-monitor/requirement-map.md new file mode 100644 index 0000000..1ab2f06 --- /dev/null +++ b/reviewer-panel-consensus-monitor/requirement-map.md @@ -0,0 +1,30 @@ +# Requirement Map: SCIBASE-AI/SCIBASE.AI#18 + +Issue #18 describes a scientific bounty marketplace with challenge posting, secure submissions, arbitration, and reward distribution. This contribution focuses on the arbitration and reviewer validation layer. + +## Challenge Posting Portal + +- Supports rubric version checks through `rubric.version`. +- Uses `rubric.criteria`, `decisionThreshold`, and `maxScoreVariance` to make evaluation criteria explicit before review. +- Preserves the review `phase` so multi-stage challenge decisions can be audited. + +## Submission Engine + +- Requires submission package evidence through `submission.artifactHashes`. +- Flags missing `manifestLockedAt` timestamps so reviewers cannot score mutable deliverables. +- Includes `submissionId` and `teamId` in the arbitration packet for standardized challenge submission records. + +## Arbitration & Reward Distribution + +- Detects reviewer conflicts and recommends replacing or quarantining conflicted scores. +- Requires at least two unconflicted independent reviewers with expertise tags. +- Computes reviewer average score and score spread against the rubric variance limit. +- Requires feedback and decision explanations before sponsor payout action. +- Emits `panelDecision` values of `approve`, `needs-arbitration`, or `block`. +- Produces a deterministic `sha256:` digest for the review packet. + +## IP And Payout Safety + +- Holds award/phase advancement when evidence is unlocked, conflicted, or under-explained. +- Produces required actions that sponsors or arbitrators can complete before reward distribution. +- Keeps the module dependency-free and synthetic-data-only; no payout action, credential access, or external service call is performed. diff --git a/reviewer-panel-consensus-monitor/test.js b/reviewer-panel-consensus-monitor/test.js new file mode 100644 index 0000000..2baa958 --- /dev/null +++ b/reviewer-panel-consensus-monitor/test.js @@ -0,0 +1,144 @@ +const assert = require("node:assert/strict"); +const { evaluateReviewerPanel } = require("./index.js"); + +function test(name, fn) { + try { + fn(); + console.log(`ok - ${name}`); + } catch (error) { + console.error(`not ok - ${name}`); + throw error; + } +} + +const riskyPanel = { + challengeTitle: "Open Biomarker Discovery Challenge", + phase: "prototype review", + submission: { + id: "sub-2026-014", + teamId: "team-alpha", + artifactHashes: ["sha256:raw-data", "sha256:notebook"], + manifestLockedAt: "" + }, + rubric: { + version: "v2.1", + criteria: ["scientific rigor", "reproducibility", "clinical relevance"], + decisionThreshold: 80, + maxScoreVariance: 14 + }, + reviewers: [ + { + id: "rev-001", + type: "sponsor", + expertise: ["clinical"], + conflicts: ["same_institution"], + scores: { "scientific rigor": 96, reproducibility: 94, "clinical relevance": 98 }, + feedback: "" + }, + { + id: "rev-002", + type: "internal", + expertise: [], + conflicts: [], + scores: { "scientific rigor": 44, reproducibility: 38, "clinical relevance": 51 }, + feedback: "Needs more evidence." + } + ], + decision: { + proposedOutcome: "advance", + explanation: "", + reviewerResponsesDueAt: "" + } +}; + +const healthyPanel = { + challengeTitle: "Climate Materials Prize", + phase: "final award review", + submission: { + id: "sub-2026-088", + teamId: "team-carbon", + artifactHashes: ["sha256:model", "sha256:dataset", "sha256:report"], + manifestLockedAt: "2026-05-18T16:40:00Z" + }, + rubric: { + version: "v3.0", + criteria: ["scientific rigor", "reproducibility", "field impact"], + decisionThreshold: 82, + maxScoreVariance: 12 + }, + reviewers: [ + { + id: "rev-independent-1", + type: "independent", + expertise: ["materials", "simulation"], + conflicts: [], + scores: { "scientific rigor": 87, reproducibility: 84, "field impact": 89 }, + feedback: "Evidence supports the award decision." + }, + { + id: "rev-independent-2", + type: "independent", + expertise: ["climate", "materials"], + conflicts: [], + scores: { "scientific rigor": 85, reproducibility: 86, "field impact": 88 }, + feedback: "Reproducibility packet is complete." + }, + { + id: "rev-sponsor-observer", + type: "sponsor", + expertise: ["deployment"], + conflicts: [], + scores: { "scientific rigor": 86, reproducibility: 83, "field impact": 90 }, + feedback: "Sponsor requirements are satisfied." + } + ], + decision: { + proposedOutcome: "award", + explanation: "Independent reviewers reached consensus above the award threshold.", + reviewerResponsesDueAt: "2026-05-22T12:00:00Z" + } +}; + +test("blocks reviewer panels with conflicts, weak independence, unlocked evidence, variance, and missing explanation", () => { + const result = evaluateReviewerPanel(riskyPanel); + const types = result.findings.map((finding) => finding.type); + + assert.equal(result.summary.panelDecision, "block"); + assert.ok(types.includes("reviewer_conflict")); + assert.ok(types.includes("independence_gap")); + assert.ok(types.includes("unlocked_submission_manifest")); + assert.ok(types.includes("score_variance")); + assert.ok(types.includes("feedback_gap")); + assert.ok(types.includes("decision_explanation_gap")); + assert.ok( + result.arbitrationPacket.requiredActions.some((action) => + action.toLowerCase().includes("replace conflicted reviewers") + ) + ); +}); + +test("approves independent panels with locked evidence, low variance, and explained consensus", () => { + const result = evaluateReviewerPanel(healthyPanel); + + assert.equal(result.summary.panelDecision, "approve"); + assert.equal(result.summary.findingCount, 0); + assert.equal(result.summary.consensusQuality, "high"); + assert.equal(result.summary.averageScore >= healthyPanel.rubric.decisionThreshold, true); + assert.deepEqual(result.findings, []); + assert.ok(result.arbitrationPacket.requiredActions.includes("Proceed with sponsor award or phase advancement.")); +}); + +test("emits arbitration-ready findings with evidence, remediation, and panel digest", () => { + const result = evaluateReviewerPanel(riskyPanel); + + for (const finding of result.findings) { + assert.ok(finding.id); + assert.ok(["critical", "high", "medium", "low"].includes(finding.severity)); + assert.ok(finding.evidence.length > 0); + assert.ok(finding.remediation.length > 10); + } + + assert.ok(result.arbitrationPacket.digest.startsWith("sha256:")); + assert.equal(result.arbitrationPacket.submissionId, "sub-2026-014"); + assert.equal(result.arbitrationPacket.phase, "prototype review"); +});