Split real-world benchmark report contracts

apps/elf-eval/src/bin/real_world_job_benchmark/external_adapter_detail_reports.rs

@@ -0,0 +1,100 @@
+use crate::{
+	AdapterCoverageStatus, Deserialize, ElfScenarioPosition, FollowUpInput,
+	ScenarioComparisonOutcome, Serialize,
+};
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub(crate) struct ExternalAdapterReport {
+	pub(crate) adapter_id: String,
+	pub(crate) project: String,
+	pub(crate) adapter_kind: String,
+	pub(crate) evidence_class: String,
+	pub(crate) docker_default: bool,
+	pub(crate) host_global_installs_required: bool,
+	pub(crate) overall_status: AdapterCoverageStatus,
+	pub(crate) setup: AdapterExecutionEvidence,
+	pub(crate) run: AdapterExecutionEvidence,
+	pub(crate) result: AdapterExecutionEvidence,
+	#[serde(default)]
+	pub(crate) capabilities: Vec<AdapterCapabilityCoverage>,
+	#[serde(default)]
+	pub(crate) suites: Vec<AdapterSuiteCoverage>,
+	#[serde(default)]
+	pub(crate) scenarios: Vec<AdapterScenarioJudgment>,
+	#[serde(default)]
+	pub(crate) evidence: Vec<AdapterEvidencePointer>,
+	#[serde(skip_serializing_if = "Option::is_none")]
+	pub(crate) execution_metadata: Option<AdapterExecutionMetadata>,
+	#[serde(default)]
+	pub(crate) notes: Vec<String>,
+	#[serde(skip_serializing_if = "Option::is_none")]
+	pub(crate) follow_up: Option<FollowUpInput>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub(crate) struct AdapterExecutionEvidence {
+	pub(crate) status: AdapterCoverageStatus,
+	pub(crate) evidence: String,
+	#[serde(skip_serializing_if = "Option::is_none")]
+	pub(crate) command: Option<String>,
+	#[serde(skip_serializing_if = "Option::is_none")]
+	pub(crate) artifact: Option<String>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub(crate) struct AdapterCapabilityCoverage {
+	pub(crate) capability: String,
+	pub(crate) status: AdapterCoverageStatus,
+	pub(crate) evidence: String,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub(crate) struct AdapterSuiteCoverage {
+	pub(crate) suite_id: String,
+	pub(crate) status: AdapterCoverageStatus,
+	pub(crate) evidence: String,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub(crate) struct AdapterScenarioJudgment {
+	pub(crate) scenario_id: String,
+	#[serde(skip_serializing_if = "Option::is_none")]
+	pub(crate) suite_id: Option<String>,
+	pub(crate) status: AdapterCoverageStatus,
+	pub(crate) elf_position: ElfScenarioPosition,
+	#[serde(skip_serializing_if = "Option::is_none")]
+	pub(crate) comparison_outcome: Option<ScenarioComparisonOutcome>,
+	pub(crate) evidence: String,
+	#[serde(skip_serializing_if = "Option::is_none")]
+	pub(crate) command: Option<String>,
+	#[serde(skip_serializing_if = "Option::is_none")]
+	pub(crate) artifact: Option<String>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub(crate) struct AdapterEvidencePointer {
+	pub(crate) kind: String,
+	#[serde(rename = "ref")]
+	pub(crate) reference: String,
+	pub(crate) status: AdapterCoverageStatus,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub(crate) struct AdapterExecutionMetadata {
+	#[serde(default)]
+	pub(crate) sources: Vec<AdapterSource>,
+	pub(crate) setup_path: String,
+	pub(crate) runtime_boundary: String,
+	pub(crate) resource_expectation: String,
+	#[serde(default)]
+	pub(crate) retry_guidance: Vec<String>,
+	#[serde(skip_serializing_if = "Option::is_none")]
+	pub(crate) research_depth: Option<String>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub(crate) struct AdapterSource {
+	pub(crate) label: String,
+	pub(crate) url: String,
+	pub(crate) evidence: String,
+}

apps/elf-eval/src/bin/real_world_job_benchmark/external_adapter_manifest_reports.rs

@@ -0,0 +1,34 @@
+use crate::{
+	Deserialize, Serialize,
+	external_adapter_reports::{ExternalAdapterReport, ExternalAdapterSummary},
+};
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct ExternalAdapterManifest {
+	pub(crate) schema: String,
+	pub(crate) manifest_id: String,
+	pub(crate) docker_isolation: ExternalDockerIsolation,
+	#[serde(default)]
+	pub(crate) adapters: Vec<ExternalAdapterReport>,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub(crate) struct ExternalAdapterSection {
+	pub(crate) schema: String,
+	pub(crate) manifest_id: String,
+	pub(crate) docker_isolation: ExternalDockerIsolation,
+	pub(crate) summary: ExternalAdapterSummary,
+	#[serde(default)]
+	pub(crate) adapters: Vec<ExternalAdapterReport>,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub(crate) struct ExternalDockerIsolation {
+	pub(crate) default: bool,
+	pub(crate) compose_file: String,
+	pub(crate) runner: String,
+	pub(crate) artifact_dir: String,
+	pub(crate) host_global_installs_required: bool,
+	#[serde(default)]
+	pub(crate) notes: Vec<String>,
+}

apps/elf-eval/src/bin/real_world_job_benchmark/external_adapter_misc_reports.rs

@@ -0,0 +1,27 @@
+use crate::{Deserialize, Serialize, TypedStatus};
+
+#[derive(Debug, Deserialize, Serialize)]
+pub(crate) struct AdapterReport {
+	pub(crate) adapter_id: String,
+	pub(crate) name: String,
+	pub(crate) behavior: String,
+	pub(crate) storage: TypedStatus,
+	pub(crate) runtime: TypedStatus,
+	pub(crate) notes: String,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub(crate) struct CaptureIntegrationReport {
+	#[serde(default)]
+	pub(crate) real: Vec<String>,
+	#[serde(default)]
+	pub(crate) fixture_backed: Vec<String>,
+	#[serde(default)]
+	pub(crate) mocked: Vec<String>,
+	#[serde(default)]
+	pub(crate) blocked: Vec<String>,
+	#[serde(default)]
+	pub(crate) not_encoded: Vec<String>,
+	#[serde(default)]
+	pub(crate) notes: Vec<String>,
+}

apps/elf-eval/src/bin/real_world_job_benchmark/external_adapter_reports.rs

@@ -1,209 +1,22 @@
-use crate::{
-	AdapterCoverageStatus, Deserialize, ElfScenarioPosition, FollowUpInput,
-	ScenarioComparisonOutcome, Serialize, TypedStatus,
+mod external_adapter_detail_reports;
+mod external_adapter_manifest_reports;
+mod external_adapter_misc_reports;
+mod external_adapter_summary_reports;
+
+pub(super) use self::{
+	external_adapter_detail_reports::{
+		AdapterScenarioJudgment, AdapterSource, AdapterSuiteCoverage, ExternalAdapterReport,
+	},
+	external_adapter_manifest_reports::{
+		ExternalAdapterManifest, ExternalAdapterSection, ExternalDockerIsolation,
+	},
+	external_adapter_misc_reports::{AdapterReport, CaptureIntegrationReport},
+	external_adapter_summary_reports::{
+		AdapterStatusCounts, ExternalAdapterSummary, ScenarioOutcomeCounts, ScenarioPositionCounts,
+	},
+};
+#[allow(unused_imports)]
+pub(super) use external_adapter_detail_reports::{
+	AdapterCapabilityCoverage, AdapterEvidencePointer, AdapterExecutionEvidence,
+	AdapterExecutionMetadata,
 };
-
-#[derive(Debug, Deserialize, Serialize)]
-pub(super) struct AdapterReport {
-	pub(super) adapter_id: String,
-	pub(super) name: String,
-	pub(super) behavior: String,
-	pub(super) storage: TypedStatus,
-	pub(super) runtime: TypedStatus,
-	pub(super) notes: String,
-}
-
-#[derive(Debug, Deserialize)]
-pub(super) struct ExternalAdapterManifest {
-	pub(super) schema: String,
-	pub(super) manifest_id: String,
-	pub(super) docker_isolation: ExternalDockerIsolation,
-	#[serde(default)]
-	pub(super) adapters: Vec<ExternalAdapterReport>,
-}
-
-#[derive(Clone, Debug, Default, Deserialize, Serialize)]
-pub(super) struct ExternalAdapterSection {
-	pub(super) schema: String,
-	pub(super) manifest_id: String,
-	pub(super) docker_isolation: ExternalDockerIsolation,
-	pub(super) summary: ExternalAdapterSummary,
-	#[serde(default)]
-	pub(super) adapters: Vec<ExternalAdapterReport>,
-}
-
-#[derive(Clone, Debug, Default, Deserialize, Serialize)]
-pub(super) struct ExternalDockerIsolation {
-	pub(super) default: bool,
-	pub(super) compose_file: String,
-	pub(super) runner: String,
-	pub(super) artifact_dir: String,
-	pub(super) host_global_installs_required: bool,
-	#[serde(default)]
-	pub(super) notes: Vec<String>,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub(super) struct ExternalAdapterReport {
-	pub(super) adapter_id: String,
-	pub(super) project: String,
-	pub(super) adapter_kind: String,
-	pub(super) evidence_class: String,
-	pub(super) docker_default: bool,
-	pub(super) host_global_installs_required: bool,
-	pub(super) overall_status: AdapterCoverageStatus,
-	pub(super) setup: AdapterExecutionEvidence,
-	pub(super) run: AdapterExecutionEvidence,
-	pub(super) result: AdapterExecutionEvidence,
-	#[serde(default)]
-	pub(super) capabilities: Vec<AdapterCapabilityCoverage>,
-	#[serde(default)]
-	pub(super) suites: Vec<AdapterSuiteCoverage>,
-	#[serde(default)]
-	pub(super) scenarios: Vec<AdapterScenarioJudgment>,
-	#[serde(default)]
-	pub(super) evidence: Vec<AdapterEvidencePointer>,
-	#[serde(skip_serializing_if = "Option::is_none")]
-	pub(super) execution_metadata: Option<AdapterExecutionMetadata>,
-	#[serde(default)]
-	pub(super) notes: Vec<String>,
-	#[serde(skip_serializing_if = "Option::is_none")]
-	pub(super) follow_up: Option<FollowUpInput>,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub(super) struct AdapterExecutionEvidence {
-	pub(super) status: AdapterCoverageStatus,
-	pub(super) evidence: String,
-	#[serde(skip_serializing_if = "Option::is_none")]
-	pub(super) command: Option<String>,
-	#[serde(skip_serializing_if = "Option::is_none")]
-	pub(super) artifact: Option<String>,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub(super) struct AdapterCapabilityCoverage {
-	pub(super) capability: String,
-	pub(super) status: AdapterCoverageStatus,
-	pub(super) evidence: String,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub(super) struct AdapterSuiteCoverage {
-	pub(super) suite_id: String,
-	pub(super) status: AdapterCoverageStatus,
-	pub(super) evidence: String,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub(super) struct AdapterScenarioJudgment {
-	pub(super) scenario_id: String,
-	#[serde(skip_serializing_if = "Option::is_none")]
-	pub(super) suite_id: Option<String>,
-	pub(super) status: AdapterCoverageStatus,
-	pub(super) elf_position: ElfScenarioPosition,
-	#[serde(skip_serializing_if = "Option::is_none")]
-	pub(super) comparison_outcome: Option<ScenarioComparisonOutcome>,
-	pub(super) evidence: String,
-	#[serde(skip_serializing_if = "Option::is_none")]
-	pub(super) command: Option<String>,
-	#[serde(skip_serializing_if = "Option::is_none")]
-	pub(super) artifact: Option<String>,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub(super) struct AdapterEvidencePointer {
-	pub(super) kind: String,
-	#[serde(rename = "ref")]
-	pub(super) reference: String,
-	pub(super) status: AdapterCoverageStatus,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub(super) struct AdapterExecutionMetadata {
-	#[serde(default)]
-	pub(super) sources: Vec<AdapterSource>,
-	pub(super) setup_path: String,
-	pub(super) runtime_boundary: String,
-	pub(super) resource_expectation: String,
-	#[serde(default)]
-	pub(super) retry_guidance: Vec<String>,
-	#[serde(skip_serializing_if = "Option::is_none")]
-	pub(super) research_depth: Option<String>,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub(super) struct AdapterSource {
-	pub(super) label: String,
-	pub(super) url: String,
-	pub(super) evidence: String,
-}
-
-#[derive(Clone, Debug, Default, Deserialize, Serialize)]
-pub(super) struct ExternalAdapterSummary {
-	pub(super) adapter_count: usize,
-	pub(super) external_project_count: usize,
-	pub(super) docker_default_count: usize,
-	pub(super) host_global_install_required_count: usize,
-	pub(super) fixture_backed_count: usize,
-	pub(super) live_baseline_only_count: usize,
-	pub(super) live_real_world_count: usize,
-	#[serde(default)]
-	pub(super) research_gate_count: usize,
-	pub(super) overall_status_counts: AdapterStatusCounts,
-	pub(super) capability_status_counts: AdapterStatusCounts,
-	pub(super) suite_status_counts: AdapterStatusCounts,
-	#[serde(default)]
-	pub(super) scenario_status_counts: AdapterStatusCounts,
-	#[serde(default)]
-	pub(super) scenario_position_counts: ScenarioPositionCounts,
-	#[serde(default)]
-	pub(super) scenario_outcome_counts: ScenarioOutcomeCounts,
-}
-
-#[derive(Clone, Debug, Default, Deserialize, Serialize)]
-pub(super) struct AdapterStatusCounts {
-	pub(super) real: usize,
-	pub(super) mocked: usize,
-	pub(super) unsupported: usize,
-	pub(super) blocked: usize,
-	pub(super) incomplete: usize,
-	pub(super) wrong_result: usize,
-	pub(super) lifecycle_fail: usize,
-	pub(super) pass: usize,
-	pub(super) not_encoded: usize,
-}
-
-#[derive(Clone, Debug, Default, Deserialize, Serialize)]
-pub(super) struct ScenarioPositionCounts {
-	pub(super) wins: usize,
-	pub(super) ties: usize,
-	pub(super) loses: usize,
-	pub(super) untested: usize,
-}
-
-#[derive(Clone, Debug, Default, Deserialize, Serialize)]
-pub(super) struct ScenarioOutcomeCounts {
-	pub(super) win: usize,
-	pub(super) tie: usize,
-	pub(super) loss: usize,
-	pub(super) not_tested: usize,
-	pub(super) blocked: usize,
-	pub(super) non_goal: usize,
-}
-
-#[derive(Clone, Debug, Default, Deserialize, Serialize)]
-pub(super) struct CaptureIntegrationReport {
-	#[serde(default)]
-	pub(super) real: Vec<String>,
-	#[serde(default)]
-	pub(super) fixture_backed: Vec<String>,
-	#[serde(default)]
-	pub(super) mocked: Vec<String>,
-	#[serde(default)]
-	pub(super) blocked: Vec<String>,
-	#[serde(default)]
-	pub(super) not_encoded: Vec<String>,
-	#[serde(default)]
-	pub(super) notes: Vec<String>,
-}