From 9715e2eeb6479c83eb7fd35acc97ebac26db8988 Mon Sep 17 00:00:00 2001 From: Ethan Jackson Date: Wed, 15 Jul 2026 20:33:36 -0400 Subject: [PATCH 1/3] test(agentic): add characterization tests before generalization refactor Pin the behaviour the domain-agnostic refactor must preserve: - AdaptiveSkillStore save/load/render/backup contract and the confirmation_threshold store parameter (shared-lib test with a minimal concrete state). - WtiStrategyState.build_markdown() renders the committed wti-strategy-trained artifact byte-identically (snapshot), plus the default oil skill name/title. - The five strategy mutation tools' evidence governance: observation/ hypothesis lifecycle, the graduation threshold guard, and the narrative-update rationale requirement. Co-Authored-By: Claude Fable 5 --- .../methods/agentic/test_adaptive_skill.py | 120 ++++++++++++++++ .../test_skill_state.py | 44 ++++++ .../test_skill_tools.py | 130 ++++++++++++++++++ 3 files changed, 294 insertions(+) create mode 100644 aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_adaptive_skill.py create mode 100644 implementations/tests/energy_oil_forecasting/test_skill_state.py create mode 100644 implementations/tests/energy_oil_forecasting/test_skill_tools.py diff --git a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_adaptive_skill.py b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_adaptive_skill.py new file mode 100644 index 00000000..3cc6fd25 --- /dev/null +++ b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_adaptive_skill.py @@ -0,0 +1,120 @@ +"""Characterization tests for the generic adaptive-skill persistence layer. + +Exercises :class:`~aieng.forecasting.methods.agentic.adaptive_skill.AdaptiveSkillStore` +against a minimal concrete :class:`~aieng.forecasting.methods.agentic.adaptive_skill.AdaptiveSkillState` +subclass so the save / load / render / backup contract is pinned before the +strategy-state machinery is promoted and generalised. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml +from aieng.forecasting.methods.agentic.adaptive_skill import ( + AdaptiveSkillState, + AdaptiveSkillStore, +) + + +class _DummyState(AdaptiveSkillState): + """Minimal concrete state used only for testing the store.""" + + approach: str + notes: list[str] = [] + + def build_markdown(self, skill_name: str | None = None) -> str: + name = skill_name or "dummy-skill" + lines = [ + "---", + f"name: {name}", + "---", + "", + "# Dummy", + "", + self.approach.strip(), + "", + ] + lines += [f"- {n}" for n in self.notes] + return "\n".join(lines) + + +def _make_store(tmp_path: Path, *, confirmation_threshold: int = 3) -> AdaptiveSkillStore[_DummyState]: + return AdaptiveSkillStore( + skill_dir=tmp_path, + state_type=_DummyState, + confirmation_threshold=confirmation_threshold, + ) + + +def test_rejects_missing_directory(tmp_path: Path) -> None: + """The store must refuse a non-existent skill directory up front.""" + missing = tmp_path / "does-not-exist" + with pytest.raises(ValueError, match="does not exist"): + AdaptiveSkillStore(skill_dir=missing, state_type=_DummyState) + + +def test_load_before_seed_raises(tmp_path: Path) -> None: + """Loading before any state is seeded raises a helpful FileNotFoundError.""" + store = _make_store(tmp_path) + with pytest.raises(FileNotFoundError, match="skill_state.yaml not found"): + store.load() + + +def test_save_writes_yaml_and_markdown(tmp_path: Path) -> None: + """``save`` writes both the YAML source of truth and rendered SKILL.md.""" + store = _make_store(tmp_path) + state = _DummyState(approach="Trust the trend.", notes=["a", "b"]) + + msg = store.save(state) + + assert store.state_path.exists() + assert store.skill_md_path.exists() + assert "SKILL.md re-rendered" in msg + + # SKILL.md is exactly what build_markdown renders with the dir name. + assert store.skill_md_path.read_text(encoding="utf-8") == state.build_markdown(skill_name=tmp_path.name) + + +def test_round_trips_state(tmp_path: Path) -> None: + """State survives a save/load cycle byte-for-byte in its fields.""" + store = _make_store(tmp_path) + state = _DummyState(approach="Weight news at long horizons.", notes=["x"]) + store.save(state) + + loaded = store.load() + + assert loaded.approach == state.approach + assert loaded.notes == state.notes + assert loaded.schema_version == state.schema_version + + +def test_second_save_backs_up_previous_state(tmp_path: Path) -> None: + """The prior YAML is copied into ``.history`` before being overwritten.""" + store = _make_store(tmp_path) + store.save(_DummyState(approach="first", notes=[])) + + # No backup after the very first save (nothing to back up beforehand). + assert not store.history_dir.exists() + + store.save(_DummyState(approach="second", notes=[])) + + backups = list(store.history_dir.glob("skill_state_*.yaml")) + assert len(backups) == 1 + backed_up = yaml.safe_load(backups[0].read_text(encoding="utf-8")) + assert backed_up["approach"] == "first" + + # The live file reflects the latest save. + assert store.load().approach == "second" + + +def test_confirmation_threshold_is_a_store_parameter(tmp_path: Path) -> None: + """The evidence bar lives on the store, not in serialised state.""" + store = _make_store(tmp_path, confirmation_threshold=5) + store.save(_DummyState(approach="a", notes=[])) + + assert store.confirmation_threshold == 5 + # It must not leak into the persisted state. + raw = yaml.safe_load(store.state_path.read_text(encoding="utf-8")) + assert "confirmation_threshold" not in raw diff --git a/implementations/tests/energy_oil_forecasting/test_skill_state.py b/implementations/tests/energy_oil_forecasting/test_skill_state.py new file mode 100644 index 00000000..2fee1a2d --- /dev/null +++ b/implementations/tests/energy_oil_forecasting/test_skill_state.py @@ -0,0 +1,44 @@ +"""Snapshot test pinning the rendered WTI strategy SKILL.md. + +Loads the committed ``wti-strategy-trained`` artifact, re-renders it through +:meth:`WtiStrategyState.build_markdown`, and asserts the output is byte-identical +to the committed ``SKILL.md``. This guards the promotion of ``WtiStrategyState`` +onto the generic shared-library ``StrategyState`` base: the rendered markdown +must stay render-identical so committed skill artifacts keep round-tripping. +""" + +from __future__ import annotations + +from pathlib import Path + +import yaml +from energy_oil_forecasting.adaptive_agent.skill_state import WtiStrategyState + + +_TRAINED_DIR = ( + Path(__file__).resolve().parents[2] + / "energy_oil_forecasting" + / "adaptive_agent" + / "skills" + / "wti-strategy-trained" +) + + +def test_trained_strategy_renders_identically() -> None: + """The committed trained strategy re-renders byte-identically.""" + state = WtiStrategyState.model_validate(yaml.safe_load((_TRAINED_DIR / "skill_state.yaml").read_text())) + + rendered = state.build_markdown(skill_name=_TRAINED_DIR.name) + committed = (_TRAINED_DIR / "SKILL.md").read_text(encoding="utf-8") + + assert rendered == committed + + +def test_default_skill_name_is_wti_strategy() -> None: + """With no explicit name, the frontmatter and heading stay oil-branded.""" + state = WtiStrategyState.model_validate(yaml.safe_load((_TRAINED_DIR / "skill_state.yaml").read_text())) + + rendered = state.build_markdown() + + assert "name: wti-strategy" in rendered + assert "# WTI Forecasting Strategy" in rendered diff --git a/implementations/tests/energy_oil_forecasting/test_skill_tools.py b/implementations/tests/energy_oil_forecasting/test_skill_tools.py new file mode 100644 index 00000000..bb12a7be --- /dev/null +++ b/implementations/tests/energy_oil_forecasting/test_skill_tools.py @@ -0,0 +1,130 @@ +"""Characterization tests for the strategy-mutation tools. + +Drives the five mutation callables produced by ``build_skill_tools`` against a +temporary strategy directory, pinning the evidence-governance behaviour (the +confirmation threshold guard, hypothesis lifecycle, and narrative update) before +the tools are promoted to the generic shared library. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from aieng.forecasting.methods.agentic.adaptive_skill import AdaptiveSkillStore +from energy_oil_forecasting.adaptive_agent.skill_state import WtiStrategyState +from energy_oil_forecasting.adaptive_agent.skill_tools import build_skill_tools + + +@pytest.fixture +def strategy_dir(tmp_path: Path) -> Path: + """Seed a minimal strategy directory and return its path.""" + store: AdaptiveSkillStore[WtiStrategyState] = AdaptiveSkillStore( + skill_dir=tmp_path, + state_type=WtiStrategyState, + ) + store.save(WtiStrategyState(approach_narrative="Seed approach.")) + return tmp_path + + +def _tools(strategy_dir: Path, *, confirmation_threshold: int = 2): + """Unpack the five mutation tools by name.""" + ( + record_observation, + open_hypothesis, + record_hypothesis_outcome, + graduate_hypothesis, + update_approach_narrative, + ) = build_skill_tools(strategy_dir, confirmation_threshold=confirmation_threshold) + return { + "record_observation": record_observation, + "open_hypothesis": open_hypothesis, + "record_hypothesis_outcome": record_hypothesis_outcome, + "graduate_hypothesis": graduate_hypothesis, + "update_approach_narrative": update_approach_narrative, + } + + +def _load(strategy_dir: Path) -> WtiStrategyState: + return AdaptiveSkillStore(skill_dir=strategy_dir, state_type=WtiStrategyState).load() + + +def test_record_observation_appends(strategy_dir: Path) -> None: + tools = _tools(strategy_dir) + tools["record_observation"]("Intervals too narrow in elevated vol.") + + state = _load(strategy_dir) + assert len(state.observations) == 1 + assert state.observations[0].finding == "Intervals too narrow in elevated vol." + assert state.observations[0].linked_hypothesis is None + + +def test_open_hypothesis_assigns_id_and_links_evidence(strategy_dir: Path) -> None: + tools = _tools(strategy_dir) + msg = tools["open_hypothesis"]("Trend hurts at long horizons.", "Backtest showed 3x MAE.") + + assert "hyp-001" in msg + state = _load(strategy_dir) + assert len(state.hypotheses) == 1 + assert state.hypotheses[0].id == "hyp-001" + assert state.hypotheses[0].status == "open" + # Initial evidence recorded as a linked observation. + assert any(o.linked_hypothesis == "hyp-001" for o in state.observations) + + +def test_record_outcome_validation(strategy_dir: Path) -> None: + tools = _tools(strategy_dir) + tools["open_hypothesis"]("claim", "evidence") + + assert "Invalid outcome" in tools["record_hypothesis_outcome"]("hyp-001", "maybe") + assert "not found" in tools["record_hypothesis_outcome"]("hyp-999", "confirmed") + + tools["record_hypothesis_outcome"]("hyp-001", "confirmed") + state = _load(strategy_dir) + assert state.hypotheses[0].confirmations == 1 + + +def test_graduate_rejected_below_threshold(strategy_dir: Path) -> None: + tools = _tools(strategy_dir, confirmation_threshold=2) + tools["open_hypothesis"]("claim", "evidence") + tools["record_hypothesis_outcome"]("hyp-001", "confirmed") + + msg = tools["graduate_hypothesis"]("hyp-001", "elevated vol", "widen CI", "all") + + assert "Cannot graduate" in msg + state = _load(strategy_dir) + assert state.calibration_corrections == [] + assert state.hypotheses[0].status == "open" + + +def test_graduate_succeeds_at_threshold(strategy_dir: Path) -> None: + tools = _tools(strategy_dir, confirmation_threshold=2) + tools["open_hypothesis"]("claim", "evidence") + tools["record_hypothesis_outcome"]("hyp-001", "confirmed") + tools["record_hypothesis_outcome"]("hyp-001", "confirmed") + + msg = tools["graduate_hypothesis"]("hyp-001", "elevated vol", "widen CI 12%", "21bd") + + assert "graduated" in msg.lower() + state = _load(strategy_dir) + assert len(state.calibration_corrections) == 1 + correction = state.calibration_corrections[0] + assert correction.condition == "elevated vol" + assert correction.adjustment == "widen CI 12%" + assert correction.horizon_scope == "21bd" + assert correction.source_hypothesis == "hyp-001" + assert state.hypotheses[0].status == "confirmed" + # Graduation is logged to version history. + assert any("hyp-001" in v.description for v in state.version_history) + + +def test_update_narrative_requires_rationale(strategy_dir: Path) -> None: + tools = _tools(strategy_dir) + + assert "rationale must not be empty" in tools["update_approach_narrative"]("New approach.", "") + assert "new_text must not be empty" in tools["update_approach_narrative"]("", "because") + + tools["update_approach_narrative"]("Weight news heavily at long horizons.", "Calibration record shifted.") + state = _load(strategy_dir) + assert state.approach_narrative == "Weight news heavily at long horizons." + assert any("approach narrative" in v.description.lower() for v in state.version_history) From 57c307ed9bb46af744d7ff955c43cf6964d95025 Mon Sep 17 00:00:00 2001 From: Ethan Jackson Date: Wed, 15 Jul 2026 20:59:18 -0400 Subject: [PATCH 2/3] refactor(agentic): make forecasting machinery domain-agnostic via DomainConfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Promote the shared agentic engine so a new target series can be configured without touching oil code, and drive the oil prompts/config from a single `DomainConfig`. Pure refactor: no new features, no study driver. Promoted to aieng/forecasting/methods/agentic/: - strategy_state.py: generic `StrategyState` (+ Observation/Hypothesis/ CalibrationCorrection/VersionEntry). Markdown title, default skill name, and frontmatter description are ClassVar params. Oil `WtiStrategyState` becomes a thin subclass pinning the WTI strings — committed SKILL.md renders identically. - adaptive_skill_tools.py: `build_skill_tools(strategy_dir, state_type, ...)` generic over any StrategyState subclass. No import-time singletons. Oil skill_tools.py delegates and re-exports the sub-models; STORE/WTI_SKILL_TOOLS removed (unused outside docstrings). - history.py: `compress_history` (was analyst_agent/agent.py); oil re-exports it. - outputs.py: generic `ScenarioCard` / `ScenarioAgentForecastOutput` with the scenario-card numeric fields parameterized via `scenario_card_template_extra`. Oil `tasks.py` subclasses them, preserving `wti_range_60d`/`point_estimate_60d` and modality="discrete". - domain.py: `DomainConfig` (identity, data/target, context-retrieval, strategy skill, skill dirs, vol-regime bands, tool bounds) plus render_* instruction templates and `build_analyst_config` / `build_adaptive_config`. Oil implementation: - energy_oil_forecasting/domain.py defines `OIL_DOMAIN`. - analyst/adaptive/starter agent instructions now render from OIL_DOMAIN; the `build_wti_*` factories are thin delegating wrappers. Rendered oil prompts are byte-identical to the originals (verified against captured goldens). Curriculum: `_vol_regime` vol bands extracted to a `bands` parameter with the 15/30/50 thresholds as `DEFAULT_VOL_REGIME_BANDS`. Invariants held: oil agent.name / predictor_id strings unchanged; committed skill artifacts render identically; nb05/nb06 import surfaces resolve. Regenerated the concierge catalog/artifacts to index the new modules. __init__ re-exports the promoted names; old oil import paths preserved. Co-Authored-By: Claude Fable 5 --- .../forecasting/methods/agentic/__init__.py | 28 ++ .../methods/agentic/adaptive_skill_tools.py | 399 ++++++++++++++++ .../forecasting/methods/agentic/curriculum.py | 29 +- .../forecasting/methods/agentic/domain.py | 422 +++++++++++++++++ .../forecasting/methods/agentic/history.py | 51 +++ .../forecasting/methods/agentic/outputs.py | 104 +++++ .../methods/agentic/strategy_state.py | 222 +++++++++ .../methods/agentic/test_adaptive_skill.py | 7 +- .../adaptive_agent/agent.py | 201 +------- .../adaptive_agent/skill_state.py | 238 ++-------- .../adaptive_agent/skill_tools.py | 406 ++--------------- .../analyst_agent/agent.py | 207 ++------- .../energy_oil_forecasting/domain.py | 118 +++++ .../starter_agent/agent.py | 36 +- .../energy_oil_forecasting/tasks.py | 111 ++--- ...ecasting__methods__agentic____init__.py.md | 28 ++ ...thods__agentic__adaptive_skill_tools.py.md | 405 +++++++++++++++++ ...asting__methods__agentic__adk_runner.py.md | 24 +- ...ing__methods__agentic__agent_factory.py.md | 268 +++++++++-- ...asting__methods__agentic__curriculum.py.md | 29 +- ...orecasting__methods__agentic__domain.py.md | 428 ++++++++++++++++++ ...recasting__methods__agentic__history.py.md | 57 +++ ...recasting__methods__agentic__outputs.py.md | 104 +++++ ...casting__methods__agentic__predictor.py.md | 8 +- ...ng__methods__agentic__strategy_state.py.md | 228 ++++++++++ ...rate_decisions__analyst_agent__agent.py.md | 17 +- ...rate_decisions__starter_agent__agent.py.md | 11 +- ...nt__skills__research-playbook__SKILL.md.md | 6 + ...ting__04_systematic_backtest_eval.ipynb.md | 194 ++++++-- ...il_forecasting__06_protected_eval.ipynb.md | 2 +- ...oil_forecasting__99_starter_agent.ipynb.md | 51 ++- ...l_forecasting__adaptive_agent__agent.py.md | 188 +------- ...casting__adaptive_agent__skill_state.py.md | 238 ++-------- ...casting__adaptive_agent__skill_tools.py.md | 406 ++--------------- ...ns__energy_oil_forecasting__analysis.py.md | 283 ++++++++++++ ...il_forecasting__analyst_agent__agent.py.md | 263 +++++------ ...ions__energy_oil_forecasting__domain.py.md | 124 +++++ ...orecasting__specs__energy_oil_eval.yaml.md | 15 +- ...ting__specs__energy_oil_eval_smoke.yaml.md | 2 +- ...forecasting__starter_agent____init__.py.md | 7 +- ...il_forecasting__starter_agent__agent.py.md | 144 +++--- ...nt__skills__research-playbook__SKILL.md.md | 6 + ...il_forecasting__starter_agent__tools.py.md | 237 ++++++++++ ...tions__energy_oil_forecasting__tasks.py.md | 111 ++--- ...tations__energy_oil_forecasting__viz.py.md | 270 +++++++++++ ...ce_forecasting__starter_agent__agent.py.md | 11 +- ...nt__skills__research-playbook__SKILL.md.md | 6 + ...00_forecasting__starter_agent__agent.py.md | 11 +- ...nt__skills__research-playbook__SKILL.md.md | 6 + .../concierge_agent/context/catalog.yaml | 181 ++++++-- .../references/catalog-summary.yaml | 10 +- 51 files changed, 4718 insertions(+), 2240 deletions(-) create mode 100644 aieng-forecasting/aieng/forecasting/methods/agentic/adaptive_skill_tools.py create mode 100644 aieng-forecasting/aieng/forecasting/methods/agentic/domain.py create mode 100644 aieng-forecasting/aieng/forecasting/methods/agentic/history.py create mode 100644 aieng-forecasting/aieng/forecasting/methods/agentic/strategy_state.py create mode 100644 implementations/energy_oil_forecasting/domain.py create mode 100644 implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adaptive_skill_tools.py.md create mode 100644 implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__domain.py.md create mode 100644 implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__history.py.md create mode 100644 implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__strategy_state.py.md create mode 100644 implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__domain.py.md create mode 100644 implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/__init__.py b/aieng-forecasting/aieng/forecasting/methods/agentic/__init__.py index 8a92c9ba..c3ec6b74 100644 --- a/aieng-forecasting/aieng/forecasting/methods/agentic/__init__.py +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/__init__.py @@ -60,6 +60,7 @@ """ from aieng.forecasting.methods.agentic.adaptive_skill import AdaptiveSkillState, AdaptiveSkillStore +from aieng.forecasting.methods.agentic.adaptive_skill_tools import build_skill_tools from aieng.forecasting.methods.agentic.adk_runner import AdkTextRunner, AdkTextRunnerConfig from aieng.forecasting.methods.agentic.agent_factory import ( AgentConfig, @@ -72,7 +73,13 @@ format_backtest_report, load_context_documents, ) +from aieng.forecasting.methods.agentic.domain import ( + DomainConfig, + build_adaptive_config, + build_analyst_config, +) from aieng.forecasting.methods.agentic.forecast_tool import ForecastTool +from aieng.forecasting.methods.agentic.history import compress_history from aieng.forecasting.methods.agentic.outputs import ( AgentCategoryProbability, AgentForecastOutput, @@ -81,8 +88,17 @@ ContinuousAgentForecastOutput, ContinuousAgentHorizonForecast, DiscreteAgentForecastOutput, + ScenarioAgentForecastOutput, + ScenarioCard, ) from aieng.forecasting.methods.agentic.predictor import AgentPredictor, ForecastPromptBuilder +from aieng.forecasting.methods.agentic.strategy_state import ( + CalibrationCorrection, + Hypothesis, + Observation, + StrategyState, + VersionEntry, +) __all__: list[str] = [ @@ -95,16 +111,28 @@ "AgentForecastOutput", "AgentPredictor", "AgentQuantileForecast", + "CalibrationCorrection", "CategoricalAgentForecastOutput", "CodeExecutionConfig", "ContinuousAgentForecastOutput", "ContinuousAgentHorizonForecast", "ContextRetrievalConfig", "DiscreteAgentForecastOutput", + "DomainConfig", "ForecastPromptBuilder", "ForecastTool", + "Hypothesis", + "Observation", + "ScenarioAgentForecastOutput", + "ScenarioCard", + "StrategyState", + "VersionEntry", + "build_adaptive_config", "build_adk_agent", + "build_analyst_config", "build_curriculum_prompt", + "build_skill_tools", + "compress_history", "format_backtest_report", "load_context_documents", ] diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/adaptive_skill_tools.py b/aieng-forecasting/aieng/forecasting/methods/agentic/adaptive_skill_tools.py new file mode 100644 index 00000000..49701c3e --- /dev/null +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/adaptive_skill_tools.py @@ -0,0 +1,399 @@ +"""Generic strategy-mutation tools for adaptive forecasting agents. + +These are plain Python callables registered as ADK ``FunctionTool`` objects via +``AgentConfig(extra_tools=build_skill_tools(strategy_dir, state_type))``. They +run in the host process — *not* inside a code-execution sandbox — so they can +read and write the skill directory on the local filesystem. + +Factory pattern +--------------- +Use :func:`build_skill_tools` to create a set of tools bound to a specific +strategy directory and :class:`StrategyState` subclass. This lets multiple +named strategy variants coexist, each with its own ``AdaptiveSkillStore`` and +``skill_state.yaml``. The factory holds **no import-time state**: nothing is +instantiated until it is called. + +Design principles +----------------- +Each tool follows the same three-step cycle: + +1. ``store.load()`` — deserialise current state from ``skill_state.yaml``. +2. Apply one typed mutation to the state model. +3. ``store.save(state)`` — write YAML, re-render ``SKILL.md``, back up. + +Tool signatures are intentionally narrow: they accept only the arguments needed +for one specific mutation. The agent cannot write arbitrary content to the +skill directory through any of these tools. + +Evidence governance +------------------- +``record_observation`` + No guard. Record any pattern-level finding (not a single-outlier surprise). + +``open_hypothesis`` + No guard. Open a hypothesis whenever you suspect a durable pattern. + +``record_hypothesis_outcome`` + Validates the hypothesis ID exists and is still open. + +``graduate_hypothesis`` + Hard guard: rejected if ``hypothesis.confirmations < store.confirmation_threshold``. + Returns a clear message stating the shortfall. + +``update_approach_narrative`` + Requires a ``rationale`` argument but no hard numeric guard. + +Scope guard +----------- +All writes go through the ``AdaptiveSkillStore`` instance passed to +:func:`build_skill_tools`. No path outside that directory can be reached. +""" + +from __future__ import annotations + +from datetime import date +from pathlib import Path +from typing import Callable + +from aieng.forecasting.methods.agentic.adaptive_skill import AdaptiveSkillStore +from aieng.forecasting.methods.agentic.strategy_state import ( + CalibrationCorrection, + Hypothesis, + Observation, + StrategyState, + VersionEntry, +) + + +# --------------------------------------------------------------------------- +# Stateless helpers +# --------------------------------------------------------------------------- + + +def _today() -> str: + return str(date.today()) + + +def _next_hypothesis_id(state: StrategyState) -> str: + """Return the next sequential hypothesis ID (e.g. ``hyp-004``).""" + n = len(state.hypotheses) + 1 + return f"hyp-{n:03d}" + + +# --------------------------------------------------------------------------- +# Tool factory +# --------------------------------------------------------------------------- + + +def build_skill_tools( # noqa: PLR0915 + strategy_dir: Path, + state_type: type[StrategyState] = StrategyState, + *, + confirmation_threshold: int = 3, +) -> list[Callable[..., str]]: + """Build a set of strategy mutation tools bound to *strategy_dir*. + + Each call returns five fresh callables (closures over a new + :class:`~aieng.forecasting.methods.agentic.adaptive_skill.AdaptiveSkillStore` + instance). Pass the returned list to + ``AgentConfig(extra_tools=build_skill_tools(strategy_dir, state_type))`` to + wire the tools into an agent that operates on a specific strategy variant. + + Parameters + ---------- + strategy_dir : Path + Directory containing the strategy skill (``skill_state.yaml``, + ``SKILL.md``, ``.history/``). Must exist and be a directory. + state_type : type[StrategyState], default=StrategyState + Concrete strategy-state class used to deserialise ``skill_state.yaml``. + Pass a domain subclass (e.g. ``WtiStrategyState``) to preserve its + rendered ``SKILL.md`` branding. + confirmation_threshold : int, default=3 + Number of confirming hypothesis outcomes required before + ``graduate_hypothesis`` is permitted. + + Returns + ------- + list[Callable[..., str]] + ``[record_observation, open_hypothesis, record_hypothesis_outcome, + graduate_hypothesis, update_approach_narrative]`` + """ + store: AdaptiveSkillStore[StrategyState] = AdaptiveSkillStore( + skill_dir=strategy_dir, + state_type=state_type, + confirmation_threshold=confirmation_threshold, + ) + + def record_observation(finding: str, linked_hypothesis: str = "") -> str: + """Record a pattern-level finding from a resolution or self-review. + + Call this whenever you observe a systematic pattern across multiple + forecasts — not after a single surprising outcome. + + Parameters + ---------- + finding : str + A concise description of the pattern observed. Be specific: include + the regime, horizon, and direction of the error where applicable. + Example: "80% intervals missed 4 of 5 actuals in the elevated vol + regime at the 21-day horizon." + linked_hypothesis : str, optional + ID of an existing open hypothesis this observation supports or + refutes (e.g. ``"hyp-001"``). Leave blank if this is a fresh + observation not yet linked to any hypothesis. + + Returns + ------- + str + Confirmation message. + """ + state = store.load() + obs = Observation( + date=_today(), + finding=finding.strip(), + linked_hypothesis=linked_hypothesis.strip() or None, + ) + state.observations.append(obs) + store.save(state) + linked_note = f" (linked to {obs.linked_hypothesis})" if obs.linked_hypothesis else "" + return f'Observation recorded{linked_note}: "{finding[:80]}{"..." if len(finding) > 80 else ""}"' + + def open_hypothesis(claim: str, initial_evidence: str) -> str: + """Open a new hypothesis about a suspected systematic forecasting pattern. + + A hypothesis is a candidate calibration correction under active testing. + Open one when you have at least one observation suggesting a durable + pattern but do not yet have enough confirming resolutions to graduate it. + + Parameters + ---------- + claim : str + A testable claim about your forecasting behaviour. State it in terms + of a specific condition and a directional error. + Example: "My 80% prediction intervals are consistently too narrow + when the vol regime is classified as elevated or extreme." + initial_evidence : str + The observation(s) that motivated opening this hypothesis. This is + for the audit record — be specific about the number of data points. + + Returns + ------- + str + Confirmation message including the assigned hypothesis ID. + """ + state = store.load() + hyp_id = _next_hypothesis_id(state) + hyp = Hypothesis( + id=hyp_id, + claim=claim.strip(), + status="open", + confirmations=0, + refutations=0, + opened_on=_today(), + ) + state.hypotheses.append(hyp) + obs = Observation( + date=_today(), + finding=initial_evidence.strip(), + linked_hypothesis=hyp_id, + ) + state.observations.append(obs) + store.save(state) + return ( + f'Hypothesis {hyp_id} opened: "{claim[:80]}{"..." if len(claim) > 80 else ""}". ' + f"Initial evidence recorded as an observation linked to {hyp_id}. " + f"Confirmations needed to graduate: {store.confirmation_threshold}." + ) + + def record_hypothesis_outcome(hypothesis_id: str, outcome: str) -> str: + """Record a confirming or refuting outcome for an open hypothesis. + + Call this after each resolution where the outcome is directly relevant + to an open hypothesis. Accumulate enough confirmations to graduate. + + Parameters + ---------- + hypothesis_id : str + ID of the hypothesis to update (e.g. ``"hyp-001"``). + outcome : str + Either ``"confirmed"`` or ``"refuted"``. A single refutation does + not automatically close the hypothesis — continue accumulating + evidence. A hypothesis should be manually closed (status → + ``"refuted"``) only when refutations clearly outweigh confirmations + across a meaningful sample. + + Returns + ------- + str + Updated confirmation / refutation counts and progress toward the + graduation threshold. + """ + if outcome not in ("confirmed", "refuted"): + return f"Invalid outcome '{outcome}'. Must be 'confirmed' or 'refuted'." + + state = store.load() + hyp = next((h for h in state.hypotheses if h.id == hypothesis_id), None) + if hyp is None: + ids = [h.id for h in state.hypotheses] + return f"Hypothesis '{hypothesis_id}' not found. Known IDs: {ids}." + if hyp.status != "open": + return f"Hypothesis {hypothesis_id} is already {hyp.status}. Only open hypotheses can receive new outcomes." + + if outcome == "confirmed": + hyp.confirmations += 1 + else: + hyp.refutations += 1 + + store.save(state) + + remaining = max(0, store.confirmation_threshold - hyp.confirmations) + if remaining == 0: + ready_msg = ( + f" Ready to graduate — call graduate_hypothesis('{hypothesis_id}', ...) " + "with a condition, adjustment, and horizon_scope." + ) + else: + ready_msg = f" {remaining} more confirmation(s) needed to graduate." + + return ( + f"{hypothesis_id} updated: {hyp.confirmations} confirmation(s), {hyp.refutations} refutation(s).{ready_msg}" + ) + + def graduate_hypothesis( + hypothesis_id: str, + condition: str, + adjustment: str, + horizon_scope: str, + ) -> str: + """Graduate a confirmed hypothesis to an active calibration correction. + + This is the primary mechanism through which the agent's strategy + improves. A calibration correction is applied at every future + prediction; it is not merely recorded — it changes behaviour. + + This tool enforces the confirmation threshold: it will reject the call + if the hypothesis has not accumulated enough confirming outcomes. + + Parameters + ---------- + hypothesis_id : str + ID of the confirmed hypothesis to graduate (e.g. ``"hyp-001"``). + condition : str + The specific condition under which this correction applies. + Example: "vol regime is elevated or extreme". + adjustment : str + The concrete adjustment to make when the condition is met. + Example: "Widen 80% CI by 12% relative to the statistical model + output." + horizon_scope : str + Which horizons this correction applies to. + One of: ``"all"``, ``"5bd"``, ``"10bd"``, ``"21bd"``, or a + combination like ``"10bd and 21bd"``. + + Returns + ------- + str + Confirmation message, or a rejection message with the shortfall. + """ + state = store.load() + hyp = next((h for h in state.hypotheses if h.id == hypothesis_id), None) + if hyp is None: + ids = [h.id for h in state.hypotheses] + return f"Hypothesis '{hypothesis_id}' not found. Known IDs: {ids}." + if hyp.status != "open": + return f"Hypothesis {hypothesis_id} is already {hyp.status}. Only open hypotheses can be graduated." + + if hyp.confirmations < store.confirmation_threshold: + shortfall = store.confirmation_threshold - hyp.confirmations + return ( + f"Cannot graduate {hypothesis_id}: " + f"{hyp.confirmations} confirmation(s), " + f"requires {store.confirmation_threshold}. " + f"Record {shortfall} more confirming outcome(s) first." + ) + + today = _today() + hyp.status = "confirmed" + correction = CalibrationCorrection( + condition=condition.strip(), + adjustment=adjustment.strip(), + horizon_scope=horizon_scope.strip(), + source_hypothesis=hypothesis_id, + confirmed_on=today, + ) + state.calibration_corrections.append(correction) + state.observations.append( + Observation( + date=today, + finding=( + f"Graduated {hypothesis_id} to calibration correction: " + f"'{condition}' → '{adjustment}' ({horizon_scope})." + ), + linked_hypothesis=hypothesis_id, + ) + ) + state.version_history.append( + VersionEntry( + date=today, + description=( + f"Graduated {hypothesis_id} to calibration correction " + f"(condition: {condition[:50]}{'...' if len(condition) > 50 else ''})." + ), + ) + ) + store.save(state) + return ( + f"Hypothesis {hypothesis_id} confirmed and graduated. " + f"Calibration correction added: when '{condition}', apply '{adjustment}' " + f"(scope: {horizon_scope})." + ) + + def update_approach_narrative(new_text: str, rationale: str) -> str: + """Replace the approach narrative with an updated strategic description. + + This is the highest-evidence-bar update. Consult ``meta-learning`` + before calling this tool — the narrative should only change when the + calibration record reveals a structural insight that the current + description no longer captures. A ``rationale`` argument is required + to force articulation of why the change is warranted. + + Parameters + ---------- + new_text : str + The complete replacement text for the ``## Approach`` section. + Write it as a self-contained description of the current forecasting + strategy — what signals are used, in what order, and with what + emphasis. + rationale : str + Why this update is warranted now. Cite the specific calibration + corrections or pattern of observations that motivated the change. + + Returns + ------- + str + Confirmation message. + """ + if not new_text.strip(): + return "new_text must not be empty." + if not rationale.strip(): + return "rationale must not be empty. Explain why the approach narrative warrants an update." + + state = store.load() + today = _today() + state.approach_narrative = new_text.strip() + state.version_history.append( + VersionEntry( + date=today, + description=f"Updated approach narrative. Rationale: {rationale[:120]}{'...' if len(rationale) > 120 else ''}", + ) + ) + store.save(state) + return f"Approach narrative updated ({len(new_text)} chars). Rationale recorded in version history." + + return [ + record_observation, + open_hypothesis, + record_hypothesis_outcome, + graduate_hypothesis, + update_approach_narrative, + ] diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/curriculum.py b/aieng-forecasting/aieng/forecasting/methods/agentic/curriculum.py index f65cc299..dc96ce0e 100644 --- a/aieng-forecasting/aieng/forecasting/methods/agentic/curriculum.py +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/curriculum.py @@ -64,12 +64,17 @@ # Vol-regime helper # --------------------------------------------------------------------------- -_VOL_REGIMES = [ +#: Default (upper-threshold, label) bands for annualised-vol regime +#: classification, in ascending order. Any value at or above the last band's +#: threshold is classified as :data:`_EXTREME_LABEL`. Override per domain via +#: the ``bands`` parameter of :func:`_vol_regime` (see +#: :attr:`~aieng.forecasting.methods.agentic.domain.DomainConfig.vol_regime_bands`). +DEFAULT_VOL_REGIME_BANDS: tuple[tuple[float, str], ...] = ( (15.0, "low"), (30.0, "medium"), (50.0, "elevated"), - (math.inf, "extreme"), -] +) +_EXTREME_LABEL = "extreme" _MIN_VOL_WINDOW = 5 @@ -81,10 +86,20 @@ _MIN_HORIZONS_FOR_NARRATIVE = 2 -def _vol_regime(price_series: pd.DataFrame, as_of: datetime, lookback: int = 21) -> str: +def _vol_regime( + price_series: pd.DataFrame, + as_of: datetime, + lookback: int = 21, + *, + bands: tuple[tuple[float, str], ...] = DEFAULT_VOL_REGIME_BANDS, + extreme_label: str = _EXTREME_LABEL, +) -> str: """Classify the vol regime at *as_of*. - Uses *lookback* trading days of log returns. + Uses *lookback* trading days of log returns. *bands* is a sequence of + ``(upper_threshold, label)`` pairs in ascending order; annualised vol at or + above the last threshold is classified as *extreme_label*. Defaults reproduce + the historical 15 / 30 / 50 thresholds. """ import pandas as pd # noqa: PLC0415 — conditional import for optional dep @@ -95,10 +110,10 @@ def _vol_regime(price_series: pd.DataFrame, as_of: datetime, lookback: int = 21) return "unknown" log_returns = np.diff(np.log(window.astype(float))) annualized_vol = float(np.std(log_returns) * np.sqrt(252) * 100) - for threshold, label in _VOL_REGIMES: + for threshold, label in bands: if annualized_vol < threshold: return label - return "extreme" + return extreme_label # --------------------------------------------------------------------------- diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/domain.py b/aieng-forecasting/aieng/forecasting/methods/agentic/domain.py new file mode 100644 index 00000000..7f2882ca --- /dev/null +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/domain.py @@ -0,0 +1,422 @@ +"""Domain configuration for agentic forecasters. + +A :class:`DomainConfig` captures everything that varies between forecasting +targets — the analyst persona, the target series and its units, the market- +intelligence search queries, the pipeline and strategy skill directories, the +volatility-regime bands, and the tool bounds — so the agent-building machinery +in this package stays domain-agnostic. The oil reference implementation defines +an ``OIL_DOMAIN`` instance; a new domain (e.g. S&P 500) is configured by +constructing another :class:`DomainConfig`, with no changes to shared code. + +The ``render_*`` functions turn a :class:`DomainConfig` into the instruction +strings the ADK agents run on, and :func:`build_analyst_config` / +:func:`build_adaptive_config` assemble those into +:class:`~aieng.forecasting.methods.agentic.agent_factory.AgentConfig` objects. +Rendering is pure string interpolation: given the oil fragments, the rendered +instructions reproduce the hand-written oil prompts. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable, Sequence + +from aieng.forecasting.methods.agentic.adaptive_skill_tools import build_skill_tools +from aieng.forecasting.methods.agentic.agent_factory import ( + AgentConfig, + CodeExecutionConfig, + ContextRetrievalConfig, +) +from aieng.forecasting.methods.agentic.outputs import ContinuousAgentForecastOutput +from aieng.forecasting.methods.agentic.strategy_state import StrategyState +from aieng.forecasting.models import LITE_MODEL +from pydantic import BaseModel, ConfigDict, Field + + +# --------------------------------------------------------------------------- +# Domain configuration +# --------------------------------------------------------------------------- + + +class DomainConfig(BaseModel): + """Everything that varies between forecasting targets. + + Grouped into identity, data/target, context-retrieval, strategy-skill, + skill-directory, volatility-regime, and tool-bound sections. All fields are + plain data; the render/build helpers below consume them. + """ + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + # ── Identity ──────────────────────────────────────────────────────────── + domain_name: str = Field(description="Human label for the domain, e.g. 'WTI crude oil'.") + analyst_persona: str = Field(description="Role noun phrase for the analyst, e.g. 'WTI crude oil market analyst'.") + analyst_forecasting_focus: str = Field( + description=( + "Completes 'You produce {...}.' in the analyst role block — the forecast " + "product and the fundamentals it is grounded in." + ) + ) + analyst_agent_name_prefix: str = Field( + default="analyst", + description="Prefix for stateless analyst agent names (a '_{suffix}' is appended).", + ) + adaptive_agent_name_prefix: str = Field( + default="adaptive_analyst", + description="Prefix for the adaptive agent name (the strategy dir name is appended).", + ) + target_short_name: str = Field(description="Short target label used inline, e.g. 'WTI'.") + starter_fluency_areas: str = Field( + default="", + description="Comma phrase of expertise areas for the hackable starter-agent persona.", + ) + + # ── Data / target ─────────────────────────────────────────────────────── + target_series_id: str = Field(description="Canonical series id the predictor forecasts.") + target_units: str = Field(description="Units of the target series, e.g. 'USD/bbl'.") + target_history_description: str = Field( + description="Describes the `target_history_csv` payload field in the analyst contract." + ) + data_ticker: str = Field(description="Market data ticker, e.g. 'CL=F'.") + data_source_name: str = Field(default="Yahoo Finance", description="Human name of the price data source.") + data_fetch_example: str = Field( + description="Verbatim code block showing a cutoff-respecting data fetch, embedded in the adaptive prompt." + ) + code_exec_preinstalled: str = Field( + default="numpy, pandas, sklearn, yfinance, statsmodels, properscoring", + description="Comma-separated list of libraries pre-installed in the code sandbox.", + ) + multitask_origin_price_field: str = Field( + default="origin_price", + description="Payload key carrying the origin close price in the multitask prompt.", + ) + + # ── Context retrieval ─────────────────────────────────────────────────── + context_retrieval_instruction: str = Field( + description="Full instruction for the web-search sub-agent (domain-specific verbatim)." + ) + recommended_search_queries: tuple[str, ...] = Field( + default=(), + description="Search queries advertised to the analyst, one `search_web` call each.", + ) + key_assumptions_hint: str = Field( + description="Parenthetical list of assumptions the analyst should document in rationales." + ) + + # ── Strategy skill ────────────────────────────────────────────────────── + strategy_skill_title: str = Field(description="Markdown H1 of the rendered strategy SKILL.md.") + strategy_skill_name: str = Field(description="Skill/dir name of the strategy skill, e.g. 'wti-strategy'.") + adaptive_calibration_example: str = Field( + description="Illustrative calibration action in the adaptive prediction-request block." + ) + + # ── Skill directories ─────────────────────────────────────────────────── + pipeline_skill_dirs: tuple[Path, ...] = Field( + default=(), + description="Ordered pipeline skill dirs (e.g. fetch, vol-regime, trend-projection).", + ) + meta_learning_skill_dir: Path | None = Field( + default=None, + description="Governance skill dir loaded after the strategy skill.", + ) + seed_strategy_dir: Path | None = Field(default=None, description="Default (seed) strategy skill dir.") + trained_strategy_dir: Path | None = Field(default=None, description="Trained strategy variant dir.") + + # ── Volatility regime ─────────────────────────────────────────────────── + vol_regime_bands: tuple[tuple[float, str], ...] = Field( + default=((15.0, "low"), (30.0, "medium"), (50.0, "elevated")), + description="(upper_threshold, label) bands for annualised-vol regime classification.", + ) + + # ── Tool bounds ───────────────────────────────────────────────────────── + frequency: str = Field(default="B", description="Pandas offset alias for the target series cadence.") + horizons: tuple[int, ...] = Field(default=(5, 10, 21), description="Default forecast horizons in steps.") + shock_threshold: float = Field(default=5.0, description="Move size defining the shock event.") + shock_horizon: int = Field(default=5, description="Horizon (steps) for the shock event.") + num_samples: int = Field(default=200, description="Monte-Carlo sample count for the statistical tool.") + + +# --------------------------------------------------------------------------- +# Instruction rendering +# --------------------------------------------------------------------------- + + +def render_analyst_instruction(domain: DomainConfig) -> str: + """Render the task-aware analyst instruction (news / code-exec / tool variants). + + Embeds the continuous output schema from + :class:`~aieng.forecasting.methods.agentic.outputs.ContinuousAgentForecastOutput` + so the required JSON block stays in sync with the schema. + """ + schema = ContinuousAgentForecastOutput.prompt_schema_json() + query_lines = "".join( + f'- ``search_web(query="{q}", cutoff_date=)``\n' for q in domain.recommended_search_queries + ) + return ( + "## Role\n\n" + f"You are an expert {domain.analyst_persona}. You produce {domain.analyst_forecasting_focus}.\n\n" + "## Forecasting contract\n\n" + "You will receive a JSON payload containing:\n" + "- `task`: the task identifier\n" + "- `as_of`: the forecast origin date in YYYY-MM-DD format\n" + "- `horizons`: a list of integer horizon steps (business days ahead)\n" + "- `standard_quantiles`: the exact quantile levels you must produce\n" + "- `target_summary`: last close price, 52-week range, and observation count\n" + f"- `target_history_csv`: {domain.target_history_description}\n\n" + "Rules:\n" + "1. Produce one forecast for each horizon listed in `horizons`.\n" + "2. Use exactly the quantile levels from `standard_quantiles` — no additions, no omissions.\n" + "3. `point_forecast` must exactly equal the 0.50 quantile value.\n" + "4. Quantile values must be strictly non-decreasing as quantile levels increase.\n" + "5. Document your reasoning in the `rationale` fields.\n" + "6. When tools are enabled, conclude with `set_model_response` to return the structured forecast.\n\n" + "## Output schema\n\n" + "Call `set_model_response` with a `json_response` string matching **exactly**:\n\n" + "```json\n" + schema + "\n```\n\n" + 'Critical: use `"horizon"` (integer, not `"horizon_days"`). ' + '`"quantiles"` is a **list** of `{"quantile": , "value": }` ' + "objects — not a dict. Omit any field not shown above.\n\n" + "## Analysis discipline\n\n" + "When context retrieval is available, call ``search_web`` to gather market " + "intelligence BEFORE producing forecasts.\n\n" + "Call ``search_web`` with ``query`` and ``cutoff_date`` (set to the ``as_of`` " + "date from the payload). The ``cutoff_date`` MUST always equal ``as_of`` — " + "this is the temporal fence that prevents post-origin information from " + "contaminating historical backtests.\n\n" + "If ``search_web`` returns a result beginning with " + "``[SEARCH_VERIFICATION_FAILED]``, treat it as no verified news context for " + "that query. Do not use your own background knowledge to fill the gap or " + "speculate about what the news might have said — proceed with price-history " + "and other available signals only, and note the gap in your rationale.\n\n" + "Recommended queries (call ``search_web`` once per topic):\n" + + query_lines + + "\n" + + f"Document your key assumptions ({domain.key_assumptions_hint}) in the `rationale` " + "fields of your forecast output." + ) + + +def render_multitask_analyst_instruction(domain: DomainConfig) -> str: + """Render the task-agnostic analyst instruction (one-agent-three-tasks demo).""" + return ( + "## Role\n\n" + f"You are an expert {domain.analyst_persona}.\n\n" + "## Input\n\n" + "You will receive a JSON payload containing:\n" + "- `task_spec`: the exact question and required JSON output schema\n" + "- `as_of`: the forecast origin date (temporal cutoff)\n" + f"- `{domain.multitask_origin_price_field}`: {domain.target_short_name} close on the origin date\n" + f"- `target_history_csv`: compressed {domain.target_short_name} daily close history\n\n" + "When context retrieval is enabled, call ``search_web`` BEFORE answering.\n\n" + "## Output contract\n\n" + "Read the data (and briefing, if retrieved) carefully, then execute the task " + "in `task_spec` precisely.\n\n" + "If a `set_model_response` tool is available, call it with your complete JSON " + "as `json_response` — the exact schema is described in `task_spec`. Otherwise " + "return the JSON directly as plain text with no preamble." + ) + + +def render_starter_instruction(domain: DomainConfig) -> str: + """Render the hackable starter-agent persona (task-agnostic, schema-free).""" + return ( + "## Role\n\n" + f"You are a {domain.analyst_persona} — fluent in {domain.starter_fluency_areas}. " + "This is a starter agent: keep your reasoning " + "transparent and your claims honest.\n\n" + "## How to respond\n\n" + "- For open-ended questions, scenario analysis, or anything " + "conversational, answer directly and concisely — do NOT ask for a JSON " + "payload.\n" + "- When you are handed a task that asks for a structured probabilistic " + "forecast, produce a calibrated one." + ) + + +def render_adaptive_analyst_instruction(domain: DomainConfig) -> str: + """Render the persistent adaptive-analyst instruction. + + Interpolates the persona, the pipeline / strategy / governance skill names, + the calibration example, the pre-installed library list, and the data-fetch + example from *domain*. The skill names default to the directory basenames. + """ + schema = ContinuousAgentForecastOutput.prompt_schema_json() + fetch, vol, trend = (d.name for d in domain.pipeline_skill_dirs) + meta = domain.meta_learning_skill_dir.name if domain.meta_learning_skill_dir else "meta-learning" + strategy = domain.strategy_skill_name + return ( + "## Identity\n\n" + f"You are a persistent {domain.analyst_persona}. You carry knowledge forward " + f"across invocations: your `{strategy}` skill captures your current forecasting " + "approach, and you update it deliberately as you learn from experience.\n\n" + "## Message types\n\n" + "You receive messages through a single chat interface. Determine from context " + "what kind of invocation this is and respond accordingly:\n\n" + "**Prediction request** — contains a JSON payload with `task`, `as_of`, " + f"`horizons`, and price history. Load `{strategy}` first to read your current " + "approach and any active calibration corrections. Then:\n" + f"1. Use `run_code` to run your full statistical analysis pipeline: fetch data " + f"via `{fetch}` (using `end=as_of` as the cutoff), classify the vol " + f"regime via `{vol}`, and project trend and intervals via `{trend}`. " + f"Apply any calibration corrections from `{strategy}` — for example, {domain.adaptive_calibration_example}.\n" + "2. Use the context-retrieval tool to gather current market news and adjust your " + "estimates where strong catalysts are present.\n" + "3. Conclude with `set_model_response` (schema below).\n\n" + "Your quantitative pipeline is your starting point — your learned strategy " + "corrections and news-grounded judgment shape the final forecast.\n\n" + "**Resolution** — describes how a past forecast resolved (actual value, error, " + "horizon). Reflect carefully. If the error points to a systematic pattern — not " + f"a one-off surprise — consult `{meta}` to assess whether a strategy update " + "is warranted.\n\n" + "**Self-review / backtesting** — you are asked to analyse your recent performance " + "or explore historical data using code execution. Compose the relevant skills, " + "write one complete code block, and summarise what you find. If the analysis " + f"surfaces a durable insight, follow the `{meta}` process.\n\n" + "**User question** — a human is asking for analysis, context, or your market " + "view. Engage directly, using code execution and web search as needed.\n\n" + "## Skills are pipeline components\n\n" + "Your skills cover specific pipeline stages. Compose them: for any task " + "involving code, load each relevant skill and its `references/examples.md`, " + "then write one complete self-contained code block combining all the patterns.\n\n" + "| Skill | Pipeline stage |\n" + "|------------------|---------------------------------------------------------|\n" + f"| {fetch:<16} | Download market / futures data from {domain.data_source_name} |\n" + f"| {vol:<16} | Classify vol regime, detect anomalies, choose window |\n" + f"| {trend:<16} | Fit trend, project to horizons, calibrate intervals |\n" + f"| {strategy:<16} | Your current forecasting strategy — load at the start of every prediction |\n" + f"| {meta:<16} | Governs when and how to update {strategy} |\n\n" + "## Strategy mutation tools\n\n" + f"These tools write directly to `{strategy}` on the host filesystem. " + "They run outside the E2B sandbox. Consult " + f"`{meta}` before calling " + "any of them.\n\n" + "| Tool | Evidence layer | Evidence bar |\n" + "|------|---------------|---------------|\n" + "| `record_observation(finding, linked_hypothesis?)` | Observations | Pattern visible across ≥2 forecasts — not a single surprise |\n" + "| `open_hypothesis(claim, initial_evidence)` | Hypotheses | One strong observation suggesting a durable pattern |\n" + "| `record_hypothesis_outcome(hypothesis_id, outcome)` | Hypotheses | Each resolution relevant to an open hypothesis |\n" + "| `graduate_hypothesis(hypothesis_id, condition, adjustment, horizon_scope)` | Calibration | Tool enforces confirmation threshold — will reject if not met |\n" + "| `update_approach_narrative(new_text, rationale)` | Approach | Only when the calibration record reveals a structural insight |\n\n" + f"Active calibration corrections from `{strategy}` are **not optional** — " + "apply every listed correction when the stated condition is met.\n\n" + "## Code execution discipline\n\n" + "Treat `run_code` like submitting to a batch queue: plan your complete " + "analysis upfront, write one self-contained script, and read the results. " + "There is no REPL, no way to inspect intermediate state between calls, and " + "no benefit to splitting work — each submission starts from zero with no " + "memory of previous calls.\n\n" + "Never make a preliminary or test call to check connectivity or verify " + "imports. Assume the environment works. Your first `run_code` call should " + "produce your complete result.\n\n" + f"Pre-installed: {domain.code_exec_preinstalled}.\n\n" + f"**Data sourcing rule:** Always use the `{fetch}` skill to load price " + "data inside `run_code`. **Never embed `target_history_csv` or any CSV " + "string literal as a data source in code.** Pasting thousands of rows of " + "data as Python string literals is fragile, wastes context, and risks hitting " + "sandbox limits. `target_history_csv` is provided in the prediction payload " + "for your reading and statistical summary only — not for copy-pasting into " + "code blocks. When a skill description says 'assume `df` is already defined', " + "that means you should define `df` via a yfinance fetch at the top of your " + "script, not by embedding raw data.\n\n" + "## Temporal discipline\n\n" + "Every forecast is anchored to an `as_of` date. Never use information beyond " + "that date — in web search, code analysis, or reasoning.\n\n" + "If `search_web` returns a result beginning with `[SEARCH_VERIFICATION_FAILED]`, " + "treat it as no verified news context for that query. Do not use your own " + "background knowledge to fill the gap — proceed on price history and other " + "available signals only, and note the gap in your reasoning.\n\n" + "When fetching data inside `run_code`, always pass `end=as_of_date` to " + "yfinance to enforce the temporal cutoff — for example:\n\n" + domain.data_fetch_example + "\n\n" + "Replace the end date with the actual `as_of` value from the prediction " + "payload. This is the only correct way to ensure the sandbox sees the same " + "data the agent would have seen on that date.\n\n" + "## Prediction output schema\n\n" + "For **prediction requests**, call `set_model_response` with `json_response` " + "matching **exactly**:\n\n" + "```json\n" + schema + "\n```\n\n" + 'Critical: use `"horizon"` (integer, not `"horizon_days"`). ' + '`"quantiles"` is a **list** of `{"quantile": , "value": }` ' + "objects — not a dict." + ) + + +# --------------------------------------------------------------------------- +# Config builders +# --------------------------------------------------------------------------- + + +def build_analyst_config( + domain: DomainConfig, + *, + name_suffix: str, + instruction: str, + model: str = LITE_MODEL, + context_retrieval: ContextRetrievalConfig | None = None, + code_execution: CodeExecutionConfig | None = None, + skills_dirs: Sequence[Path] = (), + function_tools: Sequence[Callable[..., Any]] = (), + max_output_tokens: int | None = None, +) -> AgentConfig: + """Assemble a stateless analyst :class:`AgentConfig` for *domain*. + + The agent name is ``f"{domain.analyst_agent_name_prefix}_{name_suffix}"``. + Only the knobs a variant actually uses need be passed; the rest fall back to + :class:`AgentConfig` defaults so the resulting config matches a hand-written + one field-for-field. + """ + return AgentConfig( + name=f"{domain.analyst_agent_name_prefix}_{name_suffix}", + model=model, + instruction=instruction, + context_retrieval=context_retrieval if context_retrieval is not None else ContextRetrievalConfig(), + code_execution=code_execution if code_execution is not None else CodeExecutionConfig(), + skills_dirs=tuple(skills_dirs), + function_tools=tuple(function_tools), + max_output_tokens=max_output_tokens, + ) + + +def build_adaptive_config( + domain: DomainConfig, + *, + state_type: type[StrategyState] = StrategyState, + model: str, + search_model: str = LITE_MODEL, + max_output_tokens: int = 16_384, + strategy_dir: Path | None = None, + confirmation_threshold: int = 2, +) -> AgentConfig: + """Assemble the persistent adaptive-analyst :class:`AgentConfig` for *domain*. + + Wires E2B code execution, cutoff-bounded web search, the pipeline skills, the + selected strategy skill, and the governance skill, plus the five strategy + mutation tools bound to *state_type*. The strategy directory name is baked + into the agent name so per-variant prediction caches stay separate. + """ + resolved_strategy_dir = strategy_dir or domain.seed_strategy_dir + if resolved_strategy_dir is None: + raise ValueError("No strategy_dir provided and domain.seed_strategy_dir is unset.") + agent_name = f"{domain.adaptive_agent_name_prefix}_{resolved_strategy_dir.name.replace('-', '_')}" + + skills_dirs = [*domain.pipeline_skill_dirs, resolved_strategy_dir] + if domain.meta_learning_skill_dir is not None: + skills_dirs.append(domain.meta_learning_skill_dir) + + return AgentConfig( + name=agent_name, + model=model, + instruction=render_adaptive_analyst_instruction(domain), + max_output_tokens=max_output_tokens, + context_retrieval=ContextRetrievalConfig( + enabled=True, + instruction=domain.context_retrieval_instruction, + search_model=search_model, + ), + code_execution=CodeExecutionConfig(enabled=True), + skills_dirs=skills_dirs, + extra_tools=build_skill_tools( + resolved_strategy_dir, + state_type, + confirmation_threshold=confirmation_threshold, + ), + ) diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/history.py b/aieng-forecasting/aieng/forecasting/methods/agentic/history.py new file mode 100644 index 00000000..630e8f3a --- /dev/null +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/history.py @@ -0,0 +1,51 @@ +"""History compression for agent prompt payloads. + +Serialising a full daily price history into an agent prompt is wasteful and can +overflow the context window. :func:`compress_history` keeps recent daily +resolution while down-sampling older history to weekly averages, producing a +compact ``date,close`` CSV suitable for embedding in a JSON payload. +""" + +from __future__ import annotations + +import pandas as pd + + +def compress_history(df: pd.DataFrame, *, recent_months: int = 6) -> str: + """Compress a daily price history to stay within context limits. + + Returns daily bars for the most recent ``recent_months`` months and weekly + averages for older history. The CSV header is ``date,close``. + + Parameters + ---------- + df : pd.DataFrame + DataFrame with columns ``timestamp`` and ``value``. + recent_months : int, default=6 + Number of trailing months to keep at daily resolution. Older history + is resampled to weekly averages. + + Returns + ------- + str + CSV string with header ``date,close``. + """ + df = df.copy() + df["timestamp"] = pd.to_datetime(df["timestamp"]) + cutoff = df["timestamp"].max() - pd.DateOffset(months=recent_months) + + recent = df[df["timestamp"] >= cutoff].copy() + old = df[df["timestamp"] < cutoff].copy() + + rows: list[str] = ["date,close"] + + if not old.empty: + old_indexed = old.set_index("timestamp")["value"] + weekly: pd.Series = old_indexed.resample("W").mean().dropna() + for date, val in weekly.items(): + rows.append(f"{date.date()},{val:.2f}") + + for _, row in recent.iterrows(): + rows.append(f"{row['timestamp'].date()},{row['value']:.2f}") + + return "\n".join(rows) diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/outputs.py b/aieng-forecasting/aieng/forecasting/methods/agentic/outputs.py index f62bedb6..c63e1fca 100644 --- a/aieng-forecasting/aieng/forecasting/methods/agentic/outputs.py +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/outputs.py @@ -626,3 +626,107 @@ def to_predictions( metadata=prediction_metadata, ) ] + + +class ScenarioCard(BaseModel): + """One probability-weighted scenario narrative from scenario-analysis output. + + Carries the qualitative story only. Domain implementations subclass this to + add numeric range / point-estimate fields under the field names their + downstream caches and displays expect (see + :meth:`ScenarioAgentForecastOutput.prompt_schema_json`). + """ + + model_config = {"extra": "ignore"} + + name: str + description: str + probability: float = Field(ge=0.0, le=1.0) + key_drivers: list[str] = Field(default_factory=list) + + +class ScenarioAgentForecastOutput(AgentForecastOutput): + """Scenario-analysis output: a set of probability-weighted narratives. + + The scenario cards carry the qualitative stories the market is debating; the + aggregate probability mass across cards is surfaced as a single + :class:`~aieng.forecasting.evaluation.prediction.BinaryForecast` for scoring, + while the full card set rides along in prediction metadata for display. + + Domain subclasses override :attr:`scenario_card_template_extra` (and narrow + the :attr:`scenarios` element type) to advertise the numeric scenario fields + their target series uses — e.g. a price range and point estimate at a named + horizon. + """ + + modality: ClassVar[Literal["continuous", "discrete", "categorical"]] = "discrete" + + model_config = {"extra": "ignore"} + + scenarios: list[ScenarioCard] + base_case: str + reasoning: str = "" + + #: Extra template fields injected into each scenario card in + #: :meth:`prompt_schema_json`, positioned between ``probability`` and + #: ``key_drivers``. Empty on the generic base; override in a subclass to + #: advertise domain-specific numeric fields. + scenario_card_template_extra: ClassVar[dict[str, object]] = {} + + @classmethod + def prompt_schema_json(cls) -> str: + """Return a JSON template for use in agent instruction strings. + + Returns + ------- + str + Indented JSON string showing the exact structure the agent must + pass to ``set_model_response``. + """ + card: dict[str, object] = { + "name": "", + "description": "", + "probability": "", + } + card.update(cls.scenario_card_template_extra) + card["key_drivers"] = ["", ""] + template: dict[str, object] = { + "scenarios": [card], + "base_case": "", + "reasoning": "", + } + return json.dumps(template, indent=2) + + def to_predictions( + self, + *, + task: ForecastingTask, + context: ForecastContext, + predictor_id: str, + metadata: dict[str, Any] | None = None, + ) -> list[Prediction]: + """Convert scenario output to a metadata-rich prediction.""" + if len(task.horizons) != 1: + raise ValueError("Scenario agent output expects exactly one task horizon.") + + horizon = task.horizons[0] + issued_at = datetime.utcnow() + offset = pd.tseries.frequencies.to_offset(task.frequency) + base_prob = float(sum(s.probability for s in self.scenarios)) + prediction_metadata: dict[str, Any] = dict(metadata) if metadata is not None else {} + prediction_metadata["scenarios"] = [s.model_dump() for s in self.scenarios] + prediction_metadata["base_case"] = self.base_case + if self.reasoning.strip(): + prediction_metadata["rationale"] = self.reasoning + + return [ + Prediction( + predictor_id=predictor_id, + task_id=task.task_id, + issued_at=issued_at, + as_of=context.as_of, + forecast_date=(pd.Timestamp(context.as_of) + offset * horizon).to_pydatetime(), + payload=BinaryForecast(probability=min(base_prob, 1.0)), + metadata=prediction_metadata, + ) + ] diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/strategy_state.py b/aieng-forecasting/aieng/forecasting/methods/agentic/strategy_state.py new file mode 100644 index 00000000..d1f95961 --- /dev/null +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/strategy_state.py @@ -0,0 +1,222 @@ +"""Generic adaptive forecasting-strategy state. + +``StrategyState`` is a concrete ``AdaptiveSkillState`` that models a +*learnable forecasting strategy* — the living approach an adaptive +agent refines across invocations. It captures four learning layers with +distinct evidence burdens: + +``observations`` + Append-only log of pattern-level findings. Lowest evidence bar — record + any finding that is not a single-outlier surprise. + +``hypotheses`` + Candidate systematic corrections under active testing. Accumulate + confirmation / refutation counts across resolutions. A hypothesis + graduates to a calibration correction when its confirmation count reaches + the store's ``confirmation_threshold``. + +``calibration_corrections`` + Confirmed systematic adjustments applied at prediction time. Each entry is + graduated from a confirmed hypothesis — never added directly. + +``approach_narrative`` + Free-text description of the agent's overall forecasting philosophy. + Highest evidence bar. + +The rendered ``SKILL.md`` layout is domain-agnostic; the three domain-specific +strings — the markdown heading, the default skill name, and the frontmatter +description — are ``ClassVar`` parameters. Subclass ``StrategyState`` and +override those class variables to brand the skill for a specific domain (see +:class:`~energy_oil_forecasting.adaptive_agent.skill_state.WtiStrategyState`). +""" + +from __future__ import annotations + +from typing import ClassVar, Literal + +from aieng.forecasting.methods.agentic.adaptive_skill import AdaptiveSkillState +from pydantic import BaseModel + + +# --------------------------------------------------------------------------- +# Sub-models +# --------------------------------------------------------------------------- + + +class Observation(BaseModel): + """A single pattern-level finding from a resolution or self-review.""" + + date: str + finding: str + linked_hypothesis: str | None = None + + +class Hypothesis(BaseModel): + """A candidate systematic correction under active testing. + + ``status`` progresses through ``open`` → ``confirmed`` or ``open`` → + ``refuted``. Confirmed hypotheses are graduated to + :class:`CalibrationCorrection` via the ``graduate_hypothesis`` tool. + """ + + id: str + claim: str + status: Literal["open", "confirmed", "refuted"] = "open" + confirmations: int = 0 + refutations: int = 0 + opened_on: str + + +class CalibrationCorrection(BaseModel): + """A confirmed systematic adjustment applied at prediction time. + + Every entry here was graduated from a confirmed hypothesis; the + ``source_hypothesis`` field preserves that lineage. + """ + + condition: str + adjustment: str + horizon_scope: str + source_hypothesis: str + confirmed_on: str + + +class VersionEntry(BaseModel): + """One row in the version history table.""" + + date: str + description: str + + +# --------------------------------------------------------------------------- +# Strategy state +# --------------------------------------------------------------------------- + + +class StrategyState(AdaptiveSkillState): + """Domain-agnostic structured state for an adaptive forecasting strategy. + + See the module docstring for the learning-layer hierarchy and evidence + burdens. The three ``ClassVar`` parameters below carry the only + domain-specific presentation strings; override them in a subclass to brand + the rendered ``SKILL.md`` for a particular target series. + """ + + #: Heading rendered as ``# {markdown_title}`` at the top of the body. + markdown_title: ClassVar[str] = "Forecasting Strategy" + #: Frontmatter ``name:`` used when the store does not pass an explicit name. + default_skill_name: ClassVar[str] = "strategy" + #: Content lines for the frontmatter ``description: >-`` block (rendered + #: with a two-space indent, one line per entry). + frontmatter_description_lines: ClassVar[tuple[str, ...]] = ( + "The adaptive analyst's current forecasting strategy. Load this at the", + "start of every prediction task. This file is generated — edit the state", + "through the mutation tools, not by hand.", + ) + + approach_narrative: str + calibration_corrections: list[CalibrationCorrection] = [] + hypotheses: list[Hypothesis] = [] + observations: list[Observation] = [] + version_history: list[VersionEntry] = [] + + def build_markdown(self, skill_name: str | None = None) -> str: # noqa: PLR0912 + """Render the full ``SKILL.md`` content from current state.""" + lines: list[str] = [] + + # Frontmatter — skill_name must match the containing dir name (ADK requires) + lines += [ + "---", + f"name: {skill_name or self.default_skill_name}", + "description: >-", + ] + lines += [f" {line}" for line in self.frontmatter_description_lines] + lines += [ + "---", + "", + ] + + lines += [ + f"# {self.markdown_title}", + "", + "## Approach", + "", + self.approach_narrative.strip(), + "", + ] + + # Active calibration corrections + lines += [ + "## Active calibration corrections", + "", + ] + if self.calibration_corrections: + lines += [ + "| Condition | Adjustment | Horizon scope | Confirmed on |", + "|-----------|-----------|---------------|--------------|", + ] + for c in self.calibration_corrections: + lines.append(f"| {c.condition} | {c.adjustment} | {c.horizon_scope} | {c.confirmed_on} |") + else: + lines.append("*(No calibration corrections yet. Graduate a confirmed hypothesis to add one.)*") + lines.append("") + + # Open hypotheses + lines += [ + "## Open hypotheses", + "", + ] + open_hyps = [h for h in self.hypotheses if h.status == "open"] + if open_hyps: + lines += [ + "| ID | Claim | Confirmations | Refutations |", + "|----|-------|---------------|-------------|", + ] + for h in open_hyps: + lines.append(f"| {h.id} | {h.claim} | {h.confirmations} | {h.refutations} |") + else: + lines.append("*(No open hypotheses.)*") + lines.append("") + + # Closed hypotheses (confirmed / refuted) — collapsed for readability + closed_hyps = [h for h in self.hypotheses if h.status != "open"] + if closed_hyps: + lines += [ + "## Closed hypotheses", + "", + "| ID | Claim | Status | Confirmations | Refutations |", + "|----|-------|--------|---------------|-------------|", + ] + for h in closed_hyps: + lines.append(f"| {h.id} | {h.claim} | {h.status} | {h.confirmations} | {h.refutations} |") + lines.append("") + + # Observations + lines += [ + "## Observations", + "", + ] + if self.observations: + lines += [ + "| Date | Finding | Linked hypothesis |", + "|------|---------|-------------------|", + ] + for o in self.observations: + linked = o.linked_hypothesis or "—" + lines.append(f"| {o.date} | {o.finding} | {linked} |") + else: + lines.append("*(No observations yet. Record findings from resolutions and self-reviews.)*") + lines.append("") + + # Version history + lines += [ + "## Version history", + "", + "| Date | Change |", + "|------|--------|", + ] + for v in self.version_history: + lines.append(f"| {v.date} | {v.description} |") + lines.append("") + + return "\n".join(lines) diff --git a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_adaptive_skill.py b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_adaptive_skill.py index 3cc6fd25..654fe538 100644 --- a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_adaptive_skill.py +++ b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_adaptive_skill.py @@ -1,9 +1,8 @@ """Characterization tests for the generic adaptive-skill persistence layer. -Exercises :class:`~aieng.forecasting.methods.agentic.adaptive_skill.AdaptiveSkillStore` -against a minimal concrete :class:`~aieng.forecasting.methods.agentic.adaptive_skill.AdaptiveSkillState` -subclass so the save / load / render / backup contract is pinned before the -strategy-state machinery is promoted and generalised. +Exercises ``AdaptiveSkillStore`` against a minimal concrete +``AdaptiveSkillState`` subclass so the save / load / render / backup contract is +pinned before the strategy-state machinery is promoted and generalised. """ from __future__ import annotations diff --git a/implementations/energy_oil_forecasting/adaptive_agent/agent.py b/implementations/energy_oil_forecasting/adaptive_agent/agent.py index ff4efebc..c9f11f5a 100644 --- a/implementations/energy_oil_forecasting/adaptive_agent/agent.py +++ b/implementations/energy_oil_forecasting/adaptive_agent/agent.py @@ -47,9 +47,9 @@ Pydantic model persisted in ``skills/wti-strategy/skill_state.yaml``. ``SKILL.md`` is rendered from that model on every mutation and is never hand-edited. Five typed mutation tools (from :mod:`skill_tools`) are -registered via ``AgentConfig(extra_tools=WTI_SKILL_TOOLS)`` and run in the -host process — not inside E2B. See :mod:`skill_tools` for the full tool -signatures and evidence governance rules. +registered via ``AgentConfig(extra_tools=build_skill_tools(strategy_dir))`` and +run in the host process — not inside E2B. See :mod:`skill_tools` for the full +tool signatures and evidence governance rules. """ from __future__ import annotations @@ -68,169 +68,27 @@ ContinuousAgentForecastOutput, build_adk_agent, ) -from aieng.forecasting.methods.agentic.agent_factory import ( - AgentConfig, - CodeExecutionConfig, - ContextRetrievalConfig, -) +from aieng.forecasting.methods.agentic.agent_factory import AgentConfig +from aieng.forecasting.methods.agentic.domain import build_adaptive_config from aieng.forecasting.models import ADVANCED_MODEL, LITE_MODEL -from energy_oil_forecasting.adaptive_agent.skill_tools import build_skill_tools +from energy_oil_forecasting.adaptive_agent.skill_state import WtiStrategyState from energy_oil_forecasting.analyst_agent import compress_history +from energy_oil_forecasting.domain import OIL_DOMAIN from pydantic import BaseModel logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Paths -# --------------------------------------------------------------------------- - -_SKILLS_ROOT = Path(__file__).parent / "skills" - - # --------------------------------------------------------------------------- # System prompt # --------------------------------------------------------------------------- - - -def _build_adaptive_analyst_instruction() -> str: - """Build the adaptive analyst instruction with the output schema embedded. - - Uses ``ContinuousAgentForecastOutput.prompt_schema_json()`` for the - prediction-response schema so it stays in sync with the output class. - """ - schema = ContinuousAgentForecastOutput.prompt_schema_json() - return ( - "## Identity\n\n" - "You are a persistent WTI crude oil market analyst. You carry knowledge forward " - "across invocations: your `wti-strategy` skill captures your current forecasting " - "approach, and you update it deliberately as you learn from experience.\n\n" - "## Message types\n\n" - "You receive messages through a single chat interface. Determine from context " - "what kind of invocation this is and respond accordingly:\n\n" - "**Prediction request** — contains a JSON payload with `task`, `as_of`, " - "`horizons`, and price history. Load `wti-strategy` first to read your current " - "approach and any active calibration corrections. Then:\n" - "1. Use `run_code` to run your full statistical analysis pipeline: fetch data " - "via `fetch-yfinance` (using `end=as_of` as the cutoff), classify the vol " - "regime via `vol-regime`, and project trend and intervals via `trend-projection`. " - "Apply any calibration corrections from `wti-strategy` — for example, substituting " - "a flat-trend model in elevated/extreme vol regimes if your strategy calls for it.\n" - "2. Use the context-retrieval tool to gather current market news and adjust your " - "estimates where strong catalysts are present.\n" - "3. Conclude with `set_model_response` (schema below).\n\n" - "Your quantitative pipeline is your starting point — your learned strategy " - "corrections and news-grounded judgment shape the final forecast.\n\n" - "**Resolution** — describes how a past forecast resolved (actual value, error, " - "horizon). Reflect carefully. If the error points to a systematic pattern — not " - "a one-off surprise — consult `meta-learning` to assess whether a strategy update " - "is warranted.\n\n" - "**Self-review / backtesting** — you are asked to analyse your recent performance " - "or explore historical data using code execution. Compose the relevant skills, " - "write one complete code block, and summarise what you find. If the analysis " - "surfaces a durable insight, follow the `meta-learning` process.\n\n" - "**User question** — a human is asking for analysis, context, or your market " - "view. Engage directly, using code execution and web search as needed.\n\n" - "## Skills are pipeline components\n\n" - "Your skills cover specific pipeline stages. Compose them: for any task " - "involving code, load each relevant skill and its `references/examples.md`, " - "then write one complete self-contained code block combining all the patterns.\n\n" - "| Skill | Pipeline stage |\n" - "|------------------|---------------------------------------------------------|\n" - "| fetch-yfinance | Download market / futures data from Yahoo Finance |\n" - "| vol-regime | Classify vol regime, detect anomalies, choose window |\n" - "| trend-projection | Fit trend, project to horizons, calibrate intervals |\n" - "| wti-strategy | Your current forecasting strategy — load at the start of every prediction |\n" - "| meta-learning | Governs when and how to update wti-strategy |\n\n" - "## Strategy mutation tools\n\n" - "These tools write directly to `wti-strategy` on the host filesystem. " - "They run outside the E2B sandbox. Consult `meta-learning` before calling " - "any of them.\n\n" - "| Tool | Evidence layer | Evidence bar |\n" - "|------|---------------|---------------|\n" - "| `record_observation(finding, linked_hypothesis?)` | Observations | Pattern visible across ≥2 forecasts — not a single surprise |\n" - "| `open_hypothesis(claim, initial_evidence)` | Hypotheses | One strong observation suggesting a durable pattern |\n" - "| `record_hypothesis_outcome(hypothesis_id, outcome)` | Hypotheses | Each resolution relevant to an open hypothesis |\n" - "| `graduate_hypothesis(hypothesis_id, condition, adjustment, horizon_scope)` | Calibration | Tool enforces confirmation threshold — will reject if not met |\n" - "| `update_approach_narrative(new_text, rationale)` | Approach | Only when the calibration record reveals a structural insight |\n\n" - "Active calibration corrections from `wti-strategy` are **not optional** — " - "apply every listed correction when the stated condition is met.\n\n" - "## Code execution discipline\n\n" - "Treat `run_code` like submitting to a batch queue: plan your complete " - "analysis upfront, write one self-contained script, and read the results. " - "There is no REPL, no way to inspect intermediate state between calls, and " - "no benefit to splitting work — each submission starts from zero with no " - "memory of previous calls.\n\n" - "Never make a preliminary or test call to check connectivity or verify " - "imports. Assume the environment works. Your first `run_code` call should " - "produce your complete result.\n\n" - "Pre-installed: numpy, pandas, sklearn, yfinance, statsmodels, properscoring.\n\n" - "**Data sourcing rule:** Always use the `fetch-yfinance` skill to load price " - "data inside `run_code`. **Never embed `target_history_csv` or any CSV " - "string literal as a data source in code.** Pasting thousands of rows of " - "data as Python string literals is fragile, wastes context, and risks hitting " - "sandbox limits. `target_history_csv` is provided in the prediction payload " - "for your reading and statistical summary only — not for copy-pasting into " - "code blocks. When a skill description says 'assume `df` is already defined', " - "that means you should define `df` via a yfinance fetch at the top of your " - "script, not by embedding raw data.\n\n" - "## Temporal discipline\n\n" - "Every forecast is anchored to an `as_of` date. Never use information beyond " - "that date — in web search, code analysis, or reasoning.\n\n" - "If `search_web` returns a result beginning with `[SEARCH_VERIFICATION_FAILED]`, " - "treat it as no verified news context for that query. Do not use your own " - "background knowledge to fill the gap — proceed on price history and other " - "available signals only, and note the gap in your reasoning.\n\n" - "When fetching data inside `run_code`, always pass `end=as_of_date` to " - "yfinance to enforce the temporal cutoff — for example:\n\n" - "```python\nraw = ticker.history(start='2004-01-01', end='2026-02-16', " - "auto_adjust=False)\n```\n\n" - "Replace the end date with the actual `as_of` value from the prediction " - "payload. This is the only correct way to ensure the sandbox sees the same " - "data the agent would have seen on that date.\n\n" - "## Prediction output schema\n\n" - "For **prediction requests**, call `set_model_response` with `json_response` " - "matching **exactly**:\n\n" - "```json\n" + schema + "\n```\n\n" - 'Critical: use `"horizon"` (integer, not `"horizon_days"`). ' - '`"quantiles"` is a **list** of `{"quantile": , "value": }` ' - "objects — not a dict." - ) - - -_ADAPTIVE_ANALYST_INSTRUCTION = _build_adaptive_analyst_instruction() - - -# --------------------------------------------------------------------------- -# Context retrieval instruction -# --------------------------------------------------------------------------- - -_WTI_CONTEXT_RETRIEVAL_INSTRUCTION = """\ -You are an oil market intelligence specialist with access to web search. - -Search for information relevant to the query and return a concise structured \ -markdown summary (3-5 paragraphs) covering relevant aspects of: -- WTI/Brent crude price level and recent trend -- OPEC+ production decisions and supply outlook -- Geopolitical risks in the Persian Gulf, Middle East, key shipping lanes -- US Strategic Petroleum Reserve and energy policy signals -- Notable tanker/shipping incidents or supply disruption signals -- Published analyst forecasts or unusual price-target revisions - -Ground your summary in the search results you actually retrieve. \ -When a cutoff date is specified, do not report or speculate about events \ -that occurred after that date. - -Before finalizing your summary, reason step by step: (1) for each candidate \ -fact, judge its actual recency from the substance of the result itself, \ -never from a source's claimed publish date or byline timestamp — those are \ -frequently stale or updated after original publication; (2) discard \ -anything you cannot confidently place before the cutoff date; (3) only then \ -write your summary. Do not supplement the search results with your own \ -background/training knowledge — if the results are insufficient, say so \ -explicitly rather than filling gaps from memory.\ -""" +# +# The adaptive analyst instruction and the web-search sub-agent instruction are +# rendered from the shared templates over the WTI ``OIL_DOMAIN`` fragments (see +# :mod:`energy_oil_forecasting.domain` and +# :func:`aieng.forecasting.methods.agentic.domain.build_adaptive_config`). A new +# target series is configured by supplying a different ``DomainConfig``. # --------------------------------------------------------------------------- @@ -323,30 +181,19 @@ def build_wti_adaptive_config( ------- AgentConfig """ - resolved_strategy_dir = strategy_dir or (_SKILLS_ROOT / "wti-strategy") - # Include strategy dir name in agent name so cached_multi_backtest writes a - # separate cache file per variant (cache key is derived from predictor_id, - # which is derived from agent name). - agent_name = f"wti_adaptive_analyst_{resolved_strategy_dir.name.replace('-', '_')}" - return AgentConfig( - name=agent_name, + # Delegates to the shared, domain-agnostic builder with the WTI domain and + # ``WtiStrategyState`` so the rendered SKILL.md keeps its oil branding. The + # strategy dir name is baked into the agent name (cache key is derived from + # predictor_id, which is derived from agent name) so per-variant prediction + # caches stay separate. + return build_adaptive_config( + OIL_DOMAIN, + state_type=WtiStrategyState, model=model, - instruction=_ADAPTIVE_ANALYST_INSTRUCTION, + search_model=search_model, max_output_tokens=max_output_tokens, - context_retrieval=ContextRetrievalConfig( - enabled=True, - instruction=_WTI_CONTEXT_RETRIEVAL_INSTRUCTION, - search_model=search_model, - ), - code_execution=CodeExecutionConfig(enabled=True), - skills_dirs=[ - _SKILLS_ROOT / "fetch-yfinance", - _SKILLS_ROOT / "vol-regime", - _SKILLS_ROOT / "trend-projection", - resolved_strategy_dir, - _SKILLS_ROOT / "meta-learning", - ], - extra_tools=build_skill_tools(resolved_strategy_dir, confirmation_threshold=2), + strategy_dir=strategy_dir, + confirmation_threshold=2, ) diff --git a/implementations/energy_oil_forecasting/adaptive_agent/skill_state.py b/implementations/energy_oil_forecasting/adaptive_agent/skill_state.py index 889cf8cd..c3afaf28 100644 --- a/implementations/energy_oil_forecasting/adaptive_agent/skill_state.py +++ b/implementations/energy_oil_forecasting/adaptive_agent/skill_state.py @@ -1,210 +1,54 @@ """WTI forecasting strategy state model. Defines the structured state backing the ``wti-strategy`` adaptive skill. -``WtiStrategyState`` is the single source of truth for the agent's current -forecasting approach. It is persisted to ``skills/wti-strategy/skill_state.yaml`` -and rendered to ``skills/wti-strategy/SKILL.md`` on every mutation so that the -ADK ``SkillToolset`` always reads an up-to-date version. - -Learning layers ---------------- -The four fields of ``WtiStrategyState`` map to distinct update frequencies and -evidence burdens, enforced partly by the mutation tools in ``skill_tools.py`` -and partly by the ``meta-learning`` governance skill: - -``observations`` - Append-only log of pattern-level findings. Lowest evidence bar — record - any finding that is not a single-outlier surprise. - -``hypotheses`` - Candidate systematic corrections the agent is actively testing. Open a - hypothesis when you suspect a durable pattern. Accumulate confirmation / - refutation counts across resolutions. A hypothesis graduates to a - calibration correction when its confirmation count reaches the store's - ``confirmation_threshold``. - -``calibration_corrections`` - Confirmed systematic adjustments applied at prediction time. Each entry - is graduated from a confirmed hypothesis — never added directly. - -``approach_narrative`` - Free-text description of the agent's overall forecasting philosophy. - Highest evidence bar. Update only when the calibration record reveals a - structural insight that the narrative no longer captures. +``WtiStrategyState`` is a thin oil-branded subclass of the domain-agnostic +:class:`~aieng.forecasting.methods.agentic.strategy_state.StrategyState`: it +pins the rendered ``SKILL.md`` heading, default skill name, and frontmatter +description to the WTI wording so committed artifacts round-trip byte-identically. +All fields, sub-models, and rendering logic live in the shared library. + +The state is persisted to ``skills/wti-strategy/skill_state.yaml`` and rendered +to ``skills/wti-strategy/SKILL.md`` on every mutation so that the ADK +``SkillToolset`` always reads an up-to-date version. See the shared +:mod:`aieng.forecasting.methods.agentic.strategy_state` module for the +learning-layer hierarchy and evidence burdens. """ from __future__ import annotations -from typing import Literal - -from aieng.forecasting.methods.agentic.adaptive_skill import AdaptiveSkillState -from pydantic import BaseModel - - -# --------------------------------------------------------------------------- -# Sub-models -# --------------------------------------------------------------------------- - - -class Observation(BaseModel): - """A single pattern-level finding from a resolution or self-review.""" - - date: str - finding: str - linked_hypothesis: str | None = None - - -class Hypothesis(BaseModel): - """A candidate systematic correction under active testing. - - ``status`` progresses through ``open`` → ``confirmed`` or ``open`` → - ``refuted``. Confirmed hypotheses are graduated to - :class:`CalibrationCorrection` via the ``graduate_hypothesis`` tool. - """ - - id: str - claim: str - status: Literal["open", "confirmed", "refuted"] = "open" - confirmations: int = 0 - refutations: int = 0 - opened_on: str - +from typing import ClassVar -class CalibrationCorrection(BaseModel): - """A confirmed systematic adjustment applied at prediction time. +# Re-exported so existing imports of the sub-models from this module keep working. +from aieng.forecasting.methods.agentic.strategy_state import ( + CalibrationCorrection, + Hypothesis, + Observation, + StrategyState, + VersionEntry, +) - Every entry here was graduated from a confirmed hypothesis; the - ``source_hypothesis`` field preserves that lineage. - """ - - condition: str - adjustment: str - horizon_scope: str - source_hypothesis: str - confirmed_on: str - - -class VersionEntry(BaseModel): - """One row in the version history table.""" - - date: str - description: str +class WtiStrategyState(StrategyState): + """Oil-branded strategy state for the adaptive WTI crude oil analyst. -# --------------------------------------------------------------------------- -# Strategy state -# --------------------------------------------------------------------------- - - -class WtiStrategyState(AdaptiveSkillState): - """Structured state for the adaptive WTI crude oil forecasting strategy. - - See module docstring for the learning-layer hierarchy and evidence burdens. + Identical in structure to :class:`StrategyState`; only the presentation + strings are pinned so the ``wti-strategy`` artifacts render exactly as + committed. """ - approach_narrative: str - calibration_corrections: list[CalibrationCorrection] = [] - hypotheses: list[Hypothesis] = [] - observations: list[Observation] = [] - version_history: list[VersionEntry] = [] - - def build_markdown(self, skill_name: str | None = None) -> str: # noqa: PLR0912 - """Render the full ``SKILL.md`` content from current state.""" - lines: list[str] = [] - - # Frontmatter — skill_name must match the containing directory name (ADK requirement) - lines += [ - "---", - f"name: {skill_name or 'wti-strategy'}", - "description: >-", - " The adaptive WTI analyst's current forecasting strategy. Load this at the", - " start of every prediction task. This file is generated — edit the state", - " through the mutation tools, not by hand.", - "---", - "", - ] - - lines += [ - "# WTI Forecasting Strategy", - "", - "## Approach", - "", - self.approach_narrative.strip(), - "", - ] - - # Active calibration corrections - lines += [ - "## Active calibration corrections", - "", - ] - if self.calibration_corrections: - lines += [ - "| Condition | Adjustment | Horizon scope | Confirmed on |", - "|-----------|-----------|---------------|--------------|", - ] - for c in self.calibration_corrections: - lines.append(f"| {c.condition} | {c.adjustment} | {c.horizon_scope} | {c.confirmed_on} |") - else: - lines.append("*(No calibration corrections yet. Graduate a confirmed hypothesis to add one.)*") - lines.append("") - - # Open hypotheses - lines += [ - "## Open hypotheses", - "", - ] - open_hyps = [h for h in self.hypotheses if h.status == "open"] - if open_hyps: - lines += [ - "| ID | Claim | Confirmations | Refutations |", - "|----|-------|---------------|-------------|", - ] - for h in open_hyps: - lines.append(f"| {h.id} | {h.claim} | {h.confirmations} | {h.refutations} |") - else: - lines.append("*(No open hypotheses.)*") - lines.append("") - - # Closed hypotheses (confirmed / refuted) — collapsed for readability - closed_hyps = [h for h in self.hypotheses if h.status != "open"] - if closed_hyps: - lines += [ - "## Closed hypotheses", - "", - "| ID | Claim | Status | Confirmations | Refutations |", - "|----|-------|--------|---------------|-------------|", - ] - for h in closed_hyps: - lines.append(f"| {h.id} | {h.claim} | {h.status} | {h.confirmations} | {h.refutations} |") - lines.append("") - - # Observations - lines += [ - "## Observations", - "", - ] - if self.observations: - lines += [ - "| Date | Finding | Linked hypothesis |", - "|------|---------|-------------------|", - ] - for o in self.observations: - linked = o.linked_hypothesis or "—" - lines.append(f"| {o.date} | {o.finding} | {linked} |") - else: - lines.append("*(No observations yet. Record findings from resolutions and self-reviews.)*") - lines.append("") - - # Version history - lines += [ - "## Version history", - "", - "| Date | Change |", - "|------|--------|", - ] - for v in self.version_history: - lines.append(f"| {v.date} | {v.description} |") - lines.append("") - - return "\n".join(lines) + markdown_title: ClassVar[str] = "WTI Forecasting Strategy" + default_skill_name: ClassVar[str] = "wti-strategy" + frontmatter_description_lines: ClassVar[tuple[str, ...]] = ( + "The adaptive WTI analyst's current forecasting strategy. Load this at the", + "start of every prediction task. This file is generated — edit the state", + "through the mutation tools, not by hand.", + ) + + +__all__ = [ + "CalibrationCorrection", + "Hypothesis", + "Observation", + "VersionEntry", + "WtiStrategyState", +] diff --git a/implementations/energy_oil_forecasting/adaptive_agent/skill_tools.py b/implementations/energy_oil_forecasting/adaptive_agent/skill_tools.py index b686c21d..08ce3c55 100644 --- a/implementations/energy_oil_forecasting/adaptive_agent/skill_tools.py +++ b/implementations/energy_oil_forecasting/adaptive_agent/skill_tools.py @@ -1,66 +1,25 @@ """Mutation tools for the ``wti-strategy`` adaptive skill. -These are plain Python callables registered as ADK ``FunctionTool`` objects via -``AgentConfig(extra_tools=build_skill_tools(strategy_dir))``. They run in the -host process — *not* inside the E2B sandbox — so they can read and write the -skill directory on the local filesystem. - -Factory pattern ---------------- -Use :func:`build_skill_tools` to create a set of tools bound to a specific -strategy directory. This allows multiple named strategy variants (e.g. -``wti-strategy-stats``, ``wti-strategy-news``) to coexist, each with its own -``AdaptiveSkillStore`` and ``skill_state.yaml``. - -The module-level :data:`STORE` and :data:`WTI_SKILL_TOOLS` are convenience -bindings to the **default** ``wti-strategy`` directory for backward -compatibility and interactive ``adk web`` use. - -Design principles ------------------ -Each tool follows the same three-step cycle: - -1. ``store.load()`` — deserialise current state from ``skill_state.yaml``. -2. Apply one typed mutation to the state model. -3. ``store.save(state)`` — write YAML, re-render ``SKILL.md``, back up. - -Tool signatures are intentionally narrow: they accept only the arguments -needed for one specific mutation. The agent cannot write arbitrary content to -the skill directory through any of these tools. - -Evidence governance -------------------- -``record_observation`` - No guard. Record any pattern-level finding (not a single-outlier surprise). - -``open_hypothesis`` - No guard. Open a hypothesis whenever you suspect a durable pattern. - -``record_hypothesis_outcome`` - Validates the hypothesis ID exists and is still open. - -``graduate_hypothesis`` - Hard guard: rejected if ``hypothesis.confirmations < store.confirmation_threshold``. - Returns a clear message stating the shortfall. - -``update_approach_narrative`` - Requires a ``rationale`` argument but no hard numeric guard. The - ``meta-learning`` skill governs when this is appropriate. - -Scope guard ------------ -All writes go through the :class:`~aieng.forecasting.methods.agentic.adaptive_skill.AdaptiveSkillStore` -instance passed to :func:`build_skill_tools`. No path outside that directory -can be reached. +Thin oil-side wrapper over the shared, domain-agnostic tool factory +:func:`aieng.forecasting.methods.agentic.adaptive_skill_tools.build_skill_tools`. +The wrapper binds the factory to :class:`WtiStrategyState` so the rendered +``SKILL.md`` keeps its WTI branding. The five mutation tools, their evidence +governance, and the scope guard are documented on the shared module. + +The sub-model re-exports below keep existing imports of ``Observation``, +``Hypothesis``, ``CalibrationCorrection``, and ``VersionEntry`` from this module +working. There are no import-time singletons: nothing touches the filesystem +until :func:`build_skill_tools` is called. """ from __future__ import annotations -from datetime import date from pathlib import Path from typing import Callable -from aieng.forecasting.methods.agentic.adaptive_skill import AdaptiveSkillStore +from aieng.forecasting.methods.agentic.adaptive_skill_tools import ( + build_skill_tools as _build_skill_tools, +) from energy_oil_forecasting.adaptive_agent.skill_state import ( CalibrationCorrection, Hypothesis, @@ -70,45 +29,17 @@ ) -# --------------------------------------------------------------------------- -# Default strategy directory (for backward compat and adk web) -# --------------------------------------------------------------------------- - -_SKILL_DIR = Path(__file__).parent / "skills" / "wti-strategy" - - -# --------------------------------------------------------------------------- -# Stateless helpers -# --------------------------------------------------------------------------- - - -def _today() -> str: - return str(date.today()) - - -def _next_hypothesis_id(state: WtiStrategyState) -> str: - """Return the next sequential hypothesis ID (e.g. ``hyp-004``).""" - n = len(state.hypotheses) + 1 - return f"hyp-{n:03d}" - - -# --------------------------------------------------------------------------- -# Tool factory -# --------------------------------------------------------------------------- - - -def build_skill_tools( # noqa: PLR0915 +def build_skill_tools( strategy_dir: Path, *, confirmation_threshold: int = 3, ) -> list[Callable[..., str]]: - """Build a set of strategy mutation tools bound to *strategy_dir*. + """Build the five WTI strategy mutation tools bound to *strategy_dir*. - Each call returns five fresh callables (closures over a new - :class:`~aieng.forecasting.methods.agentic.adaptive_skill.AdaptiveSkillStore` - instance). Pass the returned list to - ``AgentConfig(extra_tools=build_skill_tools(strategy_dir))`` to wire the - tools into an agent that operates on a specific strategy variant. + Delegates to the shared factory with :class:`WtiStrategyState` as the state + type. See + :func:`aieng.forecasting.methods.agentic.adaptive_skill_tools.build_skill_tools` + for the full tool signatures and evidence-governance rules. Parameters ---------- @@ -125,295 +56,18 @@ def build_skill_tools( # noqa: PLR0915 ``[record_observation, open_hypothesis, record_hypothesis_outcome, graduate_hypothesis, update_approach_narrative]`` """ - store: AdaptiveSkillStore[WtiStrategyState] = AdaptiveSkillStore( - skill_dir=strategy_dir, - state_type=WtiStrategyState, + return _build_skill_tools( + strategy_dir, + WtiStrategyState, confirmation_threshold=confirmation_threshold, ) - def record_observation(finding: str, linked_hypothesis: str = "") -> str: - """Record a pattern-level finding from a resolution or self-review. - - Call this whenever you observe a systematic pattern across multiple - forecasts — not after a single surprising outcome. - - Parameters - ---------- - finding : str - A concise description of the pattern observed. Be specific: include - the regime, horizon, and direction of the error where applicable. - Example: "80% intervals missed 4 of 5 actuals in the elevated vol - regime at the 21-day horizon." - linked_hypothesis : str, optional - ID of an existing open hypothesis this observation supports or - refutes (e.g. ``"hyp-001"``). Leave blank if this is a fresh - observation not yet linked to any hypothesis. - - Returns - ------- - str - Confirmation message. - """ - state = store.load() - obs = Observation( - date=_today(), - finding=finding.strip(), - linked_hypothesis=linked_hypothesis.strip() or None, - ) - state.observations.append(obs) - store.save(state) - linked_note = f" (linked to {obs.linked_hypothesis})" if obs.linked_hypothesis else "" - return f'Observation recorded{linked_note}: "{finding[:80]}{"..." if len(finding) > 80 else ""}"' - - def open_hypothesis(claim: str, initial_evidence: str) -> str: - """Open a new hypothesis about a suspected systematic forecasting pattern. - - A hypothesis is a candidate calibration correction under active testing. - Open one when you have at least one observation suggesting a durable - pattern but do not yet have enough confirming resolutions to graduate it. - - Parameters - ---------- - claim : str - A testable claim about your forecasting behaviour. State it in terms - of a specific condition and a directional error. - Example: "My 80% prediction intervals are consistently too narrow - when the vol regime is classified as elevated or extreme." - initial_evidence : str - The observation(s) that motivated opening this hypothesis. This is - for the audit record — be specific about the number of data points. - - Returns - ------- - str - Confirmation message including the assigned hypothesis ID. - """ - state = store.load() - hyp_id = _next_hypothesis_id(state) - hyp = Hypothesis( - id=hyp_id, - claim=claim.strip(), - status="open", - confirmations=0, - refutations=0, - opened_on=_today(), - ) - state.hypotheses.append(hyp) - obs = Observation( - date=_today(), - finding=initial_evidence.strip(), - linked_hypothesis=hyp_id, - ) - state.observations.append(obs) - store.save(state) - return ( - f'Hypothesis {hyp_id} opened: "{claim[:80]}{"..." if len(claim) > 80 else ""}". ' - f"Initial evidence recorded as an observation linked to {hyp_id}. " - f"Confirmations needed to graduate: {store.confirmation_threshold}." - ) - - def record_hypothesis_outcome(hypothesis_id: str, outcome: str) -> str: - """Record a confirming or refuting outcome for an open hypothesis. - - Call this after each resolution where the outcome is directly relevant - to an open hypothesis. Accumulate enough confirmations to graduate. - - Parameters - ---------- - hypothesis_id : str - ID of the hypothesis to update (e.g. ``"hyp-001"``). - outcome : str - Either ``"confirmed"`` or ``"refuted"``. A single refutation does - not automatically close the hypothesis — continue accumulating - evidence. A hypothesis should be manually closed (status → - ``"refuted"``) only when refutations clearly outweigh confirmations - across a meaningful sample. - - Returns - ------- - str - Updated confirmation / refutation counts and progress toward the - graduation threshold. - """ - if outcome not in ("confirmed", "refuted"): - return f"Invalid outcome '{outcome}'. Must be 'confirmed' or 'refuted'." - - state = store.load() - hyp = next((h for h in state.hypotheses if h.id == hypothesis_id), None) - if hyp is None: - ids = [h.id for h in state.hypotheses] - return f"Hypothesis '{hypothesis_id}' not found. Known IDs: {ids}." - if hyp.status != "open": - return f"Hypothesis {hypothesis_id} is already {hyp.status}. Only open hypotheses can receive new outcomes." - - if outcome == "confirmed": - hyp.confirmations += 1 - else: - hyp.refutations += 1 - - store.save(state) - - remaining = max(0, store.confirmation_threshold - hyp.confirmations) - if remaining == 0: - ready_msg = ( - f" Ready to graduate — call graduate_hypothesis('{hypothesis_id}', ...) " - "with a condition, adjustment, and horizon_scope." - ) - else: - ready_msg = f" {remaining} more confirmation(s) needed to graduate." - - return ( - f"{hypothesis_id} updated: {hyp.confirmations} confirmation(s), {hyp.refutations} refutation(s).{ready_msg}" - ) - - def graduate_hypothesis( - hypothesis_id: str, - condition: str, - adjustment: str, - horizon_scope: str, - ) -> str: - """Graduate a confirmed hypothesis to an active calibration correction. - - This is the primary mechanism through which the agent's strategy - improves. A calibration correction is applied at every future - prediction; it is not merely recorded — it changes behaviour. - - This tool enforces the confirmation threshold: it will reject the call - if the hypothesis has not accumulated enough confirming outcomes. - - Parameters - ---------- - hypothesis_id : str - ID of the confirmed hypothesis to graduate (e.g. ``"hyp-001"``). - condition : str - The specific condition under which this correction applies. - Example: "vol regime is elevated or extreme". - adjustment : str - The concrete adjustment to make when the condition is met. - Example: "Widen 80% CI by 12% relative to the statistical model - output." - horizon_scope : str - Which horizons this correction applies to. - One of: ``"all"``, ``"5bd"``, ``"10bd"``, ``"21bd"``, or a - combination like ``"10bd and 21bd"``. - - Returns - ------- - str - Confirmation message, or a rejection message with the shortfall. - """ - state = store.load() - hyp = next((h for h in state.hypotheses if h.id == hypothesis_id), None) - if hyp is None: - ids = [h.id for h in state.hypotheses] - return f"Hypothesis '{hypothesis_id}' not found. Known IDs: {ids}." - if hyp.status != "open": - return f"Hypothesis {hypothesis_id} is already {hyp.status}. Only open hypotheses can be graduated." - - if hyp.confirmations < store.confirmation_threshold: - shortfall = store.confirmation_threshold - hyp.confirmations - return ( - f"Cannot graduate {hypothesis_id}: " - f"{hyp.confirmations} confirmation(s), " - f"requires {store.confirmation_threshold}. " - f"Record {shortfall} more confirming outcome(s) first." - ) - - today = _today() - hyp.status = "confirmed" - correction = CalibrationCorrection( - condition=condition.strip(), - adjustment=adjustment.strip(), - horizon_scope=horizon_scope.strip(), - source_hypothesis=hypothesis_id, - confirmed_on=today, - ) - state.calibration_corrections.append(correction) - state.observations.append( - Observation( - date=today, - finding=( - f"Graduated {hypothesis_id} to calibration correction: " - f"'{condition}' → '{adjustment}' ({horizon_scope})." - ), - linked_hypothesis=hypothesis_id, - ) - ) - state.version_history.append( - VersionEntry( - date=today, - description=( - f"Graduated {hypothesis_id} to calibration correction " - f"(condition: {condition[:50]}{'...' if len(condition) > 50 else ''})." - ), - ) - ) - store.save(state) - return ( - f"Hypothesis {hypothesis_id} confirmed and graduated. " - f"Calibration correction added: when '{condition}', apply '{adjustment}' " - f"(scope: {horizon_scope})." - ) - - def update_approach_narrative(new_text: str, rationale: str) -> str: - """Replace the approach narrative with an updated strategic description. - - This is the highest-evidence-bar update. Consult ``meta-learning`` - before calling this tool — the narrative should only change when the - calibration record reveals a structural insight that the current - description no longer captures. A ``rationale`` argument is required - to force articulation of why the change is warranted. - - Parameters - ---------- - new_text : str - The complete replacement text for the ``## Approach`` section. - Write it as a self-contained description of the current forecasting - strategy — what signals are used, in what order, and with what - emphasis. - rationale : str - Why this update is warranted now. Cite the specific calibration - corrections or pattern of observations that motivated the change. - - Returns - ------- - str - Confirmation message. - """ - if not new_text.strip(): - return "new_text must not be empty." - if not rationale.strip(): - return "rationale must not be empty. Explain why the approach narrative warrants an update." - - state = store.load() - today = _today() - state.approach_narrative = new_text.strip() - state.version_history.append( - VersionEntry( - date=today, - description=f"Updated approach narrative. Rationale: {rationale[:120]}{'...' if len(rationale) > 120 else ''}", - ) - ) - store.save(state) - return f"Approach narrative updated ({len(new_text)} chars). Rationale recorded in version history." - - return [ - record_observation, - open_hypothesis, - record_hypothesis_outcome, - graduate_hypothesis, - update_approach_narrative, - ] - - -# --------------------------------------------------------------------------- -# Backward-compatible module-level bindings (default wti-strategy dir) -# --------------------------------------------------------------------------- - -STORE: AdaptiveSkillStore[WtiStrategyState] = AdaptiveSkillStore( - skill_dir=_SKILL_DIR, - state_type=WtiStrategyState, - confirmation_threshold=3, -) -WTI_SKILL_TOOLS: list[Callable[..., str]] = build_skill_tools(_SKILL_DIR) +__all__ = [ + "CalibrationCorrection", + "Hypothesis", + "Observation", + "VersionEntry", + "WtiStrategyState", + "build_skill_tools", +] diff --git a/implementations/energy_oil_forecasting/analyst_agent/agent.py b/implementations/energy_oil_forecasting/analyst_agent/agent.py index 77faf3e9..631ec2fa 100644 --- a/implementations/energy_oil_forecasting/analyst_agent/agent.py +++ b/implementations/energy_oil_forecasting/analyst_agent/agent.py @@ -48,129 +48,31 @@ CodeExecutionConfig, ContextRetrievalConfig, ) +from aieng.forecasting.methods.agentic.domain import ( + build_analyst_config, + render_analyst_instruction, + render_multitask_analyst_instruction, +) +from aieng.forecasting.methods.agentic.history import compress_history from aieng.forecasting.methods.numerical.darts_arima import DartsAutoARIMAPredictor from aieng.forecasting.models import ADVANCED_MODEL, LITE_MODEL from energy_oil_forecasting.data import WTI_SERIES_ID, build_wti_service +from energy_oil_forecasting.domain import OIL_DOMAIN from pydantic import BaseModel # --------------------------------------------------------------------------- -# System prompt (root analyst agent) +# System prompts — rendered from OIL_DOMAIN # --------------------------------------------------------------------------- +# +# The analyst, multitask, and context-retrieval instructions are rendered from +# the shared instruction templates over the WTI ``OIL_DOMAIN`` fragments (see +# :mod:`energy_oil_forecasting.domain`). A new target series is configured by +# supplying a different ``DomainConfig`` — no edits to this module. -_WTI_MULTITASK_ANALYST_INSTRUCTION = """\ -## Role - -You are an expert WTI crude oil market analyst. - -## Input - -You will receive a JSON payload containing: -- `task_spec`: the exact question and required JSON output schema -- `as_of`: the forecast origin date (temporal cutoff) -- `origin_price_usd_bbl`: WTI close on the origin date -- `target_history_csv`: compressed WTI daily close history - -When context retrieval is enabled, call ``search_web`` BEFORE answering. - -## Output contract - -Read the data (and briefing, if retrieved) carefully, then execute the task \ -in `task_spec` precisely. - -If a `set_model_response` tool is available, call it with your complete JSON \ -as `json_response` — the exact schema is described in `task_spec`. Otherwise \ -return the JSON directly as plain text with no preamble.\ -""" - - -def _build_wti_analyst_instruction() -> str: - """Build the WTI analyst instruction, embedding the output schema from the class. - - Using a function instead of a static string ensures the ``## Output schema`` - block is always in sync with ``ContinuousAgentForecastOutput`` — - no manual JSON to maintain. - """ - schema = ContinuousAgentForecastOutput.prompt_schema_json() - return ( - "## Role\n\n" - "You are an expert WTI crude oil market analyst. You produce calibrated " - "probabilistic price forecasts for WTI crude oil futures, grounded in " - "supply/demand fundamentals, geopolitical risk, and historical price dynamics.\n\n" - "## Forecasting contract\n\n" - "You will receive a JSON payload containing:\n" - "- `task`: the task identifier\n" - "- `as_of`: the forecast origin date in YYYY-MM-DD format\n" - "- `horizons`: a list of integer horizon steps (business days ahead)\n" - "- `standard_quantiles`: the exact quantile levels you must produce\n" - "- `target_summary`: last close price, 52-week range, and observation count\n" - "- `target_history_csv`: WTI daily close history (recent 6 months daily, " - "older history as weekly averages)\n\n" - "Rules:\n" - "1. Produce one forecast for each horizon listed in `horizons`.\n" - "2. Use exactly the quantile levels from `standard_quantiles` — no additions, no omissions.\n" - "3. `point_forecast` must exactly equal the 0.50 quantile value.\n" - "4. Quantile values must be strictly non-decreasing as quantile levels increase.\n" - "5. Document your reasoning in the `rationale` fields.\n" - "6. When tools are enabled, conclude with `set_model_response` to return the structured forecast.\n\n" - "## Output schema\n\n" - "Call `set_model_response` with a `json_response` string matching **exactly**:\n\n" - "```json\n" + schema + "\n```\n\n" - 'Critical: use `"horizon"` (integer, not `"horizon_days"`). ' - '`"quantiles"` is a **list** of `{"quantile": , "value": }` ' - "objects — not a dict. Omit any field not shown above.\n\n" - "## Analysis discipline\n\n" - "When context retrieval is available, call ``search_web`` to gather market " - "intelligence BEFORE producing forecasts.\n\n" - "Call ``search_web`` with ``query`` and ``cutoff_date`` (set to the ``as_of`` " - "date from the payload). The ``cutoff_date`` MUST always equal ``as_of`` — " - "this is the temporal fence that prevents post-origin information from " - "contaminating historical backtests.\n\n" - "If ``search_web`` returns a result beginning with " - "``[SEARCH_VERIFICATION_FAILED]``, treat it as no verified news context for " - "that query. Do not use your own background knowledge to fill the gap or " - "speculate about what the news might have said — proceed with price-history " - "and other available signals only, and note the gap in your rationale.\n\n" - "Recommended queries (call ``search_web`` once per topic):\n" - '- ``search_web(query="WTI crude oil price trend and OPEC+ supply decisions", cutoff_date=)``\n' - '- ``search_web(query="Persian Gulf geopolitical risk shipping lane disruptions", cutoff_date=)``\n' - '- ``search_web(query="US Strategic Petroleum Reserve policy and global demand outlook", cutoff_date=)``\n\n' - "Document your key assumptions (OPEC+ policy, shipping lane risk, inventory " - "levels, macro demand) in the `rationale` fields of your forecast output." - ) - - -_WTI_ANALYST_INSTRUCTION = _build_wti_analyst_instruction() - -# --------------------------------------------------------------------------- -# Context retrieval instruction (sub-agent) -# --------------------------------------------------------------------------- - -_WTI_CONTEXT_RETRIEVAL_INSTRUCTION = """\ -You are an oil market intelligence specialist with access to web search. - -Search for information relevant to the query and return a concise structured \ -markdown summary (3-5 paragraphs) covering relevant aspects of: -- WTI/Brent crude price level and recent trend -- OPEC+ production decisions and supply outlook -- Geopolitical risks in the Persian Gulf, Middle East, key shipping lanes -- US Strategic Petroleum Reserve and energy policy signals -- Notable tanker/shipping incidents or supply disruption signals -- Published analyst forecasts or unusual price-target revisions - -Ground your summary in the search results you actually retrieve. \ -When a cutoff date is specified, do not report or speculate about events \ -that occurred after that date. - -Before finalizing your summary, reason step by step: (1) for each candidate \ -fact, judge its actual recency from the substance of the result itself, \ -never from a source's claimed publish date or byline timestamp — those are \ -frequently stale or updated after original publication; (2) discard \ -anything you cannot confidently place before the cutoff date; (3) only then \ -write your summary. Do not supplement the search results with your own \ -background/training knowledge — if the results are insufficient, say so \ -explicitly rather than filling gaps from memory.\ -""" +_WTI_ANALYST_INSTRUCTION = render_analyst_instruction(OIL_DOMAIN) +_WTI_MULTITASK_ANALYST_INSTRUCTION = render_multitask_analyst_instruction(OIL_DOMAIN) +_WTI_CONTEXT_RETRIEVAL_INSTRUCTION = OIL_DOMAIN.context_retrieval_instruction # --------------------------------------------------------------------------- # Skills supplement (appended to instruction when skills are attached) @@ -239,45 +141,13 @@ def _build_wti_analyst_instruction() -> str: # --------------------------------------------------------------------------- -# History compression +# History compression — re-exported from the shared library for backward compat # --------------------------------------------------------------------------- - -def compress_history(df: pd.DataFrame) -> str: - """Compress WTI daily history to stay within context limits. - - Returns daily bars for the most recent 6 months and weekly averages for - older history. The CSV header is ``date,close``. - - Parameters - ---------- - df : pd.DataFrame - DataFrame with columns ``timestamp`` and ``value``. - - Returns - ------- - str - CSV string with header ``date,close``. - """ - df = df.copy() - df["timestamp"] = pd.to_datetime(df["timestamp"]) - cutoff = df["timestamp"].max() - pd.DateOffset(months=6) - - recent = df[df["timestamp"] >= cutoff].copy() - old = df[df["timestamp"] < cutoff].copy() - - rows: list[str] = ["date,close"] - - if not old.empty: - old_indexed = old.set_index("timestamp")["value"] - weekly: pd.Series = old_indexed.resample("W").mean().dropna() - for date, val in weekly.items(): - rows.append(f"{date.date()},{val:.2f}") - - for _, row in recent.iterrows(): - rows.append(f"{row['timestamp'].date()},{row['value']:.2f}") - - return "\n".join(rows) +# ``compress_history`` now lives in +# :mod:`aieng.forecasting.methods.agentic.history`; it is imported above and +# re-exported here so existing ``from ...analyst_agent import compress_history`` +# call sites keep resolving. # --------------------------------------------------------------------------- @@ -363,10 +233,11 @@ def build_wti_basic_config(model: str = LITE_MODEL) -> AgentConfig: ------- AgentConfig """ - return AgentConfig( - name="wti_analyst_basic", - model=model, + return build_analyst_config( + OIL_DOMAIN, + name_suffix="basic", instruction=_WTI_ANALYST_INSTRUCTION, + model=model, ) @@ -401,10 +272,11 @@ def build_wti_multitask_news_config( verifier_confidence_threshold : int Minimum verifier confidence (1-10) required to accept a result. """ - return AgentConfig( - name="wti_analyst_multitask", - model=model, + return build_analyst_config( + OIL_DOMAIN, + name_suffix="multitask", instruction=_WTI_MULTITASK_ANALYST_INSTRUCTION, + model=model, context_retrieval=ContextRetrievalConfig( enabled=True, instruction=_WTI_CONTEXT_RETRIEVAL_INSTRUCTION, @@ -454,10 +326,11 @@ def build_wti_news_config( ------- AgentConfig """ - return AgentConfig( - name="wti_analyst_news", - model=model, + return build_analyst_config( + OIL_DOMAIN, + name_suffix="news", instruction=_WTI_ANALYST_INSTRUCTION, + model=model, context_retrieval=ContextRetrievalConfig( enabled=True, instruction=_WTI_CONTEXT_RETRIEVAL_INSTRUCTION, @@ -515,10 +388,11 @@ def build_wti_code_exec_config( ------- AgentConfig """ - return AgentConfig( - name="wti_analyst_code", - model=model, + return build_analyst_config( + OIL_DOMAIN, + name_suffix="code", instruction=_WTI_ANALYST_INSTRUCTION + _CODE_EXEC_SKILLS_SUPPLEMENT, + model=model, max_output_tokens=max_output_tokens, context_retrieval=ContextRetrievalConfig( enabled=True, @@ -590,10 +464,11 @@ def build_wti_tool_config( service = data_service if data_service is not None else build_wti_service() forecast_tool = ForecastTool(service, predictor=DartsAutoARIMAPredictor(num_samples=num_samples)) - return AgentConfig( - name="wti_analyst_tool", - model=model, + return build_analyst_config( + OIL_DOMAIN, + name_suffix="tool", instruction=_WTI_ANALYST_INSTRUCTION + _FORECAST_TOOL_SUPPLEMENT, + model=model, context_retrieval=ContextRetrievalConfig( enabled=True, instruction=_WTI_CONTEXT_RETRIEVAL_INSTRUCTION, diff --git a/implementations/energy_oil_forecasting/domain.py b/implementations/energy_oil_forecasting/domain.py new file mode 100644 index 00000000..bab7179d --- /dev/null +++ b/implementations/energy_oil_forecasting/domain.py @@ -0,0 +1,118 @@ +"""WTI crude oil domain configuration. + +Defines :data:`OIL_DOMAIN`, the single +:class:`~aieng.forecasting.methods.agentic.domain.DomainConfig` instance that +supplies every WTI-specific fragment the shared agent-building machinery needs. +The analyst, multitask, and adaptive agent factories render their instructions +from this instance, so all oil-specific prompt wording lives here rather than +scattered across the agent modules. +""" + +from __future__ import annotations + +from pathlib import Path + +from aieng.forecasting.methods.agentic.domain import DomainConfig +from energy_oil_forecasting.data import WTI_SERIES_ID +from energy_oil_forecasting.paths import SHOCK_HORIZON, SHOCK_THRESHOLD + + +# --------------------------------------------------------------------------- +# Web-search sub-agent instruction (domain-specific verbatim) +# --------------------------------------------------------------------------- + +_WTI_CONTEXT_RETRIEVAL_INSTRUCTION = """\ +You are an oil market intelligence specialist with access to web search. + +Search for information relevant to the query and return a concise structured \ +markdown summary (3-5 paragraphs) covering relevant aspects of: +- WTI/Brent crude price level and recent trend +- OPEC+ production decisions and supply outlook +- Geopolitical risks in the Persian Gulf, Middle East, key shipping lanes +- US Strategic Petroleum Reserve and energy policy signals +- Notable tanker/shipping incidents or supply disruption signals +- Published analyst forecasts or unusual price-target revisions + +Ground your summary in the search results you actually retrieve. \ +When a cutoff date is specified, do not report or speculate about events \ +that occurred after that date. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ +""" + + +# --------------------------------------------------------------------------- +# Skill directories (adaptive agent) +# --------------------------------------------------------------------------- + +_ADAPTIVE_SKILLS_ROOT = Path(__file__).parent / "adaptive_agent" / "skills" + + +# --------------------------------------------------------------------------- +# The WTI domain +# --------------------------------------------------------------------------- + +OIL_DOMAIN = DomainConfig( + # Identity + domain_name="WTI crude oil", + analyst_persona="WTI crude oil market analyst", + analyst_forecasting_focus=( + "calibrated probabilistic price forecasts for WTI crude oil futures, grounded in " + "supply/demand fundamentals, geopolitical risk, and historical price dynamics" + ), + analyst_agent_name_prefix="wti_analyst", + adaptive_agent_name_prefix="wti_adaptive_analyst", + target_short_name="WTI", + starter_fluency_areas=( + "supply/demand fundamentals, OPEC+ policy, geopolitical and shipping-lane risk, and price dynamics" + ), + # Data / target + target_series_id=WTI_SERIES_ID, + target_units="USD/bbl", + target_history_description=("WTI daily close history (recent 6 months daily, older history as weekly averages)"), + data_ticker="CL=F", + data_source_name="Yahoo Finance", + data_fetch_example=( + "```python\nraw = ticker.history(start='2004-01-01', end='2026-02-16', auto_adjust=False)\n```" + ), + code_exec_preinstalled="numpy, pandas, sklearn, yfinance, statsmodels, properscoring", + multitask_origin_price_field="origin_price_usd_bbl", + # Context retrieval + context_retrieval_instruction=_WTI_CONTEXT_RETRIEVAL_INSTRUCTION, + recommended_search_queries=( + "WTI crude oil price trend and OPEC+ supply decisions", + "Persian Gulf geopolitical risk shipping lane disruptions", + "US Strategic Petroleum Reserve policy and global demand outlook", + ), + key_assumptions_hint="OPEC+ policy, shipping lane risk, inventory levels, macro demand", + # Strategy skill + strategy_skill_title="WTI Forecasting Strategy", + strategy_skill_name="wti-strategy", + adaptive_calibration_example=( + "substituting a flat-trend model in elevated/extreme vol regimes if your strategy calls for it" + ), + # Skill directories + pipeline_skill_dirs=( + _ADAPTIVE_SKILLS_ROOT / "fetch-yfinance", + _ADAPTIVE_SKILLS_ROOT / "vol-regime", + _ADAPTIVE_SKILLS_ROOT / "trend-projection", + ), + meta_learning_skill_dir=_ADAPTIVE_SKILLS_ROOT / "meta-learning", + seed_strategy_dir=_ADAPTIVE_SKILLS_ROOT / "wti-strategy", + trained_strategy_dir=_ADAPTIVE_SKILLS_ROOT / "wti-strategy-trained", + # Volatility regime + vol_regime_bands=((15.0, "low"), (30.0, "medium"), (50.0, "elevated")), + # Tool bounds + frequency="B", + horizons=(5, 10, 21), + shock_threshold=SHOCK_THRESHOLD, + shock_horizon=SHOCK_HORIZON, + num_samples=200, +) diff --git a/implementations/energy_oil_forecasting/starter_agent/agent.py b/implementations/energy_oil_forecasting/starter_agent/agent.py index 2f554617..257a1204 100644 --- a/implementations/energy_oil_forecasting/starter_agent/agent.py +++ b/implementations/energy_oil_forecasting/starter_agent/agent.py @@ -41,11 +41,13 @@ CodeExecutionConfig, ContextRetrievalConfig, ) +from aieng.forecasting.methods.agentic.domain import render_starter_instruction from aieng.forecasting.models import LITE_MODEL # Reuse the existing WTI prompt builder + history compression — these serialise # the task/context into the agent's JSON payload and are not worth duplicating. from energy_oil_forecasting.analyst_agent import WtiPriceForecastPromptBuilder +from energy_oil_forecasting.domain import OIL_DOMAIN from energy_oil_forecasting.starter_agent.tools import ToolSpec, news_search @@ -60,32 +62,14 @@ # --------------------------------------------------------------------------- -def _build_starter_instruction() -> str: - """Build the task-agnostic, skill-agnostic starter persona. - - Just the analyst's identity and how to behave — no output schema, no payload - contract, no skill or tool mechanics. ADK injects the name + description of - every attached skill (and every tool) into the system prompt, so the agent - already knows what it can load and call; repeating that here would only - duplicate dynamically-injected information. The forecasting *contract* lives - in the loadable ``forecasting`` skill. Edit the persona freely. - """ - return ( - "## Role\n\n" - "You are a WTI crude oil market analyst — fluent in supply/demand " - "fundamentals, OPEC+ policy, geopolitical and shipping-lane risk, and " - "price dynamics. This is a starter agent: keep your reasoning " - "transparent and your claims honest.\n\n" - "## How to respond\n\n" - "- For open-ended questions, scenario analysis, or anything " - "conversational, answer directly and concisely — do NOT ask for a JSON " - "payload.\n" - "- When you are handed a task that asks for a structured probabilistic " - "forecast, produce a calibrated one." - ) - - -_STARTER_INSTRUCTION = _build_starter_instruction() +# The task-agnostic, skill-agnostic starter persona — just the analyst's +# identity and how to behave. No output schema, no payload contract, no skill or +# tool mechanics: ADK injects the name + description of every attached skill (and +# every tool) into the system prompt, so the agent already knows what it can load +# and call. The forecasting *contract* lives in the loadable ``forecasting`` +# skill. The persona is rendered from the WTI ``OIL_DOMAIN`` fragments; edit it +# by supplying a different ``DomainConfig``. +_STARTER_INSTRUCTION = render_starter_instruction(OIL_DOMAIN) # --------------------------------------------------------------------------- diff --git a/implementations/energy_oil_forecasting/tasks.py b/implementations/energy_oil_forecasting/tasks.py index de12da51..75cae4f6 100644 --- a/implementations/energy_oil_forecasting/tasks.py +++ b/implementations/energy_oil_forecasting/tasks.py @@ -3,17 +3,20 @@ Implements the "one agent, three tasks" pattern: a single :class:`AgentConfig` identity with task-specific prompt builders and output schemas supplied via :class:`~aieng.forecasting.methods.agentic.predictor.AgentPredictor`. + +The scenario output classes are thin oil-branded subclasses of the shared, +domain-agnostic :class:`~aieng.forecasting.methods.agentic.outputs.ScenarioCard` +and :class:`~aieng.forecasting.methods.agentic.outputs.ScenarioAgentForecastOutput`. +They pin the numeric scenario-card fields to the exact key names +(``wti_range_60d``, ``point_estimate_60d``) the notebook caches key on. """ from __future__ import annotations import json -from datetime import datetime from typing import Any, ClassVar, Literal -import pandas as pd from aieng.forecasting.data.context import ForecastContext -from aieng.forecasting.evaluation.prediction import BinaryForecast, Prediction from aieng.forecasting.evaluation.task import ForecastingTask from aieng.forecasting.methods.agentic import ( AgentPredictor, @@ -21,7 +24,15 @@ DiscreteAgentForecastOutput, ) from aieng.forecasting.methods.agentic.agent_factory import AgentConfig -from aieng.forecasting.methods.agentic.outputs import AgentForecastOutput +from aieng.forecasting.methods.agentic.outputs import ( + AgentForecastOutput, +) +from aieng.forecasting.methods.agentic.outputs import ( + ScenarioAgentForecastOutput as _BaseScenarioAgentForecastOutput, +) +from aieng.forecasting.methods.agentic.outputs import ( + ScenarioCard as _BaseScenarioCard, +) from aieng.forecasting.models import LITE_MODEL from energy_oil_forecasting.analyst_agent import ( WtiPriceForecastPromptBuilder, @@ -30,7 +41,7 @@ compress_history, ) from energy_oil_forecasting.paths import SHOCK_HORIZON, SHOCK_THRESHOLD -from pydantic import BaseModel, Field +from pydantic import BaseModel # ── Task specification strings (embedded in user prompts for NB3) ─────────── @@ -67,89 +78,31 @@ def __call__(self, *, task: ForecastingTask, context: ForecastContext) -> str: return json.dumps(payload, indent=2) -class ScenarioCard(BaseModel): - """One scenario card from Task C agent output.""" +class ScenarioCard(_BaseScenarioCard): + """One scenario card from Task C agent output. - model_config = {"extra": "ignore"} + Adds the WTI-specific 60-day price range and point estimate to the generic + scenario card, under the exact field names the notebook caches expect. + """ - name: str - description: str - probability: float = Field(ge=0.0, le=1.0) wti_range_60d: list[float] point_estimate_60d: float - key_drivers: list[str] = Field(default_factory=list) -class ScenarioAgentForecastOutput(AgentForecastOutput): - """Track 2 scenario analysis output for the energy case study.""" +class ScenarioAgentForecastOutput(_BaseScenarioAgentForecastOutput): + """Track 2 scenario analysis output for the energy case study. - modality: ClassVar[Literal["continuous", "discrete"]] = "discrete" - - model_config = {"extra": "ignore"} + Narrows the scenario-card type to :class:`ScenarioCard` and advertises the + WTI numeric fields in the prompt schema template. Rendering and + :meth:`to_predictions` are inherited unchanged from the shared base. + """ scenarios: list[ScenarioCard] - base_case: str - reasoning: str = "" - - @classmethod - def prompt_schema_json(cls) -> str: - """Return a JSON template for use in agent instruction strings. - - Returns - ------- - str - Indented JSON string showing the exact structure the agent must - pass to ``set_model_response``. - """ - template: dict[str, object] = { - "scenarios": [ - { - "name": "", - "description": "", - "probability": "", - "wti_range_60d": ["", ""], - "point_estimate_60d": "", - "key_drivers": ["", ""], - } - ], - "base_case": "", - "reasoning": "", - } - return json.dumps(template, indent=2) - - def to_predictions( - self, - *, - task: ForecastingTask, - context: ForecastContext, - predictor_id: str, - metadata: dict[str, Any] | None = None, - ) -> list[Prediction]: - """Convert scenario output to a metadata-rich prediction (Track 2 display).""" - if len(task.horizons) != 1: - raise ValueError("Scenario agent output expects exactly one task horizon.") - - horizon = task.horizons[0] - issued_at = datetime.utcnow() - offset = pd.tseries.frequencies.to_offset(task.frequency) - base_prob = float(sum(s.probability for s in self.scenarios)) - prediction_metadata: dict[str, Any] = dict(metadata) if metadata is not None else {} - prediction_metadata["scenarios"] = [s.model_dump() for s in self.scenarios] - prediction_metadata["base_case"] = self.base_case - if self.reasoning.strip(): - prediction_metadata["rationale"] = self.reasoning - - return [ - Prediction( - predictor_id=predictor_id, - task_id=task.task_id, - issued_at=issued_at, - as_of=context.as_of, - forecast_date=(pd.Timestamp(context.as_of) + offset * horizon).to_pydatetime(), - payload=BinaryForecast(probability=min(base_prob, 1.0)), - metadata=prediction_metadata, - ) - ] + + scenario_card_template_extra: ClassVar[dict[str, object]] = { + "wti_range_60d": ["", ""], + "point_estimate_60d": "", + } # Task specification strings embedded in user prompts for NB3. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic____init__.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic____init__.py.md index 167d8280..75645755 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic____init__.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic____init__.py.md @@ -65,6 +65,7 @@ Building a predictor from a config:: """ from aieng.forecasting.methods.agentic.adaptive_skill import AdaptiveSkillState, AdaptiveSkillStore +from aieng.forecasting.methods.agentic.adaptive_skill_tools import build_skill_tools from aieng.forecasting.methods.agentic.adk_runner import AdkTextRunner, AdkTextRunnerConfig from aieng.forecasting.methods.agentic.agent_factory import ( AgentConfig, @@ -77,7 +78,13 @@ from aieng.forecasting.methods.agentic.curriculum import ( format_backtest_report, load_context_documents, ) +from aieng.forecasting.methods.agentic.domain import ( + DomainConfig, + build_adaptive_config, + build_analyst_config, +) from aieng.forecasting.methods.agentic.forecast_tool import ForecastTool +from aieng.forecasting.methods.agentic.history import compress_history from aieng.forecasting.methods.agentic.outputs import ( AgentCategoryProbability, AgentForecastOutput, @@ -86,8 +93,17 @@ from aieng.forecasting.methods.agentic.outputs import ( ContinuousAgentForecastOutput, ContinuousAgentHorizonForecast, DiscreteAgentForecastOutput, + ScenarioAgentForecastOutput, + ScenarioCard, ) from aieng.forecasting.methods.agentic.predictor import AgentPredictor, ForecastPromptBuilder +from aieng.forecasting.methods.agentic.strategy_state import ( + CalibrationCorrection, + Hypothesis, + Observation, + StrategyState, + VersionEntry, +) __all__: list[str] = [ @@ -100,16 +116,28 @@ __all__: list[str] = [ "AgentForecastOutput", "AgentPredictor", "AgentQuantileForecast", + "CalibrationCorrection", "CategoricalAgentForecastOutput", "CodeExecutionConfig", "ContinuousAgentForecastOutput", "ContinuousAgentHorizonForecast", "ContextRetrievalConfig", "DiscreteAgentForecastOutput", + "DomainConfig", "ForecastPromptBuilder", "ForecastTool", + "Hypothesis", + "Observation", + "ScenarioAgentForecastOutput", + "ScenarioCard", + "StrategyState", + "VersionEntry", + "build_adaptive_config", "build_adk_agent", + "build_analyst_config", "build_curriculum_prompt", + "build_skill_tools", + "compress_history", "format_backtest_report", "load_context_documents", ] diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adaptive_skill_tools.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adaptive_skill_tools.py.md new file mode 100644 index 00000000..0e6f0fda --- /dev/null +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adaptive_skill_tools.py.md @@ -0,0 +1,405 @@ +# Source: aieng-forecasting/aieng/forecasting/methods/agentic/adaptive_skill_tools.py + +kind: python + +```python +"""Generic strategy-mutation tools for adaptive forecasting agents. + +These are plain Python callables registered as ADK ``FunctionTool`` objects via +``AgentConfig(extra_tools=build_skill_tools(strategy_dir, state_type))``. They +run in the host process — *not* inside a code-execution sandbox — so they can +read and write the skill directory on the local filesystem. + +Factory pattern +--------------- +Use :func:`build_skill_tools` to create a set of tools bound to a specific +strategy directory and :class:`StrategyState` subclass. This lets multiple +named strategy variants coexist, each with its own ``AdaptiveSkillStore`` and +``skill_state.yaml``. The factory holds **no import-time state**: nothing is +instantiated until it is called. + +Design principles +----------------- +Each tool follows the same three-step cycle: + +1. ``store.load()`` — deserialise current state from ``skill_state.yaml``. +2. Apply one typed mutation to the state model. +3. ``store.save(state)`` — write YAML, re-render ``SKILL.md``, back up. + +Tool signatures are intentionally narrow: they accept only the arguments needed +for one specific mutation. The agent cannot write arbitrary content to the +skill directory through any of these tools. + +Evidence governance +------------------- +``record_observation`` + No guard. Record any pattern-level finding (not a single-outlier surprise). + +``open_hypothesis`` + No guard. Open a hypothesis whenever you suspect a durable pattern. + +``record_hypothesis_outcome`` + Validates the hypothesis ID exists and is still open. + +``graduate_hypothesis`` + Hard guard: rejected if ``hypothesis.confirmations < store.confirmation_threshold``. + Returns a clear message stating the shortfall. + +``update_approach_narrative`` + Requires a ``rationale`` argument but no hard numeric guard. + +Scope guard +----------- +All writes go through the ``AdaptiveSkillStore`` instance passed to +:func:`build_skill_tools`. No path outside that directory can be reached. +""" + +from __future__ import annotations + +from datetime import date +from pathlib import Path +from typing import Callable + +from aieng.forecasting.methods.agentic.adaptive_skill import AdaptiveSkillStore +from aieng.forecasting.methods.agentic.strategy_state import ( + CalibrationCorrection, + Hypothesis, + Observation, + StrategyState, + VersionEntry, +) + + +# --------------------------------------------------------------------------- +# Stateless helpers +# --------------------------------------------------------------------------- + + +def _today() -> str: + return str(date.today()) + + +def _next_hypothesis_id(state: StrategyState) -> str: + """Return the next sequential hypothesis ID (e.g. ``hyp-004``).""" + n = len(state.hypotheses) + 1 + return f"hyp-{n:03d}" + + +# --------------------------------------------------------------------------- +# Tool factory +# --------------------------------------------------------------------------- + + +def build_skill_tools( # noqa: PLR0915 + strategy_dir: Path, + state_type: type[StrategyState] = StrategyState, + *, + confirmation_threshold: int = 3, +) -> list[Callable[..., str]]: + """Build a set of strategy mutation tools bound to *strategy_dir*. + + Each call returns five fresh callables (closures over a new + :class:`~aieng.forecasting.methods.agentic.adaptive_skill.AdaptiveSkillStore` + instance). Pass the returned list to + ``AgentConfig(extra_tools=build_skill_tools(strategy_dir, state_type))`` to + wire the tools into an agent that operates on a specific strategy variant. + + Parameters + ---------- + strategy_dir : Path + Directory containing the strategy skill (``skill_state.yaml``, + ``SKILL.md``, ``.history/``). Must exist and be a directory. + state_type : type[StrategyState], default=StrategyState + Concrete strategy-state class used to deserialise ``skill_state.yaml``. + Pass a domain subclass (e.g. ``WtiStrategyState``) to preserve its + rendered ``SKILL.md`` branding. + confirmation_threshold : int, default=3 + Number of confirming hypothesis outcomes required before + ``graduate_hypothesis`` is permitted. + + Returns + ------- + list[Callable[..., str]] + ``[record_observation, open_hypothesis, record_hypothesis_outcome, + graduate_hypothesis, update_approach_narrative]`` + """ + store: AdaptiveSkillStore[StrategyState] = AdaptiveSkillStore( + skill_dir=strategy_dir, + state_type=state_type, + confirmation_threshold=confirmation_threshold, + ) + + def record_observation(finding: str, linked_hypothesis: str = "") -> str: + """Record a pattern-level finding from a resolution or self-review. + + Call this whenever you observe a systematic pattern across multiple + forecasts — not after a single surprising outcome. + + Parameters + ---------- + finding : str + A concise description of the pattern observed. Be specific: include + the regime, horizon, and direction of the error where applicable. + Example: "80% intervals missed 4 of 5 actuals in the elevated vol + regime at the 21-day horizon." + linked_hypothesis : str, optional + ID of an existing open hypothesis this observation supports or + refutes (e.g. ``"hyp-001"``). Leave blank if this is a fresh + observation not yet linked to any hypothesis. + + Returns + ------- + str + Confirmation message. + """ + state = store.load() + obs = Observation( + date=_today(), + finding=finding.strip(), + linked_hypothesis=linked_hypothesis.strip() or None, + ) + state.observations.append(obs) + store.save(state) + linked_note = f" (linked to {obs.linked_hypothesis})" if obs.linked_hypothesis else "" + return f'Observation recorded{linked_note}: "{finding[:80]}{"..." if len(finding) > 80 else ""}"' + + def open_hypothesis(claim: str, initial_evidence: str) -> str: + """Open a new hypothesis about a suspected systematic forecasting pattern. + + A hypothesis is a candidate calibration correction under active testing. + Open one when you have at least one observation suggesting a durable + pattern but do not yet have enough confirming resolutions to graduate it. + + Parameters + ---------- + claim : str + A testable claim about your forecasting behaviour. State it in terms + of a specific condition and a directional error. + Example: "My 80% prediction intervals are consistently too narrow + when the vol regime is classified as elevated or extreme." + initial_evidence : str + The observation(s) that motivated opening this hypothesis. This is + for the audit record — be specific about the number of data points. + + Returns + ------- + str + Confirmation message including the assigned hypothesis ID. + """ + state = store.load() + hyp_id = _next_hypothesis_id(state) + hyp = Hypothesis( + id=hyp_id, + claim=claim.strip(), + status="open", + confirmations=0, + refutations=0, + opened_on=_today(), + ) + state.hypotheses.append(hyp) + obs = Observation( + date=_today(), + finding=initial_evidence.strip(), + linked_hypothesis=hyp_id, + ) + state.observations.append(obs) + store.save(state) + return ( + f'Hypothesis {hyp_id} opened: "{claim[:80]}{"..." if len(claim) > 80 else ""}". ' + f"Initial evidence recorded as an observation linked to {hyp_id}. " + f"Confirmations needed to graduate: {store.confirmation_threshold}." + ) + + def record_hypothesis_outcome(hypothesis_id: str, outcome: str) -> str: + """Record a confirming or refuting outcome for an open hypothesis. + + Call this after each resolution where the outcome is directly relevant + to an open hypothesis. Accumulate enough confirmations to graduate. + + Parameters + ---------- + hypothesis_id : str + ID of the hypothesis to update (e.g. ``"hyp-001"``). + outcome : str + Either ``"confirmed"`` or ``"refuted"``. A single refutation does + not automatically close the hypothesis — continue accumulating + evidence. A hypothesis should be manually closed (status → + ``"refuted"``) only when refutations clearly outweigh confirmations + across a meaningful sample. + + Returns + ------- + str + Updated confirmation / refutation counts and progress toward the + graduation threshold. + """ + if outcome not in ("confirmed", "refuted"): + return f"Invalid outcome '{outcome}'. Must be 'confirmed' or 'refuted'." + + state = store.load() + hyp = next((h for h in state.hypotheses if h.id == hypothesis_id), None) + if hyp is None: + ids = [h.id for h in state.hypotheses] + return f"Hypothesis '{hypothesis_id}' not found. Known IDs: {ids}." + if hyp.status != "open": + return f"Hypothesis {hypothesis_id} is already {hyp.status}. Only open hypotheses can receive new outcomes." + + if outcome == "confirmed": + hyp.confirmations += 1 + else: + hyp.refutations += 1 + + store.save(state) + + remaining = max(0, store.confirmation_threshold - hyp.confirmations) + if remaining == 0: + ready_msg = ( + f" Ready to graduate — call graduate_hypothesis('{hypothesis_id}', ...) " + "with a condition, adjustment, and horizon_scope." + ) + else: + ready_msg = f" {remaining} more confirmation(s) needed to graduate." + + return ( + f"{hypothesis_id} updated: {hyp.confirmations} confirmation(s), {hyp.refutations} refutation(s).{ready_msg}" + ) + + def graduate_hypothesis( + hypothesis_id: str, + condition: str, + adjustment: str, + horizon_scope: str, + ) -> str: + """Graduate a confirmed hypothesis to an active calibration correction. + + This is the primary mechanism through which the agent's strategy + improves. A calibration correction is applied at every future + prediction; it is not merely recorded — it changes behaviour. + + This tool enforces the confirmation threshold: it will reject the call + if the hypothesis has not accumulated enough confirming outcomes. + + Parameters + ---------- + hypothesis_id : str + ID of the confirmed hypothesis to graduate (e.g. ``"hyp-001"``). + condition : str + The specific condition under which this correction applies. + Example: "vol regime is elevated or extreme". + adjustment : str + The concrete adjustment to make when the condition is met. + Example: "Widen 80% CI by 12% relative to the statistical model + output." + horizon_scope : str + Which horizons this correction applies to. + One of: ``"all"``, ``"5bd"``, ``"10bd"``, ``"21bd"``, or a + combination like ``"10bd and 21bd"``. + + Returns + ------- + str + Confirmation message, or a rejection message with the shortfall. + """ + state = store.load() + hyp = next((h for h in state.hypotheses if h.id == hypothesis_id), None) + if hyp is None: + ids = [h.id for h in state.hypotheses] + return f"Hypothesis '{hypothesis_id}' not found. Known IDs: {ids}." + if hyp.status != "open": + return f"Hypothesis {hypothesis_id} is already {hyp.status}. Only open hypotheses can be graduated." + + if hyp.confirmations < store.confirmation_threshold: + shortfall = store.confirmation_threshold - hyp.confirmations + return ( + f"Cannot graduate {hypothesis_id}: " + f"{hyp.confirmations} confirmation(s), " + f"requires {store.confirmation_threshold}. " + f"Record {shortfall} more confirming outcome(s) first." + ) + + today = _today() + hyp.status = "confirmed" + correction = CalibrationCorrection( + condition=condition.strip(), + adjustment=adjustment.strip(), + horizon_scope=horizon_scope.strip(), + source_hypothesis=hypothesis_id, + confirmed_on=today, + ) + state.calibration_corrections.append(correction) + state.observations.append( + Observation( + date=today, + finding=( + f"Graduated {hypothesis_id} to calibration correction: " + f"'{condition}' → '{adjustment}' ({horizon_scope})." + ), + linked_hypothesis=hypothesis_id, + ) + ) + state.version_history.append( + VersionEntry( + date=today, + description=( + f"Graduated {hypothesis_id} to calibration correction " + f"(condition: {condition[:50]}{'...' if len(condition) > 50 else ''})." + ), + ) + ) + store.save(state) + return ( + f"Hypothesis {hypothesis_id} confirmed and graduated. " + f"Calibration correction added: when '{condition}', apply '{adjustment}' " + f"(scope: {horizon_scope})." + ) + + def update_approach_narrative(new_text: str, rationale: str) -> str: + """Replace the approach narrative with an updated strategic description. + + This is the highest-evidence-bar update. Consult ``meta-learning`` + before calling this tool — the narrative should only change when the + calibration record reveals a structural insight that the current + description no longer captures. A ``rationale`` argument is required + to force articulation of why the change is warranted. + + Parameters + ---------- + new_text : str + The complete replacement text for the ``## Approach`` section. + Write it as a self-contained description of the current forecasting + strategy — what signals are used, in what order, and with what + emphasis. + rationale : str + Why this update is warranted now. Cite the specific calibration + corrections or pattern of observations that motivated the change. + + Returns + ------- + str + Confirmation message. + """ + if not new_text.strip(): + return "new_text must not be empty." + if not rationale.strip(): + return "rationale must not be empty. Explain why the approach narrative warrants an update." + + state = store.load() + today = _today() + state.approach_narrative = new_text.strip() + state.version_history.append( + VersionEntry( + date=today, + description=f"Updated approach narrative. Rationale: {rationale[:120]}{'...' if len(rationale) > 120 else ''}", + ) + ) + store.save(state) + return f"Approach narrative updated ({len(new_text)} chars). Rationale recorded in version history." + + return [ + record_observation, + open_hypothesis, + record_hypothesis_outcome, + graduate_hypothesis, + update_approach_narrative, + ] +``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md index 6337eb98..f1a37153 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md @@ -187,7 +187,13 @@ class AdkTextRunner: """Underlying ADK runner (session, artifact, memory services).""" return self._runner - async def _resolve_session_id(self, user_id: str | None, session_id: str | None) -> str: + async def _resolve_session_id( + self, + user_id: str | None, + session_id: str | None, + *, + initial_state: dict[str, Any] | None = None, + ) -> str: """Return the ADK session id to use for a single turn. Parameters @@ -197,6 +203,11 @@ class AdkTextRunner: session_id : str or None Explicit session id from the caller. ``None`` triggers sticky-session lookup or new-session creation depending on ``fresh_session_per_message``. + initial_state : dict[str, Any] or None + Seeded into a newly-created session's state. Only takes effect when + this call actually creates a session — has no effect when an + existing sticky session (``fresh_session_per_message=False``) is + reused, since session state can only be seeded at creation. Returns ------- @@ -210,6 +221,7 @@ class AdkTextRunner: new_session = await self._runner.session_service.create_session( app_name=self.config.app_name, user_id=user_id, + state=initial_state, ) sid = new_session.id elif session_id is not None: @@ -221,6 +233,7 @@ class AdkTextRunner: new_session = await self._runner.session_service.create_session( app_name=self.config.app_name, user_id=user_id, + state=initial_state, ) sid = new_session.id self._conversation_session_by_user[user_id] = sid @@ -234,6 +247,7 @@ class AdkTextRunner: user_id: str | None = None, session_id: str | None = None, run_config: RunConfig | None = None, + initial_state: dict[str, Any] | None = None, ) -> str: """Run one user turn; return the first final model text or an empty string. @@ -252,6 +266,12 @@ class AdkTextRunner: run_config : RunConfig | None, optional The run configuration to use for the run. If not provided, the default run configuration is used. + initial_state : dict[str, Any] | None, optional + Seeded into the session's state when this call creates a new + session (see :meth:`_resolve_session_id`). Use this to pass + harness-controlled values (e.g. a forecast's ``as_of`` date) that + tools can read via ``ToolContext.state`` without the LLM being + able to see or influence them. Returns ------- @@ -275,7 +295,7 @@ class AdkTextRunner: user_id = user_id or self.config.default_user_id - session_id = await self._resolve_session_id(user_id, session_id) + session_id = await self._resolve_session_id(user_id, session_id, initial_state=initial_state) content = genai_types.Content(role="user", parts=[genai_types.Part(text=prompt)]) diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md index 494c0ed4..db6a0513 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md @@ -17,6 +17,7 @@ raises :class:`ImportError` with installation guidance. from __future__ import annotations +import json import logging import os import warnings @@ -24,9 +25,12 @@ from pathlib import Path from typing import Any, Callable, Sequence from aieng.forecasting.methods.agentic.outputs import AgentForecastOutput -from aieng.forecasting.models import LITE_MODEL +from aieng.forecasting.models import ADVANCED_MODEL, LITE_MODEL from google.adk.models.base_llm import BaseLlm -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator + + +logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -80,6 +84,13 @@ except ModuleNotFoundError as exc: # final text, giving the predictor the structured JSON it expects. SMR_STATE_KEY = "__smr_output__" +# Session-state key AgentPredictor seeds with the current prediction's as_of +# date before each run (see AdkTextRunner.run_text_async's initial_state). +# search_web reads this via its ADK-injected ToolContext as the authoritative +# cutoff — unlike the LLM-supplied cutoff_date argument, the model can never +# see or influence this key, so it can't be silently omitted or spoofed. +AS_OF_STATE_KEY = "__as_of__" + def _build_set_model_response_tool() -> FunctionTool: """Return a proxy-compatible ``set_model_response`` shim. @@ -118,12 +129,18 @@ class ContextRetrievalConfig(BaseModel): the calling agent can retrieve grounded, sourced web context without a direct Gemini API key. - Temporal cutoff enforcement is soft (LLM-judgment-based): when - ``enforce_cutoff`` is ``True`` and the calling agent passes a - ``cutoff_date`` to the tool, the inner proxy prompt explicitly asks the - model to exclude post-cutoff sources. This is the same trust model used - by the prior Google Search sub-agent — backtest leakage is a - pedagogically useful discussion point, not a hard guarantee. + Temporal cutoff enforcement has two layers. The first is soft + (LLM-judgment-based): when ``enforce_cutoff`` is ``True`` and the calling + agent passes a ``cutoff_date`` to the tool, the inner proxy prompt + explicitly asks the model to exclude post-cutoff sources. This alone is + not a hard guarantee — backtests have shown it leak real post-cutoff + information despite the instruction. The second, hard layer is an + independent verifier call (see ``verifier_model`` etc. below): a separate + LLM call extracts and judges each factual claim in the search result + against the cutoff, strips violations, and retries the search with + feedback when it cannot produce a sufficiently confident result — + returning an explicit failure sentinel rather than silently risky + content if verification never succeeds within the attempt budget. Attributes ---------- @@ -139,12 +156,27 @@ class ContextRetrievalConfig(BaseModel): enforce_cutoff : bool, default=True When ``True``, the ``search_web`` tool appends a cutoff-date constraint to the user prompt whenever ``cutoff_date`` is supplied by - the calling agent. Set to ``False`` for live (non-backtest) agents - where no temporal fence is needed. + the calling agent, and runs the independent leakage verifier + described above. Set to ``False`` for live (non-backtest) agents + where no temporal fence is needed — the verifier is skipped entirely + in that case, at zero extra cost. temperature : float | None, default=None Sampling temperature for the inner search call. max_output_tokens : int | None, default=None Maximum output tokens for the inner search call. + verifier_model : str, default=ADVANCED_MODEL (``"gemini-3.5-flash"``) + Model used for the independent leakage-verification call. Defaults + to a different model than ``search_model`` so the verifier does not + share the same blind spot as the call it's checking. + verifier_max_attempts : int, default=3 + Maximum number of search-then-verify attempts before giving up and + returning the ``[SEARCH_VERIFICATION_FAILED]`` sentinel. + verifier_confidence_threshold : int, default=8 + Minimum self-reported confidence (1-10) the verifier must report, + alongside a clean verdict, for a result to be accepted. Kept + configurable rather than hardcoded because LLM self-reported + confidence is not well-calibrated: too strict a bar (e.g. a literal + 10) risks exhausting retries on results that were actually fine. """ model_config = {"extra": "forbid"} @@ -159,6 +191,9 @@ class ContextRetrievalConfig(BaseModel): enforce_cutoff: bool = True temperature: float | None = Field(default=None, ge=0.0, le=2.0) max_output_tokens: int | None = Field(default=None, ge=1) + verifier_model: str = ADVANCED_MODEL + verifier_max_attempts: int = Field(default=3, ge=1) + verifier_confidence_threshold: int = Field(default=8, ge=1, le=10) class CodeExecutionConfig(BaseModel): @@ -212,6 +247,106 @@ def _build_automatic_function_calling_config( return AutomaticFunctionCallingConfig(disable=True) +class _LeakageVerification(BaseModel): + """Structured verdict from the independent temporal-leakage verifier.""" + + flagged_claims: list[str] = Field(default_factory=list) + filtered_text: str = "" + confidence: int = Field(ge=1, le=10) + clean: bool + + +def _build_leakage_verification_schema() -> dict[str, Any]: + """Strict JSON schema for the verifier's structured output. + + Field order matters: the model must extract/flag claims and produce + ``filtered_text`` before declaring ``confidence``/``clean``, so the + verdict follows the claim-level reasoning instead of preceding it. + """ + return { + "type": "object", + "properties": { + "flagged_claims": {"type": "array", "items": {"type": "string"}}, + "filtered_text": {"type": "string"}, + "confidence": {"type": "integer", "minimum": 1, "maximum": 10}, + "clean": {"type": "boolean"}, + }, + "required": ["flagged_claims", "filtered_text", "confidence", "clean"], + } + + +_LEAKAGE_VERIFIER_INSTRUCTION = """\ +You are an independent fact-checker verifying that a web search result contains \ +no information published on or after a given cutoff date. + +Extract every discrete factual claim relevant to the query. For each claim, \ +judge whether its *content* (the event itself, prices/figures referenced, \ +described developments) could only be known on or after the cutoff date. \ +Do NOT trust a source's claimed publish date, byline timestamp, or URL — \ +page metadata and timestamps are frequently updated after original \ +publication and are not reliable evidence of when the underlying facts \ +became known. Reason from the substance of the claim itself. + +Remove every claim that fails this test and produce `filtered_text`: the \ +original text with only the surviving, pre-cutoff claims. Report the removed \ +claims in `flagged_claims`. Set `confidence` (1-10) to how confident you are \ +that `filtered_text` now contains zero post-cutoff leakage. Set `clean` to \ +true only if you removed all identifiable violations.""" + + +async def _verify_no_leakage( + *, + text: str, + query: str, + cutoff_date: str, + verifier_model: str, + openai_base_url: str, + openai_api_key: str | None, +) -> _LeakageVerification: + """Judge a search result for post-cutoff claims via an independent verifier call. + + Uses a different model (by default) than the search call it's checking, + so the verifier does not share the same knowledge-attribution blind spot + that caused the leak in the first place. Never raises on a malformed + verifier response — a parse failure is treated as a non-clean verdict so + it consumes a retry attempt like any other rejection. + """ + import litellm # noqa: PLC0415 + from aieng.forecasting.methods.llm_processes._client import ( # noqa: PLC0415 + make_json_schema_response_format, + strip_markdown_fence, + ) + + model = verifier_model if verifier_model.startswith("openai/") else f"openai/{verifier_model}" + resp = await litellm.acompletion( + model=model, + api_base=openai_base_url, + api_key=openai_api_key, + messages=[ + {"role": "system", "content": _LEAKAGE_VERIFIER_INSTRUCTION}, + { + "role": "user", + "content": f"Original query: {query}\nCutoff date: {cutoff_date}\n\nSearch result to verify:\n{text}", + }, + ], + response_format=make_json_schema_response_format("LeakageVerification", _build_leakage_verification_schema()), + temperature=0.0, + max_tokens=2048, + timeout=60.0, + ) + raw = resp.choices[0].message.content or "{}" + try: + return _LeakageVerification.model_validate(json.loads(strip_markdown_fence(raw))) + except (json.JSONDecodeError, ValidationError): + logger.warning("Leakage verifier returned unparseable output; treating as non-clean: %r", raw[:200]) + return _LeakageVerification( + flagged_claims=["verifier response could not be parsed"], + filtered_text=text, + confidence=1, + clean=False, + ) + + def _build_search_tool( config: ContextRetrievalConfig, *, @@ -224,25 +359,25 @@ def _build_search_tool( the proxy with ``"tools": [{"googleSearch": {}}]`` so the model does server-side grounding and returns a synthesised answer plus source URLs extracted from ``choices[0].provider_specific_fields["grounding_metadata"]``. - """ - async def search_web(query: str, cutoff_date: str | None = None) -> str: - """Search the web and return a grounded summary with source URLs. + When a ``cutoff_date`` is supplied and ``config.enforce_cutoff`` is + ``True``, the raw result is passed through an independent leakage + verifier (:func:`_verify_no_leakage`) before being returned. On a flagged + result, the search is retried (up to ``config.verifier_max_attempts`` + times) with the previously flagged claims injected as explicit negative + feedback. If no attempt is verified clean, an explicit + ``[SEARCH_VERIFICATION_FAILED]`` sentinel is returned instead of + potentially-leaky content. + """ - Args: - query: What to search for. - cutoff_date: ISO date (YYYY-MM-DD). When provided, only include - information published strictly before this date. + def _format_result(content: str, sources: list[str]) -> str: + if sources: + content += "\n\nSources:\n" + "\n".join(sources[:5]) + return content - Returns - ------- - A grounded summary of search results, with source URLs appended. - """ + async def _do_search(user_content: str) -> tuple[str, list[str]]: import litellm # noqa: PLC0415 - user_content = query - if cutoff_date and config.enforce_cutoff: - user_content += f"\n\nOnly include and cite information published strictly before {cutoff_date}." search_model = config.search_model if not search_model.startswith("openai/"): search_model = f"openai/{search_model}" @@ -265,9 +400,88 @@ def _build_search_tool( sources: list[str] = [ uri for c in gm.get("groundingChunks", []) if (uri := (c.get("web") or {}).get("uri")) is not None ] - if sources: - content += "\n\nSources:\n" + "\n".join(sources[:5]) - return content + return content, sources + + async def search_web(query: str, cutoff_date: str | None = None, tool_context: ToolContext | None = None) -> str: + """Search the web and return a grounded summary with source URLs. + + Args: + query: What to search for. + cutoff_date: ISO date (YYYY-MM-DD). When provided, only include + information published strictly before this date. + + Returns + ------- + A grounded summary of search results, with source URLs appended. + When cutoff verification is enabled and cannot be satisfied + within the attempt budget, returns a ``[SEARCH_VERIFICATION_FAILED]`` + sentinel instead of unverified content. + + Notes + ----- + ``tool_context`` is not part of the LLM-visible tool schema — ADK + injects it automatically because of its type annotation. When + :class:`AgentPredictor` runs this tool, it has already seeded the + session with the current prediction's ``as_of`` date under + :data:`AS_OF_STATE_KEY`; that harness-controlled value is the + authoritative cutoff whenever present, since (unlike ``cutoff_date``) + the calling LLM cannot see, omit, or alter it. This closes a bypass + where the model simply didn't pass ``cutoff_date`` and both the soft + cutoff instruction and the verifier below were silently skipped. + """ + harness_as_of = tool_context.state.get(AS_OF_STATE_KEY) if tool_context is not None else None + if harness_as_of and cutoff_date and harness_as_of != cutoff_date: + logger.warning( + "search_web: cutoff_date=%r disagrees with harness as_of=%r; using the harness value.", + cutoff_date, + harness_as_of, + ) + effective_cutoff = harness_as_of or cutoff_date + + needs_verification = bool(effective_cutoff and config.enforce_cutoff) + if not needs_verification: + content, sources = await _do_search(query) + return _format_result(content, sources) + + negative_guidance = "" + for attempt in range(1, config.verifier_max_attempts + 1): + user_content = ( + query + f"\n\nOnly include and cite information published strictly before {effective_cutoff}." + ) + if negative_guidance: + user_content += f"\n\n{negative_guidance}" + content, sources = await _do_search(user_content) + verdict = await _verify_no_leakage( + text=content, + query=query, + cutoff_date=effective_cutoff, # type: ignore[arg-type] + verifier_model=config.verifier_model, + openai_base_url=openai_base_url, + openai_api_key=openai_api_key, + ) + logger.info( + "search_web verification attempt %d/%d: clean=%s confidence=%d flagged=%d", + attempt, + config.verifier_max_attempts, + verdict.clean, + verdict.confidence, + len(verdict.flagged_claims), + ) + if verdict.clean and verdict.confidence >= config.verifier_confidence_threshold: + return _format_result(verdict.filtered_text, sources) + logger.warning("search_web attempt %d flagged %d claim(s); retrying.", attempt, len(verdict.flagged_claims)) + negative_guidance = ( + f"Your previous search result may have included information published on or after " + f"{effective_cutoff}. Do not repeat or rely on these claims:\n- " + + "\n- ".join(verdict.flagged_claims or ["(unspecified — be more conservative)"]) + ) + + logger.error("search_web exhausted %d attempts without a verified clean result.", config.verifier_max_attempts) + return ( + f"[SEARCH_VERIFICATION_FAILED] Could not verify search results as free of information " + f"published on or after {effective_cutoff} after {config.verifier_max_attempts} attempts. " + "Treat this as no verified news context being available for this query." + ) return search_web diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__curriculum.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__curriculum.py.md index 402af8c0..77c68803 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__curriculum.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__curriculum.py.md @@ -69,12 +69,17 @@ logger = logging.getLogger(__name__) # Vol-regime helper # --------------------------------------------------------------------------- -_VOL_REGIMES = [ +#: Default (upper-threshold, label) bands for annualised-vol regime +#: classification, in ascending order. Any value at or above the last band's +#: threshold is classified as :data:`_EXTREME_LABEL`. Override per domain via +#: the ``bands`` parameter of :func:`_vol_regime` (see +#: :attr:`~aieng.forecasting.methods.agentic.domain.DomainConfig.vol_regime_bands`). +DEFAULT_VOL_REGIME_BANDS: tuple[tuple[float, str], ...] = ( (15.0, "low"), (30.0, "medium"), (50.0, "elevated"), - (math.inf, "extreme"), -] +) +_EXTREME_LABEL = "extreme" _MIN_VOL_WINDOW = 5 @@ -86,10 +91,20 @@ _MAE_TREND_THRESHOLD = 1.1 _MIN_HORIZONS_FOR_NARRATIVE = 2 -def _vol_regime(price_series: pd.DataFrame, as_of: datetime, lookback: int = 21) -> str: +def _vol_regime( + price_series: pd.DataFrame, + as_of: datetime, + lookback: int = 21, + *, + bands: tuple[tuple[float, str], ...] = DEFAULT_VOL_REGIME_BANDS, + extreme_label: str = _EXTREME_LABEL, +) -> str: """Classify the vol regime at *as_of*. - Uses *lookback* trading days of log returns. + Uses *lookback* trading days of log returns. *bands* is a sequence of + ``(upper_threshold, label)`` pairs in ascending order; annualised vol at or + above the last threshold is classified as *extreme_label*. Defaults reproduce + the historical 15 / 30 / 50 thresholds. """ import pandas as pd # noqa: PLC0415 — conditional import for optional dep @@ -100,10 +115,10 @@ def _vol_regime(price_series: pd.DataFrame, as_of: datetime, lookback: int = 21) return "unknown" log_returns = np.diff(np.log(window.astype(float))) annualized_vol = float(np.std(log_returns) * np.sqrt(252) * 100) - for threshold, label in _VOL_REGIMES: + for threshold, label in bands: if annualized_vol < threshold: return label - return "extreme" + return extreme_label # --------------------------------------------------------------------------- diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__domain.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__domain.py.md new file mode 100644 index 00000000..5b0ea9d0 --- /dev/null +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__domain.py.md @@ -0,0 +1,428 @@ +# Source: aieng-forecasting/aieng/forecasting/methods/agentic/domain.py + +kind: python + +```python +"""Domain configuration for agentic forecasters. + +A :class:`DomainConfig` captures everything that varies between forecasting +targets — the analyst persona, the target series and its units, the market- +intelligence search queries, the pipeline and strategy skill directories, the +volatility-regime bands, and the tool bounds — so the agent-building machinery +in this package stays domain-agnostic. The oil reference implementation defines +an ``OIL_DOMAIN`` instance; a new domain (e.g. S&P 500) is configured by +constructing another :class:`DomainConfig`, with no changes to shared code. + +The ``render_*`` functions turn a :class:`DomainConfig` into the instruction +strings the ADK agents run on, and :func:`build_analyst_config` / +:func:`build_adaptive_config` assemble those into +:class:`~aieng.forecasting.methods.agentic.agent_factory.AgentConfig` objects. +Rendering is pure string interpolation: given the oil fragments, the rendered +instructions reproduce the hand-written oil prompts. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable, Sequence + +from aieng.forecasting.methods.agentic.adaptive_skill_tools import build_skill_tools +from aieng.forecasting.methods.agentic.agent_factory import ( + AgentConfig, + CodeExecutionConfig, + ContextRetrievalConfig, +) +from aieng.forecasting.methods.agentic.outputs import ContinuousAgentForecastOutput +from aieng.forecasting.methods.agentic.strategy_state import StrategyState +from aieng.forecasting.models import LITE_MODEL +from pydantic import BaseModel, ConfigDict, Field + + +# --------------------------------------------------------------------------- +# Domain configuration +# --------------------------------------------------------------------------- + + +class DomainConfig(BaseModel): + """Everything that varies between forecasting targets. + + Grouped into identity, data/target, context-retrieval, strategy-skill, + skill-directory, volatility-regime, and tool-bound sections. All fields are + plain data; the render/build helpers below consume them. + """ + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + # ── Identity ──────────────────────────────────────────────────────────── + domain_name: str = Field(description="Human label for the domain, e.g. 'WTI crude oil'.") + analyst_persona: str = Field(description="Role noun phrase for the analyst, e.g. 'WTI crude oil market analyst'.") + analyst_forecasting_focus: str = Field( + description=( + "Completes 'You produce {...}.' in the analyst role block — the forecast " + "product and the fundamentals it is grounded in." + ) + ) + analyst_agent_name_prefix: str = Field( + default="analyst", + description="Prefix for stateless analyst agent names (a '_{suffix}' is appended).", + ) + adaptive_agent_name_prefix: str = Field( + default="adaptive_analyst", + description="Prefix for the adaptive agent name (the strategy dir name is appended).", + ) + target_short_name: str = Field(description="Short target label used inline, e.g. 'WTI'.") + starter_fluency_areas: str = Field( + default="", + description="Comma phrase of expertise areas for the hackable starter-agent persona.", + ) + + # ── Data / target ─────────────────────────────────────────────────────── + target_series_id: str = Field(description="Canonical series id the predictor forecasts.") + target_units: str = Field(description="Units of the target series, e.g. 'USD/bbl'.") + target_history_description: str = Field( + description="Describes the `target_history_csv` payload field in the analyst contract." + ) + data_ticker: str = Field(description="Market data ticker, e.g. 'CL=F'.") + data_source_name: str = Field(default="Yahoo Finance", description="Human name of the price data source.") + data_fetch_example: str = Field( + description="Verbatim code block showing a cutoff-respecting data fetch, embedded in the adaptive prompt." + ) + code_exec_preinstalled: str = Field( + default="numpy, pandas, sklearn, yfinance, statsmodels, properscoring", + description="Comma-separated list of libraries pre-installed in the code sandbox.", + ) + multitask_origin_price_field: str = Field( + default="origin_price", + description="Payload key carrying the origin close price in the multitask prompt.", + ) + + # ── Context retrieval ─────────────────────────────────────────────────── + context_retrieval_instruction: str = Field( + description="Full instruction for the web-search sub-agent (domain-specific verbatim)." + ) + recommended_search_queries: tuple[str, ...] = Field( + default=(), + description="Search queries advertised to the analyst, one `search_web` call each.", + ) + key_assumptions_hint: str = Field( + description="Parenthetical list of assumptions the analyst should document in rationales." + ) + + # ── Strategy skill ────────────────────────────────────────────────────── + strategy_skill_title: str = Field(description="Markdown H1 of the rendered strategy SKILL.md.") + strategy_skill_name: str = Field(description="Skill/dir name of the strategy skill, e.g. 'wti-strategy'.") + adaptive_calibration_example: str = Field( + description="Illustrative calibration action in the adaptive prediction-request block." + ) + + # ── Skill directories ─────────────────────────────────────────────────── + pipeline_skill_dirs: tuple[Path, ...] = Field( + default=(), + description="Ordered pipeline skill dirs (e.g. fetch, vol-regime, trend-projection).", + ) + meta_learning_skill_dir: Path | None = Field( + default=None, + description="Governance skill dir loaded after the strategy skill.", + ) + seed_strategy_dir: Path | None = Field(default=None, description="Default (seed) strategy skill dir.") + trained_strategy_dir: Path | None = Field(default=None, description="Trained strategy variant dir.") + + # ── Volatility regime ─────────────────────────────────────────────────── + vol_regime_bands: tuple[tuple[float, str], ...] = Field( + default=((15.0, "low"), (30.0, "medium"), (50.0, "elevated")), + description="(upper_threshold, label) bands for annualised-vol regime classification.", + ) + + # ── Tool bounds ───────────────────────────────────────────────────────── + frequency: str = Field(default="B", description="Pandas offset alias for the target series cadence.") + horizons: tuple[int, ...] = Field(default=(5, 10, 21), description="Default forecast horizons in steps.") + shock_threshold: float = Field(default=5.0, description="Move size defining the shock event.") + shock_horizon: int = Field(default=5, description="Horizon (steps) for the shock event.") + num_samples: int = Field(default=200, description="Monte-Carlo sample count for the statistical tool.") + + +# --------------------------------------------------------------------------- +# Instruction rendering +# --------------------------------------------------------------------------- + + +def render_analyst_instruction(domain: DomainConfig) -> str: + """Render the task-aware analyst instruction (news / code-exec / tool variants). + + Embeds the continuous output schema from + :class:`~aieng.forecasting.methods.agentic.outputs.ContinuousAgentForecastOutput` + so the required JSON block stays in sync with the schema. + """ + schema = ContinuousAgentForecastOutput.prompt_schema_json() + query_lines = "".join( + f'- ``search_web(query="{q}", cutoff_date=)``\n' for q in domain.recommended_search_queries + ) + return ( + "## Role\n\n" + f"You are an expert {domain.analyst_persona}. You produce {domain.analyst_forecasting_focus}.\n\n" + "## Forecasting contract\n\n" + "You will receive a JSON payload containing:\n" + "- `task`: the task identifier\n" + "- `as_of`: the forecast origin date in YYYY-MM-DD format\n" + "- `horizons`: a list of integer horizon steps (business days ahead)\n" + "- `standard_quantiles`: the exact quantile levels you must produce\n" + "- `target_summary`: last close price, 52-week range, and observation count\n" + f"- `target_history_csv`: {domain.target_history_description}\n\n" + "Rules:\n" + "1. Produce one forecast for each horizon listed in `horizons`.\n" + "2. Use exactly the quantile levels from `standard_quantiles` — no additions, no omissions.\n" + "3. `point_forecast` must exactly equal the 0.50 quantile value.\n" + "4. Quantile values must be strictly non-decreasing as quantile levels increase.\n" + "5. Document your reasoning in the `rationale` fields.\n" + "6. When tools are enabled, conclude with `set_model_response` to return the structured forecast.\n\n" + "## Output schema\n\n" + "Call `set_model_response` with a `json_response` string matching **exactly**:\n\n" + "```json\n" + schema + "\n```\n\n" + 'Critical: use `"horizon"` (integer, not `"horizon_days"`). ' + '`"quantiles"` is a **list** of `{"quantile": , "value": }` ' + "objects — not a dict. Omit any field not shown above.\n\n" + "## Analysis discipline\n\n" + "When context retrieval is available, call ``search_web`` to gather market " + "intelligence BEFORE producing forecasts.\n\n" + "Call ``search_web`` with ``query`` and ``cutoff_date`` (set to the ``as_of`` " + "date from the payload). The ``cutoff_date`` MUST always equal ``as_of`` — " + "this is the temporal fence that prevents post-origin information from " + "contaminating historical backtests.\n\n" + "If ``search_web`` returns a result beginning with " + "``[SEARCH_VERIFICATION_FAILED]``, treat it as no verified news context for " + "that query. Do not use your own background knowledge to fill the gap or " + "speculate about what the news might have said — proceed with price-history " + "and other available signals only, and note the gap in your rationale.\n\n" + "Recommended queries (call ``search_web`` once per topic):\n" + + query_lines + + "\n" + + f"Document your key assumptions ({domain.key_assumptions_hint}) in the `rationale` " + "fields of your forecast output." + ) + + +def render_multitask_analyst_instruction(domain: DomainConfig) -> str: + """Render the task-agnostic analyst instruction (one-agent-three-tasks demo).""" + return ( + "## Role\n\n" + f"You are an expert {domain.analyst_persona}.\n\n" + "## Input\n\n" + "You will receive a JSON payload containing:\n" + "- `task_spec`: the exact question and required JSON output schema\n" + "- `as_of`: the forecast origin date (temporal cutoff)\n" + f"- `{domain.multitask_origin_price_field}`: {domain.target_short_name} close on the origin date\n" + f"- `target_history_csv`: compressed {domain.target_short_name} daily close history\n\n" + "When context retrieval is enabled, call ``search_web`` BEFORE answering.\n\n" + "## Output contract\n\n" + "Read the data (and briefing, if retrieved) carefully, then execute the task " + "in `task_spec` precisely.\n\n" + "If a `set_model_response` tool is available, call it with your complete JSON " + "as `json_response` — the exact schema is described in `task_spec`. Otherwise " + "return the JSON directly as plain text with no preamble." + ) + + +def render_starter_instruction(domain: DomainConfig) -> str: + """Render the hackable starter-agent persona (task-agnostic, schema-free).""" + return ( + "## Role\n\n" + f"You are a {domain.analyst_persona} — fluent in {domain.starter_fluency_areas}. " + "This is a starter agent: keep your reasoning " + "transparent and your claims honest.\n\n" + "## How to respond\n\n" + "- For open-ended questions, scenario analysis, or anything " + "conversational, answer directly and concisely — do NOT ask for a JSON " + "payload.\n" + "- When you are handed a task that asks for a structured probabilistic " + "forecast, produce a calibrated one." + ) + + +def render_adaptive_analyst_instruction(domain: DomainConfig) -> str: + """Render the persistent adaptive-analyst instruction. + + Interpolates the persona, the pipeline / strategy / governance skill names, + the calibration example, the pre-installed library list, and the data-fetch + example from *domain*. The skill names default to the directory basenames. + """ + schema = ContinuousAgentForecastOutput.prompt_schema_json() + fetch, vol, trend = (d.name for d in domain.pipeline_skill_dirs) + meta = domain.meta_learning_skill_dir.name if domain.meta_learning_skill_dir else "meta-learning" + strategy = domain.strategy_skill_name + return ( + "## Identity\n\n" + f"You are a persistent {domain.analyst_persona}. You carry knowledge forward " + f"across invocations: your `{strategy}` skill captures your current forecasting " + "approach, and you update it deliberately as you learn from experience.\n\n" + "## Message types\n\n" + "You receive messages through a single chat interface. Determine from context " + "what kind of invocation this is and respond accordingly:\n\n" + "**Prediction request** — contains a JSON payload with `task`, `as_of`, " + f"`horizons`, and price history. Load `{strategy}` first to read your current " + "approach and any active calibration corrections. Then:\n" + f"1. Use `run_code` to run your full statistical analysis pipeline: fetch data " + f"via `{fetch}` (using `end=as_of` as the cutoff), classify the vol " + f"regime via `{vol}`, and project trend and intervals via `{trend}`. " + f"Apply any calibration corrections from `{strategy}` — for example, {domain.adaptive_calibration_example}.\n" + "2. Use the context-retrieval tool to gather current market news and adjust your " + "estimates where strong catalysts are present.\n" + "3. Conclude with `set_model_response` (schema below).\n\n" + "Your quantitative pipeline is your starting point — your learned strategy " + "corrections and news-grounded judgment shape the final forecast.\n\n" + "**Resolution** — describes how a past forecast resolved (actual value, error, " + "horizon). Reflect carefully. If the error points to a systematic pattern — not " + f"a one-off surprise — consult `{meta}` to assess whether a strategy update " + "is warranted.\n\n" + "**Self-review / backtesting** — you are asked to analyse your recent performance " + "or explore historical data using code execution. Compose the relevant skills, " + "write one complete code block, and summarise what you find. If the analysis " + f"surfaces a durable insight, follow the `{meta}` process.\n\n" + "**User question** — a human is asking for analysis, context, or your market " + "view. Engage directly, using code execution and web search as needed.\n\n" + "## Skills are pipeline components\n\n" + "Your skills cover specific pipeline stages. Compose them: for any task " + "involving code, load each relevant skill and its `references/examples.md`, " + "then write one complete self-contained code block combining all the patterns.\n\n" + "| Skill | Pipeline stage |\n" + "|------------------|---------------------------------------------------------|\n" + f"| {fetch:<16} | Download market / futures data from {domain.data_source_name} |\n" + f"| {vol:<16} | Classify vol regime, detect anomalies, choose window |\n" + f"| {trend:<16} | Fit trend, project to horizons, calibrate intervals |\n" + f"| {strategy:<16} | Your current forecasting strategy — load at the start of every prediction |\n" + f"| {meta:<16} | Governs when and how to update {strategy} |\n\n" + "## Strategy mutation tools\n\n" + f"These tools write directly to `{strategy}` on the host filesystem. " + "They run outside the E2B sandbox. Consult " + f"`{meta}` before calling " + "any of them.\n\n" + "| Tool | Evidence layer | Evidence bar |\n" + "|------|---------------|---------------|\n" + "| `record_observation(finding, linked_hypothesis?)` | Observations | Pattern visible across ≥2 forecasts — not a single surprise |\n" + "| `open_hypothesis(claim, initial_evidence)` | Hypotheses | One strong observation suggesting a durable pattern |\n" + "| `record_hypothesis_outcome(hypothesis_id, outcome)` | Hypotheses | Each resolution relevant to an open hypothesis |\n" + "| `graduate_hypothesis(hypothesis_id, condition, adjustment, horizon_scope)` | Calibration | Tool enforces confirmation threshold — will reject if not met |\n" + "| `update_approach_narrative(new_text, rationale)` | Approach | Only when the calibration record reveals a structural insight |\n\n" + f"Active calibration corrections from `{strategy}` are **not optional** — " + "apply every listed correction when the stated condition is met.\n\n" + "## Code execution discipline\n\n" + "Treat `run_code` like submitting to a batch queue: plan your complete " + "analysis upfront, write one self-contained script, and read the results. " + "There is no REPL, no way to inspect intermediate state between calls, and " + "no benefit to splitting work — each submission starts from zero with no " + "memory of previous calls.\n\n" + "Never make a preliminary or test call to check connectivity or verify " + "imports. Assume the environment works. Your first `run_code` call should " + "produce your complete result.\n\n" + f"Pre-installed: {domain.code_exec_preinstalled}.\n\n" + f"**Data sourcing rule:** Always use the `{fetch}` skill to load price " + "data inside `run_code`. **Never embed `target_history_csv` or any CSV " + "string literal as a data source in code.** Pasting thousands of rows of " + "data as Python string literals is fragile, wastes context, and risks hitting " + "sandbox limits. `target_history_csv` is provided in the prediction payload " + "for your reading and statistical summary only — not for copy-pasting into " + "code blocks. When a skill description says 'assume `df` is already defined', " + "that means you should define `df` via a yfinance fetch at the top of your " + "script, not by embedding raw data.\n\n" + "## Temporal discipline\n\n" + "Every forecast is anchored to an `as_of` date. Never use information beyond " + "that date — in web search, code analysis, or reasoning.\n\n" + "If `search_web` returns a result beginning with `[SEARCH_VERIFICATION_FAILED]`, " + "treat it as no verified news context for that query. Do not use your own " + "background knowledge to fill the gap — proceed on price history and other " + "available signals only, and note the gap in your reasoning.\n\n" + "When fetching data inside `run_code`, always pass `end=as_of_date` to " + "yfinance to enforce the temporal cutoff — for example:\n\n" + domain.data_fetch_example + "\n\n" + "Replace the end date with the actual `as_of` value from the prediction " + "payload. This is the only correct way to ensure the sandbox sees the same " + "data the agent would have seen on that date.\n\n" + "## Prediction output schema\n\n" + "For **prediction requests**, call `set_model_response` with `json_response` " + "matching **exactly**:\n\n" + "```json\n" + schema + "\n```\n\n" + 'Critical: use `"horizon"` (integer, not `"horizon_days"`). ' + '`"quantiles"` is a **list** of `{"quantile": , "value": }` ' + "objects — not a dict." + ) + + +# --------------------------------------------------------------------------- +# Config builders +# --------------------------------------------------------------------------- + + +def build_analyst_config( + domain: DomainConfig, + *, + name_suffix: str, + instruction: str, + model: str = LITE_MODEL, + context_retrieval: ContextRetrievalConfig | None = None, + code_execution: CodeExecutionConfig | None = None, + skills_dirs: Sequence[Path] = (), + function_tools: Sequence[Callable[..., Any]] = (), + max_output_tokens: int | None = None, +) -> AgentConfig: + """Assemble a stateless analyst :class:`AgentConfig` for *domain*. + + The agent name is ``f"{domain.analyst_agent_name_prefix}_{name_suffix}"``. + Only the knobs a variant actually uses need be passed; the rest fall back to + :class:`AgentConfig` defaults so the resulting config matches a hand-written + one field-for-field. + """ + return AgentConfig( + name=f"{domain.analyst_agent_name_prefix}_{name_suffix}", + model=model, + instruction=instruction, + context_retrieval=context_retrieval if context_retrieval is not None else ContextRetrievalConfig(), + code_execution=code_execution if code_execution is not None else CodeExecutionConfig(), + skills_dirs=tuple(skills_dirs), + function_tools=tuple(function_tools), + max_output_tokens=max_output_tokens, + ) + + +def build_adaptive_config( + domain: DomainConfig, + *, + state_type: type[StrategyState] = StrategyState, + model: str, + search_model: str = LITE_MODEL, + max_output_tokens: int = 16_384, + strategy_dir: Path | None = None, + confirmation_threshold: int = 2, +) -> AgentConfig: + """Assemble the persistent adaptive-analyst :class:`AgentConfig` for *domain*. + + Wires E2B code execution, cutoff-bounded web search, the pipeline skills, the + selected strategy skill, and the governance skill, plus the five strategy + mutation tools bound to *state_type*. The strategy directory name is baked + into the agent name so per-variant prediction caches stay separate. + """ + resolved_strategy_dir = strategy_dir or domain.seed_strategy_dir + if resolved_strategy_dir is None: + raise ValueError("No strategy_dir provided and domain.seed_strategy_dir is unset.") + agent_name = f"{domain.adaptive_agent_name_prefix}_{resolved_strategy_dir.name.replace('-', '_')}" + + skills_dirs = [*domain.pipeline_skill_dirs, resolved_strategy_dir] + if domain.meta_learning_skill_dir is not None: + skills_dirs.append(domain.meta_learning_skill_dir) + + return AgentConfig( + name=agent_name, + model=model, + instruction=render_adaptive_analyst_instruction(domain), + max_output_tokens=max_output_tokens, + context_retrieval=ContextRetrievalConfig( + enabled=True, + instruction=domain.context_retrieval_instruction, + search_model=search_model, + ), + code_execution=CodeExecutionConfig(enabled=True), + skills_dirs=skills_dirs, + extra_tools=build_skill_tools( + resolved_strategy_dir, + state_type, + confirmation_threshold=confirmation_threshold, + ), + ) +``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__history.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__history.py.md new file mode 100644 index 00000000..ce20b5fc --- /dev/null +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__history.py.md @@ -0,0 +1,57 @@ +# Source: aieng-forecasting/aieng/forecasting/methods/agentic/history.py + +kind: python + +```python +"""History compression for agent prompt payloads. + +Serialising a full daily price history into an agent prompt is wasteful and can +overflow the context window. :func:`compress_history` keeps recent daily +resolution while down-sampling older history to weekly averages, producing a +compact ``date,close`` CSV suitable for embedding in a JSON payload. +""" + +from __future__ import annotations + +import pandas as pd + + +def compress_history(df: pd.DataFrame, *, recent_months: int = 6) -> str: + """Compress a daily price history to stay within context limits. + + Returns daily bars for the most recent ``recent_months`` months and weekly + averages for older history. The CSV header is ``date,close``. + + Parameters + ---------- + df : pd.DataFrame + DataFrame with columns ``timestamp`` and ``value``. + recent_months : int, default=6 + Number of trailing months to keep at daily resolution. Older history + is resampled to weekly averages. + + Returns + ------- + str + CSV string with header ``date,close``. + """ + df = df.copy() + df["timestamp"] = pd.to_datetime(df["timestamp"]) + cutoff = df["timestamp"].max() - pd.DateOffset(months=recent_months) + + recent = df[df["timestamp"] >= cutoff].copy() + old = df[df["timestamp"] < cutoff].copy() + + rows: list[str] = ["date,close"] + + if not old.empty: + old_indexed = old.set_index("timestamp")["value"] + weekly: pd.Series = old_indexed.resample("W").mean().dropna() + for date, val in weekly.items(): + rows.append(f"{date.date()},{val:.2f}") + + for _, row in recent.iterrows(): + rows.append(f"{row['timestamp'].date()},{row['value']:.2f}") + + return "\n".join(rows) +``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__outputs.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__outputs.py.md index 0a39edcf..0a4be026 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__outputs.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__outputs.py.md @@ -631,4 +631,108 @@ class CategoricalAgentForecastOutput(AgentForecastOutput): metadata=prediction_metadata, ) ] + + +class ScenarioCard(BaseModel): + """One probability-weighted scenario narrative from scenario-analysis output. + + Carries the qualitative story only. Domain implementations subclass this to + add numeric range / point-estimate fields under the field names their + downstream caches and displays expect (see + :meth:`ScenarioAgentForecastOutput.prompt_schema_json`). + """ + + model_config = {"extra": "ignore"} + + name: str + description: str + probability: float = Field(ge=0.0, le=1.0) + key_drivers: list[str] = Field(default_factory=list) + + +class ScenarioAgentForecastOutput(AgentForecastOutput): + """Scenario-analysis output: a set of probability-weighted narratives. + + The scenario cards carry the qualitative stories the market is debating; the + aggregate probability mass across cards is surfaced as a single + :class:`~aieng.forecasting.evaluation.prediction.BinaryForecast` for scoring, + while the full card set rides along in prediction metadata for display. + + Domain subclasses override :attr:`scenario_card_template_extra` (and narrow + the :attr:`scenarios` element type) to advertise the numeric scenario fields + their target series uses — e.g. a price range and point estimate at a named + horizon. + """ + + modality: ClassVar[Literal["continuous", "discrete", "categorical"]] = "discrete" + + model_config = {"extra": "ignore"} + + scenarios: list[ScenarioCard] + base_case: str + reasoning: str = "" + + #: Extra template fields injected into each scenario card in + #: :meth:`prompt_schema_json`, positioned between ``probability`` and + #: ``key_drivers``. Empty on the generic base; override in a subclass to + #: advertise domain-specific numeric fields. + scenario_card_template_extra: ClassVar[dict[str, object]] = {} + + @classmethod + def prompt_schema_json(cls) -> str: + """Return a JSON template for use in agent instruction strings. + + Returns + ------- + str + Indented JSON string showing the exact structure the agent must + pass to ``set_model_response``. + """ + card: dict[str, object] = { + "name": "", + "description": "", + "probability": "", + } + card.update(cls.scenario_card_template_extra) + card["key_drivers"] = ["", ""] + template: dict[str, object] = { + "scenarios": [card], + "base_case": "", + "reasoning": "", + } + return json.dumps(template, indent=2) + + def to_predictions( + self, + *, + task: ForecastingTask, + context: ForecastContext, + predictor_id: str, + metadata: dict[str, Any] | None = None, + ) -> list[Prediction]: + """Convert scenario output to a metadata-rich prediction.""" + if len(task.horizons) != 1: + raise ValueError("Scenario agent output expects exactly one task horizon.") + + horizon = task.horizons[0] + issued_at = datetime.utcnow() + offset = pd.tseries.frequencies.to_offset(task.frequency) + base_prob = float(sum(s.probability for s in self.scenarios)) + prediction_metadata: dict[str, Any] = dict(metadata) if metadata is not None else {} + prediction_metadata["scenarios"] = [s.model_dump() for s in self.scenarios] + prediction_metadata["base_case"] = self.base_case + if self.reasoning.strip(): + prediction_metadata["rationale"] = self.reasoning + + return [ + Prediction( + predictor_id=predictor_id, + task_id=task.task_id, + issued_at=issued_at, + as_of=context.as_of, + forecast_date=(pd.Timestamp(context.as_of) + offset * horizon).to_pydatetime(), + payload=BinaryForecast(probability=min(base_prob, 1.0)), + metadata=prediction_metadata, + ) + ] ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md index 6e613431..10baed0c 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md @@ -34,7 +34,7 @@ from aieng.forecasting.evaluation.prediction import Prediction from aieng.forecasting.evaluation.predictor import Predictor from aieng.forecasting.evaluation.task import ForecastingTask from aieng.forecasting.methods.agentic.adk_runner import AdkTextRunner, AdkTextRunnerConfig -from aieng.forecasting.methods.agentic.agent_factory import AgentConfig, build_adk_agent +from aieng.forecasting.methods.agentic.agent_factory import AS_OF_STATE_KEY, AgentConfig, build_adk_agent from aieng.forecasting.methods.agentic.outputs import AgentForecastOutput from aieng.forecasting.methods.llm_processes._client import strip_markdown_fence, trace_url_for from google.adk.agents.base_agent import BaseAgent @@ -282,7 +282,11 @@ class AgentPredictor(Predictor): validation errors on the agent's JSON are not swallowed. """ prompt = self.prompt_builder(task=task, context=context) - output_str = _run_coroutine_sync(self._runner.run_text_async(prompt)) + # Seed the harness-controlled as_of into the ADK session before the run, + # so search_web can enforce it via ToolContext.state regardless of + # whether the LLM remembers to pass a matching cutoff_date argument. + initial_state = {AS_OF_STATE_KEY: str(context.as_of)[:10]} + output_str = _run_coroutine_sync(self._runner.run_text_async(prompt, initial_state=initial_state)) # Normalise: strip markdown fences before validation so any model can # be swapped in without breaking the parse layer. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__strategy_state.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__strategy_state.py.md new file mode 100644 index 00000000..51dd9c96 --- /dev/null +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__strategy_state.py.md @@ -0,0 +1,228 @@ +# Source: aieng-forecasting/aieng/forecasting/methods/agentic/strategy_state.py + +kind: python + +```python +"""Generic adaptive forecasting-strategy state. + +``StrategyState`` is a concrete ``AdaptiveSkillState`` that models a +*learnable forecasting strategy* — the living approach an adaptive +agent refines across invocations. It captures four learning layers with +distinct evidence burdens: + +``observations`` + Append-only log of pattern-level findings. Lowest evidence bar — record + any finding that is not a single-outlier surprise. + +``hypotheses`` + Candidate systematic corrections under active testing. Accumulate + confirmation / refutation counts across resolutions. A hypothesis + graduates to a calibration correction when its confirmation count reaches + the store's ``confirmation_threshold``. + +``calibration_corrections`` + Confirmed systematic adjustments applied at prediction time. Each entry is + graduated from a confirmed hypothesis — never added directly. + +``approach_narrative`` + Free-text description of the agent's overall forecasting philosophy. + Highest evidence bar. + +The rendered ``SKILL.md`` layout is domain-agnostic; the three domain-specific +strings — the markdown heading, the default skill name, and the frontmatter +description — are ``ClassVar`` parameters. Subclass ``StrategyState`` and +override those class variables to brand the skill for a specific domain (see +:class:`~energy_oil_forecasting.adaptive_agent.skill_state.WtiStrategyState`). +""" + +from __future__ import annotations + +from typing import ClassVar, Literal + +from aieng.forecasting.methods.agentic.adaptive_skill import AdaptiveSkillState +from pydantic import BaseModel + + +# --------------------------------------------------------------------------- +# Sub-models +# --------------------------------------------------------------------------- + + +class Observation(BaseModel): + """A single pattern-level finding from a resolution or self-review.""" + + date: str + finding: str + linked_hypothesis: str | None = None + + +class Hypothesis(BaseModel): + """A candidate systematic correction under active testing. + + ``status`` progresses through ``open`` → ``confirmed`` or ``open`` → + ``refuted``. Confirmed hypotheses are graduated to + :class:`CalibrationCorrection` via the ``graduate_hypothesis`` tool. + """ + + id: str + claim: str + status: Literal["open", "confirmed", "refuted"] = "open" + confirmations: int = 0 + refutations: int = 0 + opened_on: str + + +class CalibrationCorrection(BaseModel): + """A confirmed systematic adjustment applied at prediction time. + + Every entry here was graduated from a confirmed hypothesis; the + ``source_hypothesis`` field preserves that lineage. + """ + + condition: str + adjustment: str + horizon_scope: str + source_hypothesis: str + confirmed_on: str + + +class VersionEntry(BaseModel): + """One row in the version history table.""" + + date: str + description: str + + +# --------------------------------------------------------------------------- +# Strategy state +# --------------------------------------------------------------------------- + + +class StrategyState(AdaptiveSkillState): + """Domain-agnostic structured state for an adaptive forecasting strategy. + + See the module docstring for the learning-layer hierarchy and evidence + burdens. The three ``ClassVar`` parameters below carry the only + domain-specific presentation strings; override them in a subclass to brand + the rendered ``SKILL.md`` for a particular target series. + """ + + #: Heading rendered as ``# {markdown_title}`` at the top of the body. + markdown_title: ClassVar[str] = "Forecasting Strategy" + #: Frontmatter ``name:`` used when the store does not pass an explicit name. + default_skill_name: ClassVar[str] = "strategy" + #: Content lines for the frontmatter ``description: >-`` block (rendered + #: with a two-space indent, one line per entry). + frontmatter_description_lines: ClassVar[tuple[str, ...]] = ( + "The adaptive analyst's current forecasting strategy. Load this at the", + "start of every prediction task. This file is generated — edit the state", + "through the mutation tools, not by hand.", + ) + + approach_narrative: str + calibration_corrections: list[CalibrationCorrection] = [] + hypotheses: list[Hypothesis] = [] + observations: list[Observation] = [] + version_history: list[VersionEntry] = [] + + def build_markdown(self, skill_name: str | None = None) -> str: # noqa: PLR0912 + """Render the full ``SKILL.md`` content from current state.""" + lines: list[str] = [] + + # Frontmatter — skill_name must match the containing dir name (ADK requires) + lines += [ + "---", + f"name: {skill_name or self.default_skill_name}", + "description: >-", + ] + lines += [f" {line}" for line in self.frontmatter_description_lines] + lines += [ + "---", + "", + ] + + lines += [ + f"# {self.markdown_title}", + "", + "## Approach", + "", + self.approach_narrative.strip(), + "", + ] + + # Active calibration corrections + lines += [ + "## Active calibration corrections", + "", + ] + if self.calibration_corrections: + lines += [ + "| Condition | Adjustment | Horizon scope | Confirmed on |", + "|-----------|-----------|---------------|--------------|", + ] + for c in self.calibration_corrections: + lines.append(f"| {c.condition} | {c.adjustment} | {c.horizon_scope} | {c.confirmed_on} |") + else: + lines.append("*(No calibration corrections yet. Graduate a confirmed hypothesis to add one.)*") + lines.append("") + + # Open hypotheses + lines += [ + "## Open hypotheses", + "", + ] + open_hyps = [h for h in self.hypotheses if h.status == "open"] + if open_hyps: + lines += [ + "| ID | Claim | Confirmations | Refutations |", + "|----|-------|---------------|-------------|", + ] + for h in open_hyps: + lines.append(f"| {h.id} | {h.claim} | {h.confirmations} | {h.refutations} |") + else: + lines.append("*(No open hypotheses.)*") + lines.append("") + + # Closed hypotheses (confirmed / refuted) — collapsed for readability + closed_hyps = [h for h in self.hypotheses if h.status != "open"] + if closed_hyps: + lines += [ + "## Closed hypotheses", + "", + "| ID | Claim | Status | Confirmations | Refutations |", + "|----|-------|--------|---------------|-------------|", + ] + for h in closed_hyps: + lines.append(f"| {h.id} | {h.claim} | {h.status} | {h.confirmations} | {h.refutations} |") + lines.append("") + + # Observations + lines += [ + "## Observations", + "", + ] + if self.observations: + lines += [ + "| Date | Finding | Linked hypothesis |", + "|------|---------|-------------------|", + ] + for o in self.observations: + linked = o.linked_hypothesis or "—" + lines.append(f"| {o.date} | {o.finding} | {linked} |") + else: + lines.append("*(No observations yet. Record findings from resolutions and self-reviews.)*") + lines.append("") + + # Version history + lines += [ + "## Version history", + "", + "| Date | Change |", + "|------|--------|", + ] + for v in self.version_history: + lines.append(f"| {v.date} | {v.description} |") + lines.append("") + + return "\n".join(lines) +``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__analyst_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__analyst_agent__agent.py.md index 8ead7168..d30ec295 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__analyst_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__analyst_agent__agent.py.md @@ -121,7 +121,11 @@ def _build_boc_analyst_instruction() -> str: "strongly shape which tail outcome is plausible.\n" "4. Use ONLY information available on or before `as_of`. Do not use " "knowledge of what the Bank actually decided on or after " - "`announcement_date`, even if you remember it.\n" + "`announcement_date`, even if you remember it. If `search_web` returns a " + "result beginning with `[SEARCH_VERIFICATION_FAILED]`, treat it as no " + "verified news for that query — proceed on the other signals in your " + "payload and note the gap, never filling it from your own background " + "knowledge.\n" "5. Document your reasoning in `reasoning` and list the decisive inputs " "in `key_signals` — these are compared against the Bank's own published " "rationale by a downstream evaluator, so be specific.\n\n" @@ -152,7 +156,16 @@ markdown summary (3-5 paragraphs) covering relevant aspects of: Ground your summary in the search results you actually retrieve. \ When a cutoff date is specified, do not report or speculate about events \ -that occurred after that date.\ +that occurred after that date. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md index 500357a7..0ed0cf45 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md @@ -105,7 +105,16 @@ labour market; market pricing of the upcoming decision (OIS, economist surveys); and macro shocks relevant to Canada (oil, exchange rate, US policy, trade). Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md index 96ad5034..d824d247 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md index 003940ad..a3989b15 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md @@ -14,8 +14,10 @@ This notebook simulates a rigorous production forecasting workflow: 3. Select the **top contender configurations** based solely on 2025 historical performance (no peeking at 2026). 4. Let the contenders compete in the **2026 Protected Arena** - (`energy_oil_eval.yaml`) during the geopolitical price shock — - measuring adaptive real-time responsiveness and calibration. + (`energy_oil_eval.yaml`) across the geopolitical price shock and its + aftermath — measuring adaptive real-time responsiveness and calibration. + The eval window runs through the most recent origin whose 21-business-day + horizon still resolves against cached data (see `scripts/fetch_wti.py`). The line-up spans three families behind one `Predictor` interface: **baselines** (Naive, AutoARIMA), **numerical ML** (LightGBM ± a leak-safe covariate panel), @@ -56,7 +58,7 @@ warnings.filterwarnings("ignore") # Set SMOKE_TEST = True to run a 2-origin, 1-sample version of the notebook # for fast local development and end-to-end CI testing. The full specs run # 51 backtest + 8 eval origins; smoke runs 2 + 2. -SMOKE_TEST = True +SMOKE_TEST = False # ── Models ──────────────────────────────────────────────────────────────────── # The project standardises on two Vector-proxy models. Every LLM and agent @@ -324,9 +326,11 @@ print(f"Saved {sum(n in _BASELINE_PREDICTORS for n in backtest_results)} backtes --- ## 5. 2026 Evaluation — Held-Out Test Period -We run every active predictor on **8 weekly origins in early 2026** -(`energy_oil_eval.yaml`) — a period of major geopolitical volatility not seen -during the 2025 backtest. +We run every active predictor on **18 weekly origins spanning Feb–Jun 2026** +(`energy_oil_eval.yaml`) — the major geopolitical volatility spike not seen +during the 2025 backtest, plus its aftermath. The window runs through the +most recent origin that still fully resolves against cached WTI data (the +21-business-day horizon needs data 21 business days past the origin). This evaluation serves two purposes: 1. **Measure out-of-sample robustness** — do the 2025 edges (statistical, @@ -402,55 +406,161 @@ print(df_scorecard.to_string()) ## Cell 16 (markdown) --- -## 7. Core Takeaways +## 7. Diagnostics — reading past the leaderboard -1. **Numerical methods beat the naive baseline** by extracting structure from the - price history — AutoARIMA via local autocorrelation, LightGBM via lagged - gradient-boosted quantiles. In stable regimes this translates to better CRPS. +The scorecard above is a single number per method. That hides *where* the score +comes from and *whether the ranking is even real*. The next cells decompose it +straight from the eval predictions — so they recompute on any rerun, smoke or +full: -2. **Covariates can sharpen LightGBM.** The `LightGBM + cov` variant adds a - leak-safe panel (Brent, natural gas, gasoline, gold, the USD index, the - USL/USO futures-curve contango proxy, and VIX). Comparing it to plain - `LightGBM` isolates how much the cross-market context is worth — the same - lesson that made covariates decisive in the S&P 500 study. +- **CRPS by horizon** — does a method win everywhere, or is its mean dominated by + one horizon? (For a short forecast, the 5-day calls are easy and nearly tied; + the ranking is usually decided by the longest horizon.) +- **Mean CRPS ± standard error** — with only a handful of origins, are the gaps + between methods bigger than the noise, or is the "winner" a coin flip? -3. **Tree models extrapolate poorly through regime shifts.** LightGBM forecasts - the price *level*, and gradient-boosted trees cannot predict outside the range - seen in training. When the 2026 shock pushes WTI to new levels, expect the - tree methods — like AutoARIMA — to lag and produce biased, under-confident - intervals. This is a structural limitation, not a tuning problem. +With the **smoke spec (2 origins → a few scored points)** expect wide error bars +and an unstable ranking. That is exactly why a surprising leaderboard here is not +yet evidence of anything — it is a pipeline check. -4. **LLM/agent methods bring a different prior.** The LLM-process forecasters and - the news-reading agent are run on both `gemini-3.1-flash-lite-preview` and - `gemini-3.5-flash`, so the scorecard shows both *method* and *model* effects — - and whether reading the news helps when the numerical methods are blindsided. +## Cell 17 (code) -5. **The `Predictor` abstraction makes the comparison clean.** The same harness, - scoring functions, covariate panel, and eval spec serve every family, and the - registry lets you switch any predictor on or off without touching the pipeline. +```python +from energy_oil_forecasting import viz +from energy_oil_forecasting.analysis import ( + build_price_frame, + eval_narrative_md, + extract_agent_rationales, + leaderboard_with_uncertainty, + per_horizon_crps, + predictions_to_frame, +) +from IPython.display import HTML, Markdown, display # noqa: A004 + + +# Explode every scored 2026 eval prediction into one tidy row per +# (predictor, origin, horizon): point, 80% interval, realised price, and CRPS. +# Everything in Sections 7–10 reads from this frame, so it all recomputes when +# you flip SMOKE_TEST off and rerun. +price_df = build_price_frame(data_service) +eval_frame = predictions_to_frame(eval_results, data_service) +eval_board = leaderboard_with_uncertainty(eval_frame) +ph_crps = per_horizon_crps(eval_frame) + +print("━" * 72) +print("MEAN CRPS BY PREDICTOR × HORIZON (lower = better; 'All' = overall mean):") +print("━" * 72) +print(ph_crps.round(2).to_string()) +``` + +## Cell 18 (code) + +```python +# Heatmap of the table above. Read it left-to-right: the short-horizon columns +# are usually a near-uniform green (everyone is right), and one long-horizon +# column carries the colour spread that sets the 'All' ranking. +viz.make_crps_heatmap(ph_crps) +``` + +## Cell 19 (code) + +```python +# Same leaderboard, now with a standard-error bar on each mean. If the bars of +# the top methods overlap, their ordering is not statistically distinguishable — +# the honest verdict when only a few origins have been scored. +viz.make_leaderboard_interval_chart(eval_board) +``` + +## Cell 20 (markdown) + +--- +## 8. What are the top methods actually forecasting? + +A CRPS number doesn't show *behaviour*. Below, each leading method's **median +forecast and 80% interval** are drawn against the realised WTI path at every +eval origin. This is where the leaderboard becomes legible — watch for who +tracks the move, who simply anchors to the last price, and whose intervals are +too narrow to cover the outcome when the market jumps. + +## Cell 21 (code) + +```python +# Plot the leaderboard's top methods, and always include the best LLM/agent +# method for contrast (so the chart compares families even when a baseline leads). +_leaders = list(eval_board.index[:3]) +_best_llm = next((p for p in eval_board.index if eval_board.loc[p, "family"] == "LLM / Agent"), None) +if _best_llm and _best_llm not in _leaders: + _leaders.append(_best_llm) +print(f"Showing: {', '.join(_leaders)}") +viz.make_eval_forecast_chart(eval_frame, price_df, _leaders) +``` + +## Cell 22 (markdown) + +--- +## 9. Reading the agent's reasoning + +The news-reading agent attaches a free-text **rationale** to every forecast, and +a link to the full **Langfuse trace**. These are pulled straight from the +prediction metadata. This is where a surprising score becomes interpretable: you +can read whether the agent actually saw the geopolitical risk, and *how* it +turned that into a price and an interval — including, often, an interval far too +narrow for a regime shift. + +## Cell 23 (code) + +```python +# One card per (agent, origin): the rationale, the per-horizon note, and a link +# to the full reasoning trace. Empty only if no LLM/agent predictor is enabled. +eval_rationales = extract_agent_rationales(eval_results) +display(HTML(viz.render_rationales_html(eval_rationales))) +``` + +## Cell 24 (markdown) + +--- +## 10. Takeaways — computed from this run + +The summary below is **generated from the eval results in memory, not +hard-coded**, so it always matches what actually ran: the real winner, whether +its lead clears the noise floor, the horizon that decided the ranking, the +best-performing family, and a calibration line. Flip `SMOKE_TEST` off, rerun, +and these takeaways update themselves with the full leaderboard. + +## Cell 25 (code) + +```python +display(Markdown(eval_narrative_md(eval_frame, smoke=SMOKE_TEST))) +``` + +## Cell 26 (markdown) --- -## 8. What stateless methods can't do +## 11. What stateless methods can't do -Every method here is calibrated (or prompted) once and never updated between -rounds. This is intentional — it creates a clean baseline — but it leaves a -systematic gap: +Sections 7–10 score and dissect this run on its own terms. But every method here +shares one structural limit, independent of who topped the leaderboard: it is +calibrated (or prompted) **once and never updated between rounds**. That is +intentional — it creates a clean baseline — but it leaves a systematic gap: -- **No error feedback.** If a method consistently produces intervals that are too - narrow in elevated-vol regimes, it keeps making the same mistake. There is no - mechanism to update calibration between rounds. +- **No error feedback.** If a method's intervals are consistently too narrow in + an elevated-vol regime (read the coverage line in Section 10, and the squashed + error bars in Section 8), it keeps making the same mistake. Nothing updates its + calibration between origins. -- **No strategy evolution.** Each prediction starts from the same prior (the same - fitted model, or the same prompt). Resolved outcomes disappear without +- **No strategy evolution.** Each prediction starts from the same prior — the + same fitted model, or the same prompt. Resolved outcomes disappear without influencing future forecasts. - **Context without memory.** Even the news agent re-reads the world each origin; - it does not accumulate what worked. + it does not accumulate what worked. The rationales in Section 9 are written + fresh every time, with no record of how the last one resolved. -→ **Notebook 5** introduces adaptive agents that study the 2025 backtest, -record systematic observations, and calibrate their strategies accordingly. At -inference time, each agent receives the live stateless estimate and decides how -to adjust it — applying what it learned from training. +→ **Notebook 5** introduces adaptive agents that study the 2025 backtest, record +systematic observations, and calibrate their strategies accordingly. At inference +time, each agent receives the live stateless estimate and decides how to adjust +it — applying what it learned from training. → **Notebook 6** evaluates whether any training approach actually improved -out-of-sample performance on the held-out 2026 data. +out-of-sample performance on the held-out 2026 data — measured against the +stateless baseline this notebook just established. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md index 672e8a9f..fa126a84 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md @@ -103,7 +103,7 @@ This has a concrete implication for this evaluation: fetch data via yfinance and reason from what it computed — not from memorized facts about 2025 WTI prices. -- The **evaluation period** (Feb–Mar 2026) is definitively post-cutoff. +- The **evaluation period** (Feb–Jun 2026) is definitively post-cutoff. During eval, the agent must rely entirely on: 1. Live Google Search (with `cutoff_date` enforcement per origin) 2. Code execution (for statistical analysis of fetched data) diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md index 33691c76..ceb8021c 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md @@ -8,13 +8,14 @@ kind: notebook **If you're not sure what to do next, continue from here.** -This notebook is a fresh, hackable agent for the WTI crude-oil use case — deliberately *not* wired into the numbered curriculum. It gives you our common building blocks behind simple toggles, so you can start building something of your own: +This notebook is a fresh, hackable agent for the WTI crude-oil use case — deliberately *not* wired into the numbered curriculum. An agent is a **persona** plus a **toolbelt**, and you assemble that toolbelt right here in the notebook from a menu of one-line tool factories: -- **optional news search** — bounded, cutoff-aware Google Search (proxy-only) -- **optional code execution** — an E2B Python sandbox -- **two lightweight skills** — *tool-usage playbooks* in `starter_agent/skills/` +- **`news_search()`** — bounded, cutoff-aware Google Search (proxy-only) +- **`arima_forecast()`** — an AutoARIMA statistical anchor the agent can call directly (no code-gen) +- **`code_sandbox()`** — an E2B Python sandbox for the agent to compute its own diagnostics +- each tool pulls in its own *playbook* skill from `starter_agent/skills/` -It does two things: lets you **talk to the agent** (open-ended, Track 2) and **score one real forecast** (Track 1). The live cells are gated by `RUN_AGENT` so a fresh `Run All` is safe and free; flip it to `True` to actually call the model. +The factories live in `starter_agent/tools.py` — open it to see how a tool is built, or add your own. It does two things: lets you **talk to the agent** (open-ended, Track 2) and **score one real forecast** (Track 1). The live cells are gated by `RUN_AGENT` so a fresh `Run All` is safe and free; flip it to `True` to actually call the model. ## Cell 2 (code) @@ -47,6 +48,7 @@ RUN_AGENT = False from energy_oil_forecasting.starter_agent import ( build_starter_agent_config, build_starter_agent_predictor, + tools, ) @@ -56,24 +58,31 @@ print("RUN_AGENT =", RUN_AGENT, "| model =", AGENT_MODEL) ## Cell 3 (markdown) --- -## 1. Meet your agent +## 1. Build your agent's toolbelt -`build_starter_agent_config` returns an `AgentConfig` with two toggles. The default turns **news search on** (proxy-only, no extra key) and **code execution off** (it needs `E2B_API_KEY` and is slower). Flip them and re-run — the loaded skills follow the enabled tools. +This is where you compose the agent. `build_starter_agent_config` takes a `tools=[...]` list — the toolbelt — and folds each tool onto the agent (its config, its skill, its instructions). **Comment a line to drop a tool; uncomment to add one**, then re-run. That's the whole model: an agent is a persona plus the tools you hand it. ## Cell 4 (code) ```python -config = build_starter_agent_config( - model=AGENT_MODEL, - enable_search=True, # ← cutoff-aware Google Search (proxy-only) - enable_code_exec=False, # ← E2B Python sandbox (needs E2B_API_KEY); try True! -) - -print("Agent:", config.name) -print("Search enabled: ", config.context_retrieval.enabled) -print("Code-exec enabled: ", config.code_execution.enabled) -print("Skills loaded: ", [p.name for p in config.skills_dirs]) -print("\n── System instruction (edit this in starter_agent/agent.py) ──\n") +# ── Your agent's toolbelt ────────────────────────────── +# Each factory returns one tool. Comment a line to drop it, uncomment to add it. +# See starter_agent/tools.py for how each is built — and to write your own. +toolbelt = [ + tools.news_search(), # cutoff-aware Google Search (proxy-only, no extra key) + tools.arima_forecast(), # AutoARIMA anchor — the agent calls a forecast directly, no code-gen + # tools.code_sandbox(), # E2B Python sandbox (needs E2B_API_KEY, slower) — uncomment to add +] + +config = build_starter_agent_config(model=AGENT_MODEL, tools=toolbelt) + +print("Agent: ", config.name) +print("Toolbelt:", [t.label for t in toolbelt]) +print(" search enabled: ", config.context_retrieval.enabled) +print(" forecast tool: ", bool(config.function_tools)) +print(" code-exec enabled:", config.code_execution.enabled) +print("Skills loaded: ", [p.name for p in config.skills_dirs]) +print("\n── System instruction (edit the persona in starter_agent/agent.py) ──\n") print(config.instruction[:1200], "...") ``` @@ -164,11 +173,11 @@ else: This agent is a starting point. Here are concrete next steps, easiest first — each is a small edit, then re-run the cells above. -1. **Flip code execution on.** Set `enable_code_exec=True` in §1 (needs `E2B_API_KEY`). The agent loads the `code-analysis-playbook` skill and can compute its own diagnostics before forecasting. Compare the rationale. +1. **Change the toolbelt.** In §1, uncomment `tools.code_sandbox()` (needs `E2B_API_KEY`) to let the agent compute its own diagnostics, or drop `arima_forecast()` and compare the rationale with and without a statistical anchor. Adding a tool automatically loads its playbook skill and its instructions. 2. **Edit the agent's personality.** Open `starter_agent/agent.py` and change `_build_starter_instruction()` — make it more cautious, more contrarian, focused on one driver. Re-run §1 to see the new instruction. -3. **Sharpen the skills.** The two files in `starter_agent/skills/` are short on purpose. Add your best queries to `research-playbook`, or a new diagnostic to `code-analysis-playbook`. The agent picks them up automatically. +3. **Sharpen the skills.** The files in `starter_agent/skills/` are short on purpose. Add your best queries to `research-playbook`, or a new diagnostic to `code-analysis-playbook`. The agent picks them up automatically. 4. **Change the question and the origin.** Try a different `QUESTION` in §2 and a different origin in §3. -5. **Add a tool.** Give the agent a conventional forecast tool as a statistical anchor — see `analyst_agent.build_wti_tool_config` for the `ForecastTool` pattern. +5. **Write your own tool.** Open `starter_agent/tools.py` and add a factory that returns a `ToolSpec` — point `arima_forecast()` at a different series, swap AutoARIMA for another predictor, or wrap a brand-new function tool. Then add it to the toolbelt in §1. 6. **Score it properly.** Run it across several origins with `backtest()` (see `04_systematic_backtest_eval.ipynb`) and compare CRPS against the baselines. Bigger ideas — an agent that *learns* a strategy (notebooks 05–06), news vs. no-news lift, live prospective forecasting — are in the use-case `README.md` and `planning-docs/roadmap.md`. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md index 1041fba0..d54a066f 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md @@ -52,9 +52,9 @@ The ``wti-strategy`` skill is backed by a :class:`~energy_oil_forecasting.adapti Pydantic model persisted in ``skills/wti-strategy/skill_state.yaml``. ``SKILL.md`` is rendered from that model on every mutation and is never hand-edited. Five typed mutation tools (from :mod:`skill_tools`) are -registered via ``AgentConfig(extra_tools=WTI_SKILL_TOOLS)`` and run in the -host process — not inside E2B. See :mod:`skill_tools` for the full tool -signatures and evidence governance rules. +registered via ``AgentConfig(extra_tools=build_skill_tools(strategy_dir))`` and +run in the host process — not inside E2B. See :mod:`skill_tools` for the full +tool signatures and evidence governance rules. """ from __future__ import annotations @@ -73,156 +73,27 @@ from aieng.forecasting.methods.agentic import ( ContinuousAgentForecastOutput, build_adk_agent, ) -from aieng.forecasting.methods.agentic.agent_factory import ( - AgentConfig, - CodeExecutionConfig, - ContextRetrievalConfig, -) +from aieng.forecasting.methods.agentic.agent_factory import AgentConfig +from aieng.forecasting.methods.agentic.domain import build_adaptive_config from aieng.forecasting.models import ADVANCED_MODEL, LITE_MODEL -from energy_oil_forecasting.adaptive_agent.skill_tools import build_skill_tools +from energy_oil_forecasting.adaptive_agent.skill_state import WtiStrategyState from energy_oil_forecasting.analyst_agent import compress_history +from energy_oil_forecasting.domain import OIL_DOMAIN from pydantic import BaseModel logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Paths -# --------------------------------------------------------------------------- - -_SKILLS_ROOT = Path(__file__).parent / "skills" - - # --------------------------------------------------------------------------- # System prompt # --------------------------------------------------------------------------- - - -def _build_adaptive_analyst_instruction() -> str: - """Build the adaptive analyst instruction with the output schema embedded. - - Uses ``ContinuousAgentForecastOutput.prompt_schema_json()`` for the - prediction-response schema so it stays in sync with the output class. - """ - schema = ContinuousAgentForecastOutput.prompt_schema_json() - return ( - "## Identity\n\n" - "You are a persistent WTI crude oil market analyst. You carry knowledge forward " - "across invocations: your `wti-strategy` skill captures your current forecasting " - "approach, and you update it deliberately as you learn from experience.\n\n" - "## Message types\n\n" - "You receive messages through a single chat interface. Determine from context " - "what kind of invocation this is and respond accordingly:\n\n" - "**Prediction request** — contains a JSON payload with `task`, `as_of`, " - "`horizons`, and price history. Load `wti-strategy` first to read your current " - "approach and any active calibration corrections. Then:\n" - "1. Use `run_code` to run your full statistical analysis pipeline: fetch data " - "via `fetch-yfinance` (using `end=as_of` as the cutoff), classify the vol " - "regime via `vol-regime`, and project trend and intervals via `trend-projection`. " - "Apply any calibration corrections from `wti-strategy` — for example, substituting " - "a flat-trend model in elevated/extreme vol regimes if your strategy calls for it.\n" - "2. Use the context-retrieval tool to gather current market news and adjust your " - "estimates where strong catalysts are present.\n" - "3. Conclude with `set_model_response` (schema below).\n\n" - "Your quantitative pipeline is your starting point — your learned strategy " - "corrections and news-grounded judgment shape the final forecast.\n\n" - "**Resolution** — describes how a past forecast resolved (actual value, error, " - "horizon). Reflect carefully. If the error points to a systematic pattern — not " - "a one-off surprise — consult `meta-learning` to assess whether a strategy update " - "is warranted.\n\n" - "**Self-review / backtesting** — you are asked to analyse your recent performance " - "or explore historical data using code execution. Compose the relevant skills, " - "write one complete code block, and summarise what you find. If the analysis " - "surfaces a durable insight, follow the `meta-learning` process.\n\n" - "**User question** — a human is asking for analysis, context, or your market " - "view. Engage directly, using code execution and web search as needed.\n\n" - "## Skills are pipeline components\n\n" - "Your skills cover specific pipeline stages. Compose them: for any task " - "involving code, load each relevant skill and its `references/examples.md`, " - "then write one complete self-contained code block combining all the patterns.\n\n" - "| Skill | Pipeline stage |\n" - "|------------------|---------------------------------------------------------|\n" - "| fetch-yfinance | Download market / futures data from Yahoo Finance |\n" - "| vol-regime | Classify vol regime, detect anomalies, choose window |\n" - "| trend-projection | Fit trend, project to horizons, calibrate intervals |\n" - "| wti-strategy | Your current forecasting strategy — load at the start of every prediction |\n" - "| meta-learning | Governs when and how to update wti-strategy |\n\n" - "## Strategy mutation tools\n\n" - "These tools write directly to `wti-strategy` on the host filesystem. " - "They run outside the E2B sandbox. Consult `meta-learning` before calling " - "any of them.\n\n" - "| Tool | Evidence layer | Evidence bar |\n" - "|------|---------------|---------------|\n" - "| `record_observation(finding, linked_hypothesis?)` | Observations | Pattern visible across ≥2 forecasts — not a single surprise |\n" - "| `open_hypothesis(claim, initial_evidence)` | Hypotheses | One strong observation suggesting a durable pattern |\n" - "| `record_hypothesis_outcome(hypothesis_id, outcome)` | Hypotheses | Each resolution relevant to an open hypothesis |\n" - "| `graduate_hypothesis(hypothesis_id, condition, adjustment, horizon_scope)` | Calibration | Tool enforces confirmation threshold — will reject if not met |\n" - "| `update_approach_narrative(new_text, rationale)` | Approach | Only when the calibration record reveals a structural insight |\n\n" - "Active calibration corrections from `wti-strategy` are **not optional** — " - "apply every listed correction when the stated condition is met.\n\n" - "## Code execution discipline\n\n" - "Treat `run_code` like submitting to a batch queue: plan your complete " - "analysis upfront, write one self-contained script, and read the results. " - "There is no REPL, no way to inspect intermediate state between calls, and " - "no benefit to splitting work — each submission starts from zero with no " - "memory of previous calls.\n\n" - "Never make a preliminary or test call to check connectivity or verify " - "imports. Assume the environment works. Your first `run_code` call should " - "produce your complete result.\n\n" - "Pre-installed: numpy, pandas, sklearn, yfinance, statsmodels, properscoring.\n\n" - "**Data sourcing rule:** Always use the `fetch-yfinance` skill to load price " - "data inside `run_code`. **Never embed `target_history_csv` or any CSV " - "string literal as a data source in code.** Pasting thousands of rows of " - "data as Python string literals is fragile, wastes context, and risks hitting " - "sandbox limits. `target_history_csv` is provided in the prediction payload " - "for your reading and statistical summary only — not for copy-pasting into " - "code blocks. When a skill description says 'assume `df` is already defined', " - "that means you should define `df` via a yfinance fetch at the top of your " - "script, not by embedding raw data.\n\n" - "## Temporal discipline\n\n" - "Every forecast is anchored to an `as_of` date. Never use information beyond " - "that date — in web search, code analysis, or reasoning.\n\n" - "When fetching data inside `run_code`, always pass `end=as_of_date` to " - "yfinance to enforce the temporal cutoff — for example:\n\n" - "```python\nraw = ticker.history(start='2004-01-01', end='2026-02-16', " - "auto_adjust=False)\n```\n\n" - "Replace the end date with the actual `as_of` value from the prediction " - "payload. This is the only correct way to ensure the sandbox sees the same " - "data the agent would have seen on that date.\n\n" - "## Prediction output schema\n\n" - "For **prediction requests**, call `set_model_response` with `json_response` " - "matching **exactly**:\n\n" - "```json\n" + schema + "\n```\n\n" - 'Critical: use `"horizon"` (integer, not `"horizon_days"`). ' - '`"quantiles"` is a **list** of `{"quantile": , "value": }` ' - "objects — not a dict." - ) - - -_ADAPTIVE_ANALYST_INSTRUCTION = _build_adaptive_analyst_instruction() - - -# --------------------------------------------------------------------------- -# Context retrieval instruction -# --------------------------------------------------------------------------- - -_WTI_CONTEXT_RETRIEVAL_INSTRUCTION = """\ -You are an oil market intelligence specialist with access to web search. - -Search for information relevant to the query and return a concise structured \ -markdown summary (3-5 paragraphs) covering relevant aspects of: -- WTI/Brent crude price level and recent trend -- OPEC+ production decisions and supply outlook -- Geopolitical risks in the Persian Gulf, Middle East, key shipping lanes -- US Strategic Petroleum Reserve and energy policy signals -- Notable tanker/shipping incidents or supply disruption signals -- Published analyst forecasts or unusual price-target revisions - -Ground your summary in the search results you actually retrieve. \ -When a cutoff date is specified, do not report or speculate about events \ -that occurred after that date.\ -""" +# +# The adaptive analyst instruction and the web-search sub-agent instruction are +# rendered from the shared templates over the WTI ``OIL_DOMAIN`` fragments (see +# :mod:`energy_oil_forecasting.domain` and +# :func:`aieng.forecasting.methods.agentic.domain.build_adaptive_config`). A new +# target series is configured by supplying a different ``DomainConfig``. # --------------------------------------------------------------------------- @@ -315,30 +186,19 @@ def build_wti_adaptive_config( ------- AgentConfig """ - resolved_strategy_dir = strategy_dir or (_SKILLS_ROOT / "wti-strategy") - # Include strategy dir name in agent name so cached_multi_backtest writes a - # separate cache file per variant (cache key is derived from predictor_id, - # which is derived from agent name). - agent_name = f"wti_adaptive_analyst_{resolved_strategy_dir.name.replace('-', '_')}" - return AgentConfig( - name=agent_name, + # Delegates to the shared, domain-agnostic builder with the WTI domain and + # ``WtiStrategyState`` so the rendered SKILL.md keeps its oil branding. The + # strategy dir name is baked into the agent name (cache key is derived from + # predictor_id, which is derived from agent name) so per-variant prediction + # caches stay separate. + return build_adaptive_config( + OIL_DOMAIN, + state_type=WtiStrategyState, model=model, - instruction=_ADAPTIVE_ANALYST_INSTRUCTION, + search_model=search_model, max_output_tokens=max_output_tokens, - context_retrieval=ContextRetrievalConfig( - enabled=True, - instruction=_WTI_CONTEXT_RETRIEVAL_INSTRUCTION, - search_model=search_model, - ), - code_execution=CodeExecutionConfig(enabled=True), - skills_dirs=[ - _SKILLS_ROOT / "fetch-yfinance", - _SKILLS_ROOT / "vol-regime", - _SKILLS_ROOT / "trend-projection", - resolved_strategy_dir, - _SKILLS_ROOT / "meta-learning", - ], - extra_tools=build_skill_tools(resolved_strategy_dir, confirmation_threshold=2), + strategy_dir=strategy_dir, + confirmation_threshold=2, ) diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__skill_state.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__skill_state.py.md index d897e5c7..71868028 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__skill_state.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__skill_state.py.md @@ -6,211 +6,55 @@ kind: python """WTI forecasting strategy state model. Defines the structured state backing the ``wti-strategy`` adaptive skill. -``WtiStrategyState`` is the single source of truth for the agent's current -forecasting approach. It is persisted to ``skills/wti-strategy/skill_state.yaml`` -and rendered to ``skills/wti-strategy/SKILL.md`` on every mutation so that the -ADK ``SkillToolset`` always reads an up-to-date version. - -Learning layers ---------------- -The four fields of ``WtiStrategyState`` map to distinct update frequencies and -evidence burdens, enforced partly by the mutation tools in ``skill_tools.py`` -and partly by the ``meta-learning`` governance skill: - -``observations`` - Append-only log of pattern-level findings. Lowest evidence bar — record - any finding that is not a single-outlier surprise. - -``hypotheses`` - Candidate systematic corrections the agent is actively testing. Open a - hypothesis when you suspect a durable pattern. Accumulate confirmation / - refutation counts across resolutions. A hypothesis graduates to a - calibration correction when its confirmation count reaches the store's - ``confirmation_threshold``. - -``calibration_corrections`` - Confirmed systematic adjustments applied at prediction time. Each entry - is graduated from a confirmed hypothesis — never added directly. - -``approach_narrative`` - Free-text description of the agent's overall forecasting philosophy. - Highest evidence bar. Update only when the calibration record reveals a - structural insight that the narrative no longer captures. +``WtiStrategyState`` is a thin oil-branded subclass of the domain-agnostic +:class:`~aieng.forecasting.methods.agentic.strategy_state.StrategyState`: it +pins the rendered ``SKILL.md`` heading, default skill name, and frontmatter +description to the WTI wording so committed artifacts round-trip byte-identically. +All fields, sub-models, and rendering logic live in the shared library. + +The state is persisted to ``skills/wti-strategy/skill_state.yaml`` and rendered +to ``skills/wti-strategy/SKILL.md`` on every mutation so that the ADK +``SkillToolset`` always reads an up-to-date version. See the shared +:mod:`aieng.forecasting.methods.agentic.strategy_state` module for the +learning-layer hierarchy and evidence burdens. """ from __future__ import annotations -from typing import Literal - -from aieng.forecasting.methods.agentic.adaptive_skill import AdaptiveSkillState -from pydantic import BaseModel - - -# --------------------------------------------------------------------------- -# Sub-models -# --------------------------------------------------------------------------- - - -class Observation(BaseModel): - """A single pattern-level finding from a resolution or self-review.""" - - date: str - finding: str - linked_hypothesis: str | None = None - - -class Hypothesis(BaseModel): - """A candidate systematic correction under active testing. - - ``status`` progresses through ``open`` → ``confirmed`` or ``open`` → - ``refuted``. Confirmed hypotheses are graduated to - :class:`CalibrationCorrection` via the ``graduate_hypothesis`` tool. - """ - - id: str - claim: str - status: Literal["open", "confirmed", "refuted"] = "open" - confirmations: int = 0 - refutations: int = 0 - opened_on: str - +from typing import ClassVar -class CalibrationCorrection(BaseModel): - """A confirmed systematic adjustment applied at prediction time. +# Re-exported so existing imports of the sub-models from this module keep working. +from aieng.forecasting.methods.agentic.strategy_state import ( + CalibrationCorrection, + Hypothesis, + Observation, + StrategyState, + VersionEntry, +) - Every entry here was graduated from a confirmed hypothesis; the - ``source_hypothesis`` field preserves that lineage. - """ - - condition: str - adjustment: str - horizon_scope: str - source_hypothesis: str - confirmed_on: str - - -class VersionEntry(BaseModel): - """One row in the version history table.""" - - date: str - description: str +class WtiStrategyState(StrategyState): + """Oil-branded strategy state for the adaptive WTI crude oil analyst. -# --------------------------------------------------------------------------- -# Strategy state -# --------------------------------------------------------------------------- - - -class WtiStrategyState(AdaptiveSkillState): - """Structured state for the adaptive WTI crude oil forecasting strategy. - - See module docstring for the learning-layer hierarchy and evidence burdens. + Identical in structure to :class:`StrategyState`; only the presentation + strings are pinned so the ``wti-strategy`` artifacts render exactly as + committed. """ - approach_narrative: str - calibration_corrections: list[CalibrationCorrection] = [] - hypotheses: list[Hypothesis] = [] - observations: list[Observation] = [] - version_history: list[VersionEntry] = [] - - def build_markdown(self, skill_name: str | None = None) -> str: # noqa: PLR0912 - """Render the full ``SKILL.md`` content from current state.""" - lines: list[str] = [] - - # Frontmatter — skill_name must match the containing directory name (ADK requirement) - lines += [ - "---", - f"name: {skill_name or 'wti-strategy'}", - "description: >-", - " The adaptive WTI analyst's current forecasting strategy. Load this at the", - " start of every prediction task. This file is generated — edit the state", - " through the mutation tools, not by hand.", - "---", - "", - ] - - lines += [ - "# WTI Forecasting Strategy", - "", - "## Approach", - "", - self.approach_narrative.strip(), - "", - ] - - # Active calibration corrections - lines += [ - "## Active calibration corrections", - "", - ] - if self.calibration_corrections: - lines += [ - "| Condition | Adjustment | Horizon scope | Confirmed on |", - "|-----------|-----------|---------------|--------------|", - ] - for c in self.calibration_corrections: - lines.append(f"| {c.condition} | {c.adjustment} | {c.horizon_scope} | {c.confirmed_on} |") - else: - lines.append("*(No calibration corrections yet. Graduate a confirmed hypothesis to add one.)*") - lines.append("") - - # Open hypotheses - lines += [ - "## Open hypotheses", - "", - ] - open_hyps = [h for h in self.hypotheses if h.status == "open"] - if open_hyps: - lines += [ - "| ID | Claim | Confirmations | Refutations |", - "|----|-------|---------------|-------------|", - ] - for h in open_hyps: - lines.append(f"| {h.id} | {h.claim} | {h.confirmations} | {h.refutations} |") - else: - lines.append("*(No open hypotheses.)*") - lines.append("") - - # Closed hypotheses (confirmed / refuted) — collapsed for readability - closed_hyps = [h for h in self.hypotheses if h.status != "open"] - if closed_hyps: - lines += [ - "## Closed hypotheses", - "", - "| ID | Claim | Status | Confirmations | Refutations |", - "|----|-------|--------|---------------|-------------|", - ] - for h in closed_hyps: - lines.append(f"| {h.id} | {h.claim} | {h.status} | {h.confirmations} | {h.refutations} |") - lines.append("") - - # Observations - lines += [ - "## Observations", - "", - ] - if self.observations: - lines += [ - "| Date | Finding | Linked hypothesis |", - "|------|---------|-------------------|", - ] - for o in self.observations: - linked = o.linked_hypothesis or "—" - lines.append(f"| {o.date} | {o.finding} | {linked} |") - else: - lines.append("*(No observations yet. Record findings from resolutions and self-reviews.)*") - lines.append("") - - # Version history - lines += [ - "## Version history", - "", - "| Date | Change |", - "|------|--------|", - ] - for v in self.version_history: - lines.append(f"| {v.date} | {v.description} |") - lines.append("") - - return "\n".join(lines) + markdown_title: ClassVar[str] = "WTI Forecasting Strategy" + default_skill_name: ClassVar[str] = "wti-strategy" + frontmatter_description_lines: ClassVar[tuple[str, ...]] = ( + "The adaptive WTI analyst's current forecasting strategy. Load this at the", + "start of every prediction task. This file is generated — edit the state", + "through the mutation tools, not by hand.", + ) + + +__all__ = [ + "CalibrationCorrection", + "Hypothesis", + "Observation", + "VersionEntry", + "WtiStrategyState", +] ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__skill_tools.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__skill_tools.py.md index 8469396a..b6a242b2 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__skill_tools.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__skill_tools.py.md @@ -5,67 +5,26 @@ kind: python ```python """Mutation tools for the ``wti-strategy`` adaptive skill. -These are plain Python callables registered as ADK ``FunctionTool`` objects via -``AgentConfig(extra_tools=build_skill_tools(strategy_dir))``. They run in the -host process — *not* inside the E2B sandbox — so they can read and write the -skill directory on the local filesystem. - -Factory pattern ---------------- -Use :func:`build_skill_tools` to create a set of tools bound to a specific -strategy directory. This allows multiple named strategy variants (e.g. -``wti-strategy-stats``, ``wti-strategy-news``) to coexist, each with its own -``AdaptiveSkillStore`` and ``skill_state.yaml``. - -The module-level :data:`STORE` and :data:`WTI_SKILL_TOOLS` are convenience -bindings to the **default** ``wti-strategy`` directory for backward -compatibility and interactive ``adk web`` use. - -Design principles ------------------ -Each tool follows the same three-step cycle: - -1. ``store.load()`` — deserialise current state from ``skill_state.yaml``. -2. Apply one typed mutation to the state model. -3. ``store.save(state)`` — write YAML, re-render ``SKILL.md``, back up. - -Tool signatures are intentionally narrow: they accept only the arguments -needed for one specific mutation. The agent cannot write arbitrary content to -the skill directory through any of these tools. - -Evidence governance -------------------- -``record_observation`` - No guard. Record any pattern-level finding (not a single-outlier surprise). - -``open_hypothesis`` - No guard. Open a hypothesis whenever you suspect a durable pattern. - -``record_hypothesis_outcome`` - Validates the hypothesis ID exists and is still open. - -``graduate_hypothesis`` - Hard guard: rejected if ``hypothesis.confirmations < store.confirmation_threshold``. - Returns a clear message stating the shortfall. - -``update_approach_narrative`` - Requires a ``rationale`` argument but no hard numeric guard. The - ``meta-learning`` skill governs when this is appropriate. - -Scope guard ------------ -All writes go through the :class:`~aieng.forecasting.methods.agentic.adaptive_skill.AdaptiveSkillStore` -instance passed to :func:`build_skill_tools`. No path outside that directory -can be reached. +Thin oil-side wrapper over the shared, domain-agnostic tool factory +:func:`aieng.forecasting.methods.agentic.adaptive_skill_tools.build_skill_tools`. +The wrapper binds the factory to :class:`WtiStrategyState` so the rendered +``SKILL.md`` keeps its WTI branding. The five mutation tools, their evidence +governance, and the scope guard are documented on the shared module. + +The sub-model re-exports below keep existing imports of ``Observation``, +``Hypothesis``, ``CalibrationCorrection``, and ``VersionEntry`` from this module +working. There are no import-time singletons: nothing touches the filesystem +until :func:`build_skill_tools` is called. """ from __future__ import annotations -from datetime import date from pathlib import Path from typing import Callable -from aieng.forecasting.methods.agentic.adaptive_skill import AdaptiveSkillStore +from aieng.forecasting.methods.agentic.adaptive_skill_tools import ( + build_skill_tools as _build_skill_tools, +) from energy_oil_forecasting.adaptive_agent.skill_state import ( CalibrationCorrection, Hypothesis, @@ -75,45 +34,17 @@ from energy_oil_forecasting.adaptive_agent.skill_state import ( ) -# --------------------------------------------------------------------------- -# Default strategy directory (for backward compat and adk web) -# --------------------------------------------------------------------------- - -_SKILL_DIR = Path(__file__).parent / "skills" / "wti-strategy" - - -# --------------------------------------------------------------------------- -# Stateless helpers -# --------------------------------------------------------------------------- - - -def _today() -> str: - return str(date.today()) - - -def _next_hypothesis_id(state: WtiStrategyState) -> str: - """Return the next sequential hypothesis ID (e.g. ``hyp-004``).""" - n = len(state.hypotheses) + 1 - return f"hyp-{n:03d}" - - -# --------------------------------------------------------------------------- -# Tool factory -# --------------------------------------------------------------------------- - - -def build_skill_tools( # noqa: PLR0915 +def build_skill_tools( strategy_dir: Path, *, confirmation_threshold: int = 3, ) -> list[Callable[..., str]]: - """Build a set of strategy mutation tools bound to *strategy_dir*. + """Build the five WTI strategy mutation tools bound to *strategy_dir*. - Each call returns five fresh callables (closures over a new - :class:`~aieng.forecasting.methods.agentic.adaptive_skill.AdaptiveSkillStore` - instance). Pass the returned list to - ``AgentConfig(extra_tools=build_skill_tools(strategy_dir))`` to wire the - tools into an agent that operates on a specific strategy variant. + Delegates to the shared factory with :class:`WtiStrategyState` as the state + type. See + :func:`aieng.forecasting.methods.agentic.adaptive_skill_tools.build_skill_tools` + for the full tool signatures and evidence-governance rules. Parameters ---------- @@ -130,296 +61,19 @@ def build_skill_tools( # noqa: PLR0915 ``[record_observation, open_hypothesis, record_hypothesis_outcome, graduate_hypothesis, update_approach_narrative]`` """ - store: AdaptiveSkillStore[WtiStrategyState] = AdaptiveSkillStore( - skill_dir=strategy_dir, - state_type=WtiStrategyState, + return _build_skill_tools( + strategy_dir, + WtiStrategyState, confirmation_threshold=confirmation_threshold, ) - def record_observation(finding: str, linked_hypothesis: str = "") -> str: - """Record a pattern-level finding from a resolution or self-review. - - Call this whenever you observe a systematic pattern across multiple - forecasts — not after a single surprising outcome. - - Parameters - ---------- - finding : str - A concise description of the pattern observed. Be specific: include - the regime, horizon, and direction of the error where applicable. - Example: "80% intervals missed 4 of 5 actuals in the elevated vol - regime at the 21-day horizon." - linked_hypothesis : str, optional - ID of an existing open hypothesis this observation supports or - refutes (e.g. ``"hyp-001"``). Leave blank if this is a fresh - observation not yet linked to any hypothesis. - - Returns - ------- - str - Confirmation message. - """ - state = store.load() - obs = Observation( - date=_today(), - finding=finding.strip(), - linked_hypothesis=linked_hypothesis.strip() or None, - ) - state.observations.append(obs) - store.save(state) - linked_note = f" (linked to {obs.linked_hypothesis})" if obs.linked_hypothesis else "" - return f'Observation recorded{linked_note}: "{finding[:80]}{"..." if len(finding) > 80 else ""}"' - - def open_hypothesis(claim: str, initial_evidence: str) -> str: - """Open a new hypothesis about a suspected systematic forecasting pattern. - - A hypothesis is a candidate calibration correction under active testing. - Open one when you have at least one observation suggesting a durable - pattern but do not yet have enough confirming resolutions to graduate it. - - Parameters - ---------- - claim : str - A testable claim about your forecasting behaviour. State it in terms - of a specific condition and a directional error. - Example: "My 80% prediction intervals are consistently too narrow - when the vol regime is classified as elevated or extreme." - initial_evidence : str - The observation(s) that motivated opening this hypothesis. This is - for the audit record — be specific about the number of data points. - - Returns - ------- - str - Confirmation message including the assigned hypothesis ID. - """ - state = store.load() - hyp_id = _next_hypothesis_id(state) - hyp = Hypothesis( - id=hyp_id, - claim=claim.strip(), - status="open", - confirmations=0, - refutations=0, - opened_on=_today(), - ) - state.hypotheses.append(hyp) - obs = Observation( - date=_today(), - finding=initial_evidence.strip(), - linked_hypothesis=hyp_id, - ) - state.observations.append(obs) - store.save(state) - return ( - f'Hypothesis {hyp_id} opened: "{claim[:80]}{"..." if len(claim) > 80 else ""}". ' - f"Initial evidence recorded as an observation linked to {hyp_id}. " - f"Confirmations needed to graduate: {store.confirmation_threshold}." - ) - - def record_hypothesis_outcome(hypothesis_id: str, outcome: str) -> str: - """Record a confirming or refuting outcome for an open hypothesis. - - Call this after each resolution where the outcome is directly relevant - to an open hypothesis. Accumulate enough confirmations to graduate. - - Parameters - ---------- - hypothesis_id : str - ID of the hypothesis to update (e.g. ``"hyp-001"``). - outcome : str - Either ``"confirmed"`` or ``"refuted"``. A single refutation does - not automatically close the hypothesis — continue accumulating - evidence. A hypothesis should be manually closed (status → - ``"refuted"``) only when refutations clearly outweigh confirmations - across a meaningful sample. - - Returns - ------- - str - Updated confirmation / refutation counts and progress toward the - graduation threshold. - """ - if outcome not in ("confirmed", "refuted"): - return f"Invalid outcome '{outcome}'. Must be 'confirmed' or 'refuted'." - - state = store.load() - hyp = next((h for h in state.hypotheses if h.id == hypothesis_id), None) - if hyp is None: - ids = [h.id for h in state.hypotheses] - return f"Hypothesis '{hypothesis_id}' not found. Known IDs: {ids}." - if hyp.status != "open": - return f"Hypothesis {hypothesis_id} is already {hyp.status}. Only open hypotheses can receive new outcomes." - - if outcome == "confirmed": - hyp.confirmations += 1 - else: - hyp.refutations += 1 - - store.save(state) - - remaining = max(0, store.confirmation_threshold - hyp.confirmations) - if remaining == 0: - ready_msg = ( - f" Ready to graduate — call graduate_hypothesis('{hypothesis_id}', ...) " - "with a condition, adjustment, and horizon_scope." - ) - else: - ready_msg = f" {remaining} more confirmation(s) needed to graduate." - - return ( - f"{hypothesis_id} updated: {hyp.confirmations} confirmation(s), {hyp.refutations} refutation(s).{ready_msg}" - ) - - def graduate_hypothesis( - hypothesis_id: str, - condition: str, - adjustment: str, - horizon_scope: str, - ) -> str: - """Graduate a confirmed hypothesis to an active calibration correction. - - This is the primary mechanism through which the agent's strategy - improves. A calibration correction is applied at every future - prediction; it is not merely recorded — it changes behaviour. - - This tool enforces the confirmation threshold: it will reject the call - if the hypothesis has not accumulated enough confirming outcomes. - - Parameters - ---------- - hypothesis_id : str - ID of the confirmed hypothesis to graduate (e.g. ``"hyp-001"``). - condition : str - The specific condition under which this correction applies. - Example: "vol regime is elevated or extreme". - adjustment : str - The concrete adjustment to make when the condition is met. - Example: "Widen 80% CI by 12% relative to the statistical model - output." - horizon_scope : str - Which horizons this correction applies to. - One of: ``"all"``, ``"5bd"``, ``"10bd"``, ``"21bd"``, or a - combination like ``"10bd and 21bd"``. - - Returns - ------- - str - Confirmation message, or a rejection message with the shortfall. - """ - state = store.load() - hyp = next((h for h in state.hypotheses if h.id == hypothesis_id), None) - if hyp is None: - ids = [h.id for h in state.hypotheses] - return f"Hypothesis '{hypothesis_id}' not found. Known IDs: {ids}." - if hyp.status != "open": - return f"Hypothesis {hypothesis_id} is already {hyp.status}. Only open hypotheses can be graduated." - - if hyp.confirmations < store.confirmation_threshold: - shortfall = store.confirmation_threshold - hyp.confirmations - return ( - f"Cannot graduate {hypothesis_id}: " - f"{hyp.confirmations} confirmation(s), " - f"requires {store.confirmation_threshold}. " - f"Record {shortfall} more confirming outcome(s) first." - ) - - today = _today() - hyp.status = "confirmed" - correction = CalibrationCorrection( - condition=condition.strip(), - adjustment=adjustment.strip(), - horizon_scope=horizon_scope.strip(), - source_hypothesis=hypothesis_id, - confirmed_on=today, - ) - state.calibration_corrections.append(correction) - state.observations.append( - Observation( - date=today, - finding=( - f"Graduated {hypothesis_id} to calibration correction: " - f"'{condition}' → '{adjustment}' ({horizon_scope})." - ), - linked_hypothesis=hypothesis_id, - ) - ) - state.version_history.append( - VersionEntry( - date=today, - description=( - f"Graduated {hypothesis_id} to calibration correction " - f"(condition: {condition[:50]}{'...' if len(condition) > 50 else ''})." - ), - ) - ) - store.save(state) - return ( - f"Hypothesis {hypothesis_id} confirmed and graduated. " - f"Calibration correction added: when '{condition}', apply '{adjustment}' " - f"(scope: {horizon_scope})." - ) - - def update_approach_narrative(new_text: str, rationale: str) -> str: - """Replace the approach narrative with an updated strategic description. - - This is the highest-evidence-bar update. Consult ``meta-learning`` - before calling this tool — the narrative should only change when the - calibration record reveals a structural insight that the current - description no longer captures. A ``rationale`` argument is required - to force articulation of why the change is warranted. - - Parameters - ---------- - new_text : str - The complete replacement text for the ``## Approach`` section. - Write it as a self-contained description of the current forecasting - strategy — what signals are used, in what order, and with what - emphasis. - rationale : str - Why this update is warranted now. Cite the specific calibration - corrections or pattern of observations that motivated the change. - - Returns - ------- - str - Confirmation message. - """ - if not new_text.strip(): - return "new_text must not be empty." - if not rationale.strip(): - return "rationale must not be empty. Explain why the approach narrative warrants an update." - - state = store.load() - today = _today() - state.approach_narrative = new_text.strip() - state.version_history.append( - VersionEntry( - date=today, - description=f"Updated approach narrative. Rationale: {rationale[:120]}{'...' if len(rationale) > 120 else ''}", - ) - ) - store.save(state) - return f"Approach narrative updated ({len(new_text)} chars). Rationale recorded in version history." - - return [ - record_observation, - open_hypothesis, - record_hypothesis_outcome, - graduate_hypothesis, - update_approach_narrative, - ] - - -# --------------------------------------------------------------------------- -# Backward-compatible module-level bindings (default wti-strategy dir) -# --------------------------------------------------------------------------- - -STORE: AdaptiveSkillStore[WtiStrategyState] = AdaptiveSkillStore( - skill_dir=_SKILL_DIR, - state_type=WtiStrategyState, - confirmation_threshold=3, -) -WTI_SKILL_TOOLS: list[Callable[..., str]] = build_skill_tools(_SKILL_DIR) +__all__ = [ + "CalibrationCorrection", + "Hypothesis", + "Observation", + "VersionEntry", + "WtiStrategyState", + "build_skill_tools", +] ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md index 0d07636d..6058a6d3 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md @@ -166,9 +166,292 @@ def select_top_predictors( return [str(x) for x in leaderboard.head(n)["predictor_id"].tolist()] +# ── Per-prediction diagnostics ──────────────────────────────────────────────── +# The leaderboard collapses every forecast into a single mean CRPS. To understand +# *why* a method wins or loses we need the predictions un-aggregated: one row per +# (predictor, origin, horizon) carrying the point estimate, the 80% interval, the +# realised outcome, and the CRPS the harness assigned. Everything below builds on +# this tidy frame so the notebook charts and the written narrative read from the +# same numbers. + +# Map of predictor-name fragments → family label, checked in order. Used to group +# the leaderboard into baselines / numerical-ML / LLM-agent without hard-coding a +# per-predictor table (the registry can grow without touching this). +_FAMILY_RULES: list[tuple[tuple[str, ...], str]] = [ + (("naive", "last value"), "Baseline"), + (("arima", "lightgbm", "prophet"), "Numerical ML"), + (("llmp", "agent", "news", "llm"), "LLM / Agent"), +] + + +def predictor_family(name: str) -> str: + """Classify a predictor display name into a forecasting family.""" + low = name.lower() + for fragments, label in _FAMILY_RULES: + if any(frag in low for frag in fragments): + return label + return "Other" + + +def _qval(quantiles: dict[Any, float], q: float) -> float: + """Read a quantile value tolerating both float and string dict keys.""" + for key in (q, str(q), f"{q:.2f}"): + if key in quantiles: + return float(quantiles[key]) + return float("nan") + + +def _business_horizon(as_of: pd.Timestamp, forecast_date: pd.Timestamp) -> int: + """Trading-day distance between an information cutoff and a forecast date.""" + return max(len(pd.bdate_range(as_of.normalize(), forecast_date.normalize())) - 1, 0) + + +def predictions_to_frame( + results_by_predictor: dict[str, dict[str, BacktestResult]], + data_service: DataService, + *, + actuals_as_of: datetime | None = None, +) -> pd.DataFrame: + """Explode backtest results into one tidy row per scored prediction. + + Parameters + ---------- + results_by_predictor + ``{display_name: {task_id: BacktestResult}}`` — exactly the shape the + notebook holds in ``eval_results`` / ``backtest_results``. + data_service + Used to look up realised target values for error/coverage columns. + actuals_as_of + Cutoff for realised-value lookup; defaults to *now* so every horizon that + has already resolved is scored (see :func:`score_backtest_results`). + + Returns + ------- + pd.DataFrame + Columns: ``predictor``, ``family``, ``as_of``, ``forecast_date``, + ``horizon`` (trading days), ``point``, ``q10``/``q20``/``q50``/``q80``/ + ``q90``, ``actual``, ``crps``, ``abs_error``, ``signed_error``, + ``width80`` (80% interval width), and ``inside80`` (1.0/0.0/NaN). + """ + resolved_as_of = actuals_as_of or datetime.now(tz=timezone.utc).replace(tzinfo=None) + actual_cache: dict[str, dict[pd.Timestamp, float]] = {} + + def _actuals(series_id: str) -> dict[pd.Timestamp, float]: + if series_id not in actual_cache: + df = data_service.get_series(series_id, as_of=resolved_as_of) + actual_cache[series_id] = { + pd.Timestamp(row["timestamp"]).normalize(): float(row["value"]) for _, row in df.iterrows() + } + return actual_cache[series_id] + + rows: list[dict[str, Any]] = [] + for predictor_name, task_results in results_by_predictor.items(): + for result in task_results.values(): + actual_by_date = _actuals(result.spec.task.target_series_id) + for pred, score in zip(result.predictions, result.scores, strict=False): + if not isinstance(pred.payload, ContinuousForecast): + continue + as_of = pd.Timestamp(pred.as_of) + fdate = pd.Timestamp(pred.forecast_date).normalize() + q = pred.payload.quantiles + lo80, hi80 = _qval(q, 0.2), _qval(q, 0.8) + point = float(pred.payload.point_forecast) + actual = actual_by_date.get(fdate) + rows.append( + { + "predictor": predictor_name, + "family": predictor_family(predictor_name), + "as_of": as_of, + "forecast_date": fdate, + "horizon": _business_horizon(as_of, fdate), + "point": point, + "q10": _qval(q, 0.1), + "q20": lo80, + "q50": _qval(q, 0.5), + "q80": hi80, + "q90": _qval(q, 0.9), + "actual": actual, + "crps": float(score), + "abs_error": abs(point - actual) if actual is not None else float("nan"), + "signed_error": (actual - point) if actual is not None else float("nan"), + "width80": hi80 - lo80, + "inside80": float(lo80 <= actual <= hi80) if actual is not None else float("nan"), + } + ) + return pd.DataFrame(rows) + + +def per_horizon_crps(pred_frame: pd.DataFrame) -> pd.DataFrame: + """Pivot mean CRPS to a predictor × horizon matrix with an ``All`` column. + + Rows are sorted by overall mean CRPS (best first) so the table doubles as the + leaderboard and reveals which horizon decides the ranking. + """ + if pred_frame.empty: + return pd.DataFrame() + pivot = pred_frame.pivot_table(index="predictor", columns="horizon", values="crps", aggfunc="mean") + pivot.columns = [f"h={int(h)}d" for h in pivot.columns] + pivot["All"] = pred_frame.groupby("predictor")["crps"].mean() + return pivot.sort_values("All") + + +def leaderboard_with_uncertainty(pred_frame: pd.DataFrame) -> pd.DataFrame: + """Mean CRPS per predictor with a standard error, sorted best-first. + + The ``se`` column (sample standard deviation / √n) is the lens for the + "is this lead real or noise?" question: when the gap between two predictors + is small relative to their SEs — common with only a handful of scored + origins — the ranking is not statistically meaningful. + """ + if pred_frame.empty: + return pd.DataFrame() + grp = pred_frame.groupby("predictor")["crps"] + out = pd.DataFrame( + { + "mean_crps": grp.mean(), + "se": grp.std(ddof=1) / np.sqrt(grp.count()), + "n": grp.count().astype(int), + "family": pred_frame.groupby("predictor")["family"].first(), + } + ) + return out.sort_values("mean_crps") + + +def extract_agent_rationales(results_by_predictor: dict[str, dict[str, BacktestResult]]) -> pd.DataFrame: + """Pull free-text rationale and trace links from agent/LLM prediction metadata. + + Only predictions whose ``metadata`` carries a ``rationale`` (the analyst agent + and any LLM method that returns one) produce rows. The result is the raw + material for inspecting *what the model was thinking* origin by origin. + """ + rows: list[dict[str, Any]] = [] + for predictor_name, task_results in results_by_predictor.items(): + for result in task_results.values(): + for pred in result.predictions: + meta = pred.metadata or {} + if "rationale" not in meta and "horizon_rationale" not in meta: + continue + rows.append( + { + "predictor": predictor_name, + "as_of": pd.Timestamp(pred.as_of), + "horizon": _business_horizon(pd.Timestamp(pred.as_of), pd.Timestamp(pred.forecast_date)), + "point": float(pred.payload.point_forecast) + if isinstance(pred.payload, ContinuousForecast) + else float("nan"), + "rationale": str(meta.get("rationale", "")).strip(), + "horizon_rationale": str(meta.get("horizon_rationale", "")).strip(), + "trace_url": meta.get("langfuse_trace_url", ""), + } + ) + return pd.DataFrame(rows) + + +def eval_narrative_md( + pred_frame: pd.DataFrame, + *, + smoke: bool = False, + period_label: str = "2026 evaluation", +) -> str: + """Generate the eval takeaways as Markdown computed from the results. + + Replaces hard-coded prose so the narrative always matches the run — including + after switching from smoke to the full suite. Reports the actual winner, the + gap to the runner-up relative to the noise floor, the decisive horizon, the + best family, and a calibration line, plus an explicit small-sample caveat. + """ + if pred_frame.empty: + return "_No scored predictions available to summarise._" + + board = leaderboard_with_uncertainty(pred_frame) + horizons = sorted(pred_frame["horizon"].unique()) + n_origins = pred_frame["as_of"].nunique() + n_points = len(pred_frame) + + winner = board.index[0] + win_crps, win_se = board.loc[winner, "mean_crps"], board.loc[winner, "se"] + lines: list[str] = [] + + # 1. Winner + significance vs runner-up. + if len(board) > 1: + runner = board.index[1] + gap = board.loc[runner, "mean_crps"] - win_crps + noise = float(np.nan_to_num(win_se) + np.nan_to_num(board.loc[runner, "se"])) + significant = noise > 0 and gap > noise + verdict = ( + "a gap larger than the combined standard error — a real edge over this window" + if significant + else "**well within the combined standard error**, so the ranking here is not statistically distinguishable from noise" + ) + lines.append( + f"1. **{winner}** has the best mean CRPS ({win_crps:.2f}) on the {period_label}, " + f"ahead of **{runner}** ({board.loc[runner, 'mean_crps']:.2f}) by {gap:.2f} — {verdict}." + ) + else: + lines.append(f"1. **{winner}** scored {win_crps:.2f} mean CRPS on the {period_label}.") + + # 2. Where the ranking is decided (per-horizon spread). + if len(horizons) > 1: + by_h = pred_frame.groupby("horizon")["crps"] + spread = (by_h.max() - by_h.min()).sort_values(ascending=False) + decisive_h = int(spread.index[0]) + easy_h = int(spread.index[-1]) + lines.append( + f"2. The leaderboard is **decided at h={decisive_h}d**, where CRPS ranges {by_h.min()[decisive_h]:.1f}–" + f"{by_h.max()[decisive_h]:.1f} across methods; at the short h={easy_h}d horizon the methods are nearly " + f"tied (range {by_h.min()[easy_h]:.1f}–{by_h.max()[easy_h]:.1f}). A handful of long-horizon points " + f"drives the whole ranking." + ) + + # 3. Best family — does the agentic/LLM bet pay off here? + fam = pred_frame.groupby("family")["crps"].mean().sort_values() + best_fam = fam.index[0] + fam_str = ", ".join(f"{f} {v:.2f}" for f, v in fam.items()) + lines.append(f"3. **By family** (mean CRPS): {fam_str}. Best family this window: **{best_fam}**.") + + # 4. Calibration — is the winner's 80% interval honest? + cov = pred_frame.dropna(subset=["inside80"]).groupby("predictor")["inside80"].mean() * 100 + if winner in cov.index: + n_win = int(board.loc[winner, "n"]) + lines.append( + f"4. **Calibration:** {winner}'s 80% interval covered {cov[winner]:.0f}% of outcomes " + f"(target 80%) over its {n_win} scored point(s). With this few, coverage this far from " + f"target is itself a small-sample artefact, not necessarily mis-calibration." + ) + + # 5. Sample-size caveat — the honest health warning. + caveat = ( + f"⚠️ **Smoke run:** only {n_origins} origin(s) / {n_points} scored points. Treat the ranking as a " + f"pipeline check, not evidence — rerun the full suite before drawing conclusions." + if smoke or n_origins <= 2 + else f"Based on {n_origins} origins / {n_points} scored points." + ) + lines.append(f"5. {caveat}") + return "\n".join(lines) + + +def build_price_frame(data_service: DataService, *, as_of: datetime | None = None) -> pd.DataFrame: + """Return the target price series as a ``price``-column DataFrame for plotting.""" + resolved_as_of = as_of or datetime.now(tz=timezone.utc).replace(tzinfo=None) + series = data_service.get_series("wti_crude_oil_price", as_of=resolved_as_of) + frame = pd.DataFrame( + {"price": series["value"].astype(float).to_numpy()}, + index=pd.to_datetime(series["timestamp"]), + ) + frame.index.name = "date" + return frame.sort_index() + + __all__ = [ "backtest_results_to_frame", + "build_price_frame", "compute_brier_score", + "eval_narrative_md", + "extract_agent_rationales", + "leaderboard_with_uncertainty", + "per_horizon_crps", + "predictions_to_frame", + "predictor_family", "rolling_coverage_pct", "score_backtest_results", "select_top_predictors", diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analyst_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analyst_agent__agent.py.md index b050256a..c69f4e6d 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analyst_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analyst_agent__agent.py.md @@ -53,115 +53,31 @@ from aieng.forecasting.methods.agentic.agent_factory import ( CodeExecutionConfig, ContextRetrievalConfig, ) +from aieng.forecasting.methods.agentic.domain import ( + build_analyst_config, + render_analyst_instruction, + render_multitask_analyst_instruction, +) +from aieng.forecasting.methods.agentic.history import compress_history from aieng.forecasting.methods.numerical.darts_arima import DartsAutoARIMAPredictor -from aieng.forecasting.models import LITE_MODEL +from aieng.forecasting.models import ADVANCED_MODEL, LITE_MODEL from energy_oil_forecasting.data import WTI_SERIES_ID, build_wti_service +from energy_oil_forecasting.domain import OIL_DOMAIN from pydantic import BaseModel # --------------------------------------------------------------------------- -# System prompt (root analyst agent) +# System prompts — rendered from OIL_DOMAIN # --------------------------------------------------------------------------- +# +# The analyst, multitask, and context-retrieval instructions are rendered from +# the shared instruction templates over the WTI ``OIL_DOMAIN`` fragments (see +# :mod:`energy_oil_forecasting.domain`). A new target series is configured by +# supplying a different ``DomainConfig`` — no edits to this module. -_WTI_MULTITASK_ANALYST_INSTRUCTION = """\ -## Role - -You are an expert WTI crude oil market analyst. - -## Input - -You will receive a JSON payload containing: -- `task_spec`: the exact question and required JSON output schema -- `as_of`: the forecast origin date (temporal cutoff) -- `origin_price_usd_bbl`: WTI close on the origin date -- `target_history_csv`: compressed WTI daily close history - -When context retrieval is enabled, call ``search_web`` BEFORE answering. - -## Output contract - -Read the data (and briefing, if retrieved) carefully, then execute the task \ -in `task_spec` precisely. - -If a `set_model_response` tool is available, call it with your complete JSON \ -as `json_response` — the exact schema is described in `task_spec`. Otherwise \ -return the JSON directly as plain text with no preamble.\ -""" - - -def _build_wti_analyst_instruction() -> str: - """Build the WTI analyst instruction, embedding the output schema from the class. - - Using a function instead of a static string ensures the ``## Output schema`` - block is always in sync with ``ContinuousAgentForecastOutput`` — - no manual JSON to maintain. - """ - schema = ContinuousAgentForecastOutput.prompt_schema_json() - return ( - "## Role\n\n" - "You are an expert WTI crude oil market analyst. You produce calibrated " - "probabilistic price forecasts for WTI crude oil futures, grounded in " - "supply/demand fundamentals, geopolitical risk, and historical price dynamics.\n\n" - "## Forecasting contract\n\n" - "You will receive a JSON payload containing:\n" - "- `task`: the task identifier\n" - "- `as_of`: the forecast origin date in YYYY-MM-DD format\n" - "- `horizons`: a list of integer horizon steps (business days ahead)\n" - "- `standard_quantiles`: the exact quantile levels you must produce\n" - "- `target_summary`: last close price, 52-week range, and observation count\n" - "- `target_history_csv`: WTI daily close history (recent 6 months daily, " - "older history as weekly averages)\n\n" - "Rules:\n" - "1. Produce one forecast for each horizon listed in `horizons`.\n" - "2. Use exactly the quantile levels from `standard_quantiles` — no additions, no omissions.\n" - "3. `point_forecast` must exactly equal the 0.50 quantile value.\n" - "4. Quantile values must be strictly non-decreasing as quantile levels increase.\n" - "5. Document your reasoning in the `rationale` fields.\n" - "6. When tools are enabled, conclude with `set_model_response` to return the structured forecast.\n\n" - "## Output schema\n\n" - "Call `set_model_response` with a `json_response` string matching **exactly**:\n\n" - "```json\n" + schema + "\n```\n\n" - 'Critical: use `"horizon"` (integer, not `"horizon_days"`). ' - '`"quantiles"` is a **list** of `{"quantile": , "value": }` ' - "objects — not a dict. Omit any field not shown above.\n\n" - "## Analysis discipline\n\n" - "When context retrieval is available, call ``search_web`` to gather market " - "intelligence BEFORE producing forecasts.\n\n" - "Call ``search_web`` with ``query`` and ``cutoff_date`` (set to the ``as_of`` " - "date from the payload). The ``cutoff_date`` MUST always equal ``as_of`` — " - "this is the temporal fence that prevents post-origin information from " - "contaminating historical backtests.\n\n" - "Recommended queries (call ``search_web`` once per topic):\n" - '- ``search_web(query="WTI crude oil price trend and OPEC+ supply decisions", cutoff_date=)``\n' - '- ``search_web(query="Persian Gulf geopolitical risk shipping lane disruptions", cutoff_date=)``\n' - '- ``search_web(query="US Strategic Petroleum Reserve policy and global demand outlook", cutoff_date=)``\n\n' - "Document your key assumptions (OPEC+ policy, shipping lane risk, inventory " - "levels, macro demand) in the `rationale` fields of your forecast output." - ) - - -_WTI_ANALYST_INSTRUCTION = _build_wti_analyst_instruction() - -# --------------------------------------------------------------------------- -# Context retrieval instruction (sub-agent) -# --------------------------------------------------------------------------- - -_WTI_CONTEXT_RETRIEVAL_INSTRUCTION = """\ -You are an oil market intelligence specialist with access to web search. - -Search for information relevant to the query and return a concise structured \ -markdown summary (3-5 paragraphs) covering relevant aspects of: -- WTI/Brent crude price level and recent trend -- OPEC+ production decisions and supply outlook -- Geopolitical risks in the Persian Gulf, Middle East, key shipping lanes -- US Strategic Petroleum Reserve and energy policy signals -- Notable tanker/shipping incidents or supply disruption signals -- Published analyst forecasts or unusual price-target revisions - -Ground your summary in the search results you actually retrieve. \ -When a cutoff date is specified, do not report or speculate about events \ -that occurred after that date.\ -""" +_WTI_ANALYST_INSTRUCTION = render_analyst_instruction(OIL_DOMAIN) +_WTI_MULTITASK_ANALYST_INSTRUCTION = render_multitask_analyst_instruction(OIL_DOMAIN) +_WTI_CONTEXT_RETRIEVAL_INSTRUCTION = OIL_DOMAIN.context_retrieval_instruction # --------------------------------------------------------------------------- # Skills supplement (appended to instruction when skills are attached) @@ -230,45 +146,13 @@ _SKILLS_ROOT = Path(__file__).parent / "skills" # --------------------------------------------------------------------------- -# History compression +# History compression — re-exported from the shared library for backward compat # --------------------------------------------------------------------------- - -def compress_history(df: pd.DataFrame) -> str: - """Compress WTI daily history to stay within context limits. - - Returns daily bars for the most recent 6 months and weekly averages for - older history. The CSV header is ``date,close``. - - Parameters - ---------- - df : pd.DataFrame - DataFrame with columns ``timestamp`` and ``value``. - - Returns - ------- - str - CSV string with header ``date,close``. - """ - df = df.copy() - df["timestamp"] = pd.to_datetime(df["timestamp"]) - cutoff = df["timestamp"].max() - pd.DateOffset(months=6) - - recent = df[df["timestamp"] >= cutoff].copy() - old = df[df["timestamp"] < cutoff].copy() - - rows: list[str] = ["date,close"] - - if not old.empty: - old_indexed = old.set_index("timestamp")["value"] - weekly: pd.Series = old_indexed.resample("W").mean().dropna() - for date, val in weekly.items(): - rows.append(f"{date.date()},{val:.2f}") - - for _, row in recent.iterrows(): - rows.append(f"{row['timestamp'].date()},{row['value']:.2f}") - - return "\n".join(rows) +# ``compress_history`` now lives in +# :mod:`aieng.forecasting.methods.agentic.history`; it is imported above and +# re-exported here so existing ``from ...analyst_agent import compress_history`` +# call sites keep resolving. # --------------------------------------------------------------------------- @@ -354,16 +238,20 @@ def build_wti_basic_config(model: str = LITE_MODEL) -> AgentConfig: ------- AgentConfig """ - return AgentConfig( - name="wti_analyst_basic", - model=model, + return build_analyst_config( + OIL_DOMAIN, + name_suffix="basic", instruction=_WTI_ANALYST_INSTRUCTION, + model=model, ) def build_wti_multitask_news_config( model: str = LITE_MODEL, search_model: str = LITE_MODEL, + verifier_model: str = ADVANCED_MODEL, + verifier_max_attempts: int = 3, + verifier_confidence_threshold: int = 8, ) -> AgentConfig: """News-grounded config for the one-agent-three-tasks demo (NB3). @@ -378,15 +266,29 @@ def build_wti_multitask_news_config( Model for the context-retrieval (web-search) sub-tool. Defaults to the lite model (``gemini-3.1-flash-lite-preview``) independently of ``model`` so that Gemini handles Google Search even when the analyst uses a different provider. + verifier_model : str + Model for the independent temporal-leakage verifier that audits each + ``search_web`` result against ``cutoff_date`` before it is returned. + Defaults to the advanced model so it doesn't share ``search_model``'s + blind spots. + verifier_max_attempts : int + Maximum search-then-verify attempts before giving up and returning + the ``[SEARCH_VERIFICATION_FAILED]`` sentinel. + verifier_confidence_threshold : int + Minimum verifier confidence (1-10) required to accept a result. """ - return AgentConfig( - name="wti_analyst_multitask", - model=model, + return build_analyst_config( + OIL_DOMAIN, + name_suffix="multitask", instruction=_WTI_MULTITASK_ANALYST_INSTRUCTION, + model=model, context_retrieval=ContextRetrievalConfig( enabled=True, instruction=_WTI_CONTEXT_RETRIEVAL_INSTRUCTION, search_model=search_model, + verifier_model=verifier_model, + verifier_max_attempts=verifier_max_attempts, + verifier_confidence_threshold=verifier_confidence_threshold, ), ) @@ -394,12 +296,17 @@ def build_wti_multitask_news_config( def build_wti_news_config( model: str = LITE_MODEL, search_model: str = LITE_MODEL, + verifier_model: str = ADVANCED_MODEL, + verifier_max_attempts: int = 3, + verifier_confidence_threshold: int = 8, ) -> AgentConfig: """Build an :class:`AgentConfig` with bounded Google Search. Wires a :class:`~aieng.forecasting.methods.agentic.agent_factory.ContextRetrievalConfig` sub-agent that enforces a temporal cutoff on every search call, preventing - future information from contaminating historical backtests. + future information from contaminating historical backtests. An + independent verifier call audits each search result against the cutoff + before it reaches the analyst (see :class:`ContextRetrievalConfig`). Parameters ---------- @@ -409,19 +316,33 @@ def build_wti_news_config( Model for the context-retrieval (web-search) sub-tool. Defaults to the lite model (``gemini-3.1-flash-lite-preview``) independently of ``model`` so that Gemini handles Google Search even when the analyst uses a different provider. + verifier_model : str + Model for the independent temporal-leakage verifier that audits each + ``search_web`` result against ``cutoff_date`` before it is returned. + Defaults to the advanced model so it doesn't share ``search_model``'s + blind spots. + verifier_max_attempts : int + Maximum search-then-verify attempts before giving up and returning + the ``[SEARCH_VERIFICATION_FAILED]`` sentinel. + verifier_confidence_threshold : int + Minimum verifier confidence (1-10) required to accept a result. Returns ------- AgentConfig """ - return AgentConfig( - name="wti_analyst_news", - model=model, + return build_analyst_config( + OIL_DOMAIN, + name_suffix="news", instruction=_WTI_ANALYST_INSTRUCTION, + model=model, context_retrieval=ContextRetrievalConfig( enabled=True, instruction=_WTI_CONTEXT_RETRIEVAL_INSTRUCTION, search_model=search_model, + verifier_model=verifier_model, + verifier_max_attempts=verifier_max_attempts, + verifier_confidence_threshold=verifier_confidence_threshold, ), ) @@ -430,6 +351,9 @@ def build_wti_code_exec_config( model: str = LITE_MODEL, search_model: str = LITE_MODEL, max_output_tokens: int = 16_384, + verifier_model: str = ADVANCED_MODEL, + verifier_max_attempts: int = 3, + verifier_confidence_threshold: int = 8, ) -> AgentConfig: """Build an :class:`AgentConfig` with E2B code execution and forecasting skills. @@ -454,20 +378,34 @@ def build_wti_code_exec_config( LiteLLM's OpenAI-compatible endpoint default of 4096, which is not enough for Claude to write a complete ``run_code`` Python script in a single function call — causing repeated retries with empty arguments. + verifier_model : str + Model for the independent temporal-leakage verifier that audits each + ``search_web`` result against ``cutoff_date`` before it is returned. + Defaults to the advanced model so it doesn't share ``search_model``'s + blind spots. + verifier_max_attempts : int + Maximum search-then-verify attempts before giving up and returning + the ``[SEARCH_VERIFICATION_FAILED]`` sentinel. + verifier_confidence_threshold : int + Minimum verifier confidence (1-10) required to accept a result. Returns ------- AgentConfig """ - return AgentConfig( - name="wti_analyst_code", - model=model, + return build_analyst_config( + OIL_DOMAIN, + name_suffix="code", instruction=_WTI_ANALYST_INSTRUCTION + _CODE_EXEC_SKILLS_SUPPLEMENT, + model=model, max_output_tokens=max_output_tokens, context_retrieval=ContextRetrievalConfig( enabled=True, instruction=_WTI_CONTEXT_RETRIEVAL_INSTRUCTION, search_model=search_model, + verifier_model=verifier_model, + verifier_max_attempts=verifier_max_attempts, + verifier_confidence_threshold=verifier_confidence_threshold, ), code_execution=CodeExecutionConfig(enabled=True), skills_dirs=[ @@ -483,6 +421,9 @@ def build_wti_tool_config( *, data_service: DataService | None = None, num_samples: int = 200, + verifier_model: str = ADVANCED_MODEL, + verifier_max_attempts: int = 3, + verifier_confidence_threshold: int = 8, ) -> AgentConfig: """Build an :class:`AgentConfig` with a conventional statistical forecast tool. @@ -510,6 +451,16 @@ def build_wti_tool_config( num_samples : int, default=200 Monte Carlo sample count for AutoARIMA. Kept modest to bound agent latency, since AutoARIMA can be slow per origin. + verifier_model : str + Model for the independent temporal-leakage verifier that audits each + ``search_web`` result against ``cutoff_date`` before it is returned. + Defaults to the advanced model so it doesn't share ``search_model``'s + blind spots. + verifier_max_attempts : int + Maximum search-then-verify attempts before giving up and returning + the ``[SEARCH_VERIFICATION_FAILED]`` sentinel. + verifier_confidence_threshold : int + Minimum verifier confidence (1-10) required to accept a result. Returns ------- @@ -518,14 +469,18 @@ def build_wti_tool_config( service = data_service if data_service is not None else build_wti_service() forecast_tool = ForecastTool(service, predictor=DartsAutoARIMAPredictor(num_samples=num_samples)) - return AgentConfig( - name="wti_analyst_tool", - model=model, + return build_analyst_config( + OIL_DOMAIN, + name_suffix="tool", instruction=_WTI_ANALYST_INSTRUCTION + _FORECAST_TOOL_SUPPLEMENT, + model=model, context_retrieval=ContextRetrievalConfig( enabled=True, instruction=_WTI_CONTEXT_RETRIEVAL_INSTRUCTION, search_model=search_model, + verifier_model=verifier_model, + verifier_max_attempts=verifier_max_attempts, + verifier_confidence_threshold=verifier_confidence_threshold, ), function_tools=[forecast_tool.as_function_tool()], ) diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__domain.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__domain.py.md new file mode 100644 index 00000000..40c6706f --- /dev/null +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__domain.py.md @@ -0,0 +1,124 @@ +# Source: implementations/energy_oil_forecasting/domain.py + +kind: python + +```python +"""WTI crude oil domain configuration. + +Defines :data:`OIL_DOMAIN`, the single +:class:`~aieng.forecasting.methods.agentic.domain.DomainConfig` instance that +supplies every WTI-specific fragment the shared agent-building machinery needs. +The analyst, multitask, and adaptive agent factories render their instructions +from this instance, so all oil-specific prompt wording lives here rather than +scattered across the agent modules. +""" + +from __future__ import annotations + +from pathlib import Path + +from aieng.forecasting.methods.agentic.domain import DomainConfig +from energy_oil_forecasting.data import WTI_SERIES_ID +from energy_oil_forecasting.paths import SHOCK_HORIZON, SHOCK_THRESHOLD + + +# --------------------------------------------------------------------------- +# Web-search sub-agent instruction (domain-specific verbatim) +# --------------------------------------------------------------------------- + +_WTI_CONTEXT_RETRIEVAL_INSTRUCTION = """\ +You are an oil market intelligence specialist with access to web search. + +Search for information relevant to the query and return a concise structured \ +markdown summary (3-5 paragraphs) covering relevant aspects of: +- WTI/Brent crude price level and recent trend +- OPEC+ production decisions and supply outlook +- Geopolitical risks in the Persian Gulf, Middle East, key shipping lanes +- US Strategic Petroleum Reserve and energy policy signals +- Notable tanker/shipping incidents or supply disruption signals +- Published analyst forecasts or unusual price-target revisions + +Ground your summary in the search results you actually retrieve. \ +When a cutoff date is specified, do not report or speculate about events \ +that occurred after that date. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ +""" + + +# --------------------------------------------------------------------------- +# Skill directories (adaptive agent) +# --------------------------------------------------------------------------- + +_ADAPTIVE_SKILLS_ROOT = Path(__file__).parent / "adaptive_agent" / "skills" + + +# --------------------------------------------------------------------------- +# The WTI domain +# --------------------------------------------------------------------------- + +OIL_DOMAIN = DomainConfig( + # Identity + domain_name="WTI crude oil", + analyst_persona="WTI crude oil market analyst", + analyst_forecasting_focus=( + "calibrated probabilistic price forecasts for WTI crude oil futures, grounded in " + "supply/demand fundamentals, geopolitical risk, and historical price dynamics" + ), + analyst_agent_name_prefix="wti_analyst", + adaptive_agent_name_prefix="wti_adaptive_analyst", + target_short_name="WTI", + starter_fluency_areas=( + "supply/demand fundamentals, OPEC+ policy, geopolitical and shipping-lane risk, and price dynamics" + ), + # Data / target + target_series_id=WTI_SERIES_ID, + target_units="USD/bbl", + target_history_description=("WTI daily close history (recent 6 months daily, older history as weekly averages)"), + data_ticker="CL=F", + data_source_name="Yahoo Finance", + data_fetch_example=( + "```python\nraw = ticker.history(start='2004-01-01', end='2026-02-16', auto_adjust=False)\n```" + ), + code_exec_preinstalled="numpy, pandas, sklearn, yfinance, statsmodels, properscoring", + multitask_origin_price_field="origin_price_usd_bbl", + # Context retrieval + context_retrieval_instruction=_WTI_CONTEXT_RETRIEVAL_INSTRUCTION, + recommended_search_queries=( + "WTI crude oil price trend and OPEC+ supply decisions", + "Persian Gulf geopolitical risk shipping lane disruptions", + "US Strategic Petroleum Reserve policy and global demand outlook", + ), + key_assumptions_hint="OPEC+ policy, shipping lane risk, inventory levels, macro demand", + # Strategy skill + strategy_skill_title="WTI Forecasting Strategy", + strategy_skill_name="wti-strategy", + adaptive_calibration_example=( + "substituting a flat-trend model in elevated/extreme vol regimes if your strategy calls for it" + ), + # Skill directories + pipeline_skill_dirs=( + _ADAPTIVE_SKILLS_ROOT / "fetch-yfinance", + _ADAPTIVE_SKILLS_ROOT / "vol-regime", + _ADAPTIVE_SKILLS_ROOT / "trend-projection", + ), + meta_learning_skill_dir=_ADAPTIVE_SKILLS_ROOT / "meta-learning", + seed_strategy_dir=_ADAPTIVE_SKILLS_ROOT / "wti-strategy", + trained_strategy_dir=_ADAPTIVE_SKILLS_ROOT / "wti-strategy-trained", + # Volatility regime + vol_regime_bands=((15.0, "low"), (30.0, "medium"), (50.0, "elevated")), + # Tool bounds + frequency="B", + horizons=(5, 10, 21), + shock_threshold=SHOCK_THRESHOLD, + shock_horizon=SHOCK_HORIZON, + num_samples=200, +) +``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md index 0c0a4371..80e1c480 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md @@ -5,8 +5,12 @@ kind: yaml ```yaml # Energy Oil Eval Spec — 2026 Prospective Competition # -# Runs on 8 weekly origins from Feb 2, 2026 to Mar 23, 2026. -# Covers the high-volatility Persian Gulf geopolitical price shock period. +# Runs on 18 weekly origins from Feb 2, 2026 to Jun 1, 2026. +# Covers the high-volatility Persian Gulf geopolitical price shock and its +# aftermath. The end date is set to the latest origin whose longest horizon +# (21 business days) still resolves against available data — keep it at +# most 21 business days behind the most recent cached WTI price (see +# scripts/fetch_wti.py) so every origin fully resolves. # Target is WTI Crude Oil price (yfinance ticker: CL=F). # Horizons: 5, 10, 21 business days. @@ -14,8 +18,9 @@ spec_id: energy_oil_eval description: >- Prospective/out-of-sample evaluation period in 2026 for daily WTI crude oil. - Evaluates selected contender models on 8 weekly origins during the early 2026 - geopolitical price shock to measure adaptive real-time forecasting performance. + Evaluates selected contender models on 18 weekly origins from the early 2026 + geopolitical price shock through its aftermath, to measure adaptive + real-time forecasting performance. tasks: - task_id: wti_oil_price_forecast @@ -27,7 +32,7 @@ tasks: projected 5, 10, and 21 trading days ahead. start: "2026-02-02" -end: "2026-03-23" +end: "2026-06-01" stride: 5 warmup: 250 ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md index 80597220..f4fdfe8c 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md @@ -9,7 +9,7 @@ kind: yaml # arena cheaply during development and end-to-end testing. # Use by setting SMOKE_TEST = True in the notebook setup cell. # -# Origin count : 2 (vs. 8 in the full eval) +# Origin count : 2 (vs. 18 in the full eval) # Warmup : 250 trading days (~1 year) of historical prices spec_id: energy_oil_eval_smoke diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md index 2e5a2e84..080dd814 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md @@ -5,10 +5,12 @@ kind: python ```python """WTI starter agent — a fresh, hackable template for your own exploration. -Exports the toggle-driven :class:`AgentConfig` factory and the predictor -convenience factory. See ``99_starter_agent.ipynb`` and ``agent.py``. +Exports the toolbelt-driven :class:`AgentConfig` factory, the predictor +convenience factory, and the :mod:`tools` module of per-tool factories you +compose in the notebook. See ``99_starter_agent.ipynb`` and ``agent.py``. """ +from energy_oil_forecasting.starter_agent import tools from energy_oil_forecasting.starter_agent.agent import ( build_starter_agent_config, build_starter_agent_predictor, @@ -18,5 +20,6 @@ from energy_oil_forecasting.starter_agent.agent import ( __all__ = [ "build_starter_agent_config", "build_starter_agent_predictor", + "tools", ] ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md index c7076021..b5680457 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md @@ -32,7 +32,7 @@ from __future__ import annotations import json from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Sequence from aieng.forecasting.data.context import ForecastContext from aieng.forecasting.evaluation.task import ForecastingTask @@ -46,18 +46,20 @@ from aieng.forecasting.methods.agentic.agent_factory import ( CodeExecutionConfig, ContextRetrievalConfig, ) +from aieng.forecasting.methods.agentic.domain import render_starter_instruction from aieng.forecasting.models import LITE_MODEL # Reuse the existing WTI prompt builder + history compression — these serialise # the task/context into the agent's JSON payload and are not worth duplicating. from energy_oil_forecasting.analyst_agent import WtiPriceForecastPromptBuilder +from energy_oil_forecasting.domain import OIL_DOMAIN +from energy_oil_forecasting.starter_agent.tools import ToolSpec, news_search -# Skills live next to this module. +# Skills live next to this module. The forecasting contract is always loaded; +# each tool loads its own playbook via its ToolSpec (see tools.py). _SKILLS_ROOT = Path(__file__).parent / "skills" _FORECASTING_SKILL = _SKILLS_ROOT / "forecasting" -_RESEARCH_SKILL = _SKILLS_ROOT / "research-playbook" -_CODE_ANALYSIS_SKILL = _SKILLS_ROOT / "code-analysis-playbook" # --------------------------------------------------------------------------- @@ -65,45 +67,14 @@ _CODE_ANALYSIS_SKILL = _SKILLS_ROOT / "code-analysis-playbook" # --------------------------------------------------------------------------- -def _build_starter_instruction() -> str: - """Build the task-agnostic, skill-agnostic starter persona. - - Just the analyst's identity and how to behave — no output schema, no payload - contract, no skill or tool mechanics. ADK injects the name + description of - every attached skill (and every tool) into the system prompt, so the agent - already knows what it can load and call; repeating that here would only - duplicate dynamically-injected information. The forecasting *contract* lives - in the loadable ``forecasting`` skill. Edit the persona freely. - """ - return ( - "## Role\n\n" - "You are a WTI crude oil market analyst — fluent in supply/demand " - "fundamentals, OPEC+ policy, geopolitical and shipping-lane risk, and " - "price dynamics. This is a starter agent: keep your reasoning " - "transparent and your claims honest.\n\n" - "## How to respond\n\n" - "- For open-ended questions, scenario analysis, or anything " - "conversational, answer directly and concisely — do NOT ask for a JSON " - "payload.\n" - "- When you are handed a task that asks for a structured probabilistic " - "forecast, produce a calibrated one." - ) - - -_STARTER_INSTRUCTION = _build_starter_instruction() - - -_CONTEXT_RETRIEVAL_INSTRUCTION = """\ -You are an oil-market intelligence specialist with web search. - -Return a concise structured markdown summary (3-5 paragraphs) covering, as the -query warrants: WTI/Brent price level and trend; OPEC+ supply decisions; -geopolitical risk in the Persian Gulf and key shipping lanes; US SPR / energy -policy; notable supply-disruption signals; and published analyst price targets. - -Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ -""" +# The task-agnostic, skill-agnostic starter persona — just the analyst's +# identity and how to behave. No output schema, no payload contract, no skill or +# tool mechanics: ADK injects the name + description of every attached skill (and +# every tool) into the system prompt, so the agent already knows what it can load +# and call. The forecasting *contract* lives in the loadable ``forecasting`` +# skill. The persona is rendered from the WTI ``OIL_DOMAIN`` fragments; edit it +# by supplying a different ``DomainConfig``. +_STARTER_INSTRUCTION = render_starter_instruction(OIL_DOMAIN) # --------------------------------------------------------------------------- @@ -113,59 +84,75 @@ date is specified, never report or speculate about events after it.\ def build_starter_agent_config( model: str = LITE_MODEL, - search_model: str = LITE_MODEL, *, - enable_search: bool = True, - enable_code_exec: bool = False, + tools: Sequence[ToolSpec] = (), ) -> AgentConfig: - """Build the WTI starter :class:`AgentConfig`. + """Build the WTI starter :class:`AgentConfig` from a toolbelt. + + An agent is a persona plus a list of tools. The persona is fixed here (edit + ``_build_starter_instruction``); the *toolbelt* is what you compose in the + notebook — a list of :class:`~energy_oil_forecasting.starter_agent.tools.ToolSpec` + from the factories in :mod:`energy_oil_forecasting.starter_agent.tools`:: + + from energy_oil_forecasting.starter_agent import tools, build_starter_agent_config + + config = build_starter_agent_config( + model=AGENT_MODEL, + tools=[tools.news_search(), tools.arima_forecast()], + ) + + Each spec lands in a different ``AgentConfig`` field; this function folds the + list, routing every fragment to the right place — search sub-agent, code + sandbox, function tools — and loading each tool's playbook skill and prompt + supplement. Adding or removing a tool is one line in the notebook. Parameters ---------- model : str Model for the analyst agent (default: lite). Pass the advanced model (``"gemini-3.5-flash"``) for higher-quality runs. - search_model : str - Model for the bounded web-search sub-tool. - enable_search : bool, default=True - Wire a cutoff-aware ``search_web`` tool and load the - ``research-playbook`` skill. Proxy-only — no extra API key. - enable_code_exec : bool, default=False - Wire an E2B Python sandbox and load the ``code-analysis-playbook`` - skill. Needs ``E2B_API_KEY`` and is slower, so it is off by default — - flip it on to let the agent compute its own diagnostics. + tools : Sequence[ToolSpec], default=() + The agent's toolbelt. Build entries with the factories in ``tools.py`` + (``news_search()``, ``code_sandbox()``, ``arima_forecast()``), or write + your own factory that returns a ``ToolSpec``. Returns ------- AgentConfig """ - # Every attached skill is loaded on demand: ADK injects each skill's name + - # description into the system prompt, and the agent reads the full SKILL.md - # only when relevant — so toggling a tool just adds its skill, no persona edits. + # The forecasting contract is always loaded. Each tool's own playbook is + # loaded on demand: ADK injects each skill's name + description into the + # system prompt, and the agent reads the full SKILL.md only when relevant — + # so adding a tool just adds its skill, no persona edits. skills_dirs: list[Path] = [_FORECASTING_SKILL] - if enable_search: - skills_dirs.append(_RESEARCH_SKILL) - if enable_code_exec: - skills_dirs.append(_CODE_ANALYSIS_SKILL) - - context_retrieval = ( - ContextRetrievalConfig( - enabled=True, - instruction=_CONTEXT_RETRIEVAL_INSTRUCTION, - search_model=search_model, - ) - if enable_search - else ContextRetrievalConfig() - ) + instruction = _STARTER_INSTRUCTION + context_retrieval = ContextRetrievalConfig() + code_execution = CodeExecutionConfig() + function_tools: list[Any] = [] + max_output_tokens: int | None = None + + for spec in tools: + if spec.skill_dir is not None: + skills_dirs.append(spec.skill_dir) + if spec.instruction_supplement: + instruction += spec.instruction_supplement + if spec.context_retrieval is not None: + context_retrieval = spec.context_retrieval + if spec.code_execution is not None: + code_execution = spec.code_execution + if spec.function_tool is not None: + function_tools.append(spec.function_tool) + if spec.max_output_tokens is not None: + max_output_tokens = max(max_output_tokens or 0, spec.max_output_tokens) return AgentConfig( name="wti_starter_agent", model=model, - instruction=_STARTER_INSTRUCTION, - # 16k headroom: enough for a complete run_code script + structured output. - max_output_tokens=16_384 if enable_code_exec else None, + instruction=instruction, + max_output_tokens=max_output_tokens, context_retrieval=context_retrieval, - code_execution=CodeExecutionConfig(enabled=enable_code_exec), + code_execution=code_execution, + function_tools=function_tools, skills_dirs=skills_dirs, ) @@ -228,6 +215,7 @@ def build_starter_agent_predictor(config: AgentConfig) -> AgentPredictor: def __getattr__(name: str) -> Any: """Expose ``root_agent`` lazily for schema-free interactive use via ``adk web``.""" if name == "root_agent": - return build_adk_agent(build_starter_agent_config()) + # Interactive default: news search on (proxy-only, no extra key). + return build_adk_agent(build_starter_agent_config(tools=[news_search()])) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md index a6969936..f43e96ae 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md new file mode 100644 index 00000000..1cd0911d --- /dev/null +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md @@ -0,0 +1,237 @@ +# Source: implementations/energy_oil_forecasting/starter_agent/tools.py + +kind: python + +```python +"""The starter agent's toolbelt — one factory per tool, composed in the notebook. + +An agent is a *persona* plus a *list of tools*. This module makes that list the +thing you edit: each function here returns a :class:`ToolSpec` describing one +capability, and you assemble an agent by handing a list of them to +:func:`~energy_oil_forecasting.starter_agent.agent.build_starter_agent_config`:: + + from energy_oil_forecasting.starter_agent import tools, build_starter_agent_config + + config = build_starter_agent_config( + model=AGENT_MODEL, + tools=[ + tools.news_search(), # cutoff-aware Google Search (proxy-only) + tools.arima_forecast(), # AutoARIMA statistical anchor — no code-gen + # tools.code_sandbox(), # E2B Python sandbox (needs E2B_API_KEY) + ], + ) + +Each tool lands in a *different* field of the underlying ``AgentConfig`` (search +is a sub-agent, code execution is a sandbox capability, the forecast is a +plain function tool). A :class:`ToolSpec` carries everything a tool needs — the +config fragment it fills, its playbook skill, and any prompt supplement — so the +config factory can route it without the notebook ever touching ADK plumbing. + +To add your own tool, write a factory that returns a ``ToolSpec``. Point it at a +different series, swap AutoARIMA for another predictor, or wrap a brand-new +function tool — the notebook composition and the config fold both keep working. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from aieng.forecasting.data import DataService +from aieng.forecasting.methods.agentic import ForecastTool +from aieng.forecasting.methods.agentic.agent_factory import ( + CodeExecutionConfig, + ContextRetrievalConfig, +) +from aieng.forecasting.methods.numerical.darts_arima import DartsAutoARIMAPredictor +from aieng.forecasting.models import LITE_MODEL +from energy_oil_forecasting.data import WTI_SERIES_ID, build_wti_service + + +# Skills live next to this module; each tool loads its own playbook. +_SKILLS_ROOT = Path(__file__).parent / "skills" +_RESEARCH_SKILL = _SKILLS_ROOT / "research-playbook" +_CODE_ANALYSIS_SKILL = _SKILLS_ROOT / "code-analysis-playbook" + + +# --------------------------------------------------------------------------- +# ToolSpec — the seam between the notebook's toolbelt and AgentConfig +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ToolSpec: + """One item on the agent's toolbelt. + + A tool is more than a callable: it may need a config fragment, a skill that + teaches the agent how to use it, and a line of instruction. This descriptor + bundles all of that so + :func:`~energy_oil_forecasting.starter_agent.agent.build_starter_agent_config` + can fold a list of specs onto a single ``AgentConfig`` — routing each field + to the right place — without the caller knowing the internals. + + Attributes + ---------- + label : str + Short human-readable name, shown when the config is printed. + context_retrieval : ContextRetrievalConfig or None + Web-search sub-agent config, if this tool provides search. + code_execution : CodeExecutionConfig or None + E2B sandbox config, if this tool provides code execution. + function_tool : Any or None + A ready-to-register ADK function tool (e.g. from + ``ForecastTool.as_function_tool()``). + skill_dir : Path or None + A playbook skill directory to load alongside the tool. + instruction_supplement : str + Text appended to the agent's system instruction when this tool is on. + max_output_tokens : int or None + A per-tool floor on the response budget (e.g. code execution needs + headroom for a full script). The config takes the max across all tools. + """ + + label: str + context_retrieval: ContextRetrievalConfig | None = None + code_execution: CodeExecutionConfig | None = None + function_tool: Any | None = None + skill_dir: Path | None = None + instruction_supplement: str = "" + max_output_tokens: int | None = None + + +# --------------------------------------------------------------------------- +# Tool-specific prompt text +# --------------------------------------------------------------------------- + + +_CONTEXT_RETRIEVAL_INSTRUCTION = """\ +You are an oil-market intelligence specialist with web search. + +Return a concise structured markdown summary (3-5 paragraphs) covering, as the +query warrants: WTI/Brent price level and trend; OPEC+ supply decisions; +geopolitical risk in the Persian Gulf and key shipping lanes; US SPR / energy +policy; notable supply-disruption signals; and published analyst price targets. + +Ground every claim in the search results you actually retrieve. When a cutoff +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ +""" + + +def _forecast_tool_supplement(series_id: str, frequency: str) -> str: + """Instruction appended when the statistical forecast tool is attached. + + Kept in the factory (not hard-coded) so a tool built for a different series + or frequency describes itself correctly to the agent. + """ + return f""" + +## Statistical forecast tool + +You have access to `run_forecast`, a conventional statistical baseline +(AutoARIMA) you can call directly. Unlike open-ended code, this tool has a fixed, +auditable interface and returns a structured forecast you can reason from. + +Call it ONCE before producing your forecast, with: +- `series_id`: "{series_id}" +- `cutoff_date`: the `as_of` date from the payload (YYYY-MM-DD). This is the + information cutoff — the model uses only data on or before it. +- `horizons`: the `horizons` list from the payload. +- `frequency`: "{frequency}" (the business calendar the series trades on). + +The tool returns JSON with point forecasts and 80%/90% prediction intervals per +horizon. Treat it as a disciplined statistical anchor: combine it with any +market context you have. You may adjust away from the baseline when fundamentals +or geopolitical risk justify it — document your reasoning in the `rationale` +fields.\ +""" + + +# --------------------------------------------------------------------------- +# Tool factories — each returns one ToolSpec +# --------------------------------------------------------------------------- + + +def news_search(*, search_model: str = LITE_MODEL) -> ToolSpec: + """Build a cutoff-aware Google Search tool, run by a bounded sub-agent (proxy-only). + + Wires a ``search_web`` tool and loads the ``research-playbook`` skill. No + extra API key — everything routes through the Vector proxy. + + Parameters + ---------- + search_model : str + Model for the web-search sub-agent. Defaults to the lite model. + """ + return ToolSpec( + label="news_search", + context_retrieval=ContextRetrievalConfig( + enabled=True, + instruction=_CONTEXT_RETRIEVAL_INSTRUCTION, + search_model=search_model, + ), + skill_dir=_RESEARCH_SKILL, + ) + + +def code_sandbox() -> ToolSpec: + """Build an E2B Python sandbox for the agent to compute its own diagnostics. + + Wires the code-execution capability and loads the ``code-analysis-playbook`` + skill. Needs ``E2B_API_KEY`` and is slower than the other tools, so it is + off by default — add it to the toolbelt to turn it on. + """ + return ToolSpec( + label="code_sandbox", + code_execution=CodeExecutionConfig(enabled=True), + skill_dir=_CODE_ANALYSIS_SKILL, + # 16k headroom: enough for a complete run_code script + structured output. + max_output_tokens=16_384, + ) + + +def arima_forecast( + *, + series_id: str = WTI_SERIES_ID, + frequency: str = "B", + num_samples: int = 200, + data_service: DataService | None = None, +) -> ToolSpec: + """Build a conventional statistical anchor: AutoARIMA behind a `run_forecast` tool. + + Lets the agent invoke a statistical forecast *directly* — a rigid, auditable + interface — instead of writing forecasting code. In contrast to + :func:`code_sandbox` (open-ended), this trades flexibility for control and + reproducibility. The tool reads series data server-side; it never enters the + LLM context. + + Parameters + ---------- + series_id : str + Series the agent should forecast. Defaults to the WTI target. + frequency : str + Business calendar passed to the predictor (``"B"`` for WTI). + num_samples : int + Monte Carlo sample count for AutoARIMA. Kept modest to bound latency. + data_service : DataService or None + Pre-populated data service. When ``None``, a cache-backed WTI service is + built. Pass one to point the tool at your own series (or to avoid a data + fetch in tests). + """ + service = data_service if data_service is not None else build_wti_service() + tool = ForecastTool(service, predictor=DartsAutoARIMAPredictor(num_samples=num_samples)) + return ToolSpec( + label="arima_forecast", + function_tool=tool.as_function_tool(), + instruction_supplement=_forecast_tool_supplement(series_id, frequency), + ) +``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__tasks.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__tasks.py.md index f51bbc93..2190e453 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__tasks.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__tasks.py.md @@ -8,17 +8,20 @@ kind: python Implements the "one agent, three tasks" pattern: a single :class:`AgentConfig` identity with task-specific prompt builders and output schemas supplied via :class:`~aieng.forecasting.methods.agentic.predictor.AgentPredictor`. + +The scenario output classes are thin oil-branded subclasses of the shared, +domain-agnostic :class:`~aieng.forecasting.methods.agentic.outputs.ScenarioCard` +and :class:`~aieng.forecasting.methods.agentic.outputs.ScenarioAgentForecastOutput`. +They pin the numeric scenario-card fields to the exact key names +(``wti_range_60d``, ``point_estimate_60d``) the notebook caches key on. """ from __future__ import annotations import json -from datetime import datetime from typing import Any, ClassVar, Literal -import pandas as pd from aieng.forecasting.data.context import ForecastContext -from aieng.forecasting.evaluation.prediction import BinaryForecast, Prediction from aieng.forecasting.evaluation.task import ForecastingTask from aieng.forecasting.methods.agentic import ( AgentPredictor, @@ -26,7 +29,15 @@ from aieng.forecasting.methods.agentic import ( DiscreteAgentForecastOutput, ) from aieng.forecasting.methods.agentic.agent_factory import AgentConfig -from aieng.forecasting.methods.agentic.outputs import AgentForecastOutput +from aieng.forecasting.methods.agentic.outputs import ( + AgentForecastOutput, +) +from aieng.forecasting.methods.agentic.outputs import ( + ScenarioAgentForecastOutput as _BaseScenarioAgentForecastOutput, +) +from aieng.forecasting.methods.agentic.outputs import ( + ScenarioCard as _BaseScenarioCard, +) from aieng.forecasting.models import LITE_MODEL from energy_oil_forecasting.analyst_agent import ( WtiPriceForecastPromptBuilder, @@ -35,7 +46,7 @@ from energy_oil_forecasting.analyst_agent import ( compress_history, ) from energy_oil_forecasting.paths import SHOCK_HORIZON, SHOCK_THRESHOLD -from pydantic import BaseModel, Field +from pydantic import BaseModel # ── Task specification strings (embedded in user prompts for NB3) ─────────── @@ -72,89 +83,31 @@ class WtiMultitaskPromptBuilder(BaseModel): return json.dumps(payload, indent=2) -class ScenarioCard(BaseModel): - """One scenario card from Task C agent output.""" +class ScenarioCard(_BaseScenarioCard): + """One scenario card from Task C agent output. - model_config = {"extra": "ignore"} + Adds the WTI-specific 60-day price range and point estimate to the generic + scenario card, under the exact field names the notebook caches expect. + """ - name: str - description: str - probability: float = Field(ge=0.0, le=1.0) wti_range_60d: list[float] point_estimate_60d: float - key_drivers: list[str] = Field(default_factory=list) -class ScenarioAgentForecastOutput(AgentForecastOutput): - """Track 2 scenario analysis output for the energy case study.""" +class ScenarioAgentForecastOutput(_BaseScenarioAgentForecastOutput): + """Track 2 scenario analysis output for the energy case study. - modality: ClassVar[Literal["continuous", "discrete"]] = "discrete" - - model_config = {"extra": "ignore"} + Narrows the scenario-card type to :class:`ScenarioCard` and advertises the + WTI numeric fields in the prompt schema template. Rendering and + :meth:`to_predictions` are inherited unchanged from the shared base. + """ scenarios: list[ScenarioCard] - base_case: str - reasoning: str = "" - - @classmethod - def prompt_schema_json(cls) -> str: - """Return a JSON template for use in agent instruction strings. - - Returns - ------- - str - Indented JSON string showing the exact structure the agent must - pass to ``set_model_response``. - """ - template: dict[str, object] = { - "scenarios": [ - { - "name": "", - "description": "", - "probability": "", - "wti_range_60d": ["", ""], - "point_estimate_60d": "", - "key_drivers": ["", ""], - } - ], - "base_case": "", - "reasoning": "", - } - return json.dumps(template, indent=2) - - def to_predictions( - self, - *, - task: ForecastingTask, - context: ForecastContext, - predictor_id: str, - metadata: dict[str, Any] | None = None, - ) -> list[Prediction]: - """Convert scenario output to a metadata-rich prediction (Track 2 display).""" - if len(task.horizons) != 1: - raise ValueError("Scenario agent output expects exactly one task horizon.") - - horizon = task.horizons[0] - issued_at = datetime.utcnow() - offset = pd.tseries.frequencies.to_offset(task.frequency) - base_prob = float(sum(s.probability for s in self.scenarios)) - prediction_metadata: dict[str, Any] = dict(metadata) if metadata is not None else {} - prediction_metadata["scenarios"] = [s.model_dump() for s in self.scenarios] - prediction_metadata["base_case"] = self.base_case - if self.reasoning.strip(): - prediction_metadata["rationale"] = self.reasoning - - return [ - Prediction( - predictor_id=predictor_id, - task_id=task.task_id, - issued_at=issued_at, - as_of=context.as_of, - forecast_date=(pd.Timestamp(context.as_of) + offset * horizon).to_pydatetime(), - payload=BinaryForecast(probability=min(base_prob, 1.0)), - metadata=prediction_metadata, - ) - ] + + scenario_card_template_extra: ClassVar[dict[str, object]] = { + "wti_range_60d": ["", ""], + "point_estimate_60d": "", + } # Task specification strings embedded in user prompts for NB3. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md index e8693676..ddc06844 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md @@ -1220,4 +1220,274 @@ def prob_bar(val: float, width: int = 10) -> str: def conf_bar(conf: str) -> str: """Map confidence label to emoji indicator.""" return {"high": "🟢", "medium": "🟡", "low": "🔴"}.get(conf.lower(), "⚪") + + +# ── NB4 eval-diagnostic charts ──────────────────────────────────────────────── +# These read the tidy per-prediction frame from ``analysis.predictions_to_frame`` +# (one row per predictor × origin × horizon) and answer three questions the bare +# leaderboard can't: *where* the ranking is decided (heatmap), whether a lead is +# real or noise (leaderboard with error bars), and *what* the methods actually +# forecast vs reality (trajectory chart). + +# Qualitative palette for an arbitrary, growing predictor set. Stable per call: +# colours are assigned by sorted predictor name so a method keeps its colour +# across the heatmap, leaderboard, and trajectory charts within one notebook run. +_PREDICTOR_PALETTE = [ + "#1f77b4", + "#ff7f0e", + "#2ca02c", + "#d62728", + "#9467bd", + "#8c564b", + "#e377c2", + "#7f7f7f", + "#bcbd22", + "#17becf", + "#393b79", + "#b5651d", +] + + +def predictor_colors(predictors: list[str]) -> dict[str, str]: + """Assign a stable colour to each predictor name.""" + return {name: _PREDICTOR_PALETTE[i % len(_PREDICTOR_PALETTE)] for i, name in enumerate(predictors)} + + +def make_crps_heatmap(per_horizon_df: pd.DataFrame) -> go.Figure: + """Predictor × horizon mean-CRPS heatmap (lower = better, sorted best-first). + + Expects the output of ``analysis.per_horizon_crps`` — horizon columns plus a + final ``All`` column. Reveals which horizon decides the ranking: typically the + short horizons are a wash and one long horizon dominates the mean. + """ + df = per_horizon_df.copy() + # Best predictor on top: reverse so plotly's bottom-up y-axis shows it first. + df = df.iloc[::-1] + z = df.to_numpy(dtype=float) + fig = go.Figure( + go.Heatmap( + z=z, + x=list(df.columns), + y=list(df.index), + colorscale="RdYlGn_r", + colorbar={"title": "CRPS"}, + text=[[f"{v:.2f}" if np.isfinite(v) else "" for v in row] for row in z], + texttemplate="%{text}", + textfont={"size": 12}, + hovertemplate="%{y}
%{x}: %{z:.3f}", + ) + ) + fig.update_layout( + title={"text": "Mean CRPS by Predictor × Horizon (lower = better)", "font": {"size": 16}}, + xaxis={"title": "Horizon", "side": "top"}, + yaxis={"title": ""}, + template="plotly_white", + width=720, + height=40 * len(df) + 160, + margin={"t": 90, "b": 40, "l": 230, "r": 40}, + ) + # Visually separate the "All" summary column. + if "All" in per_horizon_df.columns: + fig.add_vline(x=len(per_horizon_df.columns) - 1.5, line={"color": "#333333", "width": 1.5}) + return fig + + +def make_leaderboard_interval_chart(board_df: pd.DataFrame) -> go.Figure: + """Mean CRPS ± standard error per predictor, exposing whether a lead is noise. + + Expects ``analysis.leaderboard_with_uncertainty``. When the error bars of the + top methods overlap heavily, the ranking is not statistically meaningful — the + honest read on a short eval window. + """ + df = board_df.iloc[::-1] # best at top of the bottom-up axis + fam_colors = {"Baseline": "#7f7f7f", "Numerical ML": "#1f77b4", "LLM / Agent": "#2ca02c", "Other": "#b5651d"} + colors = [fam_colors.get(f, "#b5651d") for f in df["family"]] + fig = go.Figure( + go.Scatter( + x=df["mean_crps"], + y=df.index, + mode="markers", + marker={"size": 11, "color": colors}, + error_x={"type": "data", "array": df["se"].fillna(0.0), "thickness": 1.6, "width": 6, "color": "#888888"}, + hovertemplate="%{y}
CRPS %{x:.3f} ± %{error_x.array:.3f}", + showlegend=False, + ) + ) + best = float(df["mean_crps"].min()) + fig.add_vline( + x=best, + line={"color": "#31a354", "dash": "dot", "width": 1.5}, + annotation_text=" best", + annotation_position="top", + annotation_font={"size": 11, "color": "#31a354"}, + ) + fig.update_layout( + title={"text": "Eval Leaderboard — Mean CRPS ± 1 SE (overlap ⇒ tied)", "font": {"size": 16}}, + xaxis={"title": "Mean CRPS (lower = better)", "showgrid": True, "gridcolor": "#f0f0f0"}, + yaxis={"title": ""}, + template="plotly_white", + width=760, + height=34 * len(df) + 150, + margin={"t": 70, "b": 50, "l": 230, "r": 40}, + ) + return fig + + +def make_eval_forecast_chart( + pred_frame: pd.DataFrame, + price_df: pd.DataFrame, + predictors: list[str], + *, + history_window: int = 25, +) -> go.Figure: + """Per-origin trajectory chart: each method's median + 80% band vs reality. + + One column per forecast origin. Shows the pre-origin price history, the + realised price path, and — for each selected predictor — point forecasts at + each horizon with 80% interval error bars. This is the "what are the top + methods actually doing" view: you can see who tracks the move, who lags, and + whose intervals are too tight. + """ + origins = sorted(pred_frame["as_of"].unique()) + colors = predictor_colors(predictors) + + titles = [] + for o_raw in origins: + o = pd.Timestamp(o_raw) + rows = price_df[price_df.index >= o.normalize()] + spot = f"${float(rows.iloc[0]['price']):.0f}" if not rows.empty else "" + titles.append(f"{o.strftime('%b %d, %Y')} WTI {spot}") + + fig = psp.make_subplots( + rows=1, cols=len(origins), subplot_titles=titles, shared_yaxes=True, horizontal_spacing=0.03 + ) + + for col, origin_raw in enumerate(origins, start=1): + origin = pd.Timestamp(origin_raw) + show_legend = col == 1 + sub = pred_frame[pred_frame["as_of"] == origin] + last_fdate = pd.Timestamp(sub["forecast_date"].max()) + + # Pre-origin history + realised future path. + hist = price_df[price_df.index <= origin.normalize()].iloc[-history_window:] + future = price_df[(price_df.index > origin.normalize()) & (price_df.index <= last_fdate)] + fig.add_trace( + go.Scatter( + x=hist.index.tolist(), + y=hist["price"].tolist(), + mode="lines", + line={"color": CLR_HISTORY, "width": 1.5}, + name="WTI history", + showlegend=show_legend, + legendgroup="hist", + ), + row=1, + col=col, + ) + fig.add_trace( + go.Scatter( + x=future.index.tolist(), + y=future["price"].tolist(), + mode="lines+markers", + line={"color": CLR_ACTUAL, "width": 2.5}, + marker={"size": 5}, + name="Realised price", + showlegend=show_legend, + legendgroup="actual", + ), + row=1, + col=col, + ) + + # Each predictor's median + 80% interval at every horizon. + for name in predictors: + pr = sub[sub["predictor"] == name].sort_values("forecast_date") + if pr.empty: + continue + err_hi = (pr["q80"] - pr["point"]).clip(lower=0).fillna(0.0) + err_lo = (pr["point"] - pr["q20"]).clip(lower=0).fillna(0.0) + fig.add_trace( + go.Scatter( + x=pr["forecast_date"].tolist(), + y=pr["point"].tolist(), + mode="lines+markers", + line={"color": colors[name], "width": 1.4, "dash": "dot"}, + marker={"size": 8, "symbol": "diamond"}, + error_y={ + "type": "data", + "symmetric": False, + "array": err_hi.tolist(), + "arrayminus": err_lo.tolist(), + "color": colors[name], + "thickness": 1.4, + "width": 4, + }, + name=name, + showlegend=show_legend, + legendgroup=name, + ), + row=1, + col=col, + ) + + fig.add_vline( + x=origin.timestamp() * 1000, line={"color": "#aaaaaa", "dash": "dash", "width": 1}, row=1, col=col + ) + + fig.update_layout( + title={"text": "Eval Forecasts vs Reality — Median + 80% Interval by Origin", "font": {"size": 16}}, + template="plotly_white", + width=max(420 * len(origins), 720), + height=480, + margin={"t": 80, "b": 110, "l": 60, "r": 20}, + legend={"orientation": "h", "y": -0.18, "x": 0.0, "xanchor": "left", "font": {"size": 11}}, + ) + fig.update_xaxes(showgrid=True, gridcolor="#f0f0f0", tickfont={"size": 10}) + fig.update_yaxes(showgrid=True, gridcolor="#f0f0f0", tickfont={"size": 11}) + return fig + + +def render_rationales_html(rationale_df: pd.DataFrame, *, max_chars: int = 700) -> str: + """Render agent/LLM rationales as readable HTML cards with trace links. + + Expects ``analysis.extract_agent_rationales``. One card per (predictor, + origin), showing the overall rationale, the per-horizon note, and a link to + the Langfuse trace so the full agent reasoning is one click away. + """ + if rationale_df.empty: + return "

No agent/LLM rationales found in this run's metadata.

" + + def _clip(text: str) -> str: + text = (text or "").strip() + return text if len(text) <= max_chars else text[:max_chars].rsplit(" ", 1)[0] + " …" + + # One representative card per (predictor, origin) — the rationale is shared + # across horizons, so dedupe to the first row of each group. + seen: set[tuple[str, str]] = set() + cards: list[str] = [] + for _, r in rationale_df.sort_values(["predictor", "as_of"]).iterrows(): + key = (r["predictor"], str(pd.Timestamp(r["as_of"]).date())) + if key in seen: + continue + seen.add(key) + link = ( + f"🔗 Langfuse trace" + if r.get("trace_url") + else "" + ) + horizon_note = ( + f"
Horizon note: {_clip(r['horizon_rationale'])}
" + if r.get("horizon_rationale") + else "" + ) + cards.append( + f"
" + f"
" + f"{r['predictor']}" + f"{key[1]}   point ${r['point']:.1f}   {link}
" + f"
{_clip(r['rationale'])}
" + f"{horizon_note}
" + ) + return f"
{''.join(cards)}
" ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md index 139e8d5a..cf88644e 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md @@ -141,7 +141,16 @@ commodity and input costs (grains, energy, fertiliser); supply-chain and weather disruptions; and the CAD exchange rate. Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md index 467eb3ca..2c9e2463 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md index 03cc28e0..9819c41d 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md @@ -155,7 +155,16 @@ the VIX and credit spreads; earnings-season tone; and major geopolitical or policy shocks. Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md index dce71dea..359dd966 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/catalog.yaml b/implementations/getting_started/concierge_agent/context/catalog.yaml index 9a6f8110..980c04da 100644 --- a/implementations/getting_started/concierge_agent/context/catalog.yaml +++ b/implementations/getting_started/concierge_agent/context/catalog.yaml @@ -1,9 +1,9 @@ source_url: https://github.com/VectorInstitute/agentic-forecasting -git_ref: 0ac6b3098bdb529c08f2445895d82490de2404fd +git_ref: 9715e2eeb6479c83eb7fd35acc97ebac26db8988 branch: main -built_at: '2026-06-30T15:09:29+00:00' -ingest_source: /home/coder/agentic-forecasting -entry_count: 196 +built_at: '2026-07-16T00:57:38+00:00' +ingest_source: /Users/ethanjackson/agentic-forecasting/.claude/worktrees/agent-ad5b56acdc7a89a16 +entry_count: 202 entries: - path: AGENTS.md kind: markdown @@ -477,7 +477,7 @@ entries: summary: ADK-based agentic predictors. symbols: [] sections: [] - chars: 4216 + chars: 5001 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic____init__.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/adaptive_skill.py kind: python @@ -489,6 +489,15 @@ entries: sections: [] chars: 8583 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adaptive_skill.py.md +- path: aieng-forecasting/aieng/forecasting/methods/agentic/adaptive_skill_tools.py + kind: python + domain: core.methods + summary: Generic strategy-mutation tools for adaptive forecasting agents. + symbols: + - build_skill_tools + sections: [] + chars: 15735 + artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adaptive_skill_tools.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/adk_runner.py kind: python domain: core.methods @@ -497,7 +506,7 @@ entries: - AdkTextRunnerConfig - AdkTextRunner sections: [] - chars: 15228 + chars: 16225 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py kind: python @@ -507,10 +516,11 @@ entries: - _LiteLLMNoiseFilter - ContextRetrievalConfig - CodeExecutionConfig + - _LeakageVerification - AgentConfig - build_adk_agent sections: [] - chars: 24986 + chars: 35486 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/curriculum.py kind: python @@ -521,8 +531,23 @@ entries: - load_context_documents - build_curriculum_prompt sections: [] - chars: 20460 + chars: 21230 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__curriculum.py.md +- path: aieng-forecasting/aieng/forecasting/methods/agentic/domain.py + kind: python + domain: core.methods + summary: Domain configuration for agentic forecasters. + symbols: + - DomainConfig + - render_analyst_instruction + - render_multitask_analyst_instruction + - render_starter_instruction + - render_adaptive_analyst_instruction + - build_analyst_config + - build_adaptive_config + sections: [] + chars: 23499 + artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__domain.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/forecast_tool.py kind: python domain: core.methods @@ -532,6 +557,15 @@ entries: sections: [] chars: 10814 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__forecast_tool.py.md +- path: aieng-forecasting/aieng/forecasting/methods/agentic/history.py + kind: python + domain: core.methods + summary: History compression for agent prompt payloads. + symbols: + - compress_history + sections: [] + chars: 1821 + artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__history.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/outputs.py kind: python domain: core.methods @@ -544,8 +578,10 @@ entries: - DiscreteAgentForecastOutput - AgentCategoryProbability - CategoricalAgentForecastOutput + - ScenarioCard + - ScenarioAgentForecastOutput sections: [] - chars: 25341 + chars: 29337 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__outputs.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/predictor.py kind: python @@ -555,8 +591,21 @@ entries: - ForecastPromptBuilder - AgentPredictor sections: [] - chars: 14369 + chars: 14715 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md +- path: aieng-forecasting/aieng/forecasting/methods/agentic/strategy_state.py + kind: python + domain: core.methods + summary: Generic adaptive forecasting-strategy state. + symbols: + - Observation + - Hypothesis + - CalibrationCorrection + - VersionEntry + - StrategyState + sections: [] + chars: 7982 + artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__strategy_state.py.md - path: aieng-forecasting/aieng/forecasting/methods/baselines/__init__.py kind: python domain: core.methods @@ -925,7 +974,7 @@ entries: - build_boc_news_config - build_boc_agent_predictor sections: [] - chars: 16347 + chars: 17207 artifact: artifacts/implementations__boc_rate_decisions__analyst_agent__agent.py.md - path: implementations/boc_rate_decisions/data.py kind: python @@ -1199,7 +1248,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 9637 + chars: 10211 artifact: artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md - path: implementations/boc_rate_decisions/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -1238,7 +1287,7 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1947 + chars: 2250 artifact: artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md - path: implementations/energy_oil_forecasting/01_wti_case_study.ipynb kind: notebook @@ -1281,7 +1330,7 @@ entries: sections: - "WTI Crude Oil Price Forecasting \u2014 Stateless Methods: Systematic Backtest\ \ (Notebook 4 of 7)" - chars: 18735 + chars: 22648 artifact: artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md - path: implementations/energy_oil_forecasting/05_adaptive_agent_training.ipynb kind: notebook @@ -1321,7 +1370,7 @@ entries: symbols: [] sections: - "WTI Crude Oil \u2014 Your Starter Agent" - chars: 7573 + chars: 8586 artifact: artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md - path: implementations/energy_oil_forecasting/README.md kind: markdown @@ -1374,7 +1423,7 @@ entries: - build_wti_adaptive_config - build_wti_adaptive_predictor sections: [] - chars: 19884 + chars: 11110 artifact: artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md - path: implementations/energy_oil_forecasting/adaptive_agent/curriculum/snapshot_utils.py kind: python @@ -1392,13 +1441,14 @@ entries: domain: impl.energy_oil_forecasting summary: WTI forecasting strategy state model. symbols: - - Observation - - Hypothesis + - WtiStrategyState - CalibrationCorrection + - Hypothesis + - Observation - VersionEntry - WtiStrategyState sections: [] - chars: 7381 + chars: 2086 artifact: artifacts/implementations__energy_oil_forecasting__adaptive_agent__skill_state.py.md - path: implementations/energy_oil_forecasting/adaptive_agent/skill_tools.py kind: python @@ -1406,8 +1456,14 @@ entries: summary: Mutation tools for the ``wti-strategy`` adaptive skill. symbols: - build_skill_tools + - CalibrationCorrection + - Hypothesis + - Observation + - VersionEntry + - WtiStrategyState + - build_skill_tools sections: [] - chars: 16387 + chars: 2474 artifact: artifacts/implementations__energy_oil_forecasting__adaptive_agent__skill_tools.py.md - path: implementations/energy_oil_forecasting/adaptive_agent/skills/fetch-yfinance/SKILL.md kind: markdown @@ -1564,14 +1620,28 @@ entries: - backtest_results_to_frame - trajectory_mae_table - select_top_predictors + - predictor_family + - predictions_to_frame + - per_horizon_crps + - leaderboard_with_uncertainty + - extract_agent_rationales + - eval_narrative_md + - build_price_frame - backtest_results_to_frame + - build_price_frame - compute_brier_score + - eval_narrative_md + - extract_agent_rationales + - leaderboard_with_uncertainty + - per_horizon_crps + - predictions_to_frame + - predictor_family - rolling_coverage_pct - score_backtest_results - select_top_predictors - trajectory_mae_table sections: [] - chars: 6984 + chars: 19850 artifact: artifacts/implementations__energy_oil_forecasting__analysis.py.md - path: implementations/energy_oil_forecasting/analyst_agent/__init__.py kind: python @@ -1594,7 +1664,6 @@ entries: domain: impl.energy_oil_forecasting summary: WTI crude oil analyst agent configurations and prompt builder. symbols: - - compress_history - WtiPriceForecastPromptBuilder - build_wti_basic_config - build_wti_multitask_news_config @@ -1603,7 +1672,7 @@ entries: - build_wti_tool_config - build_wti_agent_predictor sections: [] - chars: 22599 + chars: 21151 artifact: artifacts/implementations__energy_oil_forecasting__analyst_agent__agent.py.md - path: implementations/energy_oil_forecasting/analyst_agent/skills/statistical-analysis/SKILL.md kind: markdown @@ -1704,6 +1773,14 @@ entries: sections: [] chars: 13040 artifact: artifacts/implementations__energy_oil_forecasting__data.py.md +- path: implementations/energy_oil_forecasting/domain.py + kind: python + domain: impl.energy_oil_forecasting + summary: WTI crude oil domain configuration. + symbols: [] + sections: [] + chars: 5338 + artifact: artifacts/implementations__energy_oil_forecasting__domain.py.md - path: implementations/energy_oil_forecasting/paths.py kind: python domain: impl.energy_oil_forecasting @@ -1787,11 +1864,15 @@ entries: symbols: [] sections: - "Energy Oil Eval Spec \u2014 2026 Prospective Competition" - - '# Runs on 8 weekly origins from Feb 2, 2026 to Mar 23, 2026.' - - Covers the high-volatility Persian Gulf geopolitical price shock period. + - '# Runs on 18 weekly origins from Feb 2, 2026 to Jun 1, 2026.' + - Covers the high-volatility Persian Gulf geopolitical price shock and its + - aftermath. The end date is set to the latest origin whose longest horizon + - "(21 business days) still resolves against available data \u2014 keep it at" + - most 21 business days behind the most recent cached WTI price (see + - scripts/fetch_wti.py) so every origin fully resolves. - 'Target is WTI Crude Oil price (yfinance ticker: CL=F).' - 'Horizons: 5, 10, 21 business days.' - chars: 1019 + chars: 1316 artifact: artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md - path: implementations/energy_oil_forecasting/specs/energy_oil_eval_smoke.yaml kind: yaml @@ -1803,9 +1884,9 @@ entries: - '# Two-origin subset of energy_oil_eval.yaml for running the 2026 protected' - arena cheaply during development and end-to-end testing. - Use by setting SMOKE_TEST = True in the notebook setup cell. - - '# Origin count : 2 (vs. 8 in the full eval)' + - '# Origin count : 2 (vs. 18 in the full eval)' - 'Warmup : 250 trading days (~1 year) of historical prices' - chars: 1084 + chars: 1085 artifact: artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md - path: implementations/energy_oil_forecasting/specs/energy_oil_smoke.yaml kind: yaml @@ -1828,8 +1909,9 @@ entries: symbols: - build_starter_agent_config - build_starter_agent_predictor + - tools sections: [] - chars: 542 + chars: 688 artifact: artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md - path: implementations/energy_oil_forecasting/starter_agent/agent.py kind: python @@ -1840,7 +1922,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 9265 + chars: 9273 artifact: artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md - path: implementations/energy_oil_forecasting/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -1880,8 +1962,21 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1830 + chars: 2133 artifact: artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +- path: implementations/energy_oil_forecasting/starter_agent/tools.py + kind: python + domain: impl.energy_oil_forecasting + summary: "The starter agent's toolbelt \u2014 one factory per tool, composed in\ + \ the notebook." + symbols: + - ToolSpec + - news_search + - code_sandbox + - arima_forecast + sections: [] + chars: 9826 + artifact: artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md - path: implementations/energy_oil_forecasting/tasks.py kind: python domain: impl.energy_oil_forecasting @@ -1903,7 +1998,7 @@ entries: - build_wti_agent_predictor_for_task - build_wti_news_predictor sections: [] - chars: 8968 + chars: 7388 artifact: artifacts/implementations__energy_oil_forecasting__tasks.py.md - path: implementations/energy_oil_forecasting/viz.py kind: python @@ -1923,8 +2018,13 @@ entries: - verdict_label - prob_bar - conf_bar + - predictor_colors + - make_crps_heatmap + - make_leaderboard_interval_chart + - make_eval_forecast_chart + - render_rationales_html sections: [] - chars: 43163 + chars: 54100 artifact: artifacts/implementations__energy_oil_forecasting__viz.py.md - path: implementations/food_price_forecasting/01_food_data_exploration.ipynb kind: notebook @@ -2202,7 +2302,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 10665 + chars: 11239 artifact: artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md - path: implementations/food_price_forecasting/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -2242,7 +2342,7 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1871 + chars: 2174 artifact: artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md - path: implementations/getting_started/00_environment_check.ipynb kind: notebook @@ -2645,7 +2745,7 @@ entries: - build_sp500_multivariate_service - sp500_logret_series_id sections: [] - chars: 31632 + chars: 32091 artifact: artifacts/implementations__sp500_forecasting__data.py.md - path: implementations/sp500_forecasting/leaderboard.py kind: python @@ -2799,7 +2899,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 11869 + chars: 12443 artifact: artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md - path: implementations/sp500_forecasting/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -2839,7 +2939,7 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1860 + chars: 2163 artifact: artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md - path: planning-docs/roadmap.md kind: markdown @@ -2901,12 +3001,13 @@ entries: - path: scripts/fetch_fred.py kind: python domain: scripts - summary: Populate the local FRED cache with series used by the CFPR experiment. + summary: Populate the local FRED cache with series used by the food-price and S&P + 500 experiments. symbols: - build_data_service - main sections: [] - chars: 6020 + chars: 8603 artifact: artifacts/scripts__fetch_fred.py.md - path: scripts/fetch_sp500_market.py kind: python diff --git a/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml b/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml index 14913f1b..55ef1f34 100644 --- a/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml +++ b/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml @@ -1,20 +1,20 @@ # Concierge catalog summary (regenerated by scripts/build_concierge_context.py) source_url: https://github.com/VectorInstitute/agentic-forecasting branch: main -built_at: '2026-06-30T15:09:29+00:00' -git_ref: 0ac6b3098bdb529c08f2445895d82490de2404fd -entry_count: 196 +built_at: '2026-07-16T00:57:38+00:00' +git_ref: 9715e2eeb6479c83eb7fd35acc97ebac26db8988 +entry_count: 202 domains: docs: 4 core.root: 3 core.data: 12 core.documents: 5 core.evaluation: 9 - core.methods: 26 + core.methods: 30 impl.README.md: 1 impl.__init__.py: 1 impl.boc_rate_decisions: 27 - impl.energy_oil_forecasting: 45 + impl.energy_oil_forecasting: 47 impl.food_price_forecasting: 21 impl.getting_started: 16 impl.sp500_forecasting: 19 From fe137aafbf51615924c16ff2152160239c6b062c Mon Sep 17 00:00:00 2001 From: Ethan Jackson Date: Wed, 15 Jul 2026 21:11:29 -0400 Subject: [PATCH 3/3] chore(deps): bump mistune to 3.3.3 and pillow to 12.3.0 for pip-audit New PYSEC-2026 advisories against the locked versions fail CI's pip-audit step on every branch (main last passed 2026-07-07, before publication). Full test suite re-verified after the bump: 489 passed, 7 skipped. Co-Authored-By: Claude Fable 5 --- uv.lock | 138 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 70 insertions(+), 68 deletions(-) diff --git a/uv.lock b/uv.lock index 7314a4a5..6c04371f 100644 --- a/uv.lock +++ b/uv.lock @@ -2938,11 +2938,11 @@ wheels = [ [[package]] name = "mistune" -version = "3.2.1" +version = "3.3.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/84/620cc3f7e3adf6f5067e10f4dbae71295d8f9e16d5d3f9ef97c40f2f592c/mistune-3.2.1.tar.gz", hash = "sha256:7c8e5501d38bac1582e067e46c8343f17d57ea1aaa735823f3aba1fd59c88a28", size = 98003, upload-time = "2026-05-03T14:33:22.312Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/a5/2dab368d6950e6808904dec98f54c7e726ee7be4a0c6afe00e6e011bd52d/mistune-3.3.3.tar.gz", hash = "sha256:c4c6c0c840b8637a2e9b8b6d607eb7c8f00888bf14c754409bcd339e848c2477", size = 115363, upload-time = "2026-07-09T06:18:05.268Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/7f/a946aa4f8752b37102b41e64dca18a1976ac705c3a0d1dfe74d820a02552/mistune-3.2.1-py3-none-any.whl", hash = "sha256:78cdb0ba5e938053ccf63651b352508d2efa9411dc8810bfb05f2dc5140c0048", size = 53749, upload-time = "2026-05-03T14:33:20.551Z" }, + { url = "https://files.pythonhosted.org/packages/89/70/b1e4737b84163db5bb1dfde6f216dbfbf32783330a9989c965e121172830/mistune-3.3.3-py3-none-any.whl", hash = "sha256:99de1585e42dcbd826faa9e11a202727a5e202e4e4722a4c69ac1ff615793dd7", size = 63569, upload-time = "2026-07-09T06:18:03.839Z" }, ] [[package]] @@ -3746,71 +3746,73 @@ wheels = [ [[package]] name = "pillow" -version = "12.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5", size = 5308279, upload-time = "2026-04-01T14:43:13.246Z" }, - { url = "https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421", size = 4695490, upload-time = "2026-04-01T14:43:15.584Z" }, - { url = "https://files.pythonhosted.org/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987", size = 6284462, upload-time = "2026-04-01T14:43:18.268Z" }, - { url = "https://files.pythonhosted.org/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76", size = 8094744, upload-time = "2026-04-01T14:43:20.716Z" }, - { url = "https://files.pythonhosted.org/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005", size = 6398371, upload-time = "2026-04-01T14:43:23.443Z" }, - { url = "https://files.pythonhosted.org/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780", size = 7087215, upload-time = "2026-04-01T14:43:26.758Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f8/2f6825e441d5b1959d2ca5adec984210f1ec086435b0ed5f52c19b3b8a6e/pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5", size = 6509783, upload-time = "2026-04-01T14:43:29.56Z" }, - { url = "https://files.pythonhosted.org/packages/67/f9/029a27095ad20f854f9dba026b3ea6428548316e057e6fc3545409e86651/pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5", size = 7212112, upload-time = "2026-04-01T14:43:32.091Z" }, - { url = "https://files.pythonhosted.org/packages/be/42/025cfe05d1be22dbfdb4f264fe9de1ccda83f66e4fc3aac94748e784af04/pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940", size = 6378489, upload-time = "2026-04-01T14:43:34.601Z" }, - { url = "https://files.pythonhosted.org/packages/5d/7b/25a221d2c761c6a8ae21bfa3874988ff2583e19cf8a27bf2fee358df7942/pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5", size = 7084129, upload-time = "2026-04-01T14:43:37.213Z" }, - { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612, upload-time = "2026-04-01T14:43:39.421Z" }, - { url = "https://files.pythonhosted.org/packages/4a/01/53d10cf0dbad820a8db274d259a37ba50b88b24768ddccec07355382d5ad/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c", size = 4100837, upload-time = "2026-04-01T14:43:41.506Z" }, - { url = "https://files.pythonhosted.org/packages/0f/98/f3a6657ecb698c937f6c76ee564882945f29b79bad496abcba0e84659ec5/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2", size = 4176528, upload-time = "2026-04-01T14:43:43.773Z" }, - { url = "https://files.pythonhosted.org/packages/69/bc/8986948f05e3ea490b8442ea1c1d4d990b24a7e43d8a51b2c7d8b1dced36/pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c", size = 3640401, upload-time = "2026-04-01T14:43:45.87Z" }, - { url = "https://files.pythonhosted.org/packages/34/46/6c717baadcd62bc8ed51d238d521ab651eaa74838291bda1f86fe1f864c9/pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795", size = 5308094, upload-time = "2026-04-01T14:43:48.438Z" }, - { url = "https://files.pythonhosted.org/packages/71/43/905a14a8b17fdb1ccb58d282454490662d2cb89a6bfec26af6d3520da5ec/pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f", size = 4695402, upload-time = "2026-04-01T14:43:51.292Z" }, - { url = "https://files.pythonhosted.org/packages/73/dd/42107efcb777b16fa0393317eac58f5b5cf30e8392e266e76e51cff28c3d/pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed", size = 6280005, upload-time = "2026-04-01T14:43:54.242Z" }, - { url = "https://files.pythonhosted.org/packages/a8/68/b93e09e5e8549019e61acf49f65b1a8530765a7f812c77a7461bca7e4494/pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9", size = 8090669, upload-time = "2026-04-01T14:43:57.335Z" }, - { url = "https://files.pythonhosted.org/packages/4b/6e/3ccb54ce8ec4ddd1accd2d89004308b7b0b21c4ac3d20fa70af4760a4330/pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed", size = 6395194, upload-time = "2026-04-01T14:43:59.864Z" }, - { url = "https://files.pythonhosted.org/packages/67/ee/21d4e8536afd1a328f01b359b4d3997b291ffd35a237c877b331c1c3b71c/pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3", size = 7082423, upload-time = "2026-04-01T14:44:02.74Z" }, - { url = "https://files.pythonhosted.org/packages/78/5f/e9f86ab0146464e8c133fe85df987ed9e77e08b29d8d35f9f9f4d6f917ba/pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9", size = 6505667, upload-time = "2026-04-01T14:44:05.381Z" }, - { url = "https://files.pythonhosted.org/packages/ed/1e/409007f56a2fdce61584fd3acbc2bbc259857d555196cedcadc68c015c82/pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795", size = 7208580, upload-time = "2026-04-01T14:44:08.39Z" }, - { url = "https://files.pythonhosted.org/packages/23/c4/7349421080b12fb35414607b8871e9534546c128a11965fd4a7002ccfbee/pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e", size = 6375896, upload-time = "2026-04-01T14:44:11.197Z" }, - { url = "https://files.pythonhosted.org/packages/3f/82/8a3739a5e470b3c6cbb1d21d315800d8e16bff503d1f16b03a4ec3212786/pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b", size = 7081266, upload-time = "2026-04-01T14:44:13.947Z" }, - { url = "https://files.pythonhosted.org/packages/c3/25/f968f618a062574294592f668218f8af564830ccebdd1fa6200f598e65c5/pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06", size = 2463508, upload-time = "2026-04-01T14:44:16.312Z" }, - { url = "https://files.pythonhosted.org/packages/4d/a4/b342930964e3cb4dce5038ae34b0eab4653334995336cd486c5a8c25a00c/pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b", size = 5309927, upload-time = "2026-04-01T14:44:18.89Z" }, - { url = "https://files.pythonhosted.org/packages/9f/de/23198e0a65a9cf06123f5435a5d95cea62a635697f8f03d134d3f3a96151/pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f", size = 4698624, upload-time = "2026-04-01T14:44:21.115Z" }, - { url = "https://files.pythonhosted.org/packages/01/a6/1265e977f17d93ea37aa28aa81bad4fa597933879fac2520d24e021c8da3/pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612", size = 6321252, upload-time = "2026-04-01T14:44:23.663Z" }, - { url = "https://files.pythonhosted.org/packages/3c/83/5982eb4a285967baa70340320be9f88e57665a387e3a53a7f0db8231a0cd/pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c", size = 8126550, upload-time = "2026-04-01T14:44:26.772Z" }, - { url = "https://files.pythonhosted.org/packages/4e/48/6ffc514adce69f6050d0753b1a18fd920fce8cac87620d5a31231b04bfc5/pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea", size = 6433114, upload-time = "2026-04-01T14:44:29.615Z" }, - { url = "https://files.pythonhosted.org/packages/36/a3/f9a77144231fb8d40ee27107b4463e205fa4677e2ca2548e14da5cf18dce/pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4", size = 7115667, upload-time = "2026-04-01T14:44:32.773Z" }, - { url = "https://files.pythonhosted.org/packages/c1/fc/ac4ee3041e7d5a565e1c4fd72a113f03b6394cc72ab7089d27608f8aaccb/pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4", size = 6538966, upload-time = "2026-04-01T14:44:35.252Z" }, - { url = "https://files.pythonhosted.org/packages/c0/a8/27fb307055087f3668f6d0a8ccb636e7431d56ed0750e07a60547b1e083e/pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea", size = 7238241, upload-time = "2026-04-01T14:44:37.875Z" }, - { url = "https://files.pythonhosted.org/packages/ad/4b/926ab182c07fccae9fcb120043464e1ff1564775ec8864f21a0ebce6ac25/pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24", size = 6379592, upload-time = "2026-04-01T14:44:40.336Z" }, - { url = "https://files.pythonhosted.org/packages/c2/c4/f9e476451a098181b30050cc4c9a3556b64c02cf6497ea421ac047e89e4b/pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98", size = 7085542, upload-time = "2026-04-01T14:44:43.251Z" }, - { url = "https://files.pythonhosted.org/packages/00/a4/285f12aeacbe2d6dc36c407dfbbe9e96d4a80b0fb710a337f6d2ad978c75/pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453", size = 2465765, upload-time = "2026-04-01T14:44:45.996Z" }, - { url = "https://files.pythonhosted.org/packages/bf/98/4595daa2365416a86cb0d495248a393dfc84e96d62ad080c8546256cb9c0/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8", size = 4100848, upload-time = "2026-04-01T14:44:48.48Z" }, - { url = "https://files.pythonhosted.org/packages/0b/79/40184d464cf89f6663e18dfcf7ca21aae2491fff1a16127681bf1fa9b8cf/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b", size = 4176515, upload-time = "2026-04-01T14:44:51.353Z" }, - { url = "https://files.pythonhosted.org/packages/b0/63/703f86fd4c422a9cf722833670f4f71418fb116b2853ff7da722ea43f184/pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295", size = 3640159, upload-time = "2026-04-01T14:44:53.588Z" }, - { url = "https://files.pythonhosted.org/packages/71/e0/fb22f797187d0be2270f83500aab851536101b254bfa1eae10795709d283/pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed", size = 5312185, upload-time = "2026-04-01T14:44:56.039Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8c/1a9e46228571de18f8e28f16fabdfc20212a5d019f3e3303452b3f0a580d/pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae", size = 4695386, upload-time = "2026-04-01T14:44:58.663Z" }, - { url = "https://files.pythonhosted.org/packages/70/62/98f6b7f0c88b9addd0e87c217ded307b36be024d4ff8869a812b241d1345/pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601", size = 6280384, upload-time = "2026-04-01T14:45:01.5Z" }, - { url = "https://files.pythonhosted.org/packages/5e/03/688747d2e91cfbe0e64f316cd2e8005698f76ada3130d0194664174fa5de/pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be", size = 8091599, upload-time = "2026-04-01T14:45:04.5Z" }, - { url = "https://files.pythonhosted.org/packages/f6/35/577e22b936fcdd66537329b33af0b4ccfefaeabd8aec04b266528cddb33c/pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f", size = 6396021, upload-time = "2026-04-01T14:45:07.117Z" }, - { url = "https://files.pythonhosted.org/packages/11/8d/d2532ad2a603ca2b93ad9f5135732124e57811d0168155852f37fbce2458/pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286", size = 7083360, upload-time = "2026-04-01T14:45:09.763Z" }, - { url = "https://files.pythonhosted.org/packages/5e/26/d325f9f56c7e039034897e7380e9cc202b1e368bfd04d4cbe6a441f02885/pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50", size = 6507628, upload-time = "2026-04-01T14:45:12.378Z" }, - { url = "https://files.pythonhosted.org/packages/5f/f7/769d5632ffb0988f1c5e7660b3e731e30f7f8ec4318e94d0a5d674eb65a4/pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104", size = 7209321, upload-time = "2026-04-01T14:45:15.122Z" }, - { url = "https://files.pythonhosted.org/packages/6a/7a/c253e3c645cd47f1aceea6a8bacdba9991bf45bb7dfe927f7c893e89c93c/pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7", size = 6479723, upload-time = "2026-04-01T14:45:17.797Z" }, - { url = "https://files.pythonhosted.org/packages/cd/8b/601e6566b957ca50e28725cb6c355c59c2c8609751efbecd980db44e0349/pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150", size = 7217400, upload-time = "2026-04-01T14:45:20.529Z" }, - { url = "https://files.pythonhosted.org/packages/d6/94/220e46c73065c3e2951bb91c11a1fb636c8c9ad427ac3ce7d7f3359b9b2f/pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1", size = 2554835, upload-time = "2026-04-01T14:45:23.162Z" }, - { url = "https://files.pythonhosted.org/packages/b6/ab/1b426a3974cb0e7da5c29ccff4807871d48110933a57207b5a676cccc155/pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463", size = 5314225, upload-time = "2026-04-01T14:45:25.637Z" }, - { url = "https://files.pythonhosted.org/packages/19/1e/dce46f371be2438eecfee2a1960ee2a243bbe5e961890146d2dee1ff0f12/pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3", size = 4698541, upload-time = "2026-04-01T14:45:28.355Z" }, - { url = "https://files.pythonhosted.org/packages/55/c3/7fbecf70adb3a0c33b77a300dc52e424dc22ad8cdc06557a2e49523b703d/pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166", size = 6322251, upload-time = "2026-04-01T14:45:30.924Z" }, - { url = "https://files.pythonhosted.org/packages/1c/3c/7fbc17cfb7e4fe0ef1642e0abc17fc6c94c9f7a16be41498e12e2ba60408/pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe", size = 8127807, upload-time = "2026-04-01T14:45:33.908Z" }, - { url = "https://files.pythonhosted.org/packages/ff/c3/a8ae14d6defd2e448493ff512fae903b1e9bd40b72efb6ec55ce0048c8ce/pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd", size = 6433935, upload-time = "2026-04-01T14:45:36.623Z" }, - { url = "https://files.pythonhosted.org/packages/6e/32/2880fb3a074847ac159d8f902cb43278a61e85f681661e7419e6596803ed/pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e", size = 7116720, upload-time = "2026-04-01T14:45:39.258Z" }, - { url = "https://files.pythonhosted.org/packages/46/87/495cc9c30e0129501643f24d320076f4cc54f718341df18cc70ec94c44e1/pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06", size = 6540498, upload-time = "2026-04-01T14:45:41.879Z" }, - { url = "https://files.pythonhosted.org/packages/18/53/773f5edca692009d883a72211b60fdaf8871cbef075eaa9d577f0a2f989e/pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43", size = 7239413, upload-time = "2026-04-01T14:45:44.705Z" }, - { url = "https://files.pythonhosted.org/packages/c9/e4/4b64a97d71b2a83158134abbb2f5bd3f8a2ea691361282f010998f339ec7/pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354", size = 6482084, upload-time = "2026-04-01T14:45:47.568Z" }, - { url = "https://files.pythonhosted.org/packages/ba/13/306d275efd3a3453f72114b7431c877d10b1154014c1ebbedd067770d629/pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1", size = 7225152, upload-time = "2026-04-01T14:45:50.032Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579, upload-time = "2026-04-01T14:45:52.529Z" }, +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, + { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, + { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, + { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, + { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, + { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, + { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, + { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, + { url = "https://files.pythonhosted.org/packages/42/92/2fc3ffad878ae8dd5469ec1bc8eb83b71f48e13efdf68f02709003982a32/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66", size = 5345889, upload-time = "2026-07-01T11:54:31.97Z" }, + { url = "https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35", size = 4780109, upload-time = "2026-07-01T11:54:34.026Z" }, + { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, + { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, + { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, + { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, + { url = "https://files.pythonhosted.org/packages/c9/f9/066794cca041b969964f779ee5fa66a9498bbf34248ac39c5d7954e4198f/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f", size = 6473287, upload-time = "2026-07-01T11:54:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", size = 7239691, upload-time = "2026-07-01T11:54:47.141Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b0/c4ed4f0ef8f8fa5ee8351537db6650bb8189f7e118842978dd6589065692/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b", size = 2568185, upload-time = "2026-07-01T11:54:49.137Z" }, + { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, + { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, + { url = "https://files.pythonhosted.org/packages/85/e2/73c77d218410b14f5f2d565e8a998d5317b7b9c75368d29985139f7a46f0/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8", size = 5350344, upload-time = "2026-07-01T11:54:57.657Z" }, + { url = "https://files.pythonhosted.org/packages/c7/da/32c752228ae345f489e3a42499d817b6c3996da7e8a3bc7a04fc806b243b/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0", size = 4780131, upload-time = "2026-07-01T11:54:59.713Z" }, + { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, + { url = "https://files.pythonhosted.org/packages/a3/34/77f3f793fed8efc7d243f21b33c5a3f0d1c97ee70346d3db855587e155ff/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a", size = 6467209, upload-time = "2026-07-01T11:55:10.408Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d", size = 7237707, upload-time = "2026-07-01T11:55:12.745Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ac/6b11f2875f1c2ac040d84e1bbf9cf22a88038f901ca1037898b280b38365/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838", size = 2565995, upload-time = "2026-07-01T11:55:14.736Z" }, + { url = "https://files.pythonhosted.org/packages/52/69/c2208e56af9bfc1913afb24020297a691eb1d4ef688474c8a04913f65e04/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e", size = 5352503, upload-time = "2026-07-01T11:55:17.076Z" }, + { url = "https://files.pythonhosted.org/packages/07/70/e5686d753e898a45d778ff1718dba8516ead6ab6b95d85fc8c4b70650cf2/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17", size = 4782956, upload-time = "2026-07-01T11:55:19.448Z" }, + { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, + { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, + { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, + { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" }, + { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, + { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, + { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" }, + { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" }, + { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, + { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" }, + { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" }, + { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" }, + { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, + { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, + { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, + { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" }, + { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" }, + { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" }, ] [[package]]