diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__data.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__data.py.md index 5e88ec07..a76d312f 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__data.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__data.py.md @@ -330,6 +330,12 @@ FRED_PREFETCH_REGISTRY: dict[str, tuple[str, str, str]] = { ), "UNRATE": ("Unemployment Rate", "Percent", "MS"), "DCOILWTICO": ("Crude Oil Prices: West Texas Intermediate (WTI)", "Dollars per Barrel", "D"), + # NOTE: both London gold fixing series were discontinued by FRED and no + # longer resolve (HTTP 400 "series does not exist"); there is no equivalent + # daily USD gold price on FRED. The gold covariate therefore degrades to + # absent at runtime (see the first-available fallback below and + # ``strict_covariates=False``). ``scripts/fetch_fred.py`` skips them via + # ``KNOWN_UNAVAILABLE_FRED_IDS`` so a clean run reports no spurious failure. "GOLDAMGBD228NLBM": ( "Gold Fixing Price 10:30 A.M. (London time) in London Bullion Market", "U.S. Dollars per Troy Ounce", diff --git a/implementations/getting_started/concierge_agent/context/artifacts/scripts__fetch_fred.py.md b/implementations/getting_started/concierge_agent/context/artifacts/scripts__fetch_fred.py.md index 5573a248..1cdc1c9e 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/scripts__fetch_fred.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/scripts__fetch_fred.py.md @@ -3,13 +3,23 @@ kind: python ```python -"""Populate the local FRED cache with series used by the CFPR experiment. +"""Populate the local FRED cache with series used by the food-price and S&P 500 experiments. Each FRED series in ``FRED_SERIES`` below is fetched from the FRED REST API and written to ``data/fred/{fred_id}.parquet``. Subsequent calls to :class:`~aieng.forecasting.data.adapters.FREDAdapter` read directly from those parquet files — no further network access is required. +The catalogue is the union of two experiments' covariates: + +- **Food-price forecasting** (:data:`FOOD_FRED_SERIES`): monthly US food CPI + sub-indices plus Canadian macro series, consumed directly at monthly (MS) + frequency. +- **S&P 500 forecasting** (:data:`FRED_PREFETCH_REGISTRY`, imported from + ``sp500_forecasting.data``): daily and monthly US macro series that the S&P + 500 covariate builders transform and align themselves. This script only warms + the raw parquet cache; ``fetch_sp500_market.py`` handles the Yahoo covariates. + Re-running the script is idempotent: any series already cached is re-read from disk and re-validated. Pass ``--refresh`` to force a fresh download. @@ -34,6 +44,7 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "implementations")) from dotenv import load_dotenv @@ -42,15 +53,23 @@ load_dotenv(REPO_ROOT / ".env", override=False) from aieng.forecasting.data import DataService, SeriesMetadata from aieng.forecasting.data.adapters import FREDAdapter +from sp500_forecasting.data import FRED_PREFETCH_REGISTRY DEFAULT_CACHE_DIR = REPO_ROOT / "data" / "fred" # --------------------------------------------------------------------------- -# FRED series catalogue for food price forecasting +# FRED series catalogue # -# Each entry: (series_id, fred_series_id, description, units) +# Each entry: (series_id, fred_series_id, description, units, frequency) +# +# The cache is keyed by ``fred_series_id`` (the parquet filename); ``series_id`` +# and ``frequency`` are carried through to :class:`SeriesMetadata` for the +# summary printout and downstream registration. +# --------------------------------------------------------------------------- + +# Food-price forecasting covariates. # # Rationale for inclusion: # - US food CPI sub-indices: US prices transmit to Canadian food costs @@ -61,53 +80,86 @@ DEFAULT_CACHE_DIR = REPO_ROOT / "data" / "fred" # - Canada unemployment rate: labour-market covariate for the BoC # rate-decision experiment (implementations/boc_rate_decisions/). # -# All series below are published at monthly (MS) frequency on FRED, which -# matches the Statistics Canada food CPI target frequency. Daily series -# (e.g. VXO, VIXCLS) are intentionally excluded here — the ``FREDAdapter`` -# does not resample, so mixing frequencies silently breaks the covariate -# alignment inside Darts models. -# --------------------------------------------------------------------------- +# All food series are published at monthly (MS) frequency on FRED, which +# matches the Statistics Canada food CPI target frequency. -FRED_SERIES: list[tuple[str, str, str, str]] = [ +FOOD_FRED_SERIES: list[tuple[str, str, str, str, str]] = [ ( "fred_us_cpi_food_at_home", "CPIFABSL", "US CPI: Food at Home, All Urban Consumers (1982-84=100)", "Index 1982-84=100", + "MS", ), ( "fred_us_cpi_meats_poultry_fish_eggs", "CUSR0000SAF112", "US CPI: Meats, Poultry, Fish, and Eggs, All Urban Consumers (1982-84=100)", "Index 1982-84=100", + "MS", ), ( "fred_us_cpi_fruits_vegetables", "CUSR0000SAF113", "US CPI: Fruits and Vegetables, All Urban Consumers (1982-84=100)", "Index 1982-84=100", + "MS", ), ( "fred_canada_10yr_bond_yield", "IRLTLT01CAM156N", "Canada Long-Term Government Bond Yields: 10-Year (% per annum)", "Percent per annum", + "MS", ), ( "fred_canada_us_exchange_rate", "EXCAUS", "Canada / US Foreign Exchange Rate (CAD per 1 USD, monthly average)", "CAD per USD", + "MS", ), ( "fred_canada_unemployment_rate", "LRUNTTTTCAM156S", "Unemployment Rate: Total, All Persons for Canada (seasonally adjusted, monthly)", "Percent", + "MS", ), ] +def _sp500_fred_series() -> list[tuple[str, str, str, str, str]]: + """Derive fetch entries from the S&P 500 implementation's prefetch registry. + + ``FRED_PREFETCH_REGISTRY`` maps each raw FRED id to its + ``(description, units, frequency)``. The S&P 500 covariate builders read + these parquet caches by FRED id, so warming them here is all that is + required — the ``series_id`` is synthesised only for the summary printout. + """ + return [ + (f"fred_{fred_id.lower()}", fred_id, description, units, frequency) + for fred_id, (description, units, frequency) in FRED_PREFETCH_REGISTRY.items() + ] + + +# Union of both experiments' covariates. FRED ids are unique across the two +# sets, so no de-duplication is needed. +FRED_SERIES: list[tuple[str, str, str, str, str]] = FOOD_FRED_SERIES + _sp500_fred_series() + + +# FRED ids that are permanently unavailable upstream, mapped to a short reason. +# These are skipped (not fetched, not counted as failures) so a clean run does +# not report a spurious ``[failed]``. The S&P 500 gold covariate builder tries +# both London fixing series and degrades gracefully when neither resolves +# (``strict_covariates=False``), so the covariate is simply absent — see +# ``FRED_PREFETCH_REGISTRY`` in ``sp500_forecasting/data.py``. +KNOWN_UNAVAILABLE_FRED_IDS: dict[str, str] = { + "GOLDAMGBD228NLBM": "London AM gold fix discontinued by FRED (no daily USD replacement)", + "GOLDPMGBD228NLBM": "London PM gold fix discontinued by FRED (no daily USD replacement)", +} + + def build_data_service(cache_dir: Path, refresh: bool) -> DataService: """Fetch/validate every catalogued FRED series and register it in a DataService. @@ -130,15 +182,22 @@ def build_data_service(cache_dir: Path, refresh: bool) -> DataService: succeeded = 0 failed = 0 + skipped = 0 + + for series_id, fred_id, description, units, frequency in FRED_SERIES: + reason = KNOWN_UNAVAILABLE_FRED_IDS.get(fred_id) + if reason is not None: + skipped += 1 + print(f" [ skip] {series_id:<42} ({fred_id}): {reason}") + continue - for series_id, fred_id, description, units in FRED_SERIES: adapter = FREDAdapter(fred_id, cache_dir=cache_dir, refresh=refresh) metadata = SeriesMetadata( series_id=series_id, description=description, source=f"FRED ({fred_id})", units=units, - frequency="MS", + frequency=frequency, ) try: svc.register(series_id, adapter, metadata) @@ -151,7 +210,7 @@ def build_data_service(cache_dir: Path, refresh: bool) -> DataService: print(f" [ failed] {series_id:<42} ({fred_id}): {exc}") print() - print(f"Registered {succeeded} series ({failed} failed).") + print(f"Registered {succeeded} series ({failed} failed, {skipped} skipped).") return svc diff --git a/implementations/sp500_forecasting/data.py b/implementations/sp500_forecasting/data.py index c187ec1c..03c67bb2 100644 --- a/implementations/sp500_forecasting/data.py +++ b/implementations/sp500_forecasting/data.py @@ -325,6 +325,12 @@ def _default_cache_dir() -> Path: ), "UNRATE": ("Unemployment Rate", "Percent", "MS"), "DCOILWTICO": ("Crude Oil Prices: West Texas Intermediate (WTI)", "Dollars per Barrel", "D"), + # NOTE: both London gold fixing series were discontinued by FRED and no + # longer resolve (HTTP 400 "series does not exist"); there is no equivalent + # daily USD gold price on FRED. The gold covariate therefore degrades to + # absent at runtime (see the first-available fallback below and + # ``strict_covariates=False``). ``scripts/fetch_fred.py`` skips them via + # ``KNOWN_UNAVAILABLE_FRED_IDS`` so a clean run reports no spurious failure. "GOLDAMGBD228NLBM": ( "Gold Fixing Price 10:30 A.M. (London time) in London Bullion Market", "U.S. Dollars per Troy Ounce", diff --git a/scripts/fetch_fred.py b/scripts/fetch_fred.py index f1121a42..fd89749f 100644 --- a/scripts/fetch_fred.py +++ b/scripts/fetch_fred.py @@ -1,10 +1,20 @@ -"""Populate the local FRED cache with series used by the CFPR experiment. +"""Populate the local FRED cache with series used by the food-price and S&P 500 experiments. Each FRED series in ``FRED_SERIES`` below is fetched from the FRED REST API and written to ``data/fred/{fred_id}.parquet``. Subsequent calls to :class:`~aieng.forecasting.data.adapters.FREDAdapter` read directly from those parquet files — no further network access is required. +The catalogue is the union of two experiments' covariates: + +- **Food-price forecasting** (:data:`FOOD_FRED_SERIES`): monthly US food CPI + sub-indices plus Canadian macro series, consumed directly at monthly (MS) + frequency. +- **S&P 500 forecasting** (:data:`FRED_PREFETCH_REGISTRY`, imported from + ``sp500_forecasting.data``): daily and monthly US macro series that the S&P + 500 covariate builders transform and align themselves. This script only warms + the raw parquet cache; ``fetch_sp500_market.py`` handles the Yahoo covariates. + Re-running the script is idempotent: any series already cached is re-read from disk and re-validated. Pass ``--refresh`` to force a fresh download. @@ -29,6 +39,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "implementations")) from dotenv import load_dotenv @@ -37,15 +48,23 @@ from aieng.forecasting.data import DataService, SeriesMetadata from aieng.forecasting.data.adapters import FREDAdapter +from sp500_forecasting.data import FRED_PREFETCH_REGISTRY DEFAULT_CACHE_DIR = REPO_ROOT / "data" / "fred" # --------------------------------------------------------------------------- -# FRED series catalogue for food price forecasting +# FRED series catalogue # -# Each entry: (series_id, fred_series_id, description, units) +# Each entry: (series_id, fred_series_id, description, units, frequency) +# +# The cache is keyed by ``fred_series_id`` (the parquet filename); ``series_id`` +# and ``frequency`` are carried through to :class:`SeriesMetadata` for the +# summary printout and downstream registration. +# --------------------------------------------------------------------------- + +# Food-price forecasting covariates. # # Rationale for inclusion: # - US food CPI sub-indices: US prices transmit to Canadian food costs @@ -56,53 +75,86 @@ # - Canada unemployment rate: labour-market covariate for the BoC # rate-decision experiment (implementations/boc_rate_decisions/). # -# All series below are published at monthly (MS) frequency on FRED, which -# matches the Statistics Canada food CPI target frequency. Daily series -# (e.g. VXO, VIXCLS) are intentionally excluded here — the ``FREDAdapter`` -# does not resample, so mixing frequencies silently breaks the covariate -# alignment inside Darts models. -# --------------------------------------------------------------------------- +# All food series are published at monthly (MS) frequency on FRED, which +# matches the Statistics Canada food CPI target frequency. -FRED_SERIES: list[tuple[str, str, str, str]] = [ +FOOD_FRED_SERIES: list[tuple[str, str, str, str, str]] = [ ( "fred_us_cpi_food_at_home", "CPIFABSL", "US CPI: Food at Home, All Urban Consumers (1982-84=100)", "Index 1982-84=100", + "MS", ), ( "fred_us_cpi_meats_poultry_fish_eggs", "CUSR0000SAF112", "US CPI: Meats, Poultry, Fish, and Eggs, All Urban Consumers (1982-84=100)", "Index 1982-84=100", + "MS", ), ( "fred_us_cpi_fruits_vegetables", "CUSR0000SAF113", "US CPI: Fruits and Vegetables, All Urban Consumers (1982-84=100)", "Index 1982-84=100", + "MS", ), ( "fred_canada_10yr_bond_yield", "IRLTLT01CAM156N", "Canada Long-Term Government Bond Yields: 10-Year (% per annum)", "Percent per annum", + "MS", ), ( "fred_canada_us_exchange_rate", "EXCAUS", "Canada / US Foreign Exchange Rate (CAD per 1 USD, monthly average)", "CAD per USD", + "MS", ), ( "fred_canada_unemployment_rate", "LRUNTTTTCAM156S", "Unemployment Rate: Total, All Persons for Canada (seasonally adjusted, monthly)", "Percent", + "MS", ), ] +def _sp500_fred_series() -> list[tuple[str, str, str, str, str]]: + """Derive fetch entries from the S&P 500 implementation's prefetch registry. + + ``FRED_PREFETCH_REGISTRY`` maps each raw FRED id to its + ``(description, units, frequency)``. The S&P 500 covariate builders read + these parquet caches by FRED id, so warming them here is all that is + required — the ``series_id`` is synthesised only for the summary printout. + """ + return [ + (f"fred_{fred_id.lower()}", fred_id, description, units, frequency) + for fred_id, (description, units, frequency) in FRED_PREFETCH_REGISTRY.items() + ] + + +# Union of both experiments' covariates. FRED ids are unique across the two +# sets, so no de-duplication is needed. +FRED_SERIES: list[tuple[str, str, str, str, str]] = FOOD_FRED_SERIES + _sp500_fred_series() + + +# FRED ids that are permanently unavailable upstream, mapped to a short reason. +# These are skipped (not fetched, not counted as failures) so a clean run does +# not report a spurious ``[failed]``. The S&P 500 gold covariate builder tries +# both London fixing series and degrades gracefully when neither resolves +# (``strict_covariates=False``), so the covariate is simply absent — see +# ``FRED_PREFETCH_REGISTRY`` in ``sp500_forecasting/data.py``. +KNOWN_UNAVAILABLE_FRED_IDS: dict[str, str] = { + "GOLDAMGBD228NLBM": "London AM gold fix discontinued by FRED (no daily USD replacement)", + "GOLDPMGBD228NLBM": "London PM gold fix discontinued by FRED (no daily USD replacement)", +} + + def build_data_service(cache_dir: Path, refresh: bool) -> DataService: """Fetch/validate every catalogued FRED series and register it in a DataService. @@ -125,15 +177,22 @@ def build_data_service(cache_dir: Path, refresh: bool) -> DataService: succeeded = 0 failed = 0 + skipped = 0 + + for series_id, fred_id, description, units, frequency in FRED_SERIES: + reason = KNOWN_UNAVAILABLE_FRED_IDS.get(fred_id) + if reason is not None: + skipped += 1 + print(f" [ skip] {series_id:<42} ({fred_id}): {reason}") + continue - for series_id, fred_id, description, units in FRED_SERIES: adapter = FREDAdapter(fred_id, cache_dir=cache_dir, refresh=refresh) metadata = SeriesMetadata( series_id=series_id, description=description, source=f"FRED ({fred_id})", units=units, - frequency="MS", + frequency=frequency, ) try: svc.register(series_id, adapter, metadata) @@ -146,7 +205,7 @@ def build_data_service(cache_dir: Path, refresh: bool) -> DataService: print(f" [ failed] {series_id:<42} ({fred_id}): {exc}") print() - print(f"Registered {succeeded} series ({failed} failed).") + print(f"Registered {succeeded} series ({failed} failed, {skipped} skipped).") return svc