diff --git a/apps/predbat/config.py b/apps/predbat/config.py index b633dd8b8..d1806b138 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -842,6 +842,18 @@ "type": "switch", "default": True, }, + { + "name": "rate_retention_days", + "friendly_name": "Rate Retention Days", + "type": "input_number", + "min": 1, + "max": 365, + "step": 1, + "unit": "days", + "icon": "mdi:database-clock", + "enable": "expert_mode", + "default": 7, + }, { "name": "set_charge_freeze", "friendly_name": "Set Charge Freeze", @@ -1453,6 +1465,7 @@ "days_previous": True, "days_previous_weight": True, "battery_scaling": True, + "rate_retention_days": True, "forecast_hours": True, "import_export_scaling": True, "inverter_limit_charge": True, @@ -2106,6 +2119,7 @@ "rates_export_override": {"type": "dict_list"}, "days_previous": {"type": "integer_list"}, "days_previous_weight": {"type": "float_list"}, + "rate_retention_days": {"type": "integer"}, "forecast_hours": {"type": "integer"}, "notify_devices": {"type": "string_list"}, "battery_scaling": {"type": "sensor_list", "sensor_type": "float", "entries": "num_inverters", "modify": False}, diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index 2d6beab75..d925f5e77 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -907,10 +907,30 @@ def fetch_sensor_data(self, save=True): futurerate = FutureRate(self) self.future_energy_rates_import, self.future_energy_rates_export = futurerate.futurerate_analysis(self.rate_import, self.rate_export) + # Load stored rates for past minutes (frozen historical rates) + if self.rate_store: + today = datetime.now() + stored_import, stored_export = self.rate_store.load_rates(today) + + if stored_import or stored_export: + # Merge frozen past rates into current rate tables + for minute in range(0, self.minutes_now): + if stored_import and minute in stored_import: + self.rate_import[minute] = stored_import[minute] + if stored_export and minute in stored_export: + self.rate_export[minute] = stored_export[minute] + + self.log( + "Loaded {} frozen import rates and {} frozen export rates from storage".format( + len([m for m in stored_import.keys() if m < self.minutes_now]) if stored_import else 0, len([m for m in stored_export.keys() if m < self.minutes_now]) if stored_export else 0 + ) + ) + # Replicate and scan import rates if self.rate_import: self.rate_scan(self.rate_import, print=False) self.rate_import, self.rate_import_replicated = self.rate_replicate(self.rate_import, self.io_adjusted, is_import=True) + self.rate_import_no_io = self.rate_import.copy() for car_n in range(self.num_cars): self.rate_import = self.rate_add_io_slots(car_n, self.rate_import, self.octopus_slots[car_n]) @@ -929,6 +949,7 @@ def fetch_sensor_data(self, save=True): if self.rate_export: self.rate_scan_export(self.rate_export, print=False) self.rate_export, self.rate_export_replicated = self.rate_replicate(self.rate_export, is_import=False) + # For export tariff only load the saving session if enabled if self.rate_export_max > 0: self.load_saving_slot(self.octopus_saving_slots, export=True, rate_replicate=self.rate_export_replicated) @@ -944,6 +965,11 @@ def fetch_sensor_data(self, save=True): if self.rate_import or self.rate_export: self.set_rate_thresholds() + # Save final computed rate tables to persistent storage (with frozen past slots) + if self.rate_store: + today = datetime.now() + self.rate_store.save_rates(today, self.rate_import, self.rate_export, self.minutes_now) + # Find discharging windows if self.rate_export: self.high_export_rates, lowest, highest = self.rate_scan_window(self.rate_export, 5, self.rate_export_cost_threshold, True, alt_rates=self.rate_import) @@ -1497,6 +1523,14 @@ def apply_manual_rates(self, rates, manual_items, is_import=True, rate_replicate rates[minute] = rate rate_replicate[minute] = "manual" + # Track manual override in rate store + if self.rate_store: + today = datetime.now() + if is_import: + self.rate_store.update_manual_override(today, minute, rate, None) + else: + self.rate_store.update_manual_override(today, minute, None, rate) + return rates def basic_rates(self, info, rtype, prev=None, rate_replicate=None): diff --git a/apps/predbat/persistent_store.py b/apps/predbat/persistent_store.py new file mode 100644 index 000000000..689949102 --- /dev/null +++ b/apps/predbat/persistent_store.py @@ -0,0 +1,189 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System +# Copyright Trefor Southwell 2026 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +# fmt: off +# pylint: disable=consider-using-f-string +# pylint: disable=line-too-long +# pylint: disable=attribute-defined-outside-init + +""" +Base class for persistent JSON file storage with backup and cleanup. +Provides common functionality for components needing to store state across restarts. +""" + +import json +import os +from datetime import datetime, timedelta +from pathlib import Path + + +class PersistentStore: + """ + Abstract base class for persistent JSON file storage. + Handles load/save with backup, cleanup of old files, and automatic timestamping. + """ + + def __init__(self, base): + """Initialize with reference to base PredBat instance""" + self.base = base + self.log = base.log + + def load(self, filepath): + """ + Load data from JSON file with automatic backup restoration on corruption. + + Args: + filepath: Path to JSON file to load + + Returns: + Loaded data dict or None if file doesn't exist or is corrupted + """ + try: + if not os.path.exists(filepath): + return None + + with open(filepath, 'r') as f: + data = json.load(f) + return data + + except (json.JSONDecodeError, IOError) as e: + self.log(f"Warn: Failed to load {filepath}: {e}") + + # Try to restore from backup + backup_path = filepath + '.bak' + if os.path.exists(backup_path): + try: + self.log(f"Warn: Attempting to restore from backup: {backup_path}") + with open(backup_path, 'r') as f: + data = json.load(f) + self.log(f"Warn: Successfully restored from backup") + return data + except (json.JSONDecodeError, IOError) as e2: + self.log(f"Error: Backup restoration failed: {e2}") + + return None + + def save(self, filepath, data, backup=True): + """ + Save data to JSON file with automatic backup and timestamp. + + Args: + filepath: Path to JSON file to save + data: Dict to save (will add last_updated timestamp) + backup: Whether to backup existing file before overwrite + + Returns: + True if successful, False otherwise + """ + try: + # Add timestamp + data['last_updated'] = datetime.now().astimezone().isoformat() + + # Create directory if needed + os.makedirs(os.path.dirname(filepath), exist_ok=True) + + # Backup existing file if requested + if backup and os.path.exists(filepath): + self.backup_file(filepath) + + # Write new file + with open(filepath, 'w') as f: + json.dump(data, f, indent=2) + + # Cleanup old backups + self.cleanup_backups(filepath) + + return True + + except (IOError, OSError) as e: + self.log(f"Error: Failed to save {filepath}: {e}") + return False + + def backup_file(self, filepath): + """ + Create backup copy of file. + + Args: + filepath: Path to file to backup + """ + try: + backup_path = filepath + '.bak' + if os.path.exists(filepath): + import shutil + shutil.copy2(filepath, backup_path) + except (IOError, OSError) as e: + self.log(f"Warn: Failed to backup {filepath}: {e}") + + def cleanup_backups(self, filepath): + """ + Remove backup files older than 1 day. + + Args: + filepath: Path to main file (will check for .bak file) + """ + try: + backup_path = filepath + '.bak' + if os.path.exists(backup_path): + # Check file age + file_time = datetime.fromtimestamp(os.path.getmtime(backup_path)) + age = datetime.now() - file_time + + if age > timedelta(days=1): + os.remove(backup_path) + self.log(f"Info: Cleaned up old backup: {backup_path}") + + except (IOError, OSError) as e: + self.log(f"Warn: Failed to cleanup backup for {filepath}: {e}") + + def cleanup(self, directory, pattern, retention_days): + """ + Remove files matching pattern older than retention period. + + Args: + directory: Directory to search + pattern: Glob pattern for files to cleanup + retention_days: Number of days to retain files + + Returns: + Number of files removed + """ + try: + if not os.path.exists(directory): + return 0 + + path = Path(directory) + cutoff_time = datetime.now() - timedelta(days=retention_days) + removed_count = 0 + + for file_path in path.glob(pattern): + try: + file_time = datetime.fromtimestamp(file_path.stat().st_mtime) + if file_time < cutoff_time: + file_path.unlink() + removed_count += 1 + self.log(f"Info: Cleaned up old file: {file_path}") + except (IOError, OSError) as e: + self.log(f"Warn: Failed to remove {file_path}: {e}") + + return removed_count + + except Exception as e: + self.log(f"Error: Cleanup failed for {directory}/{pattern}: {e}") + return 0 + + def get_last_updated(self, filepath): + """ + Get last_updated timestamp from JSON file. + + Args: + filepath: Path to JSON file + + Returns: + ISO 8601 timestamp string or None + """ + data = self.load(filepath) + if data and 'last_updated' in data: + return data['last_updated'] + return None diff --git a/apps/predbat/predbat.py b/apps/predbat/predbat.py index 8e6927381..1949b0599 100644 --- a/apps/predbat/predbat.py +++ b/apps/predbat/predbat.py @@ -39,7 +39,7 @@ THIS_VERSION = "v8.34.6" # fmt: off -PREDBAT_FILES = ["predbat.py", "const.py", "hass.py", "config.py", "prediction.py", "gecloud.py", "utils.py", "inverter.py", "ha.py", "download.py", "web.py", "web_helper.py", "predheat.py", "futurerate.py", "octopus.py", "solcast.py", "execute.py", "plan.py", "fetch.py", "output.py", "userinterface.py", "energydataservice.py", "alertfeed.py", "compare.py", "db_manager.py", "db_engine.py", "plugin_system.py", "ohme.py", "components.py", "fox.py", "carbon.py", "temperature.py", "web_mcp.py", "component_base.py", "axle.py", "solax.py", "solis.py", "unit_test.py", "load_ml_component.py", "load_predictor.py", "oauth_mixin.py", "predbat_metrics.py", "web_metrics_dashboard.py"] +PREDBAT_FILES = ["predbat.py", "const.py", "hass.py", "config.py", "prediction.py", "gecloud.py", "utils.py", "inverter.py", "ha.py", "download.py", "web.py", "web_helper.py", "predheat.py", "futurerate.py", "octopus.py", "solcast.py", "execute.py", "plan.py", "fetch.py", "output.py", "userinterface.py", "energydataservice.py", "alertfeed.py", "compare.py", "db_manager.py", "db_engine.py", "plugin_system.py", "ohme.py", "components.py", "fox.py", "carbon.py", "temperature.py", "web_mcp.py", "component_base.py", "axle.py", "solax.py", "solis.py", "unit_test.py", "load_ml_component.py", "load_predictor.py", "oauth_mixin.py", "predbat_metrics.py", "web_metrics_dashboard.py", "persistent_store.py", "rate_store.py"] # fmt: on from download import predbat_update_move, predbat_update_download, check_install @@ -86,6 +86,7 @@ from userinterface import UserInterface from compare import Compare from plugin_system import PluginSystem +from rate_store import RateStore class PredBat(hass.Hass, Octopus, Energidataservice, Fetch, Plan, Execute, Output, UserInterface): @@ -488,6 +489,7 @@ def reset(self): self.rate_import_no_io = {} self.rate_export = {} self.rate_gas = {} + self.rate_store = None self.rate_slots = [] self.low_rates = [] self.high_export_rates = [] @@ -1559,6 +1561,8 @@ def initialize(self): self.validate_config() self.comparison = Compare(self) + self.rate_store = RateStore(self) + self.components.initialize(phase=1) if not self.components.start(phase=1): self.log("Error: Some components failed to start (phase1)") diff --git a/apps/predbat/rate_store.py b/apps/predbat/rate_store.py new file mode 100644 index 000000000..fc2029e56 --- /dev/null +++ b/apps/predbat/rate_store.py @@ -0,0 +1,157 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System +# Copyright Trefor Southwell 2026 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +# fmt: off +# pylint: disable=consider-using-f-string +# pylint: disable=line-too-long +# pylint: disable=attribute-defined-outside-init + +""" +Persistent storage for import and export rates. +Stores the final computed rate tables from fetch, with past slots frozen +to prevent retrospective changes to historical cost calculations. +""" + +import os +from datetime import datetime +from persistent_store import PersistentStore + + +class RateStore(PersistentStore): + """ + Manages persistent storage of energy rate tables. + + Persists the final rate_import and rate_export dictionaries from fetch. + Past slots (< minutes_now) are frozen once written and cannot be overwritten, + ensuring accurate historical cost calculations even if future rates change + or overrides disappear (IOG, Axle VPP). + + File structure: predbat_save/rates_YYYY_MM_DD.json + Format: {"rates_import": {minute: rate, ...}, "rates_export": {minute: rate, ...}} + """ + + def __init__(self, base, save_dir="predbat_save"): + """ + Initialize rate store. + + Args: + base: PredBat instance + save_dir: Directory for rate files (relative to workspace root) + """ + super().__init__(base) + self.save_dir = save_dir + + # Cleanup old rate files + retention_days = base.get_arg("rate_retention_days", 7) + removed = self.cleanup_old_files(retention_days) + if removed > 0: + self.log("Cleaned up {} old rate files".format(removed)) + + def _get_filepath(self, date): + """ + Get filepath for rate file for given date. + + Args: + date: datetime object + + Returns: + Full path string to rate JSON file + """ + date_str = date.strftime("%Y_%m_%d") + filename = f"rates_{date_str}.json" + return os.path.join(self.save_dir, filename) + + def save_rates(self, date, rate_import, rate_export, freeze_before_minute): + """ + Save rate tables, freezing past slots to prevent retrospective changes. + + Args: + date: datetime object for the date + rate_import: Dict of {minute: rate} for import rates + rate_export: Dict of {minute: rate} for export rates + freeze_before_minute: Minute offset - freeze all slots before this time + + Returns: + True if successful + """ + filepath = self._get_filepath(date) + + # Load existing data to preserve frozen slots + existing_data = self.load(filepath) + if existing_data is None: + existing_data = {"rates_import": {}, "rates_export": {}} + + # Convert existing string keys to int keys for comparison + existing_import = {int(k): v for k, v in existing_data.get("rates_import", {}).items()} + existing_export = {int(k): v for k, v in existing_data.get("rates_export", {}).items()} + + # Build new data structure + new_import = {} + new_export = {} + + # Combine all minutes from both existing and new data + all_minutes = set(existing_import.keys()) | set(rate_import.keys()) | set(existing_export.keys()) | set(rate_export.keys()) + + for minute in sorted(all_minutes): + # For past slots (frozen), use existing value if present, otherwise new value + if minute < freeze_before_minute: + if minute in existing_import: + new_import[str(minute)] = existing_import[minute] + elif minute in rate_import: + new_import[str(minute)] = rate_import[minute] + + if minute in existing_export: + new_export[str(minute)] = existing_export[minute] + elif minute in rate_export: + new_export[str(minute)] = rate_export[minute] + else: + # For current/future slots, always use new value + if minute in rate_import: + new_import[str(minute)] = rate_import[minute] + if minute in rate_export: + new_export[str(minute)] = rate_export[minute] + + data = { + "rates_import": new_import, + "rates_export": new_export, + "last_updated": datetime.now().isoformat(), + "frozen_before_minute": freeze_before_minute + } + + return self.save(filepath, data, backup=True) + + def load_rates(self, date): + """ + Load stored rate tables for given date. + + Args: + date: datetime object for date to load + + Returns: + Tuple of (rate_import_dict, rate_export_dict) or (None, None) if no file + """ + filepath = self._get_filepath(date) + data = self.load(filepath) + + if data is None: + return None, None + + # Convert string keys back to integers + rate_import = {int(k): v for k, v in data.get("rates_import", {}).items()} + rate_export = {int(k): v for k, v in data.get("rates_export", {}).items()} + + return rate_import, rate_export + + def cleanup_old_files(self, retention_days): + """ + Remove rate files older than retention period. + + Args: + retention_days: Number of days to retain files + + Returns: + Number of files removed + """ + return self.cleanup(self.save_dir, "rates_*.json", retention_days) diff --git a/apps/predbat/tests/test_axle.py b/apps/predbat/tests/test_axle.py index 75c6f276d..ddfb6e59c 100644 --- a/apps/predbat/tests/test_axle.py +++ b/apps/predbat/tests/test_axle.py @@ -912,6 +912,7 @@ def __init__(self): self.minutes_now = 10 * 60 # 10:00 AM self.forecast_minutes = 24 * 60 # 24 hours self.prefix = "predbat" + self.rate_store = None # No rate persistence in tests # Initialize rate_export with base rates for each minute self.rate_export = {} diff --git a/apps/predbat/tests/test_rate_store.py b/apps/predbat/tests/test_rate_store.py new file mode 100644 index 000000000..fa12d636b --- /dev/null +++ b/apps/predbat/tests/test_rate_store.py @@ -0,0 +1,270 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System +# Copyright Trefor Southwell 2026 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +# fmt: off +# pylint: disable=consider-using-f-string +# pylint: disable=line-too-long +# pylint: disable=attribute-defined-outside-init + +import os +import json +import shutil +from datetime import datetime, timedelta +from rate_store import RateStore + + +def run_rate_store_tests(my_predbat): + """ + Run comprehensive tests for rate persistence with freeze-past-slots logic + + Args: + my_predbat: PredBat instance (unused for these tests but required for consistency) + + Returns: + bool: False if all tests pass, True if any test fails + """ + failed = False + + # Create test directory + test_dir = "test_rate_store_temp" + if os.path.exists(test_dir): + shutil.rmtree(test_dir) + os.makedirs(test_dir) + + try: + print("*** Test 1: Basic save and load") + failed |= test_basic_save_load(os.path.join(test_dir, "test1")) + + print("*** Test 2: Frozen past slots") + failed |= test_frozen_past_slots(os.path.join(test_dir, "test2")) + + print("*** Test 3: Future slots update") + failed |= test_future_slots_update(os.path.join(test_dir, "test3")) + + print("*** Test 4: Cleanup old files") + failed |= test_cleanup(os.path.join(test_dir, "test4")) + + finally: + # Cleanup + if os.path.exists(test_dir): + shutil.rmtree(test_dir) + + return failed + + +def test_basic_save_load(test_dir): + """Test basic save and load of rate tables""" + + os.makedirs(test_dir, exist_ok=True) + + class MockBase: + def log(self, msg): + print(f" {msg}") + + def get_arg(self, key, default): + if key == "rate_retention_days": + return 7 + return default + + base = MockBase() + store = RateStore(base, save_dir=test_dir) + + today = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + + # Create rate tables + rate_import = {0: 10.0, 30: 15.0, 60: 20.0, 90: 25.0} + rate_export = {0: 5.0, 30: 7.5, 60: 10.0, 90: 12.5} + + # Save rates with freeze at minute 60 + success = store.save_rates(today, rate_import, rate_export, freeze_before_minute=60) + + if not success: + print(" ERROR: Failed to save rates") + return True + + # Load rates back + loaded_import, loaded_export = store.load_rates(today) + + if loaded_import is None or loaded_export is None: + print(" ERROR: Failed to load rates") + return True + + # Verify all rates loaded correctly + for minute in rate_import: + if minute not in loaded_import or abs(loaded_import[minute] - rate_import[minute]) > 0.01: + print(f" ERROR: Import rate mismatch at minute {minute}: expected {rate_import[minute]}, got {loaded_import.get(minute)}") + return True + + for minute in rate_export: + if minute not in loaded_export or abs(loaded_export[minute] - rate_export[minute]) > 0.01: + print(f" ERROR: Export rate mismatch at minute {minute}: expected {rate_export[minute]}, got {loaded_export.get(minute)}") + return True + + print(" PASS: Basic save and load working") + return False + + +def test_frozen_past_slots(test_dir): + """Test that past slots (< freeze_before_minute) are frozen""" + + os.makedirs(test_dir, exist_ok=True) + + class MockBase: + def log(self, msg): + print(f" {msg}") + + def get_arg(self, key, default): + if key == "rate_retention_days": + return 7 + return default + + base = MockBase() + store = RateStore(base, save_dir=test_dir) + + today = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + + # Initial save with rates for minute 0-90 + rate_import_v1 = {0: 10.0, 30: 15.0, 60: 20.0, 90: 25.0} + rate_export_v1 = {0: 5.0, 30: 7.5, 60: 10.0, 90: 12.5} + + store.save_rates(today, rate_import_v1, rate_export_v1, freeze_before_minute=60) + + # Second save with different rates - past slots (< 60) should be frozen + rate_import_v2 = {0: 99.0, 30: 99.0, 60: 30.0, 90: 35.0} # Changed all + rate_export_v2 = {0: 99.0, 30: 99.0, 60: 15.0, 90: 17.5} + + store.save_rates(today, rate_import_v2, rate_export_v2, freeze_before_minute=60) + + # Load and verify + loaded_import, loaded_export = store.load_rates(today) + + # Minutes 0 and 30 should still have original values (frozen) + if abs(loaded_import[0] - 10.0) > 0.01: + print(f" ERROR: Frozen import rate at minute 0 changed from 10.0 to {loaded_import[0]}") + return True + + if abs(loaded_import[30] - 15.0) > 0.01: + print(f" ERROR: Frozen import rate at minute 30 changed from 15.0 to {loaded_import[30]}") + return True + + # Minutes 60 and 90 should have new values (not frozen) + if abs(loaded_import[60] - 30.0) > 0.01: + print(f" ERROR: Future import rate at minute 60 not updated to 30.0, got {loaded_import[60]}") + return True + + if abs(loaded_import[90] - 35.0) > 0.01: + print(f" ERROR: Future import rate at minute 90 not updated to 35.0, got {loaded_import[90]}") + return True + + print(" PASS: Past slots correctly frozen") + return False + + +def test_future_slots_update(test_dir): + """Test that future slots (>= freeze_before_minute) always get new values""" + + os.makedirs(test_dir, exist_ok=True) + + class MockBase: + def log(self, msg): + print(f" {msg}") + + def get_arg(self, key, default): + if key == "rate_retention_days": + return 7 + return default + + base = MockBase() + store = RateStore(base, save_dir=test_dir) + + today = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + + # Save initial future rates + rate_import_v1 = {120: 20.0, 150: 25.0} + rate_export_v1 = {120: 10.0, 150: 12.5} + + store.save_rates(today, rate_import_v1, rate_export_v1, freeze_before_minute=90) + + # Update future rates multiple times + for version in range(2, 5): + rate_import_vN = {120: 20.0 + version * 10, 150: 25.0 + version * 10} + rate_export_vN = {120: 10.0 + version * 5, 150: 12.5 + version * 5} + store.save_rates(today, rate_import_vN, rate_export_vN, freeze_before_minute=90) + + # Load and verify we have the latest values + loaded_import, loaded_export = store.load_rates(today) + + expected_import_120 = 20.0 + 4 * 10 # 60.0 + expected_import_150 = 25.0 + 4 * 10 # 65.0 + + if abs(loaded_import[120] - expected_import_120) > 0.01: + print(f" ERROR: Future rate at 120 not updated to {expected_import_120}, got {loaded_import[120]}") + return True + + if abs(loaded_import[150] - expected_import_150) > 0.01: + print(f" ERROR: Future rate at 150 not updated to {expected_import_150}, got {loaded_import[150]}") + return True + + print(" PASS: Future slots always updated") + return False + + +def test_cleanup(test_dir): + """Test cleanup of old rate files""" + + os.makedirs(test_dir, exist_ok=True) + + class MockBase: + def log(self, msg): + print(f" {msg}") + + def get_arg(self, key, default): + if key == "rate_retention_days": + return 7 + return default + + base = MockBase() + store = RateStore(base, save_dir=test_dir) + + # Create rate files for last 10 days + today = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + + for days_ago in range(10): + old_date = today - timedelta(days=days_ago) + date_str = old_date.strftime("%Y_%m_%d") + file_path = os.path.join(test_dir, f"rates_{date_str}.json") + + # Write dummy data + with open(file_path, "w") as f: + json.dump({ + "rates_import": {"0": 10.0}, + "rates_export": {"0": 5.0}, + "last_updated": old_date.isoformat() + }, f) + + # Set file modification time to match the date + old_timestamp = (old_date - timedelta(hours=12)).timestamp() + os.utime(file_path, (old_timestamp, old_timestamp)) + + # Run cleanup with 7 days retention + retention_days = 7 + removed = store.cleanup_old_files(retention_days) + + # Check remaining files + remaining_files = [f for f in os.listdir(test_dir) if f.startswith("rates_") and f.endswith(".json") and not f.endswith(".bak")] + + # Should have at most 7 days + today = 8 files + if len(remaining_files) > 8: + print(f" ERROR: Expected <= 8 files after cleanup, found {len(remaining_files)}") + print(f" Files: {sorted(remaining_files)}") + return True + + # At least 2 files should be removed (days 8-9), possibly more depending on time of day + if removed < 2: + print(f" ERROR: Expected to remove at least 2 files, removed {removed}") + return True + + print(" PASS: Cleanup working correctly") + return False diff --git a/apps/predbat/unit_test.py b/apps/predbat/unit_test.py index 6109d8632..428a09e07 100644 --- a/apps/predbat/unit_test.py +++ b/apps/predbat/unit_test.py @@ -103,6 +103,7 @@ from tests.test_oauth_mixin import run_oauth_mixin_tests from tests.test_fox_oauth import run_fox_oauth_tests from tests.test_band_rate_text import test_band_rate_text +from tests.test_rate_store import run_rate_store_tests # Mock the components and plugin system @@ -266,6 +267,8 @@ def main(): ("optimise_levels", run_optimise_levels_tests, "Optimise levels tests", False), ("optimise_windows", run_optimise_all_windows_tests, "Optimise all windows tests", True), ("debug_cases", run_debug_cases, "Debug case file tests", True), + # Rate Store unit tests + ("rate_store", run_rate_store_tests, "Rate Store persistence and finalization tests (write, rehydrate, finalize, priority, cleanup)", False), ] # Parse command line arguments