From b6b26e2a67c703d495a03b6257eaee82f960520c Mon Sep 17 00:00:00 2001 From: Ben Milanko Date: Sat, 14 Nov 2026 07:25:09 +1100 Subject: [PATCH] feat(web): persist the cache in IndexedDB On web, files and cache info are now stored in IndexedDB instead of memory, so the cache survives reloads and browser restarts (#480, #339, #361, #122). This is the new web default and is documented in the README; MemoryCacheSystem with NonStoringObjectProvider keeps the old in-memory behavior. - One shared schema for the file store and the cache info repository, applied by a reference-counted connection pool, so close() really closes and blocked upgrades are logged. - Files are written in 512 KiB chunks as they download; openRead streams chunks, exists() uses count() and length() reads only the header. - When the quota is reached, least recently used entries are evicted together with their data chunks, and the failed write is retried once. - Transactions request relaxed durability. - CacheStore checks existence with await file.exists() on every platform and treats a file a concurrent removal already deleted as removed. - Browser tests: flutter test --platform chrome test/web (also --wasm). Co-Authored-By: Claude Opus 5.5 --- flutter_cache_manager/CHANGELOG.md | 6 + flutter_cache_manager/README.md | 6 + .../lib/src/cache_store.dart | 11 +- .../lib/src/config/_config_web.dart | 15 +- .../lib/src/config/config.dart | 8 +- .../cache_info_repositories.dart | 4 + .../indexed_db_cache_info_repository.dart | 545 ++++++++++++++ .../indexed_db_connection_pool.dart | 133 ++++ .../indexed_db_schema.dart | 239 ++++++ .../src/storage/file_system/file_system.dart | 2 + .../storage/file_system/indexed_db_file.dart | 687 ++++++++++++++++++ .../file_system/indexed_db_file_system.dart | 16 + .../indexed_db_file_system_stub.dart | 16 + flutter_cache_manager/pubspec.yaml | 3 +- ...indexed_db_cache_info_repository_test.dart | 152 ++++ .../test/web/indexed_db_file_test.dart | 180 +++++ .../test/web/indexed_db_test_helpers.dart | 59 ++ 17 files changed, 2068 insertions(+), 14 deletions(-) create mode 100644 flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_cache_info_repository.dart create mode 100644 flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_connection_pool.dart create mode 100644 flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_schema.dart create mode 100644 flutter_cache_manager/lib/src/storage/file_system/indexed_db_file.dart create mode 100644 flutter_cache_manager/lib/src/storage/file_system/indexed_db_file_system.dart create mode 100644 flutter_cache_manager/lib/src/storage/file_system/indexed_db_file_system_stub.dart create mode 100644 flutter_cache_manager/test/web/indexed_db_cache_info_repository_test.dart create mode 100644 flutter_cache_manager/test/web/indexed_db_file_test.dart create mode 100644 flutter_cache_manager/test/web/indexed_db_test_helpers.dart diff --git a/flutter_cache_manager/CHANGELOG.md b/flutter_cache_manager/CHANGELOG.md index f1d60241..54046f30 100644 --- a/flutter_cache_manager/CHANGELOG.md +++ b/flutter_cache_manager/CHANGELOG.md @@ -1,3 +1,9 @@ +## [3.6.0] - 2026-11-14 + +* Persists the cache on web in IndexedDB, so cached files survive page reloads and browser restarts ([#480](https://github.com/Baseflow/flutter_cache_manager/issues/480), [#339](https://github.com/Baseflow/flutter_cache_manager/issues/339), [#361](https://github.com/Baseflow/flutter_cache_manager/issues/361), [#122](https://github.com/Baseflow/flutter_cache_manager/issues/122)). This changes the web default from in-memory storage; pass `fileSystem: MemoryCacheSystem()` and `repo: NonStoringObjectProvider()` to `Config` to keep the previous behavior. +* Writes and reads web files in chunks, and when the browser's storage quota is reached evicts the least recently used files together with their data. +* Checks for a cached file with `await file.exists()` on every platform, and treats a file that a concurrent removal already deleted as removed. + ## [3.5.0] - 2026-11-04 * **DEPRECATION** Deprecates `supportedFileNames`: whether an image is resized is now decided by the image engine from the file's content instead of from the file extension derived from the `Content-Type` header of the response ([#475](https://github.com/Baseflow/flutter_cache_manager/pull/475)) diff --git a/flutter_cache_manager/README.md b/flutter_cache_manager/README.md index d0e2df3e..e2b2ac15 100644 --- a/flutter_cache_manager/README.md +++ b/flutter_cache_manager/README.md @@ -93,6 +93,12 @@ By default the cached files are stored in the temporary directory of the app. Th Information about the files is stored in a database using sqflite on Android, iOS and macOs, or in a plain JSON file on other platforms. The file name of the database is the key of the cacheManager, that's why that has to be unique. +On web the files and their information are stored in **IndexedDB**, so the cache persists across page reloads and +browser restarts. Each cache manager gets its own database named `flutter_cache_manager_`. Files are written in +chunks as they download, and when the browser's storage quota is reached the least recently used files are evicted. +Browsers can still clear this storage, for example when the user clears site data. To keep the old in-memory +behaviour, pass `fileSystem: MemoryCacheSystem()` and `repo: NonStoringObjectProvider()` to `Config`. + ### When are the cached files updated? A valid url response should contain a Cache-Control header. More info on the header can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control), but in summary it says for how long diff --git a/flutter_cache_manager/lib/src/cache_store.dart b/flutter_cache_manager/lib/src/cache_store.dart index 67fad4c8..397e7884 100644 --- a/flutter_cache_manager/lib/src/cache_store.dart +++ b/flutter_cache_manager/lib/src/cache_store.dart @@ -309,12 +309,15 @@ class CacheStore { _futureCache.remove(cacheObject.key); final file = await fileSystem.createFile(cacheObject.relativePath); - if (file.existsSync()) { + if (await file.exists()) { try { await file.delete(); - // ignore: unused_catch_clause - } on PathNotFoundException catch (e) { - // File has already been deleted. Do nothing #184 + } on FileSystemException { + // A concurrent removal can delete the file between the exists() check + // and delete(), since both are awaited. Already gone is the outcome we + // wanted (#184); anything else is a real failure. Not every backend + // throws PathNotFoundException for this (MemoryFileSystem does not). + if (await file.exists()) rethrow; } } removed.add(id); diff --git a/flutter_cache_manager/lib/src/config/_config_web.dart b/flutter_cache_manager/lib/src/config/_config_web.dart index 9a019781..1aad10b7 100644 --- a/flutter_cache_manager/lib/src/config/_config_web.dart +++ b/flutter_cache_manager/lib/src/config/_config_web.dart @@ -1,8 +1,6 @@ +import 'package:flutter_cache_manager/flutter_cache_manager.dart'; import 'package:flutter_cache_manager/src/config/config.dart' as def; -import 'package:flutter_cache_manager/src/storage/cache_info_repositories/cache_info_repository.dart'; -import 'package:flutter_cache_manager/src/storage/cache_info_repositories/non_storing_object_provider.dart'; -import 'package:flutter_cache_manager/src/storage/file_system/file_system.dart'; -import 'package:flutter_cache_manager/src/web/file_service.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_cache_info_repository.dart'; class Config implements def.Config { Config( @@ -14,8 +12,13 @@ class Config implements def.Config { FileService? fileService, }) : stalePeriod = stalePeriod ?? const Duration(days: 30), maxNrOfCacheObjects = maxNrOfCacheObjects ?? 200, - repo = repo ?? NonStoringObjectProvider(), - fileSystem = fileSystem ?? MemoryCacheSystem(), + repo = + repo ?? + IndexedDbCacheInfoRepository( + databaseName: 'flutter_cache_manager_$cacheKey', + ), + fileSystem = + fileSystem ?? IndexedDbFileSystem('flutter_cache_manager_$cacheKey'), fileService = fileService ?? HttpFileService(); @override diff --git a/flutter_cache_manager/lib/src/config/config.dart b/flutter_cache_manager/lib/src/config/config.dart index 88dabb37..4af6f565 100644 --- a/flutter_cache_manager/lib/src/config/config.dart +++ b/flutter_cache_manager/lib/src/config/config.dart @@ -16,9 +16,11 @@ abstract class Config { /// time will be removed. /// [repo] is the [CacheInfoRepository] which stores the cache metadata. On /// Android, iOS and macOS this defaults to [CacheObjectProvider], a - /// sqflite implementation due to legacy. On web this defaults to - /// [NonStoringObjectProvider]. On the other platforms this defaults to - /// [JsonCacheInfoRepository]. + /// sqflite implementation due to legacy. On web this defaults to an + /// IndexedDB repository, and [fileSystem] to IndexedDB storage, so the + /// cache persists across reloads; pass [NonStoringObjectProvider] and + /// [MemoryCacheSystem] for the previous in-memory behaviour. On the other + /// platforms this defaults to [JsonCacheInfoRepository]. /// The [fileSystem] defines where the cached files are stored and the /// [fileService] defines where files are fetched, for example online. factory Config( diff --git a/flutter_cache_manager/lib/src/storage/cache_info_repositories/cache_info_repositories.dart b/flutter_cache_manager/lib/src/storage/cache_info_repositories/cache_info_repositories.dart index 14ab5ab6..b2e77fb4 100644 --- a/flutter_cache_manager/lib/src/storage/cache_info_repositories/cache_info_repositories.dart +++ b/flutter_cache_manager/lib/src/storage/cache_info_repositories/cache_info_repositories.dart @@ -2,3 +2,7 @@ export 'cache_info_repository.dart'; export 'cache_object_provider.dart'; export 'json_cache_info_repository.dart'; export 'non_storing_object_provider.dart'; + +// Note: indexed_db_cache_info_repository.dart is web-only and not exported here +// to avoid dart:js_interop import errors on non-web platforms. +// Web code should import it directly when needed. diff --git a/flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_cache_info_repository.dart b/flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_cache_info_repository.dart new file mode 100644 index 00000000..5cf7248e --- /dev/null +++ b/flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_cache_info_repository.dart @@ -0,0 +1,545 @@ +import 'dart:async'; +import 'dart:js_interop'; + +import 'package:flutter_cache_manager/src/logger.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/cache_info_repository.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/helper_methods.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_connection_pool.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_schema.dart'; +import 'package:flutter_cache_manager/src/storage/cache_object.dart'; +import 'package:web/web.dart' as web; + +/// A cache info repository implementation that stores cache metadata in IndexedDB. +class IndexedDbCacheInfoRepository extends CacheInfoRepository + with CacheInfoRepositoryHelperMethods { + IndexedDbCacheInfoRepository({required this.databaseName}); + + final String databaseName; + + static const String _metadataStoreName = IndexedDbSchema.metadataStore; + static const String _keyIndexName = IndexedDbSchema.keyIndex; + static const String _touchedIndexName = IndexedDbSchema.touchedIndex; + + late final IndexedDbConnectionPool _connectionPool = + IndexedDbConnectionPool.getInstance(databaseName: databaseName); + + Future _getDatabase() async { + return _connectionPool.getDatabase(); + } + + @override + Future open() async { + if (!shouldOpenOnNewConnection()) { + return openCompleter!.future; + } + // Held until close(), so the connection stays open while in use. + await _connectionPool.acquire(); + return opened(); + } + + web.IDBTransaction _createTransaction( + web.IDBDatabase db, + String storeName, + String mode, + ) => createIndexedDbTransaction(db, [storeName], mode); + + /// Frees space after a quota error. Removes the least recently used + /// entries together with their file contents (stored in the same database + /// by `IndexedDbFileSystem` when it uses this [databaseName], the default). + Future _handleQuotaExceeded() async { + try { + await evictLeastRecentlyUsed(await _getDatabase()); + } catch (e) { + cacheLogger.log( + 'CacheManager: Failed to free up space: $e', + CacheManagerLogLevel.warning, + ); + } + } + + @override + Future get(String key) async { + final db = await _getDatabase(); + final completer = Completer(); + + final transaction = _createTransaction(db, _metadataStoreName, 'readonly'); + final store = transaction.objectStore(_metadataStoreName); + final index = store.index(_keyIndexName); + final request = index.get(key.toJS); + + request.onsuccess = (web.Event e) { + final result = request.result; + if (result != null) { + final map = _jsToMap(result); + completer.complete(CacheObject.fromMap(map)); + } else { + completer.complete(null); + } + }.toJS; + + request.onerror = (web.Event e) { + completer.completeError( + Exception('Failed to get object from IndexedDB: ${request.error}'), + ); + }.toJS; + + return completer.future; + } + + @override + Future> getAllObjects() async { + final db = await _getDatabase(); + final completer = Completer>(); + + final transaction = _createTransaction(db, _metadataStoreName, 'readonly'); + final store = transaction.objectStore(_metadataStoreName); + final request = store.getAll(); + + request.onsuccess = (web.Event e) { + final result = request.result; + final list = []; + if (result != null) { + final jsArray = result as JSArray; + for (var i = 0; i < jsArray.length; i++) { + final item = jsArray[i]; + final map = _jsToMap(item); + list.add(CacheObject.fromMap(map)); + } + } + completer.complete(list); + }.toJS; + + request.onerror = (web.Event e) { + completer.completeError( + Exception('Failed to get all objects from IndexedDB: ${request.error}'), + ); + }.toJS; + + return completer.future; + } + + @override + Future insert( + CacheObject cacheObject, { + bool setTouchedToNow = true, + }) async { + if (cacheObject.id != null) { + throw ArgumentError("Inserted objects shouldn't have an existing id."); + } + + try { + return await _insertWithRetry(cacheObject, setTouchedToNow, retries: 1); + } catch (e) { + if (isQuotaExceeded(e)) { + cacheLogger.log( + 'CacheManager: Insert failed due to quota exceeded', + CacheManagerLogLevel.warning, + ); + } + rethrow; + } + } + + Future _insertWithRetry( + CacheObject cacheObject, + bool setTouchedToNow, { + required int retries, + }) async { + final db = await _getDatabase(); + final completer = Completer(); + + final transaction = _createTransaction(db, _metadataStoreName, 'readwrite'); + final store = transaction.objectStore(_metadataStoreName); + + final map = cacheObject.toMap(setTouchedToNow: setTouchedToNow); + map.remove(CacheObject.columnId); // Let IndexedDB auto-generate the id + + final request = store.add(map.jsify()); + + request.onsuccess = (web.Event e) { + final id = (request.result as JSNumber).toDartInt; + final newCacheObject = cacheObject.copyWith(id: id); + completer.complete(newCacheObject); + }.toJS; + + request.onerror = (web.Event e) { + e.preventDefault(); + completer.completeError( + IndexedDbException( + 'Failed to insert object into IndexedDB', + request.error, + ), + ); + }.toJS; + + try { + return await completer.future; + } catch (e) { + if (isQuotaExceeded(e) && retries > 0) { + // Try to free up space and retry once + await _handleQuotaExceeded(); + return await _insertWithRetry( + cacheObject, + setTouchedToNow, + retries: retries - 1, + ); + } + rethrow; + } + } + + @override + Future update( + CacheObject cacheObject, { + bool setTouchedToNow = true, + }) async { + if (cacheObject.id == null) { + throw ArgumentError('Updated objects should have an existing id.'); + } + + final db = await _getDatabase(); + final completer = Completer(); + + final transaction = _createTransaction(db, _metadataStoreName, 'readwrite'); + final store = transaction.objectStore(_metadataStoreName); + + final map = cacheObject.toMap(setTouchedToNow: setTouchedToNow); + final request = store.put(map.jsify()); + + request.onsuccess = (web.Event e) { + completer.complete(1); + }.toJS; + + request.onerror = (web.Event e) { + e.preventDefault(); + completer.completeError( + IndexedDbException( + 'Failed to update object in IndexedDB', + request.error, + ), + ); + }.toJS; + + return completer.future; + } + + @override + Future updateOrInsert(CacheObject cacheObject) { + return cacheObject.id == null ? insert(cacheObject) : update(cacheObject); + } + + @override + Future> getObjectsOverCapacity(int capacity) async { + final db = await _getDatabase(); + final completer = Completer>(); + + try { + final transaction = _createTransaction( + db, + _metadataStoreName, + 'readonly', + ); + final store = transaction.objectStore(_metadataStoreName); + + // First, get the count to determine if we're over capacity + final countRequest = store.count(); + + countRequest.onsuccess = (web.Event e) { + final totalCount = (countRequest.result as JSNumber).toDartInt; + + if (totalCount <= capacity) { + completer.complete([]); + return; + } + + // Use the touched index to iterate in sorted order (oldest first) + final index = store.index(_touchedIndexName); + final result = []; + final toRemoveCount = totalCount - capacity; + var count = 0; + + // Open cursor to iterate through oldest items + final cursorRequest = index.openCursor(); + + cursorRequest.onsuccess = (web.Event e) { + final cursor = cursorRequest.result as web.IDBCursorWithValue?; + + if (cursor != null) { + if (count < toRemoveCount) { + final map = _jsToMap(cursor.value); + result.add(CacheObject.fromMap(map)); + count++; + cursor.continue_(); + } else { + // We have enough items, complete + completer.complete(result); + } + } else { + // No more items + completer.complete(result); + } + }.toJS; + + cursorRequest.onerror = (web.Event e) { + completer.completeError( + Exception('Failed to iterate objects: ${cursorRequest.error}'), + ); + }.toJS; + }.toJS; + + countRequest.onerror = (web.Event e) { + completer.completeError( + Exception('Failed to count objects: ${countRequest.error}'), + ); + }.toJS; + + return await completer.future; + } catch (e) { + // Fallback to old method if cursor fails (shouldn't happen with proper index) + final allObjects = await getAllObjects(); + allObjects.sort((c1, c2) => c1.touched!.compareTo(c2.touched!)); + if (allObjects.length <= capacity) return []; + return allObjects.getRange(0, allObjects.length - capacity).toList(); + } + } + + @override + Future> getOldObjects(Duration maxAge) async { + final oldestTimestamp = DateTime.now().subtract(maxAge); + final db = await _getDatabase(); + final completer = Completer>(); + + try { + final transaction = _createTransaction( + db, + _metadataStoreName, + 'readonly', + ); + final store = transaction.objectStore(_metadataStoreName); + + // Use the touched index to efficiently find old objects + final index = store.index(_touchedIndexName); + final result = []; + + // Create a key range for items older than the threshold + // Items with touched timestamp less than oldestTimestamp + final keyRange = web.IDBKeyRange.upperBound( + oldestTimestamp.millisecondsSinceEpoch.toJS, + false, // not open (inclusive) + ); + + final cursorRequest = index.openCursor(keyRange); + + cursorRequest.onsuccess = (web.Event e) { + final cursor = cursorRequest.result as web.IDBCursorWithValue?; + if (cursor != null) { + final map = _jsToMap(cursor.value); + final obj = CacheObject.fromMap(map); + // Double-check the timestamp (defensive programming) + if (obj.touched != null && obj.touched!.isBefore(oldestTimestamp)) { + result.add(obj); + } + cursor.continue_(); + } else { + // No more items + completer.complete(result); + } + }.toJS; + + cursorRequest.onerror = (web.Event e) { + completer.completeError( + Exception('Failed to iterate old objects: ${cursorRequest.error}'), + ); + }.toJS; + + return await completer.future; + } catch (e) { + // Fallback to old method if cursor fails + final allObjects = await getAllObjects(); + return allObjects + .where((element) => element.touched!.isBefore(oldestTimestamp)) + .toList(); + } + } + + @override + Future delete(int id) async { + final db = await _getDatabase(); + final completer = Completer(); + + final transaction = _createTransaction(db, _metadataStoreName, 'readwrite'); + final store = transaction.objectStore(_metadataStoreName); + final request = store.delete(id.toJS); + + request.onsuccess = (web.Event e) { + completer.complete(1); + }.toJS; + + request.onerror = (web.Event e) { + completer.completeError( + Exception('Failed to delete object from IndexedDB: ${request.error}'), + ); + }.toJS; + + return completer.future; + } + + @override + Future deleteAll(Iterable ids) async { + if (ids.isEmpty) return 0; + + final db = await _getDatabase(); + final completer = Completer(); + + try { + final transaction = _createTransaction( + db, + _metadataStoreName, + 'readwrite', + ); + final store = transaction.objectStore(_metadataStoreName); + + // Queue all delete operations in the transaction + final deleteRequests = []; + for (final id in ids) { + deleteRequests.add(store.delete(id.toJS)); + } + + // Wait for the entire transaction to complete + // This ensures all deletes are atomic + transaction.oncomplete = (web.Event e) { + completer.complete(ids.length); + }.toJS; + + transaction.onerror = (web.Event e) { + completer.completeError( + Exception( + 'Failed to delete objects from IndexedDB: ${transaction.error}', + ), + ); + }.toJS; + + transaction.onabort = (web.Event e) { + completer.completeError( + Exception('Delete transaction aborted: ${transaction.error}'), + ); + }.toJS; + + return await completer.future; + } catch (e) { + throw Exception('Failed to delete all objects: $e'); + } + } + + @override + Future close() async { + if (!shouldClose()) { + return false; + } + // Releases the connection taken in open(); it closes once no other + // repository for this database holds it. + _connectionPool.release(); + return true; + } + + @override + Future deleteDataFile() async { + await close(); + + // Close the shared connection, or it would block the deletion. + IndexedDbConnectionPool.removeInstance(databaseName); + + final completer = Completer(); + final request = web.window.indexedDB.deleteDatabase(databaseName); + + request.onsuccess = (web.Event e) { + completer.complete(); + }.toJS; + + request.onerror = (web.Event e) { + completer.completeError( + IndexedDbException( + 'Failed to delete IndexedDB database', + request.error, + ), + ); + }.toJS; + + // A connection in another tab is still open. Those close on + // `versionchange`, after which the deletion goes ahead and onsuccess + // fires; until then, report why it is waiting. + request.onblocked = (web.Event e) { + cacheLogger.log( + 'CacheManager: deleting IndexedDB database $databaseName is blocked by ' + 'a connection in another tab; waiting for it to close', + CacheManagerLogLevel.warning, + ); + }.toJS; + + return completer.future; + } + + @override + Future exists() async { + try { + final db = await _getDatabase(); + final hasStore = db.objectStoreNames.contains(_metadataStoreName); + return hasStore; + } catch (e) { + throw Exception('Failed to check if IndexedDB exists: $e'); + } + } + + /// Converts a JavaScript value to a Dart Map. + Map _jsToMap(JSAny? jsValue) { + if (jsValue == null) { + return {}; + } + + final map = {}; + final obj = jsValue as JSObject; + + // Get all property names + final keysArray = objectKeys(obj); + final keys = keysArray.toDart; + + for (var i = 0; i < keys.length; i++) { + final keyJs = keys[i] as JSString; + final key = keyJs.toDart; + final value = obj[keyJs]; + + if (value == null) { + map[key] = null; + } else if (value.typeofEquals('string')) { + map[key] = (value as JSString).toDart; + } else if (value.typeofEquals('number')) { + final num = (value as JSNumber).toDartDouble; + // Check if it's an integer + if (num == num.truncateToDouble()) { + map[key] = num.toInt(); + } else { + map[key] = num; + } + } else if (value.typeofEquals('boolean')) { + map[key] = (value as JSBoolean).toDart; + } else { + map[key] = value; + } + } + + return map; + } +} + +@JS('Object.keys') +external JSArray objectKeys(JSObject obj); + +/// Extension to access JSObject properties using [] operator +extension JSObjectExtension on JSObject { + external JSAny? operator [](JSAny key); +} + +/// Extension to access JSArray elements using [] operator +extension JSArrayExtension on JSArray { + external JSAny? operator [](JSAny index); +} diff --git a/flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_connection_pool.dart b/flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_connection_pool.dart new file mode 100644 index 00000000..3fe0b554 --- /dev/null +++ b/flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_connection_pool.dart @@ -0,0 +1,133 @@ +import 'dart:async'; +import 'dart:js_interop'; + +import 'package:flutter_cache_manager/src/logger.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_schema.dart'; +import 'package:web/web.dart' as web; + +/// One shared IndexedDB connection per database name. +/// +/// Opening a database costs 50-200ms, so the cache info repository and every +/// file reuse one connection. The database is always opened at +/// [IndexedDbSchema.version] and upgraded with [IndexedDbSchema.upgrade]. +/// +/// Long-lived users (the cache info repository) [acquire] the connection and +/// [release] it when closed; the connection is closed when the last of them +/// releases it. Files use [getDatabase] without holding the connection, and +/// reopen it on demand. +class IndexedDbConnectionPool { + IndexedDbConnectionPool._(this.databaseName); + + static final Map _instances = {}; + + /// Get or create the connection pool for [databaseName]. + static IndexedDbConnectionPool getInstance({required String databaseName}) { + return _instances.putIfAbsent( + databaseName, + () => IndexedDbConnectionPool._(databaseName), + ); + } + + final String databaseName; + + web.IDBDatabase? _db; + Future? _opening; + int _holders = 0; + + /// Whether the connection is currently open. + bool get isOpen => _db != null; + + /// The number of [acquire]s not yet [release]d. + int get holders => _holders; + + /// Get the database connection, opening it if necessary. + Future getDatabase() { + final db = _db; + if (db != null) return Future.value(db); + return _opening ??= _open().whenComplete(() => _opening = null); + } + + /// Opens the connection and keeps it open until a matching [release]. + Future acquire() { + _holders++; + return getDatabase(); + } + + /// Releases an [acquire]. Closes the connection when nobody holds it. + void release() { + if (_holders == 0) return; + _holders--; + if (_holders == 0) close(); + } + + Future _open() { + final completer = Completer(); + final request = web.window.indexedDB.open( + databaseName, + IndexedDbSchema.version, + ); + + request.onupgradeneeded = (web.IDBVersionChangeEvent event) { + IndexedDbSchema.upgrade(request.result as web.IDBDatabase, event); + }.toJS; + + request.onsuccess = (web.Event _) { + final db = request.result as web.IDBDatabase; + // The browser closed the connection (storage cleared, for example). + db.onclose = (web.Event _) { + if (identical(_db, db)) _db = null; + }.toJS; + // Another tab wants to upgrade the schema. Close so it is not blocked; + // the next operation here reopens at the new version. + db.onversionchange = (web.IDBVersionChangeEvent _) { + db.close(); + if (identical(_db, db)) _db = null; + }.toJS; + _db = db; + completer.complete(db); + }.toJS; + + request.onerror = (web.Event _) { + completer.completeError( + IndexedDbException( + 'Failed to open IndexedDB database $databaseName', + request.error, + ), + ); + }.toJS; + + // Another tab still has the database open at an older version. It closes + // its connection on `versionchange` (see above), after which this open + // continues; a tab running an older release of this package might not, + // so say why the cache is waiting. + request.onblocked = (web.Event _) { + cacheLogger.log( + 'CacheManager: opening IndexedDB database $databaseName is blocked by ' + 'a connection in another tab; waiting for it to close', + CacheManagerLogLevel.warning, + ); + }.toJS; + + return completer.future; + } + + /// Closes the connection now, regardless of holders. It is reopened by the + /// next [getDatabase]. + void close() { + _db?.close(); + _db = null; + } + + /// Closes and forgets the pool for [databaseName]. + static void removeInstance(String databaseName) { + _instances.remove(databaseName)?.close(); + } + + /// Closes and forgets every pool (used for tests and app shutdown). + static void closeAll() { + for (final pool in _instances.values) { + pool.close(); + } + _instances.clear(); + } +} diff --git a/flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_schema.dart b/flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_schema.dart new file mode 100644 index 00000000..ae202136 --- /dev/null +++ b/flutter_cache_manager/lib/src/storage/cache_info_repositories/indexed_db_schema.dart @@ -0,0 +1,239 @@ +import 'dart:async'; +import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; + +import 'package:flutter_cache_manager/src/logger.dart'; +import 'package:flutter_cache_manager/src/storage/cache_object.dart'; +import 'package:web/web.dart' as web; + +/// The single definition of the IndexedDB schema shared by +/// `IndexedDbCacheInfoRepository` (cache info) and `IndexedDbFile` (file +/// contents). Both use the same database by default, and the connection pool +/// always upgrades it with [upgrade], so it no longer matters which of them +/// opens the database first. +abstract final class IndexedDbSchema { + /// Version history: + /// 1. `cache_metadata` (+ `key_index`) and `cache_files` (`{path, data}`). + /// 2. `touched_index` on `cache_metadata`. + /// 3. `cache_file_chunks`: file contents are stored in chunks, and a + /// `cache_files` record becomes a header (`{path, length, chunkCount, + /// chunkSize}`). Version 1/2 records (`{path, data}`) remain readable. + static const int version = 3; + + static const String metadataStore = 'cache_metadata'; + static const String keyIndex = 'key_index'; + static const String touchedIndex = 'touched_index'; + + static const String filesStore = 'cache_files'; + static const String chunksStore = 'cache_file_chunks'; + + /// Size of the chunks file contents are written in. + static const int chunkSize = 512 * 1024; + + /// Brings [db] up to [version]. Runs inside the version change transaction. + static void upgrade(web.IDBDatabase db, web.IDBVersionChangeEvent event) { + final names = db.objectStoreNames; + final transaction = (event.target as web.IDBOpenDBRequest).transaction; + + if (!names.contains(metadataStore)) { + final metadata = db.createObjectStore( + metadataStore, + web.IDBObjectStoreParameters( + keyPath: CacheObject.columnId.toJS, + autoIncrement: true, + ), + ); + metadata.createIndex( + keyIndex, + CacheObject.columnKey.toJS, + web.IDBIndexParameters(unique: true), + ); + } + final metadata = transaction?.objectStore(metadataStore); + if (metadata != null && !metadata.indexNames.contains(touchedIndex)) { + metadata.createIndex( + touchedIndex, + CacheObject.columnTouched.toJS, + web.IDBIndexParameters(unique: false), + ); + } + + if (!names.contains(filesStore)) { + db.createObjectStore( + filesStore, + web.IDBObjectStoreParameters(keyPath: 'path'.toJS), + ); + } + if (!names.contains(chunksStore)) { + db.createObjectStore( + chunksStore, + web.IDBObjectStoreParameters( + keyPath: ['path'.toJS, 'index'.toJS].toJS, + ), + ); + } + } + + /// The key of chunk [index] of the file at [path]. + static JSAny chunkKey(String path, int index) => + [path.toJS, index.toJS].toJS; + + /// A key range covering every chunk of the file at [path]. + static web.IDBKeyRange chunksOf(String path) => + web.IDBKeyRange.bound(chunkKey(path, 0), chunkKey(path, 0x7fffffff)); +} + +/// Creates a transaction over [stores]. Cache data can always be downloaded +/// again, so writes request `relaxed` durability: the browser may acknowledge +/// them before they are flushed to disk, which is much faster for many small +/// writes. Browsers without durability hints ignore the option. +web.IDBTransaction createIndexedDbTransaction( + web.IDBDatabase db, + List stores, + String mode, +) { + final names = stores.length == 1 + ? stores.single.toJS + : stores.map((s) => s.toJS).toList().toJS; + return db.transaction( + names, + mode, + web.IDBTransactionOptions(durability: 'relaxed'), + ); +} + +/// Completes with the result of [request]. +Future indexedDbRequest(web.IDBRequest request, String description) { + final completer = Completer(); + request.onsuccess = (web.Event _) { + completer.complete(request.result); + }.toJS; + request.onerror = (web.Event event) { + // Handled here; don't let the error abort the whole transaction. + event.preventDefault(); + completer.completeError(IndexedDbException(description, request.error)); + }.toJS; + return completer.future; +} + +/// Completes when [transaction] has committed. +Future indexedDbTransactionDone( + web.IDBTransaction transaction, + String description, +) { + final completer = Completer(); + transaction.oncomplete = (web.Event _) { + if (!completer.isCompleted) completer.complete(); + }.toJS; + transaction.onerror = (web.Event _) { + if (!completer.isCompleted) { + completer.completeError( + IndexedDbException(description, transaction.error), + ); + } + }.toJS; + transaction.onabort = (web.Event _) { + if (!completer.isCompleted) { + completer.completeError( + IndexedDbException('$description (aborted)', transaction.error), + ); + } + }.toJS; + return completer.future; +} + +/// An IndexedDB request or transaction failed. +class IndexedDbException implements Exception { + IndexedDbException(this.message, web.DOMException? error) + : name = error?.name, + detail = error?.message; + + final String message; + + /// The DOMException name, such as `QuotaExceededError`. + final String? name; + final String? detail; + + /// Whether the browser refused the write because storage is full. + bool get isQuotaExceeded => name == 'QuotaExceededError'; + + @override + String toString() => 'IndexedDbException: $message ($name: $detail)'; +} + +/// Whether [error] means the browser's storage quota is exhausted. +bool isQuotaExceeded(Object error) { + if (error is IndexedDbException) return error.isQuotaExceeded; + final text = error.toString().toLowerCase(); + return text.contains('quotaexceedederror'); +} + +/// Frees space after a quota error by removing the least recently used +/// tenth (at most [maxEntries]) of the cache: their cache info rows and the +/// file contents they point to, in one transaction. Removing only the rows +/// would leave the contents, which take up the space, behind. +/// +/// Returns the number of entries removed. +Future evictLeastRecentlyUsed( + web.IDBDatabase db, { + int maxEntries = 50, +}) async { + if (!db.objectStoreNames.contains(IndexedDbSchema.metadataStore)) return 0; + final hasFiles = + db.objectStoreNames.contains(IndexedDbSchema.filesStore) && + db.objectStoreNames.contains(IndexedDbSchema.chunksStore); + + // Count in its own transaction: a transaction must not wait on Dart code + // between requests, or it may commit early. + final counting = createIndexedDbTransaction(db, [ + IndexedDbSchema.metadataStore, + ], 'readonly'); + final total = + ((await indexedDbRequest( + counting.objectStore(IndexedDbSchema.metadataStore).count(), + 'Failed to count cache entries', + )) + as JSNumber) + .toDartInt; + final toRemove = (total * 0.1).ceil().clamp(0, maxEntries); + if (toRemove == 0) return 0; + + final transaction = createIndexedDbTransaction(db, [ + IndexedDbSchema.metadataStore, + if (hasFiles) IndexedDbSchema.filesStore, + if (hasFiles) IndexedDbSchema.chunksStore, + ], 'readwrite'); + final done = indexedDbTransactionDone(transaction, 'Failed to free space'); + final cursorRequest = transaction + .objectStore(IndexedDbSchema.metadataStore) + .index(IndexedDbSchema.touchedIndex) + .openCursor(); + var removed = 0; + // Every request is issued from a request callback, so the transaction + // stays active until the last one. + cursorRequest.onsuccess = (web.Event _) { + final cursor = cursorRequest.result as web.IDBCursorWithValue?; + if (cursor == null || removed >= toRemove) return; + final row = cursor.value as JSObject; + final path = row.getProperty(CacheObject.columnPath.toJS); + if (hasFiles && path != null && path.isA()) { + final relativePath = (path as JSString).toDart; + transaction + .objectStore(IndexedDbSchema.filesStore) + .delete(relativePath.toJS); + transaction + .objectStore(IndexedDbSchema.chunksStore) + .delete(IndexedDbSchema.chunksOf(relativePath)); + } + cursor.delete(); + removed++; + cursor.continue_(); + }.toJS; + await done; + cacheLogger.log( + 'CacheManager: storage quota exceeded, removed $removed least recently ' + 'used cache entries', + CacheManagerLogLevel.warning, + ); + return removed; +} diff --git a/flutter_cache_manager/lib/src/storage/file_system/file_system.dart b/flutter_cache_manager/lib/src/storage/file_system/file_system.dart index 2cd9764b..276e8ad7 100644 --- a/flutter_cache_manager/lib/src/storage/file_system/file_system.dart +++ b/flutter_cache_manager/lib/src/storage/file_system/file_system.dart @@ -1,6 +1,8 @@ export 'file_system.dart'; export 'file_system_io.dart'; export 'file_system_web.dart'; +export 'indexed_db_file_system_stub.dart' + if (dart.library.js_interop) 'indexed_db_file_system.dart'; import 'package:file/file.dart'; diff --git a/flutter_cache_manager/lib/src/storage/file_system/indexed_db_file.dart b/flutter_cache_manager/lib/src/storage/file_system/indexed_db_file.dart new file mode 100644 index 00000000..2106b91c --- /dev/null +++ b/flutter_cache_manager/lib/src/storage/file_system/indexed_db_file.dart @@ -0,0 +1,687 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; +import 'dart:math'; +import 'dart:typed_data'; + +import 'package:file/file.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_connection_pool.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_schema.dart'; +import 'package:path/path.dart' as p; +import 'package:web/web.dart' as web; + +/// A file stored in IndexedDB, for web platforms. +/// +/// The contents are stored in chunks of [IndexedDbSchema.chunkSize] in the +/// `cache_file_chunks` store, and a header with the length and chunk count in +/// `cache_files`. The header is written last, when the file is closed, so a +/// file whose write did not finish does not [exists]. Files written by +/// earlier versions (one `{path, data}` record) are still readable. +class IndexedDbFile implements File { + IndexedDbFile(this._path, this._dbName); + + final String _path; + final String _dbName; + + static const String _filesStore = IndexedDbSchema.filesStore; + static const String _chunksStore = IndexedDbSchema.chunksStore; + + IndexedDbConnectionPool get _connectionPool => + IndexedDbConnectionPool.getInstance(databaseName: _dbName); + + Future _getDatabase() => _connectionPool.getDatabase(); + + Future<_FileHeader?> _readHeader() async { + final db = await _getDatabase(); + final transaction = createIndexedDbTransaction(db, [ + _filesStore, + ], 'readonly'); + final result = await indexedDbRequest( + transaction.objectStore(_filesStore).get(_path.toJS), + 'Failed to read $_path from IndexedDB', + ); + return _FileHeader.fromJS(result); + } + + Future _readChunk(int index) async { + final db = await _getDatabase(); + final transaction = createIndexedDbTransaction(db, [ + _chunksStore, + ], 'readonly'); + final result = await indexedDbRequest( + transaction + .objectStore(_chunksStore) + .get(IndexedDbSchema.chunkKey(_path, index)), + 'Failed to read $_path from IndexedDB', + ); + if (result == null || result.isUndefinedOrNull) { + throw FileSystemException('Missing chunk $index in IndexedDB', _path); + } + final data = (result as JSObject).getProperty('data'.toJS); + if (data == null || !data.isA()) return Uint8List(0); + return (data as JSUint8Array).toDart; + } + + /// Removes the header and every chunk of this file, written or partial. + Future _deleteContents() async { + final db = await _getDatabase(); + final transaction = createIndexedDbTransaction(db, [ + _filesStore, + _chunksStore, + ], 'readwrite'); + final done = indexedDbTransactionDone( + transaction, + 'Failed to delete $_path from IndexedDB', + ); + transaction.objectStore(_filesStore).delete(_path.toJS); + transaction + .objectStore(_chunksStore) + .delete(IndexedDbSchema.chunksOf(_path)); + await done; + } + + @override + Future readAsBytes() async { + final header = await _readHeader(); + if (header == null) { + throw FileSystemException('File not found in IndexedDB', _path); + } + if (header.legacyData != null) return header.legacyData!; + if (header.chunkCount == 1) return _readChunk(0); + final builder = BytesBuilder(copy: false); + await for (final chunk in _read(header, 0, header.length)) { + builder.add(chunk); + } + return builder.takeBytes(); + } + + @override + Future writeAsBytes( + List bytes, { + FileMode mode = FileMode.write, + bool flush = false, + }) async { + final sink = openWrite(mode: mode); + sink.add(bytes); + await sink.close(); + return this; + } + + /// Opens a sink that writes the file in chunks as data arrives, instead of + /// buffering the whole file. + @override + IOSink openWrite({FileMode mode = FileMode.write, Encoding encoding = utf8}) { + return _IndexedDbIOSink(this, mode, encoding); + } + + /// Streams the file one chunk at a time. + @override + Stream> openRead([int? start, int? end]) async* { + final header = await _readHeader(); + if (header == null) { + throw FileSystemException('File not found in IndexedDB', _path); + } + final from = start ?? 0; + final to = min(end ?? header.length, header.length); + if (from >= to) return; + final legacy = header.legacyData; + if (legacy != null) { + yield Uint8List.sublistView(legacy, from, to); + return; + } + yield* _read(header, from, to); + } + + Stream _read(_FileHeader header, int from, int to) async* { + final size = header.chunkSize; + for (var index = from ~/ size; index * size < to; index++) { + final chunk = await _readChunk(index); + final chunkStart = index * size; + final sliceStart = max(from - chunkStart, 0); + final sliceEnd = min(to - chunkStart, chunk.length); + yield sliceStart == 0 && sliceEnd == chunk.length + ? chunk + : Uint8List.sublistView(chunk, sliceStart, sliceEnd); + } + } + + /// Checks the key only; the contents are not read. + @override + Future exists() async { + final db = await _getDatabase(); + final transaction = createIndexedDbTransaction(db, [ + _filesStore, + ], 'readonly'); + final count = await indexedDbRequest( + transaction.objectStore(_filesStore).count(_path.toJS), + 'Failed to look up $_path in IndexedDB', + ); + return (count as JSNumber).toDartInt > 0; + } + + @override + bool existsSync() { + throw UnsupportedError('existsSync is not supported on web'); + } + + @override + Future delete({bool recursive = false}) async { + await _deleteContents(); + return this; + } + + @override + void deleteSync({bool recursive = false}) { + throw UnsupportedError('deleteSync is not supported on web'); + } + + @override + String get path => _path; + + @override + String get basename => p.basename(_path); + + @override + String get dirname => p.dirname(_path); + + @override + Uri get uri => Uri.file(_path); + + @override + Directory get parent => + throw UnsupportedError('parent is not supported for IndexedDbFile'); + + @override + bool get isAbsolute => true; + + @override + File get absolute => this; + + @override + Future copy(String newPath) { + throw UnsupportedError('copy is not supported for IndexedDbFile'); + } + + @override + File copySync(String newPath) { + throw UnsupportedError('copySync is not supported for IndexedDbFile'); + } + + @override + Future create({bool recursive = false, bool exclusive = false}) async { + // For IndexedDB, we don't need to create the file explicitly + // It will be created when we write to it + return this; + } + + @override + void createSync({bool recursive = false, bool exclusive = false}) { + throw UnsupportedError('createSync is not supported on web'); + } + + @override + Future lastAccessed() { + throw UnsupportedError('lastAccessed is not supported for IndexedDbFile'); + } + + @override + DateTime lastAccessedSync() { + throw UnsupportedError('lastAccessedSync is not supported on web'); + } + + @override + Future lastModified() { + throw UnsupportedError('lastModified is not supported for IndexedDbFile'); + } + + @override + DateTime lastModifiedSync() { + throw UnsupportedError('lastModifiedSync is not supported on web'); + } + + /// The size recorded when the file was written; the contents are not read. + @override + Future length() async { + final header = await _readHeader(); + if (header == null) { + throw FileSystemException('File not found in IndexedDB', _path); + } + return header.length; + } + + @override + int lengthSync() { + throw UnsupportedError('lengthSync is not supported on web'); + } + + @override + Future open({FileMode mode = FileMode.read}) { + throw UnsupportedError('open is not supported for IndexedDbFile'); + } + + @override + RandomAccessFile openSync({FileMode mode = FileMode.read}) { + throw UnsupportedError('openSync is not supported on web'); + } + + @override + Stream watch({ + int events = FileSystemEvent.all, + bool recursive = false, + }) { + throw UnsupportedError('watch is not supported for IndexedDbFile'); + } + + @override + Future readAsString({Encoding encoding = utf8}) async { + final bytes = await readAsBytes(); + return encoding.decode(bytes); + } + + @override + String readAsStringSync({Encoding encoding = utf8}) { + throw UnsupportedError('readAsStringSync is not supported on web'); + } + + @override + Uint8List readAsBytesSync() { + throw UnsupportedError('readAsBytesSync is not supported on web'); + } + + @override + Future> readAsLines({Encoding encoding = utf8}) async { + final content = await readAsString(encoding: encoding); + return content.split('\n'); + } + + @override + List readAsLinesSync({Encoding encoding = utf8}) { + throw UnsupportedError('readAsLinesSync is not supported on web'); + } + + @override + Future rename(String newPath) { + throw UnsupportedError('rename is not supported for IndexedDbFile'); + } + + @override + File renameSync(String newPath) { + throw UnsupportedError('renameSync is not supported on web'); + } + + @override + Future writeAsString( + String contents, { + FileMode mode = FileMode.write, + Encoding encoding = utf8, + bool flush = false, + }) async { + final bytes = encoding.encode(contents); + return writeAsBytes(bytes, mode: mode, flush: flush); + } + + @override + void writeAsStringSync( + String contents, { + FileMode mode = FileMode.write, + Encoding encoding = utf8, + bool flush = false, + }) { + throw UnsupportedError('writeAsStringSync is not supported on web'); + } + + @override + void writeAsBytesSync( + List bytes, { + FileMode mode = FileMode.write, + bool flush = false, + }) { + throw UnsupportedError('writeAsBytesSync is not supported on web'); + } + + @override + Future resolveSymbolicLinks() async { + return _path; + } + + @override + String resolveSymbolicLinksSync() { + return _path; + } + + @override + Future stat() { + throw UnsupportedError('stat is not supported for IndexedDbFile'); + } + + @override + FileStat statSync() { + throw UnsupportedError('statSync is not supported on web'); + } + + @override + Future setLastAccessed(DateTime time) { + throw UnsupportedError( + 'setLastAccessed is not supported for IndexedDbFile', + ); + } + + @override + void setLastAccessedSync(DateTime time) { + throw UnsupportedError('setLastAccessedSync is not supported on web'); + } + + @override + Future setLastModified(DateTime time) { + throw UnsupportedError( + 'setLastModified is not supported for IndexedDbFile', + ); + } + + @override + void setLastModifiedSync(DateTime time) { + throw UnsupportedError('setLastModifiedSync is not supported on web'); + } + + @override + FileSystem get fileSystem => + throw UnsupportedError('fileSystem is not supported for IndexedDbFile'); +} + +class _FileHeader { + _FileHeader({ + required this.length, + required this.chunkCount, + required this.chunkSize, + this.legacyData, + }); + + static _FileHeader? fromJS(JSAny? value) { + if (value == null || value.isUndefinedOrNull) return null; + final record = value as JSObject; + final data = record.getProperty('data'.toJS); + if (data != null && data.isA()) { + // Written by an earlier version, as a single record. + final bytes = (data as JSUint8Array).toDart; + return _FileHeader( + length: bytes.length, + chunkCount: 1, + chunkSize: max(bytes.length, 1), + legacyData: bytes, + ); + } + int read(String name) => + (record.getProperty(name.toJS) as JSNumber?)?.toDartInt ?? 0; + return _FileHeader( + length: read('length'), + chunkCount: read('chunkCount'), + chunkSize: max(read('chunkSize'), 1), + ); + } + + final int length; + final int chunkCount; + final int chunkSize; + final Uint8List? legacyData; +} + +/// Writes an [IndexedDbFile] in chunks of [IndexedDbSchema.chunkSize] while +/// data arrives, so at most about one chunk per pending write is held in +/// memory instead of the whole file. [close] writes the header that makes the +/// file visible. If a write fails, the chunks written so far are removed and +/// [close] (and [done]) complete with the error. +class _IndexedDbIOSink implements IOSink { + _IndexedDbIOSink(this._file, FileMode mode, this.encoding) { + _pending = _prepare(mode).then( + (_) => _drainEarly(), + onError: (Object e, StackTrace s) { + _error ??= (e, s); + _early.clear(); + }, + ); + } + + final IndexedDbFile _file; + final _buffer = BytesBuilder(copy: true); + final _done = Completer(); + late Future _pending; + int _nextIndex = 0; + int _length = 0; + (Object, StackTrace)? _error; + bool _isClosed = false; + + /// Data added before [_prepare] finished. In append mode the existing + /// partial last chunk has to go into the buffer first, and the next chunk + /// index is only known once the header is read, so nothing is buffered or + /// queued until then. + final List _early = []; + bool _isPrepared = false; + + @override + Encoding encoding; + + int get _chunkSize => IndexedDbSchema.chunkSize; + + Future _prepare(FileMode mode) async { + final append = mode == FileMode.append || mode == FileMode.writeOnlyAppend; + if (!append) { + await _file._deleteContents(); + return; + } + final header = await _file._readHeader(); + if (header == null) return; + // Continue after the existing content: rewrite a legacy record or a + // partial last chunk through the buffer, so every chunk but the last is + // full and offsets stay chunk-aligned. + if (header.legacyData != null) { + await _file._deleteContents(); + _buffer.add(header.legacyData!); + return; + } + final lastSize = header.length - (header.chunkCount - 1) * _chunkSize; + if (header.chunkCount > 0 && lastSize < _chunkSize) { + _buffer.add(await _file._readChunk(header.chunkCount - 1)); + _nextIndex = header.chunkCount - 1; + } else { + _nextIndex = header.chunkCount; + } + _length = _nextIndex * _chunkSize; + } + + void _drainEarly() { + _isPrepared = true; + final early = List.of(_early); + _early.clear(); + for (final data in early) { + _addPrepared(data); + } + } + + void _checkOpen() { + if (_isClosed) throw StateError('StreamSink is closed'); + } + + @override + void add(List data) { + _checkOpen(); + if (_error != null) return; + if (!_isPrepared) { + _early.add(Uint8List.fromList(data)); + return; + } + _addPrepared(data); + } + + void _addPrepared(List data) { + _buffer.add(data); + if (_buffer.length < _chunkSize) return; + final bytes = _buffer.takeBytes(); + var offset = 0; + while (bytes.length - offset >= _chunkSize) { + _enqueue(Uint8List.sublistView(bytes, offset, offset + _chunkSize)); + offset += _chunkSize; + } + if (offset < bytes.length) { + _buffer.add(Uint8List.sublistView(bytes, offset)); + } + } + + void _enqueue(Uint8List chunk) { + final index = _nextIndex++; + _length += chunk.length; + _pending = _pending.then((_) { + if (_error != null) return null; + return _putChunk(index, chunk); + }); + } + + Future _putChunk( + int index, + Uint8List chunk, { + bool retry = true, + }) async { + try { + final db = await _file._getDatabase(); + final transaction = createIndexedDbTransaction(db, [ + IndexedDbSchema.chunksStore, + ], 'readwrite'); + final done = indexedDbTransactionDone( + transaction, + 'Failed to write ${_file.path} to IndexedDB', + ); + transaction + .objectStore(IndexedDbSchema.chunksStore) + .put( + { + 'path': _file.path, + 'index': index, + 'data': chunk, + }.jsify(), + ); + await done; + } on Object catch (e, s) { + if (retry && isQuotaExceeded(e)) { + await evictLeastRecentlyUsed(await _file._getDatabase()); + return _putChunk(index, chunk, retry: false); + } + _error ??= (e, s); + } + } + + Future _putHeader() async { + final db = await _file._getDatabase(); + final transaction = createIndexedDbTransaction(db, [ + IndexedDbSchema.filesStore, + ], 'readwrite'); + final done = indexedDbTransactionDone( + transaction, + 'Failed to write ${_file.path} to IndexedDB', + ); + transaction + .objectStore(IndexedDbSchema.filesStore) + .put( + { + 'path': _file.path, + 'length': _length, + 'chunkCount': _nextIndex, + 'chunkSize': _chunkSize, + }.jsify(), + ); + await done; + } + + @override + void write(Object? object) { + add(encoding.encode(object.toString())); + } + + @override + void writeAll(Iterable objects, [String separator = '']) { + add(encoding.encode(objects.join(separator))); + } + + @override + void writeln([Object? object = '']) { + write(object); + write('\n'); + } + + @override + void writeCharCode(int charCode) { + write(String.fromCharCode(charCode)); + } + + @override + void addError(Object error, [StackTrace? stackTrace]) { + _checkOpen(); + _error ??= (error, stackTrace ?? StackTrace.current); + } + + @override + Future addStream(Stream> stream) { + _checkOpen(); + final completer = Completer(); + stream.listen( + add, + onError: completer.completeError, + onDone: completer.complete, + cancelOnError: true, + ); + return completer.future; + } + + /// Waits for the chunks queued so far to be written. + @override + Future flush() async { + // Writes queued while waiting (early data drained once preparation + // finishes) replace [_pending], so wait until it stops changing. + Future current; + do { + current = _pending; + await current; + } while (!identical(current, _pending)); + } + + @override + Future close() { + if (_isClosed) return _done.future; + _isClosed = true; + // [_pending] starts with the preparation, so by the time this runs every + // early write has been moved into the buffer or queued. + _pending + .then((_) { + if (_buffer.isNotEmpty && _error == null) { + _enqueue(_buffer.takeBytes()); + } + return _pending; + }) + .then((_) async { + if (_error == null) { + try { + await _putHeader(); + } on Object catch (e, s) { + _error = (e, s); + } + } + final error = _error; + if (error == null) { + _done.complete(); + return; + } + try { + await _file._deleteContents(); + } on Object { + // Report the write error, not the cleanup's. + } + _done.completeError(error.$1, error.$2); + }) + .catchError((Object e, StackTrace s) { + if (!_done.isCompleted) _done.completeError(e, s); + }); + return _done.future; + } + + @override + Future get done => _done.future; +} diff --git a/flutter_cache_manager/lib/src/storage/file_system/indexed_db_file_system.dart b/flutter_cache_manager/lib/src/storage/file_system/indexed_db_file_system.dart new file mode 100644 index 00000000..cf8cb3b1 --- /dev/null +++ b/flutter_cache_manager/lib/src/storage/file_system/indexed_db_file_system.dart @@ -0,0 +1,16 @@ +import 'package:file/file.dart' as file_pkg; +import 'package:flutter_cache_manager/src/storage/file_system/file_system.dart' + as cache_fs; +import 'package:flutter_cache_manager/src/storage/file_system/indexed_db_file.dart'; + +/// A file system implementation that stores files in IndexedDB for web platforms. +class IndexedDbFileSystem implements cache_fs.FileSystem { + IndexedDbFileSystem(this.databaseName); + + final String databaseName; + + @override + Future createFile(String name) async { + return IndexedDbFile(name, databaseName); + } +} diff --git a/flutter_cache_manager/lib/src/storage/file_system/indexed_db_file_system_stub.dart b/flutter_cache_manager/lib/src/storage/file_system/indexed_db_file_system_stub.dart new file mode 100644 index 00000000..cac1a9d5 --- /dev/null +++ b/flutter_cache_manager/lib/src/storage/file_system/indexed_db_file_system_stub.dart @@ -0,0 +1,16 @@ +import 'package:file/file.dart' as file_pkg; +import 'package:flutter_cache_manager/src/storage/file_system/file_system.dart' + as cache_fs; + +/// Non-web stand-in for the IndexedDB file system, so the class can be +/// named in platform-independent code. Creating files throws. +class IndexedDbFileSystem implements cache_fs.FileSystem { + IndexedDbFileSystem(this.databaseName); + + final String databaseName; + + @override + Future createFile(String name) async { + throw UnsupportedError('IndexedDbFileSystem is only available on web'); + } +} diff --git a/flutter_cache_manager/pubspec.yaml b/flutter_cache_manager/pubspec.yaml index 49ca35b2..8b9748b5 100644 --- a/flutter_cache_manager/pubspec.yaml +++ b/flutter_cache_manager/pubspec.yaml @@ -1,6 +1,6 @@ name: flutter_cache_manager description: Generic cache manager for flutter. Saves web files on the storages of the device and saves the cache info using sqflite. -version: 3.5.0 +version: 3.6.0 homepage: https://github.com/Baseflow/flutter_cache_manager topics: - cache @@ -20,6 +20,7 @@ dependencies: rxdart: ^0.28.0 sqflite: ^2.4.2 uuid: ^4.6.0 + web: ^1.1.1 dev_dependencies: build_runner: ^2.14.0 diff --git a/flutter_cache_manager/test/web/indexed_db_cache_info_repository_test.dart b/flutter_cache_manager/test/web/indexed_db_cache_info_repository_test.dart new file mode 100644 index 00000000..51fe39bc --- /dev/null +++ b/flutter_cache_manager/test/web/indexed_db_cache_info_repository_test.dart @@ -0,0 +1,152 @@ +@TestOn('browser') +library; + +import 'dart:typed_data'; + +import 'package:flutter_cache_manager/flutter_cache_manager.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_cache_info_repository.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_connection_pool.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_schema.dart'; +import 'package:flutter_cache_manager/src/storage/file_system/indexed_db_file.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import 'indexed_db_test_helpers.dart'; + +const _db = 'indexed_db_repository_test'; + +CacheObject _object(String key, {DateTime? touched, String? path}) => + CacheObject( + 'https://example.com/$key', + key: key, + relativePath: path ?? '$key.file', + validTill: DateTime(2030), + touched: touched, + ); + +void main() { + late IndexedDbCacheInfoRepository repo; + + setUp(() async { + repo = IndexedDbCacheInfoRepository(databaseName: _db); + await repo.open(); + }); + + tearDown(() async { + await repo.close(); + await deleteDatabase(_db); + }); + + test('insert, get, update and delete a cache object', () async { + final inserted = await repo.insert(_object('a')); + expect(inserted.id, isNotNull); + + final read = await repo.get('a'); + expect(read!.id, inserted.id); + expect(read.relativePath, 'a.file'); + + await repo.update(read.copyWith(eTag: 'v2')); + expect((await repo.get('a'))!.eTag, 'v2'); + + await repo.delete(inserted.id!); + expect(await repo.get('a'), isNull); + }); + + test('updateOrInsert, getAllObjects and deleteAll', () async { + final a = await repo.updateOrInsert(_object('a')) as CacheObject; + await repo.updateOrInsert(_object('b')); + expect(await repo.getAllObjects(), hasLength(2)); + + await repo.deleteAll([a.id!]); + final left = await repo.getAllObjects(); + expect(left.map((o) => o.key), ['b']); + }); + + test('getObjectsOverCapacity returns the least recently used', () async { + for (var i = 0; i < 5; i++) { + await repo.insert( + _object('k$i', touched: DateTime(2026, 1, 1 + i)), + setTouchedToNow: false, + ); + } + final over = await repo.getObjectsOverCapacity(3); + expect(over.map((o) => o.key), ['k0', 'k1']); + }); + + test('getOldObjects returns entries older than the max age', () async { + final now = DateTime.now(); + await repo.insert( + _object('old', touched: now.subtract(const Duration(days: 40))), + setTouchedToNow: false, + ); + await repo.insert(_object('new')); + final old = await repo.getOldObjects(const Duration(days: 30)); + expect(old.map((o) => o.key), ['old']); + }); + + test('freeing space after a quota error removes the cache info rows and ' + 'the file contents they point to', () async { + for (var i = 0; i < 20; i++) { + final path = 'f$i.file'; + await repo.insert( + _object('k$i', touched: DateTime(2026, 1, 1 + i), path: path), + setTouchedToNow: false, + ); + await IndexedDbFile(path, _db).writeAsBytes(List.filled(1000, i)); + } + + final pool = IndexedDbConnectionPool.getInstance(databaseName: _db); + final removed = await evictLeastRecentlyUsed(await pool.getDatabase()); + + // 10% of 20 entries, oldest first. + expect(removed, 2); + expect(await repo.get('k0'), isNull); + expect(await repo.get('k1'), isNull); + expect(await repo.get('k2'), isNotNull); + expect(await IndexedDbFile('f0.file', _db).exists(), isFalse); + expect(await IndexedDbFile('f1.file', _db).exists(), isFalse); + expect(await countChunks(_db, 'f0.file'), 0); + expect(await IndexedDbFile('f2.file', _db).exists(), isTrue); + expect(await countRecords(_db, IndexedDbSchema.filesStore), 18); + expect(await countRecords(_db, IndexedDbSchema.chunksStore), 18); + }); + + test('close releases the shared connection', () async { + final pool = IndexedDbConnectionPool.getInstance(databaseName: _db); + expect(pool.isOpen, isTrue); + await repo.close(); + expect(pool.isOpen, isFalse); + // Reopen so tearDown's close is balanced. + await repo.open(); + }); + + test('deleteDataFile deletes the database', () async { + await repo.insert(_object('a')); + await repo.deleteDataFile(); + + repo = IndexedDbCacheInfoRepository(databaseName: _db); + await repo.open(); + expect(await repo.getAllObjects(), isEmpty); + }); + + test('a CacheManager stores and finds files across instances', () async { + final config = Config( + 'cm', + repo: IndexedDbCacheInfoRepository(databaseName: _db), + fileSystem: IndexedDbFileSystem(_db), + ); + final manager = CacheManager(config); + await manager.putFile('https://example.com/x', Uint8List.fromList([1, 2])); + await manager.dispose(); + + final again = CacheManager( + Config( + 'cm', + repo: IndexedDbCacheInfoRepository(databaseName: _db), + fileSystem: IndexedDbFileSystem(_db), + ), + ); + final info = await again.getFileFromCache('https://example.com/x'); + expect(await info!.file.readAsBytes(), [1, 2]); + await again.dispose(); + }); +} diff --git a/flutter_cache_manager/test/web/indexed_db_file_test.dart b/flutter_cache_manager/test/web/indexed_db_file_test.dart new file mode 100644 index 00000000..3342843d --- /dev/null +++ b/flutter_cache_manager/test/web/indexed_db_file_test.dart @@ -0,0 +1,180 @@ +@TestOn('browser') +library; + +import 'dart:js_interop'; +import 'dart:typed_data'; + +import 'package:file/file.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_connection_pool.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_schema.dart'; +import 'package:flutter_cache_manager/src/storage/file_system/indexed_db_file.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import 'indexed_db_test_helpers.dart'; + +const _db = 'indexed_db_file_test'; +const _chunk = IndexedDbSchema.chunkSize; + +Uint8List _pattern(int length) => + Uint8List.fromList(List.generate(length, (i) => i * 7 & 0xff)); + +void main() { + tearDown(() => deleteDatabase(_db)); + + test( + 'openWrite streams a large file into IndexedDB and reads it back', + () async { + final file = IndexedDbFile('large.file', _db); + final sink = file.openWrite(); + const writeSize = 64 * 1024; + const totalBytes = 212 * 1024 * 1024; + final chunk = Uint8List(writeSize); + for (var i = 0; i < writeSize; i++) { + chunk[i] = i & 0xff; + } + for (var written = 0; written < totalBytes; written += writeSize) { + sink.add(chunk); + } + await sink.close(); + + expect(await file.length(), totalBytes); + final bytes = await file.readAsBytes(); + expect(bytes.length, totalBytes); + expect(bytes[writeSize + 7], 7); + expect(bytes[totalBytes - 1], 0xff); + await file.delete(); + }, + timeout: const Timeout(Duration(minutes: 2)), + ); + + test('the sink writes chunks as data arrives, not on close', () async { + final file = IndexedDbFile('streamed.file', _db); + final sink = file.openWrite(); + sink.add(_pattern(_chunk * 2 + 10)); + await sink.flush(); + + // Two full chunks are stored before close; the tail is still buffered + // and the file is not visible yet. + expect(await countChunks(_db, 'streamed.file'), 2); + expect(await file.exists(), isFalse); + + await sink.close(); + expect(await countChunks(_db, 'streamed.file'), 3); + expect(await file.exists(), isTrue); + expect(await file.length(), _chunk * 2 + 10); + }); + + test('openRead streams chunk by chunk and honours start and end', () async { + final data = _pattern(_chunk * 3 + 5); + final file = IndexedDbFile('ranges.file', _db); + await file.writeAsBytes(data); + + final pieces = await file.openRead().toList(); + expect(pieces, hasLength(4)); + expect(pieces.expand((e) => e).toList(), data); + + const start = _chunk - 3; + const end = _chunk * 2 + 4; + final range = await file.openRead(start, end).expand((e) => e).toList(); + expect(range, data.sublist(start, end)); + + expect(await file.openRead(_chunk * 3 + 5).toList(), isEmpty); + }); + + test('exists and length do not read the contents, and a missing file ' + 'reports as such', () async { + final file = IndexedDbFile('missing.file', _db); + expect(await file.exists(), isFalse); + await expectLater(file.length(), throwsA(isA())); + await expectLater(file.readAsBytes(), throwsA(isA())); + }); + + test('delete removes the header and every chunk', () async { + final file = IndexedDbFile('deleted.file', _db); + await file.writeAsBytes(_pattern(_chunk * 2 + 1)); + expect(await countChunks(_db, 'deleted.file'), 3); + + await file.delete(); + expect(await file.exists(), isFalse); + expect(await countChunks(_db, 'deleted.file'), 0); + }); + + test('delete also removes the chunks of an unfinished write', () async { + final file = IndexedDbFile('partial.file', _db); + final sink = file.openWrite()..add(_pattern(_chunk + 1)); + await sink.flush(); + expect(await countChunks(_db, 'partial.file'), 1); + + await file.delete(); + expect(await countChunks(_db, 'partial.file'), 0); + }); + + test('writing again replaces the previous contents', () async { + final file = IndexedDbFile('rewrite.file', _db); + await file.writeAsBytes(_pattern(_chunk * 2)); + await file.writeAsBytes([1, 2, 3]); + + expect(await file.readAsBytes(), [1, 2, 3]); + expect(await countChunks(_db, 'rewrite.file'), 1); + }); + + test('append mode continues after a partial last chunk', () async { + final first = _pattern(_chunk + 3); + final second = _pattern(_chunk); + final file = IndexedDbFile('append.file', _db); + await file.writeAsBytes(first); + await file.writeAsBytes(second, mode: FileMode.append); + + expect(await file.readAsBytes(), [...first, ...second]); + }); + + test('write and writeAll encode text into the file', () async { + final file = IndexedDbFile('text.file', _db); + final sink = file.openWrite() + ..write('ab') + ..writeAll(['c', 'd'], '-') + ..add([0x21]); + await sink.close(); + + expect(await file.readAsString(), 'abc-d!'); + }); + + test('files written by the previous single-record layout stay ' + 'readable', () async { + final data = _pattern(1000); + // Opens the database at the current version, then stores a record the + // way schema version 2 did. + await IndexedDbFile('other', _db).exists(); + await putRecord( + _db, + IndexedDbSchema.filesStore, + {'path': 'legacy.file', 'data': data}.jsify()!, + ); + + final file = IndexedDbFile('legacy.file', _db); + expect(await file.exists(), isTrue); + expect(await file.length(), 1000); + expect(await file.readAsBytes(), data); + expect( + await file.openRead(10, 20).expand((e) => e).toList(), + data.sublist(10, 20), + ); + await file.delete(); + expect(await file.exists(), isFalse); + }); + + test('the pool closes the connection when the last holder releases ' + 'it', () async { + final pool = IndexedDbConnectionPool.getInstance(databaseName: _db); + await pool.acquire(); + await pool.acquire(); + pool.release(); + expect(pool.isOpen, isTrue); + pool.release(); + expect(pool.isOpen, isFalse); + + // Files reopen it on demand. + expect(await IndexedDbFile('x', _db).exists(), isFalse); + expect(pool.isOpen, isTrue); + }); +} diff --git a/flutter_cache_manager/test/web/indexed_db_test_helpers.dart b/flutter_cache_manager/test/web/indexed_db_test_helpers.dart new file mode 100644 index 00000000..9e55c649 --- /dev/null +++ b/flutter_cache_manager/test/web/indexed_db_test_helpers.dart @@ -0,0 +1,59 @@ +import 'dart:async'; +import 'dart:js_interop'; + +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_connection_pool.dart'; +import 'package:flutter_cache_manager/src/storage/cache_info_repositories/indexed_db_schema.dart'; +import 'package:web/web.dart' as web; + +/// Closes the pooled connection and deletes [name]. +Future deleteDatabase(String name) { + IndexedDbConnectionPool.removeInstance(name); + final completer = Completer(); + final request = web.window.indexedDB.deleteDatabase(name); + request.onsuccess = (web.Event _) { + completer.complete(); + }.toJS; + request.onerror = (web.Event _) { + completer.completeError(StateError('delete failed: ${request.error}')); + }.toJS; + return completer.future; +} + +Future _db(String name) => + IndexedDbConnectionPool.getInstance(databaseName: name).getDatabase(); + +/// The number of chunk records stored for [path]. +Future countChunks(String db, String path) async { + final transaction = createIndexedDbTransaction(await _db(db), [ + IndexedDbSchema.chunksStore, + ], 'readonly'); + final count = await indexedDbRequest( + transaction + .objectStore(IndexedDbSchema.chunksStore) + .count(IndexedDbSchema.chunksOf(path)), + 'count', + ); + return (count as JSNumber).toDartInt; +} + +/// The number of records in [store]. +Future countRecords(String db, String store) async { + final transaction = createIndexedDbTransaction(await _db(db), [ + store, + ], 'readonly'); + final count = await indexedDbRequest( + transaction.objectStore(store).count(), + 'count', + ); + return (count as JSNumber).toDartInt; +} + +/// Stores [value] in [store] directly. +Future putRecord(String db, String store, JSAny value) async { + final transaction = createIndexedDbTransaction(await _db(db), [ + store, + ], 'readwrite'); + final done = indexedDbTransactionDone(transaction, 'put'); + transaction.objectStore(store).put(value); + await done; +}