From e029e3ac04ee1f66be4da3d9144db01d01d32049 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Sat, 12 Sep 2026 23:39:07 +0400 Subject: [PATCH 1/2] feat: add a version registry that keeps release lists between runs fix: request every releases page once with `per_page=100` fix: stop `destroy()` from loading the remaining release pages Every run asked GitHub or GitLab for the release list and spent the API rate limit on it. Releases are now kept in a local version registry: a provider-neutral database of the releases and assets of every known repository, one JSON file per repository, on by default in the per-user cache directory. Versions never expire; only the last check of a repository has a TTL (`cache-ttl`, default 600 s), within which `dload get` costs no API request. A stale check fetches only the newest pages until a stored release is reached. Pages stay lazy: the first run loads what the requested version needs, older releases are fetched on demand and appended. A failed check falls back to the stored releases. `dload get --refresh` ignores the TTL once, `dload cache:clear [software...]` drops records. The page loader used to build a paginator per page and probe the next one, so every page but the first was requested twice; `destroy()` iterated the whole lazy collection and loaded every remaining page after each download. Co-Authored-By: Dmitriy Derepko Assisted-By: Claude Fable 5.1 --- README-es.md | 65 ++++++ README-ru.md | 66 ++++++ README-zh.md | 59 +++++ README.md | 68 ++++++ bin/dload | 1 + dload.xsd | 10 + src/Bootstrap.php | 34 +++ src/Command/CacheClear.php | 92 ++++++++ src/Command/Get.php | 9 + src/Module/Config/Schema/Cache.php | 43 ++++ src/Module/Downloader/Downloader.php | 6 + .../Registry/Internal/CacheDirectory.php | 57 +++++ .../Registry/Internal/FileRegistryStorage.php | 148 ++++++++++++ .../Registry/Internal/PassThroughRegistry.php | 32 +++ .../Internal/StoredVersionRegistry.php | 178 ++++++++++++++ src/Module/Registry/Record/AssetRecord.php | 69 ++++++ src/Module/Registry/Record/ReleasePage.php | 23 ++ src/Module/Registry/Record/ReleaseRecord.php | 77 ++++++ .../Registry/Record/RepositoryRecord.php | 213 +++++++++++++++++ src/Module/Registry/RegistryStorage.php | 46 ++++ src/Module/Registry/ReleaseSource.php | 29 +++ src/Module/Registry/RepositoryId.php | 48 ++++ src/Module/Registry/VersionRegistry.php | 44 ++++ .../Repository/Internal/CachedGenerator.php | 10 + src/Module/Repository/Internal/Collection.php | 12 + .../Internal/GitHub/Api/RepositoryApi.php | 138 +++++------ .../GitHub/Api/Response/AssetInfo.php | 15 ++ .../GitHub/Api/Response/ReleaseInfo.php | 17 ++ .../Repository/Internal/GitHub/Factory.php | 4 +- .../Internal/GitHub/GitHubAsset.php | 8 +- .../Internal/GitHub/GitHubRelease.php | 19 +- .../Internal/GitHub/GitHubReleaseSource.php | 34 +++ .../Internal/GitHub/GitHubRepository.php | 52 +++-- .../Internal/GitLab/Api/RepositoryApi.php | 142 +++++------ .../GitLab/Api/Response/AssetInfo.php | 10 + .../GitLab/Api/Response/ReleaseInfo.php | 17 ++ .../Repository/Internal/GitLab/Factory.php | 4 +- .../Internal/GitLab/GitLabAsset.php | 8 +- .../Internal/GitLab/GitLabRelease.php | 19 +- .../Internal/GitLab/GitLabReleaseSource.php | 34 +++ .../Internal/GitLab/GitLabRepository.php | 54 +++-- tests/Acceptance/DLoadTest.php | 6 +- .../Registry/VersionRegistryBindingTest.php | 92 ++++++++ .../Module/Registry/CacheDirectoryTest.php | 36 +++ .../Registry/FileRegistryStorageTest.php | 116 +++++++++ .../Module/Registry/RepositoryRecordTest.php | 140 +++++++++++ .../Registry/StoredVersionRegistryTest.php | 220 ++++++++++++++++++ .../Registry/Stub/ArrayReleaseSource.php | 81 +++++++ .../Registry/Stub/InMemoryRegistryStorage.php | 52 +++++ .../Internal/GitHub/GitHubRepositoryTest.php | 152 ++++++++++++ .../Internal/GitHub/Stub/PagedClientStub.php | 105 +++++++++ .../Internal/GitLab/FactoryTest.php | 8 +- .../Internal/GitLab/GitLabRepositoryTest.php | 114 +++++++++ .../Internal/GitLab/Stub/PagedClientStub.php | 106 +++++++++ 54 files changed, 3043 insertions(+), 199 deletions(-) create mode 100644 src/Command/CacheClear.php create mode 100644 src/Module/Config/Schema/Cache.php create mode 100644 src/Module/Registry/Internal/CacheDirectory.php create mode 100644 src/Module/Registry/Internal/FileRegistryStorage.php create mode 100644 src/Module/Registry/Internal/PassThroughRegistry.php create mode 100644 src/Module/Registry/Internal/StoredVersionRegistry.php create mode 100644 src/Module/Registry/Record/AssetRecord.php create mode 100644 src/Module/Registry/Record/ReleasePage.php create mode 100644 src/Module/Registry/Record/ReleaseRecord.php create mode 100644 src/Module/Registry/Record/RepositoryRecord.php create mode 100644 src/Module/Registry/RegistryStorage.php create mode 100644 src/Module/Registry/ReleaseSource.php create mode 100644 src/Module/Registry/RepositoryId.php create mode 100644 src/Module/Registry/VersionRegistry.php create mode 100644 src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php create mode 100644 src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php create mode 100644 tests/Integration/Module/Registry/VersionRegistryBindingTest.php create mode 100644 tests/Unit/Module/Registry/CacheDirectoryTest.php create mode 100644 tests/Unit/Module/Registry/FileRegistryStorageTest.php create mode 100644 tests/Unit/Module/Registry/RepositoryRecordTest.php create mode 100644 tests/Unit/Module/Registry/StoredVersionRegistryTest.php create mode 100644 tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php create mode 100644 tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php create mode 100644 tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php create mode 100644 tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php create mode 100644 tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php create mode 100644 tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php diff --git a/README-es.md b/README-es.md index 47426d0..9c9fc35 100644 --- a/README-es.md +++ b/README-es.md @@ -48,6 +48,7 @@ Con DLoad puedes: - [Tipos de Descarga](#tipos-de-descarga) - [Restricciones de Versión](#restricciones-de-versión) - [Opciones de Configuración Avanzadas](#opciones-de-configuración-avanzadas) + - [Registro de Versiones](#registro-de-versiones) - [Construir RoadRunner Personalizado](#construir-roadrunner-personalizado) - [Configuración de Acción de Construcción](#configuración-de-acción-de-construcción) - [Atributos de Acción Velox](#atributos-de-acción-velox) @@ -348,6 +349,70 @@ Usa restricciones de versión estilo Composer: ``` +### Registro de Versiones + +Resolver una versión significa pedir a GitHub o GitLab la lista de releases del repositorio. DLoad +guarda lo que aprende en un **registro de versiones** local: una pequeña base de datos con los releases +y assets de cada repositorio conocido, un archivo JSON por repositorio. Las versiones nunca expiran. +Lo que expira es la *última comprobación* del repositorio: mientras sea más reciente que `cache-ttl`, +`dload get` se responde desde el registro sin una sola petición a la API. Cuando es más antigua, DLoad +pide a la API solo los releases publicados desde entonces, normalmente una única petición. + +Las páginas de releases se siguen cargando de forma perezosa. La primera ejecución obtiene solo las +páginas necesarias para encontrar un release que cumpla la versión pedida; los releases más antiguos se +cargan después, bajo demanda. + +El registro está activado por defecto y vive en el directorio de caché del usuario +(`$XDG_CACHE_HOME/dload`, `%LOCALAPPDATA%\dload\cache` en Windows, `~/.cache/dload` en otros casos): + +```xml + + + + + +``` + +| Atributo | Variable de entorno | Por defecto | Significado | +|-------------|---------------------|--------------------------------|--------------------------------------------------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | directorio de caché del usuario | Directorio del registro de versiones. | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Segundos que sigue siendo válida la última comprobación. `0` desactiva el registro. | + +```bash +# Comprobar si hay nuevos releases aunque la última comprobación siga vigente +./vendor/bin/dload get rr --refresh + +# Olvidar los repositorios de un software, o todo el registro +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear +``` + +> [!NOTE] +> El registro solo contiene metadatos de releases: tags, nombres y enlaces de descarga. Las descargas +> no pasan por él y nunca guarda credenciales, así que el directorio puede compartirse o guardarse en +> la caché de CI sin problemas. Si una comprobación falla por un error de red o un límite de la API, se +> usan los releases almacenados; un repositorio nunca visto sigue fallando de forma visible. + +En GitHub Actions el directorio puede conservarse entre ejecuciones del workflow, de modo que cada +ejecución gasta el límite de la API solo en los releases publicados desde la anterior: + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +El `github.run_id` en la clave hace que cada ejecución guarde su registro, y `restore-keys` permite +que la siguiente parta del más reciente. Los jobs paralelos de un mismo workflow no ven la caché de los +demás, ya que `actions/cache` la guarda al terminar cada job. + ## Construir RoadRunner Personalizado DLoad soporta la construcción de binarios personalizados de RoadRunner usando la herramienta Velox. Esto es útil cuando necesitas RoadRunner con combinaciones específicas de plugins que no están disponibles en las versiones pre-construidas. diff --git a/README-ru.md b/README-ru.md index eea812f..b9a21fe 100644 --- a/README-ru.md +++ b/README-ru.md @@ -49,6 +49,7 @@ DLoad решает распространённую проблему в PHP-пр - [Типы загрузки](#типы-загрузки) - [Ограничения версий](#ограничения-версий) - [Расширенные настройки](#расширенные-настройки) + - [Реестр версий](#реестр-версий) - [Сборка кастомного RoadRunner](#сборка-кастомного-roadrunner) - [Настройка действия сборки](#настройка-действия-сборки) - [Атрибуты Velox-действия](#атрибуты-velox-действия) @@ -349,6 +350,71 @@ DLoad поддерживает три типа загрузки, которые ``` +### Реестр версий + +Чтобы определить версию, DLoad запрашивает у GitHub или GitLab список релизов репозитория. Всё, +что он узнаёт, сохраняется в локальном **реестре версий**: небольшой базе релизов и ассетов каждого +известного репозитория, по одному JSON-файлу на репозиторий. Версии из реестра не устаревают. +Устаревает только *последняя проверка* репозитория: пока она моложе `cache-ttl`, `dload get` +отвечает из реестра без единого запроса к API. Когда проверка устарела, DLoad запрашивает у API +только релизы, вышедшие после неё, и обычно это один запрос. + +Страницы релизов по-прежнему загружаются лениво. Первый запуск получает столько страниц, сколько +нужно, чтобы найти релиз под запрошенную версию, а более старые релизы догружаются позже, по мере +надобности. + +Реестр включён по умолчанию и живёт в пользовательском каталоге кэша (`$XDG_CACHE_HOME/dload`, +`%LOCALAPPDATA%\dload\cache` в Windows, иначе `~/.cache/dload`): + +```xml + + + + + +``` + +| Атрибут | Переменная окружения | По умолчанию | Значение | +|-------------|----------------------|---------------------------|---------------------------------------------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | каталог кэша пользователя | Каталог реестра версий. | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Сколько секунд действует последняя проверка репозитория. `0` отключает реестр. | + +```bash +# Проверить репозитории на новые релизы, даже если последняя проверка ещё свежая +./vendor/bin/dload get rr --refresh + +# Забыть репозитории, из которых берётся программа, или весь реестр целиком +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear +``` + +> [!NOTE] +> В реестре хранятся только метаданные релизов: теги, имена и ссылки на ассеты. Загрузки через него +> не проходят, учётные данные в нём не сохраняются, поэтому каталог можно свободно передавать между +> машинами и складывать в кэш CI. Если проверка не удалась из-за сетевой ошибки или лимита API, +> используются сохранённые релизы, а репозиторий, который раньше не встречался, по-прежнему +> завершится ошибкой. + +В GitHub Actions каталог можно переносить между запусками workflow, тогда запуск тратит лимит API +только на релизы, вышедшие после предыдущего: + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +`github.run_id` в ключе заставляет каждый запуск сохранять свой реестр, а `restore-keys` позволяет +следующему запуску начать с самого свежего. Параллельные джобы одного workflow кэш друг друга не +видят: `actions/cache` сохраняет его по завершении джобы. + ## Сборка кастомного RoadRunner DLoad поддерживает сборку кастомных бинарников RoadRunner с помощью инструмента сборки Velox. Это полезно когда нужен RoadRunner с определёнными комбинациями плагинов, которые недоступны в готовых релизах. diff --git a/README-zh.md b/README-zh.md index cd65fff..2b63606 100644 --- a/README-zh.md +++ b/README-zh.md @@ -48,6 +48,7 @@ DLoad 解决了 PHP 项目中的一个实际问题：如何在分发 PHP 代码 - [下载类型](#下载类型) - [版本约束](#版本约束) - [高级配置选项](#高级配置选项) + - [版本注册表](#版本注册表) - [构建自定义 RoadRunner](#构建自定义-roadrunner) - [构建动作配置](#构建动作配置) - [Velox 动作属性](#velox-动作属性) @@ -348,6 +349,64 @@ DLoad 支持三种下载类型，它们决定了资源的处理方式： ``` +### 版本注册表 + +解析版本意味着向 GitHub 或 GitLab 请求仓库的发布列表。DLoad 会把获取到的信息保存在本地的 +**版本注册表**中：这是一个小型数据库，记录每个已知仓库的发布版本和资产，每个仓库一个 JSON 文件。 +其中的版本永不过期，过期的只是仓库的*最近一次检查*：只要检查时间比 `cache-ttl` 更新，`dload get` +就直接从注册表返回结果，不会发出任何 API 请求。检查过期后，DLoad 只向 API 请求此后发布的版本， +通常只需一次请求。 + +发布页面仍然按需加载。首次运行只获取找到满足所需版本的发布所需的页面，更早的发布会在之后真正需要时再加载。 + +注册表默认启用，位于用户缓存目录（`$XDG_CACHE_HOME/dload`，Windows 下为 `%LOCALAPPDATA%\dload\cache`， +其他情况为 `~/.cache/dload`）： + +```xml + + + + + +``` + +| 属性 | 环境变量 | 默认值 | 含义 | +|-------------|--------------------|--------------|----------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | 用户缓存目录 | 版本注册表所在目录。 | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | 最近一次检查保持有效的秒数。`0` 表示禁用注册表。 | + +```bash +# 即使最近一次检查仍然有效，也强制检查仓库是否有新发布 +./vendor/bin/dload get rr --refresh + +# 忘记某个软件所使用的仓库，或清空整个注册表 +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear +``` + +> [!NOTE] +> 注册表只保存发布的元数据：标签、名称和资产下载链接。下载不会经过注册表，也不会保存任何凭据， +> 因此该目录可以自由共享或放入 CI 缓存。若因网络错误或 API 速率限制导致检查失败，会使用已保存的发布； +> 从未见过的仓库仍会明确报错。 + +在 GitHub Actions 中可以在多次工作流运行之间保留该目录，这样每次运行只为上次运行之后发布的版本消耗速率限制： + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +键中的 `github.run_id` 使每次运行都保存自己的注册表，而 `restore-keys` 让下一次运行从最新的注册表开始。 +同一工作流中并行运行的作业彼此看不到缓存，因为 `actions/cache` 在作业结束时才保存缓存。 + ## 构建自定义 RoadRunner DLoad 支持使用 Velox 构建工具来构建自定义 RoadRunner 二进制文件。当你需要包含特定插件组合的 RoadRunner，而这些组合在预构建版本中不可用时，这功能就很有用了。 diff --git a/README.md b/README.md index 628c879..92f62e3 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ With DLoad, you can: - [Download Types](#download-types) - [Version Constraints](#version-constraints) - [Advanced Configuration Options](#advanced-configuration-options) + - [Version Registry](#version-registry) - [Building Custom RoadRunner](#building-custom-roadrunner) - [Build Action Configuration](#build-action-configuration) - [Velox Action Attributes](#velox-action-attributes) @@ -350,6 +351,70 @@ Use Composer-style version constraints: ``` +### Version Registry + +Resolving a version means asking GitHub or GitLab for the repository's release list. DLoad keeps +what it learns in a local **version registry**: a small database of the releases and assets every +known repository offers, one JSON file per repository. Versions never expire from it. What expires +is the *last check* of a repository: while the check is younger than `cache-ttl`, `dload get` is +answered from the registry without a single API request. When it is older, DLoad asks the API only +for the releases published since the last check, which is usually one request. + +Release pages are still loaded lazily. The first run fetches only as many pages as it takes to find +a release that satisfies the requested version, and older releases are fetched later, on demand, +when a run actually needs one of them. + +The registry is on by default and lives in the per-user cache directory (`$XDG_CACHE_HOME/dload`, +`%LOCALAPPDATA%\dload\cache` on Windows, `~/.cache/dload` otherwise): + +```xml + + + + + +``` + +| Attribute | Environment variable | Default | Meaning | +|-------------|----------------------|----------------------|------------------------------------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | user cache directory | Directory of the version registry. | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Seconds the last check of a repository stays valid. `0` disables the registry. | + +```bash +# Check the repositories for new releases even if the last check is still fresh +./vendor/bin/dload get rr --refresh + +# Forget the repositories a software package is served from, or the whole registry +./vendor/bin/dload cache:clear rr +./vendor/bin/dload cache:clear +``` + +> [!NOTE] +> The registry holds release metadata only: tags, names and asset download links. Downloads never +> go through it and credentials are never stored in it, so the directory can be shared or committed +> to a CI cache freely. When a check fails because of a network error or a rate limit, the stored +> releases are used instead, and a repository that was never seen before still fails loudly. + +In GitHub Actions the directory can be carried between workflow runs, so a run spends the rate limit +only on releases published since the previous one: + +```yaml +- name: Restore DLoad version registry + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-registry-${{ github.run_id }} + restore-keys: dload-registry- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +The `github.run_id` in the key makes every workflow run save its registry, while `restore-keys` +lets the next run start from the most recent one. Jobs that run in parallel within one workflow do not +see each other's cache, since `actions/cache` saves it when a job ends. + ## Building Custom RoadRunner DLoad supports building custom RoadRunner binaries using the Velox build tool. This is useful when you need RoadRunner with custom plugin combinations that aren't available in pre-built releases. @@ -600,6 +665,9 @@ Add to CI/CD environment variables for automated downloads. > 1,000 requests per hour across all jobs of the repository. With a large job matrix the limit may run out, > and downloads from other repositories may be rejected. Use a personal access token if that happens. +Release lists are also kept in a local version registry, so repeated runs and runs that carry the +registry between them spend the rate limit only on new releases: see [Version Registry](#version-registry). + ## Failure Reporting `dload get` exits with a non-zero code when at least one requested package was not installed, and prints diff --git a/bin/dload b/bin/dload index 053b8e8..612ec56 100755 --- a/bin/dload +++ b/bin/dload @@ -51,6 +51,7 @@ use Symfony\Component\Console\CommandLoader\FactoryCommandLoader; Command\Show::getCommandName() => static fn() => new Command\Show(), Command\Init::getCommandName() => static fn() => new Command\Init(), Command\Build::getCommandName() => static fn() => new Command\Build(), + Command\CacheClear::getCommandName() => static fn() => new Command\CacheClear(), ]), ); $application->setDefaultCommand(Command\Get::getCommandName(), false); diff --git a/dload.xsd b/dload.xsd index 793255e..3246f4f 100644 --- a/dload.xsd +++ b/dload.xsd @@ -249,6 +249,16 @@ Temporary directory for downloads + + + Directory of the version registry (release lists database); the per-user cache directory when not set + + + + + Number of seconds the last check of a repository for new releases stays valid; 0 disables the version registry + + diff --git a/src/Bootstrap.php b/src/Bootstrap.php index 887153a..bdbc579 100644 --- a/src/Bootstrap.php +++ b/src/Bootstrap.php @@ -12,8 +12,15 @@ use Internal\DLoad\Module\Common\Internal\Injection\ConfigInflector; use Internal\DLoad\Module\Common\OperatingSystem; use Internal\DLoad\Module\Common\Stability; +use Internal\DLoad\Module\Config\Schema\Cache as CacheConfig; use Internal\DLoad\Module\HttpClient\Factory; use Internal\DLoad\Module\HttpClient\Internal\NyholmFactoryImpl; +use Internal\DLoad\Module\Registry\Internal\CacheDirectory; +use Internal\DLoad\Module\Registry\Internal\FileRegistryStorage; +use Internal\DLoad\Module\Registry\Internal\PassThroughRegistry; +use Internal\DLoad\Module\Registry\Internal\StoredVersionRegistry; +use Internal\DLoad\Module\Registry\RegistryStorage; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Internal\GitHub\Factory as GithubRepositoryFactory; use Internal\DLoad\Module\Repository\Internal\GitLab\Factory as GitLabRepositoryFactory; use Internal\DLoad\Module\Repository\RepositoryProvider; @@ -21,6 +28,7 @@ use Internal\DLoad\Module\Velox\Builder; use Internal\DLoad\Module\Velox\Internal\Client\BuildRoadRunner; use Internal\DLoad\Module\Velox\Internal\VeloxBuilder; +use Internal\DLoad\Service\Logger; /** * Bootstraps the application by configuring the dependency container. @@ -113,6 +121,32 @@ public function withConfig( ->addRepositoryFactory($container->get(GithubRepositoryFactory::class)) ->addRepositoryFactory($container->get(GitLabRepositoryFactory::class)), ); + $this->container->bind( + RegistryStorage::class, + static function (Container $container) use ($environment): RegistryStorage { + $config = $container->get(CacheConfig::class); + + return new FileRegistryStorage( + $config->dir ?? CacheDirectory::resolve($environment), + $container->get(Logger::class), + ); + }, + ); + $this->container->bind( + VersionRegistry::class, + static function (Container $container): VersionRegistry { + $config = $container->get(CacheConfig::class); + + return $config->ttl <= 0 + ? new PassThroughRegistry() + : new StoredVersionRegistry( + $container->get(RegistryStorage::class), + $config->ttl, + $container->get(Logger::class), + $config->refresh, + ); + }, + ); $this->container->bind(BinaryProvider::class, BinaryProviderImpl::class); $this->container->bind(Factory::class, NyholmFactoryImpl::class); $this->container->bind(Builder::class, VeloxBuilder::class); diff --git a/src/Command/CacheClear.php b/src/Command/CacheClear.php new file mode 100644 index 0000000..c94d3b7 --- /dev/null +++ b/src/Command/CacheClear.php @@ -0,0 +1,92 @@ +addArgument( + self::ARG_SOFTWARE, + InputArgument::OPTIONAL | InputArgument::IS_ARRAY, + 'Software whose repositories must be forgotten, e.g. "rr", "dolt". Everything when omitted.', + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + parent::execute($input, $output); + + $storage = $this->container->get(RegistryStorage::class); + + /** @var list $software */ + $software = \array_values(\array_filter( + (array) $input->getArgument(self::ARG_SOFTWARE), + static fn(mixed $name): bool => \is_string($name) && $name !== '', + )); + + if ($software === []) { + $storage->clear(); + $output->writeln('The version registry has been cleared.'); + + return Command::SUCCESS; + } + + $removed = 0; + foreach ($this->recordsOf($storage, $software) as $record) { + $storage->remove($record->id); + $output->writeln(\sprintf('Forgot %s', OutputFormatter::escape((string) $record->id))); + ++$removed; + } + + $output->writeln(\sprintf('%d repository listing(s) removed.', $removed)); + + return Command::SUCCESS; + } + + /** + * @param list $software + * @return \Generator + */ + private function recordsOf(RegistryStorage $storage, array $software): \Generator + { + foreach ($storage->all() as $record) { + \array_intersect($record->software, $software) === [] or yield $record; + } + } +} diff --git a/src/Command/Get.php b/src/Command/Get.php index ac23e49..3beaecf 100644 --- a/src/Command/Get.php +++ b/src/Command/Get.php @@ -43,6 +43,9 @@ * * # Force download even if binary exists * ./vendor/bin/dload get rr --force + * + * # Check for new releases even if the version registry is still fresh + * ./vendor/bin/dload get rr --refresh * ``` * * @internal @@ -72,6 +75,12 @@ public function configure(): void $this->addOption('os', null, InputOption::VALUE_OPTIONAL, 'Operating system, e.g. "linux", "darwin" etc.'); $this->addOption('stability', null, InputOption::VALUE_OPTIONAL, 'Minimum stability, e.g. "rc", "beta" etc.'); $this->addOption('force', 'f', InputOption::VALUE_NONE, 'Force download even if binary exists'); + $this->addOption( + 'refresh', + null, + InputOption::VALUE_NONE, + 'Check repositories for new releases even if the version registry is still fresh', + ); } /** diff --git a/src/Module/Config/Schema/Cache.php b/src/Module/Config/Schema/Cache.php new file mode 100644 index 0000000..2fdcb22 --- /dev/null +++ b/src/Module/Config/Schema/Cache.php @@ -0,0 +1,43 @@ +repoConfig = \array_shift($repositories); $repository = $this->repositoryProvider->getByConfig($context->repoConfig); + + // The registry keeps track of which software is served from which repository + $this->registry->attach($context->software->getId(), RepositoryId::fromConfig($context->repoConfig)); $context->repositoryAttempt = $context->diagnostics->addRepository( type: $context->repoConfig->type, name: $repository->getName(), diff --git a/src/Module/Registry/Internal/CacheDirectory.php b/src/Module/Registry/Internal/CacheDirectory.php new file mode 100644 index 0000000..1b87644 --- /dev/null +++ b/src/Module/Registry/Internal/CacheDirectory.php @@ -0,0 +1,57 @@ + $env Environment variables. + * @return non-empty-string + */ + public static function resolve(array $env): string + { + $xdg = self::variable($env, 'XDG_CACHE_HOME'); + if ($xdg !== null) { + return $xdg . \DIRECTORY_SEPARATOR . 'dload'; + } + + $localAppData = self::variable($env, 'LOCALAPPDATA'); + if ($localAppData !== null && \DIRECTORY_SEPARATOR === '\\') { + return $localAppData . \DIRECTORY_SEPARATOR . 'dload' . \DIRECTORY_SEPARATOR . 'cache'; + } + + $home = self::variable($env, 'HOME') ?? self::variable($env, 'USERPROFILE'); + if ($home !== null) { + return $home . \DIRECTORY_SEPARATOR . '.cache' . \DIRECTORY_SEPARATOR . 'dload'; + } + + return \sys_get_temp_dir() . \DIRECTORY_SEPARATOR . 'dload-cache'; + } + + /** + * @param array $env + * @return non-empty-string|null + */ + private static function variable(array $env, string $name): ?string + { + $value = $env[$name] ?? null; + + return \is_string($value) && \trim($value) !== '' ? \rtrim($value, '/\\') : null; + } +} diff --git a/src/Module/Registry/Internal/FileRegistryStorage.php b/src/Module/Registry/Internal/FileRegistryStorage.php new file mode 100644 index 0000000..f9b2a1a --- /dev/null +++ b/src/Module/Registry/Internal/FileRegistryStorage.php @@ -0,0 +1,148 @@ +/repositories/github/roadrunner-server/roadrunner.json + * /repositories/gitlab/group/project.json + * ``` + * + * Files are written aside and renamed into place, so an interrupted or parallel run cannot + * leave a half-written record for anyone to read. + * + * @internal + * @psalm-internal Internal\DLoad + */ +final class FileRegistryStorage implements RegistryStorage +{ + private const REPOSITORIES_DIR = 'repositories'; + private const EXTENSION = '.json'; + + private readonly Path $root; + + public function __construct( + Path|string $directory, + private readonly Logger $logger, + ) { + $this->root = Path::create($directory)->join(self::REPOSITORIES_DIR); + } + + public function load(RepositoryId $id): ?RepositoryRecord + { + return $this->read($this->fileOf($id)); + } + + public function save(RepositoryRecord $record): void + { + $file = $this->fileOf($record->id); + $directory = $file->parent(); + + $directory->isDir() or FS::mkdir($directory); + + $payload = \json_encode($record->toArray(), \JSON_THROW_ON_ERROR | \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES); + + $temp = Path::create((string) $file . '.' . \getmypid() . '.tmp'); + @\file_put_contents((string) $temp, $payload) === false and throw new \RuntimeException( + \sprintf('Failed to write registry record `%s`.', $temp), + ); + + if (!FS::moveFile($temp, $file)) { + FS::removeFile($temp); + throw new \RuntimeException(\sprintf('Failed to store registry record `%s`.', $file)); + } + } + + public function all(): iterable + { + if (!$this->root->isDir()) { + return; + } + + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator((string) $this->root, \FilesystemIterator::SKIP_DOTS), + ); + + /** @var \SplFileInfo $file */ + foreach ($iterator as $file) { + if (!$file->isFile() || !\str_ends_with($file->getFilename(), self::EXTENSION)) { + continue; + } + + $record = $this->read(Path::create($file->getPathname())); + $record === null or yield $record; + } + } + + public function remove(RepositoryId $id): void + { + $file = $this->fileOf($id); + $file->isFile() and FS::removeFile($file); + } + + public function clear(): void + { + $this->root->isDir() and FS::removeDir($this->root); + } + + /** + * Keeps a path segment safe for every file system: anything but plain ASCII is replaced, + * and a segment that would otherwise be empty or a directory reference gets a placeholder. + * + * @return non-empty-string + */ + private static function sanitize(string $segment): string + { + $safe = (string) \preg_replace('/[^A-Za-z0-9._-]+/', '_', $segment); + + return $safe === '' || \trim($safe, '.') === '' ? '_' : $safe; + } + + /** + * Reads a record, or returns `null` when there is none or it cannot be used. + */ + private function read(Path $file): ?RepositoryRecord + { + if (!$file->isFile()) { + return null; + } + + try { + $content = @\file_get_contents((string) $file); + $content === false and throw new \RuntimeException(\sprintf('Failed to read registry record `%s`.', $file)); + + /** @var mixed $payload */ + $payload = \json_decode($content, true, 512, \JSON_THROW_ON_ERROR); + \is_array($payload) or throw new \UnexpectedValueException('Registry record must be a JSON object.'); + + return RepositoryRecord::fromArray($payload); + } catch (\Throwable $e) { + // A half-written, hand-edited or outdated record is not worth a failed download: + // report it and let the registry fetch the releases again. + $this->logger->exception($e, important: false); + + return null; + } + } + + private function fileOf(RepositoryId $id): Path + { + $segments = \array_map(self::sanitize(...), [$id->type, ...\explode('/', $id->uri)]); + $segments[\array_key_last($segments)] .= self::EXTENSION; + + return $this->root->join(...$segments); + } +} diff --git a/src/Module/Registry/Internal/PassThroughRegistry.php b/src/Module/Registry/Internal/PassThroughRegistry.php new file mode 100644 index 0000000..d6df289 --- /dev/null +++ b/src/Module/Registry/Internal/PassThroughRegistry.php @@ -0,0 +1,32 @@ +pages() as $page) { + yield $page->releases; + } + } + + public function attach(string $software, RepositoryId $id): void + { + // Nothing to record + } +} diff --git a/src/Module/Registry/Internal/StoredVersionRegistry.php b/src/Module/Registry/Internal/StoredVersionRegistry.php new file mode 100644 index 0000000..868a9c4 --- /dev/null +++ b/src/Module/Registry/Internal/StoredVersionRegistry.php @@ -0,0 +1,178 @@ + $ttl Seconds the last check stays valid. + * @param bool $refresh Ignore the TTL and check the source for every repository once. + * @param null|\Closure(): int $clock Current unix time; defaults to `time()`. + */ + public function __construct( + private readonly RegistryStorage $storage, + private readonly int $ttl, + private readonly Logger $logger, + private readonly bool $refresh = false, + ?\Closure $clock = null, + ) { + $this->clock = $clock ?? static fn(): int => \time(); + } + + public function releases(RepositoryId $id, ReleaseSource $source): \Generator + { + $record = $this->storage->load($id) ?? RepositoryRecord::empty($id); + + if ($this->refresh || $record->isStale(($this->clock)(), $this->ttl)) { + $record = $this->check($record, $source); + } else { + $this->logger->debug('Releases of `%s` are served from the version registry.', (string) $id); + } + + $stored = $record->releases(); + $stored === [] or yield $stored; + + if ($record->complete) { + return; + } + + // Older releases are loaded only when the consumer actually needs them + yield from $this->extend($record, $source); + } + + public function attach(string $software, RepositoryId $id): void + { + $record = $this->storage->load($id) ?? RepositoryRecord::empty($id); + $updated = $record->withSoftware($software); + + $updated === $record or $this->persist($updated); + } + + /** + * @param list $page + */ + private static function hasKnown(RepositoryRecord $record, array $page): bool + { + foreach ($page as $release) { + if ($record->has($release->tag)) { + return true; + } + } + + return false; + } + + /** + * Fetches the releases published since the last check and stores the result. + * + * @throws RepositoryException When the source fails and nothing is stored to fall back on. + */ + private function check(RepositoryRecord $record, ReleaseSource $source): RepositoryRecord + { + try { + $fetched = []; + $complete = $record->complete; + + foreach ($source->pages() as $page) { + $fetched = [...$fetched, ...$page->releases]; + + // The listing ended during the check: everything is known now + $page->last and $complete = true; + + // Reaching a known release means everything newer has been fetched. A record + // without releases cannot hit one, so its check is the first page only. + if ($page->last || $record->count() === 0 || self::hasKnown($record, $page->releases)) { + break; + } + } + + $updated = $record + ->withHead($fetched) + ->withComplete($complete) + ->withCheckedAt(($this->clock)()); + + $this->persist($updated); + + return $updated; + } catch (RepositoryException $e) { + $record->count() > 0 or throw $e; + + $this->logger->exception($e, important: false); + $this->logger->info( + 'Failed to check `%s` for new releases, %d stored release(s) are used instead.', + (string) $record->id, + $record->count(), + ); + + return $record; + } + } + + /** + * Loads the releases older than the stored ones page by page, persisting every page. + * + * @return \Generator, mixed, void> + * @throws RepositoryException + */ + private function extend(RepositoryRecord $record, ReleaseSource $source): \Generator + { + foreach ($source->pages($record->count()) as $page) { + $new = \array_values(\array_filter( + $page->releases, + static fn(ReleaseRecord $release): bool => !$record->has($release->tag), + )); + + $record = $record->withTail($new)->withComplete($page->last); + $this->persist($record); + + $new === [] or yield $new; + } + + $record->complete or $this->persist($record->withComplete(true)); + } + + /** + * Stores the record; a storage failure is reported and swallowed. + */ + private function persist(RepositoryRecord $record): void + { + try { + $this->storage->save($record); + } catch (\Throwable $e) { + $this->logger->exception($e, important: false); + } + } +} diff --git a/src/Module/Registry/Record/AssetRecord.php b/src/Module/Registry/Record/AssetRecord.php new file mode 100644 index 0000000..0643f33 --- /dev/null +++ b/src/Module/Registry/Record/AssetRecord.php @@ -0,0 +1,69 @@ +|null $size Size in bytes when the source reports it. + * @param non-empty-string|null $contentType MIME type when the source reports it. + */ + public function __construct( + public readonly string $name, + public readonly string $uri, + public readonly ?int $size = null, + public readonly ?string $contentType = null, + ) {} + + /** + * @param array $data + * @throws \InvalidArgumentException When the array does not describe an asset. + */ + public static function fromArray(array $data): self + { + $name = $data['name'] ?? null; + $uri = $data['uri'] ?? null; + \is_string($name) && $name !== '' && \is_string($uri) && $uri !== '' or throw new \InvalidArgumentException( + 'Asset record requires non-empty `name` and `uri`.', + ); + + $size = $data['size'] ?? null; + $contentType = $data['content_type'] ?? null; + + return new self( + name: $name, + uri: $uri, + size: \is_int($size) && $size >= 0 ? $size : null, + contentType: \is_string($contentType) && $contentType !== '' ? $contentType : null, + ); + } + + /** + * @return AssetArray + */ + public function toArray(): array + { + $result = ['name' => $this->name, 'uri' => $this->uri]; + $this->size === null or $result['size'] = $this->size; + $this->contentType === null or $result['content_type'] = $this->contentType; + + return $result; + } +} diff --git a/src/Module/Registry/Record/ReleasePage.php b/src/Module/Registry/Record/ReleasePage.php new file mode 100644 index 0000000..64aa0db --- /dev/null +++ b/src/Module/Registry/Record/ReleasePage.php @@ -0,0 +1,23 @@ + $releases Releases of the page, newest first. + * @param bool $last Whether the listing has no page after this one. + */ + public function __construct( + public readonly array $releases, + public readonly bool $last, + ) {} +} diff --git a/src/Module/Registry/Record/ReleaseRecord.php b/src/Module/Registry/Record/ReleaseRecord.php new file mode 100644 index 0000000..4a01b28 --- /dev/null +++ b/src/Module/Registry/Record/ReleaseRecord.php @@ -0,0 +1,77 @@ +, + * } + */ +final class ReleaseRecord +{ + /** + * @param non-empty-string $tag Tag the release was made from; identifies the release within a repository. + * @param non-empty-string $name Human-readable release name. + * @param list $assets + */ + public function __construct( + public readonly string $tag, + public readonly string $name, + public readonly ?\DateTimeImmutable $publishedAt = null, + public readonly bool $prerelease = false, + public readonly array $assets = [], + ) {} + + /** + * @param array $data + * @throws \InvalidArgumentException When the array does not describe a release. + */ + public static function fromArray(array $data): self + { + $tag = $data['tag'] ?? null; + \is_string($tag) && $tag !== '' or throw new \InvalidArgumentException('Release record requires a non-empty `tag`.'); + + $name = $data['name'] ?? null; + \is_string($name) && $name !== '' or $name = $tag; + + $publishedAt = $data['published_at'] ?? null; + $assets = []; + foreach (\is_array($data['assets'] ?? null) ? $data['assets'] : [] as $asset) { + \is_array($asset) and $assets[] = AssetRecord::fromArray($asset); + } + + return new self( + tag: $tag, + name: $name, + publishedAt: \is_string($publishedAt) && $publishedAt !== '' ? new \DateTimeImmutable($publishedAt) : null, + prerelease: (bool) ($data['prerelease'] ?? false), + assets: $assets, + ); + } + + /** + * @return ReleaseArray + */ + public function toArray(): array + { + return [ + 'tag' => $this->tag, + 'name' => $this->name, + 'published_at' => $this->publishedAt?->format(\DateTimeInterface::ATOM), + 'prerelease' => $this->prerelease, + 'assets' => \array_map(static fn(AssetRecord $asset): array => $asset->toArray(), $this->assets), + ]; + } +} diff --git a/src/Module/Registry/Record/RepositoryRecord.php b/src/Module/Registry/Record/RepositoryRecord.php new file mode 100644 index 0000000..2ecc5a1 --- /dev/null +++ b/src/Module/Registry/Record/RepositoryRecord.php @@ -0,0 +1,213 @@ +, + * releases: list, + * } + */ +final class RepositoryRecord +{ + /** Format version of the stored payload; bump when the structure changes incompatibly. */ + public const FORMAT_VERSION = 1; + + /** @var array Releases keyed by tag, newest first. */ + private readonly array $releases; + + /** + * @param int|null $checkedAt Unix timestamp of the last successful check against the source. + * @param bool $complete Whether the stored releases reach the end of the source listing. + * @param list $software Identifiers of the software packages served from this repository. + * @param list $releases Releases newest first. + */ + public function __construct( + public readonly RepositoryId $id, + public readonly ?int $checkedAt = null, + public readonly bool $complete = false, + public readonly array $software = [], + array $releases = [], + ) { + $indexed = []; + foreach ($releases as $release) { + $indexed[$release->tag] ??= $release; + } + + $this->releases = $indexed; + } + + public static function empty(RepositoryId $id): self + { + return new self($id); + } + + /** + * @param array $data + * @throws \InvalidArgumentException When the array does not describe a repository record. + */ + public static function fromArray(array $data): self + { + ($data['version'] ?? null) === self::FORMAT_VERSION or throw new \InvalidArgumentException( + 'Unsupported repository record format.', + ); + + $repository = $data['repository'] ?? null; + $type = \is_array($repository) ? ($repository['type'] ?? null) : null; + $uri = \is_array($repository) ? ($repository['uri'] ?? null) : null; + \is_string($type) && $type !== '' && \is_string($uri) && $uri !== '' or throw new \InvalidArgumentException( + 'Repository record requires a repository type and URI.', + ); + + $releases = []; + foreach (\is_array($data['releases'] ?? null) ? $data['releases'] : [] as $release) { + \is_array($release) and $releases[] = ReleaseRecord::fromArray($release); + } + + /** @var list $software */ + $software = \array_values(\array_filter( + \is_array($data['software'] ?? null) ? $data['software'] : [], + static fn(mixed $name): bool => \is_string($name) && $name !== '', + )); + + $checkedAt = $data['checked_at'] ?? null; + + return new self( + id: new RepositoryId($type, $uri), + checkedAt: \is_int($checkedAt) ? $checkedAt : null, + complete: (bool) ($data['complete'] ?? false), + software: $software, + releases: $releases, + ); + } + + /** + * @return list Releases newest first. + */ + public function releases(): array + { + return \array_values($this->releases); + } + + /** + * @return int<0, max> + */ + public function count(): int + { + return \count($this->releases); + } + + /** + * @param non-empty-string $tag + */ + public function has(string $tag): bool + { + return isset($this->releases[$tag]); + } + + /** + * Whether the last check is older than the given number of seconds, or never happened. + * + * @param int<0, max> $ttl + */ + public function isStale(int $now, int $ttl): bool + { + return $this->checkedAt === null || $now - $this->checkedAt > $ttl; + } + + /** + * Replaces the head of the list with freshly fetched releases. + * + * The fetched releases are the newest ones; they overwrite the stored entries with the same + * tags (assets may have been attached after the release was created) and the remaining stored + * releases follow them, so the list stays newest first. + * + * @param list $fetched Newest first. + */ + public function withHead(array $fetched): self + { + return $this->with(releases: [...$fetched, ...$this->releases()]); + } + + /** + * Appends older releases loaded on demand; already known tags are ignored. + * + * @param list $fetched + */ + public function withTail(array $fetched): self + { + return $this->with(releases: [...$this->releases(), ...$fetched]); + } + + public function withCheckedAt(int $checkedAt): self + { + return $this->with(checkedAt: $checkedAt); + } + + public function withComplete(bool $complete): self + { + return $this->with(complete: $complete); + } + + /** + * @param non-empty-string $software + */ + public function withSoftware(string $software): self + { + return \in_array($software, $this->software, true) + ? $this + : $this->with(software: [...$this->software, $software]); + } + + /** + * @return RepositoryArray + */ + public function toArray(): array + { + return [ + 'version' => self::FORMAT_VERSION, + 'repository' => ['type' => $this->id->type, 'uri' => $this->id->uri], + 'checked_at' => $this->checkedAt, + 'complete' => $this->complete, + 'software' => $this->software, + 'releases' => \array_map(static fn(ReleaseRecord $release): array => $release->toArray(), $this->releases()), + ]; + } + + /** + * @param list|null $software + * @param list|null $releases + */ + private function with( + ?int $checkedAt = null, + ?bool $complete = null, + ?array $software = null, + ?array $releases = null, + ): self { + return new self( + id: $this->id, + checkedAt: $checkedAt ?? $this->checkedAt, + complete: $complete ?? $this->complete, + software: $software ?? $this->software, + releases: $releases ?? $this->releases(), + ); + } +} diff --git a/src/Module/Registry/RegistryStorage.php b/src/Module/Registry/RegistryStorage.php new file mode 100644 index 0000000..f4d6077 --- /dev/null +++ b/src/Module/Registry/RegistryStorage.php @@ -0,0 +1,46 @@ + + */ + public function all(): iterable; + + /** + * Removes the record of a repository; a missing record is not an error. + */ + public function remove(RepositoryId $id): void; + + /** + * Removes every record. + */ + public function clear(): void; +} diff --git a/src/Module/Registry/ReleaseSource.php b/src/Module/Registry/ReleaseSource.php new file mode 100644 index 0000000..23954e3 --- /dev/null +++ b/src/Module/Registry/ReleaseSource.php @@ -0,0 +1,29 @@ + $offset Number of newest releases to skip. + * @return \Generator Pages of releases. + * @throws RepositoryException When a page cannot be loaded. + */ + public function pages(int $offset = 0): \Generator; +} diff --git a/src/Module/Registry/RepositoryId.php b/src/Module/Registry/RepositoryId.php new file mode 100644 index 0000000..8177676 --- /dev/null +++ b/src/Module/Registry/RepositoryId.php @@ -0,0 +1,48 @@ +type, $config->uri); + } + + public function equals(self $other): bool + { + return $this->type === $other->type && $this->uri === $other->uri; + } + + /** + * @return non-empty-string + */ + public function __toString(): string + { + return $this->type . ':' . $this->uri; + } +} diff --git a/src/Module/Registry/VersionRegistry.php b/src/Module/Registry/VersionRegistry.php new file mode 100644 index 0000000..986419a --- /dev/null +++ b/src/Module/Registry/VersionRegistry.php @@ -0,0 +1,44 @@ +releases($id, $source) as $page) { + * foreach ($page as $record) { + * // ... + * } + * } + * ``` + */ +interface VersionRegistry +{ + /** + * Lists the releases of a repository newest first, page by page. + * + * Older releases that are not in the database yet are loaded from the source only when the + * iteration reaches them, so a consumer that stops early costs no extra request. + * + * @return \Generator, mixed, void> + * @throws RepositoryException When releases cannot be obtained from either the database or the source. + */ + public function releases(RepositoryId $id, ReleaseSource $source): \Generator; + + /** + * Records that a software package is served from the repository. + * + * @param non-empty-string $software Software identifier. + */ + public function attach(string $software, RepositoryId $id): void; +} diff --git a/src/Module/Repository/Internal/CachedGenerator.php b/src/Module/Repository/Internal/CachedGenerator.php index b304174..2c2ca51 100644 --- a/src/Module/Repository/Internal/CachedGenerator.php +++ b/src/Module/Repository/Internal/CachedGenerator.php @@ -63,6 +63,16 @@ public function getIterator(): \Traversable goto start; } + /** + * Returns the items produced so far without pulling anything more from the generator. + * + * @return list + */ + public function loaded(): array + { + return \array_values($this->cache); + } + /** * Returns the first item in the cache or from the generator. * diff --git a/src/Module/Repository/Internal/Collection.php b/src/Module/Repository/Internal/Collection.php index b8033b0..32dbda0 100644 --- a/src/Module/Repository/Internal/Collection.php +++ b/src/Module/Repository/Internal/Collection.php @@ -94,6 +94,18 @@ public function filter(callable $filter): static return $clone; } + /** + * Returns the items loaded so far, ignoring filters and without loading anything more. + * + * Iterating a lazy collection may cost requests; releasing what has been loaded must not. + * + * @return list + */ + public function loaded(): array + { + return \is_array($this->items) ? \array_values($this->items) : $this->items->loaded(); + } + /** * Maps each item in the collection using the provided callback. * diff --git a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php index c3ce8fb..10f87ea 100644 --- a/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php @@ -6,11 +6,11 @@ use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; use Internal\DLoad\Module\HttpClient\Method; +use Internal\DLoad\Module\Registry\Record\ReleasePage; use Internal\DLoad\Module\Repository\Exception\ApiException; use Internal\DLoad\Module\Repository\Exception\RepositoryException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\RepositoryInfo; -use Internal\DLoad\Module\Repository\Internal\Paginator; use Internal\DLoad\Service\Logger; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\UriInterface; @@ -28,6 +28,12 @@ final class RepositoryApi private const URL_REPOSITORY = 'https://api.github.com/repos/%s'; private const URL_RELEASES = 'https://api.github.com/repos/%s/releases'; + /** + * Number of releases to ask for in a single page. GitHub serves 30 by default and allows up to + * 100, so the maximum keeps the release list within as few requests as the API permits. + */ + public const RELEASES_PER_PAGE = 100; + /** * @var non-empty-string */ @@ -79,74 +85,75 @@ public function getRepository(): RepositoryInfo } /** + * Lists releases newest first, page by page, starting from the given page. + * + * A page is requested only when the generator advances to it, so a consumer that stops early + * costs no extra request. + * * @param int<1, max> $page - * @return Paginator + * @return \Generator * @throws RepositoryException */ - public function getReleases(int $page = 1): Paginator + public function releasePages(int $page = 1): \Generator { - $pageLoader = function () use ($page): \Generator { - $currentPage = $page; - - do { - $response = $this->releasesRequest($currentPage); - - /** @var list, - * prerelease: bool, - * draft: bool - * }> $data */ - $data = $this->decodeReleasesResponse($response); - - // If empty response, no more pages - if ($data === []) { - return; + $currentPage = $page; + + do { + $response = $this->releasesRequest($currentPage); + + /** @var list, + * prerelease: bool, + * draft: bool + * }> $data */ + $data = $this->decodeReleasesResponse($response); + + // If empty response, no more pages + if ($data === []) { + return; + } + + $releases = []; + $failure = null; + foreach ($data as $releaseData) { + try { + $releases[] = ReleaseInfo::fromApiResponse($releaseData)->toRecord(); + } catch (\Throwable $e) { + $failure ??= $e; + $this->logger->exception($e, important: false); + // Skip invalid releases + continue; } - - $releases = []; - $failure = null; - foreach ($data as $releaseData) { - try { - $releases[] = ReleaseInfo::fromApiResponse($releaseData); - } catch (\Throwable $e) { - $failure ??= $e; - $this->logger->exception($e, important: false); - // Skip invalid releases - continue; - } - } - - // The whole page is unreadable: the response structure is not what we expect - if ($releases === [] && $failure !== null) { - throw new ApiException( - \sprintf( - 'GitHub API returned %d release(s) for repository `%s`, but none of them could be read: %s', - \count($data), - $this->repositoryPath, - $failure->getMessage(), - ), + } + + // The whole page is unreadable: the response structure is not what we expect + if ($releases === [] && $failure !== null) { + throw new ApiException( + \sprintf( + 'GitHub API returned %d release(s) for repository `%s`, but none of them could be read: %s', + \count($data), $this->repositoryPath, - $failure, - ); - } + $failure->getMessage(), + ), + $this->repositoryPath, + $failure, + ); + } - yield $releases; + $hasMorePages = $this->hasNextPage($response); - // Check if there are more pages - $hasMorePages = $this->hasNextPage($response); - $currentPage++; - } while ($hasMorePages); - }; + yield new ReleasePage($releases, !$hasMorePages); - return Paginator::createFromGenerator($pageLoader(), null); + $currentPage++; + } while ($hasMorePages); } /** @@ -196,13 +203,12 @@ private function decodeReleasesResponse(ResponseInterface $response): array */ private function releasesRequest(int $page): ResponseInterface { - return $this->request( - Method::Get, - $this->httpFactory->uri( - \sprintf(self::URL_RELEASES, $this->repositoryPath), - ['page' => $page], - ), + $uri = $this->httpFactory->uri( + \sprintf(self::URL_RELEASES, $this->repositoryPath), + ['page' => $page, 'per_page' => self::RELEASES_PER_PAGE], ); + + return $this->request(Method::Get, $uri); } private function hasNextPage(ResponseInterface $response): bool diff --git a/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php b/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php index 43225da..af168f6 100644 --- a/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php +++ b/src/Module/Repository/Internal/GitHub/Api/Response/AssetInfo.php @@ -4,6 +4,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; + /** * GitHub Asset Data Transfer Object. * @@ -42,4 +44,17 @@ public static function fromApiResponse(array $data): self contentType: $data['content_type'], ); } + + /** + * Maps the asset into the provider-neutral registry record. + */ + public function toRecord(): AssetRecord + { + return new AssetRecord( + name: $this->name, + uri: $this->downloadUrl, + size: $this->size, + contentType: $this->contentType, + ); + } } diff --git a/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php b/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php index 6478ea4..e145b7d 100644 --- a/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php +++ b/src/Module/Repository/Internal/GitHub/Api/Response/ReleaseInfo.php @@ -4,6 +4,9 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; + /** * GitHub Release Data Transfer Object. * @@ -57,4 +60,18 @@ public static function fromApiResponse(array $data): self draft: $data['draft'], ); } + + /** + * Maps the release into the provider-neutral registry record. + */ + public function toRecord(): ReleaseRecord + { + return new ReleaseRecord( + tag: $this->tagName, + name: $this->name, + publishedAt: $this->publishedAt, + prerelease: $this->prerelease, + assets: \array_map(static fn(AssetInfo $asset): AssetRecord => $asset->toRecord(), $this->assets), + ); + } } diff --git a/src/Module/Repository/Internal/GitHub/Factory.php b/src/Module/Repository/Internal/GitHub/Factory.php index 77fd1c4..6773153 100644 --- a/src/Module/Repository/Internal/GitHub/Factory.php +++ b/src/Module/Repository/Internal/GitHub/Factory.php @@ -7,6 +7,7 @@ use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitHub; use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Client; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; use Internal\DLoad\Module\Repository\RepositoryFactory; @@ -30,6 +31,7 @@ public function __construct( private readonly HttpFactory $httpFactory, GitHub $gitHubConfig, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->gitHubClient = new Client( $httpFactory, @@ -50,7 +52,7 @@ public function create(RepositoryConfig $config): GitHubRepository $api = $this->createRepositoryApi($org, $repo); - return new GitHubRepository($api, $org, $repo, $this->logger); + return new GitHubRepository($api, $org, $repo, $this->logger, $this->registry); } /** diff --git a/src/Module/Repository/Internal/GitHub/GitHubAsset.php b/src/Module/Repository/Internal/GitHub/GitHubAsset.php index 020916c..41548f4 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubAsset.php +++ b/src/Module/Repository/Internal/GitHub/GitHubAsset.php @@ -9,8 +9,8 @@ use Internal\DLoad\Module\Common\OperatingSystem; use Internal\DLoad\Module\HttpClient\Method; use Internal\DLoad\Module\HttpClient\StreamReader; +use Internal\DLoad\Module\Registry\Record\AssetRecord; use Internal\DLoad\Module\Repository\Internal\Asset; -use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\AssetInfo; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Exception\RepositoryException; @@ -41,12 +41,12 @@ private function __construct( ); } - public static function fromDTO( + public static function fromRecord( RepositoryApi $api, GitHubRelease $release, - AssetInfo $dto, + AssetRecord $record, ): self { - return new self($api, $release, $dto->name, $dto->downloadUrl); + return new self($api, $release, $record->name, $record->uri); } /** diff --git a/src/Module/Repository/Internal/GitHub/GitHubRelease.php b/src/Module/Repository/Internal/GitHub/GitHubRelease.php index 8f0aef9..adffc49 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubRelease.php +++ b/src/Module/Repository/Internal/GitHub/GitHubRelease.php @@ -5,8 +5,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; use Internal\DLoad\Module\Repository\Collection\AssetsCollection; -use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Internal\Release; use Internal\DLoad\Module\Version\Version; @@ -30,17 +30,20 @@ private function __construct( parent::__construct($repository, $name, $version); } - public static function fromDTO( + /** + * @throws \InvalidArgumentException When the release tag is not a version. + */ + public static function fromRecord( RepositoryApi $api, GitHubRepository $repository, - ReleaseInfo $dto, + ReleaseRecord $record, ): self { - $version = Version::fromVersionString($dto->tagName); - $result = new self($repository, $dto->name, $version); + $version = Version::fromVersionString($record->tag); + $result = new self($repository, $record->name, $version); - $result->assets = AssetsCollection::create(static function () use ($api, $result, $dto): \Generator { - foreach ($dto->assets as $assetDTO) { - yield GitHubAsset::fromDTO($api, $result, $assetDTO); + $result->assets = AssetsCollection::create(static function () use ($api, $result, $record): \Generator { + foreach ($record->assets as $asset) { + yield GitHubAsset::fromRecord($api, $result, $asset); } }); diff --git a/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php b/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php new file mode 100644 index 0000000..572073d --- /dev/null +++ b/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php @@ -0,0 +1,34 @@ +api->releasePages(\intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1) as $page) { + yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last); + $skip = 0; + } + } +} diff --git a/src/Module/Repository/Internal/GitHub/GitHubRepository.php b/src/Module/Repository/Internal/GitHub/GitHubRepository.php index 2c4d9be..12a7fdf 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubRepository.php +++ b/src/Module/Repository/Internal/GitHub/GitHubRepository.php @@ -5,9 +5,12 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\RepositoryId; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Collection\ReleasesCollection; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; +use Internal\DLoad\Module\Repository\Internal\Paginator; use Internal\DLoad\Module\Repository\Repository; use Internal\DLoad\Service\Logger; @@ -19,6 +22,9 @@ */ final class GitHubRepository implements Repository, Destroyable { + /** Repository type identifier in the version registry. */ + public const TYPE = 'github'; + private ?ReleasesCollection $releases = null; /** @@ -37,13 +43,17 @@ public function __construct( string $org, string $repo, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->name = $org . '/' . $repo; } /** * Returns a lazily loaded collection of repository releases. - * Pages are loaded only when needed during iteration or filtering. + * + * Releases come from the version registry, which serves stored ones without a request and + * asks the API only for what it does not know yet. Pages are loaded only when needed during + * iteration or filtering. */ public function getReleases(): ReleasesCollection { @@ -53,32 +63,37 @@ public function getReleases(): ReleasesCollection // Create a generator function for lazy loading release pages $pageLoader = function (): \Generator { - $page = 0; + // to avoid first eager loading because of generator + yield []; + + $pages = $this->registry->releases( + new RepositoryId(self::TYPE, $this->name), + new GitHubReleaseSource($this->api), + ); $anyPageLoaded = false; - do { + while (true) { try { - // to avoid first eager loading because of generator - yield []; + // Advancing the generator is what requests the next page + $anyPageLoaded ? $pages->next() : $pages->rewind(); - $paginator = $this->api->getReleases(++$page); - $releases = $paginator->getPageItems(); + if (!$pages->valid()) { + return; + } $toYield = []; - foreach ($releases as $releaseDTO) { + foreach ($pages->current() as $record) { try { - $toYield[] = GitHubRelease::fromDTO($this->api, $this, $releaseDTO); + $toYield[] = GitHubRelease::fromRecord($this->api, $this, $record); } catch (\Throwable $e) { $this->logger->exception($e, important: false); // Skip invalid releases continue; } } - yield $toYield; - $anyPageLoaded = true; - // Check if there are more pages by getting next page - $hasMorePages = $paginator->getNextPage() !== null; + $anyPageLoaded = true; + yield $toYield; } catch (\Throwable $e) { # The first page is mandatory: when it fails, there is nothing to download and the reason # (invalid token, rate limit, missing repository, etc.) must reach the user. @@ -93,11 +108,11 @@ public function getReleases(): ReleasesCollection $this->logger->exception($e, important: false); return; } - } while ($hasMorePages); + } }; // Create paginator - $paginator = \Internal\DLoad\Module\Repository\Internal\Paginator::createFromGenerator($pageLoader(), null); + $paginator = Paginator::createFromGenerator($pageLoader(), null); // Create a collection with the paginator $this->releases = ReleasesCollection::create($paginator); @@ -112,9 +127,10 @@ public function getName(): string public function destroy(): void { - $this->releases === null or $this->releases->map( - static fn(object $release) => $release instanceof Destroyable and $release->destroy(), - ); + // Only what was loaded is released: iterating the collection would request the remaining pages + foreach ($this->releases?->loaded() ?? [] as $release) { + $release instanceof Destroyable and $release->destroy(); + } unset($this->releases); } diff --git a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php index 6af6c71..bdd6c9d 100644 --- a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php @@ -6,11 +6,11 @@ use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; use Internal\DLoad\Module\HttpClient\Method; +use Internal\DLoad\Module\Registry\Record\ReleasePage; use Internal\DLoad\Module\Repository\Exception\ApiException; use Internal\DLoad\Module\Repository\Exception\RepositoryException; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\RepositoryInfo; -use Internal\DLoad\Module\Repository\Internal\Paginator; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\UriInterface; @@ -28,6 +28,12 @@ final class RepositoryApi private const URL_RELEASES = 'https://gitlab.com/api/v4/projects/%s/releases'; private const URL_RELEASE_ASSET = 'https://gitlab.com/api/v4/projects/%s/releases/%s/downloads/%s'; + /** + * Number of releases to ask for in a single page. GitLab serves 20 by default and allows up to + * 100, so the maximum keeps the release list within as few requests as the API permits. + */ + public const RELEASES_PER_PAGE = 100; + /** * @var non-empty-string */ @@ -88,76 +94,77 @@ public function getRepository(): RepositoryInfo } /** + * Lists releases newest first, page by page, starting from the given page. + * + * A page is requested only when the generator advances to it, so a consumer that stops early + * costs no extra request. + * * @param int<1, max> $page - * @return Paginator + * @return \Generator * @throws RepositoryException */ - public function getReleases(int $page = 1): Paginator + public function releasePages(int $page = 1): \Generator { - $pageLoader = function () use ($page): \Generator { - $currentPage = $page; - - do { - $response = $this->releasesRequest($currentPage); - - /** @var list - * }, - * upcoming_release: bool - * }> $data */ - $data = $this->decodeReleasesResponse($response); - - // If empty response, no more pages - if ($data === []) { - return; + $currentPage = $page; + + do { + $response = $this->releasesRequest($currentPage); + + /** @var list + * }, + * upcoming_release: bool + * }> $data */ + $data = $this->decodeReleasesResponse($response); + + // If empty response, no more pages + if ($data === []) { + return; + } + + $releases = []; + $failure = null; + foreach ($data as $releaseData) { + try { + $releases[] = ReleaseInfo::fromApiResponse($releaseData)->toRecord(); + } catch (\Throwable $e) { + $failure ??= $e; + // Skip invalid releases + continue; } - - $releases = []; - $failure = null; - foreach ($data as $releaseData) { - try { - $releases[] = ReleaseInfo::fromApiResponse($releaseData); - } catch (\Throwable $e) { - $failure ??= $e; - // Skip invalid releases - continue; - } - } - - // The whole page is unreadable: the response structure is not what we expect - if ($releases === [] && $failure !== null) { - throw new ApiException( - \sprintf( - 'GitLab API returned %d release(s) for project `%s`, but none of them could be read: %s', - \count($data), - $this->repositoryPath, - $failure->getMessage(), - ), + } + + // The whole page is unreadable: the response structure is not what we expect + if ($releases === [] && $failure !== null) { + throw new ApiException( + \sprintf( + 'GitLab API returned %d release(s) for project `%s`, but none of them could be read: %s', + \count($data), $this->repositoryPath, - $failure, - ); - } + $failure->getMessage(), + ), + $this->repositoryPath, + $failure, + ); + } - yield $releases; + $hasMorePages = $this->hasNextPage($response); - // Check if there are more pages - $hasMorePages = $this->hasNextPage($response); - $currentPage++; - } while ($hasMorePages); - }; + yield new ReleasePage($releases, !$hasMorePages); - return Paginator::createFromGenerator($pageLoader(), null); + $currentPage++; + } while ($hasMorePages); } /** @@ -207,13 +214,12 @@ private function decodeReleasesResponse(ResponseInterface $response): array */ private function releasesRequest(int $page): ResponseInterface { - return $this->request( - Method::Get, - $this->httpFactory->uri( - \sprintf(self::URL_RELEASES, \urlencode($this->repositoryPath)), - ['page' => $page], - ), + $uri = $this->httpFactory->uri( + \sprintf(self::URL_RELEASES, \urlencode($this->repositoryPath)), + ['page' => $page, 'per_page' => self::RELEASES_PER_PAGE], ); + + return $this->request(Method::Get, $uri); } private function hasNextPage(ResponseInterface $response): bool diff --git a/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php b/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php index 46512ea..0c1a025 100644 --- a/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php +++ b/src/Module/Repository/Internal/GitLab/Api/Response/AssetInfo.php @@ -4,6 +4,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; + /** * GitLab Asset Data Transfer Object. * @@ -39,4 +41,12 @@ public static function fromApiResponse(array $data): self linkType: $data['link_type'] ?? null, ); } + + /** + * Maps the asset into the provider-neutral registry record. + */ + public function toRecord(): AssetRecord + { + return new AssetRecord(name: $this->name, uri: $this->downloadUrl); + } } diff --git a/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php b/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php index 07d6ee2..236a7da 100644 --- a/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php +++ b/src/Module/Repository/Internal/GitLab/Api/Response/ReleaseInfo.php @@ -4,6 +4,9 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response; +use Internal\DLoad\Module\Registry\Record\AssetRecord; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; + /** * GitLab Release Data Transfer Object. * @@ -58,4 +61,18 @@ public static function fromApiResponse(array $data): self prerelease: $data['upcoming_release'], ); } + + /** + * Maps the release into the provider-neutral registry record. + */ + public function toRecord(): ReleaseRecord + { + return new ReleaseRecord( + tag: $this->tagName, + name: $this->name, + publishedAt: $this->publishedAt, + prerelease: $this->prerelease, + assets: \array_map(static fn(AssetInfo $asset): AssetRecord => $asset->toRecord(), $this->assets), + ); + } } diff --git a/src/Module/Repository/Internal/GitLab/Factory.php b/src/Module/Repository/Internal/GitLab/Factory.php index 68d601d..2ccf644 100644 --- a/src/Module/Repository/Internal/GitLab/Factory.php +++ b/src/Module/Repository/Internal/GitLab/Factory.php @@ -7,6 +7,7 @@ use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitLab; use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Client; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; use Internal\DLoad\Module\Repository\RepositoryFactory; @@ -30,6 +31,7 @@ public function __construct( private readonly HttpFactory $httpFactory, GitLab $gitLabConfig, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->gitLabClient = new Client( $httpFactory, @@ -49,7 +51,7 @@ public function create(RepositoryConfig $config): GitLabRepository $uri = \is_string($path) && $path !== '' ? $path : $config->uri; $api = $this->createRepositoryApi($uri); - return new GitLabRepository($api, $uri, $this->logger); + return new GitLabRepository($api, $uri, $this->logger, $this->registry); } /** diff --git a/src/Module/Repository/Internal/GitLab/GitLabAsset.php b/src/Module/Repository/Internal/GitLab/GitLabAsset.php index 5b0f7a7..927c1b6 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabAsset.php +++ b/src/Module/Repository/Internal/GitLab/GitLabAsset.php @@ -8,8 +8,8 @@ use Internal\DLoad\Module\Common\Architecture; use Internal\DLoad\Module\Common\OperatingSystem; use Internal\DLoad\Module\HttpClient\StreamReader; +use Internal\DLoad\Module\Registry\Record\AssetRecord; use Internal\DLoad\Module\Repository\Internal\Asset; -use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\AssetInfo; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Exception\RepositoryException; @@ -40,12 +40,12 @@ private function __construct( ); } - public static function fromDTO( + public static function fromRecord( RepositoryApi $api, GitLabRelease $release, - AssetInfo $dto, + AssetRecord $record, ): self { - return new self($api, $release, $dto->name, $dto->downloadUrl); + return new self($api, $release, $record->name, $record->uri); } /** diff --git a/src/Module/Repository/Internal/GitLab/GitLabRelease.php b/src/Module/Repository/Internal/GitLab/GitLabRelease.php index f4bc854..8760d2c 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabRelease.php +++ b/src/Module/Repository/Internal/GitLab/GitLabRelease.php @@ -5,8 +5,8 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\Record\ReleaseRecord; use Internal\DLoad\Module\Repository\Collection\AssetsCollection; -use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Response\ReleaseInfo; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Internal\Release; use Internal\DLoad\Module\Version\Version; @@ -30,17 +30,20 @@ private function __construct( parent::__construct($repository, $name, $version); } - public static function fromDTO( + /** + * @throws \InvalidArgumentException When the release tag is not a version. + */ + public static function fromRecord( RepositoryApi $api, GitLabRepository $repository, - ReleaseInfo $dto, + ReleaseRecord $record, ): self { - $version = Version::fromVersionString($dto->tagName); - $result = new self($repository, $dto->name, $version); + $version = Version::fromVersionString($record->tag); + $result = new self($repository, $record->name, $version); - $result->assets = AssetsCollection::create(static function () use ($api, $result, $dto): \Generator { - foreach ($dto->assets as $assetDTO) { - yield GitLabAsset::fromDTO($api, $result, $assetDTO); + $result->assets = AssetsCollection::create(static function () use ($api, $result, $record): \Generator { + foreach ($record->assets as $asset) { + yield GitLabAsset::fromRecord($api, $result, $asset); } }); diff --git a/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php b/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php new file mode 100644 index 0000000..d07d9b6 --- /dev/null +++ b/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php @@ -0,0 +1,34 @@ +api->releasePages(\intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1) as $page) { + yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last); + $skip = 0; + } + } +} diff --git a/src/Module/Repository/Internal/GitLab/GitLabRepository.php b/src/Module/Repository/Internal/GitLab/GitLabRepository.php index 94f3dff..5fad466 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabRepository.php +++ b/src/Module/Repository/Internal/GitLab/GitLabRepository.php @@ -5,9 +5,12 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab; use Internal\Destroy\Destroyable; +use Internal\DLoad\Module\Registry\RepositoryId; +use Internal\DLoad\Module\Registry\VersionRegistry; use Internal\DLoad\Module\Repository\Collection\ReleasesCollection; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; +use Internal\DLoad\Module\Repository\Internal\Paginator; use Internal\DLoad\Module\Repository\Repository; use Internal\DLoad\Service\Logger; @@ -19,6 +22,9 @@ */ final class GitLabRepository implements Repository, Destroyable { + /** Repository type identifier in the version registry. */ + public const TYPE = 'gitlab'; + private ?ReleasesCollection $releases = null; /** @@ -35,13 +41,17 @@ public function __construct( private readonly RepositoryApi $api, string $projectPath, private readonly Logger $logger, + private readonly VersionRegistry $registry, ) { $this->name = $projectPath; } /** * Returns a lazily loaded collection of repository releases. - * Pages are loaded only when needed during iteration or filtering. + * + * Releases come from the version registry, which serves stored ones without a request and + * asks the API only for what it does not know yet. Pages are loaded only when needed during + * iteration or filtering. */ public function getReleases(): ReleasesCollection { @@ -51,38 +61,43 @@ public function getReleases(): ReleasesCollection // Create a generator function for lazy loading release pages $pageLoader = function (): \Generator { - $page = 0; + // to avoid first eager loading because of generator + yield []; + + $pages = $this->registry->releases( + new RepositoryId(self::TYPE, $this->name), + new GitLabReleaseSource($this->api), + ); $anyPageLoaded = false; - do { + while (true) { try { - // to avoid first eager loading because of generator - yield []; + // Advancing the generator is what requests the next page + $anyPageLoaded ? $pages->next() : $pages->rewind(); - $paginator = $this->api->getReleases(++$page); - $releases = $paginator->getPageItems(); + if (!$pages->valid()) { + return; + } $toYield = []; - foreach ($releases as $releaseDTO) { + foreach ($pages->current() as $record) { try { - $toYield[] = GitLabRelease::fromDTO($this->api, $this, $releaseDTO); + $toYield[] = GitLabRelease::fromRecord($this->api, $this, $record); } catch (\Throwable) { // Skip invalid releases continue; } } - yield $toYield; - $anyPageLoaded = true; - // Check if there are more pages by getting next page - $hasMorePages = $paginator->getNextPage() !== null; + $anyPageLoaded = true; + yield $toYield; } catch (\Throwable $e) { # The first page is mandatory: when it fails, there is nothing to download and the reason # (invalid token, rate limit, missing project, etc.) must reach the user. $anyPageLoaded or throw $e; # A rate limit leaves the release list incomplete: hiding it would produce a report - # that claims the project has nothing more, so it must reach the user as well. + # that claims the repository has nothing more, so it must reach the user as well. $e instanceof RateLimitException and throw $e; # Already loaded releases are enough to continue, so a failure of a subsequent page @@ -90,11 +105,11 @@ public function getReleases(): ReleasesCollection $this->logger->exception($e, important: false); return; } - } while ($hasMorePages); + } }; // Create paginator - $paginator = \Internal\DLoad\Module\Repository\Internal\Paginator::createFromGenerator($pageLoader(), null); + $paginator = Paginator::createFromGenerator($pageLoader(), null); // Create a collection with the paginator $this->releases = ReleasesCollection::create($paginator); @@ -109,9 +124,10 @@ public function getName(): string public function destroy(): void { - $this->releases === null or $this->releases->map( - static fn(object $release) => $release instanceof Destroyable and $release->destroy(), - ); + // Only what was loaded is released: iterating the collection would request the remaining pages + foreach ($this->releases?->loaded() ?? [] as $release) { + $release instanceof Destroyable and $release->destroy(); + } unset($this->releases); } diff --git a/tests/Acceptance/DLoadTest.php b/tests/Acceptance/DLoadTest.php index 4c7c19e..b22078f 100644 --- a/tests/Acceptance/DLoadTest.php +++ b/tests/Acceptance/DLoadTest.php @@ -231,8 +231,12 @@ protected function cleanup(): void */ private function buildDLoad(string $xmlConfig): DLoad { + // The version registry must not leak into the user's cache directory from a test run + $environment = \getenv() + ['DLOAD_CACHE_DIR' => (string) $this->testRuntimeDir->join('registry')]; + $environment['DLOAD_CACHE_DIR'] = (string) $this->testRuntimeDir->join('registry'); + $container = Bootstrap::init() - ->withConfig($xmlConfig, [], [], \getenv()) + ->withConfig($xmlConfig, [], [], $environment) ->finish(); $container->set($input = new ArgvInput(), InputInterface::class); $container->set($output = new BufferedOutput(), OutputInterface::class); diff --git a/tests/Integration/Module/Registry/VersionRegistryBindingTest.php b/tests/Integration/Module/Registry/VersionRegistryBindingTest.php new file mode 100644 index 0000000..0d84010 --- /dev/null +++ b/tests/Integration/Module/Registry/VersionRegistryBindingTest.php @@ -0,0 +1,92 @@ + $this->directory]); + + Assert::instanceOf($container->get(VersionRegistry::class), StoredVersionRegistry::class); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/dload/repositories/github/a/b.json')); + } + + #[Test] + public function environmentVariableSetsTheDirectory(): void + { + $container = self::bootstrap(environment: ['DLOAD_CACHE_DIR' => $this->directory]); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/repositories/github/a/b.json')); + } + + #[Test] + public function xmlAttributeSetsTheDirectory(): void + { + $container = self::bootstrap(xml: \sprintf('', $this->directory)); + + $container->get(RegistryStorage::class)->save(RepositoryRecord::empty(new RepositoryId('github', 'a/b'))); + Assert::true(\is_file($this->directory . '/repositories/github/a/b.json')); + } + + #[Test] + public function zeroTtlDisablesTheRegistry(): void + { + $container = self::bootstrap(environment: ['DLOAD_CACHE_DIR' => $this->directory, 'DLOAD_CACHE_TTL' => '0']); + + Assert::instanceOf($container->get(VersionRegistry::class), PassThroughRegistry::class); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->directory = \sys_get_temp_dir() . '/dload-registry-binding-' . \bin2hex(\random_bytes(6)); + } + + #[AfterTest] + protected function cleanup(): void + { + \is_dir($this->directory) and FS::removeDir(Path::create($this->directory)); + } + + /** + * @param array $environment + */ + private static function bootstrap(?string $xml = null, array $environment = []): Container + { + return Bootstrap::init() + ->withConfig(xml: $xml, environment: $environment) + ->finish(); + } +} diff --git a/tests/Unit/Module/Registry/CacheDirectoryTest.php b/tests/Unit/Module/Registry/CacheDirectoryTest.php new file mode 100644 index 0000000..3cfab9a --- /dev/null +++ b/tests/Unit/Module/Registry/CacheDirectoryTest.php @@ -0,0 +1,36 @@ + '/var/cache/', 'HOME' => '/home/u', 'LOCALAPPDATA' => 'C:\\x']); + + Assert::same($dir, '/var/cache' . \DIRECTORY_SEPARATOR . 'dload'); + } + + #[Test] + public function homeIsUsedWhenNothingElseIsSet(): void + { + $dir = CacheDirectory::resolve(['HOME' => '/home/u', 'XDG_CACHE_HOME' => ' ']); + + Assert::same($dir, '/home/u' . \DIRECTORY_SEPARATOR . '.cache' . \DIRECTORY_SEPARATOR . 'dload'); + } + + #[Test] + public function fallsBackToTheTemporaryDirectory(): void + { + Assert::same(CacheDirectory::resolve([]), \sys_get_temp_dir() . \DIRECTORY_SEPARATOR . 'dload-cache'); + } +} diff --git a/tests/Unit/Module/Registry/FileRegistryStorageTest.php b/tests/Unit/Module/Registry/FileRegistryStorageTest.php new file mode 100644 index 0000000..6832128 --- /dev/null +++ b/tests/Unit/Module/Registry/FileRegistryStorageTest.php @@ -0,0 +1,116 @@ +storage(); + + $storage->save(self::record('github', 'roadrunner-server/roadrunner', ['v1'])); + $storage->save(self::record('gitlab', 'group/sub/project', ['v2'])); + + Assert::true(\is_file($this->directory . '/repositories/github/roadrunner-server/roadrunner.json')); + Assert::true(\is_file($this->directory . '/repositories/gitlab/group/sub/project.json')); + + $loaded = $storage->load(new RepositoryId('github', 'roadrunner-server/roadrunner')); + Assert::same($loaded->releases()[0]->tag, 'v1'); + Assert::same($loaded->software, ['rr']); + } + + #[Test] + public function missingAndCorruptedRecordsReadAsNull(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', 'owner/repo'); + + Assert::null($storage->load($id)); + + $storage->save(self::record('github', 'owner/repo', ['v1'])); + \file_put_contents($this->directory . '/repositories/github/owner/repo.json', '{not json'); + + Assert::null($storage->load($id)); + } + + #[Test] + public function listsRemovesAndClears(): void + { + $storage = $this->storage(); + $storage->save(self::record('github', 'a/b', ['v1'])); + $storage->save(self::record('github', 'c/d', ['v1'])); + + Assert::count(\iterator_to_array($storage->all(), false), 2); + + $storage->remove(new RepositoryId('github', 'a/b')); + Assert::count(\iterator_to_array($storage->all(), false), 1); + + $storage->clear(); + Assert::count(\iterator_to_array($storage->all(), false), 0); + Assert::null($storage->load(new RepositoryId('github', 'c/d'))); + } + + #[Test] + public function unsafePathSegmentsAreSanitized(): void + { + $storage = $this->storage(); + $id = new RepositoryId('github', '../owner/re po:x'); + + $storage->save(new RepositoryRecord($id)); + + Assert::false(\is_dir(\dirname($this->directory) . '/owner')); + Assert::true(\is_file($this->directory . '/repositories/github/_/owner/re_po_x.json')); + Assert::true($storage->load($id)?->id->equals($id) ?? false); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->directory = \sys_get_temp_dir() . '/dload-registry-' . \bin2hex(\random_bytes(6)); + } + + #[AfterTest] + protected function cleanup(): void + { + \is_dir($this->directory) and FS::removeDir(Path::create($this->directory)); + } + + /** + * @param non-empty-string $type + * @param non-empty-string $uri + * @param list $tags + */ + private static function record(string $type, string $uri, array $tags): RepositoryRecord + { + return new RepositoryRecord( + id: new RepositoryId($type, $uri), + checkedAt: 1_000, + software: ['rr'], + releases: \array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags), + ); + } + + private function storage(): FileRegistryStorage + { + return new FileRegistryStorage($this->directory, new Logger()); + } +} diff --git a/tests/Unit/Module/Registry/RepositoryRecordTest.php b/tests/Unit/Module/Registry/RepositoryRecordTest.php new file mode 100644 index 0000000..9f1ab33 --- /dev/null +++ b/tests/Unit/Module/Registry/RepositoryRecordTest.php @@ -0,0 +1,140 @@ +withHead([new ReleaseRecord('v3', 'v3'), new ReleaseRecord('v2', 'new v2')]); + + Assert::same(self::tags($updated), ['v3', 'v2', 'v1']); + Assert::same($updated->releases()[1]->name, 'new v2'); + } + + #[Test] + public function tailIgnoresKnownReleases(): void + { + $record = new RepositoryRecord(self::id(), releases: [new ReleaseRecord('v2', 'v2')]); + + $updated = $record->withTail([new ReleaseRecord('v2', 'dup'), new ReleaseRecord('v1', 'v1')]); + + Assert::same(self::tags($updated), ['v2', 'v1']); + Assert::same($updated->releases()[0]->name, 'v2'); + } + + #[Test] + public function stalenessDependsOnTheLastCheck(): void + { + $never = RepositoryRecord::empty(self::id()); + $checked = $never->withCheckedAt(1_000); + + Assert::true($never->isStale(1_000, 600)); + Assert::false($checked->isStale(1_600, 600)); + Assert::true($checked->isStale(1_601, 600)); + } + + #[Test] + public function softwareIsAttachedOnce(): void + { + $record = RepositoryRecord::empty(self::id())->withSoftware('rr'); + + Assert::same($record->withSoftware('rr'), $record); + Assert::same($record->withSoftware('temporal')->software, ['rr', 'temporal']); + } + + #[Test] + public function survivesTheArrayRoundTrip(): void + { + $record = new RepositoryRecord( + id: self::id(), + checkedAt: 1_000, + complete: true, + software: ['rr'], + releases: [ + new ReleaseRecord( + tag: 'v2.0.0', + name: 'Release 2', + publishedAt: new \DateTimeImmutable('2024-01-02T03:04:05+00:00'), + prerelease: true, + assets: [new AssetRecord('rr-linux-amd64.tar.gz', 'https://x/rr.tar.gz', 42, 'application/gzip')], + ), + new ReleaseRecord('v1.0.0', 'v1.0.0'), + ], + ); + + $restored = RepositoryRecord::fromArray(\json_decode(\json_encode($record->toArray()), true)); + + Assert::same($restored->toArray(), $record->toArray()); + Assert::true($restored->id->equals(self::id())); + Assert::same($restored->releases()[0]->assets[0]->size, 42); + Assert::same($restored->releases()[0]->publishedAt?->format(\DATE_ATOM), '2024-01-02T03:04:05+00:00'); + Assert::null($restored->releases()[1]->publishedAt); + } + + #[Test] + public function rejectsUnknownFormatVersion(): void + { + try { + RepositoryRecord::fromArray(['version' => 99, 'repository' => ['type' => 'github', 'uri' => 'a/b']]); + Assert::fail('An unknown format version must be rejected.'); + } catch (\InvalidArgumentException $e) { + Assert::same($e->getMessage(), 'Unsupported repository record format.'); + } + } + + #[Test] + public function skipsBrokenReleasesButRequiresATag(): void + { + try { + ReleaseRecord::fromArray(['name' => 'no tag']); + Assert::fail('A release without a tag must be rejected.'); + } catch (\InvalidArgumentException $e) { + Assert::same($e->getMessage(), 'Release record requires a non-empty `tag`.'); + } + + // A broken asset makes the whole release unusable rather than silently dropping the asset + try { + ReleaseRecord::fromArray(['tag' => 'v1', 'assets' => [['name' => 'x']]]); + Assert::fail('An asset without a URI must be rejected.'); + } catch (\InvalidArgumentException) { + } + + $release = ReleaseRecord::fromArray(['tag' => 'v1', 'assets' => [['name' => 'ok', 'uri' => 'https://x']]]); + Assert::same($release->name, 'v1'); + Assert::count($release->assets, 1); + } + + private static function id(): RepositoryId + { + return new RepositoryId('github', 'owner/repo'); + } + + /** + * @return list + */ + private static function tags(RepositoryRecord $record): array + { + return \array_map(static fn(ReleaseRecord $release): string => $release->tag, $record->releases()); + } +} diff --git a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php new file mode 100644 index 0000000..834460f --- /dev/null +++ b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php @@ -0,0 +1,220 @@ +registry(); + + $pages = $registry->releases($this->id, $source); + $first = self::tagsOf($pages->current()); + + Assert::same($first, ['v6', 'v5']); + Assert::same($source->served, [0]); + + // The record already holds what was fetched, marked as incomplete + $record = $this->storage->load($this->id); + Assert::same(self::tagsOf($record->releases()), ['v6', 'v5']); + Assert::false($record->complete); + Assert::same($record->checkedAt, $this->now); + } + + #[Test] + public function olderReleasesAreLoadedOnDemandAndPersisted(): void + { + $source = ArrayReleaseSource::ofTags(['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + $registry = $this->registry(); + + $all = self::flatten($registry->releases($this->id, $source)); + + Assert::same($all, ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($source->served, [0, 2, 4]); + + $record = $this->storage->load($this->id); + Assert::same(self::tagsOf($record->releases()), ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::true($record->complete); + } + + #[Test] + public function freshRecordIsServedWithoutAnyRequest(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + $this->now += 100; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v3', 'v2', 'v1']); + Assert::same($source->served, []); + } + + #[Test] + public function staleRecordIsCheckedWithASinglePageWhenNothingIsNew(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v3', 'v2', 'v1']); + Assert::same($source->served, [0]); + Assert::same($this->storage->load($this->id)->checkedAt, $this->now); + } + + #[Test] + public function newReleasesAreFetchedUntilAKnownOneIsReached(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + // Three releases were published: they span two pages, the second one reaches `v3` + $source->publish('v6', 'v5', 'v4'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($source->served, [0, 2]); + } + + #[Test] + public function firstPageOverwritesStoredReleasesOnCheck(): void + { + $source = new ArrayReleaseSource([new ReleaseRecord('v1', 'v1', assets: [])]); + self::flatten($this->registry()->releases($this->id, $source)); + + // Assets were attached after the release had been stored + $updated = new ArrayReleaseSource([ + new ReleaseRecord('v1', 'v1', assets: [new \Internal\DLoad\Module\Registry\Record\AssetRecord('rr.zip', 'https://x/rr.zip')]), + ]); + $this->now += 601; + self::flatten($this->registry()->releases($this->id, $updated)); + + Assert::count($this->storage->load($this->id)->releases()[0]->assets, 1); + } + + #[Test] + public function refreshFlagIgnoresTheTtl(): void + { + $source = ArrayReleaseSource::ofTags(['v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $source->served = []; + + $source->publish('v3'); + self::flatten($this->registry(refresh: true)->releases($this->id, $source)); + + Assert::same($source->served, [0]); + Assert::same(self::tagsOf($this->storage->load($this->id)->releases()), ['v3', 'v2', 'v1']); + } + + #[Test] + public function failedCheckFallsBackToStoredReleases(): void + { + $source = ArrayReleaseSource::ofTags(['v2', 'v1']); + self::flatten($this->registry()->releases($this->id, $source)); + + $source->fail(); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v2', 'v1']); + } + + #[Test] + public function failedCheckWithoutStoredReleasesIsReported(): void + { + $source = ArrayReleaseSource::ofTags(['v1']); + $source->fail(); + + try { + self::flatten($this->registry()->releases($this->id, $source)); + Assert::fail('The failure of the source must reach the caller when nothing is stored.'); + } catch (ApiException) { + Assert::same($this->storage->records, []); + } + } + + #[Test] + public function storageFailureDoesNotBreakTheListing(): void + { + $this->storage->failOnSave = true; + $source = ArrayReleaseSource::ofTags(['v2', 'v1']); + + Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v2', 'v1']); + } + + #[Test] + public function attachRecordsTheSoftwareOnce(): void + { + $registry = $this->registry(); + + $registry->attach('rr', $this->id); + $registry->attach('rr', $this->id); + $registry->attach('roadrunner', $this->id); + + Assert::same($this->storage->load($this->id)->software, ['rr', 'roadrunner']); + Assert::same($this->storage->saves, 2); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->storage = new InMemoryRegistryStorage(); + $this->id = new RepositoryId('github', 'owner/repo'); + $this->now = 1_000_000; + } + + /** + * @param iterable> $pages + * @return list + */ + private static function flatten(iterable $pages): array + { + $tags = []; + foreach ($pages as $page) { + $tags = [...$tags, ...self::tagsOf($page)]; + } + + return $tags; + } + + /** + * @param list $releases + * @return list + */ + private static function tagsOf(array $releases): array + { + return \array_map(static fn(ReleaseRecord $release): string => $release->tag, $releases); + } + + private function registry(bool $refresh = false): StoredVersionRegistry + { + return new StoredVersionRegistry($this->storage, 600, new Logger(), $refresh, fn(): int => $this->now); + } +} diff --git a/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php new file mode 100644 index 0000000..93dda14 --- /dev/null +++ b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php @@ -0,0 +1,81 @@ + + */ + public array $served = []; + + /** When set, every page request fails with this exception. */ + public ?\Throwable $failure = null; + + /** + * @param list $releases Newest first. + * @param int<1, max> $perPage + */ + public function __construct( + private array $releases, + private readonly int $perPage = 2, + ) {} + + /** + * @param list $tags Newest first. + */ + public static function ofTags(array $tags, int $perPage = 2): self + { + return new self(\array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags), $perPage); + } + + /** + * Publishes releases on top of the list, as a repository would between two runs. + * + * @param non-empty-string ...$tags Newest first. + */ + public function publish(string ...$tags): void + { + $this->releases = [...\array_map(static fn(string $tag): ReleaseRecord => new ReleaseRecord($tag, $tag), $tags), ...$this->releases]; + } + + public function fail(?\Throwable $failure = null): void + { + $this->failure = $failure ?? new ApiException('API is unavailable.', 'stub/stub'); + } + + public function pages(int $offset = 0): \Generator + { + // Align with a page boundary and skip within the page, like a real paged API + $page = \intdiv($offset, $this->perPage); + $skip = $offset % $this->perPage; + + do { + $this->failure === null or throw $this->failure; + + $this->served[] = $page * $this->perPage; + $items = \array_slice($this->releases, $page * $this->perPage, $this->perPage); + $last = ($page + 1) * $this->perPage >= \count($this->releases); + + yield new ReleasePage(\array_slice($items, $skip), $last); + + $skip = 0; + ++$page; + } while (!$last); + } +} diff --git a/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php b/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php new file mode 100644 index 0000000..6a4a76b --- /dev/null +++ b/tests/Unit/Module/Registry/Stub/InMemoryRegistryStorage.php @@ -0,0 +1,52 @@ + */ + public array $records = []; + + /** @var int<0, max> */ + public int $saves = 0; + + public bool $failOnSave = false; + + public function load(RepositoryId $id): ?RepositoryRecord + { + return $this->records[(string) $id] ?? null; + } + + public function save(RepositoryRecord $record): void + { + $this->failOnSave and throw new \RuntimeException('Storage is read-only.'); + + ++$this->saves; + $this->records[(string) $record->id] = $record; + } + + public function all(): iterable + { + yield from \array_values($this->records); + } + + public function remove(RepositoryId $id): void + { + unset($this->records[(string) $id]); + } + + public function clear(): void + { + $this->records = []; + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php new file mode 100644 index 0000000..13a50b7 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php @@ -0,0 +1,152 @@ +getReleases(), false); + + Assert::same(\count($releases), 300); + Assert::same($client->requestedPages(), [1, 2, 3]); + } + + #[Test] + public function pagesAreLoadedOnlyWhenNeeded(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + + // Consume the whole first page, but nothing beyond it + $seen = 0; + foreach ($repository->getReleases() as $release) { + unset($release); + if (++$seen === 100) { + break; + } + } + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function destroyDoesNotLoadTheRemainingPages(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + $repository->getReleases()->first(); + + $repository->destroy(); + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function releasesAreRequestedAHundredPerPage(): void + { + $client = new PagedClientStub(pages: 1); + $repository = self::createRepository($client); + + \iterator_to_array($repository->getReleases(), false); + + Assert::same($client->requests, ['page=1&per_page=100']); + } + + #[Test] + public function secondRunIsServedFromTheRegistryWithoutRequests(): void + { + $storage = new InMemoryRegistryStorage(); + + $firstClient = new PagedClientStub(pages: 2); + $firstRun = self::names(self::createRepository($firstClient, self::registry($storage))); + + // A second run in a fresh process with the registry carried over + $secondClient = new PagedClientStub(pages: 2); + $secondRun = self::names(self::createRepository($secondClient, self::registry($storage))); + + Assert::same($firstClient->requestedPages(), [1, 2]); + Assert::same($secondClient->requestedPages(), []); + Assert::same($secondRun, $firstRun); + Assert::same(\count($secondRun), 200); + } + + #[Test] + public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void + { + $storage = new InMemoryRegistryStorage(); + + // The first run needs the first page only + $firstClient = new PagedClientStub(pages: 3); + foreach (self::createRepository($firstClient, self::registry($storage))->getReleases() as $release) { + unset($release); + break; + } + + // The second run needs everything: the stored page costs nothing, the rest is fetched + $secondClient = new PagedClientStub(pages: 3); + $all = self::names(self::createRepository($secondClient, self::registry($storage))); + + Assert::same($firstClient->requestedPages(), [1]); + Assert::same($secondClient->requestedPages(), [2, 3]); + Assert::same(\count($all), 300); + } + + private static function createRepository( + PagedClientStub $client, + VersionRegistry $registry = new PassThroughRegistry(), + ): GitHubRepository { + $logger = new Logger(); + $httpFactory = new NyholmFactoryImpl($logger); + $api = new RepositoryApi( + new Client($httpFactory, $client, new GitHubConfig()), + $httpFactory, + 'owner', + 'repo', + $logger, + ); + + return new GitHubRepository($api, 'owner', 'repo', $logger, $registry); + } + + private static function registry(InMemoryRegistryStorage $storage): StoredVersionRegistry + { + return new StoredVersionRegistry($storage, 600, new Logger()); + } + + /** + * @return list + */ + private static function names(GitHubRepository $repository): array + { + return \array_map( + static fn(ReleaseInterface $release): string => $release->getName(), + \iterator_to_array($repository->getReleases(), false), + ); + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php new file mode 100644 index 0000000..b6a04da --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php @@ -0,0 +1,105 @@ + + */ + public array $requests = []; + + /** + * @param int<1, max> $pages Number of pages the list is split into when 100 releases are requested per page. + * @param int<1, max> $releasesPerPage Number of releases on every such page. + */ + public function __construct( + private readonly int $pages = 1, + private readonly int $releasesPerPage = 100, + ) {} + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $query = $request->getUri()->getQuery(); + $this->requests[] = $query; + + $page = self::pageOf($query); + $perPage = self::perPageOf($query); + $all = $this->allReleases(); + + // Serve the slice the real API would serve for the requested page size + $releases = \array_slice($all, ($page - 1) * $perPage, $perPage); + if ($releases === []) { + return new ResponseStub(200, [], '[]'); + } + + $headers = $page * $perPage < \count($all) + ? ['link' => [\sprintf('; rel="next"', $page + 1)]] + : []; + + return new ResponseStub(200, $headers, \json_encode($releases)); + } + + /** + * Page number of every received request, in order. + * + * @return list + */ + public function requestedPages(): array + { + return \array_map(self::pageOf(...), $this->requests); + } + + private static function pageOf(string $query): int + { + \parse_str($query, $params); + + return (int) ($params['page'] ?? 1); + } + + private static function perPageOf(string $query): int + { + \parse_str($query, $params); + + return \max(1, (int) ($params['per_page'] ?? 30)); + } + + /** + * @return list> + */ + private function allReleases(): array + { + $releases = []; + $total = $this->pages * $this->releasesPerPage; + + for ($i = 1; $i <= $total; $i++) { + $tag = \sprintf('v1.0.%d', $i); + $releases[] = [ + 'name' => $tag, + 'tag_name' => $tag, + 'published_at' => '2024-01-01T00:00:00Z', + 'assets' => [], + 'prerelease' => false, + 'draft' => false, + ]; + } + + return $releases; + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php b/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php index 96b9527..8a25c34 100644 --- a/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php @@ -6,6 +6,7 @@ use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitLab as GitLabConfig; +use Internal\DLoad\Module\Registry\Internal\PassThroughRegistry; use Internal\DLoad\Module\Repository\Internal\GitLab\Factory; use Internal\DLoad\Service\Logger; use Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitLab\Stub\HttpFactoryStub; @@ -69,6 +70,11 @@ public function createDerivesTheProjectPathFromTheUri(string $uri, string $expec #[BeforeTest] protected function prepare(): void { - $this->factory = new Factory(new HttpFactoryStub(), new GitLabConfig(), new Logger()); + $this->factory = new Factory( + new HttpFactoryStub(), + new GitLabConfig(), + new Logger(), + new PassThroughRegistry(), + ); } } diff --git a/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php new file mode 100644 index 0000000..c94c5b0 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php @@ -0,0 +1,114 @@ +getReleases(), false); + + Assert::same(\count($releases), 300); + Assert::same($client->requestedPages(), [1, 2, 3]); + } + + #[Test] + public function pagesAreLoadedOnlyWhenNeeded(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + + $seen = 0; + foreach ($repository->getReleases() as $release) { + unset($release); + if (++$seen === 100) { + break; + } + } + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function destroyDoesNotLoadTheRemainingPages(): void + { + $client = new PagedClientStub(pages: 3); + $repository = self::createRepository($client); + $repository->getReleases()->first(); + + $repository->destroy(); + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function releasesAreRequestedAHundredPerPage(): void + { + $client = new PagedClientStub(pages: 1); + $repository = self::createRepository($client); + + \iterator_to_array($repository->getReleases(), false); + + Assert::same($client->requests, ['page=1&per_page=100']); + } + + #[Test] + public function secondRunIsServedFromTheRegistryWithoutRequests(): void + { + $storage = new InMemoryRegistryStorage(); + + $firstClient = new PagedClientStub(pages: 2); + \iterator_to_array(self::createRepository($firstClient, self::registry($storage))->getReleases(), false); + + $secondClient = new PagedClientStub(pages: 2); + $secondRun = \iterator_to_array(self::createRepository($secondClient, self::registry($storage))->getReleases(), false); + + Assert::same($firstClient->requestedPages(), [1, 2]); + Assert::same($secondClient->requestedPages(), []); + Assert::same(\count($secondRun), 200); + } + + private static function createRepository( + PagedClientStub $client, + VersionRegistry $registry = new PassThroughRegistry(), + ): GitLabRepository { + $logger = new Logger(); + $httpFactory = new NyholmFactoryImpl($logger); + $api = new RepositoryApi( + new Client($httpFactory, $client, new GitLabConfig()), + $httpFactory, + 'group/project', + ); + + return new GitLabRepository($api, 'group/project', $logger, $registry); + } + + private static function registry(InMemoryRegistryStorage $storage): StoredVersionRegistry + { + return new StoredVersionRegistry($storage, 600, new Logger()); + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php new file mode 100644 index 0000000..eca2b51 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php @@ -0,0 +1,106 @@ + + */ + public array $requests = []; + + /** + * @param int<1, max> $pages Number of pages the list is split into when 100 releases are requested per page. + * @param int<1, max> $releasesPerPage Number of releases on every such page. + */ + public function __construct( + private readonly int $pages = 1, + private readonly int $releasesPerPage = 100, + ) {} + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $query = $request->getUri()->getQuery(); + $this->requests[] = $query; + + $page = self::pageOf($query); + $perPage = self::perPageOf($query); + $all = $this->allReleases(); + + // Serve the slice the real API would serve for the requested page size + $releases = \array_slice($all, ($page - 1) * $perPage, $perPage); + if ($releases === []) { + return new ResponseStub(200, [], '[]'); + } + + $headers = $page * $perPage < \count($all) + ? ['link' => [\sprintf('; rel="next"', $page + 1)]] + : []; + + return new ResponseStub(200, $headers, \json_encode($releases)); + } + + /** + * Page number of every received request, in order. + * + * @return list + */ + public function requestedPages(): array + { + return \array_map(self::pageOf(...), $this->requests); + } + + private static function pageOf(string $query): int + { + \parse_str($query, $params); + + return (int) ($params['page'] ?? 1); + } + + private static function perPageOf(string $query): int + { + \parse_str($query, $params); + + return \max(1, (int) ($params['per_page'] ?? 20)); + } + + /** + * @return list> + */ + private function allReleases(): array + { + $releases = []; + $total = $this->pages * $this->releasesPerPage; + + for ($i = 1; $i <= $total; $i++) { + $tag = \sprintf('v1.0.%d', $i); + $releases[] = [ + 'name' => $tag, + 'tag_name' => $tag, + 'description' => 'Release ' . $tag, + 'created_at' => '2024-01-01T00:00:00Z', + 'released_at' => '2024-01-01T00:00:00Z', + 'assets' => ['links' => []], + 'upcoming_release' => false, + ]; + } + + return $releases; + } +} From ea36d7e47902ccc80eb3ad6c368abb87ad7438b9 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Sat, 12 Sep 2026 23:58:13 +0400 Subject: [PATCH 2/2] fix: recover from releases deleted after the version registry stored them A 404 for a release asset was reported as a missing repository, with advice about tokens and addresses, and the deleted release stayed in the registry until a later check happened to overwrite it. Asset URLs now raise `AssetNotFoundException`; when every matching asset of a release is gone, the downloader drops the release from the registry, marks the repository for a check, and fetches the list once more before giving up. Assisted-By: Claude Fable 5.1 --- README-es.md | 4 +- README-ru.md | 4 +- README-zh.md | 3 +- README.md | 2 + src/Module/Downloader/Downloader.php | 58 +++++- .../Downloader/Exception/ReleaseGone.php | 13 ++ .../Registry/Internal/PassThroughRegistry.php | 5 + .../Internal/StoredVersionRegistry.php | 13 ++ .../Registry/Record/RepositoryRecord.php | 31 +++ src/Module/Registry/VersionRegistry.php | 10 + .../Exception/AssetNotFoundException.php | 13 ++ .../Internal/GitHub/Api/ResponseValidator.php | 6 + .../Internal/GitLab/Api/ResponseValidator.php | 6 + .../Repository/Internal/ResponseValidator.php | 18 ++ .../Unit/Module/Downloader/DownloaderTest.php | 181 ++++++++++++++++++ .../Module/Downloader/Stub/GoneAssetStub.php | 58 ++++++ .../Stub/SequenceRepositoryFactoryStub.php | 39 ++++ .../Module/Registry/RepositoryRecordTest.php | 20 ++ .../Registry/StoredVersionRegistryTest.php | 35 ++++ .../Registry/Stub/RecordingRegistry.php | 39 ++++ .../GitHub/Api/ResponseValidatorTest.php | 24 ++- .../GitLab/Api/ResponseValidatorTest.php | 21 ++ 22 files changed, 592 insertions(+), 11 deletions(-) create mode 100644 src/Module/Downloader/Exception/ReleaseGone.php create mode 100644 src/Module/Repository/Exception/AssetNotFoundException.php create mode 100644 tests/Unit/Module/Downloader/DownloaderTest.php create mode 100644 tests/Unit/Module/Downloader/Stub/GoneAssetStub.php create mode 100644 tests/Unit/Module/Downloader/Stub/SequenceRepositoryFactoryStub.php create mode 100644 tests/Unit/Module/Registry/Stub/RecordingRegistry.php diff --git a/README-es.md b/README-es.md index 9c9fc35..a4e61a5 100644 --- a/README-es.md +++ b/README-es.md @@ -391,7 +391,9 @@ El registro está activado por defecto y vive en el directorio de caché del usu > El registro solo contiene metadatos de releases: tags, nombres y enlaces de descarga. Las descargas > no pasan por él y nunca guarda credenciales, así que el directorio puede compartirse o guardarse en > la caché de CI sin problemas. Si una comprobación falla por un error de red o un límite de la API, se -> usan los releases almacenados; un repositorio nunca visto sigue fallando de forma visible. +> usan los releases almacenados; un repositorio nunca visto sigue fallando de forma visible. Un +> release almacenado cuyos assets desaparecieron del origen se elimina del registro en cuanto falla +> su descarga, y la lista de releases se vuelve a obtener antes de que la ejecución se dé por vencida. En GitHub Actions el directorio puede conservarse entre ejecuciones del workflow, de modo que cada ejecución gasta el límite de la API solo en los releases publicados desde la anterior: diff --git a/README-ru.md b/README-ru.md index b9a21fe..1b3cd39 100644 --- a/README-ru.md +++ b/README-ru.md @@ -393,7 +393,9 @@ DLoad поддерживает три типа загрузки, которые > не проходят, учётные данные в нём не сохраняются, поэтому каталог можно свободно передавать между > машинами и складывать в кэш CI. Если проверка не удалась из-за сетевой ошибки или лимита API, > используются сохранённые релизы, а репозиторий, который раньше не встречался, по-прежнему -> завершится ошибкой. +> завершится ошибкой. Сохранённый релиз, ассеты которого исчезли из источника, удаляется из +> реестра сразу после неудачной загрузки, а список релизов запрашивается заново, прежде чем +> запуск завершится ошибкой. В GitHub Actions каталог можно переносить между запусками workflow, тогда запуск тратит лимит API только на релизы, вышедшие после предыдущего: diff --git a/README-zh.md b/README-zh.md index 2b63606..67c810e 100644 --- a/README-zh.md +++ b/README-zh.md @@ -387,7 +387,8 @@ DLoad 支持三种下载类型，它们决定了资源的处理方式： > [!NOTE] > 注册表只保存发布的元数据：标签、名称和资产下载链接。下载不会经过注册表，也不会保存任何凭据， > 因此该目录可以自由共享或放入 CI 缓存。若因网络错误或 API 速率限制导致检查失败，会使用已保存的发布； -> 从未见过的仓库仍会明确报错。 +> 从未见过的仓库仍会明确报错。若某个已保存发布的资产在上游已被删除，下载失败后它会立即从注册表中移除， +> 并在本次运行放弃之前重新获取发布列表。 在 GitHub Actions 中可以在多次工作流运行之间保留该目录，这样每次运行只为上次运行之后发布的版本消耗速率限制： diff --git a/README.md b/README.md index 92f62e3..220c218 100644 --- a/README.md +++ b/README.md @@ -394,6 +394,8 @@ The registry is on by default and lives in the per-user cache directory (`$XDG_C > go through it and credentials are never stored in it, so the directory can be shared or committed > to a CI cache freely. When a check fails because of a network error or a rate limit, the stored > releases are used instead, and a repository that was never seen before still fails loudly. +> A stored release whose assets have disappeared upstream is dropped from the registry as soon as +> its download fails, and the release list is fetched again before the run gives up. In GitHub Actions the directory can be carried between workflow runs, so a run spends the rate limit only on releases published since the previous one: diff --git a/src/Module/Downloader/Downloader.php b/src/Module/Downloader/Downloader.php index 4e95530..c317a95 100644 --- a/src/Module/Downloader/Downloader.php +++ b/src/Module/Downloader/Downloader.php @@ -16,6 +16,7 @@ use Internal\DLoad\Module\Config\Schema\Embed\Software; use Internal\DLoad\Module\Downloader\Exception\DownloadFailed; use Internal\DLoad\Module\Downloader\Exception\NotFound; +use Internal\DLoad\Module\Downloader\Exception\ReleaseGone; use Internal\DLoad\Module\Downloader\Internal\Diagnostics\DownloadDiagnostics; use Internal\DLoad\Module\Downloader\Internal\DownloadContext; use Internal\DLoad\Module\Downloader\Task\DownloadResult; @@ -25,6 +26,7 @@ use Internal\DLoad\Module\Repository\AssetInterface; use Internal\DLoad\Module\Repository\Collection\AssetsCollection; use Internal\DLoad\Module\Repository\Collection\ReleasesCollection; +use Internal\DLoad\Module\Repository\Exception\AssetNotFoundException; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryException; use Internal\DLoad\Module\Repository\ReleaseInterface; @@ -167,9 +169,12 @@ public function download( * @param DownloadContext $context Download context information * @return \Closure(): ReleaseInterface Closure that returns the selected release */ - private function processRepository(Repository $repository, DownloadContext $context): \Closure + private function processRepository(Repository $repository, DownloadContext $context, bool $mayRetry = true): \Closure { - return function () use ($repository, $context): ReleaseInterface { + return function () use ($repository, $context, $mayRetry): ReleaseInterface { + // Set when a release turned out to be deleted: the release list is outdated then + $forgotten = false; + $this->logger->info( 'Loading releases from `%s` repository %s', $context->repoConfig->type, @@ -206,7 +211,15 @@ private function processRepository(Repository $repository, DownloadContext $cont } process_release: - $releases === [] and throw new NotFound('No relevant release found.'); + if ($releases === []) { + // The list was outdated: ask the repository again once, with the deleted releases forgotten + if ($forgotten && $mayRetry) { + return $this->retryRepository($context); + } + + throw new NotFound('No relevant release found.'); + } + $context->release = \array_shift($releases); $context->releaseAttempt = $context->repositoryAttempt->addRelease($context->release->getName()); @@ -215,6 +228,17 @@ private function processRepository(Repository $repository, DownloadContext $cont try { await(coroutine($this->processRelease($context))); return $context->release; + } catch (ReleaseGone $e) { + // The registry must not offer this release again, and the list needs a fresh check + $this->registry->forget( + RepositoryId::fromConfig($context->repoConfig), + $context->release->getVersion()->string, + ); + $forgotten = true; + + $context->releaseAttempt->reason ??= $e->getMessage(); + $this->logger->debug($e->getMessage()); + goto process_release; } catch (NotFound $e) { $context->releaseAttempt->reason ??= $e->getMessage(); $this->logger->debug($e->getMessage()); @@ -224,6 +248,23 @@ private function processRepository(Repository $repository, DownloadContext $cont }; } + /** + * Fetches the release list anew after deleted releases were forgotten and tries once more. + * + * @throws NotFound When the fresh list has nothing suitable either. + */ + private function retryRepository(DownloadContext $context): ReleaseInterface + { + $this->logger->info('Release list of `%s` is outdated, fetching it again.', $context->repoConfig->uri); + $repository = $this->repositoryProvider->getByConfig($context->repoConfig); + + try { + return await(coroutine($this->processRepository($repository, $context, mayRetry: false))); + } finally { + $repository instanceof Destroyable and $repository->destroy(); + } + } + /** * Processes a release to find suitable assets. * @@ -397,8 +438,16 @@ private function findAssetWithGradualFiltering(DownloadContext $context): AssetI */ private function tryProcessAssets(array $assets, DownloadContext $context): AssetInterface { + // Stays true while every failed asset answered "not found": then the release itself is gone + $gone = $assets !== []; + process_asset: - $assets === [] and throw new NotFound('none of the matching assets could be downloaded'); + if ($assets === []) { + $gone and throw new ReleaseGone('every matching asset of the release is no longer available'); + + throw new NotFound('none of the matching assets could be downloaded'); + } + $context->asset = \array_shift($assets); $this->logger->debug('Trying to load asset `%s`', $context->asset->getName()); try { @@ -408,6 +457,7 @@ private function tryProcessAssets(array $assets, DownloadContext $context): Asse // Retrying other assets makes the situation worse: report the limit immediately throw $e; } catch (\Throwable $e) { + $gone = $gone && $e instanceof AssetNotFoundException; $context->releaseAttempt->addFailure($context->asset->getName(), $e); $this->logger->exception($e, important: false); goto process_asset; diff --git a/src/Module/Downloader/Exception/ReleaseGone.php b/src/Module/Downloader/Exception/ReleaseGone.php new file mode 100644 index 0000000..30b1c3f --- /dev/null +++ b/src/Module/Downloader/Exception/ReleaseGone.php @@ -0,0 +1,13 @@ +persist($updated); } + public function forget(RepositoryId $id, string $tag): void + { + $record = $this->storage->load($id); + if ($record === null || !$record->has($tag)) { + return; + } + + $this->logger->debug('Release `%s` of `%s` is gone: dropped from the version registry.', $tag, (string) $id); + + // Without the last check the next listing asks the source again + $this->persist($record->withoutRelease($tag)->withoutCheck()); + } + /** * @param list $page */ diff --git a/src/Module/Registry/Record/RepositoryRecord.php b/src/Module/Registry/Record/RepositoryRecord.php index 2ecc5a1..91c6ce6 100644 --- a/src/Module/Registry/Record/RepositoryRecord.php +++ b/src/Module/Registry/Record/RepositoryRecord.php @@ -162,6 +162,37 @@ public function withCheckedAt(int $checkedAt): self return $this->with(checkedAt: $checkedAt); } + /** + * Forgets the last check, so the record counts as stale until the source is asked again. + */ + public function withoutCheck(): self + { + return new self( + id: $this->id, + checkedAt: null, + complete: $this->complete, + software: $this->software, + releases: $this->releases(), + ); + } + + /** + * Drops a release; an unknown tag leaves the record as it is. + * + * @param non-empty-string $tag + */ + public function withoutRelease(string $tag): self + { + if (!$this->has($tag)) { + return $this; + } + + return $this->with(releases: \array_values(\array_filter( + $this->releases(), + static fn(ReleaseRecord $release): bool => $release->tag !== $tag, + ))); + } + public function withComplete(bool $complete): self { return $this->with(complete: $complete); diff --git a/src/Module/Registry/VersionRegistry.php b/src/Module/Registry/VersionRegistry.php index 986419a..a0929be 100644 --- a/src/Module/Registry/VersionRegistry.php +++ b/src/Module/Registry/VersionRegistry.php @@ -41,4 +41,14 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator; * @param non-empty-string $software Software identifier. */ public function attach(string $software, RepositoryId $id): void; + + /** + * Drops a release that turned out to be gone and marks the repository for a check. + * + * Called when the assets of a stored release cannot be downloaded any more: the next listing + * asks the source again instead of trusting the stored record. + * + * @param non-empty-string $tag Tag of the release as stored in the registry. + */ + public function forget(RepositoryId $id, string $tag): void; } diff --git a/src/Module/Repository/Exception/AssetNotFoundException.php b/src/Module/Repository/Exception/AssetNotFoundException.php new file mode 100644 index 0000000..df3a1a9 --- /dev/null +++ b/src/Module/Repository/Exception/AssetNotFoundException.php @@ -0,0 +1,13 @@ +accessDeniedMessage($apiMessage, $repository), $repository, ), + // A missing asset is not a missing repository: the listing was fine, the file is gone + $status === 404 && $this->isAssetUri((string) $request->getUri()) => new AssetNotFoundException( + \sprintf( + '%s asset is no longer available: HTTP 404 for %s. ' + . 'The release may have been deleted or its assets replaced since the release list was fetched.', + $this->providerName(), + (string) $request->getUri(), + ), + $repository, + ), $status === 404 => new RepositoryNotFoundException( $this->notFoundMessage($apiMessage, $repository, $endpoint), $repository, @@ -135,6 +146,13 @@ protected function repositoryTerm(): string */ abstract protected function repositoryFromUri(string $uri): ?string; + /** + * Whether the URI points to a release asset rather than to the API. + * + * A 404 for an asset means the release is gone, not that the repository does not exist. + */ + abstract protected function isAssetUri(string $uri): bool; + /** * @return positive-int|null Requests per hour allowed without a token. */ diff --git a/tests/Unit/Module/Downloader/DownloaderTest.php b/tests/Unit/Module/Downloader/DownloaderTest.php new file mode 100644 index 0000000..f0b459a --- /dev/null +++ b/tests/Unit/Module/Downloader/DownloaderTest.php @@ -0,0 +1,181 @@ +download([$repository]); + + Assert::same($result->version->string, 'v1.9.0'); + Assert::same($this->registry->forgotten, [['github:owner/repo', 'v2.0.0']]); + Assert::same($this->registry->attached, [['rr', 'github:owner/repo']]); + } + + #[Test] + public function outdatedListIsFetchedAgainWhenNothingIsLeft(): void + { + // The stored list knows only the deleted release; a fresh list has its replacement + $stale = new RepositoryStub('owner/repo'); + $stale = new RepositoryStub('owner/repo', ReleasesCollection::create([ + self::release($stale, 'v2.0.0', assets: false), + ])); + $fresh = new RepositoryStub('owner/repo'); + $fresh = new RepositoryStub('owner/repo', ReleasesCollection::create([ + self::release($fresh, 'v2.0.1', assets: true), + ])); + $factory = new SequenceRepositoryFactoryStub([$stale, $fresh]); + + $result = $this->download($factory); + + Assert::same($result->version->string, 'v2.0.1'); + Assert::same($factory->created, 2); + Assert::same($this->registry->forgotten, [['github:owner/repo', 'v2.0.0']]); + } + + #[Test] + public function outdatedListIsFetchedAgainOnlyOnce(): void + { + $stale = new RepositoryStub('owner/repo'); + $stale = new RepositoryStub('owner/repo', ReleasesCollection::create([ + self::release($stale, 'v2.0.0', assets: false), + ])); + $factory = new SequenceRepositoryFactoryStub([$stale]); + + try { + $this->download($factory); + Assert::fail('DownloadFailed is expected when the fresh list has nothing suitable either.'); + } catch (DownloadFailed $e) { + Assert::same($factory->created, 2); + Assert::string($e->report)->contains('no longer available'); + } + } + + #[Test] + public function releaseWithOtherFailuresIsNotForgotten(): void + { + $repository = new RepositoryStub('owner/repo'); + $broken = new ReleaseStub($repository, 'v2.0.0', Version::fromVersionString('v2.0.0')); + $broken->setAssets([ + new GoneAssetStub($broken, 'rr-linux-amd64.tar.gz'), + // A working asset: the release is not gone, the first asset just was + new AssetStub($broken, 'rr-linux-amd64.zip', 'https://x/rr.zip'), + ]); + $repository = new RepositoryStub('owner/repo', ReleasesCollection::create([$broken])); + + $result = $this->download([$repository]); + + Assert::same($result->version->string, 'v2.0.0'); + Assert::same($this->registry->forgotten, []); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->tempDir = \sys_get_temp_dir() . '/dload-downloader-' . \bin2hex(\random_bytes(6)); + $this->registry = new RecordingRegistry(); + } + + #[AfterTest] + protected function cleanup(): void + { + \is_dir($this->tempDir) and FS::removeDir(Path::create($this->tempDir)); + } + + /** + * @param non-empty-string $tag + * @param bool $assets `true` for a downloadable asset, `false` for one that is gone. + */ + private static function release(RepositoryStub $repository, string $tag, bool $assets): ReleaseStub + { + $release = new ReleaseStub($repository, $tag, Version::fromVersionString($tag)); + $release->setAssets([ + $assets + ? new AssetStub($release, 'rr-linux-amd64.tar.gz', 'https://x/' . $tag . '/rr.tar.gz') + : new GoneAssetStub($release, 'rr-linux-amd64.tar.gz'), + ]); + + return $release; + } + + /** + * @param list|SequenceRepositoryFactoryStub $repositories + */ + private function download(array|SequenceRepositoryFactoryStub $repositories): DownloadResult + { + $factory = $repositories instanceof SequenceRepositoryFactoryStub + ? $repositories + : new SequenceRepositoryFactoryStub($repositories); + + $config = new DownloaderConfig(); + $config->tmpDir = $this->tempDir; + + $downloader = new Downloader( + config: $config, + logger: new Logger(), + repositoryProvider: (new RepositoryProvider())->addRepositoryFactory($factory), + architecture: Architecture::tryFromString('amd64') ?? throw new \LogicException(), + operatingSystem: OperatingSystem::tryFromString('linux') ?? throw new \LogicException(), + stability: Stability::Stable, + archiveService: new ArchiveFactory(), + registry: $this->registry, + ); + + $software = Software::fromArray([ + 'name' => 'rr', + 'repositories' => [['type' => 'github', 'uri' => 'owner/repo']], + ]); + $task = $downloader->download($software, DownloadConfig::fromSoftwareId('rr'), static fn(): null => null); + + /** @var DownloadResult */ + return await(($task->handler)()); + } +} diff --git a/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php b/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php new file mode 100644 index 0000000..b33d2d9 --- /dev/null +++ b/tests/Unit/Module/Downloader/Stub/GoneAssetStub.php @@ -0,0 +1,58 @@ +release; + } + + public function getName(): string + { + return $this->name; + } + + public function getUri(): string + { + return 'https://github.com/owner/repo/releases/download/' . $this->release->getName() . '/' . $this->name; + } + + public function getOperatingSystem(): ?OperatingSystem + { + return null; + } + + public function getArchitecture(): ?Architecture + { + return null; + } + + public function download(): \Traversable + { + throw new AssetNotFoundException('GitHub asset is no longer available: HTTP 404 for ' . $this->getUri(), 'owner/repo'); + + /** @psalm-suppress UnevaluatedCode */ + yield ''; + } +} diff --git a/tests/Unit/Module/Downloader/Stub/SequenceRepositoryFactoryStub.php b/tests/Unit/Module/Downloader/Stub/SequenceRepositoryFactoryStub.php new file mode 100644 index 0000000..a89bb6c --- /dev/null +++ b/tests/Unit/Module/Downloader/Stub/SequenceRepositoryFactoryStub.php @@ -0,0 +1,39 @@ + Number of repositories created so far. */ + public int $created = 0; + + /** + * @param list $repositories Repositories to return, in order; the last one repeats. + */ + public function __construct( + private readonly array $repositories, + ) {} + + public function supports(RepositoryConfig $config): bool + { + return true; + } + + public function create(RepositoryConfig $config): Repository + { + $repository = $this->repositories[\min($this->created, \count($this->repositories) - 1)]; + ++$this->created; + + return $repository; + } +} diff --git a/tests/Unit/Module/Registry/RepositoryRecordTest.php b/tests/Unit/Module/Registry/RepositoryRecordTest.php index 9f1ab33..ae6570c 100644 --- a/tests/Unit/Module/Registry/RepositoryRecordTest.php +++ b/tests/Unit/Module/Registry/RepositoryRecordTest.php @@ -43,6 +43,26 @@ public function tailIgnoresKnownReleases(): void Assert::same($updated->releases()[0]->name, 'v2'); } + #[Test] + public function releaseCanBeDroppedAndTheCheckForgotten(): void + { + $record = new RepositoryRecord(self::id(), checkedAt: 1_000, releases: [ + new ReleaseRecord('v2', 'v2'), + new ReleaseRecord('v1', 'v1'), + ]); + + $dropped = $record->withoutRelease('v2'); + + Assert::same(self::tags($dropped), ['v1']); + Assert::same($dropped->checkedAt, 1_000); + Assert::same($record->withoutRelease('v9'), $record); + + $stale = $dropped->withoutCheck(); + Assert::null($stale->checkedAt); + Assert::same(self::tags($stale), ['v1']); + Assert::true($stale->isStale(1_000, 600)); + } + #[Test] public function stalenessDependsOnTheLastCheck(): void { diff --git a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php index 834460f..5a32f23 100644 --- a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php +++ b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php @@ -169,6 +169,41 @@ public function storageFailureDoesNotBreakTheListing(): void Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v2', 'v1']); } + #[Test] + public function forgetDropsTheReleaseAndForcesTheNextCheck(): void + { + $source = ArrayReleaseSource::ofTags(['v3', 'v2', 'v1']); + $registry = $this->registry(); + self::flatten($registry->releases($this->id, $source)); + $source->served = []; + + // `v3` was deleted upstream and its download failed + $registry->forget($this->id, 'v3'); + + $record = $this->storage->load($this->id); + Assert::same(self::tagsOf($record->releases()), ['v2', 'v1']); + Assert::null($record->checkedAt); + + // The record is fresh by time, yet the next listing asks the source again + $again = self::flatten($this->registry()->releases($this->id, $source)); + Assert::same($source->served, [0]); + Assert::same($again, ['v3', 'v2', 'v1']); + } + + #[Test] + public function forgetOfAnUnknownReleaseChangesNothing(): void + { + $source = ArrayReleaseSource::ofTags(['v1']); + self::flatten($this->registry()->releases($this->id, $source)); + $saves = $this->storage->saves; + + $this->registry()->forget($this->id, 'v9'); + $this->registry()->forget(new RepositoryId('github', 'other/repo'), 'v1'); + + Assert::same($this->storage->saves, $saves); + Assert::same($this->storage->load($this->id)->checkedAt, $this->now); + } + #[Test] public function attachRecordsTheSoftwareOnce(): void { diff --git a/tests/Unit/Module/Registry/Stub/RecordingRegistry.php b/tests/Unit/Module/Registry/Stub/RecordingRegistry.php new file mode 100644 index 0000000..95a7603 --- /dev/null +++ b/tests/Unit/Module/Registry/Stub/RecordingRegistry.php @@ -0,0 +1,39 @@ + Software attached, as `[software, repository id]`. */ + public array $attached = []; + + /** @var list Releases forgotten, as `[repository id, tag]`. */ + public array $forgotten = []; + + public function releases(RepositoryId $id, ReleaseSource $source): \Generator + { + foreach ($source->pages() as $page) { + yield $page->releases; + } + } + + public function attach(string $software, RepositoryId $id): void + { + $this->attached[] = [$software, (string) $id]; + } + + public function forget(RepositoryId $id, string $tag): void + { + $this->forgotten[] = [(string) $id, $tag]; + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php b/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php index 6d13d98..a5f51e7 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/Api/ResponseValidatorTest.php @@ -6,6 +6,7 @@ use Internal\DLoad\Module\Repository\Exception\AccessDeniedException; use Internal\DLoad\Module\Repository\Exception\ApiException; +use Internal\DLoad\Module\Repository\Exception\AssetNotFoundException; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryNotFoundException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\ResponseValidator; @@ -126,20 +127,35 @@ public function serverErrorIsReportedAsTemporaryFailure(): void } #[Test] - public function repositoryIsResolvedFromAssetDownloadUrl(): void + public function missingAssetIsNotReportedAsMissingRepository(): void { $validator = new ResponseValidator(authenticated: false); $request = new Request('GET', 'https://github.com/owner/repo/releases/download/v1.0.0/asset.zip'); - $response = new ResponseStub(404, [], \json_encode(['message' => 'Not Found'])); + $response = new ResponseStub(404, [], 'Not Found'); try { $validator->validate($request, $response); - Assert::fail('RepositoryNotFoundException is expected.'); - } catch (RepositoryNotFoundException $e) { + Assert::fail('AssetNotFoundException is expected.'); + } catch (AssetNotFoundException $e) { + // The repository is still known, but the advice about tokens and addresses would mislead Assert::same($e->repository, 'owner/repo'); + Assert::string($e->getMessage())->contains('asset is no longer available'); + Assert::string($e->getMessage())->contains('release may have been deleted'); + Assert::string($e->getMessage())->notContains('GITHUB_TOKEN'); } } + #[Test] + public function forbiddenAssetIsStillAnAccessProblem(): void + { + $validator = new ResponseValidator(authenticated: false); + $request = new Request('GET', 'https://github.com/owner/repo/releases/download/v1.0.0/asset.zip'); + + Expect::exception(AccessDeniedException::class); + + $validator->validate($request, new ResponseStub(403, [], 'Forbidden')); + } + #[Test] public function transportFailureKeepsTheOriginalError(): void { diff --git a/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php b/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php index fd8e2ea..ad45478 100644 --- a/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php +++ b/tests/Unit/Module/Repository/Internal/GitLab/Api/ResponseValidatorTest.php @@ -4,6 +4,7 @@ namespace Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitLab\Api; +use Internal\DLoad\Module\Repository\Exception\AssetNotFoundException; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Exception\RepositoryNotFoundException; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\ResponseValidator; @@ -16,6 +17,26 @@ #[Covers(ResponseValidator::class)] final class ResponseValidatorTest { + #[Test] + public function missingAssetIsNotReportedAsMissingProject(): void + { + $validator = new ResponseValidator(authenticated: false); + $request = new Request( + 'GET', + 'https://gitlab.com/api/v4/projects/group%2Fproject/releases/v1.0.0/downloads/asset.zip', + ); + $response = new ResponseStub(404, [], \json_encode(['message' => '404 Not Found'])); + + try { + $validator->validate($request, $response); + Assert::fail('AssetNotFoundException is expected.'); + } catch (AssetNotFoundException $e) { + Assert::same($e->repository, 'group/project'); + Assert::string($e->getMessage())->contains('asset is no longer available'); + Assert::string($e->getMessage())->notContains('GITLAB_TOKEN'); + } + } + #[Test] public function projectPathIsDecodedFromApiUrl(): void {