diff --git a/README.md b/README.md index 628c879..03a250c 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ With DLoad, you can: - [Download Types](#download-types) - [Version Constraints](#version-constraints) - [Advanced Configuration Options](#advanced-configuration-options) + - [Caching Release Lists](#caching-release-lists) - [Building Custom RoadRunner](#building-custom-roadrunner) - [Build Action Configuration](#build-action-configuration) - [Velox Action Attributes](#velox-action-attributes) @@ -350,6 +351,50 @@ Use Composer-style version constraints: ``` +### Caching Release Lists + +Resolving a version means asking GitHub or GitLab for the repository's release list. DLoad can keep +those listings in a directory and reuse them, so repeated runs resolve the same versions without +spending the API rate limit: + +```xml + + + + + +``` + +| Attribute | Environment variable | Default | Meaning | +|-------------|----------------------|---------|---------------------------------------------------------------| +| `cache-dir` | `DLOAD_CACHE_DIR` | not set | Directory to store cached release listings in. Caching is off until it is set. | +| `cache-ttl` | `DLOAD_CACHE_TTL` | `600` | Seconds a cached listing stays usable. `0` disables caching. | + +> [!NOTE] +> Only successful release listings are cached. Failed requests are never stored, so a rate limit +> answer is not replayed after the limit is gone, and asset downloads do not go through the cache: +> the directory holds listings only, never the downloaded binaries. + +In GitHub Actions the directory can be carried between jobs, so only the first job of a workflow run +spends any rate limit on listings: + +```yaml +- name: Cache DLoad release lists + uses: actions/cache@v4 + with: + path: ./runtime/dload-cache + key: dload-cache-${{ github.run_id }} + restore-keys: dload-cache- + +- run: ./vendor/bin/dload get + env: + DLOAD_CACHE_DIR: ./runtime/dload-cache +``` + +The `github.run_id` in the key makes every workflow run write a fresh entry, while `restore-keys` +lets the remaining jobs of that run restore it. A static key would never be written again and the +cached listings would stay stale forever. + ## Building Custom RoadRunner DLoad supports building custom RoadRunner binaries using the Velox build tool. This is useful when you need RoadRunner with custom plugin combinations that aren't available in pre-built releases. @@ -600,6 +645,9 @@ Add to CI/CD environment variables for automated downloads. > 1,000 requests per hour across all jobs of the repository. With a large job matrix the limit may run out, > and downloads from other repositories may be rejected. Use a personal access token if that happens. +Release listings can also be cached between runs, which removes them from the rate limit budget +entirely: see [Caching Release Lists](#caching-release-lists). + ## Failure Reporting `dload get` exits with a non-zero code when at least one requested package was not installed, and prints diff --git a/dload.xsd b/dload.xsd index 793255e..5fbf7ae 100644 --- a/dload.xsd +++ b/dload.xsd @@ -249,6 +249,16 @@ Temporary directory for downloads + + + Directory to cache release listings in; caching is disabled when not set + + + + + Number of seconds a cached release listing stays usable; 0 disables caching + + diff --git a/src/Bootstrap.php b/src/Bootstrap.php index 887153a..a795003 100644 --- a/src/Bootstrap.php +++ b/src/Bootstrap.php @@ -8,10 +8,14 @@ use Internal\Container\ObjectContainer; use Internal\DLoad\Module\Binary\BinaryProvider; use Internal\DLoad\Module\Binary\Internal\BinaryProviderImpl; +use Internal\DLoad\Module\Cache\Internal\FileResponseCache; +use Internal\DLoad\Module\Cache\Internal\NullResponseCache; +use Internal\DLoad\Module\Cache\ResponseCache; use Internal\DLoad\Module\Common\Architecture; use Internal\DLoad\Module\Common\Internal\Injection\ConfigInflector; use Internal\DLoad\Module\Common\OperatingSystem; use Internal\DLoad\Module\Common\Stability; +use Internal\DLoad\Module\Config\Schema\Cache as CacheConfig; use Internal\DLoad\Module\HttpClient\Factory; use Internal\DLoad\Module\HttpClient\Internal\NyholmFactoryImpl; use Internal\DLoad\Module\Repository\Internal\GitHub\Factory as GithubRepositoryFactory; @@ -21,6 +25,7 @@ use Internal\DLoad\Module\Velox\Builder; use Internal\DLoad\Module\Velox\Internal\Client\BuildRoadRunner; use Internal\DLoad\Module\Velox\Internal\VeloxBuilder; +use Internal\DLoad\Service\Logger; /** * Bootstraps the application by configuring the dependency container. @@ -113,6 +118,16 @@ public function withConfig( ->addRepositoryFactory($container->get(GithubRepositoryFactory::class)) ->addRepositoryFactory($container->get(GitLabRepositoryFactory::class)), ); + $this->container->bind( + ResponseCache::class, + static function (Container $container): ResponseCache { + $config = $container->get(CacheConfig::class); + + return $config->dir === null || $config->ttl <= 0 + ? new NullResponseCache() + : new FileResponseCache($config->dir, $config->ttl, $container->get(Logger::class)); + }, + ); $this->container->bind(BinaryProvider::class, BinaryProviderImpl::class); $this->container->bind(Factory::class, NyholmFactoryImpl::class); $this->container->bind(Builder::class, VeloxBuilder::class); diff --git a/src/Module/Cache/Internal/FileResponseCache.php b/src/Module/Cache/Internal/FileResponseCache.php new file mode 100644 index 0000000..8e5ccbd --- /dev/null +++ b/src/Module/Cache/Internal/FileResponseCache.php @@ -0,0 +1,134 @@ + $ttl + */ + public function __construct( + private readonly string $directory, + private readonly int $ttl, + private readonly Logger $logger, + ) {} + + public function remember(string $key, \Closure $fetch): ResponseInterface + { + $file = $this->fileOf($key); + + $cached = $this->read($file); + if ($cached !== null) { + return $cached; + } + + $response = $fetch(); + $this->write($file, $response); + + return $response; + } + + private static function readBody(ResponseInterface $response): string + { + $stream = $response->getBody(); + + $stream->isSeekable() and $stream->rewind(); + $body = $stream->getContents(); + $stream->isSeekable() and $stream->rewind(); + + return $body; + } + + private function read(string $file): ?ResponseInterface + { + if (!\is_file($file)) { + return null; + } + + $content = @\file_get_contents($file); + if ($content === false) { + $this->discard($file); + return null; + } + + try { + /** @var mixed $payload */ + $payload = \json_decode($content, true, 512, JSON_THROW_ON_ERROR); + + \is_array($payload) + && \is_int($payload['created_at'] ?? null) + && \is_int($payload['status'] ?? null) + && \is_array($payload['headers'] ?? null) + && \is_string($payload['body'] ?? null) + or throw new \UnexpectedValueException('Unexpected cache entry structure.'); + } catch (\Throwable) { + $this->discard($file); + return null; + } + + /** @var array{created_at: int, status: int, headers: array>, body: string} $payload */ + + if (\time() - $payload['created_at'] > $this->ttl) { + return null; + } + + return new Response($payload['status'], $payload['headers'], $payload['body']); + } + + private function write(string $file, ResponseInterface $response): void + { + $status = $response->getStatusCode(); + + if ($status < 200 || $status > 299) { + return; + } + + try { + $payload = \json_encode([ + 'created_at' => \time(), + 'status' => $status, + 'headers' => $response->getHeaders(), + 'body' => self::readBody($response), + ], JSON_THROW_ON_ERROR); + + if (!\is_dir($this->directory) && !@\mkdir($this->directory, 0777, true) && !\is_dir($this->directory)) { + throw new \RuntimeException(\sprintf('Failed to create cache directory `%s`.', $this->directory)); + } + + $temp = $file . '.' . (string) \getmypid() . '.tmp'; + + if (@\file_put_contents($temp, $payload) === false) { + throw new \RuntimeException(\sprintf('Failed to write cache entry `%s`.', $temp)); + } + + if (!@\rename($temp, $file)) { + @\unlink($temp); + throw new \RuntimeException(\sprintf('Failed to store cache entry `%s`.', $file)); + } + } catch (\Throwable $e) { + $this->logger->exception($e, important: false); + } + } + + private function discard(string $file): void + { + @\unlink($file); + } + + private function fileOf(string $key): string + { + return $this->directory . \DIRECTORY_SEPARATOR . \hash('xxh128', $key) . '.json'; + } +} diff --git a/src/Module/Cache/Internal/NullResponseCache.php b/src/Module/Cache/Internal/NullResponseCache.php new file mode 100644 index 0000000..6f1cecf --- /dev/null +++ b/src/Module/Cache/Internal/NullResponseCache.php @@ -0,0 +1,20 @@ +repositoryPath = $owner . '/' . $repo; } @@ -196,13 +199,12 @@ private function decodeReleasesResponse(ResponseInterface $response): array */ private function releasesRequest(int $page): ResponseInterface { - return $this->request( - Method::Get, - $this->httpFactory->uri( - \sprintf(self::URL_RELEASES, $this->repositoryPath), - ['page' => $page], - ), + $uri = $this->httpFactory->uri( + \sprintf(self::URL_RELEASES, $this->repositoryPath), + ['page' => $page, 'per_page' => self::RELEASES_PER_PAGE], ); + + return $this->cache->remember((string) $uri, fn(): ResponseInterface => $this->request(Method::Get, $uri)); } private function hasNextPage(ResponseInterface $response): bool diff --git a/src/Module/Repository/Internal/GitHub/Factory.php b/src/Module/Repository/Internal/GitHub/Factory.php index 77fd1c4..8f74d95 100644 --- a/src/Module/Repository/Internal/GitHub/Factory.php +++ b/src/Module/Repository/Internal/GitHub/Factory.php @@ -4,6 +4,7 @@ namespace Internal\DLoad\Module\Repository\Internal\GitHub; +use Internal\DLoad\Module\Cache\ResponseCache; use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitHub; use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; @@ -30,6 +31,7 @@ public function __construct( private readonly HttpFactory $httpFactory, GitHub $gitHubConfig, private readonly Logger $logger, + private readonly ResponseCache $cache, ) { $this->gitHubClient = new Client( $httpFactory, @@ -59,6 +61,13 @@ public function create(RepositoryConfig $config): GitHubRepository */ private function createRepositoryApi(string $owner, string $repo): RepositoryApi { - return new RepositoryApi($this->gitHubClient, $this->httpFactory, $owner, $repo, $this->logger); + return new RepositoryApi( + $this->gitHubClient, + $this->httpFactory, + $owner, + $repo, + $this->logger, + $this->cache, + ); } } diff --git a/src/Module/Repository/Internal/GitHub/GitHubRepository.php b/src/Module/Repository/Internal/GitHub/GitHubRepository.php index 2c4d9be..a56a211 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubRepository.php +++ b/src/Module/Repository/Internal/GitHub/GitHubRepository.php @@ -53,7 +53,8 @@ public function getReleases(): ReleasesCollection // Create a generator function for lazy loading release pages $pageLoader = function (): \Generator { - $page = 0; + /** @var \Internal\DLoad\Module\Repository\Internal\Paginator|null $page */ + $page = null; $anyPageLoaded = false; do { @@ -61,11 +62,14 @@ public function getReleases(): ReleasesCollection // to avoid first eager loading because of generator yield []; - $paginator = $this->api->getReleases(++$page); - $releases = $paginator->getPageItems(); + $page = $page === null ? $this->api->getReleases() : $page->getNextPage(); + + if ($page === null) { + return; + } $toYield = []; - foreach ($releases as $releaseDTO) { + foreach ($page->getPageItems() as $releaseDTO) { try { $toYield[] = GitHubRelease::fromDTO($this->api, $this, $releaseDTO); } catch (\Throwable $e) { @@ -76,9 +80,6 @@ public function getReleases(): ReleasesCollection } yield $toYield; $anyPageLoaded = true; - - // Check if there are more pages by getting next page - $hasMorePages = $paginator->getNextPage() !== null; } catch (\Throwable $e) { # The first page is mandatory: when it fails, there is nothing to download and the reason # (invalid token, rate limit, missing repository, etc.) must reach the user. @@ -93,7 +94,7 @@ public function getReleases(): ReleasesCollection $this->logger->exception($e, important: false); return; } - } while ($hasMorePages); + } while (true); }; // Create paginator diff --git a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php index 6af6c71..93395e6 100644 --- a/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php +++ b/src/Module/Repository/Internal/GitLab/Api/RepositoryApi.php @@ -4,6 +4,7 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab\Api; +use Internal\DLoad\Module\Cache\ResponseCache; use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; use Internal\DLoad\Module\HttpClient\Method; use Internal\DLoad\Module\Repository\Exception\ApiException; @@ -27,6 +28,7 @@ final class RepositoryApi private const URL_REPOSITORY = 'https://gitlab.com/api/v4/projects/%s'; private const URL_RELEASES = 'https://gitlab.com/api/v4/projects/%s/releases'; private const URL_RELEASE_ASSET = 'https://gitlab.com/api/v4/projects/%s/releases/%s/downloads/%s'; + private const RELEASES_PER_PAGE = 100; /** * @var non-empty-string @@ -40,6 +42,7 @@ public function __construct( private readonly Client $client, private readonly HttpFactory $httpFactory, string $projectPath, + private readonly ResponseCache $cache, ) { $this->repositoryPath = $projectPath; } @@ -207,13 +210,12 @@ private function decodeReleasesResponse(ResponseInterface $response): array */ private function releasesRequest(int $page): ResponseInterface { - return $this->request( - Method::Get, - $this->httpFactory->uri( - \sprintf(self::URL_RELEASES, \urlencode($this->repositoryPath)), - ['page' => $page], - ), + $uri = $this->httpFactory->uri( + \sprintf(self::URL_RELEASES, \urlencode($this->repositoryPath)), + ['page' => $page, 'per_page' => self::RELEASES_PER_PAGE], ); + + return $this->cache->remember((string) $uri, fn(): ResponseInterface => $this->request(Method::Get, $uri)); } private function hasNextPage(ResponseInterface $response): bool diff --git a/src/Module/Repository/Internal/GitLab/Factory.php b/src/Module/Repository/Internal/GitLab/Factory.php index 68d601d..ef4263c 100644 --- a/src/Module/Repository/Internal/GitLab/Factory.php +++ b/src/Module/Repository/Internal/GitLab/Factory.php @@ -4,6 +4,7 @@ namespace Internal\DLoad\Module\Repository\Internal\GitLab; +use Internal\DLoad\Module\Cache\ResponseCache; use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitLab; use Internal\DLoad\Module\HttpClient\Factory as HttpFactory; @@ -30,6 +31,7 @@ public function __construct( private readonly HttpFactory $httpFactory, GitLab $gitLabConfig, private readonly Logger $logger, + private readonly ResponseCache $cache, ) { $this->gitLabClient = new Client( $httpFactory, @@ -57,6 +59,6 @@ public function create(RepositoryConfig $config): GitLabRepository */ private function createRepositoryApi(string $projectPath): RepositoryApi { - return new RepositoryApi($this->gitLabClient, $this->httpFactory, $projectPath); + return new RepositoryApi($this->gitLabClient, $this->httpFactory, $projectPath, $this->cache); } } diff --git a/src/Module/Repository/Internal/GitLab/GitLabRepository.php b/src/Module/Repository/Internal/GitLab/GitLabRepository.php index 94f3dff..066ca1a 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabRepository.php +++ b/src/Module/Repository/Internal/GitLab/GitLabRepository.php @@ -51,7 +51,8 @@ public function getReleases(): ReleasesCollection // Create a generator function for lazy loading release pages $pageLoader = function (): \Generator { - $page = 0; + /** @var \Internal\DLoad\Module\Repository\Internal\Paginator|null $page */ + $page = null; $anyPageLoaded = false; do { @@ -59,11 +60,14 @@ public function getReleases(): ReleasesCollection // to avoid first eager loading because of generator yield []; - $paginator = $this->api->getReleases(++$page); - $releases = $paginator->getPageItems(); + $page = $page === null ? $this->api->getReleases() : $page->getNextPage(); + + if ($page === null) { + return; + } $toYield = []; - foreach ($releases as $releaseDTO) { + foreach ($page->getPageItems() as $releaseDTO) { try { $toYield[] = GitLabRelease::fromDTO($this->api, $this, $releaseDTO); } catch (\Throwable) { @@ -73,9 +77,6 @@ public function getReleases(): ReleasesCollection } yield $toYield; $anyPageLoaded = true; - - // Check if there are more pages by getting next page - $hasMorePages = $paginator->getNextPage() !== null; } catch (\Throwable $e) { # The first page is mandatory: when it fails, there is nothing to download and the reason # (invalid token, rate limit, missing project, etc.) must reach the user. @@ -90,7 +91,7 @@ public function getReleases(): ReleasesCollection $this->logger->exception($e, important: false); return; } - } while ($hasMorePages); + } while (true); }; // Create paginator diff --git a/tests/Integration/Module/Cache/ResponseCacheBindingTest.php b/tests/Integration/Module/Cache/ResponseCacheBindingTest.php new file mode 100644 index 0000000..9b89d82 --- /dev/null +++ b/tests/Integration/Module/Cache/ResponseCacheBindingTest.php @@ -0,0 +1,69 @@ + \sys_get_temp_dir() . '/dload-cache-binding']), + FileResponseCache::class, + ); + } + + #[Test] + public function xmlAttributeEnablesTheFileCache(): void + { + Assert::instanceOf( + self::resolve(xml: \sprintf( + '', + \sys_get_temp_dir() . '/dload-cache-binding', + )), + FileResponseCache::class, + ); + } + + #[Test] + public function zeroTtlDisablesTheCache(): void + { + Assert::instanceOf( + self::resolve(environment: [ + 'DLOAD_CACHE_DIR' => \sys_get_temp_dir() . '/dload-cache-binding', + 'DLOAD_CACHE_TTL' => '0', + ]), + NullResponseCache::class, + ); + } + + /** + * @param array $environment + */ + private static function resolve(?string $xml = null, array $environment = []): ResponseCache + { + return Bootstrap::init() + ->withConfig(xml: $xml, environment: $environment) + ->finish() + ->get(ResponseCache::class); + } +} diff --git a/tests/Unit/Module/Cache/FileResponseCacheTest.php b/tests/Unit/Module/Cache/FileResponseCacheTest.php new file mode 100644 index 0000000..4a4b579 --- /dev/null +++ b/tests/Unit/Module/Cache/FileResponseCacheTest.php @@ -0,0 +1,249 @@ +cache(); + $calls = 0; + $fetch = static function () use (&$calls): ResponseInterface { + ++$calls; + return new Response(200, ['link' => '; rel="next"'], 'first'); + }; + + $first = $cache->remember('https://api.github.com/x', $fetch); + $second = $cache->remember('https://api.github.com/x', $fetch); + + Assert::same($calls, 1); + Assert::same((string) $second->getBody(), (string) $first->getBody()); + Assert::same($second->getStatusCode(), 200); + + Assert::same($second->getHeaderLine('link'), '; rel="next"'); + } + + #[Test] + public function differentKeysAreCachedApart(): void + { + $cache = $this->cache(); + + $first = $cache->remember('https://api.github.com/a', static fn(): ResponseInterface => new Response(200, [], 'a')); + $second = $cache->remember('https://api.github.com/b', static fn(): ResponseInterface => new Response(200, [], 'b')); + + Assert::same((string) $first->getBody(), 'a'); + Assert::same((string) $second->getBody(), 'b'); + Assert::same((string) $cache->remember('https://api.github.com/a', self::unexpectedFetch(...))->getBody(), 'a'); + } + + #[Test] + public function entryOlderThanTheTtlIsFetchedAgain(): void + { + $cache = $this->cache(ttl: 60); + $cache->remember('https://api.github.com/x', static fn(): ResponseInterface => new Response(200, [], 'stale')); + + $this->ageStoredEntries(120); + + $response = $cache->remember('https://api.github.com/x', static fn(): ResponseInterface => new Response(200, [], 'fresh')); + + Assert::same((string) $response->getBody(), 'fresh'); + } + + #[Test] + public function entryWithinTheTtlIsKept(): void + { + $cache = $this->cache(ttl: 600); + $cache->remember('https://api.github.com/x', static fn(): ResponseInterface => new Response(200, [], 'cached')); + + $this->ageStoredEntries(60, touchFiles: true); + + $response = $cache->remember('https://api.github.com/x', self::unexpectedFetch(...)); + + Assert::same((string) $response->getBody(), 'cached'); + } + + #[Test] + public function unsuccessfulResponseIsNotCached(): void + { + $cache = $this->cache(); + $calls = 0; + $fetch = static function () use (&$calls): ResponseInterface { + ++$calls; + return new Response(403, [], 'rate limit exceeded'); + }; + + $cache->remember('https://api.github.com/x', $fetch); + $cache->remember('https://api.github.com/x', $fetch); + + Assert::same($calls, 2); + Assert::same(\glob($this->directory . '/*.json'), []); + } + + #[Test] + public function corruptedEntryIsIgnored(): void + { + $cache = $this->cache(); + $cache->remember('https://api.github.com/x', static fn(): ResponseInterface => new Response(200, [], 'cached')); + + foreach (\glob($this->directory . '/*.json') as $file) { + \file_put_contents($file, 'not a json payload'); + } + + $response = $cache->remember('https://api.github.com/x', static fn(): ResponseInterface => new Response(200, [], 'refetched')); + + Assert::same((string) $response->getBody(), 'refetched'); + } + + #[Test] + public function nullCacheAlwaysFetches(): void + { + $cache = new NullResponseCache(); + $calls = 0; + $fetch = static function () use (&$calls): ResponseInterface { + ++$calls; + return new Response(200, [], 'body'); + }; + + $cache->remember('https://api.github.com/x', $fetch); + $response = $cache->remember('https://api.github.com/x', $fetch); + + Assert::same($calls, 2); + Assert::same((string) $response->getBody(), 'body'); + } + + #[Test] + public function unreadableEntryIsFetchedAgain(): void + { + $cache = $this->cache(); + $cache->remember('https://api.github.com/x', static fn(): ResponseInterface => new Response(200, [], 'cached')); + + foreach (\glob($this->directory . '/*.json') as $file) { + \chmod($file, 0000); + \is_readable($file) and throw new SkipTest('The entry stays readable for this user.'); + } + + $response = $cache->remember('https://api.github.com/x', static fn(): ResponseInterface => new Response(200, [], 'refetched')); + + Assert::same((string) $response->getBody(), 'refetched'); + } + + #[Test] + public function responseIsReturnedWhenTheDirectoryCannotBeCreated(): void + { + \file_put_contents($this->directory, 'a file where the cache directory should be'); + $cache = new FileResponseCache($this->directory . '/entries', 600, new Logger()); + + $response = $cache->remember('https://api.github.com/x', static fn(): ResponseInterface => new Response(200, [], 'body')); + + Assert::same((string) $response->getBody(), 'body'); + } + + #[Test] + public function responseIsReturnedWhenTheEntryCannotBeWritten(): void + { + $cache = $this->cache(); + $file = $this->storeAndForget($cache); + + \mkdir($file . '.' . \getmypid() . '.tmp'); + + $response = $cache->remember('https://api.github.com/x', static fn(): ResponseInterface => new Response(200, [], 'body')); + + Assert::same((string) $response->getBody(), 'body'); + } + + #[Test] + public function responseIsReturnedWhenTheEntryCannotBeMovedIntoPlace(): void + { + $cache = $this->cache(); + $file = $this->storeAndForget($cache); + + \mkdir($file); + \file_put_contents($file . '/occupied', 'x'); + + $response = $cache->remember('https://api.github.com/x', static fn(): ResponseInterface => new Response(200, [], 'body')); + + Assert::same((string) $response->getBody(), 'body'); + Assert::same(\file_get_contents($file . '/occupied'), 'x'); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->directory = \sys_get_temp_dir() . '/dload-response-cache-' . \bin2hex(\random_bytes(6)); + } + + #[AfterTest] + protected function cleanup(): void + { + self::erase($this->directory); + } + + private static function erase(string $path): void + { + if (\is_dir($path)) { + foreach (\glob($path . '/*') as $child) { + self::erase($child); + } + + \rmdir($path); + return; + } + + \is_file($path) and \unlink($path); + } + + private static function unexpectedFetch(): ResponseInterface + { + throw new \LogicException('The cached entry must be used instead of fetching again.'); + } + + private function cache(int $ttl = 600): FileResponseCache + { + return new FileResponseCache($this->directory, $ttl, new Logger()); + } + + /** + * @return non-empty-string + */ + private function storeAndForget(FileResponseCache $cache): string + { + $cache->remember('https://api.github.com/x', static fn(): ResponseInterface => new Response(200, [], 'stored')); + + $files = \glob($this->directory . '/*.json'); + Assert::same(\count($files), 1); + + \unlink($files[0]); + + return $files[0]; + } + + private function ageStoredEntries(int $seconds, bool $touchFiles = false): void + { + foreach (\glob($this->directory . '/*.json') as $file) { + $payload = \json_decode(\file_get_contents($file), true, 512, JSON_THROW_ON_ERROR); + $payload['created_at'] -= $seconds; + \file_put_contents($file, \json_encode($payload, JSON_THROW_ON_ERROR)); + + $touchFiles and \touch($file); + } + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitHub/FactoryTest.php b/tests/Unit/Module/Repository/Internal/GitHub/FactoryTest.php new file mode 100644 index 0000000..1f39be2 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitHub/FactoryTest.php @@ -0,0 +1,79 @@ + ['github', true]; + yield 'mixed case' => ['GitHub', true]; + yield 'uppercase' => ['GITHUB', true]; + yield 'gitlab' => ['gitlab', false]; + yield 'unknown' => ['custom', false]; + } + + public static function provideRepositoryUris(): \Generator + { + yield 'bare repository path' => ['owner/repo', 'owner/repo']; + yield 'full url' => ['https://github.com/owner/repo', 'owner/repo']; + } + + #[DataProvider('provideSupportedTypes')] + #[Test] + public function supportsOnlyGithubRegardlessOfCase(string $type, bool $expected): void + { + $config = new RepositoryConfig(); + $config->type = $type; + $config->uri = 'owner/repo'; + + Assert::same($this->factory->supports($config), $expected); + } + + #[DataProvider('provideRepositoryUris')] + #[Test] + public function createDerivesTheRepositoryNameFromTheUri(string $uri, string $expectedName): void + { + $config = new RepositoryConfig(); + $config->type = 'github'; + $config->uri = $uri; + + Assert::same($this->factory->create($config)->getName(), $expectedName); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->factory = new Factory( + new HttpFactoryStub( + static fn(): UriInterface&MockInterface => \Mockery::mock(UriInterface::class)->shouldIgnoreMissing(), + static fn(): RequestInterface&MockInterface => \Mockery::mock(RequestInterface::class)->shouldIgnoreMissing(), + static fn(): ClientInterface&MockInterface => \Mockery::mock(ClientInterface::class)->shouldIgnoreMissing(), + ), + new GitHubConfig(), + new Logger(), + new NullResponseCache(), + ); + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php new file mode 100644 index 0000000..d1b1956 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php @@ -0,0 +1,123 @@ +getReleases(), false); + + Assert::same(\count($releases), 6); + Assert::same($client->requestedPages(), [1, 2, 3]); + } + + #[Test] + public function pagesAreLoadedOnlyWhenNeeded(): void + { + $client = new PagedClientStub(pages: 3, releasesPerPage: 2); + $repository = self::createRepository($client); + + foreach ($repository->getReleases() as $release) { + unset($release); + break; + } + + Assert::same($client->requestedPages(), [1]); + } + + #[Test] + public function releasesAreRequestedAHundredPerPage(): void + { + $client = new PagedClientStub(pages: 1, releasesPerPage: 2); + $repository = self::createRepository($client); + + \iterator_to_array($repository->getReleases(), false); + + Assert::same($client->requests, ['page=1&per_page=100']); + } + + #[Test] + public function cachedListingsCostNoRequestsOnTheNextRun(): void + { + $firstClient = new PagedClientStub(pages: 2, releasesPerPage: 2); + $firstRun = \iterator_to_array(self::createRepository($firstClient, $this->cache())->getReleases(), false); + + $secondClient = new PagedClientStub(pages: 2, releasesPerPage: 2); + $secondRun = \iterator_to_array(self::createRepository($secondClient, $this->cache())->getReleases(), false); + + Assert::same($firstClient->requestedPages(), [1, 2]); + Assert::same($secondClient->requestedPages(), []); + Assert::same(\count($firstRun), 4); + Assert::same(\count($secondRun), 4); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->cacheDirectory = \sys_get_temp_dir() . '/dload-github-cache-' . \bin2hex(\random_bytes(6)); + } + + #[AfterTest] + protected function cleanup(): void + { + if (!\is_dir($this->cacheDirectory)) { + return; + } + + foreach (\glob($this->cacheDirectory . '/*') as $file) { + \is_file($file) and \unlink($file); + } + + \rmdir($this->cacheDirectory); + } + + private static function createRepository( + PagedClientStub $client, + ResponseCache $cache = new NullResponseCache(), + ): GitHubRepository { + $logger = new Logger(); + $httpFactory = new NyholmFactoryImpl($logger); + $api = new RepositoryApi( + new Client($httpFactory, $client, new GitHubConfig()), + $httpFactory, + 'owner', + 'repo', + $logger, + $cache, + ); + + return new GitHubRepository($api, 'owner', 'repo', $logger); + } + + private function cache(): FileResponseCache + { + return new FileResponseCache($this->cacheDirectory, 600, new Logger()); + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php new file mode 100644 index 0000000..4102336 --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php @@ -0,0 +1,84 @@ + + */ + public array $requests = []; + + /** + * @param int<1, max> $pages + * @param int<1, max> $releasesPerPage + */ + public function __construct( + private readonly int $pages = 1, + private readonly int $releasesPerPage = 2, + ) {} + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $query = $request->getUri()->getQuery(); + $this->requests[] = $query; + + $page = self::pageOf($query); + + if ($page > $this->pages) { + return new ResponseStub(200, [], '[]'); + } + + $headers = $page < $this->pages + ? ['link' => [\sprintf('; rel="next"', $page + 1)]] + : []; + + return new ResponseStub(200, $headers, \json_encode($this->releasesOfPage($page))); + } + + /** + * @return list + */ + public function requestedPages(): array + { + return \array_map(self::pageOf(...), $this->requests); + } + + private static function pageOf(string $query): int + { + \parse_str($query, $params); + + return (int) ($params['page'] ?? 1); + } + + /** + * @param int<1, max> $page + * @return list> + */ + private function releasesOfPage(int $page): array + { + $releases = []; + $offset = ($page - 1) * $this->releasesPerPage; + + for ($i = 1; $i <= $this->releasesPerPage; $i++) { + $tag = \sprintf('v1.0.%d', $offset + $i); + $releases[] = [ + 'name' => $tag, + 'tag_name' => $tag, + 'published_at' => '2024-01-01T00:00:00Z', + 'assets' => [], + 'prerelease' => false, + 'draft' => false, + ]; + } + + return $releases; + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php b/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php index 96b9527..74f5e40 100644 --- a/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitLab/FactoryTest.php @@ -4,6 +4,7 @@ namespace Internal\DLoad\Tests\Unit\Module\Repository\Internal\GitLab; +use Internal\DLoad\Module\Cache\Internal\NullResponseCache; use Internal\DLoad\Module\Config\Schema\Embed\Repository as RepositoryConfig; use Internal\DLoad\Module\Config\Schema\GitLab as GitLabConfig; use Internal\DLoad\Module\Repository\Internal\GitLab\Factory; @@ -69,6 +70,11 @@ public function createDerivesTheProjectPathFromTheUri(string $uri, string $expec #[BeforeTest] protected function prepare(): void { - $this->factory = new Factory(new HttpFactoryStub(), new GitLabConfig(), new Logger()); + $this->factory = new Factory( + new HttpFactoryStub(), + new GitLabConfig(), + new Logger(), + new NullResponseCache(), + ); } } diff --git a/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php new file mode 100644 index 0000000..4bb885d --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php @@ -0,0 +1,59 @@ +getReleases(), false); + + Assert::same(\count($releases), 6); + Assert::same($client->requestedPages(), [1, 2, 3]); + } + + #[Test] + public function releasesAreRequestedAHundredPerPage(): void + { + $client = new PagedClientStub(pages: 1, releasesPerPage: 2); + $repository = self::createRepository($client); + + \iterator_to_array($repository->getReleases(), false); + + Assert::same($client->requests, ['page=1&per_page=100']); + } + + private static function createRepository(PagedClientStub $client): GitLabRepository + { + $logger = new Logger(); + $httpFactory = new NyholmFactoryImpl($logger); + $api = new RepositoryApi( + new Client($httpFactory, $client, new GitLabConfig()), + $httpFactory, + 'group/project', + new NullResponseCache(), + ); + + return new GitLabRepository($api, 'group/project', $logger); + } +} diff --git a/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php new file mode 100644 index 0000000..01fcb9b --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php @@ -0,0 +1,85 @@ + + */ + public array $requests = []; + + /** + * @param int<1, max> $pages + * @param int<1, max> $releasesPerPage + */ + public function __construct( + private readonly int $pages = 1, + private readonly int $releasesPerPage = 2, + ) {} + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $query = $request->getUri()->getQuery(); + $this->requests[] = $query; + + $page = self::pageOf($query); + + if ($page > $this->pages) { + return new ResponseStub(200, [], '[]'); + } + + $headers = $page < $this->pages + ? ['link' => [\sprintf('; rel="next"', $page + 1)]] + : []; + + return new ResponseStub(200, $headers, \json_encode($this->releasesOfPage($page))); + } + + /** + * @return list + */ + public function requestedPages(): array + { + return \array_map(self::pageOf(...), $this->requests); + } + + private static function pageOf(string $query): int + { + \parse_str($query, $params); + + return (int) ($params['page'] ?? 1); + } + + /** + * @param int<1, max> $page + * @return list> + */ + private function releasesOfPage(int $page): array + { + $releases = []; + $offset = ($page - 1) * $this->releasesPerPage; + + for ($i = 1; $i <= $this->releasesPerPage; $i++) { + $tag = \sprintf('v1.0.%d', $offset + $i); + $releases[] = [ + 'name' => $tag, + 'tag_name' => $tag, + 'description' => 'Release ' . $tag, + 'created_at' => '2024-01-01T00:00:00Z', + 'released_at' => '2024-01-01T00:00:00Z', + 'assets' => ['links' => []], + 'upcoming_release' => false, + ]; + } + + return $releases; + } +}