diff --git a/Classes/BackendUi/BackendUiDataService.php b/Classes/BackendUi/BackendUiDataService.php index a908836..cb8cde3 100644 --- a/Classes/BackendUi/BackendUiDataService.php +++ b/Classes/BackendUi/BackendUiDataService.php @@ -88,6 +88,16 @@ public function loadBackendOverviewData(RedisInstanceIdentifier $redisInstanceId ...$contentReleaseIds, ); + try { + $jobsById = []; + foreach ($this->prunnerApiService->loadPipelinesAndJobs()->getJobs() as $job) { + $jobsById[$job->getId()->getId()] = $job; + } + } catch (\Throwable $throwable) { + // the overview also holds the controls to pause releases, which must keep working while prunner is down + $jobsById = null; + } + $result = []; foreach ($contentReleaseIds as $contentReleaseId) { $lastRendering = RenderingStatistics::fromJsonString($lastRenderingStatisticsEntries->getResultForContentRelease( @@ -98,13 +108,27 @@ public function loadBackendOverviewData(RedisInstanceIdentifier $redisInstanceId )); $metadataForContentRelease = $metadata->getResultForContentRelease($contentReleaseId); + if (!$metadataForContentRelease instanceof ContentReleaseMetadata) { + $metadataForContentRelease = null; + } + $jobForContentRelease = $metadataForContentRelease !== null + ? $jobsById[$metadataForContentRelease->getPrunnerJobId()->getIdentifier()] ?? null + : null; + $status = match (true) { + $jobForContentRelease !== null => ContentReleaseStatus::fromJob( + $jobForContentRelease, + $metadataForContentRelease, + ), + $metadataForContentRelease === null || $jobsById === null => ContentReleaseStatus::UNKNOWN, + default => ContentReleaseStatus::jobGone($metadataForContentRelease), + }; $countForContentRelease = $counts->getResultForContentRelease($contentReleaseId); $iterationsCountForContentRelease = $iterationsCounts->getResultForContentRelease($contentReleaseId); $errorCountForContentRelease = $errorCounts->getResultForContentRelease($contentReleaseId); $result[] = new ContentReleaseOverviewRow( $contentReleaseId, - $metadataForContentRelease instanceof ContentReleaseMetadata ? $metadataForContentRelease : null, + $metadataForContentRelease, is_int($countForContentRelease) ? $countForContentRelease : 0, is_int($iterationsCountForContentRelease) ? $iterationsCountForContentRelease : 0, is_int($errorCountForContentRelease) ? $errorCountForContentRelease : 0, @@ -115,9 +139,8 @@ public function loadBackendOverviewData(RedisInstanceIdentifier $redisInstanceId $contentReleaseId->equals($this->redisReleaseSwitchService->getCurrentRelease( $redisInstanceIdentifier, )), - $metadataForContentRelease instanceof ContentReleaseMetadata - ? $metadataForContentRelease->getContentReleaseSize() - : null, + $metadataForContentRelease?->getContentReleaseSize(), + $status, ); } diff --git a/Classes/BackendUi/ContentReleaseStatus.php b/Classes/BackendUi/ContentReleaseStatus.php new file mode 100644 index 0000000..96a4da6 --- /dev/null +++ b/Classes/BackendUi/ContentReleaseStatus.php @@ -0,0 +1,71 @@ +isCanceled()) { + return self::CANCELED; + } + if ($job->isErrored() || ($metadata?->getStatus()->isFailed() ?? false)) { + return self::FAILED; + } + if ($job->isCompleted()) { + return self::SUCCESS; + } + return $job->getStart() === null ? self::SCHEDULED : self::RUNNING; + } + + /** + * Prunner answered, but no longer holds the job: it has fallen out of the pipeline's retention, so the release is + * not being built anymore. One which was switched live succeeded; one which was neither switched nor marked + * failed was cancelled or killed with its worker. + */ + public static function jobGone(ContentReleaseMetadata $metadata): self + { + if ($metadata->getStatus()->isFailed()) { + return self::FAILED; + } + return $metadata->getSwitchTime() !== null ? self::SUCCESS : self::ABORTED; + } + + /** + * A getter, so Fusion reads it as the property "badgeClass": Eel only calls methods of objects which allow it. + */ + public function getBadgeClass(): string + { + return match ($this) { + self::SUCCESS => 'neos-badge neos-badge-success', + self::FAILED => 'neos-badge neos-badge-important', + self::SCHEDULED, self::RUNNING, self::CANCELED, self::ABORTED, self::UNKNOWN => 'neos-badge', + }; + } +} diff --git a/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php b/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php index fbdebe4..3e03328 100644 --- a/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php +++ b/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php @@ -4,6 +4,7 @@ namespace Flowpack\DecoupledContentStore\BackendUi\Dto; +use Flowpack\DecoupledContentStore\BackendUi\ContentReleaseStatus; use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier; use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata; use Neos\Flow\Annotations as Flow; @@ -22,6 +23,7 @@ class ContentReleaseOverviewRow private int $renderedUrlCount; private bool $isActive; private ?float $releaseSize; + private ContentReleaseStatus $status; public function __construct( ContentReleaseIdentifier $contentReleaseIdentifier, @@ -33,6 +35,7 @@ public function __construct( int $renderedUrlCount, bool $isActive, ?float $releaseSize, + ContentReleaseStatus $status, ) { $this->contentReleaseIdentifier = $contentReleaseIdentifier; $this->metadata = $metadata; @@ -43,6 +46,7 @@ public function __construct( $this->renderedUrlCount = $renderedUrlCount; $this->isActive = $isActive; $this->releaseSize = $releaseSize; + $this->status = $status; } public function getMetadata(): ?ContentReleaseMetadata @@ -92,4 +96,9 @@ public function getReleaseSize(): ?float { return $this->releaseSize; } + + public function getStatus(): ContentReleaseStatus + { + return $this->status; + } } diff --git a/Classes/BackendUi/Dto/RemovedContentReleaseOverviewRow.php b/Classes/BackendUi/Dto/RemovedContentReleaseOverviewRow.php index 3941b36..7fc3333 100644 --- a/Classes/BackendUi/Dto/RemovedContentReleaseOverviewRow.php +++ b/Classes/BackendUi/Dto/RemovedContentReleaseOverviewRow.php @@ -4,6 +4,7 @@ namespace Flowpack\DecoupledContentStore\BackendUi\Dto; +use Flowpack\DecoupledContentStore\BackendUi\ContentReleaseStatus; use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier; use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata; use Flowpack\Prunner\Dto\Job; @@ -45,17 +46,8 @@ public function getErrorCount(): int return $this->errorCount; } - public function getStatus(): string + public function getStatus(): ContentReleaseStatus { - if ($this->metadata !== null) { - return $this->metadata->getStatus()->getDisplayName(); - } - if ($this->job->isCanceled()) { - return 'canceled'; - } - if ($this->job->isErrored()) { - return 'failed'; - } - return $this->job->isCompleted() ? 'done' : 'running'; + return ContentReleaseStatus::fromJob($this->job, $this->metadata); } } diff --git a/Classes/Command/ContentReleasePrepareCommandController.php b/Classes/Command/ContentReleasePrepareCommandController.php index a64da0a..62a98fc 100644 --- a/Classes/Command/ContentReleasePrepareCommandController.php +++ b/Classes/Command/ContentReleasePrepareCommandController.php @@ -7,7 +7,9 @@ use Flowpack\DecoupledContentStore\Core\ConcurrentBuildLockService; use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier; use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\PrunnerJobId; +use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\RedisInstanceIdentifier; use Flowpack\DecoupledContentStore\Core\Infrastructure\ContentReleaseLogger; +use Flowpack\DecoupledContentStore\NodeRendering\Dto\NodeRenderingCompletionStatus; use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService; use Neos\Flow\Annotations as Flow; use Neos\Flow\Cli\CommandController; @@ -70,6 +72,29 @@ public function registerManualTransferJobCommand(string $contentReleaseIdentifie $this->redisContentReleaseService->registerManualTransferJob($contentReleaseIdentifier, $prunnerJobId, $logger); } + /** + * Mark a release as failed from the pipeline's on_error hook + * + * Rendering sets the status to "success" before validation, transfer and switch have run, and only rendering + * failures set "failed" - so without this, a release which fails later keeps reading "success" once prunner has + * dropped its job. Pipelines which merely act on an existing release (e.g. a manual transfer) share that hook, so + * only the job which built the release may mark it. + */ + public function markFailedCommand(string $contentReleaseIdentifier, string $prunnerJobId): void + { + $contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier); + $metadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier); + if ($metadata === null || $metadata->getPrunnerJobId()->getIdentifier() !== $prunnerJobId) { + return; + } + + $this->redisContentReleaseService->setContentReleaseMetadata( + $contentReleaseIdentifier, + $metadata->withStatus(NodeRenderingCompletionStatus::failed()), + RedisInstanceIdentifier::primary(), + ); + } + public function flushContentCacheIfRequiredCommand( string $contentReleaseIdentifier, bool $flushContentCache = false, diff --git a/README.md b/README.md index 26ee094..342b961 100644 --- a/README.md +++ b/README.md @@ -368,13 +368,13 @@ Prunner keeps the logs of a release independently of Redis - its pipeline's `ret started job of `do_content_release` and `do_quick_content_release` whose release has been removed, and links to its logs. This only happens for the primary content store, the one releases are built in. -To keep the status, author, rendering errors and rendering statistics next to those logs, the keys flagged with -`keepAfterRemoval` in `redisKeyPostfixesForEachRelease` (by default `meta:info`, `renderingErrors` and -`renderingStatistics`) are not deleted with their release. Redis expires them after `removedReleaseRetentionSeconds` -of the content store instead (5 days on the primary one, unset - so deleted right away - everywhere else). The release -itself leaves `contentStore:registeredReleases` as before, so it can neither be switched to nor built upon, and -*Prune content store* leaves keys alone which expire on their own. Set `removedReleaseRetentionSeconds` to about the -prunner `retention_period` of your release pipelines; once both have run out, the release is gone from the UI. +The status badge of a release comes from its prunner job while prunner still has it. Afterwards only the release +itself is left, and its metadata status is a rendering status: it reads `success` before validation, transfer and +switch have run. So the pipeline's `on_error` hook has to call +`./flow contentReleasePrepare:markFailed {{ .contentReleaseId }} {{ .__jobID }}` (see `pipelines_template.yml`); +the command ignores jobs which did not build the release, so the hook can be shared with other pipelines. A release +without a job is then shown as `failed` if it was marked so, `success` if it was switched live, and `aborted` +otherwise - canceled, or killed together with its worker. ## Quick Content Releases @@ -747,7 +747,7 @@ Example: ## Development - You need [pnpm](https://github.com/pnpm/pnpm) as package panager installed: - `curl -f https://get.pnpm.io/v6.js | node - add --global pnpm` + `curl -fsSL https://get.pnpm.io | sh -` - Run `pnpm install` in this folder - Then run `pnpm watch` for development and `pnpm build` for prod build. diff --git a/Resources/Private/BackendFusion/Integration/Backend.Index.fusion b/Resources/Private/BackendFusion/Integration/Backend.Index.fusion index e48db6e..762edae 100644 --- a/Resources/Private/BackendFusion/Integration/Backend.Index.fusion +++ b/Resources/Private/BackendFusion/Integration/Backend.Index.fusion @@ -31,7 +31,7 @@ Flowpack.DecoupledContentStore.BackendController.index = Neos.Fusion:Component { {props.contentReleaseIdentifier} {" "} - {item.metadata.status.displayName} + active {item.errorCount} @@ -104,6 +104,15 @@ Flowpack.DecoupledContentStore.BackendController.index = Neos.Fusion:Component { } } +prototype(Flowpack.DecoupledContentStore:StatusBadge) < prototype(Neos.Fusion:Component) { + contentReleaseStatus = null + + renderer = afx` + {props.contentReleaseStatus.value} + ` +} + // Releases which Redis has dropped, but whose prunner logs are still there. The keys a removed release keeps carry the // status, author and errors for a while; once they have expired, the prunner job is all that is shown. prototype(Flowpack.DecoupledContentStore:RemovedReleases) < prototype(Neos.Fusion:Component) { @@ -124,7 +133,7 @@ prototype(Flowpack.DecoupledContentStore:RemovedReleases) < prototype(Neos.Fusio {item.contentReleaseIdentifier.identifier} {" "} - {item.status} + {item.errorCount} {item.job.pipeline} diff --git a/Resources/Public/BackendCompiled/out.css b/Resources/Public/BackendCompiled/out.css index d94deb7..06cae87 100644 --- a/Resources/Public/BackendCompiled/out.css +++ b/Resources/Public/BackendCompiled/out.css @@ -1,4 +1,4 @@ -/* ../../../../../../../../private/var/folders/fd/vt4bk4351l97704fxm4zmmyr0000gn/T/tmp-48963-VFAUMDKqBjLj/Flowpack.DecoupledContentStore/Resources/Private/Js/1786957409186-styles.css */ +/* ../../../../../../../../private/var/folders/fd/vt4bk4351l97704fxm4zmmyr0000gn/T/tmp-76571-TNACfhTZ9WWw/Flowpack.DecoupledContentStore/Resources/Private/Js/1791884818496-styles.css */ #app *, #app ::before, #app ::after { diff --git a/Tests/Unit/BackendUi/ContentReleaseStatusTest.php b/Tests/Unit/BackendUi/ContentReleaseStatusTest.php new file mode 100644 index 0000000..14a98f6 --- /dev/null +++ b/Tests/Unit/BackendUi/ContentReleaseStatusTest.php @@ -0,0 +1,127 @@ +withSwitchTime(new \DateTimeImmutable()), + ), + ); + } + + public function testAReleaseMarkedFailedAfterItWasSwitchedIsFailed(): void + { + // e.g. sending the events to Spryker failed after the switch + self::assertSame( + ContentReleaseStatus::FAILED, + ContentReleaseStatus::jobGone( + self::metadata(NodeRenderingCompletionStatus::failed())->withSwitchTime(new \DateTimeImmutable()), + ), + ); + } + + public function testARenderedButNeverSwitchedReleaseWhoseJobIsGoneIsAborted(): void + { + // killed together with prunner during validation, so no on_error hook marked it failed + self::assertSame( + ContentReleaseStatus::ABORTED, + ContentReleaseStatus::jobGone(self::metadata(NodeRenderingCompletionStatus::success())), + ); + } + + public function testAnUnfinishedReleaseWhoseJobIsGoneIsAborted(): void + { + self::assertSame( + ContentReleaseStatus::ABORTED, + ContentReleaseStatus::jobGone(self::metadata(NodeRenderingCompletionStatus::running())), + ); + } + + public function testOnlySuccessAndFailureAreColoured(): void + { + self::assertSame('neos-badge neos-badge-success', ContentReleaseStatus::SUCCESS->getBadgeClass()); + self::assertSame('neos-badge neos-badge-important', ContentReleaseStatus::FAILED->getBadgeClass()); + self::assertSame('neos-badge', ContentReleaseStatus::RUNNING->getBadgeClass()); + } + + private static function metadata(NodeRenderingCompletionStatus $status): ContentReleaseMetadata + { + return ContentReleaseMetadata::create(PrunnerJobId::fromString('job'), new \DateTimeImmutable()) + ->withStatus($status); + } + + private static function job(bool $completed, bool $canceled, bool $errored): Job + { + return Job::fromJsonArray([ + 'id' => 'job', + 'pipeline' => 'do_content_release', + 'tasks' => [], + 'completed' => $completed, + 'canceled' => $canceled, + 'errored' => $errored, + 'created' => '2026-10-01T07:00:00+00:00', + 'start' => '2026-10-01T08:00:00+00:00', + 'variables' => ['contentReleaseId' => '80'], + 'user' => 'cli', + ]); + } +} diff --git a/Tests/Unit/Command/ContentReleasePrepareCommandControllerTest.php b/Tests/Unit/Command/ContentReleasePrepareCommandControllerTest.php new file mode 100644 index 0000000..5074e64 --- /dev/null +++ b/Tests/Unit/Command/ContentReleasePrepareCommandControllerTest.php @@ -0,0 +1,66 @@ +serviceHoldingReleaseBuiltBy('build-job'); + $redisContentReleaseService->expects(self::once()) + ->method('setContentReleaseMetadata') + ->with( + self::anything(), + self::callback(static fn(ContentReleaseMetadata $metadata): bool => $metadata->getStatus()->isFailed()), + self::anything(), + ); + + $this->controllerWith($redisContentReleaseService)->markFailedCommand('80', 'build-job'); + } + + public function testAnotherPipelineSharingTheHookLeavesTheReleaseAlone(): void + { + // e.g. a failed manual transfer of a release which is fine + $redisContentReleaseService = $this->serviceHoldingReleaseBuiltBy('build-job'); + $redisContentReleaseService->expects(self::never())->method('setContentReleaseMetadata'); + + $this->controllerWith($redisContentReleaseService)->markFailedCommand('80', 'transfer-job'); + } + + public function testAReleaseWhichIsGoneIsIgnored(): void + { + $redisContentReleaseService = $this->createMock(RedisContentReleaseService::class); + $redisContentReleaseService->method('fetchMetadataForContentRelease')->willReturn(null); + $redisContentReleaseService->expects(self::never())->method('setContentReleaseMetadata'); + + $this->controllerWith($redisContentReleaseService)->markFailedCommand('80', 'build-job'); + } + + private function serviceHoldingReleaseBuiltBy(string $prunnerJobId): RedisContentReleaseService&MockObject + { + $redisContentReleaseService = $this->createMock(RedisContentReleaseService::class); + $redisContentReleaseService->method('fetchMetadataForContentRelease') + ->willReturn(ContentReleaseMetadata::create( + PrunnerJobId::fromString($prunnerJobId), + new \DateTimeImmutable(), + )); + return $redisContentReleaseService; + } + + private function controllerWith( + RedisContentReleaseService $redisContentReleaseService, + ): ContentReleasePrepareCommandController { + $controller = new ContentReleasePrepareCommandController(); + $this->inject($controller, 'redisContentReleaseService', $redisContentReleaseService); + return $controller; + } +} diff --git a/pipelines_template.yml b/pipelines_template.yml index 9fbb990..fb63194 100644 --- a/pipelines_template.yml +++ b/pipelines_template.yml @@ -27,6 +27,11 @@ pipelines: # retention_period_hours: 72 retention_count: 10 + # Rendering reports "success" before validation, transfer and switch have run, so a later failure has to be + # written to the release, or the Backend UI shows it as successful once prunner has dropped this job. + on_error: &mark_release_failed + script: + - ./flow contentReleasePrepare:markFailed {{ .contentReleaseId }} {{ .__jobID }} tasks: ################################################################################ @@ -224,6 +229,8 @@ pipelines: # see the note on do_content_release above retention_count: 10 + on_error: *mark_release_failed + tasks: ################################################################################ # 0) PREPARE diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 81af8a6..06ca3d0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -194,8 +194,8 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - baseline-browser-mapping@2.10.21: - resolution: {integrity: sha512-Q+rUQ7Uz8AHM7DEaNdwvfFCTq7a43lNTzuS94eiWqwyxfV/wJv+oUivef51T91mmRY4d4A1u9rcSvkeufCVXlA==} + baseline-browser-mapping@2.11.26: + resolution: {integrity: sha512-GLQdD3y6UF8iVuMJl5fHgE4jdn/ua7n+toKfLgNlg3BqQtOZjpy68T8Tup8/wGWZCDlm7KMg7tPb4MPn7oN0TQ==} engines: {node: '>=6.0.0'} hasBin: true @@ -219,8 +219,8 @@ packages: resolution: {integrity: sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==} engines: {node: '>= 6'} - caniuse-lite@1.0.30001790: - resolution: {integrity: sha512-bOoxfJPyYo+ds6W0YfptaCWbFnJYjh2Y1Eow5lRv+vI2u8ganPZqNm1JwNh0t2ELQCqIWg4B3dWEusgAmsoyOw==} + caniuse-lite@1.0.30001813: + resolution: {integrity: sha512-zfjJo4rM0+fUomGDBW/xcDjhIwz/210DGvip2MAMDZ8KHcRPnOHmEgHPZP0UHlZoxr8fYKVJOqcORhYQcG4FKQ==} chokidar@3.6.0: resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} @@ -1013,7 +1013,7 @@ snapshots: autoprefixer@10.5.0(postcss@8.5.10): dependencies: browserslist: 4.28.2 - caniuse-lite: 1.0.30001790 + caniuse-lite: 1.0.30001813 fraction.js: 5.3.4 picocolors: 1.1.1 postcss: 8.5.10 @@ -1021,7 +1021,7 @@ snapshots: balanced-match@1.0.2: {} - baseline-browser-mapping@2.10.21: {} + baseline-browser-mapping@2.11.26: {} binary-extensions@2.3.0: {} @@ -1035,15 +1035,15 @@ snapshots: browserslist@4.28.2: dependencies: - baseline-browser-mapping: 2.10.21 - caniuse-lite: 1.0.30001790 + baseline-browser-mapping: 2.11.26 + caniuse-lite: 1.0.30001813 electron-to-chromium: 1.5.344 node-releases: 2.0.38 update-browserslist-db: 1.2.3(browserslist@4.28.2) camelcase-css@2.0.1: {} - caniuse-lite@1.0.30001790: {} + caniuse-lite@1.0.30001813: {} chokidar@3.6.0: dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 6c2ce91..03ed224 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,2 +1,5 @@ +allowBuilds: + '@parcel/watcher': true + esbuild: true # to prevent installation of malicious dependencies (came up with increasing Supply-Chain-Attacks in npm packages) minimumReleaseAge: 20160 # 14 days \ No newline at end of file