Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 27 additions & 4 deletions Classes/BackendUi/BackendUiDataService.php
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,16 @@ public function loadBackendOverviewData(RedisInstanceIdentifier $redisInstanceId
...$contentReleaseIds,
);

try {
$jobsById = [];
foreach ($this->prunnerApiService->loadPipelinesAndJobs()->getJobs() as $job) {
$jobsById[$job->getId()->getId()] = $job;
}
} catch (\Throwable $throwable) {
// the overview also holds the controls to pause releases, which must keep working while prunner is down
$jobsById = null;
}

$result = [];
foreach ($contentReleaseIds as $contentReleaseId) {
$lastRendering = RenderingStatistics::fromJsonString($lastRenderingStatisticsEntries->getResultForContentRelease(
Expand All @@ -98,13 +108,27 @@ public function loadBackendOverviewData(RedisInstanceIdentifier $redisInstanceId
));

$metadataForContentRelease = $metadata->getResultForContentRelease($contentReleaseId);
if (!$metadataForContentRelease instanceof ContentReleaseMetadata) {
$metadataForContentRelease = null;
}
$jobForContentRelease = $metadataForContentRelease !== null
? $jobsById[$metadataForContentRelease->getPrunnerJobId()->getIdentifier()] ?? null
: null;
$status = match (true) {
$jobForContentRelease !== null => ContentReleaseStatus::fromJob(
$jobForContentRelease,
$metadataForContentRelease,
),
$metadataForContentRelease === null || $jobsById === null => ContentReleaseStatus::UNKNOWN,
default => ContentReleaseStatus::jobGone($metadataForContentRelease),
};
$countForContentRelease = $counts->getResultForContentRelease($contentReleaseId);
$iterationsCountForContentRelease = $iterationsCounts->getResultForContentRelease($contentReleaseId);
$errorCountForContentRelease = $errorCounts->getResultForContentRelease($contentReleaseId);

$result[] = new ContentReleaseOverviewRow(
$contentReleaseId,
$metadataForContentRelease instanceof ContentReleaseMetadata ? $metadataForContentRelease : null,
$metadataForContentRelease,
is_int($countForContentRelease) ? $countForContentRelease : 0,
is_int($iterationsCountForContentRelease) ? $iterationsCountForContentRelease : 0,
is_int($errorCountForContentRelease) ? $errorCountForContentRelease : 0,
Expand All @@ -115,9 +139,8 @@ public function loadBackendOverviewData(RedisInstanceIdentifier $redisInstanceId
$contentReleaseId->equals($this->redisReleaseSwitchService->getCurrentRelease(
$redisInstanceIdentifier,
)),
$metadataForContentRelease instanceof ContentReleaseMetadata
? $metadataForContentRelease->getContentReleaseSize()
: null,
$metadataForContentRelease?->getContentReleaseSize(),
$status,
);
}

Expand Down
71 changes: 71 additions & 0 deletions Classes/BackendUi/ContentReleaseStatus.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
<?php

declare(strict_types=1);

namespace Flowpack\DecoupledContentStore\BackendUi;

use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata;
use Flowpack\Prunner\Dto\Job;
use Neos\Flow\Annotations as Flow;

/**
* The status the backend shows for a release.
*
* The status in the release metadata reads "success" as soon as rendering is done, while validation, transfer and
* switch are still ahead. "failed" is reliable: the render orchestrator and the pipeline's on_error hook set it. So
* the prunner job, which spans the whole pipeline, decides wherever it is known.
*/
#[Flow\Proxy(false)]
enum ContentReleaseStatus: string
{
case SCHEDULED = 'scheduled';
case RUNNING = 'running';
case SUCCESS = 'success';
case FAILED = 'failed';
case CANCELED = 'canceled';
case ABORTED = 'aborted';
/**
* Prunner cannot be reached, or the release has no metadata to find its job by. The rendering status is not
* shown instead, as it reads "success" for a release which is still validating.
*/
case UNKNOWN = 'unknown';

public static function fromJob(Job $job, ?ContentReleaseMetadata $metadata): self
{
if ($job->isCanceled()) {
return self::CANCELED;
}
if ($job->isErrored() || ($metadata?->getStatus()->isFailed() ?? false)) {
return self::FAILED;
}
if ($job->isCompleted()) {
return self::SUCCESS;
}
return $job->getStart() === null ? self::SCHEDULED : self::RUNNING;
}

/**
* Prunner answered, but no longer holds the job: it has fallen out of the pipeline's retention, so the release is
* not being built anymore. One which was switched live succeeded; one which was neither switched nor marked
* failed was cancelled or killed with its worker.
*/
public static function jobGone(ContentReleaseMetadata $metadata): self
{
if ($metadata->getStatus()->isFailed()) {
return self::FAILED;
}
return $metadata->getSwitchTime() !== null ? self::SUCCESS : self::ABORTED;
}

/**
* A getter, so Fusion reads it as the property "badgeClass": Eel only calls methods of objects which allow it.
*/
public function getBadgeClass(): string
{
return match ($this) {
self::SUCCESS => 'neos-badge neos-badge-success',
self::FAILED => 'neos-badge neos-badge-important',
self::SCHEDULED, self::RUNNING, self::CANCELED, self::ABORTED, self::UNKNOWN => 'neos-badge',
};
}
}
9 changes: 9 additions & 0 deletions Classes/BackendUi/Dto/ContentReleaseOverviewRow.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

namespace Flowpack\DecoupledContentStore\BackendUi\Dto;

use Flowpack\DecoupledContentStore\BackendUi\ContentReleaseStatus;
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata;
use Neos\Flow\Annotations as Flow;
Expand All @@ -22,6 +23,7 @@ class ContentReleaseOverviewRow
private int $renderedUrlCount;
private bool $isActive;
private ?float $releaseSize;
private ContentReleaseStatus $status;

public function __construct(
ContentReleaseIdentifier $contentReleaseIdentifier,
Expand All @@ -33,6 +35,7 @@ public function __construct(
int $renderedUrlCount,
bool $isActive,
?float $releaseSize,
ContentReleaseStatus $status,
) {
$this->contentReleaseIdentifier = $contentReleaseIdentifier;
$this->metadata = $metadata;
Expand All @@ -43,6 +46,7 @@ public function __construct(
$this->renderedUrlCount = $renderedUrlCount;
$this->isActive = $isActive;
$this->releaseSize = $releaseSize;
$this->status = $status;
}

public function getMetadata(): ?ContentReleaseMetadata
Expand Down Expand Up @@ -92,4 +96,9 @@ public function getReleaseSize(): ?float
{
return $this->releaseSize;
}

public function getStatus(): ContentReleaseStatus
{
return $this->status;
}
}
14 changes: 3 additions & 11 deletions Classes/BackendUi/Dto/RemovedContentReleaseOverviewRow.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

namespace Flowpack\DecoupledContentStore\BackendUi\Dto;

use Flowpack\DecoupledContentStore\BackendUi\ContentReleaseStatus;
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata;
use Flowpack\Prunner\Dto\Job;
Expand Down Expand Up @@ -45,17 +46,8 @@ public function getErrorCount(): int
return $this->errorCount;
}

public function getStatus(): string
public function getStatus(): ContentReleaseStatus
{
if ($this->metadata !== null) {
return $this->metadata->getStatus()->getDisplayName();
}
if ($this->job->isCanceled()) {
return 'canceled';
}
if ($this->job->isErrored()) {
return 'failed';
}
return $this->job->isCompleted() ? 'done' : 'running';
return ContentReleaseStatus::fromJob($this->job, $this->metadata);
}
}
25 changes: 25 additions & 0 deletions Classes/Command/ContentReleasePrepareCommandController.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
use Flowpack\DecoupledContentStore\Core\ConcurrentBuildLockService;
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\PrunnerJobId;
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\RedisInstanceIdentifier;
use Flowpack\DecoupledContentStore\Core\Infrastructure\ContentReleaseLogger;
use Flowpack\DecoupledContentStore\NodeRendering\Dto\NodeRenderingCompletionStatus;
use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService;
use Neos\Flow\Annotations as Flow;
use Neos\Flow\Cli\CommandController;
Expand Down Expand Up @@ -70,6 +72,29 @@ public function registerManualTransferJobCommand(string $contentReleaseIdentifie
$this->redisContentReleaseService->registerManualTransferJob($contentReleaseIdentifier, $prunnerJobId, $logger);
}

/**
* Mark a release as failed from the pipeline's on_error hook
*
* Rendering sets the status to "success" before validation, transfer and switch have run, and only rendering
* failures set "failed" - so without this, a release which fails later keeps reading "success" once prunner has
* dropped its job. Pipelines which merely act on an existing release (e.g. a manual transfer) share that hook, so
* only the job which built the release may mark it.
*/
public function markFailedCommand(string $contentReleaseIdentifier, string $prunnerJobId): void
{
$contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier);
$metadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier);
if ($metadata === null || $metadata->getPrunnerJobId()->getIdentifier() !== $prunnerJobId) {
return;
}

$this->redisContentReleaseService->setContentReleaseMetadata(
$contentReleaseIdentifier,
$metadata->withStatus(NodeRenderingCompletionStatus::failed()),
RedisInstanceIdentifier::primary(),
);
}

public function flushContentCacheIfRequiredCommand(
string $contentReleaseIdentifier,
bool $flushContentCache = false,
Expand Down
16 changes: 8 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -368,13 +368,13 @@ Prunner keeps the logs of a release independently of Redis - its pipeline's `ret
started job of `do_content_release` and `do_quick_content_release` whose release has been removed, and links to its
logs. This only happens for the primary content store, the one releases are built in.

To keep the status, author, rendering errors and rendering statistics next to those logs, the keys flagged with
`keepAfterRemoval` in `redisKeyPostfixesForEachRelease` (by default `meta:info`, `renderingErrors` and
`renderingStatistics`) are not deleted with their release. Redis expires them after `removedReleaseRetentionSeconds`
of the content store instead (5 days on the primary one, unset - so deleted right away - everywhere else). The release
itself leaves `contentStore:registeredReleases` as before, so it can neither be switched to nor built upon, and
*Prune content store* leaves keys alone which expire on their own. Set `removedReleaseRetentionSeconds` to about the
prunner `retention_period` of your release pipelines; once both have run out, the release is gone from the UI.
The status badge of a release comes from its prunner job while prunner still has it. Afterwards only the release
itself is left, and its metadata status is a rendering status: it reads `success` before validation, transfer and
switch have run. So the pipeline's `on_error` hook has to call
`./flow contentReleasePrepare:markFailed {{ .contentReleaseId }} {{ .__jobID }}` (see `pipelines_template.yml`);
the command ignores jobs which did not build the release, so the hook can be shared with other pipelines. A release
without a job is then shown as `failed` if it was marked so, `success` if it was switched live, and `aborted`
otherwise - canceled, or killed together with its worker.

## Quick Content Releases

Expand Down Expand Up @@ -747,7 +747,7 @@ Example:
## Development

- You need [pnpm](https://github.com/pnpm/pnpm) as package panager installed:
`curl -f https://get.pnpm.io/v6.js | node - add --global pnpm`
`curl -fsSL https://get.pnpm.io | sh -`
- Run `pnpm install` in this folder
- Then run `pnpm watch` for development and `pnpm build` for prod build.

Expand Down
13 changes: 11 additions & 2 deletions Resources/Private/BackendFusion/Integration/Backend.Index.fusion
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ Flowpack.DecoupledContentStore.BackendController.index = Neos.Fusion:Component {
{props.contentReleaseIdentifier}
</Neos.Fusion:Link.Action>
{" "}
<span class="neos-badge" title="status" data-neos-toggle="tooltip">{item.metadata.status.displayName}</span>
<Flowpack.DecoupledContentStore:StatusBadge contentReleaseStatus={item.status}/>
<span @if.isActive={item.active} class="neos-badge neos-badge-info">active</span>
<span @if.errors={item.errorCount} class="neos-badge neos-badge-important" title="errors" data-neos-toggle="tooltip">{item.errorCount}</span>
</td>
Expand Down Expand Up @@ -104,6 +104,15 @@ Flowpack.DecoupledContentStore.BackendController.index = Neos.Fusion:Component {
}
}

prototype(Flowpack.DecoupledContentStore:StatusBadge) < prototype(Neos.Fusion:Component) {
contentReleaseStatus = null

renderer = afx`
<span class={props.contentReleaseStatus.badgeClass} title="status"
data-neos-toggle="tooltip">{props.contentReleaseStatus.value}</span>
`
}

// Releases which Redis has dropped, but whose prunner logs are still there. The keys a removed release keeps carry the
// status, author and errors for a while; once they have expired, the prunner job is all that is shown.
prototype(Flowpack.DecoupledContentStore:RemovedReleases) < prototype(Neos.Fusion:Component) {
Expand All @@ -124,7 +133,7 @@ prototype(Flowpack.DecoupledContentStore:RemovedReleases) < prototype(Neos.Fusio
{item.contentReleaseIdentifier.identifier}
</Neos.Fusion:Link.Action>
{" "}
<span class="neos-badge" title="status" data-neos-toggle="tooltip">{item.status}</span>
<Flowpack.DecoupledContentStore:StatusBadge contentReleaseStatus={item.status}/>
<span @if.errors={item.errorCount} class="neos-badge neos-badge-important" title="errors" data-neos-toggle="tooltip">{item.errorCount}</span>
</td>
<td>{item.job.pipeline}</td>
Expand Down
2 changes: 1 addition & 1 deletion Resources/Public/BackendCompiled/out.css
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* ../../../../../../../../private/var/folders/fd/vt4bk4351l97704fxm4zmmyr0000gn/T/tmp-48963-VFAUMDKqBjLj/Flowpack.DecoupledContentStore/Resources/Private/Js/1786957409186-styles.css */
/* ../../../../../../../../private/var/folders/fd/vt4bk4351l97704fxm4zmmyr0000gn/T/tmp-76571-TNACfhTZ9WWw/Flowpack.DecoupledContentStore/Resources/Private/Js/1791884818496-styles.css */
#app *,
#app ::before,
#app ::after {
Expand Down
Loading
Loading