diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 5167ccda..4109558a 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -117,9 +117,10 @@ External data connections, Power Query queries and links in the workbook are never opened. Macro-enabled workbooks (`.xlsm`), old Excel files (`.xls`) and CSV files are not accepted. -**Placeholder values.** The CMDB sheets fill some empty cells with a -placeholder. These are read as empty: `NB` in `BNN Classificatie`, and the -date 2036-01-01 (Excel serial 49675) in `End-of-Life Functioneel`. +**Placeholder values.** The CMDB sheets fill an empty `BNN Classificatie` +with `NB` ("niet bekend"); that is read as empty. Dates are stored as the +file has them: the CMDB's 2036-01-01 in `End-of-Life Functioneel` is imported +as 2036-01-01. ### Columns and where they go @@ -130,14 +131,14 @@ date 2036-01-01 (Excel serial 49675) in `End-of-Life Functioneel`. | Applicatie Code | module external id | reference only; it can change in TOPdesk, so it is not the match key | | Roepnaam, Nickname | module short description | Roepnaam when filled, otherwise Nickname (only on `Beheerde Applicaties CMDB`) | | Functionele Omschrijving | module long description | | -| Applicatiesoort | module hosting model (`cloudDienstverleningsmodel`) | `Saas` → SaaS, `PaaS` → PaaS, `IaaS` → IaaS, `On-premise(s)` → On-premises (self-managed); another value is dropped with a warning | -| BNN Classificatie | module BBN level | `BBN1`/`BBN 1`/`BNN1` etc. become `BBN1`, `BBN2`, `BBN3`; `NB` is empty; another value is dropped with a warning | +| Applicatiesoort | module application type (`applicationType`) and hosting model (`cloudDienstverleningsmodel`) | stored as is as the application type (Webapplicatie, Client/server, Saas, …); `Saas` → SaaS, `PaaS` → PaaS, `IaaS` → IaaS, `On-premise(s)` → On-premises (self-managed) also set the hosting model, any other kind leaves it empty | +| BNN Classificatie | module BBN level | `1`, `2`, `2+`, `3` (and `BBN1`/`BBN 1`/`BNN1` etc.) become `BBN1`, `BBN2`, `BBN2+`, `BBN3`; `NB` is empty; another value is dropped with a warning | | Datum | module external creation date | Excel date | | Referentie datum wijziging | module external modification date | Excel date | | Vendor | Supplier organisation, set as provider on the module and the usage | one organisation per name, see below | -| Applicatie Status | usage status | In productie → In production, In voorraad → Planned, In ontwikkeling → Acquisition, Uit te faseren → To be phased out, Uitgefaseerd → Phased out; another value is dropped with a warning | -| Classificatie | usage TIME classification | Tolereren/Tolerate, Investeren/Invest, Migreren/Migrate, Elimineren/Eliminate | -| End-of-Life Functioneel | usage phase-out date | Excel date; 2036-01-01 is empty | +| Applicatie Status | usage status | In productie → In production, In voorraad → Planned, In ontwikkeling → Acquisition, Uit te faseren and Moet verwijderd worden → To be phased out, Uitgefaseerd and Verwijderd → Phased out, Besteld and Wordt getest → Acquisition, Stand-by voor continuïteit → In production; another value is dropped with a warning | +| Classificatie | usage TIME classification | Tolereren/Tolerate (also `1. Tolereren (wordt ingelezen)`), Investeren/Invest, Migreren/Migrate, Elimineren/Eliminate | +| End-of-Life Functioneel | usage phase-out date | Excel date, stored as is | | (the sheet), Cluster, Applicatie Eigenaar (Afdeling) | usage internal annotation | `Beheer geregeld: ja` or `nee`, the cluster and the department, joined with ` / `; written only when the usage is new or the note is empty | | Applicatie Eigenaar (Persoon), Applicatie Eigenaar (Functie) | usage business owner (contact person) | see [Owners](#owners) | diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 793e9b83..e97ef326 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -624,7 +624,10 @@ private function map(string $target, array $values, int $rowNumber, array &$warn } } - $data = ($result['data'] ?? []); + // A lookup whose default is null maps a known "no value" (an Applicatiesoort + // that is not a hosting model) to null; that field is left out, as an empty + // cell is, so an update never blanks what the field already holds. + $data = array_filter(($result['data'] ?? []), static fn ($value): bool => $value !== null); unset($data['id']); return ['data' => $data, 'missing' => array_values(array_unique($missing))]; diff --git a/lib/Settings/cmdb-import/topdesk-module.json b/lib/Settings/cmdb-import/topdesk-module.json index e5580d97..3a7ca749 100644 --- a/lib/Settings/cmdb-import/topdesk-module.json +++ b/lib/Settings/cmdb-import/topdesk-module.json @@ -1,7 +1,7 @@ { "id": "stackiq-topdesk-module", "name": "TOPdesk CMDB export to stackiq module", - "description": "One application row of a CMDB sheet becomes a stackiq module. A mapping marked required that fails skips the row; any other failing mapping drops that field with a warning. Nickname and Roepnaam both feed shortDescription; Roepnaam wins when both are filled.", + "description": "One application row of a CMDB sheet becomes a stackiq module. A mapping marked required that fails skips the row; any other failing mapping drops that field with a warning. Nickname and Roepnaam both feed shortDescription; Roepnaam wins when both are filled. Applicatiesoort is kept as is in applicationType; only its SaaS/PaaS/IaaS/on-premises values also set the hosting model, every other kind leaves that empty without a warning.", "sourceFormat": "excel", "version": "2.0.0", "fieldMappings": [ @@ -11,6 +11,7 @@ { "source": "Nickname", "target": "shortDescription", "transform": { "type": "trim" } }, { "source": "Roepnaam", "target": "shortDescription", "transform": { "type": "trim" } }, { "source": "Functionele Omschrijving", "target": "longDescription", "transform": { "type": "trim" } }, + { "source": "Applicatiesoort", "target": "applicationType", "transform": { "type": "trim" } }, { "source": "Applicatiesoort", "target": "cloudDienstverleningsmodel", @@ -21,7 +22,8 @@ "Paas": ["PaaS"], "PaaS": ["PaaS"], "PAAS": ["PaaS"], "Iaas": ["IaaS"], "IaaS": ["IaaS"], "IAAS": ["IaaS"], "On-premise": ["On-premises (self-managed)"], "On-premises": ["On-premises (self-managed)"], "On premise": ["On-premises (self-managed)"] - } + }, + "default": null } }, { @@ -32,7 +34,9 @@ "map": { "BBN1": "BBN1", "BBN 1": "BBN1", "bbn1": "BBN1", "bbn 1": "BBN1", "BNN1": "BBN1", "BNN 1": "BBN1", "BBN2": "BBN2", "BBN 2": "BBN2", "bbn2": "BBN2", "bbn 2": "BBN2", "BNN2": "BBN2", "BNN 2": "BBN2", - "BBN3": "BBN3", "BBN 3": "BBN3", "bbn3": "BBN3", "bbn 3": "BBN3", "BNN3": "BBN3", "BNN 3": "BBN3" + "BBN3": "BBN3", "BBN 3": "BBN3", "bbn3": "BBN3", "bbn 3": "BBN3", "BNN3": "BBN3", "BNN 3": "BBN3", + "1": "BBN1", "2": "BBN2", "2+": "BBN2+", "3": "BBN3", + "BBN2+": "BBN2+", "BBN 2+": "BBN2+", "BNN2+": "BBN2+", "BNN 2+": "BBN2+" } } }, diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json index 88ddf79d..cc18881d 100644 --- a/lib/Settings/cmdb-import/topdesk-profile.json +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -22,8 +22,7 @@ "dateColumns": ["Datum", "Referentie datum wijziging", "End-of-Life Functioneel"], "idColumns": ["APPID"], "emptyValues": { - "BNN Classificatie": ["NB"], - "End-of-Life Functioneel": ["49675"] + "BNN Classificatie": ["NB"] }, "externalKeyPrefix": "topdesk", "packs": { diff --git a/lib/Settings/cmdb-import/topdesk-usage.json b/lib/Settings/cmdb-import/topdesk-usage.json index 12ae459f..0038df19 100644 --- a/lib/Settings/cmdb-import/topdesk-usage.json +++ b/lib/Settings/cmdb-import/topdesk-usage.json @@ -15,7 +15,12 @@ "In voorraad": "Planned", "In ontwikkeling": "Acquisition", "Uit te faseren": "To be phased out", - "Uitgefaseerd": "Phased out" + "Uitgefaseerd": "Phased out", + "Moet verwijderd worden": "To be phased out", + "Verwijderd": "Phased out", + "Besteld": "Acquisition", + "Wordt getest": "Acquisition", + "Stand-by voor continuïteit": "In production" } } }, @@ -28,7 +33,8 @@ "Tolerate": "Tolerate", "Tolereren": "Tolerate", "Invest": "Invest", "Investeren": "Invest", "Migrate": "Migrate", "Migreren": "Migrate", - "Eliminate": "Eliminate", "Elimineren": "Eliminate" + "Eliminate": "Eliminate", "Elimineren": "Eliminate", + "1. Tolereren (wordt ingelezen)": "Tolerate" } } }, diff --git a/lib/Settings/register.d/topdesk-cmdb-import.json b/lib/Settings/register.d/topdesk-cmdb-import.json index 1f226ed5..2bd310c3 100644 --- a/lib/Settings/register.d/topdesk-cmdb-import.json +++ b/lib/Settings/register.d/topdesk-cmdb-import.json @@ -2,7 +2,7 @@ "components": { "schemas": { "module": { - "version": "0.3.5", + "version": "0.3.7", "properties": { "externalId": { "type": "string", @@ -51,6 +51,18 @@ "visible": true, "facetable": false, "order": 64 + }, + "applicationType": { + "type": "string", + "title": "Application type", + "description": "The kind of application as the source system records it, such as the TOPdesk Applicatiesoort (Webapplicatie, Client/server, Saas). Kept as the source has it.", + "maxLength": 100, + "visible": true, + "facetable": true, + "order": 65 + }, + "bbnLevel": { + "enum": ["BBN2+"] } } } diff --git a/openspec/changes/cmdb-export-import/design.md b/openspec/changes/cmdb-export-import/design.md index 5d620080..1994c055 100644 --- a/openspec/changes/cmdb-export-import/design.md +++ b/openspec/changes/cmdb-export-import/design.md @@ -107,7 +107,7 @@ External connections, the Power Query package and hyperlinks are never resolved: `CmdbRowNormaliser` turns reader output into the flat `column => string` row the engine expects: -- Values listed in the profile's `emptyValues` for their column become empty, compared case-insensitively before any conversion: `NB` in "BNN Classificatie" (the CMDB sheet's "niet bekend") and `49675` (2036-01-01) in "End-of-Life Functioneel" (the CMDB sheet's placeholder for "no end-of-life date"; its formula turns an empty date, or TOPdesk's 2099-12-31, into 49675). +- Values listed in the profile's `emptyValues` for their column become empty, compared case-insensitively before any conversion: `NB` in "BNN Classificatie" (the CMDB sheet's "niet bekend"). Dates are stored as the file has them: "End-of-Life Functioneel" `49675` (2036-01-01) is imported as that date, even though the CMDB sheet's formula writes it for an empty date or TOPdesk's 2099-12-31; the municipality decided to keep the file's value (2026-10-02). - Columns listed in the profile's `dateColumns` ("Datum", "Referentie datum wijziging", "End-of-Life Functioneel"): a numeric value is converted with `PhpOffice\PhpSpreadsheet\Shared\Date::excelToDateTimeObject()` in UTC and written as `Y-m-d`. For example, `45111.38…` becomes `2023-07-04` and `53359` becomes `2046-02-01`. A non-numeric value stays as it is, so the pack's `date` transform (`sourceFormat: Y-m-d`) either accepts it or reports a warning. - Columns listed in `idColumns` ("APPID"): a whole number becomes a string without a decimal part (`1234.0` becomes `"1234"`). - The constants of the row's sheet are added before mapping (`Beheer` = `Beheer geregeld: nee` on "Onbeh", `ja` on "Beheerde"), so the usage pack can map the sheet like a column. @@ -206,14 +206,14 @@ Source columns of "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB" and w | Roepnaam | both | module.shortDescription | trim; wins over Nickname | | Nickname | Beheerde | module.shortDescription | trim; used when Roepnaam is empty; listed as absent on Onbeh | | Functionele Omschrijving | both | module.longDescription | trim | -| Applicatiesoort | both | module.cloudDienstverleningsmodel | lookup: `Saas`/`SaaS` → `["SaaS"]`, `PaaS` → `["PaaS"]`, `IaaS` → `["IaaS"]`, `On-premise(s)` → `["On-premises (self-managed)"]`; another value (such as `Webapplicatie`): warning, field dropped | -| BNN Classificatie | both | module.bbnLevel | `NB` is empty; lookup "BBN1"/"BBN 1"/"BNN1" etc. to `BBN1`/`BBN2`/`BBN3`; unknown value: warning | +| Applicatiesoort | both | module.applicationType and module.cloudDienstverleningsmodel | applicationType: as is; hosting model lookup: `Saas`/`SaaS` → `["SaaS"]`, `PaaS` → `["PaaS"]`, `IaaS` → `["IaaS"]`, `On-premise(s)` → `["On-premises (self-managed)"]`; any other kind (`Webapplicatie`, `Client/server`, …) has default `null`: no hosting model, no warning | +| BNN Classificatie | both | module.bbnLevel | `NB` is empty; lookup `1`/`2`/`2+`/`3` and "BBN1"/"BBN 1"/"BNN1" etc. to `BBN1`/`BBN2`/`BBN2+`/`BBN3` (the fragment adds `BBN2+` to the enum); unknown value: warning | | Datum | both | module.externalCreatedAt | Excel serial to date | | Referentie datum wijziging | both | module.externalModifiedAt | Excel serial to date | | Vendor | both | organization (Supplier) via module.provider and usage.provider | dedup on normalised name (D7) | -| Applicatie Status | both | usage.status | lookup: In productie → In production, In voorraad → Planned, In ontwikkeling → Acquisition, Uit te faseren → To be phased out, Uitgefaseerd → Phased out; unknown value: warning | -| Classificatie | both | usage.timeClassification | lookup Tolereren/Tolerate, Investeren/Invest, Migreren/Migrate, Elimineren/Eliminate | -| End-of-Life Functioneel | both | usage.startDateOutPhased | `49675` (2036-01-01) is empty; Excel serial to date | +| Applicatie Status | both | usage.status | lookup: In productie → In production, In voorraad → Planned, In ontwikkeling → Acquisition, Uit te faseren and Moet verwijderd worden → To be phased out, Uitgefaseerd and Verwijderd → Phased out, Besteld and Wordt getest → Acquisition, Stand-by voor continuïteit → In production; unknown value: warning | +| Classificatie | both | usage.timeClassification | lookup Tolereren/Tolerate (also `1. Tolereren (wordt ingelezen)`), Investeren/Invest, Migreren/Migrate, Elimineren/Eliminate | +| End-of-Life Functioneel | both | usage.startDateOutPhased | Excel serial to date, stored as is (2036-01-01 included) | | (sheet constant `Beheer`), Cluster, Applicatie Eigenaar (Afdeling) | both | usage.interneAnnotation | concat with " / ", empty parts dropped, create-only; `Beheer geregeld: nee` (Onbeh) or `ja` (Beheerde) | | Applicatie Eigenaar (Persoon), Applicatie Eigenaar (Functie) | both | usage.businessOwner (contactPerson + Nextcloud contact; role = Functie) | D8; the person column may hold a function | | Hostingpartij | both | not mapped | follow-up; "Leverancier" (where the municipality buys the software) is not on the CMDB sheets | @@ -414,8 +414,8 @@ The seeds show the new properties in a fresh install. They carry no `publication - [Owner contacts land in the importing admin's address book] → `StackiqContactSyncService` writes to the first writable address book of the acting user, the same as every other stackiq contact path. The docs say so. A dedicated system address book is a follow-up. - [Long synchronous request] → Per-row progress, cancel, and "unchanged" rows skip the save. About 1,100 rows is expected to fit. A background job is a follow-up if it does not. - [OpenRegister internals (`MappingEngine`, `PackDefinitionValidator`, PhpSpreadsheet) change shape] → Guarded resolution with 503, and a contract test that maps the fixture through the real engine in the dev environment. -- [Provisional lookups for Applicatiesoort and BNN Classificatie] → The values of the first real import were not kept (the report lives in the progress cache for an hour). The maps hold the values the anonymised export and the CMDB formulas show (`Saas`, `Webapplicatie`, `NB`) plus the usual spellings. An unknown value is a warning, never a wrong value; once the municipality lists its values, the maps in the JSON are extended, with no code change. -- [CMDB placeholders] → "End-of-Life Functioneel" `2036-01-01` and "BNN Classificatie" `NB` are read as empty. A real end-of-life date of exactly 2036-01-01 would be lost; the municipality confirms. "Classificatie" defaults to `Tolereren` on "Beheerde" when TOPdesk has none; that cannot be told apart from a real `Tolereren` and is imported as Tolerate. +- [Lookups from the real export] → The maps hold the values the municipality's export of 2026-09-22 contains (2026-10-02 import report): "Applicatiesoort" is an application kind, kept as is in `applicationType`; "BNN Classificatie" holds `NB`, `1`, `2` and `2+`; "Applicatie Status" adds five Dutch statuses; "Classificatie" one numbered form. A new value is a warning, never a wrong value, and is added to the JSON map with no code change. A lookup `default` of `null` means "known, no value": the service leaves the field out. +- [CMDB placeholders] → "BNN Classificatie" `NB` is read as empty. "End-of-Life Functioneel" and "Classificatie" are stored as the file has them: the sheet's 2036-01-01 (written for an empty date) is imported as that date, and the `Tolereren` the "Beheerde" formula writes when TOPdesk has none is imported as Tolerate. - [An application moves between the sheets] → Same APPID, so the same module and usage; the usage note (create-only) keeps its old `Beheer geregeld` line when it is not empty. - [An unknown status on create falls back to the usage schema's default "In production"] → Accepted. The warning in the report makes it visible. - [The fragment version is overwritten by merge order] → Filename ordering plus a unit test on the merged version (Mixed-spec rationale). @@ -426,7 +426,5 @@ No data migration. The register fragment deploys with the existing repair-step r ## Open Questions -- Which "Applicatiesoort" and "BNN Classificatie" values occur in the municipality's real export, and which hosting model does each mean (lookup maps)? -- Is 2036-01-01 in "End-of-Life Functioneel" always the placeholder, and should a "Beheerde" row without a TIME class really be Tolerate? - Should the maintenance status update the usage note on re-import (it is create-only today), or get a field of its own? - Should OpenRegister promote `MigrationPack\MappingEngine` to its `Contract` namespace? diff --git a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md index 92ccf2d7..96e818c0 100644 --- a/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md @@ -133,7 +133,7 @@ The request SHALL carry either `municipalityUuid`, the uuid of an existing stack ### Requirement: REQ-CMDB-005 Field mapping SHALL be declarative and executed by OpenRegister's mapping engine -The service SHALL map each normalised row with OpenRegister's `MigrationPack\MappingEngine::mapRow()`, once per target pack: module, manufacturer, municipality, usage, business owner. The packs and the import profile SHALL ship as JSON under `lib/Settings/cmdb-import/`. Each pack SHALL pass OpenRegister's `PackDefinitionValidator` when the import starts; an invalid pack, or a missing `MappingEngine`, SHALL stop the import with 503 `MAPPING_UNAVAILABLE` before any row is read. Before mapping, the service SHALL convert the cells of the profile's date columns from Excel serial numbers to `Y-m-d`, SHALL turn numeric id cells into strings without a decimal part, SHALL read a value the profile lists as empty for its column (`NB` in "BNN Classificatie", serial `49675` in "End-of-Life Functioneel") as empty, and SHALL add the constants of the row's sheet (`Beheer` = `Beheer geregeld: nee` or `ja`). A mapping error on a mapping marked `required` in the module pack SHALL skip the row. In the manufacturer and owner packs it SHALL mean the row has no manufacturer or no such owner, without a warning. A mapping error on any other mapping SHALL drop only that field and add a row warning naming the column and the value. The reader SHALL keep only the columns that the profile or a pack references, and SHALL discard every other cell when it reads the row. +The service SHALL map each normalised row with OpenRegister's `MigrationPack\MappingEngine::mapRow()`, once per target pack: module, manufacturer, municipality, usage, business owner. The packs and the import profile SHALL ship as JSON under `lib/Settings/cmdb-import/`. Each pack SHALL pass OpenRegister's `PackDefinitionValidator` when the import starts; an invalid pack, or a missing `MappingEngine`, SHALL stop the import with 503 `MAPPING_UNAVAILABLE` before any row is read. Before mapping, the service SHALL convert the cells of the profile's date columns from Excel serial numbers to `Y-m-d`, SHALL turn numeric id cells into strings without a decimal part, SHALL read a value the profile lists as empty for its column (`NB` in "BNN Classificatie"; dates, "End-of-Life Functioneel" included, are kept as the file has them) as empty, and SHALL add the constants of the row's sheet (`Beheer` = `Beheer geregeld: nee` or `ja`). A mapping error on a mapping marked `required` in the module pack SHALL skip the row. In the manufacturer and owner packs it SHALL mean the row has no manufacturer or no such owner, without a warning. A mapping error on any other mapping SHALL drop only that field and add a row warning naming the column and the value. The reader SHALL keep only the columns that the profile or a pack references, and SHALL discard every other cell when it reads the row. #### Scenario: Excel serial dates are converted before mapping @e2e exclude Pure transformation; tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php asserts the conversions below. @@ -142,7 +142,8 @@ The service SHALL map each normalised row with OpenRegister's `MigrationPack\Map - **WHEN** the rows are normalised - **THEN** "Datum" SHALL be `2023-07-04`, "Referentie datum wijziging" SHALL be `2026-07-29`, and "End-of-Life Functioneel" SHALL be `2046-02-01` - **AND** "APPID" `1234` SHALL be the string `"1234"` -- **AND** "End-of-Life Functioneel" `49675` and "BNN Classificatie" `NB` SHALL be empty +- **AND** "BNN Classificatie" `NB` SHALL be empty +- **AND** "End-of-Life Functioneel" `49675` SHALL be `2036-01-01` #### Scenario: Changing a pack changes the mapping without code @e2e exclude Configuration behaviour; tests/Unit/Service/CmdbExportImportServiceTest.php loads an alternate module pack that maps "Software Suite" to licentietype and asserts the mapped module. @@ -165,9 +166,10 @@ The service SHALL map each normalised row with OpenRegister's `MigrationPack\Map - **GIVEN** the "Beheerde" row of the anonymised export with "Applicatiesoort" `Saas`, "BNN Classificatie" `BBN2`, "Classificatie" `Tolereren` and "End-of-Life Functioneel" `53359` - **WHEN** it is imported -- **THEN** the module SHALL have `cloudDienstverleningsmodel` = `["SaaS"]` and `bbnLevel` = `BBN2` +- **THEN** the module SHALL have `applicationType` = `Saas`, `cloudDienstverleningsmodel` = `["SaaS"]` and `bbnLevel` = `BBN2` - **AND** the usage SHALL have `timeClassification` = `Tolerate` and `startDateOutPhased` = `2046-02-01` -- **AND** the "Onbeh" row's "Applicatiesoort" `Webapplicatie`, which is not a hosting model, SHALL be dropped with a warning +- **AND** the "Onbeh" row's "Applicatiesoort" `Webapplicatie`, which is not a hosting model, SHALL be stored as `applicationType` and SHALL leave `cloudDienstverleningsmodel` empty without a warning +- **AND** "BNN Classificatie" `1`, `2`, `2+` SHALL be `BBN1`, `BBN2`, `BBN2+` ### Requirement: REQ-CMDB-006 A module SHALL be matched on its TOPdesk APPID, so a re-import updates instead of duplicating diff --git a/openspec/changes/cmdb-export-import/tasks.md b/openspec/changes/cmdb-export-import/tasks.md index a04d3196..8e0c3c39 100644 --- a/openspec/changes/cmdb-export-import/tasks.md +++ b/openspec/changes/cmdb-export-import/tasks.md @@ -42,7 +42,7 @@ Spec: `openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md` (`S - GIVEN the formula/connection fixture WHEN it is read THEN "Applicatie Naam" is `Rekenmodel`, "Roepnaam" is empty and listed in the row's `uncached`, `getCalculatedValue()` is never called, and no HTTP client is involved - GIVEN the shuffled fixture WHEN it is read THEN rows equal those of the original; GIVEN the missing-column fixture THEN `MISSING_COLUMN` names `APPID` and `Beheerde Applicaties CMDB`; GIVEN only "Blad1" THEN `NO_SOURCE_SHEET`; GIVEN more than `maxRowsPerSheet` rows THEN `TOO_MANY_ROWS` - GIVEN a text file named `.xlsx`, or a `.xlsm` WHEN checked THEN `NOT_XLSX` before PhpSpreadsheet is touched; GIVEN PhpSpreadsheet absent THEN `READER_UNAVAILABLE` - - GIVEN serials `45111.380322627316`, `46232.552113113423`, `53359` and id `1234.0` WHEN normalised THEN `2023-07-04`, `2026-07-29`, `2046-02-01` and `"1234"`; GIVEN "BNN Classificatie" `NB` and "End-of-Life Functioneel" `49675` THEN both are empty + - GIVEN serials `45111.380322627316`, `46232.552113113423`, `53359` and id `1234.0` WHEN normalised THEN `2023-07-04`, `2026-07-29`, `2046-02-01` and `"1234"`; GIVEN "BNN Classificatie" `NB` THEN it is empty; GIVEN "End-of-Life Functioneel" `49675` THEN it is `2036-01-01` - [x] Implement - [x] Test diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index fe36e789..7f231a1e 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -146,7 +146,7 @@ public function testThePacksImplementTheColumnTable(): void { $this->assertSame(['Beheer' => 'Beheer geregeld: ja'], $profile->sheetConstants(sheetName: 'Beheerde Applicaties CMDB')); $this->assertSame(['Nickname'], $profile->absentColumns(sheetName: 'Onbeh Applicaties CMDB')); $this->assertSame([], $profile->absentColumns(sheetName: 'Beheerde Applicaties CMDB')); - $this->assertSame(['BNN Classificatie' => ['NB'], 'End-of-Life Functioneel' => ['49675']], $profile->emptyValues()); + $this->assertSame(['BNN Classificatie' => ['NB']], $profile->emptyValues()); $this->assertSame(10485760, $profile->maxFileBytes()); $this->assertSame(10000, $profile->maxRowsPerSheet()); }//end testThePacksImplementTheColumnTable() @@ -198,8 +198,9 @@ public function testTheLookupsMapThroughTheEngine(): void { $this->assertStringContainsString('Onbekende status', $unknown['errors'][0]['message']); $soort = $engine->mapRow($profile->pack(target: 'module'), ['Applicatie Naam' => 'X', 'APPID' => '1', 'Applicatiesoort' => 'Webapplicatie'], 3); - $this->assertArrayNotHasKey('cloudDienstverleningsmodel', $soort['data'], 'an application kind is not a hosting model'); - $this->assertSame('Applicatiesoort', $soort['errors'][0]['source']); + $this->assertNull($soort['data']['cloudDienstverleningsmodel'], 'an application kind is not a hosting model; the service leaves the null out'); + $this->assertSame('Webapplicatie', $soort['data']['applicationType']); + $this->assertSame([], $soort['errors']); }//end testTheLookupsMapThroughTheEngine() /** diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index cde666c2..f483fe35 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -520,12 +520,12 @@ public function testTheFixtureCreatesModulesUsagesAndSuppliers(): void { $this->assertTrue($report['success']); $this->assertFalse($report['cancelled']); $this->assertSame('cmdb-test-0001', $report['operationId']); - $this->assertSame(['rowsRead' => 2, 'processed' => 2, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 0, 'warnings' => 1], $report['summary']); + $this->assertSame(['rowsRead' => 2, 'processed' => 2, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 0, 'warnings' => 0], $report['summary']); $this->assertSame('Gemeente Voorbeeldstad', $report['municipality']['name']); $this->assertTrue($report['municipality']['created']); $this->assertSame([], $report['importWarnings']); - // "Webapplicatie" is an application kind, not a hosting model: the field is dropped with a warning. - $this->assertSame(['Column "Applicatiesoort": Value "Webapplicatie" has no mapping and no default is configured'], $report['rows'][0]['warnings']); + // "Webapplicatie" is an application kind, not a hosting model: kept as the kind, no hosting model, no warning. + $this->assertSame([], $report['rows'][0]['warnings']); $municipality = $report['municipality']['uuid']; $this->assertSame('Municipality', $this->store[self::ORGANIZATION][$municipality]['type']); @@ -551,6 +551,7 @@ public function testTheFixtureCreatesModulesUsagesAndSuppliers(): void { $this->assertSame('Functionele omschrijving test123', $onbeh['longDescription']); $this->assertArrayNotHasKey('bbnLevel', $onbeh, '"NB" means unknown'); $this->assertArrayNotHasKey('cloudDienstverleningsmodel', $onbeh); + $this->assertSame('Webapplicatie', $onbeh['applicationType']); $publication = new \DateTimeImmutable($onbeh['publicationDate']); $this->assertGreaterThanOrEqual($before, $publication); $this->assertLessThanOrEqual(new \DateTimeImmutable('now'), $publication); @@ -563,6 +564,7 @@ public function testTheFixtureCreatesModulesUsagesAndSuppliers(): void { $this->assertSame('Naamtest', $beheerd['shortDescription'], 'Roepnaam wins over Nickname'); $this->assertSame('Accomodatieplanning.', $beheerd['longDescription']); $this->assertSame(['SaaS'], $beheerd['cloudDienstverleningsmodel']); + $this->assertSame('Saas', $beheerd['applicationType']); $this->assertSame('BBN2', $beheerd['bbnLevel']); $supplierByName = array_column($suppliers, 'id', 'name'); @@ -576,7 +578,7 @@ public function testTheFixtureCreatesModulesUsagesAndSuppliers(): void { $this->assertSame($municipality, $aia['consumer']); $this->assertSame('Planned', $aia['status']); $this->assertSame('Beheer geregeld: nee / H10 / H10 Accounting', $aia['interneAnnotation']); - $this->assertArrayNotHasKey('startDateOutPhased', $aia, 'the CMDB placeholder 2036-01-01 means no date'); + $this->assertSame('2036-01-01', $aia['startDateOutPhased'], 'the end-of-life date is stored as the file has it'); $this->assertArrayNotHasKey('timeClassification', $aia); $this->assertSame($supplierByName['Aangetekend B.V.'], $aia['provider']); $app = $usageByModule[$beheerd['id']]; @@ -821,11 +823,11 @@ public function testTheSheetRecordsWhetherMaintenanceIsArranged(): void { }//end testTheSheetRecordsWhetherMaintenanceIsArranged() /** - * "NB" in BNN Classificatie and the CMDB end-of-life placeholder (serial 49675) mean empty: no field, no warning. + * "NB" in BNN Classificatie means empty (no field, no warning); an end-of-life date is stored as the file has it, 2036-01-01 included. * * @return void */ - public function testTheCmdbPlaceholdersMeanEmpty(): void { + public function testNbMeansEmptyAndEndOfLifeIsStoredAsIs(): void { $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); $rows = [ $this->row(appId: '1', cells: ['BNN Classificatie' => 'NB', 'End-of-Life Functioneel' => 49675]), @@ -839,10 +841,40 @@ public function testTheCmdbPlaceholdersMeanEmpty(): void { $this->assertArrayNotHasKey('bbnLevel', $modules[1]); $this->assertSame('BBN3', $modules[2]['bbnLevel']); $usages = array_column($this->objects(self::USAGE), null, 'module'); - $this->assertArrayNotHasKey('startDateOutPhased', $usages[$modules[1]['id']]); + $this->assertSame('2036-01-01', $usages[$modules[1]['id']]['startDateOutPhased']); $this->assertSame('2046-02-01', $usages[$modules[2]['id']]['startDateOutPhased']); $this->assertSame('Migrate', $usages[$modules[2]['id']]['timeClassification']); - }//end testTheCmdbPlaceholdersMeanEmpty() + }//end testNbMeansEmptyAndEndOfLifeIsStoredAsIs() + + /** + * The values the municipality's real export holds map without a warning: BNN 1/2/2+, the extra + * statuses, the numbered TIME class, and application kinds that are not a hosting model. + * + * @return void + */ + public function testTheValuesOfTheRealExportMapWithoutWarnings(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $rows = [ + $this->row(appId: '1', cells: ['BNN Classificatie' => '1', 'Applicatiesoort' => 'Client/server', 'Applicatie Status' => 'Moet verwijderd worden', 'Classificatie' => '1. Tolereren (wordt ingelezen)']), + $this->row(appId: '2', cells: ['BNN Classificatie' => '2', 'Applicatiesoort' => 'Saas', 'Applicatie Status' => 'Wordt getest'], row: 3), + $this->row(appId: '3', cells: ['BNN Classificatie' => '2+', 'Applicatiesoort' => 'Beheertool', 'Applicatie Status' => 'Stand-by voor continuïteit'], row: 4), + $this->row(appId: '4', cells: ['Applicatie Status' => 'Besteld'], row: 5), + $this->row(appId: '5', cells: ['Applicatie Status' => 'Verwijderd'], row: 6), + ]; + + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(0, $report['summary']['warnings']); + $modules = array_column($this->objects(self::MODULE), null, 'externalNumber'); + $this->assertSame(['BBN1', 'BBN2', 'BBN2+'], [$modules[1]['bbnLevel'], $modules[2]['bbnLevel'], $modules[3]['bbnLevel']]); + $this->assertSame(['Client/server', 'Saas', 'Beheertool'], [$modules[1]['applicationType'], $modules[2]['applicationType'], $modules[3]['applicationType']]); + $this->assertArrayNotHasKey('cloudDienstverleningsmodel', $modules[1], 'Client/server is no hosting model'); + $this->assertSame(['SaaS'], $modules[2]['cloudDienstverleningsmodel']); + $usages = array_column($this->objects(self::USAGE), null, 'module'); + $status = array_map(fn (int $appId): string => $usages[$modules[$appId]['id']]['status'], [1, 2, 3, 4, 5]); + $this->assertSame(['To be phased out', 'Acquisition', 'In production', 'Acquisition', 'Phased out'], $status); + $this->assertSame('Tolerate', $usages[$modules[1]['id']]['timeClassification']); + }//end testTheValuesOfTheRealExportMapWithoutWarnings() /** * A formula without a cached value reads as empty and warns on its row; the row is still imported. diff --git a/tests/Unit/Settings/TopdeskCmdbFragmentTest.php b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php index 71badecf..13e592f3 100644 --- a/tests/Unit/Settings/TopdeskCmdbFragmentTest.php +++ b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php @@ -30,11 +30,11 @@ */ class TopdeskCmdbFragmentTest extends TestCase { /** - * The external-id properties the fragment adds. + * The properties the fragment adds. * * @var array */ - private const PROPERTIES = ['externalId', 'externalNumber', 'externalKey', 'externalCreatedAt', 'externalModifiedAt']; + private const PROPERTIES = ['externalId', 'externalNumber', 'externalKey', 'externalCreatedAt', 'externalModifiedAt', 'applicationType']; /** * The register after merging every fragment in sorted filename order. @@ -56,14 +56,14 @@ private function mergedRegister(): array { }//end mergedRegister() /** - * The merged module is 0.3.5 and carries the five optional, titled properties. + * The merged module is 0.3.7, carries the six optional, titled properties and allows BBN2+. * * @return void */ - public function testTheMergedModuleIsVersion035WithTheExternalIds(): void { + public function testTheMergedModuleIsVersion037WithTheExternalIds(): void { $module = $this->mergedRegister()['components']['schemas']['module']; - $this->assertSame('0.3.5', $module['version'], 'a fragment sorting after topdesk-cmdb-import.json overwrote the bump'); + $this->assertSame('0.3.7', $module['version'], 'a fragment sorting after topdesk-cmdb-import.json overwrote the bump'); foreach (self::PROPERTIES as $property) { $this->assertArrayHasKey($property, $module['properties']); $this->assertNotEmpty($module['properties'][$property]['title'] ?? '', $property); @@ -79,9 +79,11 @@ public function testTheMergedModuleIsVersion035WithTheExternalIds(): void { $this->assertSame(['default' => false], $module['properties']['externalKey']['table']); $this->assertSame('date', $module['properties']['externalCreatedAt']['format']); $this->assertSame('date', $module['properties']['externalModifiedAt']['format']); + $this->assertSame(100, $module['properties']['applicationType']['maxLength']); + $this->assertSame(['BBN1', 'BBN2', 'BBN3', 'BBN2+'], $module['properties']['bbnLevel']['enum'], 'the fragment adds BBN2+ to the BIO levels'); $this->assertArrayHasKey('roadmapStatement', $module['properties'], 'the 0.3.4 fragment still applies'); $this->assertSame(['name'], $module['required']); - }//end testTheMergedModuleIsVersion035WithTheExternalIds() + }//end testTheMergedModuleIsVersion037WithTheExternalIds() /** * The fragment sorts after the fragment that set module 0.3.4.