diff --git a/.env b/.env index 999fa0f..819e914 100644 --- a/.env +++ b/.env @@ -32,4 +32,9 @@ DEFAULT_URI=http://localhost ###> app ### APP_BROKER_BASE_URI=http://scorpio.local:9090/ + +# JSON-LD contexts attached to every entity, outermost last so the ETSI core +# context resolves the NGSI-LD terms and the domain context resolves the +# Smart Data Models ones. +ENTER_NGSI_CONTEXT_URLS='https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld' ###< app ### diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..333ba2f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,18 @@ +#### Link to ticket + +Please add a link to the ticket being addressed by this change. + +#### Description + +Please include a short description of the suggested change and the reasoning behind the approach you have chosen. + +#### Screenshot of the result + +If your change affects the user interface you should include a screenshot of the result with the pull request. + +#### Checklist + +- [ ] My code is covered by test cases. +- [ ] My code passes our test (all our tests). +- [ ] My code passes our static analysis suite. +- [ ] My code passes our continuous integration process. diff --git a/CHANGELOG.md b/CHANGELOG.md index bf48d94..e6e1c78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,4 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +* [#3](https://github.com/itk-dev/enter/pull/3) + * Import command that reads a geospatial feed, reprojects it to WGS84 and upserts it to an NGSI-LD broker. + * A committed record per data set — feed, CRS, model, DCAT-AP metadata. + * An extension point for adding data sets, a test suite, and architecture decision records. + * Source manifest validated against a Symfony config tree and read during container warm-up, so a malformed + entry fails the build rather than the one import that selects it. + [Unreleased]: https://github.com/itk-dev/enter diff --git a/README.md b/README.md index 95afbea..c80ab34 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ We use [DDEV](https://ddev.com/) and [Task](https://taskfile.dev/) for developme ``` shell task site:install +``` ``` shell task site:update @@ -12,150 +13,43 @@ ddev launch Run `task` to see what cool task are available. Running `ddev` can help with other stuff. -## Broker - -A [Scorpio Broker](https://scorpio.readthedocs.io/) is part of the development setup. - -``` shell -ddev exec "curl --silent http://scorpio.local:9090/ngsi-ld/v1/types | jq" -``` - -Load some example data: +## Adapter -``` shell name=import-toilet -ddev console app:broker:entity:delete toilet -ddev console app:broker:import:geojson toilet 'https://webkort.aarhuskommune.dk/spatialmap?page=get_geojson_opendata&datasource=andre_toiletter' -ddev exec "curl --silent http://scorpio.local:9090/ngsi-ld/v1/entities --get --data-urlencode type=toilet" | jq '.[]|with_entries(select([.key] | inside(["id", "type", "location"])))' -``` +Takes an open-data set, converts it to [NGSI-LD], and upserts it into the +context broker. -``` shell name=import-handicapparkering -ddev console app:broker:entity:delete handicapparkering -ddev console app:broker:import:geojson handicapparkering 'https://webkort.aarhuskommune.dk/spatialmap?page=get_geojson_opendata&datasource=invap' -ddev exec "curl --silent http://scorpio.local:9090/ngsi-ld/v1/entities --get --data-urlencode type=handicapparkering" | jq '.[]|with_entries(select([.key] | inside(["id", "type", "location"])))' +``` text +source feed (JSON) + → SourceInterface implementation maps fields, fixes quirks, picks the data model + → NgsiEntity normalized NGSI-LD: Property / GeoProperty / Relationship + → NgsiLdBroker POST /ngsi-ld/v1/entityOperations/upsert + → context broker ``` -``` shell name=import-hundeskov -ddev console app:broker:entity:delete hundeskov -ddev console app:broker:import:geojson hundeskov 'https://webkort.aarhuskommune.dk/spatialmap?page=get_geojson_opendata&datasource=hundeskove_friluftsliv_aarhus' -ddev exec "curl --silent http://scorpio.local:9090/ngsi-ld/v1/entities --get --data-urlencode type=hundeskov" | jq '.[]|with_entries(select([.key] | inside(["id", "type", "location", "geometry"])))' +``` shell +task import # list the available sources +task import -- mtm_spatialmaps-handicap-parking # import one +task import -- mtm_spatialmaps-handicap-parking --dry-run --limit 5 # print the payload instead ``` -### Broker API request examples - - - -``` shell name=scorpio-entity-create substitutions="{«entity-type»: Room, «entity-id»: 'house2:smartrooms:room1'}" -ddev exec "curl --silent http://scorpio.local:9090/ngsi-ld/v1/entities --header 'content-type: application/json' --data @-" <<'JSON' -{ - "type": "«entity-type»", - "id": "«entity-id»" -} -JSON - +### Source manifest -# EPSG:4326?! -ddev exec "curl --silent http://scorpio.local:9090/ngsi-ld/v1/entities/«entity-id»/attrs --header 'content-type: application/json' --data @-" <<'JSON' -{ - "location": { - "type": "geo:json", - "value": { - "type": "Point", - "coordinates": [ - 10.15711687080293, 56.126271111641266 - ] - } - } -} -JSON +Every data set is recorded in [config/sources.yaml](config/sources.yaml), keyed +by the identifier `app:import` takes as its argument. See [ADR 007](docs/adr/007-source-manifest.md). +Adding a data set means adding one `SourceInterface` implementation and one +manifest entry. The class is discovered through +`#[AutoconfigureTag('app.source')]` and shows up as an `app:import` argument +with no further wiring. -# Get the entities -ddev exec "curl --silent --header 'accept: application/ld+json' http://scorpio.local:9090/ngsi-ld/v1/entities?type=«entity-type» | jq" +Design decisions are recorded in [docs/adr](docs/adr/README.md). -# Get the entities as GeoJSON -ddev exec "curl --silent --header 'accept: application/geo+json' http://scorpio.local:9090/ngsi-ld/v1/entities?type=«entity-type» | jq" -``` - -> [!CAUTION] -> Excuse me what?! -> -> ``` shell -> ddev exec "curl --silent --header 'accept: application/geo+json' 'http://scorpio.local:9090/ngsi-ld/v1/entities?georel=near;maxDistance%3D%3D2000&geometry=Point&coordinates=%5B8,40%5D'" -> ddev exec "curl --silent --header 'accept: application/geo+json' 'http://scorpio.local:9090/ngsi-ld/v1/entities?georel=near;maxDistance==2000&geometry=Point&coordinates=%5B8,40%5D'" -> ddev exec "curl --silent --header 'accept: application/geo+json' 'http://scorpio.local:9090/ngsi-ld/v1/entities?georel=near;maxDistance==2000&geometry=Point&coordinates=[10,56]'" -> > ``` - -## GeoJSON - -* "[Position](https://datatracker.ietf.org/doc/html/rfc7946#section-3.1.1)": `(longitude, latitude)` - * "[Coordinate Reference System](https://datatracker.ietf.org/doc/html/rfc7946#section-4)" - * But … - - → : - - ```json - { - "type": "FeatureCollection", - "crs": { - "type": "name", - "properties": { - "name": "EPSG:25832" - } - }, - "bbox": [563262.0903961, 6209995.05346621, 579587.800694027, 6231954.36230685], - "features": [ - { - "type": "Feature", - "geometry": { - "type": "MultiPoint", - "coordinates": [ - [576933.018139537, 6218035.22691216] - ] - }, - "properties": { - … - ``` - -* - ---- - -* -* -* -* - -Must `location` be a `Point` in ngsi-ld? +[NGSI-LD]: https://www.etsi.org/committee/cim -``` shell -ddev exec --service scorpio-db "psql ngb ngb" -``` - -``` sql -2026-08-30 10:26:19.789 UTC [41] LOG: execute 0000000: WITH D0 AS (SELECT ID, ENTITY, TRUE as PARENT FROM ENTITY WHERE ST_DWithin( location::geography, ST_SetSRID(ST_GeomFromGeoJSON('{"type": "Point", "coordinates": [10.236808047161853,56.101226991176155] }'), 4326)::geography, 2000.0) ORDER BY createdAt limit $1 offset $2) SELECT ID, ENTITY, PARENT FROM D0 -``` +## Broker - +A [Scorpio Broker](https://scorpio.readthedocs.io/) is part of the development setup. ``` shell -ddev exec --service scorpio-db "psql ngb ngb" <<< "SELECT id , e_types, location FROM entity;" -``` - -``` shell name=hmm -ddev console app:import:geojson toilet 'https://webkort.aarhuskommune.dk/spatialmap?page=get_geojson_opendata&datasource=andre_toiletter' \ - && ddev exec --service scorpio-db "psql ngb ngb" <<< "SELECT id , e_types, ST_AsText(location) AS location FROM entity WHERE id = 'toilet:0000';" \ - && ddev exec --service scorpio-db "psql ngb ngb" <<< "SELECT temporalentity_id, ST_AsText(location) AS location, ST_asText(geovalue) AS geovalue, createdat FROM temporalentityattrinstance WHERE temporalentity_id = 'toilet:0000' ORDER BY createdat DESC LIMIT 10;" +ddev exec "curl --silent http://scorpio.local:9090/ngsi-ld/v1/types | jq" ``` - - - -* [Public toilets in Aarhus Municipality](https://www.opendata.dk/city-of-aarhus/toiletter-i-aarhus-kommune) - * [Offentlige - toiletter](http://webkort.aarhuskommune.dk/spatialmap?page=get_geojson_opendata&datasource=andre_toiletter): - -* [Parking in Aarhus Municipality – Zones, Permits and - Spaces](https://www.opendata.dk/city-of-aarhus/parkering-i-aarhus-kommune) - * [Handicapparkering](https://webkort.aarhuskommune.dk/spatialmap?page=get_geojson_opendata&datasource=invap): - - -* diff --git a/Taskfile.yml b/Taskfile.yml index 49f5e72..84fbd88 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -36,6 +36,10 @@ tasks: test:integration: *test_task test:application: *test_task + import: + desc: 'Import a source into the broker, e.g. task import -- mtm_spatialmaps-handicap-parking' + cmd: ddev console app:import {{.CLI_ARGS}} + coding-standards:apply: desc: 'Apply coding standards' cmds: diff --git a/composer.json b/composer.json index b3bcf60..fad8d55 100644 --- a/composer.json +++ b/composer.json @@ -13,6 +13,7 @@ "proj4php/proj4php": "^2.0", "symfony/asset": "~8.1.0", "symfony/asset-mapper": "~8.1.0", + "symfony/config": "~8.1.0", "symfony/console": "~8.1.6", "symfony/dotenv": "~8.1.6", "symfony/flex": "^2", diff --git a/composer.lock b/composer.lock index 2b3d45f..2d8ddb4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "89753fcad5de669970a5e88087a1a523", + "content-hash": "dfaceee7643a3761619ee1e8ae943e6a", "packages": [ { "name": "composer/semver", diff --git a/config/sources.yaml b/config/sources.yaml new file mode 100644 index 0000000..bebb760 --- /dev/null +++ b/config/sources.yaml @@ -0,0 +1,37 @@ +# The data sets this application publishes, one entry per source key. +# +# Field names follow DCAT-AP, the metadata profile European data portals +# harvest, so registering a data set is a translation of its entry rather than +# a new survey. See docs/adr/007-source-manifest.md. + +sources: + mtm_spatialmaps-handicap-parking: + title: 'Handicapparkering, Aarhus Kommune' + description: >- + Disabled parking bays in Aarhus Municipality, with the number of + reserved bays per location. + publisher: 'Aarhus Kommune' + contact: ppg@aarhus.dk + landing_page: 'https://www.opendata.dk/city-of-aarhus/parkering-i-aarhus-kommune' + access_url: 'https://webkort.aarhuskommune.dk/spatialmap?page=get_geojson_opendata&datasource=invap' + media_type: application/geo+json + crs: 'EPSG:25832' + model: OnStreetParking + context_url: 'https://raw.githubusercontent.com/smart-data-models/dataModel.Parking/master/context.jsonld' + update_frequency: continuous + + # The portal states no licence for this data set. DCAT-AP requires + # one, so it has to be settled with the data owner before the + # catalogue can be registered anywhere. + licence: ~ + + # Fields the feed carries that are not published. Recorded here + # because the source class shows what is mapped but cannot show what + # was left out, or why. + omitted_fields: + ident: 'Single-letter code; its meaning is not documented and not confirmed by the data owner.' + oprettet_af: 'Directory username of the municipal employee who created the record.' + rettet_af: 'Directory username of the municipal employee who last edited the record.' + oprettet_dato: 'Describes the register record.' + rettet_dato: 'Describes the register record.' + mi_style: 'MapInfo rendering style, empty throughout the export.' diff --git a/docs/adr/001-architecture-symfony-docker.md b/docs/adr/001-architecture-symfony-docker.md new file mode 100644 index 0000000..1431f7b --- /dev/null +++ b/docs/adr/001-architecture-symfony-docker.md @@ -0,0 +1,74 @@ +# 001: Architecture — Symfony 8 on the ITK Dev Docker template + +| Field | Value | +|--------------------|----------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-24 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, future maintainers | +| **Status** | Draft | + +## Context + +This application reads open data sets, converts them to a standard smart-city +representation, and publishes them to a context broker. The organisation +maintains its PHP services on a versioned Docker template carrying shared CI +and coding-standards configuration. This ADR serves to decide the runtime, +framework and development environment the application is built on. + +### Drivers + +- **Functional:** scheduled console commands, outbound HTTP, and a local broker + to import into. No database and no HTTP surface of its own. +- **Non-functional:** shared tooling rather than reimplemented tooling, minimal + onboarding, reproducible across developers and CI, long-term vendor support. + +### Options Considered + +1. **PHP 8.4 / Symfony 8 on the maintained template.** CI, coding standards and + task runner come for free, and its console suits scheduled imports; it + provisions services this application never uses, and its PHP runs ahead of + developer hosts. +2. **A minimal project on the host, without the template.** No unused services + and no container requirement, but shared configuration is rebuilt by hand + and a local broker needs containers anyway, moving the requirement rather + than removing it. +3. **A second entry point in an existing internal application.** One deployment + to operate, but couples a batch importer to a user-facing release cycle and + inherits dependencies it has no use for. +4. **A different language ecosystem on a bespoke setup.** Richer geospatial + libraries, but no internal expertise and no shared tooling; the needed + transformations exist as mature libraries in the established stack. + +## Decision + +**PHP 8.4 + Symfony 8** on the ITK Dev Docker template, as its **own deployable +service**, with a containerised broker overlay for local development. + +- Standardising costs less over the application's lifetime than trimming unused + services: a second toolchain must be learned and patched; idle containers + cost only disk. +- A batch importer's lifecycle and failure modes differ from a user-facing + application's, so it stays its own service. +- No domain persistence is needed — the broker is the system of record — so the + template's database is left unused rather than removed, keeping template + updates a clean diff. +- Local development includes a real broker, so imports are verified end to end + rather than only as serialised output. + +## Consequences + +### Positive + +- Onboarding cost close to zero; CI and coding standards work from the first + commit. +- No schema, no migrations, no state to keep consistent with the broker. + +### Negative / Trade-offs + +- Containers are mandatory; dependency management, console commands and tests + cannot run natively. +- A web server, database and mail catcher are provisioned and never used. +- Broker images are not published for every CPU architecture, so local start-up + may be slow under emulation. +- The application follows the template's choices; deviating later has a cost. diff --git a/docs/adr/002-publish-to-a-context-broker.md b/docs/adr/002-publish-to-a-context-broker.md new file mode 100644 index 0000000..a1225c6 --- /dev/null +++ b/docs/adr/002-publish-to-a-context-broker.md @@ -0,0 +1,86 @@ +# 002: Publication mechanism — publish to a context broker + +| Field | Value | +|--------------------|--------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-24 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, data consumers, future maintainers | +| **Status** | Draft | + +## Context + +This application republishes data held in operational systems, in heterogeneous +formats and coordinate systems, to consumers the organisation neither controls +nor can brief. The source systems stay authoritative; the published copy is not +a system of record. This ADR serves to decide the mechanism by which data is +published to consumers. + +### Drivers + +- **Functional:** query by location and attribute rather than bulk download + only, several data sets on one consumer-facing surface, change notification, + and adding a data set without changing consumer integrations. +- **Non-functional:** interpretable by consumers never spoken to, served through + client tooling that already exists, operational cost proportionate to the data + and its consumers, and an interface that outlives individual data sets. + +### Options Considered + +1. **An NGSI-LD context broker.** Geospatial and attribute queries, pagination, + subscriptions and a temporal interface without implementing them, and + payloads carrying a vocabulary reference; substantial operational weight, and + the strictness of a particular implementation is inherited. +2. **A bespoke REST API** over a datastore of our own. Exact fit and full + control of the query surface and its semantics; every capability from + geo-queries to notifications, clients and versioning is ours to build, and + consumers must learn an interface that exists nowhere else. +3. **Direct database access or a read replica.** No API layer, powerful ad-hoc + querying; exposes internal schema as a public contract, needs per-consumer + credentials and network access, and is unusable by browser-based consumers. + +## Decision + +Publish to an **NGSI-LD context broker**. + +1. Consumers of geographic data ask for an area or a kind more often than for a + whole data set, and a broker offers that as a standard interface rather than + a per-data-set feature. +2. Structure published without a vocabulary reference obliges every consumer to + hold separate documentation; a broker payload carries the reference. +3. A single small data set would be served better by a static export. Across + several heterogeneous ones the fixed operational cost is paid once, the + per-data-set cost approaches zero, and consumers integrate once rather than + once per source. +4. New consumers require no change here, new data sets none from consumers, and + existing viewers, dashboards and connectors already speak this interface. + +The value taken is interoperability and query, not storage. Revisit once +consumers have had time to appear: if none read the data through the interface, +a static export was the better decision. + +## Consequences + +### Positive + +- Geospatial and attribute queries, pagination, subscriptions and a temporal + interface, none of which are implemented here. +- Payloads reference a shared vocabulary, so they need no bespoke documentation. +- Additional data sets reach every existing consumer with no integration work. +- This application keeps no database and no read surface of its own. + +### Negative / Trade-offs + +- Operational weight out of proportion to a small data set: several services to + run, patch, monitor and back up. +- Broker implementations impose constraints beyond the standard. Those met + include one accepted spelling of a UTC timestamp while an equivalent is + rejected, the vocabulary reference required on reads where omission returns an + empty success rather than an error, and single-element lists collapsed to + scalars. +- Vocabulary documents may be fetched over the network during writes, making + third-party availability part of the import path. +- Upsert creates and updates but never removes, so records that disappear + upstream persist until reconciliation is built. +- JSON-LD is a learning curve for maintainers and consumers, and fitting data to + a shared vocabulary costs effort that publishing as-is would not. diff --git a/docs/adr/003-ngsi-ld-representation.md b/docs/adr/003-ngsi-ld-representation.md new file mode 100644 index 0000000..ffbb16f --- /dev/null +++ b/docs/adr/003-ngsi-ld-representation.md @@ -0,0 +1,65 @@ +# 003: NGSI-LD representation — normalized form and batch upsert + +| Field | Value | +|--------------------|--------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-24 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, data consumers, future maintainers | +| **Status** | Draft | + +## Context + +ADR 002 chooses an NGSI-LD context broker, which leaves the representation +open. + +This ADR serves to decide how attributes are shaped and which write operation +is used. + +### Drivers + +- **Functional:** the broker must accept the payload on write, attribute + metadata must be expressible, and repeated imports must not duplicate + entities. +- **Non-functional:** idempotency, payload size, readability for consumers. + +### Options Considered + +#### Attribute form + +1. **Normalized.** Every attribute an object naming its kind. Verbose, but the + form brokers accept on write and the only one carrying metadata. +2. **Key-values.** Flat `name: value`, smaller and easier to read, but + read-only and unable to carry attribute metadata. + +#### Write operation + +1. **Batch upsert.** Creates or updates, so a re-import refreshes in place. +2. **Create.** Fails with `409` for an identifier that already exists, so a + re-import errors rather than refreshing. +3. **Batch replace.** Silently drops attributes absent from the payload, + making a partial payload destructive. + +## Decision + +Publish **normalized** NGSI-LD with `Content-Type: application/ld+json`, via +**batch upsert**. + +- Normalized is the only form accepted on write, so key-values is not + available to a producer. +- Identifiers derive from each data set's primary key, so upsert makes a + re-run update in place rather than add. + +## Consequences + +### Positive + +- Re-imports produce no duplicates and need no prior state. +- Attribute metadata stays expressible where a data set supplies it. + +### Negative / Trade-offs + +- Payloads are considerably larger than key-values. +- Upsert never deletes, so records removed upstream persist until + reconciliation is built (see ADR 002). +- Consumers unfamiliar with JSON-LD face a learning curve. diff --git a/docs/adr/004-coordinate-reference-system.md b/docs/adr/004-coordinate-reference-system.md new file mode 100644 index 0000000..f77f88a --- /dev/null +++ b/docs/adr/004-coordinate-reference-system.md @@ -0,0 +1,93 @@ +# 004: Coordinate reference system — publish WGS84 (EPSG:4326) + +| Field | Value | +|--------------------|----------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-27 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, broker consumers, future maintainers | +| **Status** | Draft | + +## Context + +Every entity the adapter publishes carries a `location` GeoProperty, so the +coordinate reference system is cross-cutting rather than a per-input detail. +Inputs arrive in whatever CRS their publisher uses — Danish municipal data is +commonly projected, typically EPSG:25832 (ETRS89 / UTM zone 32N) as eastings and +northings in metres — and no single input CRS can be assumed. A GeoJSON envelope +states a geometry type but not units, so projected coordinates arrive inside one +undetected. + +This ADR serves to decide which coordinate reference system is published, and at +what precision. + +### Drivers + +- **Functional:** `location` is interpretable unbriefed, geo-queries return + correct results, and clients render without preprocessing. +- **Non-functional:** conformance and self-description; one rule for every + input; precision no worse than the input. + +### Options Considered + +1. **Normalise everything to WGS84, reprojecting in the adapter.** One rule + however many input CRSs accumulate. Needs a reprojection dependency, and each + input must declare its CRS. +2. **Pass each input's native CRS through unchanged.** No transformation and no + dependency, but the GeoJSON is invalid, entities from different inputs are + incomparable, and distances read as degrees. +3. **Publish WGS84 and also keep the original coordinates in an extra + attribute.** Saves consumers a round trip, but the attribute has no stable + shape — CRS and geometry type differ per input, and it is absent for inputs + already in WGS84 — so nothing can be coded against it. +4. **Pass native CRSs through under RFC 7946's "prior arrangement" clause.** + Permitted, but the clause requires all parties to have agreed, which a broker + whose consumers are unknown by design cannot satisfy. + +## Decision + +Every `location` the adapter emits is **EPSG:4326 (WGS84) longitude/latitude**, +at **full precision — coordinates are not rounded**. Each input declares its own +CRS; reprojection happens at the boundary between reading an input and building +an entity, and nowhere else. + +1. **The specification leaves no choice.** RFC 7946 §4 fixes the CRS for all + GeoJSON coordinates as geographic, WGS 84 datum, longitude and latitude in + decimal degrees; NGSI-LD GeoProperty values are GeoJSON and inherit it. +2. **There is no way to declare otherwise.** RFC 7946 Appendix B.1 removed the + `crs` member of the 2008 specification, so projected coordinates publish an + undeclarable assumption. An input may still carry that deprecated member; it + can be read, not passed on. +3. Nothing catches the mistake: a broker accepts projected coordinates, answers + geo-queries incorrectly and places points wrongly, raising no error; and + entities from different inputs are queried together, so a query spanning two + CRSs returns meaningless results. +4. **A round trip is lossless at full precision.** Projected to geographic and + back returns the input exactly when nothing is rounded; rounding trades + accuracy for a marginal payload reduction. +5. `source` and `seeAlso` can reference the originating export, which states its + own CRS. + +## Consequences + +### Positive + +- Payloads are valid GeoJSON and NGSI-LD, comparable across inputs, and render + in any client unmodified, under one rule for every present and future input. +- Reprojection is isolated in one tested component, verified against + independently known reference coordinates, so a regression fails loudly rather + than silently relocating data. + +### Negative / Trade-offs + +- Adds a reprojection dependency, and national grid definitions are not always + shipped, so registering them becomes load-bearing project code. +- Datum shifts are approximated: ETRS89-based grids are treated as equivalent + to WGS84 via a null datum transformation, so what is published is ETRS89 + labelled WGS84. Coincident in 1989, the two have diverged by roughly 0.5–1 m + at about 2.5 cm per year — standard practice in web GIS, and the largest + error introduced, so a consumer transforming rigorously with an explicit + epoch lands about a metre away. +- Consumers with natively projected stacks must convert. +- Every new input must declare its CRS, and unsupported ones need adding. +- Each geometry type needs its own reprojection, not points alone. diff --git a/docs/adr/005-smart-data-models-as-vocabulary.md b/docs/adr/005-smart-data-models-as-vocabulary.md new file mode 100644 index 0000000..7cba550 --- /dev/null +++ b/docs/adr/005-smart-data-models-as-vocabulary.md @@ -0,0 +1,96 @@ +# 005: Vocabulary — adopt Smart Data Models + +| Field | Value | +|--------------------|--------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-31 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, data consumers, future maintainers | +| **Status** | Draft | + +## Context + +NGSI-LD defines how attributes are carried and how a vocabulary is referenced, +not which entity types and attribute names exist. Without one the JSON-LD +context resolves to nothing and consumers need our documentation to interpret +anything. This ADR serves to decide which vocabulary supplies entity types and +attribute names, and the rules for mapping sources onto it. + +### Drivers + +- **Functional:** types and attributes interpretable without our documentation, + expressible as a JSON-LD context a broker can resolve, covering the domains in + scope. +- **Non-functional:** a vocabulary consumers plausibly already know, maintained + by someone else, at a mapping cost that does not dominate onboarding. + +### Options Considered + +1. **Smart Data Models.** The openly governed reference vocabulary of the + NGSI-LD ecosystem, with broad coverage, per-domain context documents, and a + schema and examples per model to conform against. Depth varies, many models + assume real-time sensing, required attributes can presuppose a hierarchy a + source lacks, and versioning is loose. +2. **A vocabulary of our own, with self-hosted context documents.** Exact fit + and full control of naming and versioning, but nobody else speaks it, + governance and documentation stay ours indefinitely, and no existing tooling + recognises the types. +3. **A general-purpose web vocabulary.** Widely recognised and stably governed, + adequate for names, addresses and descriptions, but with no NGSI-LD + conventions for geometry or relationships and no domain terms, leaving the + domains in scope unmodelled. + +## Decision + +Adopt **Smart Data Models**, referencing the relevant domain context documents +alongside the NGSI-LD core context. Which model a given data set uses is +recorded in its own ADR; this one states policy. + +1. **Use an existing model; do not invent a type.** An imperfect standard type + is more useful to a consumer than a precise private one. +2. **Never fabricate a value to satisfy a model.** An absent attribute is + honest; a fabricated one is indistinguishable from a measured one. + - Attributes the source cannot fill are left unset, not approximated, + defaulted or inferred. + - Where a model *requires* an attribute the source cannot supply, choose a + sibling model without the requirement, even if its terms are less precise. + - In a hierarchy where each level requires a relationship upward, publish at + the highest level the source can populate — typically the top, for a data + set giving a location and a count of units within it. + +Rationale: + +- A context must resolve to terms a consumer recognises, or publishing gains + nothing over a file, and Smart Data Models ships those documents, so adoption + is a URL rather than a project. +- Shipped schemas and examples make modelling disagreements checkable against a + specification instead of settled by preference. +- Required relationships propagate downward, so publishing below the top level + defers the need for a parent rather than avoiding it, and the costs are + asymmetric: acquiring real parents later is a one-off migration, whereas never + acquiring them means maintaining an invented entity that every consumer + following the relationship receives as meaningless. + +## Consequences + +### Positive + +- Types and attributes resolve to shared global identifiers consumers may + already have code for. +- A later data set is likely covered already, so onboarding it does not start + with vocabulary design. +- Published entities are self-contained, with nothing invented to keep in sync, + and finer granularity can be added beneath them later. + +### Negative / Trade-offs + +- Models built around real-time sensing carry attributes static inventory + cannot fill, so many will always be empty. +- Model choice is embedded in entity identifiers, so changing model later means + deleting and re-publishing rather than updating in place. +- Enum values must be read from the schema rather than the examples: where the + two disagree the schema is authoritative, and equivalent-looking values differ + between sibling models, so they cannot be copied across. +- Versioning is loose, so a model can change without an obvious signal. +- Mapping a source to a model takes longer than exposing its fields verbatim, + and occasionally the fit is poor. diff --git a/docs/adr/006-onstreetparking-over-parkinggroup.md b/docs/adr/006-onstreetparking-over-parkinggroup.md new file mode 100644 index 0000000..f72fe3f --- /dev/null +++ b/docs/adr/006-onstreetparking-over-parkinggroup.md @@ -0,0 +1,85 @@ +# 006: Model selection — OnStreetParking over ParkingGroup + +| Field | Value | +|--------------------|--------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-08-31 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, data consumers, future maintainers | +| **Status** | Draft | + +## Context + +The parking domain is a hierarchy. A site — `OnStreetParking` or +`OffStreetParking` — requires only `id`, `type` and `location`. Below it, a +`ParkingGroup` subdivision and a `ParkingSpot` unit must each reference a site, +and a spot also requires `status` and `category`. + +This decision applies where a data set provides a count of units per location +with a point geometry, no reference to a containing site, and neither per-unit +geometry nor occupancy. + +This ADR serves to decide which model in the parking hierarchy is published +under those conditions. + +### Drivers + +- **Functional:** every mandatory relationship points at an entity that exists; + the restriction on who may park is expressible unambiguously; finer + granularity is addable later without restructuring. +- **Non-functional:** nothing invented purely to satisfy a schema; a reversible + choice in preference to one that is not. + +### Options Considered + +1. **`ParkingGroup`, creating the missing parent site.** `onlyDisabled` states + exclusivity by name, and the model's reference example for disabled parking + sits at this level — but points at a real street-address site. + `refParkingSite` is mandatory with no value available, so a parent must be + invented; one spanning the administrative area asserts a false containment + and its own mandatory geometry carries no meaning. +2. **`ParkingGroup`, omitting `refParkingSite`.** Nothing invented, smallest + change — but knowingly non-conformant, and a schema validator flags every + entity. +3. **`OnStreetParking`.** Everything it requires is available, and it is the + entity both lower levels must reference, so granularity can be added beneath + it and migration downward stays possible if real sites appear. `category` + offers only `forDisabled`, which states exclusivity less plainly. +4. **`ParkingSpot`.** Models the individual unit, but only a count per location + is available, `status` is mandatory with no occupancy data, and a parent site + is required. Rejected outright. + +## Decision + +Publish each record as an **`OnStreetParking`** entity, with +`category: ["forDisabled"]` and no `refParkingSite`. + +- The only option that invents nothing: everything it requires is available. +- Mandatory references propagate downward, so `ParkingGroup` would schedule the + invented parent rather than avoid it — `ParkingSpot` requires a site too. +- The costs are asymmetric. Site level, then finding real sites exist, is a + one-off migration; subdivision level, then never acquiring real sites, means + maintaining an invented entity indefinitely. + +## Consequences + +### Positive + +- No dangling relationship: every entity is self-contained, and nothing invented + has to be created, documented or kept in sync. +- Conformant to the model's schema without exceptions. +- `ParkingGroup` or `ParkingSpot` entities can be attached beneath these later + without changing them. + +### Negative / Trade-offs + +- `forDisabled` does not state exclusivity. The schema documents `category` only + as "Street parking category" with an enum list and defines no individual + value, and the two models prefix apparently identical concepts inconsistently + — `forDisabled` / `forResidents` against `onlyDisabled` / `onlyResidents` — + while both carry `onlyWithPermit`, so values must not be copied between the + enums. +- Entity type is embedded in identifiers, so any later change means deleting and + re-publishing. +- Domain experts may find a single address described as a "site" + counter-intuitive. diff --git a/docs/adr/007-source-manifest.md b/docs/adr/007-source-manifest.md new file mode 100644 index 0000000..8bb90c3 --- /dev/null +++ b/docs/adr/007-source-manifest.md @@ -0,0 +1,70 @@ +# 007: Data set metadata — a committed source manifest + +| Field | Value | +|--------------------|--------------------------------------------------------| +| **Created By** | Jeppe Krogh | +| **Date** | 2026-09-02 | +| **Decision Maker** | ITK Dev team | +| **Stakeholders** | ITK Dev developers, data consumers, future maintainers | +| **Status** | Draft | + +## Context + +This application publishes data sets it does not own. This ADR serves +to decide where the data sets and their specifications live. + +### Drivers + +- **Functional:** each data set is listed with what is needed to read and + republish it. +- **Non-functional:** adding a data set needs no deployment change, and its + specification is reviewable as a diff. + +### Options Considered + +1. **One environment variable per data set.** Variable names grow with the + collection, and the environment holds only strings. +2. **Every fact in the class that maps the data set.** Nothing can diverge, but + the specification is readable only by opening code, and correcting a licence + becomes a code change. +3. **A committed manifest the code reads.** One record per data set, holding + the specification beside the values the import needs; its shape has to be + declared. +4. **An external catalogue or registry service.** The eventual home of + published specifications, but a second system to operate, populated before + anything can be published from it. + +## Decision + +Keep one **committed manifest** listing every data set this application +publishes, keyed by the identifier the import selects a data set by, and read +from it every fact the code needs. + +1. **Record only what the code cannot state.** Field mappings and feed quirks + stay in the class that maps the data set. +2. **A fact the manifest records is not restated elsewhere**, in code, a + comment, or documentation. +3. **An incomplete record is an error.** Required fields raise rather than + default, an unknown fact is recorded as unknown, and the shape is a schema + the framework validates when the application is built. +4. **Structure each record after DCAT-AP**, the specification the data will be + published under, so publication is a translation rather than a redesign. + +## Consequences + +### Positive + +- Every data set is listed in one place, reviewable as a diff and versioned + with the code. +- Publishing a specification onward, and answering the data owner, translate + records that already exist. +- A wrong record, or a field the manifest does not define, fails the build. + +### Negative / Trade-offs + +- Values are identical in every environment, so pointing a data set at a test + copy means editing a committed file. +- A wrong reference system looks less like code than it is, and yields + coordinates that are well-formed and misplaced. +- A malformed record blocks every build, not just the import that reads it. +- Fields no code reads have only review keeping them current. diff --git a/docs/adr/CLAUDE.md b/docs/adr/CLAUDE.md new file mode 100644 index 0000000..679a5cd --- /dev/null +++ b/docs/adr/CLAUDE.md @@ -0,0 +1,78 @@ +# CLAUDE.md — writing ADRs in this project + +Guidance for Claude Code when creating or editing files in `docs/adr/`. + +An ADR here **decides a specific choice on general grounds**. The choice is +concrete; the reasoning must hold beyond any one data set, developer or machine. + +## Never include + +- **Local project or repository names.** These are organisation-level documents; + someone's working copies have no place in them. +- **Specific data sources, their fields, or their quirks.** Not field names, not + envelope shapes, not "the source provides…". State the *condition* the + decision applies under instead: "This decision applies where a data set + provides a count of units per location…". +- **Measurements or counts taken from a data set.** No record counts, no + "measured across N records", no byte or centimetre figures derived from one + export. Standards identifiers (EPSG codes, RFC numbers, framework versions) + and general domain facts are fine. +- **Educational or self-referential framing.** No "two things make this easy to + get wrong", no "note that", no "the real lesson is", no bolded aphorism + followed by the actual fact, and no commentary on the ADR's own importance + ("this is the hardest to reverse"). State the fact and stop. + +## Structure + +```text +# NNN: Area — the choice + +| Field | Value | Created By, Date, Decision Maker, Stakeholders, Status + +## Context the situation; what is undecided + ends with: "This ADR serves to decide …" +### Drivers Functional / Non-functional, brief +### Options Considered one paragraph each, upsides then downsides +## Decision the choice, then terse rationale bullets +## Consequences +### Positive +### Negative / Trade-offs the honest costs, including ones found by hitting them +``` + +The Context section **must end with a single "This ADR serves to …" sentence** +so the purpose is unmissable. + +Status is `Draft` while it needs review, then `Accepted`. Other values: +`Rejected`, `Deprecated by NNN`, `Supersedes NNN`. + +## Numbering + +Numbering follows **dependency order**: every ADR may reference only +lower-numbered ones. Check for forward references after adding or renumbering. +Dates therefore need not run in the same order as numbers. + +Renumbering is only acceptable while nothing external cites the numbers. Once +cited, supersede instead. + +## Before writing, ask whether it earns its place + +- **Is there a real alternative?** An ADR comparing one viable candidate against + a dead end documents a decision that made itself, and dilutes the ones + recording genuine trade-offs. Reframe it around the decisions that did have + alternatives, or fold it into a neighbour. +- **Would folding it into an existing ADR be better?** Prefer fewer, shorter + documents. Consolidation has been chosen over adding a document more than + once. +- **Are the rules distinct?** One principle restated against several targets is + one rule with sub-points, not several rules. + +## Before finishing + +- Grep for source field names, model names in the general ADRs, project names, + record counts and measured figures. +- Grep for `Note that`, `worth noting`, `is the lesson`, `easy to get wrong`, + `Two things`, `earns its`, `is the point`. +- Check no ADR cites a higher number than its own. +- `task coding-standards:markdown:check` — recurring failures are line length + over 120 (MD013), misaligned table pipes (MD060), and bold used as a heading + (MD036); use `####` for sub-headings inside a section. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..0560124 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,22 @@ +# Architecture Decision Records + +This directory contains Architecture Decision Records (ADRs) for this project. + +See [adr.github.io](https://adr.github.io/) for background on the format. + +| Number | Title | Status | Date | +| ----------------------------------------------- | ------------------------------------------------------- | ------ | ---------- | +| [001](001-architecture-symfony-docker.md) | Architecture — Symfony 8 on the ITK Dev Docker template | Draft | 2026-08-24 | +| [002](002-publish-to-a-context-broker.md) | Publication mechanism — publish to a context broker | Draft | 2026-08-24 | +| [003](003-ngsi-ld-representation.md) | NGSI-LD representation — normalized form, batch upsert | Draft | 2026-08-24 | +| [004](004-coordinate-reference-system.md) | Coordinate reference system — publish WGS84 | Draft | 2026-08-27 | +| [005](005-smart-data-models-as-vocabulary.md) | Vocabulary — adopt Smart Data Models | Draft | 2026-08-31 | +| [006](006-onstreetparking-over-parkinggroup.md) | Model selection — OnStreetParking over ParkingGroup | Draft | 2026-08-31 | +| [007](007-source-manifest.md) | Data set metadata — a committed source manifest | Draft | 2026-09-02 | + +Numbering follows dependency order: each ADR relies only on lower-numbered +ones. Dates therefore do not run in the same order as numbers. + +All ADRs state general policy and name no data set. Concrete per-data-set +facts are recorded in `config/sources.yaml`, and the mappings in the source +classes themselves. diff --git a/src/Broker/NgsiLdBroker.php b/src/Broker/NgsiLdBroker.php new file mode 100644 index 0000000..806a99f --- /dev/null +++ b/src/Broker/NgsiLdBroker.php @@ -0,0 +1,69 @@ +> $entities + * + * @return int the broker's HTTP status code + */ + public function upsert(array $entities): int + { + if ([] === $entities) { + return 204; + } + + $response = $this->client->request( + 'POST', + rtrim($this->brokerUrl, '/').self::UPSERT_PATH, + [ + 'headers' => ['Content-Type' => self::CONTENT_TYPE], + 'json' => $entities, + ] + ); + + $status = $response->getStatusCode(); + + if ($status >= 400) { + throw new \RuntimeException(\sprintf('Broker rejected the upsert with HTTP %d: %s', $status, $response->getContent(false))); + } + + return $status; + } + + public function brokerUrl(): string + { + return $this->brokerUrl; + } +} diff --git a/src/Command/ImportCommand.php b/src/Command/ImportCommand.php new file mode 100644 index 0000000..e1c4fa9 --- /dev/null +++ b/src/Command/ImportCommand.php @@ -0,0 +1,128 @@ +addArgument('source', InputArgument::OPTIONAL, 'Source to import. Omit to list the available sources.') + ->addOption('dry-run', null, InputOption::VALUE_NONE, 'Print the NGSI-LD payload instead of sending it.') + ->addOption('limit', 'l', InputOption::VALUE_REQUIRED, 'Import at most this many entities.'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + + $keys = $this->importer->keys(); + + if ([] === $keys) { + $io->error('No data sources are registered.'); + $io->listing([ + 'A source must implement App\Source\SourceInterface.', + 'Implementations are picked up automatically — check the class exists and is not excluded from the container.', + ]); + + return Command::FAILURE; + } + + $argument = $input->getArgument('source'); + + if (null === $argument) { + // Options only make sense together with a source. Listing the + // sources and exiting successfully would look like an import ran. + if ($input->getOption('dry-run') || null !== $input->getOption('limit')) { + $io->error(\sprintf( + 'No source given. Available: %s.', + implode(', ', $keys) + )); + + return Command::INVALID; + } + + $io->section('Available sources'); + $io->listing($keys); + + return Command::SUCCESS; + } + + $key = (string) $argument; + $limit = null !== $input->getOption('limit') ? (int) $input->getOption('limit') : null; + + try { + if ($input->getOption('dry-run')) { + $payload = $this->importer->payload($key, $limit); + + $output->writeln(json_encode($payload, self::JSON_FLAGS)); + $io->note(\sprintf('Dry run: %d entities were not sent.', \count($payload))); + + return Command::SUCCESS; + } + + $result = $this->importer->import($key, $limit); + } catch (UnknownSourceException $exception) { + $io->error($exception->getMessage()); + + return Command::INVALID; + } catch (EmptySourceException $exception) { + $io->error($exception->getMessage()); + $io->text( + 'The source ran to completion without raising an exception, so every record was ' + .'discarded by the source\'s own guards rather than failing. Verbosity flags will ' + .'not reveal more: there is no exception to show.' + ); + $io->listing([ + 'Does the configured path or URL point at the intended document?', + 'Does the document match the shape the source expects — envelope, nesting, field names?', + 'Which guard returns early — a missing identifier, or a missing geometry?', + ]); + + return Command::FAILURE; + } catch (UpsertFailedException $exception) { + $io->error($exception->getMessage()); + + return Command::FAILURE; + } + + $io->success(\sprintf( + 'Upserted %d entities into %s (HTTP %d).', + $result->count, + $result->brokerUrl, + $result->status + )); + + return Command::SUCCESS; + } +} diff --git a/src/Geo/Wgs84Transformer.php b/src/Geo/Wgs84Transformer.php new file mode 100644 index 0000000..19489ba --- /dev/null +++ b/src/Geo/Wgs84Transformer.php @@ -0,0 +1,157 @@ + + */ + private const array DEFINITIONS = [ + // ETRS89 / UTM zone 32N — most of Denmark. + 'EPSG:25832' => '+proj=utm +zone=32 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs', + // ETRS89 / UTM zone 33N — Bornholm and eastwards. + 'EPSG:25833' => '+proj=utm +zone=33 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs', + // Web Mercator, in case an input arrives as tile coordinates. + 'EPSG:3857' => '+proj=merc +a=6378137 +b=6378137 +lat_ts=0 +lon_0=0 +x_0=0 +y_0=0 +k=1 +units=m +nadgrids=@null +no_defs', + self::TARGET_SRID => '+proj=longlat +datum=WGS84 +no_defs', + ]; + + private readonly Proj4php $proj4; + + /** @var array */ + private readonly array $definitions; + + /** @var array */ + private array $projections = []; + + /** + * @param array $definitions additional PROJ definitions, keyed by SRID + */ + public function __construct(array $definitions = []) + { + $this->definitions = [...self::DEFINITIONS, ...$definitions]; + + $this->proj4 = new Proj4php(); + foreach ($this->definitions as $srid => $definition) { + $this->proj4->addDef($srid, $definition); + } + } + + /** + * Coordinates are not rounded. + * + * @param string $srid source CRS + * + * @return array{float, float} GeoJSON coordinate order: [longitude, latitude] + */ + public function toWgs84(string $srid, float $x, float $y): array + { + if (self::TARGET_SRID === $srid) { + return [$x, $y]; + } + + $transformed = $this->proj4->transform( + $this->projection(self::TARGET_SRID), + new Point($x, $y, $this->projection($srid)) + ); + + return [(float) $transformed->x, (float) $transformed->y]; + } + + /** + * @param string $srid source CRS + * + * @return array{type: string, coordinates: array{float, float}} GeoJSON Point + */ + public function point(string $srid, float $x, float $y): array + { + return [ + 'type' => 'Point', + 'coordinates' => $this->toWgs84($srid, $x, $y), + ]; + } + + /** + * Reprojects a whole GeoJSON geometry, whatever its type. + * + * Handles Point, LineString, Polygon, MultiPoint, MultiLineString and + * MultiPolygon — every type an NGSI-LD GeoProperty accepts. GeometryCollection + * is not supported, because it carries `geometries` rather than `coordinates`. + * + * A third ordinate (elevation) is dropped: the inputs are two-dimensional, + * and vertical datums are a separate concern this class does not model. + * + * @param string $srid source CRS + * @param array $geometry GeoJSON geometry object + * + * @return array{type: string, coordinates: mixed} + */ + public function transformGeometry(string $srid, array $geometry): array + { + $type = $geometry['type'] ?? null; + + if (!\is_string($type) || !\array_key_exists('coordinates', $geometry)) { + throw new \InvalidArgumentException('Not a GeoJSON geometry: "type" and "coordinates" are both required.'); + } + + return [ + 'type' => $type, + 'coordinates' => $this->transformCoordinates($srid, $geometry['coordinates']), + ]; + } + + /** + * GeoJSON nests coordinates to a depth that depends on the geometry type: + * a bare position for Point, an array of positions for LineString, an array + * of those for Polygon, and so on. Recursing until the first element is + * numeric handles every depth without enumerating the types. + * + * @param array $coordinates + * + * @return array + */ + private function transformCoordinates(string $srid, array $coordinates): array + { + if ([] === $coordinates) { + throw new \InvalidArgumentException('GeoJSON coordinates must be a non-empty array.'); + } + + if (is_numeric($coordinates[0] ?? null)) { + if (!is_numeric($coordinates[1] ?? null)) { + throw new \InvalidArgumentException('A GeoJSON position needs at least two ordinates.'); + } + + return $this->toWgs84($srid, (float) $coordinates[0], (float) $coordinates[1]); + } + + return array_map( + fn (mixed $nested): array => $this->transformCoordinates($srid, $nested), + array_values($coordinates) + ); + } + + private function projection(string $srid): Proj + { + if (!isset($this->definitions[$srid])) { + throw new \InvalidArgumentException(\sprintf('Unknown CRS "%s". Register a PROJ definition for it before use. Known: %s.', $srid, implode(', ', array_keys($this->definitions)))); + } + + return $this->projections[$srid] ??= new Proj($srid, $this->proj4); + } +} diff --git a/src/Import/DataSourceImporter.php b/src/Import/DataSourceImporter.php new file mode 100644 index 0000000..f2a7fa7 --- /dev/null +++ b/src/Import/DataSourceImporter.php @@ -0,0 +1,136 @@ + $sources + */ + public function __construct( + #[AutowireIterator('app.source')] + private iterable $sources, + private Catalog $catalog, + private NgsiLdBroker $broker, + #[Autowire(env: 'ENTER_NGSI_CONTEXT_URLS')] + private string $contextUrls, + ) { + } + + /** + * @return list every registered source key, in registration order + */ + public function keys(): array + { + return array_keys($this->registry()); + } + + /** + * Converts a source to NGSI-LD. + * + * @return non-empty-list> + * + * @throws UnknownSourceException when no source is registered under the key + * @throws EmptySourceException when the source yields no entities + */ + public function payload(string $key, ?int $limit = null): array + { + // Collect all dataset keys. + $registry = $this->registry(); + + // Check if requested dataset key exists. + if (!isset($registry[$key])) { + throw new UnknownSourceException($key, array_keys($registry)); + } + + // Define minimum limit, in case of limit defined as less than 1. + $limit = null === $limit ? null : max(1, $limit); + + // Load context for given dataset. + $contexts = $this->contexts($key); + + $payload = []; + foreach ($registry[$key]->entities() as $entity) { + $payload[] = $entity->toArray($contexts); + + // Break upon limit. + if (null !== $limit && \count($payload) >= $limit) { + break; + } + } + + if ([] === $payload) { + throw new EmptySourceException($key); + } + + return $payload; + } + + /** + * @throws UnknownSourceException when no source is registered under the key + * @throws EmptySourceException when the source yields no entities + * @throws UpsertFailedException when the broker cannot be written to + */ + public function import(string $key, ?int $limit = null): ImportResult + { + // Get payload from dataset. + $payload = $this->payload($key, $limit); + + // Try to upsert broker with payload. + try { + $status = $this->broker->upsert($payload); + } catch (\Throwable $exception) { + throw new UpsertFailedException($exception); + } + + // Return result. + return new ImportResult(\count($payload), $status, $this->broker->brokerUrl()); + } + + /** + * Get list of registered datasets. + * + * @see config/sources.yaml + * + * @return array keyed by source key + */ + private function registry(): array + { + $registry = []; + + foreach ($this->sources as $source) { + $registry[$source->key()] = $source; + } + + return $registry; + } + + /** + * Return an array of contexts. Each dataset holds its own context. + * + * @see config/sources.yaml + * + * @return array + */ + private function contexts(string $key): array + { + return [ + $this->catalog->get($key)->contextUrl, + ...array_values(array_filter(array_map(trim(...), explode(',', $this->contextUrls)))), + ]; + } +} diff --git a/src/Import/Exception/EmptySourceException.php b/src/Import/Exception/EmptySourceException.php new file mode 100644 index 0000000..d6bcc3f --- /dev/null +++ b/src/Import/Exception/EmptySourceException.php @@ -0,0 +1,16 @@ + $known every key that is registered + */ + public function __construct( + public readonly string $key, + public readonly array $known, + ) { + parent::__construct(\sprintf( + 'Unknown source "%s". Available: %s.', + $key, + [] === $known ? 'none' : implode(', ', $known) + )); + } +} diff --git a/src/Import/Exception/UpsertFailedException.php b/src/Import/Exception/UpsertFailedException.php new file mode 100644 index 0000000..de80c2a --- /dev/null +++ b/src/Import/Exception/UpsertFailedException.php @@ -0,0 +1,16 @@ +getMessage(), previous: $previous); + } +} diff --git a/src/Import/ImportResult.php b/src/Import/ImportResult.php new file mode 100644 index 0000000..f891f65 --- /dev/null +++ b/src/Import/ImportResult.php @@ -0,0 +1,23 @@ +> */ + private array $attributes = []; + + public function __construct( + private readonly string $id, + private readonly string $type, + ) { + } + + public function id(): string + { + return $this->id; + } + + /** + * Null and empty-string values are dropped rather than emitted as null, + * because the source data uses "" for "not filled in" and a broker would + * otherwise store the emptiness as a fact. + */ + public function setProperty(string $name, mixed $value, ?string $observedAt = null): self + { + if (null === $value || '' === $value || [] === $value) { + return $this; + } + + $attribute = ['type' => 'Property', 'value' => $value]; + + if (null !== $observedAt) { + $attribute['observedAt'] = $observedAt; + } + + $this->attributes[$name] = $attribute; + + return $this; + } + + /** + * @param array{type: string, coordinates: mixed} $geoJson + */ + public function geoProperty(string $name, array $geoJson): self + { + $this->attributes[$name] = ['type' => 'GeoProperty', 'value' => $geoJson]; + + return $this; + } + + public function relationship(string $name, string $object): self + { + $this->attributes[$name] = ['type' => 'Relationship', 'object' => $object]; + + return $this; + } + + /** + * @param list $contextUrls + * + * @return array + */ + public function toArray(array $contextUrls): array + { + return [ + 'id' => $this->id, + 'type' => $this->type, + ...$this->attributes, + '@context' => $contextUrls, + ]; + } +} diff --git a/src/Source/DataSourceReader.php b/src/Source/DataSourceReader.php new file mode 100644 index 0000000..c8f3751 --- /dev/null +++ b/src/Source/DataSourceReader.php @@ -0,0 +1,58 @@ + the decoded document + * + * @throws \RuntimeException when the location is not an http(s) URL, cannot + * be fetched, does not contain valid JSON, or + * does not decode to an array + */ + public function read(string $url): array + { + if (!str_starts_with($url, 'http://') && !str_starts_with($url, 'https://')) { + throw new \RuntimeException(\sprintf('Feed location must be an http(s) URL, got "%s".', $url)); + } + + $json = $this->fetch($url); + + try { + $decoded = json_decode($json, true, 512, \JSON_THROW_ON_ERROR); + } catch (\JsonException $exception) { + throw new \RuntimeException(\sprintf('Invalid JSON in "%s": %s', $url, $exception->getMessage()), previous: $exception); + } + + // A JSON document may legally be a scalar. Every feed we consume is a + // list or an object, and a scalar here means the location is wrong + // rather than that the feed is empty. + if (!\is_array($decoded)) { + throw new \RuntimeException(\sprintf('Expected a JSON array or object in "%s", got %s.', $url, get_debug_type($decoded))); + } + + return $decoded; + } + + private function fetch(string $url): string + { + try { + return $this->client->request('GET', $url)->getContent(); + } catch (\Throwable $exception) { + throw new \RuntimeException(\sprintf('Could not fetch "%s": %s', $url, $exception->getMessage()), previous: $exception); + } + } +} diff --git a/src/Source/Manifest/Catalog.php b/src/Source/Manifest/Catalog.php new file mode 100644 index 0000000..71ad47f --- /dev/null +++ b/src/Source/Manifest/Catalog.php @@ -0,0 +1,114 @@ +|null */ + private ?array $descriptors = null; + + public function __construct( + #[Autowire('%kernel.project_dir%/config/sources.yaml')] + private readonly string $manifest, + ) { + } + + /** + * @throws \RuntimeException when the manifest cannot be read, or carries no entry for the key + */ + public function get(string $key): Descriptor + { + $descriptors = $this->all(); + + if (!isset($descriptors[$key])) { + throw new \RuntimeException(\sprintf('No entry for source "%s" in %s. Entries: %s.', $key, $this->manifest, [] === $descriptors ? 'none' : implode(', ', array_keys($descriptors)))); + } + + return $descriptors[$key]; + } + + /** + * @return array keyed by source key + * + * @throws \RuntimeException when the manifest cannot be read + */ + public function all(): array + { + return $this->descriptors ??= $this->load(); + } + + /** + * @return array + */ + private function load(): array + { + $descriptors = []; + + foreach ($this->validated() as $key => $entry) { + $descriptors[$key] = new Descriptor( + key: $key, + title: $entry['title'], + accessUrl: $entry['access_url'], + crs: $entry['crs'], + model: $entry['model'], + contextUrl: $entry['context_url'], + description: $entry['description'], + publisher: $entry['publisher'], + contact: $entry['contact'], + landingPage: $entry['landing_page'], + mediaType: $entry['media_type'], + updateFrequency: $entry['update_frequency'], + licence: $entry['licence'], + omittedFields: $entry['omitted_fields'], + ); + } + + return $descriptors; + } + + /** + * @return array}> + */ + private function validated(): array + { + if (!is_file($this->manifest)) { + throw new \RuntimeException(\sprintf('Source manifest "%s" does not exist.', $this->manifest)); + } + + try { + $parsed = Yaml::parseFile($this->manifest); + } catch (ParseException $exception) { + throw new \RuntimeException(\sprintf('Source manifest "%s" is not valid YAML: %s', $this->manifest, $exception->getMessage()), previous: $exception); + } + + $sources = \is_array($parsed) ? $parsed['sources'] ?? null : null; + if (!\is_array($sources)) { + throw new \RuntimeException(\sprintf('Source manifest "%s" must contain a "sources" mapping at the top level.', $this->manifest)); + } + + try { + /** @var array}> $processed */ + $processed = new Processor()->process(Schema::tree(), [$sources]); + } catch (InvalidConfigurationException $exception) { + throw new \RuntimeException(\sprintf('Source manifest "%s" is invalid: %s', $this->manifest, $exception->getMessage()), previous: $exception); + } + + return $processed; + } +} diff --git a/src/Source/Manifest/Descriptor.php b/src/Source/Manifest/Descriptor.php new file mode 100644 index 0000000..46d0631 --- /dev/null +++ b/src/Source/Manifest/Descriptor.php @@ -0,0 +1,40 @@ + $omittedFields source field name => why it is not published + */ + public function __construct( + public string $key, + public string $title, + public string $accessUrl, + public string $crs, + public string $model, + public string $contextUrl, + public ?string $description = null, + public ?string $publisher = null, + public ?string $contact = null, + public ?string $landingPage = null, + public ?string $mediaType = null, + public ?string $updateFrequency = null, + public ?string $licence = null, + public array $omittedFields = [], + ) { + } +} diff --git a/src/Source/Manifest/Schema.php b/src/Source/Manifest/Schema.php new file mode 100644 index 0000000..bbaf41c --- /dev/null +++ b/src/Source/Manifest/Schema.php @@ -0,0 +1,115 @@ +getRootNode() + // The import selects a data set by its key exactly as written in + // the manifest. Key normalization rewrites a key that contains + // dashes and no underscore, which breaks that lookup silently. + ->normalizeKeys(false) + ->requiresAtLeastOneElement() + ->arrayPrototype() + ->beforeNormalization() + ->always(self::blankToNull()) + ->end() + ->children() + ->scalarNode('title') + ->isRequired() + ->cannotBeEmpty() + ->info('What the data set is called where it is published.') + ->end() + ->scalarNode('access_url') + ->isRequired() + ->cannotBeEmpty() + ->info('Where the feed is read from; an import cannot run without it.') + ->end() + ->scalarNode('crs') + ->isRequired() + ->cannotBeEmpty() + ->info('The CRS the feed publishes coordinates in, e.g. "EPSG:25832". A wrong value yields well-formed coordinates in the wrong place.') + ->end() + ->scalarNode('model') + ->isRequired() + ->cannotBeEmpty() + ->info('Smart Data Model the data set is published as.') + ->end() + ->scalarNode('context_url') + ->isRequired() + ->cannotBeEmpty() + ->info('JSON-LD context defining the model\'s terms. Without the right one a consumer reads the entity\'s attributes as undefined strings.') + ->end() + ->scalarNode('description')->defaultNull()->end() + ->scalarNode('publisher')->defaultNull()->end() + ->scalarNode('contact')->defaultNull()->end() + ->scalarNode('landing_page')->defaultNull()->end() + ->scalarNode('media_type')->defaultNull()->end() + ->scalarNode('update_frequency')->defaultNull()->end() + ->scalarNode('licence') + ->defaultNull() + ->info('Empty means the terms are unsettled; DCAT-AP requires one before the data set can be registered.') + ->end() + ->arrayNode('omitted_fields') + ->defaultValue([]) + ->normalizeKeys(false) + ->scalarPrototype() + ->beforeNormalization() + ->always(self::blankToNullValue()) + ->end() + ->cannotBeEmpty() + ->info('The reason a field is withheld cannot be recovered from the code, so a bare list of names is not accepted.') + ->end() + ->end() + ->end() + ->end(); + + return $tree->buildTree(); + } + + /** + * An empty value means "not filled in", the same as an absent key, so both + * become null rather than an empty string. + * + * Normalization runs before the type check, so an entry that is not a + * mapping has to pass through untouched for the tree to report it as one. + */ + private static function blankToNull(): \Closure + { + return static function (mixed $entry): mixed { + if (!\is_array($entry)) { + return $entry; + } + + return array_map(self::blankToNullValue(), $entry); + }; + } + + private static function blankToNullValue(): \Closure + { + return static function (mixed $value): mixed { + if (!\is_string($value)) { + return $value; + } + + $value = trim($value); + + return '' === $value ? null : $value; + }; + } +} diff --git a/src/Source/Manifest/Validator.php b/src/Source/Manifest/Validator.php new file mode 100644 index 0000000..01d54dd --- /dev/null +++ b/src/Source/Manifest/Validator.php @@ -0,0 +1,34 @@ + nothing is written, so nothing is preloaded + */ + public function warmUp(string $cacheDir, ?string $buildDir = null): array + { + $this->catalog->all(); + + return []; + } +} diff --git a/src/Source/MtmSpatialMaps/HandicapParking.php b/src/Source/MtmSpatialMaps/HandicapParking.php new file mode 100644 index 0000000..1612cf0 --- /dev/null +++ b/src/Source/MtmSpatialMaps/HandicapParking.php @@ -0,0 +1,96 @@ +catalog->get(self::KEY); + + // The export is a GeoJSON FeatureCollection, so the records live under + // `features`. Iterating the document itself would walk its two + // top-level keys instead. + foreach ($this->reader->read($source->accessUrl)['features'] ?? [] as $feature) { + if (\is_array($feature) && null !== $entity = $this->toEntity($feature, $source)) { + yield $entity; + } + } + } + + /** + * Maps one feed record onto an NgsiEntity. + * + * @param array $feature GeoJSON Feature + */ + private function toEntity(array $feature, Descriptor $source): ?NgsiEntity + { + $row = $feature['properties'] ?? null; + $geometry = $feature['geometry'] ?? null; + + if (!\is_array($row) || !\is_array($geometry)) { + return null; + } + + // mi_prinx is the feed's stable primary key. Without it there is no + // way to address the same bay again on the next import, and an upsert + // would create duplicates instead of updating. + $key = $row['mi_prinx'] ?? null; + if (null === $key || '' === $key) { + return null; + } + + $entity = new NgsiEntity( + \sprintf('urn:ngsi-ld:%s:aarhus-handicap-%s', $source->model, $key), + $source->model + ); + + return $entity + ->setProperty('name', $this->address($row)) + ->setProperty('description', trim((string) ($row['bemrk'] ?? ''))) + ->setProperty('category', ['forDisabled']) + ->setProperty('totalSpotNumber', (int) ($row['invalidepladser'] ?? 0)) + ->setProperty('source', $source->accessUrl) + ->geoProperty('location', $this->transformer->transformGeometry($source->crs, $geometry)); + } + + /** + * @param array $row + */ + private function address(array $row): string + { + return trim(\sprintf( + '%s %s', + trim((string) ($row['vejnavn'] ?? '')), + trim((string) ($row['husnnr'] ?? '')) + )); + } +} diff --git a/src/Source/SourceInterface.php b/src/Source/SourceInterface.php new file mode 100644 index 0000000..f985048 --- /dev/null +++ b/src/Source/SourceInterface.php @@ -0,0 +1,29 @@ + + */ + public function entities(): iterable; +} diff --git a/tests/Command/ImportCommandTest.php b/tests/Command/ImportCommandTest.php new file mode 100644 index 0000000..6d4c7ca --- /dev/null +++ b/tests/Command/ImportCommandTest.php @@ -0,0 +1,149 @@ + $sources + */ + private function tester(iterable $sources, ?MockHttpClient $client = null): CommandTester + { + $keys = []; + foreach ($sources as $source) { + $keys[] = $source->key(); + } + + return new CommandTester(new ImportCommand(new DataSourceImporter( + $sources, + new Catalog($this->manifestFor($keys)), + new NgsiLdBroker($client ?? new MockHttpClient(), 'http://broker.invalid'), + 'https://example.com/core.jsonld', + ))); + } + + public function testItListsTheSourcesWhenCalledBare(): void + { + $tester = $this->tester([FakeSource::withEntities('some-source')]); + + $status = $tester->execute([]); + + $this->assertSame(Command::SUCCESS, $status); + $this->assertStringContainsString('some-source', $tester->getDisplay()); + } + + public function testItFailsWhenNoSourcesAreRegistered(): void + { + $tester = $this->tester([]); + + $status = $tester->execute([]); + + $this->assertSame(Command::FAILURE, $status); + $this->assertStringContainsString('No data sources are registered', $tester->getDisplay()); + } + + /** + * Passing --dry-run with no source used to print the source listing and + * exit successfully, silently ignoring the flag. + */ + public function testItRejectsOptionsWithoutASource(): void + { + $tester = $this->tester([FakeSource::withEntities('some-source')]); + + $status = $tester->execute(['--dry-run' => true]); + + $this->assertSame(Command::INVALID, $status); + $this->assertStringContainsString('No source given', $tester->getDisplay()); + } + + public function testAnUnknownSourceIsTheCallersMistake(): void + { + $tester = $this->tester([FakeSource::withEntities('some-source')]); + + $status = $tester->execute(['source' => 'nope']); + + $this->assertSame(Command::INVALID, $status); + $this->assertStringContainsString('Unknown source "nope"', $tester->getDisplay()); + } + + /** + * The failure carries no exception to show, so the command has to supply + * the places worth looking itself. + */ + public function testAnEmptySourceFailsAndSuggestsCauses(): void + { + $tester = $this->tester([new FakeSource('empty-source')]); + + $status = $tester->execute(['source' => 'empty-source']); + $display = $tester->getDisplay(); + + $this->assertSame(Command::FAILURE, $status); + $this->assertStringContainsString('produced no entities', $display); + $this->assertStringContainsString('path or URL', $display); + $this->assertStringContainsString('envelope, nesting, field names', $display); + } + + public function testDryRunPrintsThePayloadAndSendsNothing(): void + { + $client = new MockHttpClient(); + $tester = $this->tester([FakeSource::withEntities('one-entity', 'urn:ngsi-ld:Example:1')], $client); + + $status = $tester->execute(['source' => 'one-entity', '--dry-run' => true]); + $display = $tester->getDisplay(); + + $this->assertSame(Command::SUCCESS, $status); + $this->assertStringContainsString('"urn:ngsi-ld:Example:1"', $display); + $this->assertStringContainsString('1 entities were not sent', $display); + $this->assertSame(0, $client->getRequestsCount()); + } + + public function testItReportsWhatWasUpserted(): void + { + $client = new MockHttpClient(new MockResponse('', ['http_code' => 204])); + $tester = $this->tester([FakeSource::withEntities('one-entity', 'urn:ngsi-ld:Example:1')], $client); + + $status = $tester->execute(['source' => 'one-entity']); + $display = $tester->getDisplay(); + + $this->assertSame(Command::SUCCESS, $status); + $this->assertStringContainsString('Upserted 1 entities', $display); + $this->assertStringContainsString('HTTP 204', $display); + } + + /** + * The broker being down is an operational condition rather than a bug, so + * it is reported as a message instead of an uncaught exception. + */ + public function testABrokerFailureIsReportedAsAnError(): void + { + $client = new MockHttpClient(new MockResponse('', ['http_code' => 500])); + $tester = $this->tester([FakeSource::withEntities('one-entity', 'urn:ngsi-ld:Example:1')], $client); + + $status = $tester->execute(['source' => 'one-entity']); + + $this->assertSame(Command::FAILURE, $status); + $this->assertStringContainsString('HTTP 500', $tester->getDisplay()); + } +} diff --git a/tests/Geo/Wgs84TransformerTest.php b/tests/Geo/Wgs84TransformerTest.php new file mode 100644 index 0000000..2f2192b --- /dev/null +++ b/tests/Geo/Wgs84TransformerTest.php @@ -0,0 +1,182 @@ +transformer = new Wgs84Transformer(); + } + + public function testItReprojectsFromUtm32ToWgs84(): void + { + [$longitude, $latitude] = $this->transformer->toWgs84( + self::UTM32, + self::REFERENCE_EASTING, + self::REFERENCE_NORTHING + ); + + $this->assertEqualsWithDelta(10.2152, $longitude, 0.001); + $this->assertEqualsWithDelta(56.1592, $latitude, 0.001); + } + + public function testItReturnsLongitudeLatitudeOrder(): void + { + $point = $this->transformer->point(self::UTM32, 574108.2557507273, 6222343.6199512165); + + $this->assertSame('Point', $point['type']); + + // Longitude first, per GeoJSON. A swapped pair is easy to spot here: + // latitude cannot be 10.19. + $this->assertGreaterThan(9.0, $point['coordinates'][0]); + $this->assertLessThan(11.0, $point['coordinates'][0]); + $this->assertGreaterThan(55.0, $point['coordinates'][1]); + $this->assertLessThan(57.0, $point['coordinates'][1]); + } + + public function testItPassesThroughCoordinatesAlreadyInWgs84(): void + { + $this->assertSame( + [10.199512, 56.149753], + $this->transformer->toWgs84(Wgs84Transformer::TARGET_SRID, 10.199512, 56.149753) + ); + } + + public function testItSupportsAnotherRegisteredCrs(): void + { + // UTM zone 33N: same northing, easting near the zone's central + // meridian, so the result must land east of zone 32's output. + [$longitude] = $this->transformer->toWgs84('EPSG:25833', 500000.0, 6224460.0); + + $this->assertEqualsWithDelta(15.0, $longitude, 0.001); + } + + public function testItRejectsAnUnregisteredCrs(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessageMatches('/Unknown CRS "EPSG:31700"/'); + + $this->transformer->toWgs84('EPSG:31700', 500000.0, 6224460.0); + } + + public function testItAcceptsAdditionalDefinitions(): void + { + $transformer = new Wgs84Transformer([ + 'EPSG:23032' => '+proj=utm +zone=32 +ellps=intl +units=m +no_defs', + ]); + + [$longitude, $latitude] = $transformer->toWgs84('EPSG:23032', self::REFERENCE_EASTING, self::REFERENCE_NORTHING); + + // Different datum, so not identical to EPSG:25832 — but the same + // corner of the world, which proves the definition was registered. + $this->assertEqualsWithDelta(10.2152, $longitude, 0.01); + $this->assertEqualsWithDelta(56.1592, $latitude, 0.01); + } + + public function testItReprojectsAPointGeometry(): void + { + $geometry = $this->transformer->transformGeometry(self::UTM32, [ + 'type' => 'Point', + 'coordinates' => [self::REFERENCE_EASTING, self::REFERENCE_NORTHING], + ]); + + $this->assertSame('Point', $geometry['type']); + $this->assertEqualsWithDelta(10.2152, $geometry['coordinates'][0], 0.001); + } + + public function testItReprojectsALineString(): void + { + $geometry = $this->transformer->transformGeometry(self::UTM32, [ + 'type' => 'LineString', + 'coordinates' => [ + [574108.2557507273, 6222343.6199512165], + [self::REFERENCE_EASTING, self::REFERENCE_NORTHING], + ], + ]); + + $this->assertSame('LineString', $geometry['type']); + $this->assertCount(2, $geometry['coordinates']); + $this->assertEqualsWithDelta(10.1926, $geometry['coordinates'][0][0], 0.001); + $this->assertEqualsWithDelta(10.2152, $geometry['coordinates'][1][0], 0.001); + } + + public function testItReprojectsAPolygonPreservingNesting(): void + { + $geometry = $this->transformer->transformGeometry(self::UTM32, [ + 'type' => 'Polygon', + 'coordinates' => [ + [ + [574108.0, 6222343.0], + [574208.0, 6222343.0], + [574208.0, 6222443.0], + [574108.0, 6222343.0], + ], + ], + ]); + + $this->assertSame('Polygon', $geometry['type']); + $this->assertCount(1, $geometry['coordinates']); + $this->assertCount(4, $geometry['coordinates'][0]); + $this->assertIsFloat($geometry['coordinates'][0][0][0]); + } + + public function testItReprojectsAMultiPolygon(): void + { + $ring = [ + [574108.0, 6222343.0], + [574208.0, 6222343.0], + [574208.0, 6222443.0], + [574108.0, 6222343.0], + ]; + + $geometry = $this->transformer->transformGeometry(self::UTM32, [ + 'type' => 'MultiPolygon', + 'coordinates' => [[$ring], [$ring]], + ]); + + $this->assertCount(2, $geometry['coordinates']); + $this->assertCount(4, $geometry['coordinates'][1][0]); + } + + public function testItRejectsAGeometryWithoutCoordinates(): void + { + $this->expectException(\InvalidArgumentException::class); + + $this->transformer->transformGeometry(self::UTM32, ['type' => 'Point']); + } + + public function testItRejectsAGeometryCollection(): void + { + $this->expectException(\InvalidArgumentException::class); + + $this->transformer->transformGeometry(self::UTM32, [ + 'type' => 'GeometryCollection', + 'geometries' => [], + ]); + } + + public function testItRejectsAPositionWithASingleOrdinate(): void + { + $this->expectException(\InvalidArgumentException::class); + + $this->transformer->transformGeometry(self::UTM32, ['type' => 'Point', 'coordinates' => [574108.0]]); + } +} diff --git a/tests/Import/DataSourceImporterTest.php b/tests/Import/DataSourceImporterTest.php new file mode 100644 index 0000000..3e838a7 --- /dev/null +++ b/tests/Import/DataSourceImporterTest.php @@ -0,0 +1,259 @@ + $sources each is registered in the manifest under its own key + */ + private function importer( + iterable $sources, + ?MockHttpClient $client = null, + string $contextUrls = self::CORE_CONTEXT, + ): DataSourceImporter { + $keys = []; + foreach ($sources as $source) { + $keys[] = $source->key(); + } + + return new DataSourceImporter( + $sources, + new Catalog($this->manifestFor($keys)), + new NgsiLdBroker($client ?? new MockHttpClient(), self::BROKER_URL), + $contextUrls, + ); + } + + public function testItListsTheRegisteredSourceKeys(): void + { + $importer = $this->importer([FakeSource::withEntities('a-source'), FakeSource::withEntities('b-source')]); + + $this->assertSame(['a-source', 'b-source'], $importer->keys()); + } + + public function testItListsNothingWhenNoSourceIsRegistered(): void + { + $this->assertSame([], $this->importer([])->keys()); + } + + public function testItRejectsAnUnknownSourceAndNamesTheKnownOnes(): void + { + $importer = $this->importer([FakeSource::withEntities('some-source')]); + + $this->expectException(UnknownSourceException::class); + $this->expectExceptionMessage('Unknown source "nope". Available: some-source.'); + + $importer->payload('nope'); + } + + /** + * The important one: a source yielding nothing used to be reported as a + * completed import, which is indistinguishable from a working one. + */ + public function testItFailsWhenASourceProducesNothing(): void + { + $importer = $this->importer([new FakeSource('empty-source')]); + + $this->expectException(EmptySourceException::class); + $this->expectExceptionMessage('Source "empty-source" produced no entities.'); + + $importer->payload('empty-source'); + } + + /** + * The guard belongs to the conversion rather than to the dry run, so an + * import that intends to send is held to it too. + */ + public function testItSendsNothingWhenASourceProducesNothing(): void + { + $client = new MockHttpClient(); + + try { + $this->importer([new FakeSource('empty-source')], $client)->import('empty-source'); + $this->fail('An empty source was reported as a completed import.'); + } catch (EmptySourceException) { + $this->assertSame(0, $client->getRequestsCount(), 'An empty payload was sent to the broker.'); + } + } + + /** + * Later entries win term conflicts, so the data set's own context comes + * first and the core context last, where it stays authoritative over the + * NGSI-LD terms a domain context may also define. + */ + public function testEveryEntityCarriesItsDataSetContextThenTheCoreContext(): void + { + $importer = $this->importer([FakeSource::withEntities('one-entity', 'urn:ngsi-ld:Example:1')]); + + $payload = $importer->payload('one-entity'); + + $this->assertSame( + [self::dataSetContext('one-entity'), self::CORE_CONTEXT], + $payload[0]['@context'] + ); + } + + /** + * Two data sets published under different models must not advertise each + * other's vocabulary, which a single app-wide list cannot avoid. + */ + public function testEachDataSetCarriesOnlyItsOwnContext(): void + { + $importer = $this->importer([ + FakeSource::withEntities('first-source', 'urn:ngsi-ld:Example:1'), + FakeSource::withEntities('second-source', 'urn:ngsi-ld:Example:2'), + ]); + + $this->assertSame( + [self::dataSetContext('first-source'), self::CORE_CONTEXT], + $importer->payload('first-source')[0]['@context'] + ); + $this->assertSame( + [self::dataSetContext('second-source'), self::CORE_CONTEXT], + $importer->payload('second-source')[0]['@context'] + ); + } + + /** + * A source registered in the container but not in the manifest has no + * context to publish under, so it fails rather than emitting entities a + * consumer cannot resolve. + */ + public function testItFailsWhenASourceHasNoManifestEntry(): void + { + $importer = new DataSourceImporter( + [FakeSource::withEntities('unregistered', 'urn:ngsi-ld:Example:1')], + new Catalog($this->manifestFor(['other-source'])), + new NgsiLdBroker(new MockHttpClient(), self::BROKER_URL), + self::CORE_CONTEXT, + ); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('No entry for source "unregistered"'); + + $importer->payload('unregistered'); + } + + /** + * The contexts arrive as one comma-separated environment variable, so they + * are written by hand and carry whatever spacing that produces. + */ + public function testItIgnoresSpacingAndEmptyEntriesInTheConfiguredContexts(): void + { + $importer = $this->importer( + [FakeSource::withEntities('one-entity', 'urn:ngsi-ld:Example:1')], + contextUrls: ' '.self::CORE_CONTEXT.' , ,', + ); + + $payload = $importer->payload('one-entity'); + + $this->assertSame( + [self::dataSetContext('one-entity'), self::CORE_CONTEXT], + $payload[0]['@context'] + ); + } + + public function testALimitCapsThePayload(): void + { + $importer = $this->importer([FakeSource::withEntities('many', 'urn:1', 'urn:2', 'urn:3', 'urn:4', 'urn:5')]); + + $this->assertCount(2, $importer->payload('many', 2)); + } + + /** + * A real source reads a whole feed, so a limit that converts everything + * and then trims would do all the work it was given to avoid. + */ + public function testALimitStopsPullingFromTheSource(): void + { + $source = FakeSource::withEntities('many', 'urn:1', 'urn:2', 'urn:3', 'urn:4', 'urn:5'); + + $this->importer([$source])->payload('many', 2); + + $this->assertSame(2, $source->produced()); + } + + /** + * --limit 0 is a mistyped option. Honouring it would produce an empty + * payload, which is the one outcome an import refuses to call a success. + */ + public function testALimitBelowOneStillImportsOneEntity(): void + { + $importer = $this->importer([FakeSource::withEntities('many', 'urn:1', 'urn:2')]); + + $this->assertCount(1, $importer->payload('many', 0)); + } + + public function testBuildingThePayloadSendsNothing(): void + { + $client = new MockHttpClient(); + + $this->importer([FakeSource::withEntities('one-entity', 'urn:1')], $client)->payload('one-entity'); + + $this->assertSame(0, $client->getRequestsCount()); + } + + public function testItUpsertsThePayloadAndReportsWhatTheBrokerDid(): void + { + $client = new MockHttpClient(new MockResponse('', ['http_code' => 204])); + + $result = $this->importer([FakeSource::withEntities('many', 'urn:1', 'urn:2')], $client)->import('many'); + + $this->assertSame(2, $result->count); + $this->assertSame(204, $result->status); + $this->assertSame(self::BROKER_URL, $result->brokerUrl); + $this->assertSame(1, $client->getRequestsCount(), 'The entities were not sent as one batch.'); + } + + public function testItReportsWhatTheBrokerSaidWhenTheUpsertIsRejected(): void + { + $client = new MockHttpClient(new MockResponse('{"title":"Bad Request"}', ['http_code' => 400])); + + try { + $this->importer([FakeSource::withEntities('one-entity', 'urn:1')], $client)->import('one-entity'); + $this->fail('A rejected upsert was reported as a completed import.'); + } catch (UpsertFailedException $exception) { + $this->assertStringContainsString('HTTP 400', $exception->getMessage()); + $this->assertStringContainsString('Bad Request', $exception->getMessage()); + $this->assertNotNull($exception->getPrevious(), 'The broker\'s own exception was discarded.'); + } + } + + /** + * A broker that cannot be reached fails in the HTTP client rather than in + * the broker's status check, and the two are the same thing to a caller. + */ + public function testItFailsTheSameWayWhenTheBrokerCannotBeReached(): void + { + $client = new MockHttpClient(static function (): never { + throw new TransportException('Connection refused'); + }); + + $this->expectException(UpsertFailedException::class); + $this->expectExceptionMessage('Connection refused'); + + $this->importer([FakeSource::withEntities('one-entity', 'urn:1')], $client)->import('one-entity'); + } +} diff --git a/tests/Source/DataSourceReaderTest.php b/tests/Source/DataSourceReaderTest.php new file mode 100644 index 0000000..9c866ca --- /dev/null +++ b/tests/Source/DataSourceReaderTest.php @@ -0,0 +1,114 @@ +reader($client)->read('https://example.com/feed.json'); + + $this->assertSame([['example' => 1]], $document); + } + + public function testItAcceptsABareArrayDocument(): void + { + $client = new MockHttpClient(new MockResponse('[1, 2, 3]')); + + $this->assertSame([1, 2, 3], $this->reader($client)->read('http://example.com/feed.json')); + } + + /** + * The envelope is feed-specific knowledge, so it must survive intact for + * the caller to interpret. Returning the features list directly would push + * that knowledge into the wrong class. + */ + public function testItDoesNotUnwrapTheEnvelope(): void + { + $client = new MockHttpClient(new MockResponse(self::FEATURE_COLLECTION)); + + $document = $this->reader($client)->read('https://example.com/feed.json'); + + $this->assertArrayHasKey('features', $document); + $this->assertArrayNotHasKey(0, $document); + $this->assertCount(2, $document['features']); + } + + public function testItRejectsALocationThatIsNotAUrl(): void + { + $client = new MockHttpClient(); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessageMatches('/must be an http\(s\) URL/'); + + try { + $this->reader($client)->read('data/feed.json'); + } finally { + // Rejection has to happen before the request, or a mistyped + // location becomes an opaque transport error instead. + $this->assertSame(0, $client->getRequestsCount()); + } + } + + /** + * A string that merely begins with "http" is not a URL. + */ + public function testItRejectsAPathThatMerelyStartsWithHttp(): void + { + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessageMatches('/must be an http\(s\) URL/'); + + $this->reader()->read('https-export.json'); + } + + public function testItFailsOnInvalidJson(): void + { + $client = new MockHttpClient(new MockResponse('{ not json')); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessageMatches('/Invalid JSON/'); + + $this->reader($client)->read('https://example.com/feed.json'); + } + + public function testItFailsWhenTheDocumentIsAScalar(): void + { + $client = new MockHttpClient(new MockResponse('"just a string"')); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessageMatches('/Expected a JSON array or object/'); + + $this->reader($client)->read('https://example.com/feed.json'); + } + + public function testItWrapsTransportFailures(): void + { + $client = new MockHttpClient(new MockResponse('', ['http_code' => 500])); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessageMatches('/Could not fetch/'); + + $this->reader($client)->read('https://example.com/feed.json'); + } +} diff --git a/tests/Source/FakeSource.php b/tests/Source/FakeSource.php new file mode 100644 index 0000000..052ab33 --- /dev/null +++ b/tests/Source/FakeSource.php @@ -0,0 +1,61 @@ + $entities + */ + public function __construct( + private readonly string $key, + private readonly array $entities = [], + ) { + } + + /** + * @param string ...$ids entity ids, one entity each + */ + public static function withEntities(string $key, string ...$ids): self + { + return new self($key, array_map( + static fn (string $id): NgsiEntity => new NgsiEntity($id, 'Example'), + array_values($ids) + )); + } + + public function key(): string + { + return $this->key; + } + + public function entities(): iterable + { + foreach ($this->entities as $entity) { + ++$this->produced; + + yield $entity; + } + } + + /** + * How many entities were actually pulled from this source. + */ + public function produced(): int + { + return $this->produced; + } +} diff --git a/tests/Source/Manifest/CatalogTest.php b/tests/Source/Manifest/CatalogTest.php new file mode 100644 index 0000000..2707d49 --- /dev/null +++ b/tests/Source/Manifest/CatalogTest.php @@ -0,0 +1,238 @@ + */ + private array $written = []; + + protected function tearDown(): void + { + foreach ($this->written as $path) { + if (is_file($path)) { + unlink($path); + } + } + + $this->written = []; + } + + public function testTheShippedManifestIsUsable(): void + { + $catalog = new Catalog(\dirname(__DIR__, 3).'/config/sources.yaml'); + + $this->assertNotSame([], $catalog->all(), 'The manifest registers no data sets.'); + } + + /** + * A wrong URL scheme or a CRS the transformer does not know only surfaces + * mid-import otherwise, after the feed has been fetched. + */ + public function testEveryShippedEntryCanBeImportedFrom(): void + { + $catalog = new Catalog(\dirname(__DIR__, 3).'/config/sources.yaml'); + + foreach ($catalog->all() as $key => $descriptor) { + $this->assertSame($key, $descriptor->key); + $this->assertMatchesRegularExpression('#^https?://#', $descriptor->accessUrl, $key); + $this->assertMatchesRegularExpression('/^EPSG:\d+$/', $descriptor->crs, $key); + $this->assertNotSame('', $descriptor->model, $key); + $this->assertMatchesRegularExpression('#^https?://#', $descriptor->contextUrl, $key); + } + } + + public function testItNamesTheKnownEntriesWhenAskedForAnUnknownOne(): void + { + $catalog = new Catalog($this->manifest(<<<'YAML' + sources: + a-source: + title: A source + access_url: https://example.com/feed.json + crs: EPSG:25832 + model: Example + context_url: https://example.com/context.jsonld + YAML)); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('Entries: a-source.'); + + $catalog->get('no-such-source'); + } + + public function testItRejectsAManifestWithoutASourcesMapping(): void + { + $catalog = new Catalog($this->manifest("data_sets:\n a-source: {}\n")); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('must contain a "sources" mapping'); + + $catalog->all(); + } + + public function testItRejectsAnEntryMissingAFieldTheImportNeeds(): void + { + $catalog = new Catalog($this->manifest(<<<'YAML' + sources: + a-source: + title: A source + access_url: https://example.com/feed.json + model: Example + context_url: https://example.com/context.jsonld + YAML)); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('The child config "crs" under "sources.a-source" must be configured'); + + $catalog->all(); + } + + public function testItRejectsAnOmittedFieldWithoutAReason(): void + { + $catalog = new Catalog($this->manifest(<<<'YAML' + sources: + a-source: + title: A source + access_url: https://example.com/feed.json + crs: EPSG:25832 + model: Example + context_url: https://example.com/context.jsonld + omitted_fields: + some_field: ~ + YAML)); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('"sources.a-source.omitted_fields.some_field" cannot contain an empty value'); + + $catalog->all(); + } + + /** + * The import selects a data set by the key written in the manifest, and the + * config tree rewrites a key that has dashes and no underscore unless told + * otherwise. A rewritten key stops matching without saying so. + */ + public function testItKeepsADashedSourceKeyIntact(): void + { + $catalog = new Catalog($this->manifest(<<<'YAML' + sources: + handicap-parking: + title: A source + access_url: https://example.com/feed.json + crs: EPSG:25832 + model: Example + context_url: https://example.com/context.jsonld + YAML)); + + $this->assertSame(['handicap-parking'], array_keys($catalog->all())); + $this->assertSame('handicap-parking', $catalog->get('handicap-parking')->key); + } + + /** + * A misspelled optional field was dropped in silence before, which loses a + * fact the record exists to carry. + */ + public function testItRejectsAFieldTheManifestDoesNotDefine(): void + { + $catalog = new Catalog($this->manifest(<<<'YAML' + sources: + a-source: + title: A source + access_url: https://example.com/feed.json + crs: EPSG:25832 + model: Example + context_url: https://example.com/context.jsonld + license: CC-BY-4.0 + YAML)); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('Unrecognized option "license" under "sources.a-source"'); + + $catalog->all(); + } + + public function testItRejectsAnEntryThatIsNotAMapping(): void + { + $catalog = new Catalog($this->manifest("sources:\n a-source: just a string\n")); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('Invalid type for path "sources.a-source"'); + + $catalog->all(); + } + + public function testItRejectsAManifestThatRegistersNothing(): void + { + $catalog = new Catalog($this->manifest("sources: {}\n")); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('should have at least 1 element'); + + $catalog->all(); + } + + /** + * An empty value means "not filled in", the same as an absent key, so an + * unanswered question reads the same either way. + */ + public function testItReadsABlankOptionalFieldAsUnknown(): void + { + $catalog = new Catalog($this->manifest(<<<'YAML' + sources: + a-source: + title: A source + access_url: https://example.com/feed.json + crs: EPSG:25832 + model: Example + context_url: https://example.com/context.jsonld + publisher: ' Aarhus Kommune ' + contact: '' + licence: ~ + YAML)); + + $descriptor = $catalog->get('a-source'); + + $this->assertSame('Aarhus Kommune', $descriptor->publisher); + $this->assertNull($descriptor->contact); + $this->assertNull($descriptor->licence); + } + + public function testItReportsAManifestThatIsNotThere(): void + { + $catalog = new Catalog('/no/such/sources.yaml'); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('does not exist'); + + $catalog->all(); + } + + public function testItReportsUnparsableYaml(): void + { + $catalog = new Catalog($this->manifest("sources:\n - [unbalanced\n")); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('is not valid YAML'); + + $catalog->all(); + } + + private function manifest(string $yaml): string + { + $path = tempnam(sys_get_temp_dir(), 'sources-'); + + if (false === $path) { + $this->fail('Could not create a temporary manifest.'); + } + + file_put_contents($path, $yaml); + $this->written[] = $path; + + return $path; + } +} diff --git a/tests/Source/Manifest/ValidatorTest.php b/tests/Source/Manifest/ValidatorTest.php new file mode 100644 index 0000000..53230bb --- /dev/null +++ b/tests/Source/Manifest/ValidatorTest.php @@ -0,0 +1,90 @@ + */ + private array $written = []; + + protected function tearDown(): void + { + foreach ($this->written as $path) { + if (is_file($path)) { + unlink($path); + } + } + + $this->written = []; + } + + /** + * A check that can be skipped is not a check. + */ + public function testItIsNotOptional(): void + { + $this->assertFalse($this->validator(\dirname(__DIR__, 3).'/config/sources.yaml')->isOptional()); + } + + /** + * It validates rather than caches, so it leaves nothing behind to preload. + */ + public function testItAcceptsTheShippedManifestAndWritesNothing(): void + { + $validator = $this->validator(\dirname(__DIR__, 3).'/config/sources.yaml'); + + $this->assertSame([], $validator->warmUp(sys_get_temp_dir(), sys_get_temp_dir())); + } + + /** + * The reason for checking at build time: an entry no import selects still + * fails the build rather than waiting to be discovered. + */ + public function testItFailsOnAnEntryNoImportWouldReach(): void + { + $validator = $this->validator($this->manifest(<<<'YAML' + sources: + a-source: + title: A source + access_url: https://example.com/feed.json + crs: EPSG:25832 + model: Example + context_url: https://example.com/context.jsonld + unreached-source: + title: Another source + access_url: https://example.com/other.json + model: Example + context_url: https://example.com/context.jsonld + YAML)); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('The child config "crs" under "sources.unreached-source" must be configured'); + + $validator->warmUp(sys_get_temp_dir(), sys_get_temp_dir()); + } + + private function validator(string $manifest): Validator + { + return new Validator(new Catalog($manifest)); + } + + private function manifest(string $yaml): string + { + $path = tempnam(sys_get_temp_dir(), 'sources-'); + + if (false === $path) { + $this->fail('Could not create a temporary manifest.'); + } + + file_put_contents($path, $yaml); + $this->written[] = $path; + + return $path; + } +} diff --git a/tests/Source/Manifest/WritesManifests.php b/tests/Source/Manifest/WritesManifests.php new file mode 100644 index 0000000..121509f --- /dev/null +++ b/tests/Source/Manifest/WritesManifests.php @@ -0,0 +1,61 @@ + */ + private array $manifests = []; + + protected function tearDown(): void + { + foreach ($this->manifests as $path) { + if (is_file($path)) { + unlink($path); + } + } + + $this->manifests = []; + } + + protected static function dataSetContext(string $key): string + { + return \sprintf('https://example.com/%s.jsonld', $key); + } + + /** + * @param list $keys + * + * @return string path to the manifest + */ + protected function manifestFor(array $keys): string + { + $entries = array_map(static fn (string $key): string => \sprintf( + " %s:\n title: %s\n access_url: https://example.com/%s.json\n crs: EPSG:25832\n model: Example\n context_url: %s", + $key, + $key, + $key, + self::dataSetContext($key), + ), $keys); + + $path = tempnam(sys_get_temp_dir(), 'sources-'); + + if (false === $path) { + $this->fail('Could not create a temporary manifest.'); + } + + file_put_contents($path, "sources:\n".implode("\n", $entries)."\n"); + $this->manifests[] = $path; + + return $path; + } +} diff --git a/tests/Source/MtmSpatialMaps/HandicapParkingTest.php b/tests/Source/MtmSpatialMaps/HandicapParkingTest.php new file mode 100644 index 0000000..6517824 --- /dev/null +++ b/tests/Source/MtmSpatialMaps/HandicapParkingTest.php @@ -0,0 +1,190 @@ +> */ + private array $entities; + + protected function setUp(): void + { + $catalog = new Catalog(\dirname(__DIR__, 3).'/config/sources.yaml'); + $this->source = $catalog->get(self::KEY); + + $client = new MockHttpClient(function (string $method, string $url): MockResponse { + $this->requestedUrl = $url; + + return new MockResponse(json_encode($this->feed(), \JSON_THROW_ON_ERROR)); + }); + + $source = new HandicapParking(new DataSourceReader($client), new Wgs84Transformer(), $catalog); + + $this->entities = array_map( + static fn (NgsiEntity $entity): array => $entity->toArray(['https://example.com/context.jsonld']), + iterator_to_array($source->entities(), false) + ); + } + + public function testItReadsTheFeedTheManifestPointsAt(): void + { + $this->assertSame($this->source->accessUrl, $this->requestedUrl); + } + + public function testItSkipsRecordsWithoutAnIdentifierOrGeometry(): void + { + // Four features, of which one has no mi_prinx and one no geometry. + $this->assertCount(2, $this->entities); + } + + public function testItTakesIdentifierAndTypeFromTheManifestModel(): void + { + $first = $this->entities[0]; + + // The id is derived from mi_prinx so that re-importing upserts the + // same bay instead of creating a duplicate. + $this->assertSame( + \sprintf('urn:ngsi-ld:%s:aarhus-handicap-172', $this->source->model), + $first['id'] + ); + $this->assertSame($this->source->model, $first['type']); + } + + public function testItJoinsStreetAndHouseNumberIntoName(): void + { + $this->assertSame('Domkirkeplads/Bispegade 1', $this->entities[0]['name']['value']); + } + + public function testItOmitsTheHouseNumberWhenBlank(): void + { + // husnnr is "" for this row, so the name must not end in a space. + $this->assertSame('Brammersgade', $this->entities[1]['name']['value']); + } + + public function testItMarksEveryEntityAsDisabledParking(): void + { + foreach ($this->entities as $entity) { + $this->assertSame(['forDisabled'], $entity['category']['value']); + } + } + + public function testItCarriesTheBayCountAsTotalSpotNumber(): void + { + $this->assertSame(6, $this->entities[0]['totalSpotNumber']['value']); + $this->assertSame(1, $this->entities[1]['totalSpotNumber']['value']); + } + + public function testItDropsEmptyDescriptions(): void + { + // bemrk is null for the first row and filled in for the second. + $this->assertArrayNotHasKey('description', $this->entities[0]); + $this->assertSame('Ved indgangen', $this->entities[1]['description']['value']); + } + + public function testItReprojectsLocationIntoWgs84(): void + { + $location = $this->entities[0]['location']; + + $this->assertSame('GeoProperty', $location['type']); + $this->assertSame('Point', $location['value']['type']); + + // The feed publishes metres in the manifest's CRS, so degrees within + // Denmark are the evidence that the reprojection ran. + [$longitude, $latitude] = $location['value']['coordinates']; + $this->assertGreaterThan(8.0, $longitude); + $this->assertLessThan(13.0, $longitude); + $this->assertGreaterThan(54.5, $latitude); + $this->assertLessThan(57.8, $latitude); + } + + public function testItRecordsTheManifestUrlAsTheEntitySource(): void + { + $this->assertSame($this->source->accessUrl, $this->entities[0]['source']['value']); + } + + /** + * The first feature is a record from the live export, kept verbatim. The + * rest are constructed to exercise a blank house number and the two guards + * that discard a record. + * + * @return array + */ + private function feed(): array + { + return [ + 'type' => 'FeatureCollection', + 'crs' => ['type' => 'name', 'properties' => ['name' => 'EPSG:25832']], + 'features' => [ + [ + 'type' => 'Feature', + 'geometry' => ['type' => 'Point', 'coordinates' => [575153.9524951308, 6224260.609753487]], + 'properties' => [ + 'vejnavn' => 'Domkirkeplads/Bispegade', + 'husnnr' => '1', + 'invalidepladser' => 6, + 'bemrk' => null, + 'ident' => 'P', + 'oprettet_af' => 'ADM\\aztnbnd', + 'oprettet_dato' => '2019-03-19 14:51:23.91', + 'rettet_af' => 'ADM\\aztnbnd', + 'rettet_dato' => '2019-03-19 14:51:23.91', + 'mi_style' => null, + 'mi_prinx' => 172, + ], + ], + [ + 'type' => 'Feature', + 'geometry' => ['type' => 'Point', 'coordinates' => [574000.0, 6223000.0]], + 'properties' => [ + 'vejnavn' => 'Brammersgade', + 'husnnr' => '', + 'invalidepladser' => 1, + 'bemrk' => 'Ved indgangen', + 'mi_prinx' => 378, + ], + ], + [ + 'type' => 'Feature', + 'geometry' => ['type' => 'Point', 'coordinates' => [574100.0, 6223100.0]], + 'properties' => [ + 'vejnavn' => 'Uden nøgle', + 'husnnr' => '3', + 'invalidepladser' => 2, + ], + ], + [ + 'type' => 'Feature', + 'geometry' => null, + 'properties' => [ + 'vejnavn' => 'Uden geometri', + 'husnnr' => '5', + 'invalidepladser' => 2, + 'mi_prinx' => 999, + ], + ], + ], + ]; + } +}