From 9136e1a8bbfb89252f17ec889d23b659fd0206de Mon Sep 17 00:00:00 2001 From: Mikkel Ricky Date: Wed, 9 Sep 2026 14:03:03 +0200 Subject: [PATCH 1/4] Added Monolog --- composer.json | 1 + composer.lock | 264 +++++++++++++++++- config/bundles.php | 1 + config/packages/monolog.yaml | 55 ++++ config/reference.php | 149 ++++++++++ ...eadCommand.php => SourceImportCommand.php} | 0 symfony.lock | 12 + 7 files changed, 481 insertions(+), 1 deletion(-) create mode 100644 config/packages/monolog.yaml rename src/Command/{SourceReadCommand.php => SourceImportCommand.php} (100%) diff --git a/composer.json b/composer.json index fad8d55..d749026 100644 --- a/composer.json +++ b/composer.json @@ -19,6 +19,7 @@ "symfony/flex": "^2", "symfony/framework-bundle": "~8.1.6", "symfony/http-client": "~8.1.6", + "symfony/monolog-bundle": "^4.0", "symfony/property-access": "~8.1.0", "symfony/property-info": "~8.1.6", "symfony/runtime": "~8.1.0", diff --git a/composer.lock b/composer.lock index 2d8ddb4..f525ffb 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "dfaceee7643a3761619ee1e8ae943e6a", + "content-hash": "44363235a88ae03aec157c6d6f78089a", "packages": [ { "name": "composer/semver", @@ -178,6 +178,111 @@ }, "time": "2026-04-02T10:02:52+00:00" }, + { + "name": "monolog/monolog", + "version": "3.12.0", + "source": { + "type": "git", + "url": "https://github.com/Seldaek/monolog.git", + "reference": "72c534fc0ab181ef52d92a68382318631e301608" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Seldaek/monolog/zipball/72c534fc0ab181ef52d92a68382318631e301608", + "reference": "72c534fc0ab181ef52d92a68382318631e301608", + "shasum": "" + }, + "require": { + "php": ">=8.1", + "psr/log": "^2.0 || ^3.0" + }, + "provide": { + "psr/log-implementation": "3.0.0" + }, + "require-dev": { + "aws/aws-sdk-php": "^3.0", + "doctrine/couchdb": "~1.0@dev", + "elasticsearch/elasticsearch": "^7 || ^8", + "ext-json": "*", + "graylog2/gelf-php": "^1.4.2 || ^2.0", + "guzzlehttp/guzzle": "^7.4.5", + "guzzlehttp/psr7": "^2.2", + "mongodb/mongodb": "^1.8 || ^2.0", + "php-amqplib/php-amqplib": "~2.4 || ^3", + "php-console/php-console": "^3.1.8", + "phpstan/phpstan": "^2", + "phpstan/phpstan-deprecation-rules": "^2", + "phpstan/phpstan-strict-rules": "^2", + "phpunit/phpunit": "^10.5.17 || ^11.0.7", + "predis/predis": "^1.1 || ^2", + "psr/clock": "^1.0", + "rollbar/rollbar": "^4.0", + "ruflin/elastica": "^7 || ^8", + "symfony/mailer": "^5.4 || ^6", + "symfony/mime": "^5.4 || ^6" + }, + "suggest": { + "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", + "doctrine/couchdb": "Allow sending log messages to a CouchDB server", + "elasticsearch/elasticsearch": "Allow sending log messages to an Elasticsearch server via official client", + "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", + "ext-curl": "Required to send log messages using the IFTTTHandler, the LogglyHandler, the SendGridHandler, the SlackWebhookHandler or the TelegramBotHandler", + "ext-mbstring": "Allow to work properly with unicode symbols", + "ext-mongodb": "Allow sending log messages to a MongoDB server (via driver)", + "ext-openssl": "Required to send log messages using SSL", + "ext-sockets": "Allow sending log messages to a Syslog server (via UDP driver)", + "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", + "mongodb/mongodb": "Allow sending log messages to a MongoDB server (via library)", + "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", + "psr/clock": "Required to pass a clock to the Logger and control the timestamp of log records", + "rollbar/rollbar": "Allow sending log messages to Rollbar", + "ruflin/elastica": "Allow sending log messages to an Elastic Search server" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.x-dev" + } + }, + "autoload": { + "psr-4": { + "Monolog\\": "src/Monolog" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jordi Boggiano", + "email": "j.boggiano@seld.be", + "homepage": "https://seld.be" + } + ], + "description": "Sends your logs to files, sockets, inboxes, databases and various web services", + "homepage": "https://github.com/Seldaek/monolog", + "keywords": [ + "log", + "logging", + "psr-3" + ], + "support": { + "issues": "https://github.com/Seldaek/monolog/issues", + "source": "https://github.com/Seldaek/monolog/tree/3.12.0" + }, + "funding": [ + { + "url": "https://github.com/Seldaek", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/monolog/monolog", + "type": "tidelift" + } + ], + "time": "2026-09-09T08:34:20+00:00" + }, { "name": "phpdocumentor/reflection-common", "version": "2.2.0", @@ -2374,6 +2479,163 @@ ], "time": "2026-08-30T21:40:49+00:00" }, + { + "name": "symfony/monolog-bridge", + "version": "v8.1.6", + "source": { + "type": "git", + "url": "https://github.com/symfony/monolog-bridge.git", + "reference": "90747e82447b92c0fc701bfaf99bac0d54127039" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/monolog-bridge/zipball/90747e82447b92c0fc701bfaf99bac0d54127039", + "reference": "90747e82447b92c0fc701bfaf99bac0d54127039", + "shasum": "" + }, + "require": { + "monolog/monolog": "^3", + "php": ">=8.4.1", + "symfony/http-kernel": "^7.4|^8.0", + "symfony/service-contracts": "^2.5|^3" + }, + "require-dev": { + "symfony/console": "^7.4|^8.0", + "symfony/http-client": "^7.4|^8.0", + "symfony/mailer": "^7.4|^8.0", + "symfony/messenger": "^7.4|^8.0", + "symfony/mime": "^7.4|^8.0", + "symfony/notifier": "^7.4|^8.0", + "symfony/security-core": "^7.4|^8.0", + "symfony/var-dumper": "^7.4|^8.0" + }, + "type": "symfony-bridge", + "autoload": { + "psr-4": { + "Symfony\\Bridge\\Monolog\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Provides integration for Monolog with various Symfony components", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/monolog-bridge/tree/v8.1.6" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-08-30T01:03:44+00:00" + }, + { + "name": "symfony/monolog-bundle", + "version": "v4.0.2", + "source": { + "type": "git", + "url": "https://github.com/symfony/monolog-bundle.git", + "reference": "c012c6aba13129eb02aa7dd61e66e720911d8598" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/monolog-bundle/zipball/c012c6aba13129eb02aa7dd61e66e720911d8598", + "reference": "c012c6aba13129eb02aa7dd61e66e720911d8598", + "shasum": "" + }, + "require": { + "composer-runtime-api": "^2.0", + "monolog/monolog": "^3.5", + "php": ">=8.2", + "symfony/config": "^7.3 || ^8.0", + "symfony/dependency-injection": "^7.3 || ^8.0", + "symfony/http-kernel": "^7.3 || ^8.0", + "symfony/monolog-bridge": "^7.3 || ^8.0", + "symfony/polyfill-php84": "^1.30" + }, + "require-dev": { + "phpunit/phpunit": "^11.5.41 || ^12.3", + "symfony/console": "^7.3 || ^8.0", + "symfony/yaml": "^7.3 || ^8.0" + }, + "type": "symfony-bundle", + "autoload": { + "psr-4": { + "Symfony\\Bundle\\MonologBundle\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony MonologBundle", + "homepage": "https://symfony.com", + "keywords": [ + "log", + "logging" + ], + "support": { + "issues": "https://github.com/symfony/monolog-bundle/issues", + "source": "https://github.com/symfony/monolog-bundle/tree/v4.0.2" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-04-02T18:27:21+00:00" + }, { "name": "symfony/polyfill-deepclone", "version": "v1.42.0", diff --git a/config/bundles.php b/config/bundles.php index 93280e5..5b067eb 100644 --- a/config/bundles.php +++ b/config/bundles.php @@ -7,4 +7,5 @@ Twig\Extra\TwigExtraBundle\TwigExtraBundle::class => ['all' => true], Symfony\Bundle\WebProfilerBundle\WebProfilerBundle::class => ['dev' => true, 'test' => true], Symfony\Bundle\DebugBundle\DebugBundle::class => ['dev' => true], + Symfony\Bundle\MonologBundle\MonologBundle::class => ['all' => true], ]; diff --git a/config/packages/monolog.yaml b/config/packages/monolog.yaml new file mode 100644 index 0000000..e09985f --- /dev/null +++ b/config/packages/monolog.yaml @@ -0,0 +1,55 @@ +monolog: + channels: # To see all channels, run bin/console debug:autowiring logger + - deprecation # Deprecations are logged in the dedicated "deprecation" channel when it exists + +when@dev: + monolog: + handlers: + main: + type: stream + path: '%kernel.logs_dir%/%kernel.environment%.log' + level: debug + channels: ['!event'] + console: + type: console + process_psr_3_messages: false + channels: ['!event', '!doctrine', '!console'] + +when@test: + monolog: + handlers: + main: + type: fingers_crossed + action_level: error + handler: nested + excluded_http_codes: [404, 405] + channels: ['!event'] + nested: + type: stream + path: '%kernel.logs_dir%/%kernel.environment%.log' + level: debug + +when@prod: + monolog: + handlers: + main: + type: fingers_crossed + action_level: error + handler: nested + excluded_http_codes: [404, 405] + channels: ['!deprecation'] + buffer_size: 50 # How many messages should be saved? Prevent memory leaks + nested: + type: stream + path: php://stderr + level: debug + formatter: monolog.formatter.json + console: + type: console + process_psr_3_messages: false + channels: ['!event', '!doctrine'] + deprecation: + type: stream + channels: [deprecation] + path: php://stderr + formatter: monolog.formatter.json diff --git a/config/reference.php b/config/reference.php index 1af6c7d..aea1b6f 100644 --- a/config/reference.php +++ b/config/reference.php @@ -801,6 +801,151 @@ * dump_destination?: scalar|Param|null, // A stream URL where dumps should be written to. // Default: null * theme?: "dark"|"light"|Param, // Changes the color of the dump() output when rendered directly on the templating. "dark" (default) or "light". // Default: "dark" * } + * @psalm-type MonologConfig = array{ + * use_microseconds?: scalar|Param|null, // Default: true + * channels?: list, + * handlers?: array + * }, + * path?: scalar|Param|null, // Default: "%kernel.logs_dir%/%kernel.environment%.log" + * file_permission?: scalar|Param|null, // Default: null + * use_locking?: bool|Param, // Default: false + * filename_format?: scalar|Param|null, // Default: "{filename}-{date}" + * date_format?: scalar|Param|null, // Default: "Y-m-d" + * ident?: scalar|Param|null, // Default: false + * logopts?: scalar|Param|null, // Default: 1 + * facility?: scalar|Param|null, // Default: "user" + * max_files?: scalar|Param|null, // Default: 0 + * action_level?: scalar|Param|null, // Default: "WARNING" + * activation_strategy?: scalar|Param|null, // Default: null + * stop_buffering?: bool|Param, // Default: true + * passthru_level?: scalar|Param|null, // Default: null + * excluded_http_codes?: list, + * }>, + * accepted_levels?: list, + * min_level?: scalar|Param|null, // Default: "DEBUG" + * max_level?: scalar|Param|null, // Default: "EMERGENCY" + * buffer_size?: scalar|Param|null, // Default: 0 + * flush_on_overflow?: bool|Param, // Default: false + * handler?: scalar|Param|null, + * url?: scalar|Param|null, + * exchange?: scalar|Param|null, + * exchange_name?: scalar|Param|null, // Default: "log" + * channel?: scalar|Param|null, // Default: null + * bot_name?: scalar|Param|null, // Default: "Monolog" + * use_attachment?: scalar|Param|null, // Default: true + * use_short_attachment?: scalar|Param|null, // Default: false + * include_extra?: scalar|Param|null, // Default: false + * icon_emoji?: scalar|Param|null, // Default: null + * webhook_url?: scalar|Param|null, + * exclude_fields?: list, + * token?: scalar|Param|null, + * region?: scalar|Param|null, + * source?: scalar|Param|null, + * use_ssl?: bool|Param, // Default: true + * user?: mixed, + * title?: scalar|Param|null, // Default: null + * host?: scalar|Param|null, // Default: null + * port?: scalar|Param|null, // Default: 514 + * config?: list, + * members?: list, + * connection_string?: scalar|Param|null, + * timeout?: scalar|Param|null, + * time?: scalar|Param|null, // Default: 60 + * deduplication_level?: scalar|Param|null, // Default: 400 + * store?: scalar|Param|null, // Default: null + * connection_timeout?: scalar|Param|null, + * persistent?: bool|Param, + * message_type?: scalar|Param|null, // Default: 0 + * parse_mode?: scalar|Param|null, // Default: null + * disable_webpage_preview?: bool|Param|null, // Default: null + * disable_notification?: bool|Param|null, // Default: null + * split_long_messages?: bool|Param, // Default: false + * delay_between_messages?: bool|Param, // Default: false + * topic?: int|Param, // Default: null + * factor?: int|Param, // Default: 1 + * tags?: Param|string|list, + * console_formatter_options?: mixed, // Default: [] + * formatter?: scalar|Param|null, + * nested?: bool|Param, // Default: false + * publisher?: Param|string|array{ + * id?: scalar|Param|null, + * hostname?: scalar|Param|null, + * port?: scalar|Param|null, // Default: 12201 + * chunk_size?: scalar|Param|null, // Default: 1420 + * encoder?: "json"|"compressed_json"|Param, + * }, + * mongodb?: Param|string|array{ + * id?: scalar|Param|null, // ID of a MongoDB\Client service + * uri?: scalar|Param|null, + * username?: scalar|Param|null, + * password?: scalar|Param|null, + * database?: scalar|Param|null, // Default: "monolog" + * collection?: scalar|Param|null, // Default: "logs" + * }, + * elasticsearch?: Param|string|array{ + * id?: scalar|Param|null, + * hosts?: list, + * host?: scalar|Param|null, + * port?: scalar|Param|null, // Default: 9200 + * transport?: scalar|Param|null, // Default: "Http" + * user?: scalar|Param|null, // Default: null + * password?: scalar|Param|null, // Default: null + * }, + * index?: scalar|Param|null, // Default: "monolog" + * document_type?: scalar|Param|null, // Default: "logs" + * ignore_error?: scalar|Param|null, // Default: false + * redis?: Param|string|array{ + * id?: scalar|Param|null, + * host?: scalar|Param|null, + * password?: scalar|Param|null, // Default: null + * port?: scalar|Param|null, // Default: 6379 + * database?: scalar|Param|null, // Default: 0 + * key_name?: scalar|Param|null, // Default: "monolog_redis" + * }, + * predis?: Param|string|array{ + * id?: scalar|Param|null, + * host?: scalar|Param|null, + * }, + * from_email?: scalar|Param|null, + * to_email?: Param|string|list, + * subject?: scalar|Param|null, + * content_type?: scalar|Param|null, // Default: null + * headers?: list, + * mailer?: scalar|Param|null, // Default: null + * email_prototype?: Param|string|array{ + * id?: scalar|Param|null, + * method?: scalar|Param|null, // Default: null + * }, + * verbosity_levels?: array{ + * VERBOSITY_QUIET?: scalar|Param|null, // Default: "ERROR" + * VERBOSITY_NORMAL?: scalar|Param|null, // Default: "WARNING" + * VERBOSITY_VERBOSE?: scalar|Param|null, // Default: "NOTICE" + * VERBOSITY_VERY_VERBOSE?: scalar|Param|null, // Default: "INFO" + * VERBOSITY_DEBUG?: scalar|Param|null, // Default: "DEBUG" + * }, + * channels?: Param|string|array{ + * type?: scalar|Param|null, + * elements?: list, + * ... + * }, + * }>, + * } * @psalm-type ConfigType = array{ * imports?: ImportsConfig, * parameters?: ParametersConfig, @@ -808,6 +953,7 @@ * framework?: FrameworkConfig, * twig?: TwigConfig, * twig_extra?: TwigExtraConfig, + * monolog?: MonologConfig, * "when@dev"?: array{ * imports?: ImportsConfig, * parameters?: ParametersConfig, @@ -818,6 +964,7 @@ * twig_extra?: TwigExtraConfig, * web_profiler?: WebProfilerConfig, * debug?: DebugConfig, + * monolog?: MonologConfig, * }, * "when@prod"?: array{ * imports?: ImportsConfig, @@ -826,6 +973,7 @@ * framework?: FrameworkConfig, * twig?: TwigConfig, * twig_extra?: TwigExtraConfig, + * monolog?: MonologConfig, * }, * "when@test"?: array{ * imports?: ImportsConfig, @@ -835,6 +983,7 @@ * twig?: TwigConfig, * twig_extra?: TwigExtraConfig, * web_profiler?: WebProfilerConfig, + * monolog?: MonologConfig, * }, * ... Date: Wed, 9 Sep 2026 14:09:28 +0200 Subject: [PATCH 2/4] Refactored source import stuff --- .env | 10 +- src/Command/SourceImportCommand.php | 22 +++-- src/Ngsi/NgsiEntity.php | 2 +- src/Source/MtmSpatialMaps/HandicapParking.php | 37 ++----- src/Source/SourceInterface.php | 5 +- src/SourceImporterFactory.php | 35 +++++++ src/SourceImporterGeoJson.php | 97 +++++++++++++++++++ src/SourceImporterInterface.php | 18 ++++ src/SourceReaderGeoJson.php | 49 ++++++++++ src/SourceReaderInterface.php | 13 +++ .../MtmSpatialMaps/HandicapParkingTest.php | 2 +- 11 files changed, 246 insertions(+), 44 deletions(-) create mode 100644 src/SourceImporterFactory.php create mode 100644 src/SourceImporterGeoJson.php create mode 100644 src/SourceImporterInterface.php create mode 100644 src/SourceReaderGeoJson.php create mode 100644 src/SourceReaderInterface.php diff --git a/.env b/.env index 819e914..45fd128 100644 --- a/.env +++ b/.env @@ -33,8 +33,10 @@ DEFAULT_URI=http://localhost ###> app ### APP_BROKER_BASE_URI=http://scorpio.local:9090/ -# JSON-LD contexts attached to every entity, outermost last so the ETSI core -# context resolves the NGSI-LD terms and the domain context resolves the -# Smart Data Models ones. -ENTER_NGSI_CONTEXT_URLS='https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld' +# A (JSON) list of JSON-LD contexts attached to every entity, outermost last so the +# ETSI core context resolves the NGSI-LD terms and the domain context resolves +# the Smart Data Models ones. +APP_NGSI_CONTEXT_URLS='[ +"https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" +]' ###< app ### diff --git a/src/Command/SourceImportCommand.php b/src/Command/SourceImportCommand.php index cbbaba6..2ab733c 100644 --- a/src/Command/SourceImportCommand.php +++ b/src/Command/SourceImportCommand.php @@ -4,23 +4,33 @@ use App\Source\DataSourceReader; use App\Source\SourceInterface; +use App\SourceImporterFactory; use Symfony\Component\Console\Attribute\Argument; use Symfony\Component\Console\Attribute\AsCommand; +use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Style\SymfonyStyle; #[AsCommand( - name: 'app:source:read', + name: 'app:source:import', )] -class SourceReadCommand +class SourceImportCommand { public function __invoke(SymfonyStyle $io, DataSourceReader $reader, + SourceImporterFactory $factory, #[Argument] SourceInterface $source): int { - throw new \RuntimeException('Lazy programmer exception!'); - // $reader = $readerFactory->getReader($source); - // $reader->read($source); - // … + $importer = $factory->getSourceImporter($source); + $result = $importer->import($source); + + $io->success(sprintf( + 'Upserted %d entities into %s (HTTP %d).', + $result->count, + $result->brokerUrl, + $result->status + )); + + return Command::SUCCESS; } } diff --git a/src/Ngsi/NgsiEntity.php b/src/Ngsi/NgsiEntity.php index dbc2f79..97d781a 100644 --- a/src/Ngsi/NgsiEntity.php +++ b/src/Ngsi/NgsiEntity.php @@ -69,7 +69,7 @@ public function relationship(string $name, string $object): self * * @return array */ - public function toArray(array $contextUrls): array + public function toPayload(array $contextUrls): array { return [ 'id' => $this->id, diff --git a/src/Source/MtmSpatialMaps/HandicapParking.php b/src/Source/MtmSpatialMaps/HandicapParking.php index 3ea503c..1ccf26a 100644 --- a/src/Source/MtmSpatialMaps/HandicapParking.php +++ b/src/Source/MtmSpatialMaps/HandicapParking.php @@ -7,21 +7,14 @@ use App\Geo\Wgs84Transformer; use App\Ngsi\NgsiEntity; use App\Source\AbstractSource; -use App\Source\DataSourceReader; -use App\Source\Manifest\Catalog; -use App\Source\Manifest\Descriptor; /** * Disabled parking bays in Aarhus Municipality. */ final readonly class HandicapParking extends AbstractSource { - private const string KEY = 'mtm_spatialmaps-handicap-parking'; - public function __construct( - private DataSourceReader $reader, private Wgs84Transformer $transformer, - private Catalog $catalog, ) { parent::__construct( id: 'mtm_spatialmaps-handicap-parking', @@ -48,29 +41,15 @@ public function __construct( ); } - public function entities(): iterable - { - $source = $this->catalog->get(self::KEY); - - // The export is a GeoJSON FeatureCollection, so the records live under - // `features`. Iterating the document itself would walk its two - // top-level keys instead. - foreach ($this->reader->read($source->accessUrl)['features'] ?? [] as $feature) { - if (\is_array($feature) && null !== $entity = $this->toEntity($feature, $source)) { - yield $entity; - } - } - } - /** * Maps one feed record onto an NgsiEntity. * - * @param array $feature GeoJSON Feature + * @param array $data GeoJSON Feature */ - private function toEntity(array $feature, Descriptor $source): ?NgsiEntity + public function createNgsiEntity(array $data): ?NgsiEntity { - $row = $feature['properties'] ?? null; - $geometry = $feature['geometry'] ?? null; + $row = $data['properties'] ?? null; + $geometry = $data['geometry'] ?? null; if (!\is_array($row) || !\is_array($geometry)) { return null; @@ -85,8 +64,8 @@ private function toEntity(array $feature, Descriptor $source): ?NgsiEntity } $entity = new NgsiEntity( - \sprintf('urn:ngsi-ld:%s:aarhus-handicap-%s', $source->model, $key), - $source->model + \sprintf('urn:ngsi-ld:%s:aarhus-handicap-%s', $this->model, $key), + $this->model ); return $entity @@ -94,8 +73,8 @@ private function toEntity(array $feature, Descriptor $source): ?NgsiEntity ->setProperty('description', trim((string) ($row['bemrk'] ?? ''))) ->setProperty('category', ['forDisabled']) ->setProperty('totalSpotNumber', (int) ($row['invalidepladser'] ?? 0)) - ->setProperty('source', $source->accessUrl) - ->geoProperty('location', $this->transformer->transformGeometry($source->crs, $geometry)); + ->setProperty('source', $this->accessUrl) + ->geoProperty('location', $this->transformer->transformGeometry($this->crs, $geometry)); } /** diff --git a/src/Source/SourceInterface.php b/src/Source/SourceInterface.php index 1d6d68d..e010081 100644 --- a/src/Source/SourceInterface.php +++ b/src/Source/SourceInterface.php @@ -78,10 +78,9 @@ interface SourceInterface extends \Stringable, \JsonSerializable } /** - * @return iterable + * @param array $data */ - // We should let the (data) source reader read. - public function entities(): iterable; + public function createNgsiEntity(array $data): ?NgsiEntity; /** * @return array diff --git a/src/SourceImporterFactory.php b/src/SourceImporterFactory.php new file mode 100644 index 0000000..aee58ea --- /dev/null +++ b/src/SourceImporterFactory.php @@ -0,0 +1,35 @@ + $importers + */ + public function __construct( + #[AutowireIterator('app.source_importer')] + private iterable $importers, + ) { + } + + public function getSourceImporter(SourceInterface $source): SourceImporterInterface + { + foreach ($this->importers as $importer) { + if ($importer->supports($source)) { + return $importer; + } + } + + // @todo + throw new \RuntimeException('Cannot get importer for source'); + } +} diff --git a/src/SourceImporterGeoJson.php b/src/SourceImporterGeoJson.php new file mode 100644 index 0000000..8acd977 --- /dev/null +++ b/src/SourceImporterGeoJson.php @@ -0,0 +1,97 @@ + $contextUrls + */ + public function __construct( + private readonly SourceReaderGeoJson $reader, + private readonly NgsiLdBroker $broker, + #[Autowire(env: 'json:APP_NGSI_CONTEXT_URLS')] + private readonly array $contextUrls, + LoggerInterface $logger, + ) { + $this->setLogger($logger); + } + + public function supports(SourceInterface $source): bool + { + // @todo + return true; + } + + public function import(SourceInterface $source): ImportResult + { + $this->info('Processing source {source}', ['source' => $source->__toString()]); + + $contextUrls = array_merge([$source->contextUrl], $this->contextUrls); + $payload = []; + foreach ($this->read($source) as $entity) { + $this->info('Building payload for {entity}', ['entity' => $entity->id()]); + $payload[] = $entity->toPayload($contextUrls); + } + + if (1 === count($payload)) { + $this->info('Upserting 1 entity'); + } else { + $this->info('Upserting {count} entities', ['count' => count($payload)]); + } + + try { + $status = $this->broker->upsert($payload); + } catch (\Throwable $exception) { + throw new UpsertFailedException($exception); + } + + return new ImportResult(\count($payload), $status, $this->broker->brokerUrl()); + } + + /** + * @return iterable + */ + private function read(SourceInterface $source): iterable + { + $data = $this->reader->read($source); + foreach ($data as $item) { + if (is_array($item)) { + try { + if ($entity = $source->createNgsiEntity($item)) { + yield $entity; + } + } catch (\Exception $e) { + // @todo Log in database? + $this->error('error: {message} ', ['message' => $e->getMessage()]); + } + } + } + } + + /** + * @param array $context + */ + public function log($level, \Stringable|string $message, array $context = []): void + { + $this->logger->log($level, $message, $context); + } +} diff --git a/src/SourceImporterInterface.php b/src/SourceImporterInterface.php new file mode 100644 index 0000000..65391fd --- /dev/null +++ b/src/SourceImporterInterface.php @@ -0,0 +1,18 @@ +getData($source->accessUrl); + + $features = $data['features']; + + if (!is_array($features) || !array_is_list($features)) { + throw new \RuntimeException('Invalid features array'); + } + + return $features; + } + + /** + * @return array + */ + private function getData(string $url): iterable + { + return $this->client->request('GET', $url)->toArray(); + } + + /** + * @param array $context + */ + public function log($level, \Stringable|string $message, array $context = []): void + { + $this->logger->log($level, $message, $context); + } +} diff --git a/src/SourceReaderInterface.php b/src/SourceReaderInterface.php new file mode 100644 index 0000000..311f398 --- /dev/null +++ b/src/SourceReaderInterface.php @@ -0,0 +1,13 @@ + + */ + public function read(SourceInterface $source): iterable; +} diff --git a/tests/Source/MtmSpatialMaps/HandicapParkingTest.php b/tests/Source/MtmSpatialMaps/HandicapParkingTest.php index 6517824..ff6b8f9 100644 --- a/tests/Source/MtmSpatialMaps/HandicapParkingTest.php +++ b/tests/Source/MtmSpatialMaps/HandicapParkingTest.php @@ -43,7 +43,7 @@ protected function setUp(): void $source = new HandicapParking(new DataSourceReader($client), new Wgs84Transformer(), $catalog); $this->entities = array_map( - static fn (NgsiEntity $entity): array => $entity->toArray(['https://example.com/context.jsonld']), + static fn (NgsiEntity $entity): array => $entity->toPayload(['https://example.com/context.jsonld']), iterator_to_array($source->entities(), false) ); } From 3644286d578822505a9d605d71b0b966a83ae6d2 Mon Sep 17 00:00:00 2001 From: Mikkel Ricky Date: Wed, 9 Sep 2026 14:36:54 +0200 Subject: [PATCH 3/4] Cleaned up --- phpstan.dist.neon | 6 +- src/Command/BrokerEntityDelete.php | 20 -- src/Command/BrokerImportGeoJson.php | 93 ------- src/Command/ImportCommand.php | 128 --------- src/Command/SourceImportCommand.php | 2 - src/Import/DataSourceImporter.php | 136 --------- src/Source/DataSourceReader.php | 58 ---- src/Source/Manifest/Catalog.php | 114 -------- src/Source/Manifest/Descriptor.php | 40 --- src/Source/Manifest/Schema.php | 115 -------- src/Source/Manifest/Validator.php | 34 --- tests/Command/ImportCommandTest.php | 149 ---------- tests/Import/DataSourceImporterTest.php | 259 ------------------ tests/Source/DataSourceReaderTest.php | 114 -------- tests/Source/FakeSource.php | 61 ----- tests/Source/Manifest/CatalogTest.php | 238 ---------------- tests/Source/Manifest/ValidatorTest.php | 90 ------ tests/Source/Manifest/WritesManifests.php | 61 ----- .../MtmSpatialMaps/HandicapParkingTest.php | 190 ------------- 19 files changed, 2 insertions(+), 1906 deletions(-) delete mode 100644 src/Command/BrokerImportGeoJson.php delete mode 100644 src/Command/ImportCommand.php delete mode 100644 src/Import/DataSourceImporter.php delete mode 100644 src/Source/DataSourceReader.php delete mode 100644 src/Source/Manifest/Catalog.php delete mode 100644 src/Source/Manifest/Descriptor.php delete mode 100644 src/Source/Manifest/Schema.php delete mode 100644 src/Source/Manifest/Validator.php delete mode 100644 tests/Command/ImportCommandTest.php delete mode 100644 tests/Import/DataSourceImporterTest.php delete mode 100644 tests/Source/DataSourceReaderTest.php delete mode 100644 tests/Source/FakeSource.php delete mode 100644 tests/Source/Manifest/CatalogTest.php delete mode 100644 tests/Source/Manifest/ValidatorTest.php delete mode 100644 tests/Source/Manifest/WritesManifests.php delete mode 100644 tests/Source/MtmSpatialMaps/HandicapParkingTest.php diff --git a/phpstan.dist.neon b/phpstan.dist.neon index afeb348..0fbcd82 100644 --- a/phpstan.dist.neon +++ b/phpstan.dist.neon @@ -5,17 +5,15 @@ parameters: - config/ - public/ - src/ - # - tests/ + - tests/ excludePaths: - src/Kernel.php + - tests/bootstrap.php treatPhpDocTypesAsCertain: false ignoreErrors: - - message: '#Access to protected property proj4php\\Point::\$(x|y).#' - path: src/Command/BrokerImportGeoJson.php - - messages: - "#contains generic class RecursiveArrayIterator but does not specify its types#" - '#Call to an undefined method RecursiveIterator::getArrayCopy\(\).#' diff --git a/src/Command/BrokerEntityDelete.php b/src/Command/BrokerEntityDelete.php index d1da87d..3e585cf 100644 --- a/src/Command/BrokerEntityDelete.php +++ b/src/Command/BrokerEntityDelete.php @@ -9,7 +9,6 @@ use Symfony\Component\Console\Style\SymfonyStyle; use Symfony\Component\HttpFoundation\Request; use Symfony\Contracts\HttpClient\HttpClientInterface; -use Symfony\Contracts\HttpClient\ResponseInterface; #[AsCommand( name: 'app:broker:entity:delete', @@ -73,23 +72,4 @@ public function __invoke( return Command::SUCCESS; } - - /** - * Get link URL. - * - * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link - */ - private function getLinkUrl(ResponseInterface $response, string $rel): ?string - { - $links = $response->getHeaders()['link'] ?? []; - foreach ($links as $link) { - if (preg_match_all('/<(?P[^>]+)>;\s*rel="(?P[^"]+)"/', (string) $link, $matches, PREG_SET_ORDER)) { - if ($rel === ($matches[0]['rel'] ?? null)) { - return $matches[0]['url'] ?? null; - } - } - } - - return null; - } } diff --git a/src/Command/BrokerImportGeoJson.php b/src/Command/BrokerImportGeoJson.php deleted file mode 100644 index da2839d..0000000 --- a/src/Command/BrokerImportGeoJson.php +++ /dev/null @@ -1,93 +0,0 @@ -request(Request::METHOD_GET, $dataUrl)->toArray(); - $features = $data['features'] ?? null; - $sourceProjection = $data['crs']['properties']['name'] ?? null; - $sourceProjection = null !== $sourceProjection - ? Crs::from($sourceProjection) - : Crs::geoJson(); - - $count = count($features); - if ($count > 0) { - $io->writeln(match ($count) { - 1 => sprintf('Creating one %s entity …', $entityType), - default => sprintf('Creating %d %s entities …', $count, $entityType), - }); - $io->progressStart($count); - foreach ($features as $index => $feature) { - $entity = [ - 'type' => $entityType, - 'id' => sprintf('%s:%04d', $entityType, $index), - ]; - foreach ($feature['properties'] as $name => $value) { - if (null === $value) { - continue; - } - if (in_array($name, ['id', 'type'])) { - $name = self::RESERVED_PROPERTY_NAME_PREFIX.$name; - } - $entity[$name] = [ - 'type' => 'Property', - 'value' => $value, - ]; - } - $geometry = $geometryHelper->transformGeoJsonGeometry($feature['geometry'], from: $sourceProjection); - $entity['geometry'] = $geometry; - $centroid = $geometryHelper->getCentroid($geometry); - // Compute location from geometry. The location must be a single point (the "spatial location" of the entity). - $entity['location'] = [ - 'type' => 'geo:json', - 'value' => [ - 'type' => 'Point', - 'coordinates' => [$centroid->x, $centroid->y], - ], - ]; - $response = $brokerClient->request(Request::METHOD_POST, '/ngsi-ld/v1/entities', [ - 'json' => $entity, - ]); - - if (Response::HTTP_CREATED !== $response->getStatusCode()) { - $io->error($response->getContent(false)); - $io->writeln(json_encode($entity, JSON_PRETTY_PRINT)); - - return Command::FAILURE; - } - $io->progressAdvance(); - } - $io->progressFinish(); - } - - return Command::SUCCESS; - } -} diff --git a/src/Command/ImportCommand.php b/src/Command/ImportCommand.php deleted file mode 100644 index e1c4fa9..0000000 --- a/src/Command/ImportCommand.php +++ /dev/null @@ -1,128 +0,0 @@ -addArgument('source', InputArgument::OPTIONAL, 'Source to import. Omit to list the available sources.') - ->addOption('dry-run', null, InputOption::VALUE_NONE, 'Print the NGSI-LD payload instead of sending it.') - ->addOption('limit', 'l', InputOption::VALUE_REQUIRED, 'Import at most this many entities.'); - } - - protected function execute(InputInterface $input, OutputInterface $output): int - { - $io = new SymfonyStyle($input, $output); - - $keys = $this->importer->keys(); - - if ([] === $keys) { - $io->error('No data sources are registered.'); - $io->listing([ - 'A source must implement App\Source\SourceInterface.', - 'Implementations are picked up automatically — check the class exists and is not excluded from the container.', - ]); - - return Command::FAILURE; - } - - $argument = $input->getArgument('source'); - - if (null === $argument) { - // Options only make sense together with a source. Listing the - // sources and exiting successfully would look like an import ran. - if ($input->getOption('dry-run') || null !== $input->getOption('limit')) { - $io->error(\sprintf( - 'No source given. Available: %s.', - implode(', ', $keys) - )); - - return Command::INVALID; - } - - $io->section('Available sources'); - $io->listing($keys); - - return Command::SUCCESS; - } - - $key = (string) $argument; - $limit = null !== $input->getOption('limit') ? (int) $input->getOption('limit') : null; - - try { - if ($input->getOption('dry-run')) { - $payload = $this->importer->payload($key, $limit); - - $output->writeln(json_encode($payload, self::JSON_FLAGS)); - $io->note(\sprintf('Dry run: %d entities were not sent.', \count($payload))); - - return Command::SUCCESS; - } - - $result = $this->importer->import($key, $limit); - } catch (UnknownSourceException $exception) { - $io->error($exception->getMessage()); - - return Command::INVALID; - } catch (EmptySourceException $exception) { - $io->error($exception->getMessage()); - $io->text( - 'The source ran to completion without raising an exception, so every record was ' - .'discarded by the source\'s own guards rather than failing. Verbosity flags will ' - .'not reveal more: there is no exception to show.' - ); - $io->listing([ - 'Does the configured path or URL point at the intended document?', - 'Does the document match the shape the source expects — envelope, nesting, field names?', - 'Which guard returns early — a missing identifier, or a missing geometry?', - ]); - - return Command::FAILURE; - } catch (UpsertFailedException $exception) { - $io->error($exception->getMessage()); - - return Command::FAILURE; - } - - $io->success(\sprintf( - 'Upserted %d entities into %s (HTTP %d).', - $result->count, - $result->brokerUrl, - $result->status - )); - - return Command::SUCCESS; - } -} diff --git a/src/Command/SourceImportCommand.php b/src/Command/SourceImportCommand.php index 2ab733c..8f5e561 100644 --- a/src/Command/SourceImportCommand.php +++ b/src/Command/SourceImportCommand.php @@ -2,7 +2,6 @@ namespace App\Command; -use App\Source\DataSourceReader; use App\Source\SourceInterface; use App\SourceImporterFactory; use Symfony\Component\Console\Attribute\Argument; @@ -16,7 +15,6 @@ class SourceImportCommand { public function __invoke(SymfonyStyle $io, - DataSourceReader $reader, SourceImporterFactory $factory, #[Argument] SourceInterface $source): int diff --git a/src/Import/DataSourceImporter.php b/src/Import/DataSourceImporter.php deleted file mode 100644 index 597646b..0000000 --- a/src/Import/DataSourceImporter.php +++ /dev/null @@ -1,136 +0,0 @@ - $sources - */ - public function __construct( - #[AutowireIterator('app.source')] - private iterable $sources, - private Catalog $catalog, - private NgsiLdBroker $broker, - #[Autowire(env: 'ENTER_NGSI_CONTEXT_URLS')] - private string $contextUrls, - ) { - } - - /** - * @return list every registered source key, in registration order - */ - public function keys(): array - { - return array_keys($this->registry()); - } - - /** - * Converts a source to NGSI-LD. - * - * @return non-empty-list> - * - * @throws UnknownSourceException when no source is registered under the key - * @throws EmptySourceException when the source yields no entities - */ - public function payload(string $key, ?int $limit = null): array - { - // Collect all dataset keys. - $registry = $this->registry(); - - // Check if requested dataset key exists. - if (!isset($registry[$key])) { - throw new UnknownSourceException($key, array_keys($registry)); - } - - // Define minimum limit, in case of limit defined as less than 1. - $limit = null === $limit ? null : max(1, $limit); - - // Load context for given dataset. - $contexts = $this->contexts($key); - - $payload = []; - foreach ($registry[$key]->entities() as $entity) { - $payload[] = $entity->toArray($contexts); - - // Break upon limit. - if (null !== $limit && \count($payload) >= $limit) { - break; - } - } - - if ([] === $payload) { - throw new EmptySourceException($key); - } - - return $payload; - } - - /** - * @throws UnknownSourceException when no source is registered under the key - * @throws EmptySourceException when the source yields no entities - * @throws UpsertFailedException when the broker cannot be written to - */ - public function import(string $key, ?int $limit = null): ImportResult - { - // Get payload from dataset. - $payload = $this->payload($key, $limit); - - // Try to upsert broker with payload. - try { - $status = $this->broker->upsert($payload); - } catch (\Throwable $exception) { - throw new UpsertFailedException($exception); - } - - // Return result. - return new ImportResult(\count($payload), $status, $this->broker->brokerUrl()); - } - - /** - * Get list of registered datasets. - * - * @see config/sources.yaml - * - * @return array keyed by source key - */ - private function registry(): array - { - $registry = []; - - foreach ($this->sources as $source) { - $registry[$source->id] = $source; - } - - return $registry; - } - - /** - * Return an array of contexts. Each dataset holds its own context. - * - * @see config/sources.yaml - * - * @return array - */ - private function contexts(string $key): array - { - return [ - $this->catalog->get($key)->contextUrl, - ...array_values(array_filter(array_map(trim(...), explode(',', $this->contextUrls)))), - ]; - } -} diff --git a/src/Source/DataSourceReader.php b/src/Source/DataSourceReader.php deleted file mode 100644 index c8f3751..0000000 --- a/src/Source/DataSourceReader.php +++ /dev/null @@ -1,58 +0,0 @@ - the decoded document - * - * @throws \RuntimeException when the location is not an http(s) URL, cannot - * be fetched, does not contain valid JSON, or - * does not decode to an array - */ - public function read(string $url): array - { - if (!str_starts_with($url, 'http://') && !str_starts_with($url, 'https://')) { - throw new \RuntimeException(\sprintf('Feed location must be an http(s) URL, got "%s".', $url)); - } - - $json = $this->fetch($url); - - try { - $decoded = json_decode($json, true, 512, \JSON_THROW_ON_ERROR); - } catch (\JsonException $exception) { - throw new \RuntimeException(\sprintf('Invalid JSON in "%s": %s', $url, $exception->getMessage()), previous: $exception); - } - - // A JSON document may legally be a scalar. Every feed we consume is a - // list or an object, and a scalar here means the location is wrong - // rather than that the feed is empty. - if (!\is_array($decoded)) { - throw new \RuntimeException(\sprintf('Expected a JSON array or object in "%s", got %s.', $url, get_debug_type($decoded))); - } - - return $decoded; - } - - private function fetch(string $url): string - { - try { - return $this->client->request('GET', $url)->getContent(); - } catch (\Throwable $exception) { - throw new \RuntimeException(\sprintf('Could not fetch "%s": %s', $url, $exception->getMessage()), previous: $exception); - } - } -} diff --git a/src/Source/Manifest/Catalog.php b/src/Source/Manifest/Catalog.php deleted file mode 100644 index 71ad47f..0000000 --- a/src/Source/Manifest/Catalog.php +++ /dev/null @@ -1,114 +0,0 @@ -|null */ - private ?array $descriptors = null; - - public function __construct( - #[Autowire('%kernel.project_dir%/config/sources.yaml')] - private readonly string $manifest, - ) { - } - - /** - * @throws \RuntimeException when the manifest cannot be read, or carries no entry for the key - */ - public function get(string $key): Descriptor - { - $descriptors = $this->all(); - - if (!isset($descriptors[$key])) { - throw new \RuntimeException(\sprintf('No entry for source "%s" in %s. Entries: %s.', $key, $this->manifest, [] === $descriptors ? 'none' : implode(', ', array_keys($descriptors)))); - } - - return $descriptors[$key]; - } - - /** - * @return array keyed by source key - * - * @throws \RuntimeException when the manifest cannot be read - */ - public function all(): array - { - return $this->descriptors ??= $this->load(); - } - - /** - * @return array - */ - private function load(): array - { - $descriptors = []; - - foreach ($this->validated() as $key => $entry) { - $descriptors[$key] = new Descriptor( - key: $key, - title: $entry['title'], - accessUrl: $entry['access_url'], - crs: $entry['crs'], - model: $entry['model'], - contextUrl: $entry['context_url'], - description: $entry['description'], - publisher: $entry['publisher'], - contact: $entry['contact'], - landingPage: $entry['landing_page'], - mediaType: $entry['media_type'], - updateFrequency: $entry['update_frequency'], - licence: $entry['licence'], - omittedFields: $entry['omitted_fields'], - ); - } - - return $descriptors; - } - - /** - * @return array}> - */ - private function validated(): array - { - if (!is_file($this->manifest)) { - throw new \RuntimeException(\sprintf('Source manifest "%s" does not exist.', $this->manifest)); - } - - try { - $parsed = Yaml::parseFile($this->manifest); - } catch (ParseException $exception) { - throw new \RuntimeException(\sprintf('Source manifest "%s" is not valid YAML: %s', $this->manifest, $exception->getMessage()), previous: $exception); - } - - $sources = \is_array($parsed) ? $parsed['sources'] ?? null : null; - if (!\is_array($sources)) { - throw new \RuntimeException(\sprintf('Source manifest "%s" must contain a "sources" mapping at the top level.', $this->manifest)); - } - - try { - /** @var array}> $processed */ - $processed = new Processor()->process(Schema::tree(), [$sources]); - } catch (InvalidConfigurationException $exception) { - throw new \RuntimeException(\sprintf('Source manifest "%s" is invalid: %s', $this->manifest, $exception->getMessage()), previous: $exception); - } - - return $processed; - } -} diff --git a/src/Source/Manifest/Descriptor.php b/src/Source/Manifest/Descriptor.php deleted file mode 100644 index 46d0631..0000000 --- a/src/Source/Manifest/Descriptor.php +++ /dev/null @@ -1,40 +0,0 @@ - $omittedFields source field name => why it is not published - */ - public function __construct( - public string $key, - public string $title, - public string $accessUrl, - public string $crs, - public string $model, - public string $contextUrl, - public ?string $description = null, - public ?string $publisher = null, - public ?string $contact = null, - public ?string $landingPage = null, - public ?string $mediaType = null, - public ?string $updateFrequency = null, - public ?string $licence = null, - public array $omittedFields = [], - ) { - } -} diff --git a/src/Source/Manifest/Schema.php b/src/Source/Manifest/Schema.php deleted file mode 100644 index bbaf41c..0000000 --- a/src/Source/Manifest/Schema.php +++ /dev/null @@ -1,115 +0,0 @@ -getRootNode() - // The import selects a data set by its key exactly as written in - // the manifest. Key normalization rewrites a key that contains - // dashes and no underscore, which breaks that lookup silently. - ->normalizeKeys(false) - ->requiresAtLeastOneElement() - ->arrayPrototype() - ->beforeNormalization() - ->always(self::blankToNull()) - ->end() - ->children() - ->scalarNode('title') - ->isRequired() - ->cannotBeEmpty() - ->info('What the data set is called where it is published.') - ->end() - ->scalarNode('access_url') - ->isRequired() - ->cannotBeEmpty() - ->info('Where the feed is read from; an import cannot run without it.') - ->end() - ->scalarNode('crs') - ->isRequired() - ->cannotBeEmpty() - ->info('The CRS the feed publishes coordinates in, e.g. "EPSG:25832". A wrong value yields well-formed coordinates in the wrong place.') - ->end() - ->scalarNode('model') - ->isRequired() - ->cannotBeEmpty() - ->info('Smart Data Model the data set is published as.') - ->end() - ->scalarNode('context_url') - ->isRequired() - ->cannotBeEmpty() - ->info('JSON-LD context defining the model\'s terms. Without the right one a consumer reads the entity\'s attributes as undefined strings.') - ->end() - ->scalarNode('description')->defaultNull()->end() - ->scalarNode('publisher')->defaultNull()->end() - ->scalarNode('contact')->defaultNull()->end() - ->scalarNode('landing_page')->defaultNull()->end() - ->scalarNode('media_type')->defaultNull()->end() - ->scalarNode('update_frequency')->defaultNull()->end() - ->scalarNode('licence') - ->defaultNull() - ->info('Empty means the terms are unsettled; DCAT-AP requires one before the data set can be registered.') - ->end() - ->arrayNode('omitted_fields') - ->defaultValue([]) - ->normalizeKeys(false) - ->scalarPrototype() - ->beforeNormalization() - ->always(self::blankToNullValue()) - ->end() - ->cannotBeEmpty() - ->info('The reason a field is withheld cannot be recovered from the code, so a bare list of names is not accepted.') - ->end() - ->end() - ->end() - ->end(); - - return $tree->buildTree(); - } - - /** - * An empty value means "not filled in", the same as an absent key, so both - * become null rather than an empty string. - * - * Normalization runs before the type check, so an entry that is not a - * mapping has to pass through untouched for the tree to report it as one. - */ - private static function blankToNull(): \Closure - { - return static function (mixed $entry): mixed { - if (!\is_array($entry)) { - return $entry; - } - - return array_map(self::blankToNullValue(), $entry); - }; - } - - private static function blankToNullValue(): \Closure - { - return static function (mixed $value): mixed { - if (!\is_string($value)) { - return $value; - } - - $value = trim($value); - - return '' === $value ? null : $value; - }; - } -} diff --git a/src/Source/Manifest/Validator.php b/src/Source/Manifest/Validator.php deleted file mode 100644 index 01d54dd..0000000 --- a/src/Source/Manifest/Validator.php +++ /dev/null @@ -1,34 +0,0 @@ - nothing is written, so nothing is preloaded - */ - public function warmUp(string $cacheDir, ?string $buildDir = null): array - { - $this->catalog->all(); - - return []; - } -} diff --git a/tests/Command/ImportCommandTest.php b/tests/Command/ImportCommandTest.php deleted file mode 100644 index 828e17f..0000000 --- a/tests/Command/ImportCommandTest.php +++ /dev/null @@ -1,149 +0,0 @@ - $sources - */ - private function tester(iterable $sources, ?MockHttpClient $client = null): CommandTester - { - $keys = []; - foreach ($sources as $source) { - $keys[] = $source->id; - } - - return new CommandTester(new ImportCommand(new DataSourceImporter( - $sources, - new Catalog($this->manifestFor($keys)), - new NgsiLdBroker($client ?? new MockHttpClient(), 'http://broker.invalid'), - 'https://example.com/core.jsonld', - ))); - } - - public function testItListsTheSourcesWhenCalledBare(): void - { - $tester = $this->tester([FakeSource::withEntities('some-source')]); - - $status = $tester->execute([]); - - $this->assertSame(Command::SUCCESS, $status); - $this->assertStringContainsString('some-source', $tester->getDisplay()); - } - - public function testItFailsWhenNoSourcesAreRegistered(): void - { - $tester = $this->tester([]); - - $status = $tester->execute([]); - - $this->assertSame(Command::FAILURE, $status); - $this->assertStringContainsString('No data sources are registered', $tester->getDisplay()); - } - - /** - * Passing --dry-run with no source used to print the source listing and - * exit successfully, silently ignoring the flag. - */ - public function testItRejectsOptionsWithoutASource(): void - { - $tester = $this->tester([FakeSource::withEntities('some-source')]); - - $status = $tester->execute(['--dry-run' => true]); - - $this->assertSame(Command::INVALID, $status); - $this->assertStringContainsString('No source given', $tester->getDisplay()); - } - - public function testAnUnknownSourceIsTheCallersMistake(): void - { - $tester = $this->tester([FakeSource::withEntities('some-source')]); - - $status = $tester->execute(['source' => 'nope']); - - $this->assertSame(Command::INVALID, $status); - $this->assertStringContainsString('Unknown source "nope"', $tester->getDisplay()); - } - - /** - * The failure carries no exception to show, so the command has to supply - * the places worth looking itself. - */ - public function testAnEmptySourceFailsAndSuggestsCauses(): void - { - $tester = $this->tester([new FakeSource('empty-source')]); - - $status = $tester->execute(['source' => 'empty-source']); - $display = $tester->getDisplay(); - - $this->assertSame(Command::FAILURE, $status); - $this->assertStringContainsString('produced no entities', $display); - $this->assertStringContainsString('path or URL', $display); - $this->assertStringContainsString('envelope, nesting, field names', $display); - } - - public function testDryRunPrintsThePayloadAndSendsNothing(): void - { - $client = new MockHttpClient(); - $tester = $this->tester([FakeSource::withEntities('one-entity', 'urn:ngsi-ld:Example:1')], $client); - - $status = $tester->execute(['source' => 'one-entity', '--dry-run' => true]); - $display = $tester->getDisplay(); - - $this->assertSame(Command::SUCCESS, $status); - $this->assertStringContainsString('"urn:ngsi-ld:Example:1"', $display); - $this->assertStringContainsString('1 entities were not sent', $display); - $this->assertSame(0, $client->getRequestsCount()); - } - - public function testItReportsWhatWasUpserted(): void - { - $client = new MockHttpClient(new MockResponse('', ['http_code' => 204])); - $tester = $this->tester([FakeSource::withEntities('one-entity', 'urn:ngsi-ld:Example:1')], $client); - - $status = $tester->execute(['source' => 'one-entity']); - $display = $tester->getDisplay(); - - $this->assertSame(Command::SUCCESS, $status); - $this->assertStringContainsString('Upserted 1 entities', $display); - $this->assertStringContainsString('HTTP 204', $display); - } - - /** - * The broker being down is an operational condition rather than a bug, so - * it is reported as a message instead of an uncaught exception. - */ - public function testABrokerFailureIsReportedAsAnError(): void - { - $client = new MockHttpClient(new MockResponse('', ['http_code' => 500])); - $tester = $this->tester([FakeSource::withEntities('one-entity', 'urn:ngsi-ld:Example:1')], $client); - - $status = $tester->execute(['source' => 'one-entity']); - - $this->assertSame(Command::FAILURE, $status); - $this->assertStringContainsString('HTTP 500', $tester->getDisplay()); - } -} diff --git a/tests/Import/DataSourceImporterTest.php b/tests/Import/DataSourceImporterTest.php deleted file mode 100644 index 6686577..0000000 --- a/tests/Import/DataSourceImporterTest.php +++ /dev/null @@ -1,259 +0,0 @@ - $sources each is registered in the manifest under its own key - */ - private function importer( - iterable $sources, - ?MockHttpClient $client = null, - string $contextUrls = self::CORE_CONTEXT, - ): DataSourceImporter { - $keys = []; - foreach ($sources as $source) { - $keys[] = $source->id; - } - - return new DataSourceImporter( - $sources, - new Catalog($this->manifestFor($keys)), - new NgsiLdBroker($client ?? new MockHttpClient(), self::BROKER_URL), - $contextUrls, - ); - } - - public function testItListsTheRegisteredSourceKeys(): void - { - $importer = $this->importer([FakeSource::withEntities('a-source'), FakeSource::withEntities('b-source')]); - - $this->assertSame(['a-source', 'b-source'], $importer->keys()); - } - - public function testItListsNothingWhenNoSourceIsRegistered(): void - { - $this->assertSame([], $this->importer([])->keys()); - } - - public function testItRejectsAnUnknownSourceAndNamesTheKnownOnes(): void - { - $importer = $this->importer([FakeSource::withEntities('some-source')]); - - $this->expectException(UnknownSourceException::class); - $this->expectExceptionMessage('Unknown source "nope". Available: some-source.'); - - $importer->payload('nope'); - } - - /** - * The important one: a source yielding nothing used to be reported as a - * completed import, which is indistinguishable from a working one. - */ - public function testItFailsWhenASourceProducesNothing(): void - { - $importer = $this->importer([new FakeSource('empty-source')]); - - $this->expectException(EmptySourceException::class); - $this->expectExceptionMessage('Source "empty-source" produced no entities.'); - - $importer->payload('empty-source'); - } - - /** - * The guard belongs to the conversion rather than to the dry run, so an - * import that intends to send is held to it too. - */ - public function testItSendsNothingWhenASourceProducesNothing(): void - { - $client = new MockHttpClient(); - - try { - $this->importer([new FakeSource('empty-source')], $client)->import('empty-source'); - $this->fail('An empty source was reported as a completed import.'); - } catch (EmptySourceException) { - $this->assertSame(0, $client->getRequestsCount(), 'An empty payload was sent to the broker.'); - } - } - - /** - * Later entries win term conflicts, so the data set's own context comes - * first and the core context last, where it stays authoritative over the - * NGSI-LD terms a domain context may also define. - */ - public function testEveryEntityCarriesItsDataSetContextThenTheCoreContext(): void - { - $importer = $this->importer([FakeSource::withEntities('one-entity', 'urn:ngsi-ld:Example:1')]); - - $payload = $importer->payload('one-entity'); - - $this->assertSame( - [self::dataSetContext('one-entity'), self::CORE_CONTEXT], - $payload[0]['@context'] - ); - } - - /** - * Two data sets published under different models must not advertise each - * other's vocabulary, which a single app-wide list cannot avoid. - */ - public function testEachDataSetCarriesOnlyItsOwnContext(): void - { - $importer = $this->importer([ - FakeSource::withEntities('first-source', 'urn:ngsi-ld:Example:1'), - FakeSource::withEntities('second-source', 'urn:ngsi-ld:Example:2'), - ]); - - $this->assertSame( - [self::dataSetContext('first-source'), self::CORE_CONTEXT], - $importer->payload('first-source')[0]['@context'] - ); - $this->assertSame( - [self::dataSetContext('second-source'), self::CORE_CONTEXT], - $importer->payload('second-source')[0]['@context'] - ); - } - - /** - * A source registered in the container but not in the manifest has no - * context to publish under, so it fails rather than emitting entities a - * consumer cannot resolve. - */ - public function testItFailsWhenASourceHasNoManifestEntry(): void - { - $importer = new DataSourceImporter( - [FakeSource::withEntities('unregistered', 'urn:ngsi-ld:Example:1')], - new Catalog($this->manifestFor(['other-source'])), - new NgsiLdBroker(new MockHttpClient(), self::BROKER_URL), - self::CORE_CONTEXT, - ); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('No entry for source "unregistered"'); - - $importer->payload('unregistered'); - } - - /** - * The contexts arrive as one comma-separated environment variable, so they - * are written by hand and carry whatever spacing that produces. - */ - public function testItIgnoresSpacingAndEmptyEntriesInTheConfiguredContexts(): void - { - $importer = $this->importer( - [FakeSource::withEntities('one-entity', 'urn:ngsi-ld:Example:1')], - contextUrls: ' '.self::CORE_CONTEXT.' , ,', - ); - - $payload = $importer->payload('one-entity'); - - $this->assertSame( - [self::dataSetContext('one-entity'), self::CORE_CONTEXT], - $payload[0]['@context'] - ); - } - - public function testALimitCapsThePayload(): void - { - $importer = $this->importer([FakeSource::withEntities('many', 'urn:1', 'urn:2', 'urn:3', 'urn:4', 'urn:5')]); - - $this->assertCount(2, $importer->payload('many', 2)); - } - - /** - * A real source reads a whole feed, so a limit that converts everything - * and then trims would do all the work it was given to avoid. - */ - public function testALimitStopsPullingFromTheSource(): void - { - $source = FakeSource::withEntities('many', 'urn:1', 'urn:2', 'urn:3', 'urn:4', 'urn:5'); - - $this->importer([$source])->payload('many', 2); - - $this->assertSame(2, $source->produced()); - } - - /** - * --limit 0 is a mistyped option. Honouring it would produce an empty - * payload, which is the one outcome an import refuses to call a success. - */ - public function testALimitBelowOneStillImportsOneEntity(): void - { - $importer = $this->importer([FakeSource::withEntities('many', 'urn:1', 'urn:2')]); - - $this->assertCount(1, $importer->payload('many', 0)); - } - - public function testBuildingThePayloadSendsNothing(): void - { - $client = new MockHttpClient(); - - $this->importer([FakeSource::withEntities('one-entity', 'urn:1')], $client)->payload('one-entity'); - - $this->assertSame(0, $client->getRequestsCount()); - } - - public function testItUpsertsThePayloadAndReportsWhatTheBrokerDid(): void - { - $client = new MockHttpClient(new MockResponse('', ['http_code' => 204])); - - $result = $this->importer([FakeSource::withEntities('many', 'urn:1', 'urn:2')], $client)->import('many'); - - $this->assertSame(2, $result->count); - $this->assertSame(204, $result->status); - $this->assertSame(self::BROKER_URL, $result->brokerUrl); - $this->assertSame(1, $client->getRequestsCount(), 'The entities were not sent as one batch.'); - } - - public function testItReportsWhatTheBrokerSaidWhenTheUpsertIsRejected(): void - { - $client = new MockHttpClient(new MockResponse('{"title":"Bad Request"}', ['http_code' => 400])); - - try { - $this->importer([FakeSource::withEntities('one-entity', 'urn:1')], $client)->import('one-entity'); - $this->fail('A rejected upsert was reported as a completed import.'); - } catch (UpsertFailedException $exception) { - $this->assertStringContainsString('HTTP 400', $exception->getMessage()); - $this->assertStringContainsString('Bad Request', $exception->getMessage()); - $this->assertNotNull($exception->getPrevious(), 'The broker\'s own exception was discarded.'); - } - } - - /** - * A broker that cannot be reached fails in the HTTP client rather than in - * the broker's status check, and the two are the same thing to a caller. - */ - public function testItFailsTheSameWayWhenTheBrokerCannotBeReached(): void - { - $client = new MockHttpClient(static function (): never { - throw new TransportException('Connection refused'); - }); - - $this->expectException(UpsertFailedException::class); - $this->expectExceptionMessage('Connection refused'); - - $this->importer([FakeSource::withEntities('one-entity', 'urn:1')], $client)->import('one-entity'); - } -} diff --git a/tests/Source/DataSourceReaderTest.php b/tests/Source/DataSourceReaderTest.php deleted file mode 100644 index 9c866ca..0000000 --- a/tests/Source/DataSourceReaderTest.php +++ /dev/null @@ -1,114 +0,0 @@ -reader($client)->read('https://example.com/feed.json'); - - $this->assertSame([['example' => 1]], $document); - } - - public function testItAcceptsABareArrayDocument(): void - { - $client = new MockHttpClient(new MockResponse('[1, 2, 3]')); - - $this->assertSame([1, 2, 3], $this->reader($client)->read('http://example.com/feed.json')); - } - - /** - * The envelope is feed-specific knowledge, so it must survive intact for - * the caller to interpret. Returning the features list directly would push - * that knowledge into the wrong class. - */ - public function testItDoesNotUnwrapTheEnvelope(): void - { - $client = new MockHttpClient(new MockResponse(self::FEATURE_COLLECTION)); - - $document = $this->reader($client)->read('https://example.com/feed.json'); - - $this->assertArrayHasKey('features', $document); - $this->assertArrayNotHasKey(0, $document); - $this->assertCount(2, $document['features']); - } - - public function testItRejectsALocationThatIsNotAUrl(): void - { - $client = new MockHttpClient(); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessageMatches('/must be an http\(s\) URL/'); - - try { - $this->reader($client)->read('data/feed.json'); - } finally { - // Rejection has to happen before the request, or a mistyped - // location becomes an opaque transport error instead. - $this->assertSame(0, $client->getRequestsCount()); - } - } - - /** - * A string that merely begins with "http" is not a URL. - */ - public function testItRejectsAPathThatMerelyStartsWithHttp(): void - { - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessageMatches('/must be an http\(s\) URL/'); - - $this->reader()->read('https-export.json'); - } - - public function testItFailsOnInvalidJson(): void - { - $client = new MockHttpClient(new MockResponse('{ not json')); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessageMatches('/Invalid JSON/'); - - $this->reader($client)->read('https://example.com/feed.json'); - } - - public function testItFailsWhenTheDocumentIsAScalar(): void - { - $client = new MockHttpClient(new MockResponse('"just a string"')); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessageMatches('/Expected a JSON array or object/'); - - $this->reader($client)->read('https://example.com/feed.json'); - } - - public function testItWrapsTransportFailures(): void - { - $client = new MockHttpClient(new MockResponse('', ['http_code' => 500])); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessageMatches('/Could not fetch/'); - - $this->reader($client)->read('https://example.com/feed.json'); - } -} diff --git a/tests/Source/FakeSource.php b/tests/Source/FakeSource.php deleted file mode 100644 index 052ab33..0000000 --- a/tests/Source/FakeSource.php +++ /dev/null @@ -1,61 +0,0 @@ - $entities - */ - public function __construct( - private readonly string $key, - private readonly array $entities = [], - ) { - } - - /** - * @param string ...$ids entity ids, one entity each - */ - public static function withEntities(string $key, string ...$ids): self - { - return new self($key, array_map( - static fn (string $id): NgsiEntity => new NgsiEntity($id, 'Example'), - array_values($ids) - )); - } - - public function key(): string - { - return $this->key; - } - - public function entities(): iterable - { - foreach ($this->entities as $entity) { - ++$this->produced; - - yield $entity; - } - } - - /** - * How many entities were actually pulled from this source. - */ - public function produced(): int - { - return $this->produced; - } -} diff --git a/tests/Source/Manifest/CatalogTest.php b/tests/Source/Manifest/CatalogTest.php deleted file mode 100644 index 2707d49..0000000 --- a/tests/Source/Manifest/CatalogTest.php +++ /dev/null @@ -1,238 +0,0 @@ - */ - private array $written = []; - - protected function tearDown(): void - { - foreach ($this->written as $path) { - if (is_file($path)) { - unlink($path); - } - } - - $this->written = []; - } - - public function testTheShippedManifestIsUsable(): void - { - $catalog = new Catalog(\dirname(__DIR__, 3).'/config/sources.yaml'); - - $this->assertNotSame([], $catalog->all(), 'The manifest registers no data sets.'); - } - - /** - * A wrong URL scheme or a CRS the transformer does not know only surfaces - * mid-import otherwise, after the feed has been fetched. - */ - public function testEveryShippedEntryCanBeImportedFrom(): void - { - $catalog = new Catalog(\dirname(__DIR__, 3).'/config/sources.yaml'); - - foreach ($catalog->all() as $key => $descriptor) { - $this->assertSame($key, $descriptor->key); - $this->assertMatchesRegularExpression('#^https?://#', $descriptor->accessUrl, $key); - $this->assertMatchesRegularExpression('/^EPSG:\d+$/', $descriptor->crs, $key); - $this->assertNotSame('', $descriptor->model, $key); - $this->assertMatchesRegularExpression('#^https?://#', $descriptor->contextUrl, $key); - } - } - - public function testItNamesTheKnownEntriesWhenAskedForAnUnknownOne(): void - { - $catalog = new Catalog($this->manifest(<<<'YAML' - sources: - a-source: - title: A source - access_url: https://example.com/feed.json - crs: EPSG:25832 - model: Example - context_url: https://example.com/context.jsonld - YAML)); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('Entries: a-source.'); - - $catalog->get('no-such-source'); - } - - public function testItRejectsAManifestWithoutASourcesMapping(): void - { - $catalog = new Catalog($this->manifest("data_sets:\n a-source: {}\n")); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('must contain a "sources" mapping'); - - $catalog->all(); - } - - public function testItRejectsAnEntryMissingAFieldTheImportNeeds(): void - { - $catalog = new Catalog($this->manifest(<<<'YAML' - sources: - a-source: - title: A source - access_url: https://example.com/feed.json - model: Example - context_url: https://example.com/context.jsonld - YAML)); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('The child config "crs" under "sources.a-source" must be configured'); - - $catalog->all(); - } - - public function testItRejectsAnOmittedFieldWithoutAReason(): void - { - $catalog = new Catalog($this->manifest(<<<'YAML' - sources: - a-source: - title: A source - access_url: https://example.com/feed.json - crs: EPSG:25832 - model: Example - context_url: https://example.com/context.jsonld - omitted_fields: - some_field: ~ - YAML)); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('"sources.a-source.omitted_fields.some_field" cannot contain an empty value'); - - $catalog->all(); - } - - /** - * The import selects a data set by the key written in the manifest, and the - * config tree rewrites a key that has dashes and no underscore unless told - * otherwise. A rewritten key stops matching without saying so. - */ - public function testItKeepsADashedSourceKeyIntact(): void - { - $catalog = new Catalog($this->manifest(<<<'YAML' - sources: - handicap-parking: - title: A source - access_url: https://example.com/feed.json - crs: EPSG:25832 - model: Example - context_url: https://example.com/context.jsonld - YAML)); - - $this->assertSame(['handicap-parking'], array_keys($catalog->all())); - $this->assertSame('handicap-parking', $catalog->get('handicap-parking')->key); - } - - /** - * A misspelled optional field was dropped in silence before, which loses a - * fact the record exists to carry. - */ - public function testItRejectsAFieldTheManifestDoesNotDefine(): void - { - $catalog = new Catalog($this->manifest(<<<'YAML' - sources: - a-source: - title: A source - access_url: https://example.com/feed.json - crs: EPSG:25832 - model: Example - context_url: https://example.com/context.jsonld - license: CC-BY-4.0 - YAML)); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('Unrecognized option "license" under "sources.a-source"'); - - $catalog->all(); - } - - public function testItRejectsAnEntryThatIsNotAMapping(): void - { - $catalog = new Catalog($this->manifest("sources:\n a-source: just a string\n")); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('Invalid type for path "sources.a-source"'); - - $catalog->all(); - } - - public function testItRejectsAManifestThatRegistersNothing(): void - { - $catalog = new Catalog($this->manifest("sources: {}\n")); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('should have at least 1 element'); - - $catalog->all(); - } - - /** - * An empty value means "not filled in", the same as an absent key, so an - * unanswered question reads the same either way. - */ - public function testItReadsABlankOptionalFieldAsUnknown(): void - { - $catalog = new Catalog($this->manifest(<<<'YAML' - sources: - a-source: - title: A source - access_url: https://example.com/feed.json - crs: EPSG:25832 - model: Example - context_url: https://example.com/context.jsonld - publisher: ' Aarhus Kommune ' - contact: '' - licence: ~ - YAML)); - - $descriptor = $catalog->get('a-source'); - - $this->assertSame('Aarhus Kommune', $descriptor->publisher); - $this->assertNull($descriptor->contact); - $this->assertNull($descriptor->licence); - } - - public function testItReportsAManifestThatIsNotThere(): void - { - $catalog = new Catalog('/no/such/sources.yaml'); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('does not exist'); - - $catalog->all(); - } - - public function testItReportsUnparsableYaml(): void - { - $catalog = new Catalog($this->manifest("sources:\n - [unbalanced\n")); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('is not valid YAML'); - - $catalog->all(); - } - - private function manifest(string $yaml): string - { - $path = tempnam(sys_get_temp_dir(), 'sources-'); - - if (false === $path) { - $this->fail('Could not create a temporary manifest.'); - } - - file_put_contents($path, $yaml); - $this->written[] = $path; - - return $path; - } -} diff --git a/tests/Source/Manifest/ValidatorTest.php b/tests/Source/Manifest/ValidatorTest.php deleted file mode 100644 index 53230bb..0000000 --- a/tests/Source/Manifest/ValidatorTest.php +++ /dev/null @@ -1,90 +0,0 @@ - */ - private array $written = []; - - protected function tearDown(): void - { - foreach ($this->written as $path) { - if (is_file($path)) { - unlink($path); - } - } - - $this->written = []; - } - - /** - * A check that can be skipped is not a check. - */ - public function testItIsNotOptional(): void - { - $this->assertFalse($this->validator(\dirname(__DIR__, 3).'/config/sources.yaml')->isOptional()); - } - - /** - * It validates rather than caches, so it leaves nothing behind to preload. - */ - public function testItAcceptsTheShippedManifestAndWritesNothing(): void - { - $validator = $this->validator(\dirname(__DIR__, 3).'/config/sources.yaml'); - - $this->assertSame([], $validator->warmUp(sys_get_temp_dir(), sys_get_temp_dir())); - } - - /** - * The reason for checking at build time: an entry no import selects still - * fails the build rather than waiting to be discovered. - */ - public function testItFailsOnAnEntryNoImportWouldReach(): void - { - $validator = $this->validator($this->manifest(<<<'YAML' - sources: - a-source: - title: A source - access_url: https://example.com/feed.json - crs: EPSG:25832 - model: Example - context_url: https://example.com/context.jsonld - unreached-source: - title: Another source - access_url: https://example.com/other.json - model: Example - context_url: https://example.com/context.jsonld - YAML)); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('The child config "crs" under "sources.unreached-source" must be configured'); - - $validator->warmUp(sys_get_temp_dir(), sys_get_temp_dir()); - } - - private function validator(string $manifest): Validator - { - return new Validator(new Catalog($manifest)); - } - - private function manifest(string $yaml): string - { - $path = tempnam(sys_get_temp_dir(), 'sources-'); - - if (false === $path) { - $this->fail('Could not create a temporary manifest.'); - } - - file_put_contents($path, $yaml); - $this->written[] = $path; - - return $path; - } -} diff --git a/tests/Source/Manifest/WritesManifests.php b/tests/Source/Manifest/WritesManifests.php deleted file mode 100644 index 121509f..0000000 --- a/tests/Source/Manifest/WritesManifests.php +++ /dev/null @@ -1,61 +0,0 @@ - */ - private array $manifests = []; - - protected function tearDown(): void - { - foreach ($this->manifests as $path) { - if (is_file($path)) { - unlink($path); - } - } - - $this->manifests = []; - } - - protected static function dataSetContext(string $key): string - { - return \sprintf('https://example.com/%s.jsonld', $key); - } - - /** - * @param list $keys - * - * @return string path to the manifest - */ - protected function manifestFor(array $keys): string - { - $entries = array_map(static fn (string $key): string => \sprintf( - " %s:\n title: %s\n access_url: https://example.com/%s.json\n crs: EPSG:25832\n model: Example\n context_url: %s", - $key, - $key, - $key, - self::dataSetContext($key), - ), $keys); - - $path = tempnam(sys_get_temp_dir(), 'sources-'); - - if (false === $path) { - $this->fail('Could not create a temporary manifest.'); - } - - file_put_contents($path, "sources:\n".implode("\n", $entries)."\n"); - $this->manifests[] = $path; - - return $path; - } -} diff --git a/tests/Source/MtmSpatialMaps/HandicapParkingTest.php b/tests/Source/MtmSpatialMaps/HandicapParkingTest.php deleted file mode 100644 index ff6b8f9..0000000 --- a/tests/Source/MtmSpatialMaps/HandicapParkingTest.php +++ /dev/null @@ -1,190 +0,0 @@ -> */ - private array $entities; - - protected function setUp(): void - { - $catalog = new Catalog(\dirname(__DIR__, 3).'/config/sources.yaml'); - $this->source = $catalog->get(self::KEY); - - $client = new MockHttpClient(function (string $method, string $url): MockResponse { - $this->requestedUrl = $url; - - return new MockResponse(json_encode($this->feed(), \JSON_THROW_ON_ERROR)); - }); - - $source = new HandicapParking(new DataSourceReader($client), new Wgs84Transformer(), $catalog); - - $this->entities = array_map( - static fn (NgsiEntity $entity): array => $entity->toPayload(['https://example.com/context.jsonld']), - iterator_to_array($source->entities(), false) - ); - } - - public function testItReadsTheFeedTheManifestPointsAt(): void - { - $this->assertSame($this->source->accessUrl, $this->requestedUrl); - } - - public function testItSkipsRecordsWithoutAnIdentifierOrGeometry(): void - { - // Four features, of which one has no mi_prinx and one no geometry. - $this->assertCount(2, $this->entities); - } - - public function testItTakesIdentifierAndTypeFromTheManifestModel(): void - { - $first = $this->entities[0]; - - // The id is derived from mi_prinx so that re-importing upserts the - // same bay instead of creating a duplicate. - $this->assertSame( - \sprintf('urn:ngsi-ld:%s:aarhus-handicap-172', $this->source->model), - $first['id'] - ); - $this->assertSame($this->source->model, $first['type']); - } - - public function testItJoinsStreetAndHouseNumberIntoName(): void - { - $this->assertSame('Domkirkeplads/Bispegade 1', $this->entities[0]['name']['value']); - } - - public function testItOmitsTheHouseNumberWhenBlank(): void - { - // husnnr is "" for this row, so the name must not end in a space. - $this->assertSame('Brammersgade', $this->entities[1]['name']['value']); - } - - public function testItMarksEveryEntityAsDisabledParking(): void - { - foreach ($this->entities as $entity) { - $this->assertSame(['forDisabled'], $entity['category']['value']); - } - } - - public function testItCarriesTheBayCountAsTotalSpotNumber(): void - { - $this->assertSame(6, $this->entities[0]['totalSpotNumber']['value']); - $this->assertSame(1, $this->entities[1]['totalSpotNumber']['value']); - } - - public function testItDropsEmptyDescriptions(): void - { - // bemrk is null for the first row and filled in for the second. - $this->assertArrayNotHasKey('description', $this->entities[0]); - $this->assertSame('Ved indgangen', $this->entities[1]['description']['value']); - } - - public function testItReprojectsLocationIntoWgs84(): void - { - $location = $this->entities[0]['location']; - - $this->assertSame('GeoProperty', $location['type']); - $this->assertSame('Point', $location['value']['type']); - - // The feed publishes metres in the manifest's CRS, so degrees within - // Denmark are the evidence that the reprojection ran. - [$longitude, $latitude] = $location['value']['coordinates']; - $this->assertGreaterThan(8.0, $longitude); - $this->assertLessThan(13.0, $longitude); - $this->assertGreaterThan(54.5, $latitude); - $this->assertLessThan(57.8, $latitude); - } - - public function testItRecordsTheManifestUrlAsTheEntitySource(): void - { - $this->assertSame($this->source->accessUrl, $this->entities[0]['source']['value']); - } - - /** - * The first feature is a record from the live export, kept verbatim. The - * rest are constructed to exercise a blank house number and the two guards - * that discard a record. - * - * @return array - */ - private function feed(): array - { - return [ - 'type' => 'FeatureCollection', - 'crs' => ['type' => 'name', 'properties' => ['name' => 'EPSG:25832']], - 'features' => [ - [ - 'type' => 'Feature', - 'geometry' => ['type' => 'Point', 'coordinates' => [575153.9524951308, 6224260.609753487]], - 'properties' => [ - 'vejnavn' => 'Domkirkeplads/Bispegade', - 'husnnr' => '1', - 'invalidepladser' => 6, - 'bemrk' => null, - 'ident' => 'P', - 'oprettet_af' => 'ADM\\aztnbnd', - 'oprettet_dato' => '2019-03-19 14:51:23.91', - 'rettet_af' => 'ADM\\aztnbnd', - 'rettet_dato' => '2019-03-19 14:51:23.91', - 'mi_style' => null, - 'mi_prinx' => 172, - ], - ], - [ - 'type' => 'Feature', - 'geometry' => ['type' => 'Point', 'coordinates' => [574000.0, 6223000.0]], - 'properties' => [ - 'vejnavn' => 'Brammersgade', - 'husnnr' => '', - 'invalidepladser' => 1, - 'bemrk' => 'Ved indgangen', - 'mi_prinx' => 378, - ], - ], - [ - 'type' => 'Feature', - 'geometry' => ['type' => 'Point', 'coordinates' => [574100.0, 6223100.0]], - 'properties' => [ - 'vejnavn' => 'Uden nøgle', - 'husnnr' => '3', - 'invalidepladser' => 2, - ], - ], - [ - 'type' => 'Feature', - 'geometry' => null, - 'properties' => [ - 'vejnavn' => 'Uden geometri', - 'husnnr' => '5', - 'invalidepladser' => 2, - 'mi_prinx' => 999, - ], - ], - ], - ]; - } -} From 5598a299be57d650c933cb4aff342c2a4bcb3cf2 Mon Sep 17 00:00:00 2001 From: Mikkel Ricky Date: Wed, 9 Sep 2026 14:38:19 +0200 Subject: [PATCH 4/4] Moved stuff around --- CHANGELOG.md | 2 ++ src/Command/SourceImportCommand.php | 2 +- src/{ => SourceImporter}/SourceImporterFactory.php | 2 +- src/{ => SourceImporter}/SourceImporterGeoJson.php | 3 ++- src/{ => SourceImporter}/SourceImporterInterface.php | 2 +- src/{ => SourceReader}/SourceReaderGeoJson.php | 2 +- src/{ => SourceReader}/SourceReaderInterface.php | 2 +- 7 files changed, 9 insertions(+), 6 deletions(-) rename src/{ => SourceImporter}/SourceImporterFactory.php (96%) rename src/{ => SourceImporter}/SourceImporterGeoJson.php (97%) rename src/{ => SourceImporter}/SourceImporterInterface.php (93%) rename src/{ => SourceReader}/SourceReaderGeoJson.php (97%) rename src/{ => SourceReader}/SourceReaderInterface.php (86%) diff --git a/CHANGELOG.md b/CHANGELOG.md index c67b120..9a8dae0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +* [PR-9](https://github.com/itk-dev/enter/pull/9) + Refactored source import * [PR-7](https://github.com/itk-dev/enter/pull/7) Refactored source definition * [#3](https://github.com/itk-dev/enter/pull/3) diff --git a/src/Command/SourceImportCommand.php b/src/Command/SourceImportCommand.php index 8f5e561..94945a7 100644 --- a/src/Command/SourceImportCommand.php +++ b/src/Command/SourceImportCommand.php @@ -3,7 +3,7 @@ namespace App\Command; use App\Source\SourceInterface; -use App\SourceImporterFactory; +use App\SourceImporter\SourceImporterFactory; use Symfony\Component\Console\Attribute\Argument; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; diff --git a/src/SourceImporterFactory.php b/src/SourceImporter/SourceImporterFactory.php similarity index 96% rename from src/SourceImporterFactory.php rename to src/SourceImporter/SourceImporterFactory.php index aee58ea..4dce0d3 100644 --- a/src/SourceImporterFactory.php +++ b/src/SourceImporter/SourceImporterFactory.php @@ -1,6 +1,6 @@