From d52766db2d8d5b386e294a8c58d2c7aa18519e0f Mon Sep 17 00:00:00 2001 From: Norbert Orzechowicz Date: Sat, 31 Oct 2026 13:22:07 -0500 Subject: [PATCH] feat(flow-php/etl): corrupted records are refused or routed, never repaired silently - readers refuse records that don't match the header or schema with CorruptedRecordException - withCorruptedRecords($loader) routes them through the run's own loader lifecycle - withMissingColumnsAsNull() replaces padding, truncation and Google Sheet withDropExtraColumns() - schema errors for records or keys past the inference sample and missing declared columns - Rust CSV and JSON readers match the PHP readers - to_date_time(null) returns null; Corrupted Records documentation page --- documentation/components/adapters/csv.md | 12 +- documentation/components/adapters/excel.md | 11 +- .../components/adapters/google-sheet.md | 11 +- documentation/components/adapters/json.md | 4 + documentation/components/core/core.md | 1 + .../components/core/corrupted-records.md | 96 +++++ .../components/core/error-handling.md | 2 + documentation/upgrading.md | 27 +- .../Adapter/CSV/IO/Extractor/CSVDecoder.php | 28 +- .../Adapter/CSV/IO/Extractor/CSVExtractor.php | 32 +- .../CSV/IO/Extractor/CSVFileBatches.php | 33 +- .../CSV/IO/Extractor/CSVOpenSource.php | 15 +- .../CSV/IO/Extractor/CSVReadOptions.php | 26 +- .../CSV/IO/Extractor/CSVSourceOpener.php | 2 + .../CSV/IO/Extractor/PhpCSVOpenSource.php | 73 +++- .../CSV/Tests/Context/CSVFixtureContext.php | 59 ++- .../IO/Extractor/CSVExtractorTest.php | 336 +++++++++++++---- .../CSVExtractorTypedColumnsTest.php | 19 +- .../IO/Extractor/CSVSplitReadTest.php | 70 +++- .../IO/Extractor/PhpCSVOpenSourceTest.php | 127 ++++--- .../IO/Extractor/RustCSVOpenSourceTest.php | 177 ++++++--- .../Unit/IO/Extractor/CSVDecoderTest.php | 105 +++--- .../Unit/IO/Extractor/CSVFileBatchesTest.php | 11 +- .../Unit/IO/Extractor/CSVReadOptionsTest.php | 5 +- .../Excel/IO/Extractor/ExcelExtractor.php | 32 +- .../Excel/IO/Extractor/ExcelFileBatches.php | 46 ++- .../Excel/IO/Extractor/ExcelReadOptions.php | 20 +- .../Excel/IO/Extractor/Sheet/SheetCells.php | 13 +- .../IO/Extractor/Sheet/XlsxSheetRows.php | 7 +- .../Excel/IO/Extractor/WorkbookReader.php | 6 +- .../Excel/IO/Extractor/WorkbookSheet.php | 53 ++- .../Excel/IO/Inference/ExcelSniffer.php | 2 +- .../IO/Inference/WorkbookSheetSample.php | 16 +- .../Tests/Context/ExcelFixtureContext.php | 21 ++ .../IO/Extractor/ExcelExtractorTest.php | 151 ++++++-- .../ExcelExtractorTypedColumnsTest.php | 12 +- .../IO/Extractor/WorkbookReaderTest.php | 12 +- .../IO/Extractor/WorkbookSheetTest.php | 10 +- .../IO/Extractor/ExcelFileBatchesTest.php | 9 +- .../IO/Extractor/ExcelReadOptionsTest.php | 6 + .../IO/Inference/WorkbookSheetSampleTest.php | 47 ++- .../IO/Extractor/GoogleSheetExtractor.php | 160 +++++--- .../IO/Extractor/GoogleSheetReadOptions.php | 18 +- .../IO/Extractor/GoogleSheetReader.php | 48 +-- .../IO/Inference/GoogleSheetSample.php | 4 +- .../IO/Inference/GoogleSheetSniffer.php | 30 +- .../Tests/Fixtures/extra-empty-rows.json | 2 +- .../Fixtures/sample-extra-empty-rows.json | 3 +- .../IO/Extractor/GoogleSheetExtractorTest.php | 78 ++-- .../GoogleSheetExtractorTypedColumnsTest.php | 12 +- .../IO/Extractor/GoogleSheetExtractorTest.php | 205 +++++++---- .../Extractor/GoogleSheetReadOptionsTest.php | 38 +- .../IO/Extractor/GoogleSheetReaderTest.php | 57 ++- .../IO/Inference/GoogleSheetSampleTest.php | 9 +- .../JSON/IO/Extractor/JsonExtractor.php | 22 +- .../JSON/IO/Extractor/JsonFileBatches.php | 10 +- .../JSON/IO/Extractor/JsonFileReader.php | 23 +- .../JSON/IO/Extractor/JsonOpenSource.php | 20 +- .../JSON/IO/Extractor/JsonReadBytes.php | 29 -- .../JSON/IO/Extractor/JsonReadProgress.php | 66 ++++ .../JSON/IO/Extractor/PhpJsonOpenSource.php | 31 +- .../JSON/IO/Inference/JsonFileSample.php | 4 +- .../JSON/Tests/Context/JsonFixtureContext.php | 22 +- .../IO/Extractor/JsonExtractorTest.php | 102 +++-- .../JsonExtractorTypedColumnsTest.php | 18 +- .../IO/Extractor/JsonFileBatchesTest.php | 2 + .../IO/Extractor/JsonLinesExtractorTest.php | 141 +++++-- .../JsonLinesExtractorTypedColumnsTest.php | 6 +- .../IO/Extractor/JsonSplitReadTest.php | 3 +- .../IO/Extractor/PhpJsonOpenSourceTest.php | 34 +- .../IO/Extractor/RustJsonOpenSourceTest.php | 90 ++++- .../Unit/IO/Extractor/JsonFileBatchesTest.php | 9 +- .../Unit/IO/Extractor/JsonReadBytesTest.php | 25 -- .../IO/Extractor/JsonReadProgressTest.php | 41 +++ .../Parquet/IO/Extractor/ParquetExtractor.php | 3 +- .../Extractor/ParquetExtractorFooterTest.php | 19 +- .../IO/Extractor/ParquetExtractorTest.php | 114 ++++-- .../Tests/Integration/PaginationTest.php | 42 ++- .../Text/IO/Extractor/TextExtractor.php | 3 +- .../IO/Extractor/TextExtractorTest.php | 47 ++- .../TextExtractorTypedColumnsTest.php | 12 +- .../XML/IO/Extractor/XMLParserExtractor.php | 3 +- .../XMLExtractorTypedColumnsTest.php | 42 ++- .../IO/Extractor/XMLParserExtractorTest.php | 49 ++- .../Unit/AsyncAWSS3DestinationStreamTest.php | 6 +- .../Exception/ColumnMismatchException.php | 7 +- .../Exception/CorruptedRecordException.php | 178 +++++++++ .../Exception/InferredSchemaException.php | 32 +- .../Exception/SchemaValidationException.php | 19 + .../etl/src/Flow/ETL/Functions/ToDateTime.php | 10 +- .../Flow/ETL/IO/CorruptedRecordsRouter.php | 15 + src/core/etl/src/Flow/ETL/IO/Extractor.php | 6 +- .../Flow/ETL/IO/Extractor/ArrayExtractor.php | 45 ++- .../Flow/ETL/IO/Extractor/CacheExtractor.php | 8 +- .../Flow/ETL/IO/Extractor/ChainExtractor.php | 12 +- .../ETL/IO/Extractor/File/FileBatches.php | 6 +- .../Flow/ETL/IO/Extractor/File/FileRead.php | 3 +- .../ETL/IO/Extractor/File/FileReadLoop.php | 14 +- .../ETL/IO/Extractor/File/FileSplitReader.php | 28 +- .../ETL/IO/Extractor/Grid/RecordDecoder.php | 71 ++-- .../etl/src/Flow/ETL/IO/Extractor/Reader.php | 5 +- .../ETL/IO/Extractor/SequenceExtractor.php | 56 ++- .../src/Flow/ETL/IO/Floe/FloeExtractor.php | 3 +- .../Transformer/JoinEachRowsTransformer.php | 8 +- .../Runtime/ErrorHandler/ExtractionError.php | 3 +- .../ETL/Runtime/Executor/AdoptedBatches.php | 13 +- .../Flow/ETL/Runtime/Executor/Executor.php | 5 +- .../Flow/ETL/Runtime/Executor/SourceCount.php | 13 +- .../ETL/Runtime/Executor/SourceCounts.php | 7 +- .../Flow/ETL/Runtime/Executor/StepRunner.php | 214 ++++++++--- .../src/Flow/ETL/Values/Rows/RecordBatch.php | 245 +++++++++++- src/core/etl/src/Flow/ETL/Values/Schema.php | 13 + .../Flow/ETL/Tests/Context/ExtractedRows.php | 6 +- .../ETL/Tests/Context/FileFormatContract.php | 4 +- .../tests/Flow/ETL/Tests/Context/IdRows.php | 5 +- .../ETL/Tests/Context/ReaderRoundTrip.php | 7 +- .../Flow/ETL/Tests/Context/SplitCoverage.php | 7 +- .../etl/tests/Flow/ETL/Tests/FlowTestCase.php | 42 ++- .../DataFrame/ChainedBatchSizesTest.php | 5 +- .../Integration/DataFrame/SchemaTest.php | 4 +- .../IO/Extractor/CacheExtractorTest.php | 66 +++- .../Integration/IO/Floe/FloeExtractorTest.php | 6 +- .../Integration/IO/Floe/FloeSplitReadTest.php | 17 +- .../Exception/ColumnMismatchExceptionTest.php | 8 + .../CorruptedRecordExceptionTest.php | 113 ++++++ .../Exception/InferredSchemaExceptionTest.php | 30 ++ .../SchemaValidationExceptionTest.php | 14 + .../Tests/Unit/Functions/ToDateTimeTest.php | 32 ++ .../Unit/IO/Extractor/ArrayExtractorTest.php | 35 ++ .../Unit/IO/Extractor/ChainExtractorTest.php | 43 ++- .../IO/Extractor/File/FileReadLoopTest.php | 110 +++--- .../Unit/IO/Extractor/File/FileReadTest.php | 32 +- .../IO/Extractor/File/FileSplitReaderTest.php | 22 +- .../IO/Extractor/Grid/RecordDecoderTest.php | 348 +++--------------- .../IO/Extractor/SequenceExtractorTest.php | 47 ++- .../Tests/Unit/IO/Floe/FloeExtractorTest.php | 88 +++-- .../ETL/Tests/Unit/IO/Floe/FloeLoaderTest.php | 16 +- .../Unit/Operators/Join/HashJoinTest.php | 2 +- .../Processor/BatchingByProcessorTest.php | 14 +- .../Processor/BatchingProcessorTest.php | 20 +- .../Processor/CachingProcessorTest.php | 2 +- .../Processor/ConstrainedProcessorTest.php | 2 +- .../Processor/ExpandingProcessorTest.php | 14 +- .../Processor/OffsetProcessorTest.php | 2 +- .../JoinEachRowsTransformerTest.php | 12 + .../Runtime/Executor/AdoptedBatchesTest.php | 24 +- .../Unit/Runtime/Executor/SourceCountTest.php | 23 ++ .../Runtime/Executor/SourceCountsTest.php | 70 +++- .../Unit/Runtime/Executor/StepRunnerTest.php | 164 +++++++++ .../Unit/Values/Rows/RecordBatchTest.php | 263 ++++++++++++- .../Flow/ETL/Tests/Unit/Values/SchemaTest.php | 9 + .../crates/flow-csv-tokenizer/src/lib.rs | 83 ++++- .../crates/flow-json-records/src/lib.rs | 106 +++++- .../CSV/IO/Extractor/RustCSVOpenSource.php | 3 +- .../JSON/IO/Extractor/RustJsonOpenSource.php | 9 +- .../flow-php-ext/src/batch_columns.rs | 61 +-- src/extension/flow-php-ext/src/corrupted.rs | 131 +++++++ src/extension/flow-php-ext/src/csv/columns.rs | 142 +++++-- src/extension/flow-php-ext/src/csv/mod.rs | 213 +++++++---- src/extension/flow-php-ext/src/csv/source.rs | 57 +-- src/extension/flow-php-ext/src/interfaces.rs | 9 +- src/extension/flow-php-ext/src/json/read.rs | 252 ++++++++++--- src/extension/flow-php-ext/src/json/source.rs | 57 ++- src/extension/flow-php-ext/src/lib.rs | 1 + src/extension/flow-php-ext/src/source.rs | 39 +- .../tests/phpt/040_csv_tokenizer_parity.phpt | 6 +- .../tests/phpt/044_csv_ragged_rows.phpt | 14 +- .../phpt/064_csv_next_columns_no_leaks.phpt | 12 +- .../tests/phpt/086_json_read_parity.phpt | 9 +- .../tests/phpt/087_json_read_no_leaks.phpt | 18 +- .../flow-php-ext/tests/phpt/bootstrap.php | 10 +- .../corrupted_records_routing/_meta.yaml | 2 + .../corrupted_records_routing/code.php | 26 ++ .../corrupted_records_routing/description.md | 2 + .../documentation.md | 4 + .../topics/reading/csv_cells/code.php | 2 +- .../topics/reading/csv_cells/description.md | 5 +- .../documentation/navigation_left.html.twig | 1 + 178 files changed, 5517 insertions(+), 1971 deletions(-) create mode 100644 documentation/components/core/corrupted-records.md delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonReadBytes.php create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonReadProgress.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonReadBytesTest.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonReadProgressTest.php create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/CorruptedRecordException.php create mode 100644 src/core/etl/src/Flow/ETL/IO/CorruptedRecordsRouter.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/CorruptedRecordExceptionTest.php create mode 100644 src/extension/flow-php-ext/src/corrupted.rs create mode 100644 web/landing/content/examples/topics/reading/corrupted_records_routing/_meta.yaml create mode 100644 web/landing/content/examples/topics/reading/corrupted_records_routing/code.php create mode 100644 web/landing/content/examples/topics/reading/corrupted_records_routing/description.md create mode 100644 web/landing/content/examples/topics/reading/corrupted_records_routing/documentation.md diff --git a/documentation/components/adapters/csv.md b/documentation/components/adapters/csv.md index b5f988c396..140fc18bd1 100644 --- a/documentation/components/adapters/csv.md +++ b/documentation/components/adapters/csv.md @@ -36,11 +36,17 @@ $rows = data_frame() ### Header and missing cells ``` -header row [' id ', '', 'name'] → columns ['id', 'e01', 'name'] (trimmed, a blank cell named e + its position) -record ['1', 'a'] → ['id' => 1, 'e01' => 'a', 'name' => null] (a cell the record lacks is null) -record ['1', '', 'x'] → 'e01' => null (a present '' is null; `withEmptyToNull(false)` keeps it '') +header row [' id ', '', 'name'] -> columns ['id', 'e01', 'name'] (trimmed, a blank cell named e + its position) +record ['1', 'a'] -> corrupted record (2 cells, the header has 3); `withMissingColumnsAsNull(true)` makes 'name' null +record ['1', 'a', 'x', ''] -> corrupted record (a non-empty cell past the header), an empty one is dropped +record ['1', '', 'x'] -> 'e01' => null (a present '' is null; `withEmptyToNull(false)` keeps it '') +empty line -> skipped ``` +A corrupted record throws `CorruptedRecordException`, or goes to `->withCorruptedRecords($loader)`, see +[Corrupted Records](/documentation/components/core/corrupted-records.md). A column of `->withSchema()` missing from the +header throws `SchemaValidationException`. + ## Loader ```php diff --git a/documentation/components/adapters/excel.md b/documentation/components/adapters/excel.md index 8cfbffd237..9372a86486 100644 --- a/documentation/components/adapters/excel.md +++ b/documentation/components/adapters/excel.md @@ -71,11 +71,16 @@ starts at row 5. ### Header and missing cells ``` -header row [' id ', '', 'name'] → columns ['id', 'e01', 'name'] (trimmed, a blank cell named e + its position) -record ['1', 'a'] → ['id' => '1', 'e01' => 'a', 'name' => null] (a cell the record lacks is null) -record ['1', '', 'x'] → 'e01' => null (a present '' is null; `withConvertEmptyToNull(false)` keeps it '') +header row [' id ', '', 'name'] -> columns ['id', 'e01', 'name'] (trimmed, a blank cell named e + its position) +record ['1', 'a'] -> ['id' => '1', 'e01' => 'a', 'name' => null]; `withMissingColumnsAsNull(false)` makes it a corrupted record +record ['1', 'a', 'x', ''] -> corrupted record (a non-empty cell past the header), an empty one is dropped +record ['1', '', 'x'] -> 'e01' => null (a present '' is null; `withConvertEmptyToNull(false)` keeps it '') ``` +A corrupted record throws `CorruptedRecordException`, or goes to `->withCorruptedRecords($loader)`, see +[Corrupted Records](/documentation/components/core/corrupted-records.md). A column of `->withSchema()` missing from the +header throws `SchemaValidationException`. + A header cell that is not text (a date, a number) is named like a blank one. ### Schema diff --git a/documentation/components/adapters/google-sheet.md b/documentation/components/adapters/google-sheet.md index a0472be025..f9854d5206 100644 --- a/documentation/components/adapters/google-sheet.md +++ b/documentation/components/adapters/google-sheet.md @@ -32,11 +32,16 @@ data_frame() ### Header and missing cells ``` -header row [' id ', '', 'name'] → columns ['id', 'e01', 'name'] (trimmed, a blank cell named e + its position) -record ['1', 'a'] → ['id' => '1', 'e01' => 'a', 'name' => null] (a cell the record lacks is null) -record ['1', '', 'x'] → 'e01' => null (a present '' is null; `withEmptyToNull(false)` keeps it '') +header row [' id ', '', 'name'] -> columns ['id', 'e01', 'name'] (trimmed, a blank cell named e + its position) +record ['1', 'a'] -> corrupted record (2 cells, the header has 3); `withMissingColumnsAsNull(true)` makes 'name' null +record ['1', 'a', 'x', ''] -> corrupted record (a non-empty cell past the header), an empty one is dropped +record ['1', '', 'x'] -> 'e01' => null (a present '' is null; `withEmptyToNull(false)` keeps it '') ``` +A corrupted record throws `CorruptedRecordException`, or goes to `->withCorruptedRecords($loader)`, see +[Corrupted Records](/documentation/components/core/corrupted-records.md). A column of `->withSchema()` missing from the +header throws `SchemaValidationException`. + Blank rows before the header are skipped, also under `withHeader(false)`. ## Needed parameters diff --git a/documentation/components/adapters/json.md b/documentation/components/adapters/json.md index 723bc3969e..961d763df7 100644 --- a/documentation/components/adapters/json.md +++ b/documentation/components/adapters/json.md @@ -54,6 +54,10 @@ holding a top-level array are read natively, as strict JSON: malformed input thr `RuntimeException('Malformed JSON in "" at line N: ...')` (`at element N` in a document). A pointer, or a document that is not an array, is read by the PHP reader. +A value that does not cast to the type `->withSchema()` declares throws `CorruptedRecordException`, or goes to +`->withCorruptedRecords($loader)`, see [Corrupted Records](/documentation/components/core/corrupted-records.md). Under an +inferred schema, a key first seen after the inference sample throws `InferredSchemaException`. + ## Loader ```php diff --git a/documentation/components/core/core.md b/documentation/components/core/core.md index 4e97b2befb..78970f3ec0 100644 --- a/documentation/components/core/core.md +++ b/documentation/components/core/core.md @@ -343,6 +343,7 @@ For detailed information about specific DataFrame operations, see the following - **[Schema](/documentation/components/core/schema.md)** - Schema management and validation - **[Constraints](/documentation/components/core/constraints.md)** - Data integrity constraints and business rules - **[Error Handling](/documentation/components/core/error-handling.md)** - Error management strategies +- **[Corrupted Records](/documentation/components/core/corrupted-records.md)** - Routing records that cannot become rows ### Observability - **[Telemetry](/documentation/components/core/telemetry.md)** - Distributed tracing, metrics, and logging integration diff --git a/documentation/components/core/corrupted-records.md b/documentation/components/core/corrupted-records.md new file mode 100644 index 0000000000..912125fedb --- /dev/null +++ b/documentation/components/core/corrupted-records.md @@ -0,0 +1,96 @@ +# Corrupted Records + +[DOC_LINK:/documentation/components/core/core.md] + +A corrupted record is a source record that cannot become a row without losing or inventing data: a CSV line or a +sheet row whose cells do not match the header, or a value that does not cast to the declared type. A read throws +`CorruptedRecordException` at the first one. Route them to any loader instead: + +```php +greaterThan(lit(0)); + +// read, route corrupted records, split valid and invalid rows +data_frame() + ->read(from_csv('orders.csv') + ->withSchema(schema(int_schema('id'), float_schema('amount'))) + ->withCorruptedRecords(to_json_lines('rejects/broken.jsonl'))) + ->write(to_branch($valid, to_json('clean.json'))) + ->write(to_branch(not($valid), to_json('rejects/invalid.json'))) + ->run(); + +// report +data_frame() + ->read(from_json_lines('rejects/broken.jsonl')) + ->groupBy(ref('reason')) + ->aggregate(count(ref('offset'))) + ->write(to_output()) + ->run(); + +// repair: a record with an extra field keeps its first two +data_frame() + ->read(from_json_lines('rejects/broken.jsonl')) + ->filter(ref('fields')->size()->greaterThan(lit(2))) + ->select('offset', 'fields') + ->withColumn('id', ref('fields')->arrayGet('0')->cast('int')) + ->withColumn('amount', ref('fields')->arrayGet('1')->cast('float')) + ->drop('fields') + ->write(to_output()) + ->run(); +``` + +`orders.csv`: + +``` +id,amount +1,10.5 +2,abc +3,-4 +4,7,extra +5,1.5 +``` + +Output: + +``` ++----------------------+--------------+ +| reason | offset_count | ++----------------------+--------------+ +| 3 fields, the header | 1 | +| column "amount": cou | 1 | ++----------------------+--------------+ +2 rows ++--------+----+----------+ +| offset | id | amount | ++--------+----+----------+ +| 28 | 4 | 7.000000 | ++--------+----+----------+ +1 rows +``` + +## The routed row + +| Column | Type | Value | +|----------|-----------------|--------------------------------------------------------------------------| +| `uri` | `?string` | the file, null for `from_array()` and sequences | +| `offset` | `int` | where the record starts, see below | +| `line` | `?int` | if the read starts at the first byte of the file, the line of the record | +| `reason` | `string` | `3 fields, the header has 2`, `column "amount": could not convert ...` | +| `record` | `string` | the record as the source holds it, else its cells or values as JSON | +| `fields` | `?list` | the cells a CSV, Excel or Google Sheet reader split, null for JSON | + +| Reader | `offset` | +|---------------------------|------------------------------| +| CSV, JSON lines | byte offset of the record | +| Excel, Google Sheet | 1-based sheet row number | +| JSON array | 1-based element number | +| `from_array()`, sequences | 1-based entry number | + +A run that routed records logs how many and where. A corrupted record is not an extraction error. Without +`withCorruptedRecords()` it fails the run, whatever the `ErrorHandler`. Route it to skip it. `limit()` and `offset()` +count rows only. diff --git a/documentation/components/core/error-handling.md b/documentation/components/core/error-handling.md index dd84a20080..8e1e76ea97 100644 --- a/documentation/components/core/error-handling.md +++ b/documentation/components/core/error-handling.md @@ -11,6 +11,8 @@ By default an exception thrown while reading, transforming or loading stops the | `onTransformation(TransformationError): TransformationAction` | `cause`, `transformer`, `rows` | `skipBatch` - drop the batch, continue with the next one | | `onLoading(LoadingError): LoadingAction` | `cause`, `loader`, `rows` | `skipLoader` - the failing loader misses this batch, the remaining steps still run | +A corrupted record never reaches the `ErrorHandler`, see [Corrupted Records](/documentation/components/core/corrupted-records.md). + Built-in handlers: | Handler | Extraction | Transformation | Loading | diff --git a/documentation/upgrading.md b/documentation/upgrading.md index 4687465160..58040a8367 100644 --- a/documentation/upgrading.md +++ b/documentation/upgrading.md @@ -1347,7 +1347,7 @@ wrappers are gone, the class that builds the sink or source picks the lane | `Flow\ETL\Adapter\JSON\{JsonOpenSource, PhpJsonOpenSource, RustJsonOpenSource}` | `Flow\ETL\Adapter\JSON\IO\Extractor\{JsonOpenSource, PhpJsonOpenSource, RustJsonOpenSource}` | | `Flow\ETL\Adapter\JSON\JSONMachine\{JsonArrayElements, JsonExtractor, JsonFileBatches, JsonFileReader}` | `Flow\ETL\Adapter\JSON\IO\Extractor\{JsonArrayElements, JsonExtractor, JsonFileBatches, JsonFileReader}` | | `Flow\ETL\Adapter\JSON\JSONMachine\JsonFileSample` | `Flow\ETL\Adapter\JSON\IO\Inference\JsonFileSample` | -| `Flow\ETL\Adapter\JSON\JSONMachine\JsonReadBytes` | `Flow\ETL\Adapter\JSON\IO\Extractor\JsonReadBytes` | +| `Flow\ETL\Adapter\JSON\JSONMachine\JsonReadBytes` | `Flow\ETL\Adapter\JSON\IO\Extractor\JsonReadProgress` | | `Flow\ETL\Adapter\JSON\JsonSchema\{JsonSchemaMetadata, ResolvedReference}` | `Flow\ETL\Adapter\JSON\Values\Schema\JsonSchema\{JsonSchemaMetadata, ResolvedReference}` | | `Flow\ETL\Adapter\JSON\JsonSchema\ReferenceResolver` | `Flow\ETL\Adapter\JSON\IO\Schema\JsonSchema\ReferenceResolver` | | `Flow\ETL\Adapter\JSON\JsonSchema\Exception\{CircularReferenceException, UnresolvableReferenceException, UnsupportedKeywordException}` | `Flow\ETL\Adapter\JSON\Foundation\Exception\{CircularReferenceException, UnresolvableReferenceException, UnsupportedKeywordException}` | @@ -1889,6 +1889,31 @@ by Spark's hidden-path rule. |------------------------------------------------------|-----------------------------------------------------------------------| | the first filled block put an empty blob at the path | nothing is visible at the path until `close()` commits the block list | +### 126) `flow-php/etl` - extractors yield corrupted records, the run routes or refuses them + +| Before | After | +|------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Extractor::extract()`, `Reader::extract()` yielded `Rows` | they may also yield a `CorruptedRecordException`, one per record that cannot become a row; a direct caller routes or refuses it (`instanceof Rows`) | +| a reader refused a corrupted record itself | the run routes it to `withCorruptedRecords($loader)`, loaded, closed and discarded like the run's other loaders; without a loader it fails the run, whatever the `ErrorHandler` - a corrupted record is not an extraction error | +| `ExtractionError::$extractor` was an `Extractor` | `Extractor\|Reader`: a task's source is a `Reader` | +| `StepRunner::steps()`, `run()`: `?Extractor $source` | `Extractor\|Reader\|null $source`; a source implementing `CorruptedRecordsRouter` routes its corrupted records | + +### 127) `flow-php/etl`, `flow-php/etl-adapter-csv`, `-json`, `-excel`, `-google-sheet` - a record that cannot become a row throws + +| Before | After | +|-----------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| a CSV line or a sheet row with fewer cells than the header was padded with nulls | `CorruptedRecordException`; `->withMissingColumnsAsNull(true)` pads it (CSV, Google Sheet), Excel pads by default (`->withMissingColumnsAsNull(false)` refuses) | +| a CSV line or an Excel row with more cells than the header lost them | `CorruptedRecordException` for a non-empty extra cell; an empty one is dropped | +| Google Sheet `->withDropExtraColumns(bool)` | removed: a non-empty extra cell is a corrupted record; `->withMissingColumnsAsNull(bool)` pads a short row | +| an empty CSV line was a row of nulls | skipped | +| a value that does not cast to the type `->withSchema()` declares threw `SchemaMismatchException` | `CorruptedRecordException` (route it with `->withCorruptedRecords($loader)`) | +| a value or a JSON key a file shows only after the inference sample: `SchemaMismatchException`, or the key was dropped | `InferredSchemaException::outsideTheSample()`, naming the file, offset, line and sample size; `from_array()` and sequences throw `InferredSchemaException` | +| a `->withSchema()` column missing from a CSV, Excel or Google Sheet header was null on every row | `SchemaValidationException` | +| `new RecordDecoder($withHeader, $emptyToNull, $dropExtraColumns, $headers)`, `decode(list $batch): list` | `new RecordDecoder($withHeader, $emptyToNull, $missingColumnsAsNull, $headers)`, `decode(array $cells): null\|array\|string` (null: header or empty record, string: why it is corrupted) | +| `CSVDecoder::decode(list $lines)` | `CSVDecoder::decode(CSVDecoder::cells(string $line))`, one record | +| `CSVOpenSource::batches($schema, $batchSize, $backend)` | `+ ?int $sampleSize = null`; `JsonOpenSource::batches()` also `+ ?Schema $known = null`; both yield `CorruptedRecordException\|Rows` | +| `to_date_time(null)` threw `InvalidArgumentException` | returns null; a null format or time zone still throws | + ## Upgrading from 0.45.x to 0.46.x ### 1) `flow-php/etl-adapter-postgresql`, `flow-php/etl-adapter-doctrine` - keyset reads reject NULL keys up front diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVDecoder.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVDecoder.php index ce265bfc0d..a1919f581f 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVDecoder.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVDecoder.php @@ -6,7 +6,6 @@ use Flow\ETL\IO\Extractor\Grid\RecordDecoder; -use function array_map; use function str_getcsv; final class CSVDecoder @@ -23,24 +22,29 @@ public function __construct( private string $escape = '\\', bool $emptyToNull = true, ?array $headers = null, + bool $missingColumnsAsNull = false, ) { - $this->records = new RecordDecoder($withHeader, $emptyToNull, headers: $headers); + $this->records = new RecordDecoder($withHeader, $emptyToNull, $missingColumnsAsNull, $headers); } /** - * @param list $batch + * @return list + */ + public function cells(string $line): array + { + /** @var list */ + return str_getcsv($line, $this->separator, $this->enclosure, $this->escape); + } + + /** + * @param list $cells * - * @return list> + * @return null|array|string see RecordDecoder::decode() */ - public function decode(array $batch): array + public function decode(array $cells): array|string|null { - /** @var list> */ - return $this->records->decode(array_map(fn(string $line): array => str_getcsv( - $line, - $this->separator, - $this->enclosure, - $this->escape, - ), $batch)); + /** @var null|array|string */ + return $this->records->decode($cells); } /** diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVExtractor.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVExtractor.php index 6ed1067e5a..4bea1f9e26 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVExtractor.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVExtractor.php @@ -5,9 +5,11 @@ namespace Flow\ETL\Adapter\CSV\IO\Extractor; use Flow\ETL\Adapter\CSV\IO\Inference\CSVSniffer; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; use Flow\ETL\Functions\ScalarFunction; +use Flow\ETL\IO\CorruptedRecordsRouter; use Flow\ETL\IO\Extractor; use Flow\ETL\IO\Extractor\BatchableExtractor; use Flow\ETL\IO\Extractor\Criteria; @@ -29,6 +31,7 @@ use Flow\ETL\IO\Inference\SchemaInference; use Flow\ETL\IO\Inference\SchemaInferenceBuilder; use Flow\ETL\IO\Inference\SchemaInferrer; +use Flow\ETL\IO\Loader; use Flow\ETL\Values\BatchSize; use Flow\ETL\Values\Cardinality; use Flow\ETL\Values\Column\Backend; @@ -47,6 +50,7 @@ final class CSVExtractor implements FileExtractor, InfersSchema, MetadataColumnsExtractor, + CorruptedRecordsRouter, SplittableExtractor { private BatchSize $batchSize; @@ -70,6 +74,8 @@ final class CSVExtractor implements * The source the inferred header was read from; '' until an inference ran. */ + private ?Loader $corruptedRecordsLoader = null; + private ?Schema $schema = null; private ?Statistics $statistics = null; @@ -107,8 +113,13 @@ public function isRepeatable(): bool return true; } + public function corruptedRecordsLoader(): ?Loader + { + return $this->corruptedRecordsLoader; + } + /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { @@ -132,7 +143,9 @@ public function reader(): FileSplitReader $this->derived->from(), ) : null, + $this->schema === null ? $this->inference->sampleSize : null, ), + $this->corruptedRecordsLoader, ); } @@ -285,6 +298,13 @@ public function withCharactersReadInLine(int $charactersReadInLine): self return $this; } + public function withCorruptedRecords(?Loader $loader): static + { + $this->corruptedRecordsLoader = $loader; + + return $this; + } + public function withEmptyToNull(bool $emptyToNull): self { $this->derived = new DerivedFromFiles(); @@ -336,6 +356,16 @@ public function withMetadataColumns(bool $addMetadataColumns): static return $this; } + public function withMissingColumnsAsNull(bool $missingColumnsAsNull): self + { + $this->derived = new DerivedFromFiles(); + $this->sampled = null; + $this->statistics = null; + $this->readOptions = $this->readOptions->withMissingColumnsAsNull($missingColumnsAsNull); + + return $this; + } + public function withSchema(Schema $schema): static { $this->schema = $schema; diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVFileBatches.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVFileBatches.php index 928b3c417f..194b32a1ee 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVFileBatches.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVFileBatches.php @@ -6,7 +6,9 @@ use Flow\ETL\Foundation\Describe\Description; use Flow\ETL\Foundation\Describe\Rebuild; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; +use Flow\ETL\Foundation\Exception\SchemaValidationException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\File\FileBatches; use Flow\ETL\IO\Extractor\File\FileRead; @@ -20,6 +22,7 @@ use SensitiveParameter; use function Flow\Types\DSL\type_array; +use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_optional; final readonly class CSVFileBatches implements FileBatches @@ -27,9 +30,13 @@ /** * @param null|InferredColumns $inferred the columns every file must carry; null when declared or union by name */ + /** + * @param null|int $sampleSize the inference's sample size when the schema was inferred; null when it was declared + */ public function __construct( private CSVSourceOpener $opener, private ?InferredColumns $inferred, + private ?int $sampleSize = null, ) {} public static function fromDescription(#[SensitiveParameter] array $data, Rebuild $rebuild): static @@ -39,6 +46,7 @@ public static function fromDescription(#[SensitiveParameter] array $data, Rebuil return new self( $rebuild->of($data['opener'] ?? null, CSVSourceOpener::class), $inferred === null ? null : InferredColumns::fromArray($inferred), + type_optional(type_integer())->assert($data['sample_size'] ?? null), ); } @@ -47,6 +55,7 @@ public function describe(): ?Description return Description::compose(self::class, [ 'opener' => $this->opener, 'inferred' => $this->inferred?->normalize(), + 'sample_size' => $this->sampleSize, ]); } @@ -55,9 +64,9 @@ public function describe(): ?Description * the file's first batch is built, so a cell the inferred types refuse cannot hide a file whose columns diverge. * A split reads only its byte window of the file. * - * @throws InferredSchemaException + * @throws InferredSchemaException|SchemaValidationException * - * @return Generator + * @return Generator */ public function batches( FileRead $read, @@ -72,7 +81,25 @@ public function batches( : $this->opener->openSplit($source, $criteria->split, $this->inferred); try { - yield from $open->batches($body, $read->batchSize, $backend); + // the header is resolved by the time the first batch or corrupted record is handed out + $checked = $this->sampleSize !== null; + + foreach ($open->batches($body, $read->batchSize, $backend, $this->sampleSize) as $batch) { + if (!$checked) { + $checked = true; + $missing = $body->missingFrom($open->headers()); + + if ($missing !== []) { + throw SchemaValidationException::declaredColumnsMissing( + $source->uri(), + $missing, + $open->headers(), + ); + } + } + + yield $batch; + } } finally { $open->close(); } diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVOpenSource.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVOpenSource.php index e0911b2e79..e07a3384c9 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVOpenSource.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVOpenSource.php @@ -4,6 +4,8 @@ namespace Flow\ETL\Adapter\CSV\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; +use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\SchemaMismatchException; use Flow\ETL\IO\Inference\ColumnTypes; use Flow\ETL\Values\Column\Backend; @@ -44,16 +46,17 @@ public function producedRows(): int; public function columns(): array; /** - * Batches of exactly $batchSize rows (the last may be shorter), keyed and ordered by $schema; row indexes in - * refusals are relative to the source. This instance is consumed afterwards. + * Batches of exactly $batchSize rows (the last may be shorter), keyed and ordered by $schema, and every corrupted + * record handed up in record order. This instance is consumed afterwards. * * @param int<1, max> $batchSize + * @param null|int $sampleSize the inference's sample size when $schema was inferred; null when it was declared * - * @throws SchemaMismatchException + * @throws InferredSchemaException|SchemaMismatchException a refused record under an inferred schema * - * @return Iterator + * @return Iterator */ - public function batches(Schema $schema, int $batchSize, Backend $backend): Iterator; + public function batches(Schema $schema, int $batchSize, Backend $backend, ?int $sampleSize = null): Iterator; /** * The header records() resolved; [] before it ran or for a 0-byte source. @@ -63,7 +66,7 @@ public function batches(Schema $schema, int $batchSize, Backend $backend): Itera public function headers(): array; /** - * This instance is consumed afterwards. + * The rows of the records that are not corrupted. This instance is consumed afterwards. * * @return Iterator> */ diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVReadOptions.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVReadOptions.php index f3c318569f..8063bf1a73 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVReadOptions.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVReadOptions.php @@ -25,6 +25,7 @@ public function __construct( public ?string $enclosure = null, public ?string $escape = null, public ?int $charactersReadInLine = null, + public bool $missingColumnsAsNull = false, ) { CSVDialect::assertParts($separator, $enclosure, $escape); } @@ -42,6 +43,7 @@ public static function fromArray(array $data): self type_optional(type_string())->assert($data['enclosure'] ?? null), type_optional(type_string())->assert($data['escape'] ?? null), self::positive(type_optional(type_integer())->assert($data['characters_read_in_line'] ?? null)), + type_boolean()->assert($data['missing_columns_as_null'] ?? null), ); } @@ -54,7 +56,7 @@ public static function positive(?int $value): ?int } /** - * @return array{with_header: bool, empty_to_null: bool, remove_bom: bool, separator: null|string, enclosure: null|string, escape: null|string, characters_read_in_line: null|int} + * @return array{with_header: bool, empty_to_null: bool, remove_bom: bool, separator: null|string, enclosure: null|string, escape: null|string, characters_read_in_line: null|int, missing_columns_as_null: bool} */ public function normalize(): array { @@ -66,6 +68,7 @@ public function normalize(): array 'enclosure' => $this->enclosure, 'escape' => $this->escape, 'characters_read_in_line' => $this->charactersReadInLine, + 'missing_columns_as_null' => $this->missingColumnsAsNull, ]; } @@ -82,6 +85,7 @@ public function withCharactersReadInLine(int $charactersReadInLine): self enclosure: $this->enclosure, escape: $this->escape, charactersReadInLine: $charactersReadInLine, + missingColumnsAsNull: $this->missingColumnsAsNull, ); } @@ -95,6 +99,7 @@ public function withEmptyToNull(bool $emptyToNull): self enclosure: $this->enclosure, escape: $this->escape, charactersReadInLine: $this->charactersReadInLine, + missingColumnsAsNull: $this->missingColumnsAsNull, ); } @@ -108,6 +113,7 @@ public function withEnclosure(string $enclosure): self enclosure: $enclosure, escape: $this->escape, charactersReadInLine: $this->charactersReadInLine, + missingColumnsAsNull: $this->missingColumnsAsNull, ); } @@ -121,6 +127,7 @@ public function withEscape(string $escape): self enclosure: $this->enclosure, escape: $escape, charactersReadInLine: $this->charactersReadInLine, + missingColumnsAsNull: $this->missingColumnsAsNull, ); } @@ -134,6 +141,21 @@ public function withHeader(bool $withHeader): self enclosure: $this->enclosure, escape: $this->escape, charactersReadInLine: $this->charactersReadInLine, + missingColumnsAsNull: $this->missingColumnsAsNull, + ); + } + + public function withMissingColumnsAsNull(bool $missingColumnsAsNull): self + { + return new self( + withHeader: $this->withHeader, + emptyToNull: $this->emptyToNull, + removeBOM: $this->removeBOM, + separator: $this->separator, + enclosure: $this->enclosure, + escape: $this->escape, + charactersReadInLine: $this->charactersReadInLine, + missingColumnsAsNull: $missingColumnsAsNull, ); } @@ -147,6 +169,7 @@ public function withRemoveBOM(bool $removeBOM): self enclosure: $this->enclosure, escape: $this->escape, charactersReadInLine: $this->charactersReadInLine, + missingColumnsAsNull: $this->missingColumnsAsNull, ); } @@ -160,6 +183,7 @@ public function withSeparator(string $separator): self enclosure: $this->enclosure, escape: $this->escape, charactersReadInLine: $this->charactersReadInLine, + missingColumnsAsNull: $this->missingColumnsAsNull, ); } } diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVSourceOpener.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVSourceOpener.php index 17c6a8163f..ab168e9069 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVSourceOpener.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVSourceOpener.php @@ -204,6 +204,7 @@ public function over( $removeBOM, $charactersReadInLine, $headers, + $this->options->missingColumnsAsNull, ); } @@ -216,6 +217,7 @@ public function over( escape: $dialect->escape, emptyToNull: $this->options->emptyToNull, headers: $headers, + missingColumnsAsNull: $this->options->missingColumnsAsNull, ), new CSVLineReader( $dialect->enclosure, diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/PhpCSVOpenSource.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/PhpCSVOpenSource.php index 349a281813..2d8616477c 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/PhpCSVOpenSource.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/PhpCSVOpenSource.php @@ -4,15 +4,21 @@ namespace Flow\ETL\Adapter\CSV\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Inference\ColumnTypes; use Flow\ETL\Values\Column\Backend; use Flow\ETL\Values\Rows; use Flow\ETL\Values\Rows\RecordBatch; use Flow\ETL\Values\Schema; use Flow\Filesystem\SourceStream; +use Flow\Filesystem\Stream\RangeSourceStream; use Flow\Types\Type\TypeNarrower; use Generator; +use function is_array; +use function is_string; +use function substr_count; + final class PhpCSVOpenSource implements CSVOpenSource { /** @@ -32,25 +38,52 @@ public function __construct( ) {} /** - * @return Generator + * @return Generator */ - public function batches(Schema $schema, int $batchSize, Backend $backend): Generator + public function batches(Schema $schema, int $batchSize, Backend $backend, ?int $sampleSize = null): Generator { - $batch = new RecordBatch($schema, $backend, $batchSize); + $uri = $this->stream->path()->uri(); + $position = $this->stream instanceof RangeSourceStream ? $this->stream->start() : 0; + // a window cut past the file's start cannot know how many lines came before it + $line = $position === 0 ? 1 : null; + $batch = new RecordBatch($schema, $backend, $batchSize, $uri, $sampleSize); + + foreach ($this->lineReader->readLines($this->stream) as $record) { + $offset = $position; + $recordLine = $line; + $position += $this->lineReader->lastRecordBytes(); + + if ($line !== null) { + $line += 1 + substr_count($record, "\n"); + } - foreach ($this->records() as $values) { - $rows = $batch->add($values); + $cells = $this->decoder->cells($record); + $values = $this->decoder->decode($cells); - if ($rows !== null) { - yield $rows; + if ($values === null) { + continue; + } + + if (is_string($values)) { + yield from $batch->refuse(CorruptedRecordException::fromCells( + $uri, + $offset, + $recordLine, + $values, + $cells, + $record, + )); + + continue; } - } - $rest = $batch->flush(); + $this->producedBytes += $this->lineReader->lastRecordBytes(); + $this->producedRows++; - if ($rest !== null) { - yield $rest; + yield from $batch->add($values, $offset, $recordLine, $record, $cells); } + + yield from $batch->flush(); } public function close(): void @@ -66,9 +99,11 @@ public function close(): void public function columns(): array { foreach ($this->lineReader->readLines($this->stream) as $line) { - $this->decoder->decode([$line]); + $this->decoder->decode($this->decoder->cells($line)); - break; + if ($this->decoder->headers() !== null) { + break; + } } return $this->decoder->headers() ?? []; @@ -88,12 +123,16 @@ public function headers(): array public function records(): Generator { foreach ($this->lineReader->readLines($this->stream) as $line) { - foreach ($this->decoder->decode([$line]) as $values) { - $this->producedBytes += $this->lineReader->lastRecordBytes(); - $this->producedRows++; + $values = $this->decoder->decode($this->decoder->cells($line)); - yield $values; + if (!is_array($values)) { + continue; } + + $this->producedBytes += $this->lineReader->lastRecordBytes(); + $this->producedRows++; + + yield $values; } } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php index 9723166e84..564be2c420 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFixtureContext.php @@ -14,6 +14,8 @@ use Flow\ETL\Adapter\CSV\IO\Extractor\PhpCSVOpenSource; use Flow\ETL\Adapter\CSV\IO\Extractor\RustCSVOpenSource; use Flow\ETL\Adapter\CSV\IO\Inference\CSVSniffer; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; +use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\SchemaMismatchException; use Flow\ETL\IO\Extractor\File\SourceFile; use Flow\ETL\IO\Inference\ColumnTypes; @@ -41,6 +43,7 @@ use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; use function Flow\Filesystem\DSL\path_real; +use function in_array; use function iterator_to_array; use function range; use function serialize; @@ -103,6 +106,32 @@ public static function fixtures(): Generator } } + /** + * fixtures() without the ones holding a corrupted record, which a read refuses. + * + * @return Generator + */ + public static function wellFormedFixtures(): Generator + { + foreach (self::fixtures() as $name => $fixture) { + if (!in_array( + $name, + [ + 'corrupted_row.csv', + 'file_with_empty_columns.csv', + 'file_with_empty_headers.csv', + 'more_columns_than_headers.csv', + 'more_headers_than_columns.csv', + 'ragged.csv', + 'single_quotes_csv.csv', + ], + true, + )) { + yield $name => $fixture; + } + } + } + /** * The dialect CSVSourceOpener resolves: pinned options first, detection for the rest. */ @@ -146,6 +175,7 @@ public static function openNativeStream( $options->emptyToNull, $options->removeBOM, $options->charactersReadInLine, + missingColumnsAsNull: $options->missingColumnsAsNull, ); } @@ -171,6 +201,7 @@ public static function openPhpStream( enclosure: $dialect->enclosure, escape: $dialect->escape, emptyToNull: $options->emptyToNull, + missingColumnsAsNull: $options->missingColumnsAsNull, ), new CSVLineReader( $dialect->enclosure, @@ -195,23 +226,34 @@ public static function content(string $content): SourceStream * * @param int<1, max> $batchSize * - * @return list>|SchemaMismatchException + * @return InferredSchemaException|list}|array>>|SchemaMismatchException + * rows, and a corrupted record as ['corrupted' => its identity] */ public static function batches( CSVOpenSource $open, Schema $schema, int $batchSize, Backend $backend = new PhpBackend(), - ): array|SchemaMismatchException { + ?int $sampleSize = null, + ): array|InferredSchemaException|SchemaMismatchException { $batches = []; try { - foreach ($open->batches($schema, $batchSize, $backend) as $batch) { - $batches[] = $batch->toArray(); + foreach ($open->batches($schema, $batchSize, $backend, $sampleSize) as $batch) { + $batches[] = $batch instanceof CorruptedRecordException + ? ['corrupted' => [ + $batch->uri, + $batch->offset, + $batch->lineNumber, + $batch->reason, + $batch->record, + $batch->fields, + ]] + : $batch->toArray(); } return $batches; - } catch (SchemaMismatchException $e) { + } catch (InferredSchemaException|SchemaMismatchException $e) { return $e; } finally { $open->close(); @@ -228,8 +270,13 @@ public static function batchesOutcome( Schema $schema, int $batchSize, Backend $backend = new PhpBackend(), + ?int $sampleSize = null, ): string { - $batches = self::batches($open, $schema, $batchSize, $backend); + $batches = self::batches($open, $schema, $batchSize, $backend, $sampleSize); + + if ($batches instanceof InferredSchemaException) { + return $batches::class . ': ' . $batches->getMessage(); + } return $batches instanceof SchemaMismatchException ? $batches::class diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVExtractorTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVExtractorTest.php index f6c4dc8ea2..6bb656dde0 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVExtractorTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVExtractorTest.php @@ -8,10 +8,11 @@ use DateTimeImmutable; use Flow\ETL\Adapter\CSV\IO\Extractor\CSVExtractor; use Flow\ETL\Adapter\CSV\Tests\Context\CSVFixtureContext; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; use Flow\ETL\Foundation\Exception\SchemaDefinitionNotUniqueException; -use Flow\ETL\Foundation\Exception\SchemaMismatchException; +use Flow\ETL\Foundation\Exception\SchemaValidationException; use Flow\ETL\Functions\ScalarFunction; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\File\FileSplit; @@ -33,6 +34,7 @@ use PHPUnit\Framework\Attributes\TestWith; use RuntimeException; +use function array_column; use function array_filter; use function array_keys; use function array_map; @@ -49,6 +51,7 @@ use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\df; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_all; use function Flow\ETL\DSL\from_array; @@ -63,6 +66,7 @@ use function Flow\ETL\DSL\schema_metadata; use function Flow\ETL\DSL\schema_to_ascii; use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\to_array; use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\native_local_filesystem; use function Flow\Filesystem\DSL\path; @@ -80,6 +84,74 @@ final class CSVExtractorTest extends FlowTestCase { + #[TestWith(["a,b,c\n1,2,3\n4,5,6,7,8\n", '4,5,6,7,8', ['4', '5', '6', '7', '8'], '5 fields, the header has 3'])] + #[TestWith(["a,b,c\n1,2,3\n4,5\n", '4,5', ['4', '5'], '2 fields, the header has 3'])] + /** + * @param list $fields + */ + public function test_a_record_whose_field_count_differs_from_the_header_is_refused( + string $content, + string $record, + array $fields, + string $reason, + ): void { + try { + ExtractedRows::of(from_csv('memory://source.csv', filesystem: CSVFixtureContext::memory($content))); + static::fail('Expected the corrupted record to be refused'); + } catch (CorruptedRecordException $e) { + static::assertSame(['memory://source.csv', 12, 3, $reason, $record, $fields], [ + $e->uri, + $e->offset, + $e->lineNumber, + $e->reason, + $e->record, + $e->fields, + ]); + } + } + + public function test_an_empty_line_is_skipped(): void + { + $rows = ExtractedRows::of(from_csv( + 'memory://source.csv', + filesystem: CSVFixtureContext::memory("a,b,c\n1,2,3\n\n4,5,6\n"), + )); + + static::assertSame([1, 4], $rows->reduceToArray('a')); + } + + public function test_a_value_that_does_not_cast_to_the_declared_type_is_a_corrupted_record(): void + { + try { + ExtractedRows::of(from_csv( + 'memory://source.csv', + filesystem: CSVFixtureContext::memory("id,amount\n1,10.5\n2,N/A\n3,7\n"), + )->withSchema(schema(int_schema('id'), float_schema('amount', true)))); + static::fail('Expected the corrupted record to be refused'); + } catch (CorruptedRecordException $e) { + static::assertSame(['memory://source.csv', 17, 3, '2,N/A', ['2', 'N/A']], [ + $e->uri, + $e->offset, + $e->lineNumber, + $e->record, + $e->fields, + ]); + static::assertStringContainsString('"amount"', $e->reason); + static::assertStringContainsString('N/A', $e->reason); + } + } + + public function test_a_declared_column_the_header_lacks_is_a_schema_error(): void + { + $this->expectException(SchemaValidationException::class); + $this->expectExceptionMessage('"x"'); + + ExtractedRows::of(from_csv( + 'memory://source.csv', + filesystem: CSVFixtureContext::memory("a,b,c\n1,2,3\n"), + )->withSchema(schema(int_schema('a'), int_schema('b'), int_schema('x', true)))); + } + public function test_an_empty_glob_refuses_to_infer_a_schema(): void { $this->expectException(InferredSchemaException::class); @@ -112,7 +184,7 @@ public function test_an_empty_glob_with_a_declared_schema_answers_it_and_extract static::assertEquals(schema(int_schema('id')), $extractor->schema()); static::assertSame( [], - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()), false), + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false), ); } @@ -144,7 +216,7 @@ public function test_declared_partition_types_reach_the_rows(): void ->withSchema(schema(int_schema('id'), str_schema('value'))) ->partitionTypes(partition_types(group: type_integer())); - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { static::assertEquals($extractor->schema(), $rows->schema()); static::assertSame(1, $rows->column('group')->value(0)); @@ -160,7 +232,7 @@ public function test_partition_columns_are_typed_without_a_declared_schema(): vo partition_types(group: type_integer()), ); - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { static::assertSame(1, $rows->column('group')->value(0)); return; @@ -252,7 +324,11 @@ public function test_every_batch_carries_the_inferred_schema(): void $schema = $extractor->schema(); $sizes = []; - foreach ($extractor->withBatchSize(3)->extract(flow_context()->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor->withBatchSize(3), + flow_context()->backend(), + new Criteria(), + ) as $rows) { static::assertTrue($schema->isSame($rows->schema())); $sizes[] = $rows->count(); @@ -275,7 +351,7 @@ public function test_a_declared_schema_opens_each_file_once(): void )); $extractor->schema(); - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())); static::assertSame(1, $counting->readFromCalls); } @@ -379,7 +455,7 @@ public function test_the_inferred_schema_is_memoised_across_schema_and_extract() $extractor = from_csv(CSVFixtureContext::path('two_rows.csv'), filesystem: $counting); $extractor->schema(); - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())); static::assertSame(3, $counting->readFromCalls, 'header() + sample + read'); static::assertSame(3, $counting->closedStreams()); @@ -407,7 +483,10 @@ public function test_an_explain_before_the_run_lists_and_infers_no_second_time() public function test_a_shape_changing_setter_drops_the_inferred_schema(Closure $setter): void { $counting = new CountingFilesystem(new NativeLocalFilesystem()); - $extractor = from_csv(CSVFixtureContext::path('file_with_empty_columns.csv'), filesystem: $counting); + $extractor = from_csv( + CSVFixtureContext::path('file_with_empty_columns.csv'), + filesystem: $counting, + )->withMissingColumnsAsNull(true); $extractor->schema(); $cold = $counting->readFromCalls; @@ -435,7 +514,10 @@ public static function shapeChangingSetters(): Generator public function test_infer_schema_drops_the_inferred_schema_and_all_strings_floors_it(): void { $counting = new CountingFilesystem(new NativeLocalFilesystem()); - $extractor = from_csv(CSVFixtureContext::path('file_with_empty_columns.csv'), filesystem: $counting); + $extractor = from_csv( + CSVFixtureContext::path('file_with_empty_columns.csv'), + filesystem: $counting, + )->withMissingColumnsAsNull(true); $extractor->schema(); $cold = $counting->readFromCalls; @@ -453,7 +535,10 @@ public function test_infer_schema_drops_the_inferred_schema_and_all_strings_floo public function test_without_header_generates_the_column_names(): void { - $schema = from_csv(CSVFixtureContext::path('file_with_empty_columns.csv'))->withHeader(false)->schema(); + $schema = from_csv(CSVFixtureContext::path('file_with_empty_columns.csv')) + ->withMissingColumnsAsNull(true) + ->withHeader(false) + ->schema(); static::assertSame(['e00', 'e01', 'e02'], array_keys($schema->definitions())); @@ -522,13 +607,14 @@ public function test_sample_size_bounds_the_inferred_schema(): void public function test_a_value_past_the_sample_is_reported(): void { - $this->expectException(SchemaMismatchException::class); + $this->expectException(InferredSchemaException::class); - iterator_to_array( + iterator_to_array(FlowTestCase::extracted( from_csv(CSVFixtureContext::path('annual-enterprise-survey-2019-financial-year-provisional-csv.csv')) - ->inferSchema(infer_schema()->sampleSize(10)) - ->extract(flow_context(config())->backend(), new Criteria()), - ); + ->inferSchema(infer_schema()->sampleSize(10)), + flow_context(config())->backend(), + new Criteria(), + )); } public function test_files_to_sniff_bounds_the_sources_opened(): void @@ -547,11 +633,11 @@ public function test_columns_that_diverge_from_the_inferred_schema_are_rejected( $this->expectException(InferredSchemaException::class); $this->expectExceptionMessageMatches('/unexpected \[label\].+missing \[name\]/'); - iterator_to_array( - from_csv(CSVFixtureContext::path('columns_diverge/*.csv')) - ->inferSchema(infer_schema()->filesToSniff(1)) - ->extract(flow_context(config())->backend(), new Criteria()), - ); + iterator_to_array(FlowTestCase::extracted( + from_csv(CSVFixtureContext::path('columns_diverge/*.csv'))->inferSchema(infer_schema()->filesToSniff(1)), + flow_context(config())->backend(), + new Criteria(), + )); } /** @@ -567,7 +653,7 @@ public function test_union_by_name_reads_diverging_columns_as_one_wider_schema() $rows = []; - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $batch) { foreach ($batch->toArray() as $row) { $rows[] = $row; } @@ -586,11 +672,12 @@ public function test_union_by_name_reads_diverging_columns_as_one_wider_schema() public function test_the_divergence_message_names_both_sources(): void { try { - iterator_to_array( + iterator_to_array(FlowTestCase::extracted( from_csv(CSVFixtureContext::path('columns_diverge/*.csv')) - ->inferSchema(infer_schema()->filesToSniff(1)) - ->extract(flow_context(config())->backend(), new Criteria()), - ); + ->inferSchema(infer_schema()->filesToSniff(1)), + flow_context(config())->backend(), + new Criteria(), + )); static::fail('a divergent column set must throw'); } catch (InferredSchemaException $e) { static::assertStringContainsString('b.csv', $e->getMessage()); @@ -605,21 +692,23 @@ public function test_divergence_is_reported_before_a_refused_cell(): void $this->expectException(InferredSchemaException::class); $this->expectExceptionMessageMatches('/unexpected \[label\]/'); - iterator_to_array( + iterator_to_array(FlowTestCase::extracted( from_csv(CSVFixtureContext::path('columns_diverge_refused/*.csv')) - ->inferSchema(infer_schema()->filesToSniff(1)) - ->extract(flow_context(config())->backend(), new Criteria()), - ); + ->inferSchema(infer_schema()->filesToSniff(1)), + flow_context(config())->backend(), + new Criteria(), + )); } public function test_a_header_only_file_with_a_divergent_header_is_rejected(): void { try { - iterator_to_array( + iterator_to_array(FlowTestCase::extracted( from_csv(CSVFixtureContext::path('columns_diverge_header_only/*.csv')) - ->inferSchema(infer_schema()->filesToSniff(1)) - ->extract(flow_context(config())->backend(), new Criteria()), - ); + ->inferSchema(infer_schema()->filesToSniff(1)), + flow_context(config())->backend(), + new Criteria(), + )); static::fail('a divergent header-only source must throw'); } catch (InferredSchemaException $e) { static::assertMatchesRegularExpression('/unexpected \[label\].+missing \[name\]/', $e->getMessage()); @@ -637,7 +726,7 @@ public function test_a_header_only_file_infers_every_column_as_string(): void static::assertSame('string', $extractor->schema()->get('name')->type()->toString()); static::assertSame( [], - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())), + iterator_to_array(FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())), ); } @@ -647,7 +736,7 @@ public function test_an_empty_file_refuses_to_infer_a_schema_and_extracts_nothin static::assertSame( [], - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())), + iterator_to_array(FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())), ); $this->expectException(InferredSchemaException::class); @@ -690,13 +779,15 @@ public function test_a_glob_of_an_empty_and_a_header_only_file_infers_the_header static::assertSame(['id', 'name'], array_keys($extractor->schema()->definitions())); static::assertSame( [], - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())), + iterator_to_array(FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())), ); } public function test_empty_cells_are_not_type_evidence(): void { - $schema = from_csv(CSVFixtureContext::path('file_with_empty_columns.csv'))->schema(); + $schema = from_csv(CSVFixtureContext::path('file_with_empty_columns.csv')) + ->withMissingColumnsAsNull(true) + ->schema(); static::assertSame('integer', $schema->get('id')->type()->toString()); static::assertSame('string', $schema->get('name')->type()->toString()); @@ -705,7 +796,10 @@ public function test_empty_cells_are_not_type_evidence(): void public function test_without_empty_to_null_empty_cells_are_string_evidence(): void { - $schema = from_csv(CSVFixtureContext::path('file_with_empty_columns.csv'))->withEmptyToNull(false)->schema(); + $schema = from_csv(CSVFixtureContext::path('file_with_empty_columns.csv')) + ->withMissingColumnsAsNull(true) + ->withEmptyToNull(false) + ->schema(); static::assertSame(['id', 'name', 'active'], array_keys($schema->definitions())); static::assertSame('string', $schema->get('id')->type()->toString()); @@ -720,7 +814,7 @@ public function test_partition_columns_are_not_typed_by_the_sample(): void static::assertSame('string', $extractor->schema()->get('group')->type()->toString()); - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { static::assertSame('1', $rows->toArray()[0]['group']); break; @@ -741,17 +835,25 @@ public function test_a_stream_is_closed_when_the_read_stops_early(string $mode): )->inferSchema(infer_schema()->sampleSize(10)); try { - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted( + $extractor, + flow_context(config())->backend(), + new Criteria(), + )); static::fail('a value past the sample must stop the read'); - } catch (SchemaMismatchException) { + } catch (InferredSchemaException) { } } else { $extractor = from_csv(CSVFixtureContext::path('orders_flow.csv'), filesystem: $counting); if ($mode === 'limit') { - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria(limit: 2))); + iterator_to_array(FlowTestCase::extracted( + $extractor, + flow_context(config())->backend(), + new Criteria(limit: 2), + )); } else { - $rows = $extractor->extract(flow_context(config())->backend(), new Criteria()); + $rows = FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()); $rows->current(); $rows->send(Signal::STOP); } @@ -812,7 +914,8 @@ public function test_bom_removal_utf16_be(): void ], array_map( static fn(Rows $r) => $r->toArray(), - iterator_to_array($extractor->extract( + iterator_to_array(FlowTestCase::extracted( + $extractor, flow_context(config_builder()->build())->backend(), new Criteria(), )), @@ -839,7 +942,8 @@ public function test_bom_removal_utf16_le(): void ], array_map( static fn(Rows $r) => $r->toArray(), - iterator_to_array($extractor->extract( + iterator_to_array(FlowTestCase::extracted( + $extractor, flow_context(config_builder()->build())->backend(), new Criteria(), )), @@ -870,7 +974,8 @@ public function test_bom_removal_utf32_be(): void ], array_map( static fn(Rows $r) => $r->toArray(), - iterator_to_array($extractor->extract( + iterator_to_array(FlowTestCase::extracted( + $extractor, flow_context(config_builder()->build())->backend(), new Criteria(), )), @@ -901,7 +1006,8 @@ public function test_bom_removal_utf32_le(): void ], array_map( static fn(Rows $r) => $r->toArray(), - iterator_to_array($extractor->extract( + iterator_to_array(FlowTestCase::extracted( + $extractor, flow_context(config_builder()->build())->backend(), new Criteria(), )), @@ -929,7 +1035,8 @@ public function test_bom_removal_utf8(): void ], array_map( static fn(Rows $r) => $r->toArray(), - iterator_to_array($extractor->extract( + iterator_to_array(FlowTestCase::extracted( + $extractor, flow_context(config_builder()->build())->backend(), new Criteria(), )), @@ -975,7 +1082,9 @@ public function test_extracting_csv_empty_columns_as_empty_strings(): void $extractor = from_csv( $path = path_real(__DIR__ . '/../../../Fixtures/file_with_empty_columns.csv'), empty_to_null: false, - )->withMetadataColumns(true); + ) + ->withMissingColumnsAsNull(true) + ->withMetadataColumns(true); static::assertSame( [ @@ -1012,7 +1121,7 @@ public function test_extracting_csv_empty_columns_as_null(): void ], ], ExtractedRows::of( - from_csv(__DIR__ . '/../../../Fixtures/file_with_empty_columns.csv'), + from_csv(__DIR__ . '/../../../Fixtures/file_with_empty_columns.csv')->withMissingColumnsAsNull(true), flow_context(config()), )->toArray(), ); @@ -1026,7 +1135,7 @@ public function test_extracting_csv_empty_headers(): void ['e00' => 1, 'name' => 'Norbert', 'active' => null], ], ExtractedRows::of( - from_csv(__DIR__ . '/../../../Fixtures/file_with_empty_headers.csv'), + from_csv(__DIR__ . '/../../../Fixtures/file_with_empty_headers.csv')->withMissingColumnsAsNull(true), flow_context(config()), )->toArray(), ); @@ -1112,7 +1221,7 @@ public function test_extracting_csv_files_without_header(): void $total = 0; - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { foreach ($rows->toArray() as $row) { static::assertSame( ['e00', 'e01', 'e02', 'e03', 'e04', 'e05', 'e06', 'e07', 'e08', 'e09'], @@ -1127,38 +1236,28 @@ public function test_extracting_csv_files_without_header(): void public function test_extracting_csv_with_corrupted_row(): void { - static::assertSame( - 3, - df() - ->read(from_csv(__DIR__ . '/../../../Fixtures/corrupted_row.csv')) - ->fetch() - ->count(), - ); + $this->expectException(CorruptedRecordException::class); + $this->expectExceptionMessage('5 fields, the header has 2'); + + df()->read(from_csv(__DIR__ . '/../../../Fixtures/corrupted_row.csv'))->fetch(); } public function test_extracting_csv_with_more_columns_than_headers(): void { - $extractor = from_csv(__DIR__ . '/../../../Fixtures/more_columns_than_headers.csv'); - - $total = 0; + $this->expectException(CorruptedRecordException::class); + $this->expectExceptionMessage('3 fields, the header has 2'); - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { - foreach ($rows->toArray() as $row) { - static::assertSame(['id', 'name'], array_keys($row)); - } - $total += $rows->count(); - } - - static::assertSame(1, $total); + ExtractedRows::of(from_csv(__DIR__ . '/../../../Fixtures/more_columns_than_headers.csv')); } public function test_extracting_csv_with_more_headers_than_columns(): void { - $extractor = from_csv(path_real(__DIR__ . '/../../../Fixtures/more_headers_than_columns.csv')); + $extractor = from_csv(path_real(__DIR__ . '/../../../Fixtures/more_headers_than_columns.csv')) + ->withMissingColumnsAsNull(true); $total = 0; - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { foreach ($rows->toArray() as $row) { static::assertSame(['id', 'name', 'active'], array_keys($row)); } @@ -1235,7 +1334,11 @@ public function test_schema_and_extract_agree_on_partition_columns(): void static::assertSame(['id', 'value', 'group'], $extractor->schema()->names()); - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { static::assertEquals($extractor->schema(), $rows->schema()); } } @@ -1292,8 +1395,11 @@ public function test_partition_columns_are_not_leaking_between_streams(): void public function test_signal_stop(): void { - $generator = from_csv(path_real(__DIR__ . '/../../../Fixtures/orders_flow.csv')) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_csv(path_real(__DIR__ . '/../../../Fixtures/orders_flow.csv')), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->next(); @@ -1326,7 +1432,8 @@ public function test_without_bom_removal_utf8(): void ], array_map( static fn(Rows $r) => $r->toArray(), - iterator_to_array($extractor->extract( + iterator_to_array(FlowTestCase::extracted( + $extractor, flow_context(config_builder()->build())->backend(), new Criteria(), )), @@ -1416,7 +1523,7 @@ public function test_an_escaped_quote_read_without_an_escape_character_closes_th ); } - #[DataProviderExternal(CSVFixtureContext::class, 'fixtures')] + #[DataProviderExternal(CSVFixtureContext::class, 'wellFormedFixtures')] public function test_record_counts_match_fgetcsv_for_every_fixture(string $fixture): void { $path = CSVFixtureContext::path($fixture); @@ -1589,7 +1696,8 @@ public function test_extract_split_stops_on_the_stop_signal(): void filesystem: CSVFixtureContext::memory(CSVFixtureContext::tenByteRows(30)), ); $extractor->withBatchSize(1); - $generator = $extractor->extract( + $generator = FlowTestCase::extracted( + $extractor, flow_context()->backend(), (new Criteria())->withSplit($extractor->splits(new Criteria(), 100)[1]), ); @@ -1612,7 +1720,8 @@ public function test_extract_split_reads_no_more_than_the_limit(): void $extractor->withBatchSize(2); $total = 0; - foreach ($extractor->extract( + foreach (FlowTestCase::extracted( + $extractor, flow_context()->backend(), (new Criteria(3))->withSplit($extractor->splits(new Criteria(), 100)[1]), ) as $rows) { @@ -1790,4 +1899,73 @@ public function test_a_partition_filter_never_opens_the_files_of_other_partition static::assertSame(['file_01.csv', 'file_02.csv'], $read); static::assertSame([1, 2, 3, 4], $rows->reduceToArray('id')); } + + public function test_corrupted_records_are_routed_to_the_reject_loader_and_the_read_goes_on(): void + { + $rejects = []; + + $rows = df() + ->read( + from_csv( + 'memory://source.csv', + filesystem: CSVFixtureContext::memory("id,name\n1,a\n2,b,c\nx,d\n4,e\n"), + ) + ->withSchema(schema(int_schema('id'), str_schema('name'))) + ->withCorruptedRecords(to_array($rejects)), + ) + ->fetch(); + + static::assertSame([['id' => 1, 'name' => 'a'], ['id' => 4, 'name' => 'e']], $rows->toArray()); + static::assertSame( + [ + [ + 'uri' => 'memory://source.csv', + 'offset' => 12, + 'line' => 3, + 'reason' => '3 fields, the header has 2', + 'record' => '2,b,c', + 'fields' => ['2', 'b', 'c'], + ], + [ + 'uri' => 'memory://source.csv', + 'offset' => 18, + 'line' => 4, + 'reason' => 'column "id": could not convert \'x\' (string) to integer', + 'record' => 'x,d', + 'fields' => ['x', 'd'], + ], + ], + $rejects, + ); + } + + public function test_a_limit_counts_rows_and_never_the_corrupted_records_routed_before_them(): void + { + $rejects = []; + + $rows = df() + ->read( + from_csv('memory://source.csv', filesystem: CSVFixtureContext::memory("id\nx\n1\ny\n2\n3\n")) + ->withSchema(schema(int_schema('id'))) + ->withCorruptedRecords(to_array($rejects)), + ) + ->limit(2) + ->fetch(); + + static::assertSame([1, 2], $rows->reduceToArray('id')); + static::assertSame(['x', 'y'], array_column($rejects, 'record')); + } + + public function test_a_decode_refusal_and_a_cast_refusal_in_one_batch_are_routed_in_offset_order(): void + { + $rejects = []; + + df()->read( + from_csv('memory://source.csv', filesystem: CSVFixtureContext::memory("id,name\nx,a\n2,b,c\n3,d\n")) + ->withSchema(schema(int_schema('id'), str_schema('name'))) + ->withCorruptedRecords(to_array($rejects)), + )->run(); + + static::assertSame([8, 12], array_column($rejects, 'offset')); + } } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVExtractorTypedColumnsTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVExtractorTypedColumnsTest.php index efd18b52e3..5295098c70 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVExtractorTypedColumnsTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVExtractorTypedColumnsTest.php @@ -22,6 +22,7 @@ final class CSVExtractorTypedColumnsTest extends FlowTestCase public function test_hydrates_typed_columns_with_empty_to_null_and_input_uri(): void { $extractor = from_csv($path = path_real(__DIR__ . '/../../../Fixtures/file_with_empty_columns.csv')) + ->withMissingColumnsAsNull(true) ->withSchema(schema( int_schema('id', nullable: true), str_schema('name', nullable: true), @@ -31,7 +32,11 @@ public function test_hydrates_typed_columns_with_empty_to_null_and_input_uri(): $actual = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { foreach ($rows->toArray() as $row) { $actual[] = $row; } @@ -56,7 +61,11 @@ public function test_reads_partitioned_files_with_schema_typed_partition_values( $actual = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { foreach ($rows->toArray() as $row) { $actual[] = $row; } @@ -88,7 +97,11 @@ public function test_appends_partition_columns_absent_from_the_schema(): void $actual = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { foreach ($rows->toArray() as $row) { $actual[] = $row; } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVSplitReadTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVSplitReadTest.php index 783e4a7da4..ba29ba212d 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVSplitReadTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVSplitReadTest.php @@ -7,12 +7,20 @@ use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\File\FileSplit; +use Flow\ETL\IO\Loader\File\FileLoader; +use Flow\ETL\IO\Loader\File\Part; +use Flow\ETL\Planning\Physical\PipelineSteps; +use Flow\ETL\Runtime\Executor\StepRunner; use Flow\ETL\Tests\Context\DescribeRoundTrip; use Flow\ETL\Tests\Context\ExtractedRows; +use Flow\ETL\Tests\Context\MemoryTelemetryContext; use Flow\ETL\Tests\Context\ReaderRoundTrip; use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Values\Rows; +use function array_map; +use function array_merge; use function file_put_contents; use function filesize; use function Flow\ETL\Adapter\CSV\from_csv; @@ -20,10 +28,14 @@ use function Flow\ETL\DSL\infer_schema; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\to_array; use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\native_local_filesystem; use function Flow\Filesystem\DSL\path; use function Flow\Filesystem\DSL\path_real; +use function Flow\Floe\DSL\from_floe; +use function Flow\Floe\DSL\to_floe; +use function Flow\Types\DSL\type_instance_of; use function iterator_to_array; use function mkdir; @@ -84,7 +96,8 @@ public function test_a_sized_split_is_read_without_listing(): void static::assertSame( [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => 'b']], iterator_to_array( - $reader->extract( + FlowTestCase::extracted( + $reader, flow_context()->backend(), (new Criteria())->withSplit( new FileSplit($file, 0, null, (int) filesize($root->path() . '/report.csv')), @@ -118,4 +131,59 @@ public function test_a_memory_filesystem_does_not_describe(): void ->describe(), ); } + + public function test_a_reader_routes_its_rejects_after_a_rebuild_through_the_runs_lifecycle(): void + { + $root = ReaderRoundTrip::scratch('csv-rejects'); + file_put_contents($root->path() . '/orders.csv', "id\n1\nx\n3\n"); + $reader = from_csv($root->suffix('/orders.csv')) + ->withSchema(schema(int_schema('id'))) + ->withCorruptedRecords(to_floe($root->suffix('/rejects/part-{part}.floe'))) + ->reader(); + $rebuilt = DescribeRoundTrip::rebuilt($reader); + $loader = type_instance_of(FileLoader::class)->assert($rebuilt->corruptedRecordsLoader()); + $inTask = $loader->inTask(new Part(0, 1, 'run1', false)); + $task = $rebuilt->withCorruptedRecords($inTask); + + $telemetry = new MemoryTelemetryContext(); + $rows = iterator_to_array( + (new StepRunner())->steps( + new PipelineSteps(), + $task->extract(flow_context()->backend(), new Criteria()), + $telemetry->flowContext, + $task, + ), + false, + ); + $loader->commit('run1', 1)?->commit([0]); + + static::assertSame( + [1, 3], + array_merge(...array_map(static fn(Rows $batch): array => $batch->reduceToArray('id'), $rows)), + ); + static::assertSame( + ['x'], + ExtractedRows::of(from_floe($root->suffix('/rejects/*.floe')))->reduceToArray('record'), + ); + static::assertCount( + 1, + $telemetry->logs->entriesContaining( + 'Routed 1 corrupted records to ' . $root->suffix('/rejects/part-{part}.floe')->uri() . '.', + ), + ); + + native_local_filesystem()->rm($root); + } + + public function test_a_reader_whose_reject_loader_does_not_describe_does_not_describe(): void + { + $rejects = []; + + static::assertNull( + from_csv(path_real(__DIR__ . '/../../../Fixtures/partitioned/group=*/*.csv')) + ->withCorruptedRecords(to_array($rejects)) + ->reader() + ->describe(), + ); + } } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/PhpCSVOpenSourceTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/PhpCSVOpenSourceTest.php index bea066df16..ed5c6fa646 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/PhpCSVOpenSourceTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/PhpCSVOpenSourceTest.php @@ -7,13 +7,13 @@ use Flow\ETL\Adapter\CSV\IO\Extractor\CSVReadOptions; use Flow\ETL\Adapter\CSV\Tests\Context\CSVFixtureContext; use Flow\ETL\Foundation\Exception\ColumnMismatchException; -use Flow\ETL\Foundation\Exception\SchemaMismatchException; use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Local\NativeLocalFilesystem; -use function array_keys; +use function array_key_exists; use function array_map; +use function array_slice; use function count; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\schema; @@ -90,18 +90,12 @@ public function test_columns_without_a_header_line_are_generated(): void } } - public function test_records_yields_every_row_with_the_full_key_set(): void + public function test_records_yields_only_the_rows_of_records_that_are_not_corrupted(): void { $open = CSVFixtureContext::openPhp('ragged.csv'); try { - $records = iterator_to_array($open->records(), false); - - static::assertCount(3, $records); - - foreach ($records as $record) { - static::assertSame(['id', 'name', 'v'], array_keys($record)); - } + static::assertSame([['id' => '1', 'name' => 'a', 'v' => '10']], iterator_to_array($open->records(), false)); } finally { $open->close(); } @@ -171,80 +165,123 @@ public function test_batches_pad_a_missing_nullable_column(): void ); } - public function test_batches_refuse_a_missing_not_null_column(): void + public function test_batches_hand_up_a_record_missing_a_not_null_column(): void { - $refusal = CSVFixtureContext::batches( + $batches = CSVFixtureContext::batches( CSVFixtureContext::openPhpStream(CSVFixtureContext::content("id\n1\n")), schema(int_schema('id'), str_schema('name')), 10, ); - static::assertInstanceOf(SchemaMismatchException::class, $refusal); + static::assertIsArray($batches); static::assertSame( - (new SchemaMismatchException(0, ColumnMismatchException::missingColumn('name')))->getMessage(), - $refusal->getMessage(), + [[ + 3, + 2, + 'column "' + . ColumnMismatchException::missingColumn('name')->column + . '"' + . ColumnMismatchException::missingColumn('name')->detail, + '1', + ['1'], + ]], + array_map(static fn(array $batch): array => array_key_exists('corrupted', $batch) + ? array_slice($batch['corrupted'], 1) + : $batch, $batches), ); - static::assertSame(0, $refusal->rowIndex); } - public function test_batches_refused_value_wins_over_absence(): void + public function test_batches_hand_up_corrupted_records_in_source_order(): void { - $refusal = CSVFixtureContext::batches( + $batches = CSVFixtureContext::batches( CSVFixtureContext::openPhpStream(CSVFixtureContext::content("id\n1\nx\n")), schema(int_schema('id'), str_schema('name')), 10, ); - static::assertInstanceOf(SchemaMismatchException::class, $refusal); + static::assertIsArray($batches); static::assertSame( - (new SchemaMismatchException(1, ColumnMismatchException::valueDoesNotMatch( - 'id', - 'integer', - 'x', - )))->getMessage(), - $refusal->getMessage(), + [ + [ + 3, + 2, + 'column "' + . ColumnMismatchException::missingColumn('name')->column + . '"' + . ColumnMismatchException::missingColumn('name')->detail, + '1', + ['1'], + ], + [ + 5, + 3, + 'column "' + . ColumnMismatchException::valueDoesNotMatch('id', 'integer', 'x')->column + . '"' + . ColumnMismatchException::valueDoesNotMatch('id', 'integer', 'x')->detail, + 'x', + ['x'], + ], + ], + array_map(static fn(array $batch): array => array_key_exists('corrupted', $batch) + ? array_slice($batch['corrupted'], 1) + : $batch, $batches), ); - static::assertSame(1, $refusal->rowIndex); } public function test_batches_refusal_tie_goes_to_the_first_definition(): void { - $refusal = CSVFixtureContext::batches( + $batches = CSVFixtureContext::batches( CSVFixtureContext::openPhpStream(CSVFixtureContext::content("a,b\nx,y\n")), schema(int_schema('b'), int_schema('a')), 10, ); - static::assertInstanceOf(SchemaMismatchException::class, $refusal); + static::assertIsArray($batches); static::assertSame( - (new SchemaMismatchException(0, ColumnMismatchException::valueDoesNotMatch( - 'b', - 'integer', - 'y', - )))->getMessage(), - $refusal->getMessage(), + [[ + 4, + 2, + 'column "' + . ColumnMismatchException::valueDoesNotMatch('b', 'integer', 'y')->column + . '"' + . ColumnMismatchException::valueDoesNotMatch('b', 'integer', 'y')->detail, + 'x,y', + ['x', 'y'], + ]], + array_map(static fn(array $batch): array => array_key_exists('corrupted', $batch) + ? array_slice($batch['corrupted'], 1) + : $batch, $batches), ); - static::assertSame(0, $refusal->rowIndex); } - public function test_batches_refuse_an_empty_to_null_null_under_not_null(): void + public function test_batches_hand_up_an_empty_to_null_null_under_not_null(): void { - $refusal = CSVFixtureContext::batches( + $batches = CSVFixtureContext::batches( CSVFixtureContext::openPhpStream(CSVFixtureContext::content("id,name\n1,a\n2,\n")), schema(int_schema('id'), str_schema('name')), 10, ); - static::assertInstanceOf(SchemaMismatchException::class, $refusal); + static::assertIsArray($batches); static::assertSame( - (new SchemaMismatchException(1, ColumnMismatchException::valueDoesNotMatch( - 'name', - 'string', - null, - )))->getMessage(), - $refusal->getMessage(), + [ + [ + 12, + 3, + 'column "' + . ColumnMismatchException::valueDoesNotMatch('name', 'string', null)->column + . '"' + . ColumnMismatchException::valueDoesNotMatch('name', 'string', null)->detail, + '2,', + ['2', ''], + ], + [['id' => 1, 'name' => 'a']], + ], + array_map(static fn(array $batch): array => array_key_exists('corrupted', $batch) + ? array_slice($batch['corrupted'], 1) + : $batch, $batches), ); - static::assertSame(1, $refusal->rowIndex); } public function test_batches_of_a_header_only_source_are_empty(): void diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/RustCSVOpenSourceTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/RustCSVOpenSourceTest.php index 3a59144293..e70bc11bc9 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/RustCSVOpenSourceTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/RustCSVOpenSourceTest.php @@ -8,7 +8,6 @@ use Flow\ETL\Adapter\CSV\Tests\Context\CSVFixtureContext; use Flow\ETL\Adapter\CSV\Tests\Double\LengthCapturingSourceStream; use Flow\ETL\Foundation\Exception\ColumnMismatchException; -use Flow\ETL\Foundation\Exception\SchemaMismatchException; use Flow\ETL\IO\Inference\ColumnTypes; use Flow\ETL\IO\Inference\InferredTypes; use Flow\ETL\IO\Inference\SchemaInference; @@ -18,6 +17,8 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Schema; use Flow\Filesystem\Local\NativeLocalFilesystem; +use Flow\Filesystem\SourceStream; +use Flow\Filesystem\Stream\RangeSourceStream; use Flow\Filesystem\Telemetry\FilesystemTelemetryAttributes; use Flow\Filesystem\Telemetry\TraceableFilesystem; use Flow\Filesystem\Tests\Mother\FilesystemTelemetryConfigMother; @@ -27,8 +28,9 @@ use PHPUnit\Framework\Attributes\DataProviderExternal; use PHPUnit\Framework\Attributes\TestWith; -use function array_keys; +use function array_key_exists; use function array_map; +use function array_slice; use function count; use function extension_loaded; use function file_get_contents; @@ -41,6 +43,7 @@ use function Flow\Telemetry\DSL\void_exporter; use function implode; use function iterator_to_array; +use function strlen; final class RustCSVOpenSourceTest extends FlowTestCase { @@ -134,7 +137,7 @@ public function test_columns_without_a_header_line_are_generated(): void } } - public function test_records_yields_every_row_with_the_full_key_set(): void + public function test_records_yields_only_the_rows_of_records_that_are_not_corrupted(): void { if (!extension_loaded('flow_php')) { static::markTestSkipped('flow_php extension is not loaded'); @@ -143,13 +146,7 @@ public function test_records_yields_every_row_with_the_full_key_set(): void $open = CSVFixtureContext::openNative('ragged.csv'); try { - $records = iterator_to_array($open->records(), false); - - static::assertCount(3, $records); - - foreach ($records as $record) { - static::assertSame(['id', 'name', 'v'], array_keys($record)); - } + static::assertSame([['id' => '1', 'name' => 'a', 'v' => '10']], iterator_to_array($open->records(), false)); } finally { $open->close(); } @@ -479,48 +476,76 @@ public function test_batches_pad_a_missing_nullable_column(): void ); } - public function test_batches_refuse_a_missing_not_null_column(): void + public function test_batches_hand_up_a_record_missing_a_not_null_column(): void { if (!extension_loaded('flow_php')) { static::markTestSkipped('flow_php extension is not loaded'); } - $refusal = CSVFixtureContext::batches( + $batches = CSVFixtureContext::batches( CSVFixtureContext::openNativeStream(CSVFixtureContext::content("id\n1\n")), schema(int_schema('id'), str_schema('name')), 10, ); - static::assertInstanceOf(SchemaMismatchException::class, $refusal); + static::assertIsArray($batches); static::assertSame( - (new SchemaMismatchException(0, ColumnMismatchException::missingColumn('name')))->getMessage(), - $refusal->getMessage(), + [[ + 3, + 2, + 'column "' + . ColumnMismatchException::missingColumn('name')->column + . '"' + . ColumnMismatchException::missingColumn('name')->detail, + '1', + ['1'], + ]], + array_map(static fn(array $batch): array => array_key_exists('corrupted', $batch) + ? array_slice($batch['corrupted'], 1) + : $batch, $batches), ); - static::assertSame(0, $refusal->rowIndex); } - public function test_batches_refused_value_wins_over_absence(): void + public function test_batches_hand_up_corrupted_records_in_source_order(): void { if (!extension_loaded('flow_php')) { static::markTestSkipped('flow_php extension is not loaded'); } - $refusal = CSVFixtureContext::batches( + $batches = CSVFixtureContext::batches( CSVFixtureContext::openNativeStream(CSVFixtureContext::content("id\n1\nx\n")), schema(int_schema('id'), str_schema('name')), 10, ); - static::assertInstanceOf(SchemaMismatchException::class, $refusal); + static::assertIsArray($batches); static::assertSame( - (new SchemaMismatchException(1, ColumnMismatchException::valueDoesNotMatch( - 'id', - 'integer', - 'x', - )))->getMessage(), - $refusal->getMessage(), + [ + [ + 3, + 2, + 'column "' + . ColumnMismatchException::missingColumn('name')->column + . '"' + . ColumnMismatchException::missingColumn('name')->detail, + '1', + ['1'], + ], + [ + 5, + 3, + 'column "' + . ColumnMismatchException::valueDoesNotMatch('id', 'integer', 'x')->column + . '"' + . ColumnMismatchException::valueDoesNotMatch('id', 'integer', 'x')->detail, + 'x', + ['x'], + ], + ], + array_map(static fn(array $batch): array => array_key_exists('corrupted', $batch) + ? array_slice($batch['corrupted'], 1) + : $batch, $batches), ); - static::assertSame(1, $refusal->rowIndex); } public function test_batches_refusal_tie_goes_to_the_first_definition(): void @@ -529,46 +554,61 @@ public function test_batches_refusal_tie_goes_to_the_first_definition(): void static::markTestSkipped('flow_php extension is not loaded'); } - $refusal = CSVFixtureContext::batches( + $batches = CSVFixtureContext::batches( CSVFixtureContext::openNativeStream(CSVFixtureContext::content("a,b\nx,y\n")), schema(int_schema('b'), int_schema('a')), 10, ); - static::assertInstanceOf(SchemaMismatchException::class, $refusal); + static::assertIsArray($batches); static::assertSame( - (new SchemaMismatchException(0, ColumnMismatchException::valueDoesNotMatch( - 'b', - 'integer', - 'y', - )))->getMessage(), - $refusal->getMessage(), + [[ + 4, + 2, + 'column "' + . ColumnMismatchException::valueDoesNotMatch('b', 'integer', 'y')->column + . '"' + . ColumnMismatchException::valueDoesNotMatch('b', 'integer', 'y')->detail, + 'x,y', + ['x', 'y'], + ]], + array_map(static fn(array $batch): array => array_key_exists('corrupted', $batch) + ? array_slice($batch['corrupted'], 1) + : $batch, $batches), ); - static::assertSame(0, $refusal->rowIndex); } - public function test_batches_refuse_an_empty_to_null_null_under_not_null(): void + public function test_batches_hand_up_an_empty_to_null_null_under_not_null(): void { if (!extension_loaded('flow_php')) { static::markTestSkipped('flow_php extension is not loaded'); } - $refusal = CSVFixtureContext::batches( + $batches = CSVFixtureContext::batches( CSVFixtureContext::openNativeStream(CSVFixtureContext::content("id,name\n1,a\n2,\n")), schema(int_schema('id'), str_schema('name')), 10, ); - static::assertInstanceOf(SchemaMismatchException::class, $refusal); + static::assertIsArray($batches); static::assertSame( - (new SchemaMismatchException(1, ColumnMismatchException::valueDoesNotMatch( - 'name', - 'string', - null, - )))->getMessage(), - $refusal->getMessage(), + [ + [ + 12, + 3, + 'column "' + . ColumnMismatchException::valueDoesNotMatch('name', 'string', null)->column + . '"' + . ColumnMismatchException::valueDoesNotMatch('name', 'string', null)->detail, + '2,', + ['2', ''], + ], + [['id' => 1, 'name' => 'a']], + ], + array_map(static fn(array $batch): array => array_key_exists('corrupted', $batch) + ? array_slice($batch['corrupted'], 1) + : $batch, $batches), ); - static::assertSame(1, $refusal->rowIndex); } public function test_batches_of_a_header_only_source_are_empty(): void @@ -588,7 +628,7 @@ public function test_batches_of_a_header_only_source_are_empty(): void } /** - * @return Generator + * @return Generator}> */ public static function batch_parity_cases(): Generator { @@ -622,28 +662,65 @@ public static function batch_parity_cases(): Generator ]; yield 'numeric header' => ["1,2\na,3\n", schema(str_schema('1'), int_schema('2')), 2]; yield 'refusal in the second batch' => ["id,name\n1,a\n2,b\n3,c\nx,d\n", $idName, 2]; + + $idNullableName = schema(int_schema('id'), str_schema('name', nullable: true)); + + yield 'bom, crlf, multiline, blank lines and corrupted records' => [ + "\xEF\xBB\xBFid,name\r\n1,\"a\r\nb\"\r\n\r\n2\r\n3,c,x\r\nx,d\r\n4,", + $idNullableName, + 2, + ]; + yield 'a cast refusal before a decode refusal in one batch' => ["id,name\nx,a\n2,b,c\n3,d\n", $idName, 10]; + yield 'missing columns as null' => [ + "id,name\n1\n2,b\n3,c,x\n", + $idNullableName, + 2, + (new CSVReadOptions())->withMissingColumnsAsNull(true), + ]; + yield 'refused past the sample' => ["id,name\n1,a\n2,b\nx,c\n", $idName, 2, new CSVReadOptions(), 2]; + yield 'window past the first byte' => [ + "e00,e01\n1,a\n2\n3,c\n", + schema(str_schema('e00'), str_schema('e01', nullable: true)), + 2, + (new CSVReadOptions())->withHeader(false), + null, + 8, + ]; } /** * @param positive-int $batchSize + * @param int<0, max> $windowStart */ #[DataProvider('batch_parity_cases')] - public function test_native_and_php_batches_are_identical(string $content, Schema $schema, int $batchSize): void - { + public function test_native_and_php_batches_are_identical( + string $content, + Schema $schema, + int $batchSize, + CSVReadOptions $options = new CSVReadOptions(), + ?int $sampleSize = null, + int $windowStart = 0, + ): void { if (!extension_loaded('flow_php')) { static::markTestSkipped('flow_php extension is not loaded'); } + $stream = static fn(): SourceStream => $windowStart === 0 + ? CSVFixtureContext::content($content) + : new RangeSourceStream(CSVFixtureContext::content($content), $windowStart, strlen($content)); + static::assertSame( CSVFixtureContext::batchesOutcome( - CSVFixtureContext::openPhpStream(CSVFixtureContext::content($content)), + CSVFixtureContext::openPhpStream($stream(), $options), $schema, $batchSize, + sampleSize: $sampleSize, ), CSVFixtureContext::batchesOutcome( - CSVFixtureContext::openNativeStream(CSVFixtureContext::content($content)), + CSVFixtureContext::openNativeStream($stream(), $options), $schema, $batchSize, + sampleSize: $sampleSize, ), ); } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVDecoderTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVDecoderTest.php index 81ae9ea8f8..2fea839550 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVDecoderTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVDecoderTest.php @@ -9,99 +9,104 @@ final class CSVDecoderTest extends FlowTestCase { - public function test_decode_captures_headers_from_the_first_row_and_emits_no_row_for_it(): void + public function test_the_header_line_is_no_row(): void { $decoder = new CSVDecoder(); - static::assertSame([], $decoder->decode(['id,name'])); - static::assertSame(['id' => '1', 'name' => 'Norbert'], $decoder->decode(['1,Norbert'])[0]); + static::assertNull($decoder->decode($decoder->cells('id,name'))); + static::assertSame(['id', 'name'], $decoder->headers()); } - public function test_decode_generates_auto_headers_when_header_is_disabled(): void + public function test_fields_are_mapped_to_headers(): void { - static::assertSame( - ['e00' => '1', 'e01' => 'Norbert'], - (new CSVDecoder(withHeader: false))->decode(['1,Norbert'])[0], - ); - } + $decoder = new CSVDecoder(); + $decoder->decode($decoder->cells('id,name')); - public function test_decode_keeps_empty_fields_when_empty_to_null_is_disabled(): void - { - static::assertSame( - ['id' => '1', 'name' => ''], - (new CSVDecoder(emptyToNull: false))->decode(['id,name', '1,'])[0], - ); + static::assertSame(['id' => '1', 'name' => 'Norbert'], $decoder->decode($decoder->cells('1,Norbert'))); } - public function test_decode_maps_fields_to_headers(): void + public function test_auto_headers_are_generated_when_header_is_disabled(): void { - static::assertSame(['id' => '1', 'name' => 'Norbert'], (new CSVDecoder())->decode(['id,name', '1,Norbert'])[0]); + $decoder = new CSVDecoder(withHeader: false); + + static::assertSame(['e00' => '1', 'e01' => 'Norbert'], $decoder->decode($decoder->cells('1,Norbert'))); + static::assertSame(['e00', 'e01'], $decoder->headers()); } - public function test_decode_pads_short_rows_with_null(): void + public function test_an_empty_field_is_null_by_default_and_kept_when_empty_to_null_is_disabled(): void { - static::assertSame(['id' => '1', 'name' => null], (new CSVDecoder())->decode(['id,name', '1'])[0]); + $nulls = new CSVDecoder(); + $nulls->decode($nulls->cells('id,name')); + $strings = new CSVDecoder(emptyToNull: false); + $strings->decode($strings->cells('id,name')); + + static::assertSame(['id' => '1', 'name' => null], $nulls->decode($nulls->cells('1,'))); + static::assertSame(['id' => '1', 'name' => ''], $strings->decode($strings->cells('1,'))); } - public function test_decode_truncates_rows_longer_than_the_headers(): void + public function test_a_short_line_is_corrupted(): void { - static::assertSame( - ['id' => '1', 'name' => 'Norbert'], - (new CSVDecoder())->decode(['id,name', '1,Norbert,extra'])[0], - ); + $decoder = new CSVDecoder(); + $decoder->decode($decoder->cells('id,name')); + + static::assertSame('1 fields, the header has 2', $decoder->decode($decoder->cells('1'))); } - public function test_decode_turns_empty_fields_into_null_by_default(): void + public function test_a_short_line_is_padded_under_missing_columns_as_null(): void { - static::assertSame(['id' => '1', 'name' => null], (new CSVDecoder())->decode(['id,name', '1,'])[0]); + $decoder = new CSVDecoder(missingColumnsAsNull: true); + $decoder->decode($decoder->cells('id,name')); + + static::assertSame(['id' => '1', 'name' => null], $decoder->decode($decoder->cells('1'))); } - public function test_decoding_a_blank_line_yields_a_single_null_field(): void + public function test_a_line_with_a_non_empty_extra_field_is_corrupted(): void { - static::assertSame(['e00' => null], (new CSVDecoder(withHeader: false, emptyToNull: false))->decode([''])[0]); + $decoder = new CSVDecoder(); + $decoder->decode($decoder->cells('id,name')); + + static::assertSame('3 fields, the header has 2', $decoder->decode($decoder->cells('1,Norbert,extra'))); } - public function test_decoding_a_line_with_a_custom_separator_enclosure_and_escape_yields_strings(): void + public function test_a_trailing_comma_loses_nothing(): void { - static::assertSame( - ['id' => '1', 'name' => 'a;b'], - (new CSVDecoder(separator: ';', enclosure: "'", escape: '|'))->decode(['id;name', "1;'a;b'"])[0], - ); + $decoder = new CSVDecoder(); + $decoder->decode($decoder->cells('id,name')); + + static::assertSame(['id' => '1', 'name' => 'Norbert'], $decoder->decode($decoder->cells('1,Norbert,'))); } - public function test_decoding_a_line_with_empty_quoted_fields_yields_empty_strings(): void + public function test_a_blank_line_is_no_row(): void { - static::assertSame( - ['id' => '', 'name' => ''], - (new CSVDecoder(emptyToNull: false))->decode(['id,name', '"",""'])[0], - ); + $decoder = new CSVDecoder(withHeader: false); + + static::assertNull($decoder->decode($decoder->cells(''))); } - public function test_headers_are_null_before_any_line_is_decoded(): void + public function test_a_custom_separator_enclosure_and_escape_split_the_fields(): void { - static::assertNull((new CSVDecoder())->headers()); + $decoder = new CSVDecoder(separator: ';', enclosure: "'", escape: '|'); + + static::assertSame(['id', 'a;b'], $decoder->cells("id;'a;b'")); } - public function test_the_generated_headers_are_exposed_without_a_header_line(): void + public function test_empty_quoted_fields_are_empty_strings(): void { - $decoder = new CSVDecoder(withHeader: false); - $decoder->decode(['1,a']); + $decoder = new CSVDecoder(emptyToNull: false); + $decoder->decode($decoder->cells('id,name')); - static::assertSame(['e00', 'e01'], $decoder->headers()); + static::assertSame(['id' => '', 'name' => ''], $decoder->decode($decoder->cells('"",""'))); } - public function test_the_resolved_headers_are_exposed(): void + public function test_headers_are_null_before_any_line_is_decoded(): void { - $decoder = new CSVDecoder(); - - static::assertSame([], $decoder->decode(['id,name'])); - static::assertSame(['id', 'name'], $decoder->headers()); + static::assertNull((new CSVDecoder())->headers()); } public function test_an_empty_header_cell_is_named_by_position(): void { $decoder = new CSVDecoder(); - $decoder->decode([',name']); + $decoder->decode($decoder->cells(',name')); static::assertSame(['e00', 'name'], $decoder->headers()); } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVFileBatchesTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVFileBatchesTest.php index f29e2dd19f..a3ec5833fc 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVFileBatchesTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVFileBatchesTest.php @@ -8,6 +8,7 @@ use Flow\ETL\Adapter\CSV\IO\Extractor\CSVReadOptions; use Flow\ETL\Adapter\CSV\IO\Extractor\CSVSourceOpener; use Flow\ETL\Adapter\CSV\Tests\Context\CSVFixtureContext; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\File\FileSplit; @@ -50,7 +51,12 @@ public function test_a_file_yields_its_rows_and_consumes_no_offset(): void static::assertSame( [[['id' => 1], ['id' => 2]], [['id' => 3]]], - array_map(static fn(Rows $rows): array => $rows->toArray(), iterator_to_array($file, false)), + array_map( + static fn(CorruptedRecordException|Rows $rows): array => ( + $rows instanceof Rows ? $rows->toArray() : [$rows->getMessage()] + ), + iterator_to_array($file, false), + ), ); static::assertSame(0, $file->getReturn()); } @@ -91,6 +97,7 @@ public function test_batches_are_sized_and_the_tail_is_shorter(): void new PhpBackend(), new ReadWindow(), ) as $batch) { + static::assertInstanceOf(Rows::class, $batch); $sizes[] = count($batch); } @@ -147,6 +154,8 @@ public function test_batches_of_a_split_read_only_its_records(): void new PhpBackend(), new ReadWindow(), ) as $batch) { + static::assertInstanceOf(Rows::class, $batch); + foreach ($batch->toArray() as $row) { $values[] = $row; } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVReadOptionsTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVReadOptionsTest.php index ebf37bca3b..0e9f1aa6eb 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVReadOptionsTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVReadOptionsTest.php @@ -84,7 +84,9 @@ public function test_the_withers_compose(): void public function test_normalize_round_trips_a_null_separator_and_a_set_escape(): void { - $options = (new CSVReadOptions(separator: null, escape: '\\'))->withCharactersReadInLine(64); + $options = (new CSVReadOptions(separator: null, escape: '\\')) + ->withCharactersReadInLine(64) + ->withMissingColumnsAsNull(true); static::assertSame( [ @@ -95,6 +97,7 @@ public function test_normalize_round_trips_a_null_separator_and_a_set_escape(): 'enclosure' => null, 'escape' => '\\', 'characters_read_in_line' => 64, + 'missing_columns_as_null' => true, ], $options->normalize(), ); diff --git a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelExtractor.php b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelExtractor.php index 1b419f51e4..3e2a9626e6 100644 --- a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelExtractor.php +++ b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelExtractor.php @@ -5,9 +5,11 @@ namespace Flow\ETL\Adapter\Excel\IO\Extractor; use Flow\ETL\Adapter\Excel\IO\Inference\ExcelSniffer; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; use Flow\ETL\Functions\ScalarFunction; +use Flow\ETL\IO\CorruptedRecordsRouter; use Flow\ETL\IO\Extractor; use Flow\ETL\IO\Extractor\BatchableExtractor; use Flow\ETL\IO\Extractor\Criteria; @@ -29,6 +31,7 @@ use Flow\ETL\IO\Inference\SchemaInference; use Flow\ETL\IO\Inference\SchemaInferenceBuilder; use Flow\ETL\IO\Inference\SchemaInferrer; +use Flow\ETL\IO\Loader; use Flow\ETL\Values\BatchSize; use Flow\ETL\Values\Cardinality; use Flow\ETL\Values\Column\Backend; @@ -49,6 +52,7 @@ final class ExcelExtractor implements FileExtractor, InfersSchema, MetadataColumnsExtractor, + CorruptedRecordsRouter, SplittableExtractor { private BatchSize $batchSize; @@ -76,6 +80,8 @@ final class ExcelExtractor implements */ private ?SampledUnits $sampledUnits = null; + private ?Loader $corruptedRecordsLoader = null; + private ?Schema $schema = null; private ?Statistics $statistics = null; @@ -129,8 +135,13 @@ public function isRepeatable(): bool return true; } + public function corruptedRecordsLoader(): ?Loader + { + return $this->corruptedRecordsLoader; + } + /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { @@ -154,7 +165,9 @@ public function extract(Backend $backend, Criteria $criteria): Generator $this->derived->from(), ) : null, + $this->schema === null ? $this->inference->sampleSize : null, ), + $this->corruptedRecordsLoader, ))->extract($backend, $criteria); } @@ -176,7 +189,9 @@ public function reader(): FileSplitReader $this->derived->from(), ) : null, + $this->schema === null ? $this->inference->sampleSize : null, ), + $this->corruptedRecordsLoader, ); } @@ -316,6 +331,21 @@ public function withConvertEmptyToNull(bool $convertEmptyToNull): self return $this; } + public function withCorruptedRecords(?Loader $loader): static + { + $this->corruptedRecordsLoader = $loader; + + return $this; + } + + public function withMissingColumnsAsNull(bool $missingColumnsAsNull): self + { + $this->readOptions = $this->readOptions->withMissingColumnsAsNull($missingColumnsAsNull); + $this->forgetInference(); + + return $this; + } + public function withHeader(bool $withHeader): self { $this->readOptions = $this->readOptions->withHeader($withHeader); diff --git a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelFileBatches.php b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelFileBatches.php index bbd8dbb7de..f902fae05e 100644 --- a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelFileBatches.php +++ b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelFileBatches.php @@ -7,7 +7,9 @@ use Flow\ETL\Adapter\Excel\IO\Inference\ExcelSniffer; use Flow\ETL\Foundation\Describe\Description; use Flow\ETL\Foundation\Describe\Rebuild; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; +use Flow\ETL\Foundation\Exception\SchemaValidationException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\File\FileBatches; use Flow\ETL\IO\Extractor\File\FileRead; @@ -22,7 +24,9 @@ use SensitiveParameter; use function Flow\Types\DSL\type_array; +use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_optional; +use function is_string; final readonly class ExcelFileBatches implements FileBatches { @@ -30,11 +34,13 @@ * @param null|ExcelSniffer $sample the open sheets of the inference this read follows, read on from where the * sample stopped; null to parse every sheet afresh; never described * @param null|InferredColumns $inferred the columns every sheet must carry; null when declared or union by name + * @param null|int $sampleSize the inference's sample size when the schema was inferred; null when it was declared */ public function __construct( private WorkbookReader $workbook, private ?ExcelSniffer $sample, private ?InferredColumns $inferred, + private ?int $sampleSize = null, ) {} public static function fromDescription(#[SensitiveParameter] array $data, Rebuild $rebuild): static @@ -45,6 +51,7 @@ public static function fromDescription(#[SensitiveParameter] array $data, Rebuil $rebuild->of($data['workbook'] ?? null, WorkbookReader::class), null, $inferred === null ? null : InferredColumns::fromArray($inferred), + type_optional(type_integer())->assert($data['sample_size'] ?? null), ); } @@ -53,6 +60,7 @@ public function describe(): ?Description return Description::compose(self::class, [ 'workbook' => $this->workbook, 'inferred' => $this->inferred?->normalize(), + 'sample_size' => $this->sampleSize, ]); } @@ -62,7 +70,7 @@ public function describe(): ?Description * * @throws InferredSchemaException * - * @return Generator + * @return Generator */ public function batches( FileRead $read, @@ -75,23 +83,39 @@ public function batches( $sheet = $this->sample?->take($source) ?? $this->workbook->sheet($source); try { - $this->inferred?->refuseDivergence($source->uri(), $sheet->columns()); + $columns = $sheet->columns(); + $this->inferred?->refuseDivergence($source->uri(), $columns); + $missing = $this->sampleSize === null ? $body->missingFrom($columns) : []; - $batch = new RecordBatch($body, $backend, $read->batchSize); + if ($columns !== [] && $missing !== []) { + throw SchemaValidationException::declaredColumnsMissing($source->uri(), $missing, $columns); + } + + $batch = new RecordBatch($body, $backend, $read->batchSize, $source->uri(), $this->sampleSize); - foreach ($sheet->rows() as $rowValues) { - $rows = $batch->add($rowValues); + foreach ($sheet->rows() as $number => $cells) { + $values = $sheet->decode($cells); - if ($rows !== null) { - yield $rows; + if ($values === null) { + continue; } - } - $rest = $batch->flush(); + if (is_string($values)) { + yield from $batch->refuse(CorruptedRecordException::fromCells( + $source->uri(), + $number, + null, + $values, + $cells, + )); - if ($rest !== null) { - yield $rest; + continue; + } + + yield from $batch->add($values, $number, fields: $cells); } + + yield from $batch->flush(); } finally { $sheet->close(); } diff --git a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelReadOptions.php b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelReadOptions.php index 993a8c5ace..ad522b48f1 100644 --- a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelReadOptions.php +++ b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelReadOptions.php @@ -17,6 +17,7 @@ public function __construct( public bool $convertEmptyToNull = true, public ?string $sheetName = null, public ?ExcelReader $format = null, + public bool $missingColumnsAsNull = true, ) { if ($sheetName !== null) { SheetNameAssertion::assert($sheetName); @@ -35,11 +36,12 @@ public static function fromArray(array $data): self type_boolean()->assert($data['convert_empty_to_null'] ?? null), type_optional(type_string())->assert($data['sheet_name'] ?? null), $format === null ? null : ExcelReader::from($format), + type_boolean()->assert($data['missing_columns_as_null'] ?? null), ); } /** - * @return array{with_header: bool, convert_empty_to_null: bool, sheet_name: null|string, format: null|string} + * @return array{with_header: bool, convert_empty_to_null: bool, sheet_name: null|string, format: null|string, missing_columns_as_null: bool} */ public function normalize(): array { @@ -48,6 +50,7 @@ public function normalize(): array 'convert_empty_to_null' => $this->convertEmptyToNull, 'sheet_name' => $this->sheetName, 'format' => $this->format?->value, + 'missing_columns_as_null' => $this->missingColumnsAsNull, ]; } @@ -58,6 +61,7 @@ public function withConvertEmptyToNull(bool $convertEmptyToNull): self convertEmptyToNull: $convertEmptyToNull, sheetName: $this->sheetName, format: $this->format, + missingColumnsAsNull: $this->missingColumnsAsNull, ); } @@ -68,6 +72,7 @@ public function withFormat(ExcelReader $format): self convertEmptyToNull: $this->convertEmptyToNull, sheetName: $this->sheetName, format: $format, + missingColumnsAsNull: $this->missingColumnsAsNull, ); } @@ -78,6 +83,18 @@ public function withHeader(bool $withHeader): self convertEmptyToNull: $this->convertEmptyToNull, sheetName: $this->sheetName, format: $this->format, + missingColumnsAsNull: $this->missingColumnsAsNull, + ); + } + + public function withMissingColumnsAsNull(bool $missingColumnsAsNull): self + { + return new self( + withHeader: $this->withHeader, + convertEmptyToNull: $this->convertEmptyToNull, + sheetName: $this->sheetName, + format: $this->format, + missingColumnsAsNull: $missingColumnsAsNull, ); } @@ -88,6 +105,7 @@ public function withSheetName(string $sheetName): self convertEmptyToNull: $this->convertEmptyToNull, sheetName: $sheetName, format: $this->format, + missingColumnsAsNull: $this->missingColumnsAsNull, ); } } diff --git a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/Sheet/SheetCells.php b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/Sheet/SheetCells.php index 6be509904b..6d9f45cbc0 100644 --- a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/Sheet/SheetCells.php +++ b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/Sheet/SheetCells.php @@ -28,6 +28,8 @@ public function __construct( ) {} /** + * Keyed by the row's 1-based number in the sheet. + * * @return Generator> */ public function rows(): Generator @@ -35,11 +37,11 @@ public function rows(): Generator $previousWidth = 0; $index = 0; - foreach ($this->values() as $row) { + foreach ($this->values() as $number => $row) { $index++; if ($index === 1 && $this->withHeader) { - yield $row; + yield $number => $row; continue; } @@ -51,7 +53,7 @@ public function rows(): Generator $previousWidth = count($row); - yield $row; + yield $number => $row; } } @@ -68,8 +70,9 @@ private function values(): Generator static $utc = new DateTimeZone('UTC'); - foreach ($this->sheet->getRowIterator() as $sheetRow) { - yield array_map(static fn(Cell $cell) => $cell instanceof DateTimeCell + /** @var int $number */ + foreach ($this->sheet->getRowIterator() as $number => $sheetRow) { + yield $number => array_map(static fn(Cell $cell) => $cell instanceof DateTimeCell ? new DateTimeImmutable($cell->getValue()->format('Y-m-d H:i:s.u'), $utc) : $cell->getValue(), $sheetRow->cells); } diff --git a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/Sheet/XlsxSheetRows.php b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/Sheet/XlsxSheetRows.php index 7a85185c86..da65c8688e 100644 --- a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/Sheet/XlsxSheetRows.php +++ b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/Sheet/XlsxSheetRows.php @@ -76,7 +76,7 @@ public function __construct(Sheet $sheet) /** * Each non-empty row's cell values by column index, as OpenSpout's RowIterator yields them with the default * options Flow opens every workbook with: a row is padded to its spans, else to the sheet's dimension, and a - * sheet that declares no dimension gets the gaps below its cell count filled. + * sheet that declares no dimension gets the gaps below its cell count filled. Keyed by the row's 1-based number. * * @return Generator> */ @@ -100,6 +100,7 @@ public function values(): Generator $dimension = 0; $values = []; $column = -1; + $rowNumber = 0; $moved = $reader->read(); while ($moved) { @@ -127,6 +128,8 @@ public function values(): Generator $width = $spans !== null && $spans !== '' ? (int) (explode(':', $spans)[1] ?? 0) : $dimension; $values = $width > 0 ? array_fill(0, $width, '') : []; $column = -1; + $reference = $reader->getAttribute('r'); + $rowNumber = $reference === null || $reference === '' ? $rowNumber + 1 : (int) $reference; } elseif ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'dimension') { $match = []; @@ -141,7 +144,7 @@ public function values(): Generator if (!self::isEmpty($values)) { libxml_use_internal_errors($internalErrors); - yield $dimension === 0 ? self::filled($values) : $values; + yield $rowNumber => $dimension === 0 ? self::filled($values) : $values; $internalErrors = libxml_use_internal_errors(true); libxml_clear_errors(); diff --git a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/WorkbookReader.php b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/WorkbookReader.php index 425c97f249..b05aad0453 100644 --- a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/WorkbookReader.php +++ b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/WorkbookReader.php @@ -76,7 +76,11 @@ public function open(Path $path): OpenSheet return new OpenSheet( $reader, (new SheetCells($sheet, $this->options->withHeader))->rows(), - new RecordDecoder(withHeader: $this->options->withHeader, emptyToNull: $this->options->convertEmptyToNull), + new RecordDecoder( + withHeader: $this->options->withHeader, + emptyToNull: $this->options->convertEmptyToNull, + missingColumnsAsNull: $this->options->missingColumnsAsNull, + ), ); } diff --git a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/WorkbookSheet.php b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/WorkbookSheet.php index b5ba47cab1..b6e904b490 100644 --- a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/WorkbookSheet.php +++ b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/WorkbookSheet.php @@ -11,7 +11,7 @@ final class WorkbookSheet { /** - * @var list> + * @var array> raw cells by their row number */ private array $buffered = []; @@ -41,48 +41,63 @@ public function columns(): array if (!$this->headerResolved) { $this->headerResolved = true; - $row = $sheet->cells->current(); + $cells = $sheet->cells->current(); - if ($row !== null) { - $this->buffered = $sheet->decoder->decode([$row]); + if ($cells !== null) { + $number = $sheet->cells->key(); $sheet->cells->next(); + + // a sheet without a header row sizes its names by its first row, which stays a row + if ($sheet->decoder->decode($cells) !== null) { + $this->buffered[$number] = $cells; + } } } return $sheet->decoder->headers() ?? []; } + /** + * @param array $cells a row rows() or sample() yielded + * + * @return null|array|string see RecordDecoder::decode() + */ + public function decode(array $cells): array|string|null + { + $sheet = $this->sheet ??= $this->reader->open($this->path); + + return $sheet->decoder->decode($cells); + } + /** * rows() for a bounded schema sample: every row it hands out stays buffered and the sheet stays open, so the * rows() that follows replays the sample and parses on from where the sample stopped, instead of parsing the * sample a second time. Closing the sheet stays with its owner. * - * @return Generator> + * @return Generator> raw cells by their row number */ public function sample(): Generator { $sheet = $this->sheet ??= $this->reader->open($this->path); $this->columns(); - foreach ($this->buffered as $rowValues) { - yield $rowValues; + foreach ($this->buffered as $number => $cells) { + yield $number => $cells; } - while (($row = $sheet->cells->current()) !== null) { + while (($cells = $sheet->cells->current()) !== null) { + $number = $sheet->cells->key(); // advanced before the row is handed out: a sampler that stops here leaves the cursor past everything // buffered, so rows() neither repeats nor skips a row $sheet->cells->next(); + $this->buffered[$number] = $cells; - foreach ($sheet->decoder->decode([$row]) as $rowValues) { - $this->buffered[] = $rowValues; - - yield $rowValues; - } + yield $number => $cells; } } /** - * @return Generator> + * @return Generator> raw cells by their row number, the header row excluded */ public function rows(): Generator { @@ -93,16 +108,14 @@ public function rows(): Generator try { $this->columns(); - foreach ($this->buffered as $rowValues) { - yield $rowValues; + foreach ($this->buffered as $number => $cells) { + yield $number => $cells; } $this->buffered = []; - while (($row = $sheet->cells->current()) !== null) { - foreach ($sheet->decoder->decode([$row]) as $rowValues) { - yield $rowValues; - } + while (($cells = $sheet->cells->current()) !== null) { + yield $sheet->cells->key() => $cells; $sheet->cells->next(); } diff --git a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Inference/ExcelSniffer.php b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Inference/ExcelSniffer.php index ac3f0435ec..30824cdcd1 100644 --- a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Inference/ExcelSniffer.php +++ b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Inference/ExcelSniffer.php @@ -130,7 +130,7 @@ public function names(): array public function sniff(int $unit, int $rowBudget): Sample { $sheet = $this->sheets[$unit] ??= $this->workbook->sheet($this->files[$unit]); - $sample = new WorkbookSheetSample($rowBudget === -1 ? $sheet->rows() : $sheet->sample()); + $sample = new WorkbookSheetSample($sheet, $rowBudget === -1 ? $sheet->rows() : $sheet->sample()); return new Sample( ColumnTypes::sniff($this->names(), $sample, $rowBudget, $this->typer)->folded(), diff --git a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Inference/WorkbookSheetSample.php b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Inference/WorkbookSheetSample.php index 27af72db15..7537b67faf 100644 --- a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Inference/WorkbookSheetSample.php +++ b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Inference/WorkbookSheetSample.php @@ -4,9 +4,12 @@ namespace Flow\ETL\Adapter\Excel\IO\Inference; +use Flow\ETL\Adapter\Excel\IO\Extractor\WorkbookSheet; use Generator; use IteratorAggregate; +use function is_array; + /** * @implements IteratorAggregate> */ @@ -15,19 +18,26 @@ final class WorkbookSheetSample implements IteratorAggregate private bool $wholeSheet = false; /** - * @param Generator> $sheetRows + * @param Generator> $sheetRows the sheet's raw rows, decoded by $sheet */ public function __construct( + private readonly WorkbookSheet $sheet, private readonly Generator $sheetRows, ) {} /** + * The rows of the records that are not corrupted. + * * @return Generator> */ public function getIterator(): Generator { - foreach ($this->sheetRows as $rowValues) { - yield $rowValues; + foreach ($this->sheetRows as $cells) { + $values = $this->sheet->decode($cells); + + if (is_array($values)) { + yield $values; + } } $this->wholeSheet = true; diff --git a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Context/ExcelFixtureContext.php b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Context/ExcelFixtureContext.php index 1eac03ece6..8047bffa4b 100644 --- a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Context/ExcelFixtureContext.php +++ b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Context/ExcelFixtureContext.php @@ -29,6 +29,7 @@ use function Flow\Filesystem\DSL\path; use function Flow\Filesystem\DSL\path_real; use function glob; +use function is_array; use function sys_get_temp_dir; final class ExcelFixtureContext @@ -150,6 +151,26 @@ public static function sheet( return self::reader($filesystem, $options)->sheet(self::source($fixture)); } + /** + * The rows of $sheet's records that are not corrupted, decoded as the read decodes them. + * + * @return list> + */ + public static function decodedRows(WorkbookSheet $sheet): array + { + $rows = []; + + foreach ($sheet->rows() as $cells) { + $values = $sheet->decode($cells); + + if (is_array($values)) { + $rows[] = $values; + } + } + + return $rows; + } + public static function source(string $fixture): SourceFile { return new SourceFile(self::path($fixture)); diff --git a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelExtractorTest.php b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelExtractorTest.php index c650eb7610..d5b106ad7b 100644 --- a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelExtractorTest.php +++ b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelExtractorTest.php @@ -8,10 +8,12 @@ use DateTimeImmutable; use Flow\ETL\Adapter\Excel\IO\Extractor\ExcelExtractor; use Flow\ETL\Adapter\Excel\IO\Extractor\ExcelReader; +use Flow\ETL\Adapter\Excel\Tests\Context\CraftedWorkbook; use Flow\ETL\Adapter\Excel\Tests\Context\ExcelFixtureContext; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; -use Flow\ETL\Foundation\Exception\SchemaMismatchException; +use Flow\ETL\Foundation\Exception\SchemaValidationException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\File\FileSplit; use Flow\ETL\IO\Extractor\Signal; @@ -30,6 +32,7 @@ use PHPUnit\Framework\Attributes\TestWith; use function array_keys; +use function array_map; use function count; use function Flow\ETL\Adapter\Excel\DSL\from_excel; use function Flow\ETL\Adapter\Excel\DSL\is_valid_excel_sheet_name; @@ -51,6 +54,7 @@ use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\string_schema; +use function Flow\ETL\DSL\to_array; use function Flow\Filesystem\DSL\native_local_filesystem; use function Flow\Filesystem\DSL\path_real; use function Flow\Types\DSL\type_datetime; @@ -79,6 +83,52 @@ public static function provide_nullable_fixtures(): iterable yield 'xlsx' => [__DIR__ . '/../../../Fixtures/nullable_fixture.xlsx']; } + public function test_a_row_with_more_cells_than_the_header_is_refused(): void + { + $file = CraftedWorkbook::with(CraftedWorkbook::sheet( + 'abc12345678', + )); + + try { + ExtractedRows::of(from_excel($file)); + static::fail('Expected the corrupted record to be refused'); + } catch (CorruptedRecordException $e) { + static::assertSame([path_real($file)->uri(), 3, null, ['4', '5', '6', '7', '8']], [ + $e->uri, + $e->offset, + $e->lineNumber, + $e->fields, + ]); + } + } + + public function test_a_corrupted_row_after_skipped_empty_rows_reports_its_sheet_row_number(): void + { + $file = CraftedWorkbook::with(CraftedWorkbook::sheet( + 'abc1234567', + )); + + try { + ExtractedRows::of(from_excel($file)); + static::fail('Expected the corrupted record to be refused'); + } catch (CorruptedRecordException $e) { + static::assertSame([5, ['4', '5', '6', '7']], [$e->offset, $e->fields]); + } + } + + public function test_a_declared_column_the_header_lacks_is_a_schema_error(): void + { + $this->expectException(SchemaValidationException::class); + $this->expectExceptionMessage('"x"'); + + ExtractedRows::of( + from_excel(CraftedWorkbook::with(CraftedWorkbook::sheet( + 'abc123', + ))) + ->withSchema(schema(str_schema('a'), str_schema('b'), str_schema('x', true))), + ); + } + public function test_an_empty_glob_refuses_to_infer_a_schema(): void { $this->expectException(InferredSchemaException::class); @@ -110,7 +160,7 @@ public function test_an_empty_glob_with_a_declared_schema_answers_it_and_extract static::assertEquals(schema(int_schema('id')), $extractor->schema()); static::assertSame( [], - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()), false), + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false), ); } @@ -226,22 +276,19 @@ public function test_extract_excel_nullable_file(string $fixtureName): void } #[DataProvider('provide_fixtures')] - public function test_extract_excel_puts_null_in_a_declared_nullable_column_the_sheet_lacks(string $fixtureName): void + public function test_extract_excel_refuses_a_declared_nullable_column_the_sheet_lacks(string $fixtureName): void { - $rows = df() + $this->expectException(SchemaValidationException::class); + $this->expectExceptionMessage('Columns "missing" declared by the schema are not in the header of'); + + df() ->read(from_excel($fixtureName)->withSchema(schema( int_schema('id'), string_schema('name'), string_schema('email', nullable: true), string_schema('missing', nullable: true), ))) - ->fetch() - ->toArray(); - - foreach ($rows as $row) { - static::assertNotSame([], $row); - static::assertNull($row['missing']); - } + ->fetch(); } /** @@ -249,10 +296,10 @@ public function test_extract_excel_puts_null_in_a_declared_nullable_column_the_s * declaration. Declare the column nullable if that is what the data is. */ #[DataProvider('provide_fixtures')] - public function test_extract_excel_refuses_a_not_null_column_the_sheet_lacks(string $fixtureName): void + public function test_extract_excel_refuses_a_declared_column_the_sheet_lacks(string $fixtureName): void { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage('column "missing" (row 0) declared by the schema is missing from the row'); + $this->expectException(SchemaValidationException::class); + $this->expectExceptionMessage('Columns "missing" declared by the schema are not in the header of'); df() ->read(from_excel($fixtureName)->withSchema(schema( @@ -383,7 +430,11 @@ public function test_every_batch_carries_the_inferred_schema(string $fixtureName $emails = []; $sizes = []; - foreach ($extractor->withBatchSize(2)->extract(flow_context()->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor->withBatchSize(2), + flow_context()->backend(), + new Criteria(), + ) as $rows) { static::assertEquals($extractor->schema(), $rows->schema()); $sizes[] = $rows->count(); @@ -466,7 +517,7 @@ public function test_a_value_past_the_sample_that_does_not_fit_fails_naming_colu static::assertEquals(schema(int_schema('v', nullable: true)), $extractor->schema()); - $this->expectException(SchemaMismatchException::class); + $this->expectException(InferredSchemaException::class); $this->expectExceptionMessage('column "v"'); df()->read($extractor)->fetch(); @@ -515,7 +566,7 @@ public function test_an_empty_sheet_refuses_to_infer_a_schema_and_extracts_nothi static::assertSame( [], - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()), false), + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false), ); $this->expectException(InferredSchemaException::class); @@ -589,7 +640,7 @@ public function test_partitioned_glob_infers_from_the_first_file_only(): void $count = 0; - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { static::assertEquals($extractor->schema(), $rows->schema()); static::assertContains('group', $rows->schema()->names()); @@ -671,8 +722,11 @@ public function test_signal_stop_leaves_no_shared_strings_temp_folder(): void { $before = ExcelFixtureContext::sharedStringsFolders(); - $generator = from_excel(ExcelFixtureContext::path('orders_flow.xlsx')) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_excel(ExcelFixtureContext::path('orders_flow.xlsx')), + flow_context(config())->backend(), + new Criteria(), + ); $generator->current(); $generator->send(Signal::STOP); unset($generator); @@ -685,7 +739,7 @@ public function test_extract_twice_on_the_same_instance(string $fixtureName): vo { $extractor = from_excel($fixtureName); - $generator = $extractor->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()); $generator->current(); $generator->next(); $generator->send(Signal::STOP); @@ -793,9 +847,11 @@ public function test_a_declared_schema_is_never_sampled_for(): void public function test_signal_stop(): void { - $generator = from_excel(path_real(__DIR__ . '/../../../Fixtures/fixture.xlsx')) - ->withBatchSize(1) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_excel(path_real(__DIR__ . '/../../../Fixtures/fixture.xlsx'))->withBatchSize(1), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->next(); @@ -808,9 +864,11 @@ public function test_signal_stop(): void public function test_signal_stop_on_the_first_file_tail_batch_skips_the_remaining_files(): void { - $generator = from_excel(ExcelFixtureContext::file('cross_stream/*/*.xlsx')) - ->withBatchSize(10) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_excel(ExcelFixtureContext::file('cross_stream/*/*.xlsx'))->withBatchSize(10), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->send(Signal::STOP); @@ -889,7 +947,7 @@ public function test_extract_samples_like_schema_does(): void { $extractor = from_excel(ExcelFixtureContext::path('sniff/a')); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()), false); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false); static::assertEquals(Cardinality::exact(10), $extractor->statistics()->rows); } @@ -939,9 +997,13 @@ public function test_a_filtered_extract_infers_over_every_listed_file(): void $filtered = from_excel(ExcelFixtureContext::path('sniff/*')); $whole = from_excel(ExcelFixtureContext::path('sniff/*')); - iterator_to_array($filtered->extract(flow_context()->backend(), new Criteria(pathFilter: new KeepPaths([ - ExcelFixtureContext::path('sniff/a'), - ]))), false); + iterator_to_array(FlowTestCase::extracted( + $filtered, + flow_context()->backend(), + new Criteria(pathFilter: new KeepPaths([ + ExcelFixtureContext::path('sniff/a'), + ])), + ), false); $whole->schema(); static::assertEquals($whole->statistics()->rows, $filtered->statistics()->rows); @@ -1006,7 +1068,8 @@ public function test_extract_builds_through_the_config_backend(): void $backend = new SpyBackend(); iterator_to_array( - from_excel(__DIR__ . '/../../../Fixtures/fixture.xlsx')->extract( + FlowTestCase::extracted( + from_excel(__DIR__ . '/../../../Fixtures/fixture.xlsx'), flow_context(config_builder()->backend($backend)->build())->backend(), new Criteria(), ), @@ -1052,7 +1115,8 @@ public function test_extract_split_stops_on_the_stop_signal(): void { $extractor = from_excel(__DIR__ . '/../../../Fixtures/orders_1k.xlsx'); $extractor->withBatchSize(1); - $generator = $extractor->extract( + $generator = FlowTestCase::extracted( + $extractor, flow_context()->backend(), (new Criteria())->withSplit($extractor->splits(new Criteria(), 1)[0]), ); @@ -1124,4 +1188,25 @@ public function test_a_pushed_path_filter_prunes_the_files_splits_are_cut_from() $extractor->splits((new Criteria())->withPathFilter(new RejectingFilter()), 1), ); } + + public function test_corrupted_records_are_routed_to_the_reject_loader(): void + { + $rejects = []; + $file = CraftedWorkbook::with(CraftedWorkbook::sheet( + 'a12x3', + )); + + $rows = df()->read(from_excel($file)->withCorruptedRecords(to_array($rejects)))->fetch(); + + static::assertSame(['1', '3'], $rows->reduceToArray('a')); + static::assertSame( + [[path_real($file)->uri(), 3, null, ['2', 'x']]], + array_map(static fn(array $reject): array => [ + $reject['uri'], + $reject['offset'], + $reject['line'], + $reject['fields'], + ], $rejects), + ); + } } diff --git a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelExtractorTypedColumnsTest.php b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelExtractorTypedColumnsTest.php index 11695b63c2..ce34dca64e 100644 --- a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelExtractorTypedColumnsTest.php +++ b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelExtractorTypedColumnsTest.php @@ -25,7 +25,11 @@ public function test_reads_typed_columns_with_a_schema(): void $count = 0; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { static::assertEquals(type_integer(), $rows->schema()->get('id')->type()); $count += $rows->count(); @@ -40,7 +44,11 @@ public function test_appends_input_file_uri_when_metadata_columns_are_enabled(): ->withSchema(schema(int_schema('id'), string_schema('name'), string_schema('email', nullable: true))) ->withMetadataColumns(true); - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { static::assertSame([$path->uri()], array_values(array_unique($rows->column('_input_file_uri')->values()))); } } diff --git a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/WorkbookReaderTest.php b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/WorkbookReaderTest.php index 98f72996ca..d0f65b92db 100644 --- a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/WorkbookReaderTest.php +++ b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/WorkbookReaderTest.php @@ -16,19 +16,15 @@ use function array_keys; use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Types\DSL\type_array; -use function iterator_to_array; final class WorkbookReaderTest extends FlowTestCase { public function test_options_reach_the_sheet(): void { - $rows = iterator_to_array( - ExcelFixtureContext::sheet( - 'nullable_fixture.xlsx', - options: new ExcelReadOptions(withHeader: false, convertEmptyToNull: false), - )->rows(), - false, - ); + $rows = ExcelFixtureContext::decodedRows(ExcelFixtureContext::sheet( + 'nullable_fixture.xlsx', + options: new ExcelReadOptions(withHeader: false, convertEmptyToNull: false), + )); static::assertCount(6, $rows); static::assertSame(['e00', 'e01', 'e02'], array_keys($rows[0])); diff --git a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/WorkbookSheetTest.php b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/WorkbookSheetTest.php index 0f4cb9c898..96391bda44 100644 --- a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/WorkbookSheetTest.php +++ b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/WorkbookSheetTest.php @@ -160,10 +160,10 @@ public function test_rows_close_the_reader_when_fully_read(string $fixture): voi public function test_rows_re_emit_the_row_the_header_was_resolved_from(): void { - $rows = iterator_to_array( - ExcelFixtureContext::sheet('fixture.xlsx', options: new ExcelReadOptions(withHeader: false))->rows(), - false, - ); + $rows = ExcelFixtureContext::decodedRows(ExcelFixtureContext::sheet( + 'fixture.xlsx', + options: new ExcelReadOptions(withHeader: false), + )); static::assertCount(11, $rows); static::assertSame(['e00' => 'id', 'e01' => 'name', 'e02' => 'email'], $rows[0]); @@ -171,7 +171,7 @@ public function test_rows_re_emit_the_row_the_header_was_resolved_from(): void public function test_rows_resolve_the_columns_without_a_columns_call_first(): void { - $rows = iterator_to_array(ExcelFixtureContext::sheet('fixture.xlsx')->rows(), false); + $rows = ExcelFixtureContext::decodedRows(ExcelFixtureContext::sheet('fixture.xlsx')); static::assertCount(10, $rows); static::assertSame(['id', 'name', 'email'], array_keys($rows[0])); diff --git a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelFileBatchesTest.php b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelFileBatchesTest.php index f4fd1c49d8..9d662aa62b 100644 --- a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelFileBatchesTest.php +++ b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelFileBatchesTest.php @@ -6,6 +6,7 @@ use Flow\ETL\Adapter\Excel\IO\Extractor\ExcelFileBatches; use Flow\ETL\Adapter\Excel\Tests\Context\ExcelFixtureContext; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\File\InferredColumns; @@ -39,7 +40,9 @@ public function test_a_sampled_sheet_is_read_on_and_gives_the_rows_a_fresh_read_ new ReadWindow(), ); $rows = array_merge(...array_map( - static fn(Rows $batch): array => $batch->toArray(), + static fn(CorruptedRecordException|Rows $rows): array => ( + $rows instanceof Rows ? $rows->toArray() : [$rows->getMessage()] + ), iterator_to_array($fromSample, false), )); $sampled->close(); @@ -49,7 +52,9 @@ public function test_a_sampled_sheet_is_read_on_and_gives_the_rows_a_fresh_read_ static::assertSame( $rows, array_merge(...array_map( - static fn(Rows $batch): array => $batch->toArray(), + static fn(CorruptedRecordException|Rows $rows): array => ( + $rows instanceof Rows ? $rows->toArray() : [$rows->getMessage()] + ), iterator_to_array( $fresh->batches( FileReadMother::of($schema, null, 2), diff --git a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelReadOptionsTest.php b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelReadOptionsTest.php index 24a91f3222..f4b50a4f4e 100644 --- a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelReadOptionsTest.php +++ b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelReadOptionsTest.php @@ -19,6 +19,7 @@ public function test_defaults_read_the_first_sheet_with_a_header(): void static::assertTrue($options->convertEmptyToNull); static::assertNull($options->sheetName); static::assertNull($options->format); + static::assertTrue($options->missingColumnsAsNull); } public function test_an_invalid_sheet_name_is_refused(): void @@ -45,6 +46,10 @@ public function test_each_wither_changes_one_knob_and_keeps_the_rest(): void static::assertEquals(new ExcelReadOptions(convertEmptyToNull: false), $options->withConvertEmptyToNull(false)); static::assertEquals(new ExcelReadOptions(sheetName: 'Sheet2'), $options->withSheetName('Sheet2')); static::assertEquals(new ExcelReadOptions(format: ExcelReader::ODS), $options->withFormat(ExcelReader::ODS)); + static::assertEquals( + new ExcelReadOptions(missingColumnsAsNull: false), + $options->withMissingColumnsAsNull(false), + ); } public function test_withers_compose_without_losing_earlier_knobs(): void @@ -69,6 +74,7 @@ public function test_normalize_round_trips_with_and_without_a_format(): void 'convert_empty_to_null' => true, 'sheet_name' => 'Data', 'format' => 'xlsx', + 'missing_columns_as_null' => true, ], $xlsx->normalize(), ); diff --git a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Inference/WorkbookSheetSampleTest.php b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Inference/WorkbookSheetSampleTest.php index 3c63840ab8..f8d1e7dd9e 100644 --- a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Inference/WorkbookSheetSampleTest.php +++ b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Inference/WorkbookSheetSampleTest.php @@ -5,50 +5,57 @@ namespace Flow\ETL\Adapter\Excel\Tests\Unit\IO\Inference; use Flow\ETL\Adapter\Excel\IO\Inference\WorkbookSheetSample; +use Flow\ETL\Adapter\Excel\Tests\Context\CraftedWorkbook; +use Flow\ETL\Adapter\Excel\Tests\Context\ExcelFixtureContext; +use Flow\ETL\IO\Extractor\File\SourceFile; use Flow\ETL\Tests\FlowTestCase; +use function Flow\Filesystem\DSL\path_real; use function iterator_to_array; final class WorkbookSheetSampleTest extends FlowTestCase { public function test_a_sample_stopped_early_is_not_whole(): void { - $sample = new WorkbookSheetSample( - (static function () { - yield ['id' => 1]; - yield ['id' => 2]; - })(), - ); + $sheet = ExcelFixtureContext::sheet('fixture.xlsx'); + $sample = new WorkbookSheetSample($sheet, $sheet->sample()); foreach ($sample as $_rowValues) { break; } static::assertFalse($sample->wholeSheet()); + $sheet->close(); } public function test_a_sample_read_to_the_end_is_whole(): void { - $sample = new WorkbookSheetSample( - (static function () { - yield ['id' => 1]; - yield ['id' => 2]; - })(), - ); - - iterator_to_array($sample, false); + $sheet = ExcelFixtureContext::sheet('fixture.xlsx'); + $sample = new WorkbookSheetSample($sheet, $sheet->sample()); + static::assertCount(10, iterator_to_array($sample, false)); static::assertTrue($sample->wholeSheet()); + $sheet->close(); } public function test_a_sample_never_iterated_is_not_whole(): void { - $sample = new WorkbookSheetSample( - (static function () { - yield ['id' => 1]; - })(), - ); + $sheet = ExcelFixtureContext::sheet('fixture.xlsx'); - static::assertFalse($sample->wholeSheet()); + static::assertFalse((new WorkbookSheetSample($sheet, $sheet->sample()))->wholeSheet()); + $sheet->close(); + } + + public function test_a_corrupted_record_is_left_out_of_the_sample(): void + { + $sheet = ExcelFixtureContext::reader()->sheet(new SourceFile(path_real(CraftedWorkbook::with(CraftedWorkbook::sheet( + 'a' + . '12' + . '3', + ))))); + $sample = new WorkbookSheetSample($sheet, $sheet->sample()); + + static::assertSame([['a' => 3]], iterator_to_array($sample, false)); + $sheet->close(); } } diff --git a/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetExtractor.php b/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetExtractor.php index 951bc8fb4b..1146f2b0bb 100644 --- a/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetExtractor.php +++ b/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetExtractor.php @@ -5,8 +5,11 @@ namespace Flow\ETL\Adapter\GoogleSheet\IO\Extractor; use Flow\ETL\Adapter\GoogleSheet\IO\Inference\GoogleSheetSniffer; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; +use Flow\ETL\Foundation\Exception\SchemaValidationException; +use Flow\ETL\IO\CorruptedRecordsRouter; use Flow\ETL\IO\Extractor; use Flow\ETL\IO\Extractor\BatchableExtractor; use Flow\ETL\IO\Extractor\Criteria; @@ -17,26 +20,33 @@ use Flow\ETL\IO\Inference\SchemaInference; use Flow\ETL\IO\Inference\SchemaInferenceBuilder; use Flow\ETL\IO\Inference\SchemaInferrer; +use Flow\ETL\IO\Loader; use Flow\ETL\Values\BatchSize; use Flow\ETL\Values\Cardinality; use Flow\ETL\Values\Column\Backend; use Flow\ETL\Values\Rows; -use Flow\ETL\Values\Rows\RowsBuilder; +use Flow\ETL\Values\Rows\RecordBatch; use Flow\ETL\Values\Schema; use Flow\ETL\Values\Schema\Definition\StringDefinition; use Generator; use Google\Service\Sheets; use function array_diff; -use function array_keys; -use function array_map; +use function is_string; use function sprintf; /** * @import-type GoogleSheetOptions from GoogleSheetReadOptions */ -final class GoogleSheetExtractor implements BatchableExtractor, Extractor, InfersSchema, MetadataColumnsExtractor +final class GoogleSheetExtractor implements + BatchableExtractor, + Extractor, + InfersSchema, + MetadataColumnsExtractor, + CorruptedRecordsRouter { + private ?Loader $corruptedRecordsLoader = null; + private BatchSize $batchSize; /** @@ -88,8 +98,13 @@ public function isRepeatable(): bool return true; } + public function corruptedRecordsLoader(): ?Loader + { + return $this->corruptedRecordsLoader; + } + /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { @@ -112,60 +127,23 @@ public function extract(Backend $backend, Criteria $criteria): Generator $schema = $this->addMetadataColumns ? $base->add(new StringDefinition('_spread_sheet_id'), new StringDefinition('_sheet_name')) : $base; - $checked = $this->schema !== null || $this->inference->unionByName; + $checked = $this->schema === null && $this->inference->unionByName; - $batchSize = $this->batchSize(); $yielded = 0; // the sample already holds every row when its range covered the grid, so the read is served from it rather // than fetching the same rows again; nothing can have diverged from a schema read off these very rows - $batches = ($sampledNow ? $sniffer->batches($batchSize) : null) ?? $reader->batches( - $this->rowsPerPage, - $batchSize, + $sheetRows = ($sampledNow ? $sniffer->rows() : null) ?? $reader->rows($this->rowsPerPage); + $uri = sprintf('google-sheet://%s/%s', $this->spreadsheetId, $this->columnRange->sheetName); + $batch = new RecordBatch( + $schema, + $backend, + $this->batchSize(), + $uri, + $this->schema === null ? $this->inference->sampleSize : null, ); - foreach ($batches as $batch) { - if (!$checked) { - $columns = array_map(static fn(int|string $name): string => (string) $name, array_keys($batch[0])); - - // a row that carries no columns is not a divergence: withHeader(false) decodes a leading blank - // row to one, and the names come from the row after it - if ($columns !== []) { - $expected = $base->names(); - - if (array_diff($columns, $expected) !== [] || array_diff($expected, $columns) !== []) { - // a sheet is one source, so it is named as both the read and the inferred-from side - throw InferredSchemaException::columnsDiverge( - sprintf('spreadsheet "%s" sheet "%s"', $this->spreadsheetId, $this->columnRange->sheetName), - sprintf('spreadsheet "%s" sheet "%s"', $this->spreadsheetId, $this->columnRange->sheetName), - $expected, - $columns, - $this->inference->sampleSize, - $this->inference->filesToSniff, - ); - } - - $checked = true; - } - } - - if ($this->addMetadataColumns) { - $batch = array_map( - function (array $values): array { - // assigned, never unpacked: a numeric column name is an int array key and ... renumbers it - $values['_spread_sheet_id'] = $this->spreadsheetId; - $values['_sheet_name'] = $this->columnRange->sheetName; - - return $values; - }, - $batch, - ); - } - - $rows = (new RowsBuilder($schema, $backend)) - ->appendRows($batch) - ->finish(); - - $yielded += $rows->count(); + foreach ($this->recordBatches($batch, $uri, $sheetRows, $base, $checked) as $rows) { + $yielded += $rows instanceof Rows ? $rows->count() : 0; $signal = yield $rows; @@ -239,9 +217,16 @@ public function withBatchSize(int $batchSize): static return $this; } - public function withDropExtraColumns(bool $dropExtraColumns): self + public function withCorruptedRecords(?Loader $loader): static { - $this->readOptions = $this->readOptions->withDropExtraColumns($dropExtraColumns); + $this->corruptedRecordsLoader = $loader; + + return $this; + } + + public function withMissingColumnsAsNull(bool $missingColumnsAsNull): self + { + $this->readOptions = $this->readOptions->withMissingColumnsAsNull($missingColumnsAsNull); $this->derivedSchema = null; $this->gridRowCount = null; $this->statistics = null; @@ -306,4 +291,69 @@ public function withSchema(Schema $schema): static return $this; } + + /** + * @param Generator> $sheetRows the raw cells of every row, keyed by its number in the sheet + * + * @throws InferredSchemaException when the sheet's header diverges from the inferred schema + * + * @return Generator + */ + private function recordBatches( + RecordBatch $batch, + string $uri, + Generator $sheetRows, + Schema $base, + bool $checked, + ): Generator { + $decoder = $this->readOptions->decoder(); + $sheet = sprintf('spreadsheet "%s" sheet "%s"', $this->spreadsheetId, $this->columnRange->sheetName); + + foreach ($sheetRows as $number => $cells) { + $values = $decoder->decode($cells); + $columns = $decoder->headers(); + + if (!$checked && $columns !== null) { + $checked = true; + $expected = $base->names(); + $missing = $base->missingFrom($columns); + + if ($this->schema !== null && $missing !== []) { + throw SchemaValidationException::declaredColumnsMissing($uri, $missing, $columns); + } + + if ($this->schema === null && (array_diff($columns, $expected) !== [] || $missing !== [])) { + // a sheet is one source, so it is named as both the read and the inferred-from side + throw InferredSchemaException::columnsDiverge( + $sheet, + $sheet, + $expected, + $columns, + $this->inference->sampleSize, + $this->inference->filesToSniff, + ); + } + } + + if ($values === null) { + continue; + } + + if (is_string($values)) { + yield from $batch->refuse(CorruptedRecordException::fromCells($uri, $number, null, $values, $cells)); + + continue; + } + + if ($this->addMetadataColumns) { + // assigned, never unpacked: a numeric column name is an int array key and ... renumbers it + $values['_spread_sheet_id'] = $this->spreadsheetId; + $values['_sheet_name'] = $this->columnRange->sheetName; + } + + yield from $batch->add($values, $number, fields: $cells); + } + + yield from $batch->flush(); + } } diff --git a/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetReadOptions.php b/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetReadOptions.php index e7e54c15c8..31ede58f48 100644 --- a/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetReadOptions.php +++ b/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetReadOptions.php @@ -20,7 +20,7 @@ */ public function __construct( public bool $withHeader = true, - public bool $dropExtraColumns = true, + public bool $missingColumnsAsNull = false, public bool $emptyToNull = true, public array $options = [], ) {} @@ -30,7 +30,7 @@ public function __construct( */ public function decoder(): RecordDecoder { - return new RecordDecoder($this->withHeader, $this->emptyToNull, $this->dropExtraColumns); + return new RecordDecoder($this->withHeader, $this->emptyToNull, $this->missingColumnsAsNull); } /** @@ -44,19 +44,19 @@ public function typer(InferredTypes $candidates): TypeNarrower : new InstanceOfTypeNarrower(); } - public function withDropExtraColumns(bool $dropExtraColumns): self + public function withEmptyToNull(bool $emptyToNull): self { - return new self($this->withHeader, $dropExtraColumns, $this->emptyToNull, $this->options); + return new self($this->withHeader, $this->missingColumnsAsNull, $emptyToNull, $this->options); } - public function withEmptyToNull(bool $emptyToNull): self + public function withHeader(bool $withHeader): self { - return new self($this->withHeader, $this->dropExtraColumns, $emptyToNull, $this->options); + return new self($withHeader, $this->missingColumnsAsNull, $this->emptyToNull, $this->options); } - public function withHeader(bool $withHeader): self + public function withMissingColumnsAsNull(bool $missingColumnsAsNull): self { - return new self($withHeader, $this->dropExtraColumns, $this->emptyToNull, $this->options); + return new self($this->withHeader, $missingColumnsAsNull, $this->emptyToNull, $this->options); } /** @@ -64,6 +64,6 @@ public function withHeader(bool $withHeader): self */ public function withOptions(array $options): self { - return new self($this->withHeader, $this->dropExtraColumns, $this->emptyToNull, $options); + return new self($this->withHeader, $this->missingColumnsAsNull, $this->emptyToNull, $options); } } diff --git a/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetReader.php b/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetReader.php index a73d061243..6ca4b46020 100644 --- a/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetReader.php +++ b/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetReader.php @@ -10,7 +10,7 @@ use function array_merge; use function array_values; -use function count; +use function is_array; final readonly class GoogleSheetReader { @@ -22,9 +22,9 @@ public function __construct( ) {} /** - * @return Generator>> + * @return Generator> the raw cells of every row, keyed by its 1-based number in the sheet */ - public function batches(int $rowsPerPage, int $batchSize): Generator + public function rows(int $rowsPerPage): Generator { $range = new SheetRange($this->columns, 1, $rowsPerPage, $this->rowCount()); $ranges = []; @@ -36,26 +36,16 @@ public function batches(int $rowsPerPage, int $batchSize): Generator /** @var Sheets\Resource\SpreadsheetsValues $values */ $values = $this->service->spreadsheets_values; - $decoder = $this->readOptions->decoder(); - $batch = []; - foreach ($values->batchGet($this->spreadsheetId, array_merge($this->readOptions->options, [ + foreach (array_values($values->batchGet($this->spreadsheetId, array_merge($this->readOptions->options, [ 'ranges' => $ranges, - ]))->getValueRanges() as $valueRange) { + ]))->getValueRanges()) as $page => $valueRange) { // @mago-ignore analysis:redundant-null-coalesce - foreach ($decoder->decode(array_values($valueRange->getValues() ?? [])) as $rowValues) { - $batch[] = $rowValues; - - if (count($batch) >= $batchSize) { - yield $batch; - $batch = []; - } + foreach (array_values($valueRange->getValues() ?? []) as $index => $cells) { + // page $page covers the rows from 1 + $page * $rowsPerPage on; the API leaves out a page's trailing empty rows + yield 1 + ($page * $rowsPerPage) + $index => array_values($cells); } } - - if ($batch !== []) { - yield $batch; - } } public function rowCount(): int @@ -90,11 +80,27 @@ public function sample(int $rows): GoogleSheetSample /** @var Sheets\Resource\SpreadsheetsValues $values */ $values = $this->service->spreadsheets_values; $decoder = $this->readOptions->decoder(); + $cells = []; + $decoded = []; + // @mago-ignore analysis:redundant-null-coalesce - $decoded = $decoder->decode(array_values( + foreach (array_values( $values->get($this->spreadsheetId, $range->toString(), $this->readOptions->options)->getValues() ?? [], - )); + ) as $index => $row) { + $cells[$index + 1] = array_values($row); + $rowValues = $decoder->decode($cells[$index + 1]); + + if (is_array($rowValues)) { + $decoded[] = $rowValues; + } + } - return new GoogleSheetSample($decoder->headers() ?? [], $decoded, $range->endRow >= $rowCount, $rowCount); + return new GoogleSheetSample( + $decoder->headers() ?? [], + $decoded, + $cells, + $range->endRow >= $rowCount, + $rowCount, + ); } } diff --git a/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Inference/GoogleSheetSample.php b/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Inference/GoogleSheetSample.php index 2e5ae69017..2877e52f2b 100644 --- a/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Inference/GoogleSheetSample.php +++ b/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Inference/GoogleSheetSample.php @@ -8,7 +8,8 @@ { /** * @param list $names the header (or generated names) - known even when no data row follows - * @param list> $rows + * @param list> $rows the rows of the records that are not corrupted + * @param array> $cells the raw cells of every row, keyed by its 1-based number in the sheet * @param bool $wholeSheet the range reached the last row of the grid, so $rows IS the sheet and a read can be * served from it instead of fetching the same rows again * @param int $gridRowCount the rows the grid allocates - an upper bound on the rows it holds @@ -16,6 +17,7 @@ public function __construct( public array $names, public array $rows, + public array $cells, public bool $wholeSheet, public int $gridRowCount, ) {} diff --git a/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Inference/GoogleSheetSniffer.php b/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Inference/GoogleSheetSniffer.php index cce11193af..d436a516de 100644 --- a/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Inference/GoogleSheetSniffer.php +++ b/src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Inference/GoogleSheetSniffer.php @@ -11,8 +11,6 @@ use Flow\Types\Type\TypeNarrower; use Generator; -use function count; - final class GoogleSheetSniffer implements Sniffer { private ?GoogleSheetSample $sample = null; @@ -27,34 +25,20 @@ public function __construct( ) {} /** - * The rows the sample already holds, batched, when its range covered the whole grid - so a read does not fetch - * the same rows a second time. Null when it did not, and the caller must read the sheet itself. - * - * @param int<1, max> $batchSize + * The raw rows the sample already holds, keyed by their number in the sheet, when its range covered the whole + * grid - so a read does not fetch the same rows a second time. Null when it did not, and the caller must read the + * sheet itself. * - * @return null|Generator>> + * @return null|Generator> */ - public function batches(int $batchSize): ?Generator + public function rows(): ?Generator { if (!$this->sample()->wholeSheet) { return null; } - return (function () use ($batchSize): Generator { - $batch = []; - - foreach ($this->sample()->rows as $rowValues) { - $batch[] = $rowValues; - - if (count($batch) >= $batchSize) { - yield $batch; - $batch = []; - } - } - - if ($batch !== []) { - yield $batch; - } + return (function (): Generator { + yield from $this->sample()->cells; })(); } diff --git a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/extra-empty-rows.json b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/extra-empty-rows.json index 8a030290b7..daa1229811 100644 --- a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/extra-empty-rows.json +++ b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/extra-empty-rows.json @@ -16,7 +16,7 @@ ["A8", "B8", "C8"], ["A9", "B9", "C9"], ["A10", "B10", "C10"], - ["A11", "B11", "C11", "D11"] + ["A11", "B11", "C11"] ] } ] diff --git a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/sample-extra-empty-rows.json b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/sample-extra-empty-rows.json index 050e4d489a..2c0f9da591 100644 --- a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/sample-extra-empty-rows.json +++ b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/sample-extra-empty-rows.json @@ -56,8 +56,7 @@ [ "A11", "B11", - "C11", - "D11" + "C11" ] ] } diff --git a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Integration/IO/Extractor/GoogleSheetExtractorTest.php b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Integration/IO/Extractor/GoogleSheetExtractorTest.php index 250c7de3b8..930b26ea87 100644 --- a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Integration/IO/Extractor/GoogleSheetExtractorTest.php +++ b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Integration/IO/Extractor/GoogleSheetExtractorTest.php @@ -5,12 +5,13 @@ namespace Flow\ETL\Adapter\GoogleSheet\Tests\Integration\IO\Extractor; use Flow\ETL\Adapter\GoogleSheet\Tests\Context\GoogleSheetsContext; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; -use Flow\ETL\Foundation\Exception\InvalidArgumentException; -use Flow\ETL\Foundation\Exception\SchemaMismatchException; +use Flow\ETL\Foundation\Exception\SchemaValidationException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Rows; +use PHPUnit\Framework\Attributes\TestWith; use function array_map; use function Flow\ETL\Adapter\GoogleSheet\from_google_sheet; @@ -47,7 +48,7 @@ public function test_extract_expand_missing_columns(): void ), '1234567890', 'Sheet', - )) + )->withMissingColumnsAsNull(true)) ->fetch() ->toArray(); @@ -58,45 +59,21 @@ public function test_extract_expand_missing_columns(): void } } - public function test_extract_puts_null_in_a_declared_nullable_column_the_sheet_lacks(): void - { - $rows = df() - ->read(from_google_sheet( - $this->context->sheets(__DIR__ . '/../../../Fixtures/extra-empty-rows.json'), - '1234567890', - 'Sheet', - )->withSchema(schema( - string_schema('Header 1'), - string_schema('Header 2'), - int_schema('id', nullable: true), - ))) - ->fetch() - ->toArray(); - - static::assertCount(10, $rows); - - foreach ($rows as $row) { - static::assertNotSame([], $row); - static::assertArrayNotHasKey('Header 3', $row); - static::assertNull($row['id']); - } - } - - /** - * b57: the sheet has no such column, so every row would carry a null under a NOT NULL - * declaration. Declare the column nullable if that is what the data is. - */ - public function test_extract_refuses_a_not_null_column_the_sheet_lacks(): void + #[TestWith([true])] + #[TestWith([false])] + public function test_extract_refuses_a_declared_column_the_sheet_lacks(bool $nullable): void { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage('column "id" (row 0) declared by the schema is missing from the row'); + $this->expectException(SchemaValidationException::class); + $this->expectExceptionMessage( + 'Columns "id" declared by the schema are not in the header of google-sheet://1234567890/Sheet', + ); df() ->read(from_google_sheet( $this->context->sheets(__DIR__ . '/../../../Fixtures/extra-empty-rows.json'), '1234567890', 'Sheet', - )->withSchema(schema(string_schema('Header 1'), string_schema('Header 2'), int_schema('id')))) + )->withSchema(schema(string_schema('Header 1'), string_schema('Header 2'), int_schema('id', $nullable)))) ->fetch(); } @@ -167,9 +144,12 @@ public function test_extract_with_batches_without_header(): void static::assertCount(20, df()->read($extractor)->fetch()->toArray()); } - public function test_extract_with_cut_extra_columns(): void + public function test_extract_refuses_a_row_with_an_extra_cell(): void { - $rows = df() + $this->expectException(CorruptedRecordException::class); + $this->expectExceptionMessage('4 fields, the header has 3'); + + df() ->read(from_google_sheet( $this->context->sheets( __DIR__ . '/../../../Fixtures/sample-extra-columns.json', @@ -178,14 +158,7 @@ public function test_extract_with_cut_extra_columns(): void '1234567890', 'Sheet', )) - ->fetch() - ->toArray(); - - static::assertCount(10, $rows); - - foreach ($rows as $row) { - static::assertCount(3, $row); - } + ->fetch(); } public function test_extract_with_limit(): void @@ -203,10 +176,10 @@ public function test_extract_with_limit(): void static::assertCount(2, df()->read($extractor)->limit(2)->fetch()->toArray()); } - public function test_extract_without_cut_extra_columns(): void + public function test_missing_columns_as_null_still_refuses_a_row_with_an_extra_cell(): void { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Row has more columns (4) than headers (3)'); + $this->expectException(CorruptedRecordException::class); + $this->expectExceptionMessage('4 fields, the header has 3'); df() ->read(from_google_sheet( @@ -216,9 +189,8 @@ public function test_extract_without_cut_extra_columns(): void ), '1234567890', 'Sheet', - )->withDropExtraColumns(false)) - ->fetch() - ->toArray(); + )->withMissingColumnsAsNull(true)) + ->fetch(); } public function test_schema_is_inferred_before_any_row_is_read(): void @@ -362,9 +334,9 @@ public function test_a_header_that_changed_since_the_sample_is_refused(): void public function test_a_value_past_the_sample_that_does_not_fit_is_refused(): void { - $this->expectException(SchemaMismatchException::class); + $this->expectException(InferredSchemaException::class); $this->expectExceptionMessageMatches('/column "price"/'); - $this->expectExceptionMessageMatches('/row 3/'); + $this->expectExceptionMessageMatches('/offset 5 of google-sheet:\/\/1234567890\/Sheet/'); df() ->read(from_google_sheet( diff --git a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Integration/IO/Extractor/GoogleSheetExtractorTypedColumnsTest.php b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Integration/IO/Extractor/GoogleSheetExtractorTypedColumnsTest.php index ab05b2f9b6..4d9b62d570 100644 --- a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Integration/IO/Extractor/GoogleSheetExtractorTypedColumnsTest.php +++ b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Integration/IO/Extractor/GoogleSheetExtractorTypedColumnsTest.php @@ -34,7 +34,11 @@ public function test_appends_spread_sheet_id_and_sheet_name_to_the_schema_when_m $rows = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $batch) { foreach ($batch->toArray() as $row) { $rows[] = $row; } @@ -62,7 +66,11 @@ public function test_infers_typed_rows(): void $rows = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $batch) { static::assertTrue( $batch ->schema() diff --git a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetExtractorTest.php b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetExtractorTest.php index b7f9b33e09..35bb928f00 100644 --- a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetExtractorTest.php +++ b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetExtractorTest.php @@ -9,9 +9,10 @@ use Flow\ETL\Adapter\GoogleSheet\Tests\Context\GoogleSheetFixtureContext; use Flow\ETL\Adapter\GoogleSheet\Tests\Double\StubSpreadsheetsResource; use Flow\ETL\Adapter\GoogleSheet\Tests\Mother\SheetValuesMother; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; -use Flow\ETL\Foundation\Exception\SchemaMismatchException; +use Flow\ETL\Foundation\Exception\SchemaValidationException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\Statistics; use Flow\ETL\Tests\Context\ExtractedRows; @@ -21,14 +22,18 @@ use Flow\ETL\Values\Rows; use Google\Service\Sheets; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; +use function array_map; use function array_sum; use function Flow\ETL\DSL\config_builder; +use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\infer_schema; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\to_array; use function iterator_to_array; use function max; @@ -43,7 +48,9 @@ public static function memo_keeping_setters(): iterable public static function memo_resetting_setters(): iterable { yield 'withHeader' => [static fn(GoogleSheetExtractor $e): mixed => $e->withHeader(false)]; - yield 'withDropExtraColumns' => [static fn(GoogleSheetExtractor $e): mixed => $e->withDropExtraColumns(false)]; + yield 'withMissingColumnsAsNull' => [ + static fn(GoogleSheetExtractor $e): mixed => $e->withMissingColumnsAsNull(true), + ]; yield 'withOptions' => [ static fn(GoogleSheetExtractor $e): mixed => $e->withOptions(['valueRenderOption' => 'UNFORMATTED_VALUE']), ]; @@ -64,8 +71,8 @@ public function test_extract_does_not_mutate_user_provided_schema(): void ->withSchema($userSchema) ->withMetadataColumns(true); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria())); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria())); static::assertNull($userSchema->findDefinition('_spread_sheet_id')); static::assertNull($userSchema->findDefinition('_sheet_name')); @@ -83,7 +90,7 @@ public function test_its_fails_if_sheet_not_found(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Max rows "0" must be greater than 0'); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria())); } public function test_schema_fails_the_same_way_for_a_missing_sheet(): void @@ -108,6 +115,33 @@ public function test_schema_of_everything_fails_on_a_missing_sheet(): void ->schema(); } + /** + * @param list $cells + */ + #[TestWith([['4', '5', '6', '7', '8']])] + #[TestWith([['4', '5']])] + public function test_a_row_whose_cell_count_differs_from_the_header_is_refused(array $cells): void + { + try { + ExtractedRows::of(GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, GoogleSheetFixtureContext::valuesAndBatches([], [ + SheetValuesMother::batch([[['a', 'b', 'c'], ['1', '2', '3'], $cells]]), + ])))->withSchema(schema(str_schema('a'), str_schema('b'), str_schema('c', true)))); + static::fail('Expected the corrupted record to be refused'); + } catch (CorruptedRecordException $e) { + static::assertSame([3, null, $cells], [$e->offset, $e->lineNumber, $e->fields]); + } + } + + public function test_a_declared_column_the_header_lacks_is_a_schema_error(): void + { + $this->expectException(SchemaValidationException::class); + $this->expectExceptionMessage('"x"'); + + ExtractedRows::of(GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, GoogleSheetFixtureContext::valuesAndBatches([], [ + SheetValuesMother::batch([[['a', 'b', 'c'], ['1', '2', '3']]]), + ])))->withSchema(schema(str_schema('a'), str_schema('b'), str_schema('x', true)))); + } + public function test_its_stop_fetching_data_if_processed_row_count_is_less_then_last_range_end_row(): void { $rows = ExtractedRows::of( @@ -225,7 +259,7 @@ public function test_the_sample_is_taken_once(): void $extractor->schema(); $extractor->schema(); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria())); static::assertCount(1, $values->getCalls); @@ -305,7 +339,11 @@ public function test_extract_seeds_every_batch_with_the_inferred_schema(): void $sizes = []; $first = null; - foreach ($extractor->withBatchSize(2)->extract(flow_context()->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor->withBatchSize(2), + flow_context()->backend(), + new Criteria(), + ) as $rows) { static::assertTrue($rows->schema()->isSame($extractor->schema())); $sizes[] = $rows->count(); $first ??= $rows->toArray()[0]; @@ -331,24 +369,27 @@ public function test_extract_refuses_a_header_that_diverges_from_the_sample(): v $this->expectExceptionMessageMatches('/spreadsheet "spread-id" sheet "sheet"/'); $this->expectExceptionMessageMatches('/unexpected \[title\], missing \[name\]/'); - iterator_to_array(GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service( - 500, - $values, - ))->extract(flow_context()->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted( + GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(500, $values)), + flow_context()->backend(), + new Criteria(), + )); } public function test_union_by_name_skips_the_divergence_check(): void { static::assertCount( 1, - iterator_to_array( + iterator_to_array(FlowTestCase::extracted( GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, GoogleSheetFixtureContext::valuesAndBatches([SheetValuesMother::range([ ['id', 'name'], ['1', 'a'], - ])], [SheetValuesMother::batch([[['id', 'title'], ['1', 'a']]])]))) - ->inferSchema(infer_schema()->unionByName()) - ->extract(flow_context()->backend(), new Criteria()), - ), + ])], [SheetValuesMother::batch([[['id', 'title'], ['1', 'a']]])])))->inferSchema( + infer_schema()->unionByName(), + ), + flow_context()->backend(), + new Criteria(), + )), ); } @@ -362,10 +403,11 @@ public function test_extract_refuses_a_batch_missing_a_sampled_column(): void $this->expectException(InferredSchemaException::class); $this->expectExceptionMessageMatches('/unexpected \[\], missing \[name\]/'); - iterator_to_array(GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service( - 500, - $values, - ))->extract(flow_context()->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted( + GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(500, $values)), + flow_context()->backend(), + new Criteria(), + )); } public function test_extract_refuses_a_batch_with_a_column_the_sample_never_saw(): void @@ -381,23 +423,24 @@ public function test_extract_refuses_a_batch_with_a_column_the_sample_never_saw( $this->expectException(InferredSchemaException::class); $this->expectExceptionMessageMatches('/unexpected \[extra\], missing \[\]/'); - iterator_to_array(GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service( - 500, - $values, - ))->extract(flow_context()->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted( + GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(500, $values)), + flow_context()->backend(), + new Criteria(), + )); } public function test_metadata_columns_keep_a_numeric_column_name(): void { /** @var array $rowsArray */ - $rowsArray = iterator_to_array( + $rowsArray = iterator_to_array(FlowTestCase::extracted( GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, GoogleSheetFixtureContext::valuesAndBatches([SheetValuesMother::range([ ['3', '7'], ['a', 'b'], - ])], [SheetValuesMother::batch([[['3', '7'], ['a', 'b']]])]))) - ->withMetadataColumns(true) - ->extract(flow_context()->backend(), new Criteria()), - ); + ])], [SheetValuesMother::batch([[['3', '7'], ['a', 'b']]])])))->withMetadataColumns(true), + flow_context()->backend(), + new Criteria(), + )); static::assertSame( ['3' => 'a', '7' => 'b', '_spread_sheet_id' => 'spread-id', '_sheet_name' => 'sheet'], @@ -409,10 +452,11 @@ public function test_extract_accepts_the_sampled_columns_in_another_order(): voi { static::assertCount( 1, - iterator_to_array(GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, GoogleSheetFixtureContext::valuesAndBatches([SheetValuesMother::range([ - ['id', 'name'], - ['1', 'a'], - ])], [SheetValuesMother::batch([[['name', 'id'], ['a', '1']]])])))->extract( + iterator_to_array(FlowTestCase::extracted( + GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, GoogleSheetFixtureContext::valuesAndBatches([SheetValuesMother::range([ + ['id', 'name'], + ['1', 'a'], + ])], [SheetValuesMother::batch([[['name', 'id'], ['a', '1']]])]))), flow_context()->backend(), new Criteria(), )), @@ -428,11 +472,14 @@ public function test_a_declared_schema_skips_the_sample_and_the_check(): void static::assertCount( 1, - iterator_to_array( - GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, $values)) - ->withSchema(schema(str_schema('id'))) - ->extract(flow_context()->backend(), new Criteria()), - ), + iterator_to_array(FlowTestCase::extracted( + GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service( + 100, + $values, + ))->withSchema(schema(str_schema('id'))), + flow_context()->backend(), + new Criteria(), + )), ); static::assertSame([], $values->getCalls); } @@ -440,14 +487,14 @@ public function test_a_declared_schema_skips_the_sample_and_the_check(): void public function test_empty_cells_are_null_by_default(): void { /** @var array $rowsArray */ - $rowsArray = iterator_to_array( + $rowsArray = iterator_to_array(FlowTestCase::extracted( GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, GoogleSheetFixtureContext::valuesAndBatches([], [SheetValuesMother::batch([[ ['id', 'name'], ['1', ''], - ]])]))) - ->withSchema(schema(str_schema('id'), str_schema('name', nullable: true))) - ->extract(flow_context()->backend(), new Criteria()), - ); + ]])])))->withSchema(schema(str_schema('id'), str_schema('name', nullable: true))), + flow_context()->backend(), + new Criteria(), + )); static::assertNull($rowsArray[0]->toArray()[0]['name']); } @@ -455,15 +502,16 @@ public function test_empty_cells_are_null_by_default(): void public function test_empty_cells_stay_strings_when_asked(): void { /** @var array $rowsArray */ - $rowsArray = iterator_to_array( + $rowsArray = iterator_to_array(FlowTestCase::extracted( GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, GoogleSheetFixtureContext::valuesAndBatches([], [SheetValuesMother::batch([[ ['id', 'name'], ['1', ''], ]])]))) ->withEmptyToNull(false) - ->withSchema(schema(str_schema('id'), str_schema('name', nullable: true))) - ->extract(flow_context()->backend(), new Criteria()), - ); + ->withSchema(schema(str_schema('id'), str_schema('name', nullable: true))), + flow_context()->backend(), + new Criteria(), + )); static::assertSame('', $rowsArray[0]->toArray()[0]['name']); } @@ -475,15 +523,18 @@ public function test_a_value_past_the_sample_that_does_not_fit_is_refused(): voi ['1'], ])], [SheetValuesMother::batch([[['id'], ['1'], ['n/a']]])]); - $this->expectException(SchemaMismatchException::class); + $this->expectException(InferredSchemaException::class); $this->expectExceptionMessageMatches('/column "id"/'); - $this->expectExceptionMessageMatches('/row 1/'); + $this->expectExceptionMessageMatches('/offset 3 of google-sheet:\/\/spread-id\/sheet/'); - iterator_to_array( - GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(500, $values)) - ->inferSchema(infer_schema()->sampleSize(1)) - ->extract(flow_context()->backend(), new Criteria()), - ); + iterator_to_array(FlowTestCase::extracted( + GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service( + 500, + $values, + ))->inferSchema(infer_schema()->sampleSize(1)), + flow_context()->backend(), + new Criteria(), + )); } public function test_a_leading_blank_row_without_a_header_is_not_a_divergence(): void @@ -505,7 +556,10 @@ public function test_works_for_no_data(): void null, )], [SheetValuesMother::batch([null])])))->withRowsPerPage(20); - static::assertCount(0, iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()))); + static::assertCount( + 0, + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria())), + ); $this->expectException(InferredSchemaException::class); $this->expectExceptionMessage( @@ -593,7 +647,7 @@ public function test_a_read_samples_the_grid_row_count_too(): void ['2'], ])], []))); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()), false); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false); static::assertEquals(Cardinality::atMost(3), $extractor->statistics()->rows); } @@ -618,15 +672,42 @@ public function test_extract_builds_through_the_config_backend(): void $backend = new SpyBackend(); iterator_to_array( - GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, GoogleSheetFixtureContext::valuesAndBatches([], [SheetValuesMother::batch([[ - ['header'], - ['row1'], - ]])]))) - ->withSchema(schema(str_schema('header'))) - ->extract(flow_context(config_builder()->backend($backend)->build())->backend(), new Criteria()), + FlowTestCase::extracted( + GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, GoogleSheetFixtureContext::valuesAndBatches([], [SheetValuesMother::batch([[ + ['header'], + ['row1'], + ]])])))->withSchema(schema(str_schema('header'))), + flow_context(config_builder()->backend($backend)->build())->backend(), + new Criteria(), + ), false, ); static::assertGreaterThanOrEqual(1, $backend->builders()); } + + public function test_corrupted_records_are_routed_to_the_reject_loader(): void + { + $rejects = []; + + $rows = df() + ->read( + GoogleSheetFixtureContext::extractor(GoogleSheetFixtureContext::service(100, GoogleSheetFixtureContext::valuesAndBatches([], [ + SheetValuesMother::batch([[['a', 'b'], ['1', '2'], ['3', '4', '5'], ['6', '7']]]), + ]))) + ->withSchema(schema(str_schema('a'), str_schema('b'))) + ->withCorruptedRecords(to_array($rejects)), + ) + ->fetch(); + + static::assertSame(['1', '6'], $rows->reduceToArray('a')); + static::assertSame( + [[3, '3 fields, the header has 2', ['3', '4', '5']]], + array_map(static fn(array $reject): array => [ + $reject['offset'], + $reject['reason'], + $reject['fields'], + ], $rejects), + ); + } } diff --git a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetReadOptionsTest.php b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetReadOptionsTest.php index 1c4b4ef48d..4559e34dcd 100644 --- a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetReadOptionsTest.php +++ b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetReadOptionsTest.php @@ -5,7 +5,6 @@ namespace Flow\ETL\Adapter\GoogleSheet\Tests\Unit\IO\Extractor; use Flow\ETL\Adapter\GoogleSheet\IO\Extractor\GoogleSheetReadOptions; -use Flow\ETL\Foundation\Exception\InvalidArgumentException; use Flow\ETL\IO\Inference\InferredTypes; use Flow\ETL\Tests\FlowTestCase; use Flow\Types\Type\Logical\InstanceOfTypeNarrower; @@ -18,7 +17,7 @@ public function test_defaults(): void $options = new GoogleSheetReadOptions(); static::assertTrue($options->withHeader); - static::assertTrue($options->dropExtraColumns); + static::assertFalse($options->missingColumnsAsNull); static::assertTrue($options->emptyToNull); static::assertSame([], $options->options); } @@ -28,17 +27,17 @@ public function test_with_header_changes_only_the_header_flag(): void $options = (new GoogleSheetReadOptions())->withHeader(false); static::assertFalse($options->withHeader); - static::assertTrue($options->dropExtraColumns); + static::assertFalse($options->missingColumnsAsNull); static::assertTrue($options->emptyToNull); static::assertSame([], $options->options); } - public function test_with_drop_extra_columns_changes_only_that_flag(): void + public function test_with_missing_columns_as_null_changes_only_that_flag(): void { - $options = (new GoogleSheetReadOptions())->withDropExtraColumns(false); + $options = (new GoogleSheetReadOptions())->withMissingColumnsAsNull(true); static::assertTrue($options->withHeader); - static::assertFalse($options->dropExtraColumns); + static::assertTrue($options->missingColumnsAsNull); static::assertTrue($options->emptyToNull); static::assertSame([], $options->options); } @@ -48,7 +47,7 @@ public function test_with_empty_to_null_changes_only_that_flag(): void $options = (new GoogleSheetReadOptions())->withEmptyToNull(false); static::assertTrue($options->withHeader); - static::assertTrue($options->dropExtraColumns); + static::assertFalse($options->missingColumnsAsNull); static::assertFalse($options->emptyToNull); static::assertSame([], $options->options); } @@ -58,7 +57,7 @@ public function test_with_options_changes_only_the_options(): void $options = (new GoogleSheetReadOptions())->withOptions(['valueRenderOption' => 'UNFORMATTED_VALUE']); static::assertTrue($options->withHeader); - static::assertTrue($options->dropExtraColumns); + static::assertFalse($options->missingColumnsAsNull); static::assertTrue($options->emptyToNull); static::assertSame(['valueRenderOption' => 'UNFORMATTED_VALUE'], $options->options); } @@ -72,25 +71,22 @@ public function test_decoder_is_a_fresh_instance_per_call(): void public function test_decoder_carries_the_empty_to_null_flag(): void { - static::assertSame( - [['a' => '']], - (new GoogleSheetReadOptions()) - ->withEmptyToNull(false) - ->decoder() - ->decode([['a'], ['']]), - ); + $decoder = (new GoogleSheetReadOptions()) + ->withEmptyToNull(false) + ->decoder(); + $decoder->decode(['a']); + + static::assertSame(['a' => ''], $decoder->decode([''])); } - public function test_decoder_carries_the_drop_extra_columns_flag(): void + public function test_decoder_carries_the_missing_columns_as_null_flag(): void { $decoder = (new GoogleSheetReadOptions()) - ->withDropExtraColumns(false) + ->withMissingColumnsAsNull(true) ->decoder(); + $decoder->decode(['a', 'b']); - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Row has more columns (2) than headers (1)'); - - $decoder->decode([['a'], ['x', 'y']]); + static::assertSame(['a' => 'x', 'b' => null], $decoder->decode(['x'])); } public function test_typer_runs_the_string_ladder_under_the_default_render_option(): void diff --git a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetReaderTest.php b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetReaderTest.php index 1ebbc21772..ea1003741e 100644 --- a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetReaderTest.php +++ b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetReaderTest.php @@ -162,7 +162,7 @@ public function test_sample_of_everything_on_an_absent_sheet_fails(): void )->sample(-1); } - public function test_batches_page_the_sheet_by_rows_per_page(): void + public function test_rows_page_the_sheet_by_rows_per_page(): void { $values = GoogleSheetFixtureContext::valuesAndBatches([], [SheetValuesMother::batch([ [['id'], ['1']], @@ -170,67 +170,60 @@ public function test_batches_page_the_sheet_by_rows_per_page(): void [['4']], ])]); - iterator_to_array( - GoogleSheetFixtureContext::reader(GoogleSheetFixtureContext::service(5, $values))->batches(2, 100), - false, - ); + iterator_to_array(GoogleSheetFixtureContext::reader(GoogleSheetFixtureContext::service(5, $values))->rows(2)); static::assertSame(['sheet!A1:B2', 'sheet!A3:B4', 'sheet!A5:B5'], $values->batchGetCalls[0][1]['ranges']); } - public function test_batches_forward_the_render_options(): void + public function test_rows_forward_the_render_options(): void { $values = GoogleSheetFixtureContext::valuesAndBatches([], [SheetValuesMother::batch([[['id'], ['1']]])]); - iterator_to_array( - GoogleSheetFixtureContext::reader( - GoogleSheetFixtureContext::service(2, $values), - new GoogleSheetReadOptions(options: ['valueRenderOption' => 'UNFORMATTED_VALUE']), - )->batches(2, 100), - false, - ); + iterator_to_array(GoogleSheetFixtureContext::reader( + GoogleSheetFixtureContext::service(2, $values), + new GoogleSheetReadOptions(options: ['valueRenderOption' => 'UNFORMATTED_VALUE']), + )->rows(2)); static::assertSame('UNFORMATTED_VALUE', $values->batchGetCalls[0][1]['valueRenderOption']); static::assertSame(['sheet!A1:B2'], $values->batchGetCalls[0][1]['ranges']); } - public function test_batches_are_cut_at_the_batch_size_across_pages(): void + public function test_rows_are_keyed_by_their_number_in_the_sheet_across_pages(): void { $values = GoogleSheetFixtureContext::valuesAndBatches([], [SheetValuesMother::batch([ - [['id'], ['1'], ['2'], ['3']], - [['4'], ['5'], ['6']], + [['id'], ['1'], ['2']], + [['4'], ['5']], ])]); - $batches = iterator_to_array( - GoogleSheetFixtureContext::reader(GoogleSheetFixtureContext::service(10, $values))->batches(10, 4), - false, + static::assertSame( + [1 => ['id'], 2 => ['1'], 3 => ['2'], 5 => ['4'], 6 => ['5']], + iterator_to_array(GoogleSheetFixtureContext::reader(GoogleSheetFixtureContext::service(8, $values))->rows( + 4, + )), ); - - static::assertSame([4, 2], array_map('count', $batches)); } - public function test_batches_skip_pages_without_values(): void + public function test_rows_skip_pages_without_values(): void { $values = GoogleSheetFixtureContext::valuesAndBatches([], [SheetValuesMother::batch([[['id'], ['1']], null])]); - $batches = iterator_to_array( - GoogleSheetFixtureContext::reader(GoogleSheetFixtureContext::service(4, $values))->batches(2, 100), - false, + static::assertSame( + [1 => ['id'], 2 => ['1']], + iterator_to_array(GoogleSheetFixtureContext::reader(GoogleSheetFixtureContext::service(4, $values))->rows( + 2, + )), ); - - static::assertSame([1], array_map('count', $batches)); } - public function test_batches_yield_nothing_for_an_empty_sheet(): void + public function test_rows_yield_nothing_for_an_empty_sheet(): void { $values = GoogleSheetFixtureContext::valuesAndBatches([], [SheetValuesMother::batch([null])]); static::assertSame( [], - iterator_to_array( - GoogleSheetFixtureContext::reader(GoogleSheetFixtureContext::service(2, $values))->batches(2, 100), - false, - ), + iterator_to_array(GoogleSheetFixtureContext::reader(GoogleSheetFixtureContext::service(2, $values))->rows( + 2, + )), ); } diff --git a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Inference/GoogleSheetSampleTest.php b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Inference/GoogleSheetSampleTest.php index 435169872e..277a36c4d9 100644 --- a/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Inference/GoogleSheetSampleTest.php +++ b/src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Inference/GoogleSheetSampleTest.php @@ -11,10 +11,17 @@ final class GoogleSheetSampleTest extends FlowTestCase { public function test_carries_names_and_rows(): void { - $sample = new GoogleSheetSample(['a'], [['a' => 1]], wholeSheet: true, gridRowCount: 1000); + $sample = new GoogleSheetSample( + ['a'], + [['a' => 1]], + [1 => ['a'], 2 => [1]], + wholeSheet: true, + gridRowCount: 1000, + ); static::assertSame(['a'], $sample->names); static::assertSame([['a' => 1]], [$sample->rows[0]]); + static::assertSame([1 => ['a'], 2 => [1]], $sample->cells); static::assertTrue($sample->wholeSheet); static::assertSame(1000, $sample->gridRowCount); } diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonExtractor.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonExtractor.php index b7e72ec8f4..2a8e50dc8b 100644 --- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonExtractor.php +++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonExtractor.php @@ -6,9 +6,11 @@ use Flow\ETL\Adapter\JSON\IO\Inference\JsonSniffer; use Flow\ETL\Adapter\JSON\IO\JsonFraming; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; use Flow\ETL\Functions\ScalarFunction; +use Flow\ETL\IO\CorruptedRecordsRouter; use Flow\ETL\IO\Extractor; use Flow\ETL\IO\Extractor\BatchableExtractor; use Flow\ETL\IO\Extractor\Criteria; @@ -29,6 +31,7 @@ use Flow\ETL\IO\Inference\SchemaInference; use Flow\ETL\IO\Inference\SchemaInferenceBuilder; use Flow\ETL\IO\Inference\SchemaInferrer; +use Flow\ETL\IO\Loader; use Flow\ETL\Values\BatchSize; use Flow\ETL\Values\Cardinality; use Flow\ETL\Values\Column\Backend; @@ -47,6 +50,7 @@ final class JsonExtractor implements FileExtractor, InfersSchema, MetadataColumnsExtractor, + CorruptedRecordsRouter, SplittableExtractor { private BatchSize $batchSize; @@ -63,11 +67,20 @@ final class JsonExtractor implements private bool $pointerToColumnName = false; + public function withCorruptedRecords(?Loader $loader): static + { + $this->corruptedRecordsLoader = $loader; + + return $this; + } + /** * What the last schema inference sampled; null until one ran. */ private ?SampledUnits $sampled = null; + private ?Loader $corruptedRecordsLoader = null; + private ?Schema $schema = null; private ?Statistics $statistics = null; @@ -107,8 +120,13 @@ public function isRepeatable(): bool return true; } + public function corruptedRecordsLoader(): ?Loader + { + return $this->corruptedRecordsLoader; + } + /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { @@ -122,7 +140,9 @@ public function reader(): FileSplitReader new JsonFileBatches( $this->files->filesystem, new JsonFileReader($this->files->filesystem, $this->format, $this->pointer, $this->pointerToColumnName), + sampleSize: $this->schema === null ? $this->inference->sampleSize : null, ), + $this->corruptedRecordsLoader, ); } diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonFileBatches.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonFileBatches.php index 266889d9b7..9445d4eb90 100644 --- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonFileBatches.php +++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonFileBatches.php @@ -8,6 +8,7 @@ use Flow\ETL\Foundation\Describe\ConfiguredFilesystem; use Flow\ETL\Foundation\Describe\Description; use Flow\ETL\Foundation\Describe\Rebuild; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\FlowPhpExtension; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\File\FileBatches; @@ -23,6 +24,8 @@ use SensitiveParameter; use Throwable; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_optional; use function str_starts_with; use function strlen; use function strspn; @@ -43,6 +46,7 @@ public function __construct( private Filesystem $filesystem, private JsonFileReader $reader, private ?FlowPhpExtension $extension = null, + private ?int $sampleSize = null, ) {} public static function fromDescription(#[SensitiveParameter] array $data, Rebuild $rebuild): static @@ -50,6 +54,7 @@ public static function fromDescription(#[SensitiveParameter] array $data, Rebuil return new self( $rebuild->of($data['filesystem'] ?? null, ConfiguredFilesystem::class)->filesystem(), $rebuild->of($data['reader'] ?? null, JsonFileReader::class), + sampleSize: type_optional(type_integer())->assert($data['sample_size'] ?? null), ); } @@ -58,11 +63,12 @@ public function describe(): ?Description return Description::compose(self::class, [ 'filesystem' => new ConfiguredFilesystem($this->filesystem), 'reader' => $this->reader, + 'sample_size' => $this->sampleSize, ]); } /** - * @return Generator + * @return Generator */ public function batches( FileRead $read, @@ -75,7 +81,7 @@ public function batches( $open = $this->open($source, $criteria->split); try { - yield from $open->batches($body, $read->batchSize, $backend); + yield from $open->batches($body, $read->batchSize, $backend, $this->sampleSize, $read->schema); } finally { $open->close(); } diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonFileReader.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonFileReader.php index 4aaff85d0d..178b2fc1ed 100644 --- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonFileReader.php +++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonFileReader.php @@ -14,6 +14,7 @@ use Flow\ETL\IO\Extractor\File\SourceFile; use Flow\Filesystem\Filesystem; use Flow\Filesystem\SourceStream; +use Flow\Filesystem\Stream\RangeSourceStream; use Generator; use JsonMachine\Items; use JsonMachine\JsonDecoder\ExtJsonDecoder; @@ -153,16 +154,28 @@ public function documentItems(SourceStream $stream): Generator * * @return Generator */ - public function lineItems(SourceStream $stream, JsonReadBytes $read = new JsonReadBytes()): Generator + public function lineItems(SourceStream $stream, JsonReadProgress $read = new JsonReadProgress()): Generator { + $offset = $stream instanceof RangeSourceStream ? $stream->start() : 0; + // a window cut past the file's start cannot know how many lines came before it + $number = $offset === 0 ? 0 : null; + foreach ($stream->readLines() as $line) { + $start = $offset; // readLines() strips the "\n" and hides whether the last line had one, so that line counts 1 byte more + $offset += strlen($line) + 1; $read->add(strlen($line) + 1); + if ($number !== null) { + $number++; + } + if (trim($line, self::C_ISSPACE) === '') { continue; } + $read->recordAt($start, $number, $line); + if ($this->pointer === null) { // json_decode is an order of magnitude faster and agrees with JSON Machine on every object or // array line; a line it rejects (a BOM, two objects, a bare scalar) keeps JSON Machine's verdict @@ -188,7 +201,7 @@ public function lineItems(SourceStream $stream, JsonReadBytes $read = new JsonRe */ public function sample( SourceFile $source, - JsonReadBytes $read = new JsonReadBytes(), + JsonReadProgress $read = new JsonReadProgress(), ?FileSplit $window = null, ): Generator { $stream = $this->filesystem->readFrom($source->path); @@ -208,8 +221,14 @@ public function sample( JsonFraming::Lines => $this->lineItems($stream, $read), }; + $element = 0; + // @mago-ignore analysis:mixed-assignment foreach ($items as $row) { + if ($this->format !== JsonFraming::Lines) { + $read->recordAt(++$element, null, null); + } + if ($this->pointer !== null && $this->pointerToColumnName) { $row = [$this->pointer => $row]; } diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonOpenSource.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonOpenSource.php index 52f763ce5a..50cfd2a71c 100644 --- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonOpenSource.php +++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonOpenSource.php @@ -4,6 +4,8 @@ namespace Flow\ETL\Adapter\JSON\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; +use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\SchemaMismatchException; use Flow\ETL\Values\Column\Backend; use Flow\ETL\Values\Rows; @@ -21,14 +23,22 @@ interface JsonOpenSource public function close(): void; /** - * Batches of exactly $batchSize rows (the last may be shorter), keyed and ordered by $schema; row indexes in - * refusals are relative to the source. This instance is consumed afterwards. + * Batches of exactly $batchSize rows (the last may be shorter), keyed and ordered by $schema, and every corrupted + * record handed up in record order. This instance is consumed afterwards. * * @param int<1, max> $batchSize + * @param null|int $sampleSize the inference's sample size when $schema was inferred; null when it was declared + * @param null|Schema $known the schema the user sees, $schema plus the file's own columns * - * @throws SchemaMismatchException + * @throws InferredSchemaException|SchemaMismatchException a refused record under an inferred schema * - * @return Iterator + * @return Iterator */ - public function batches(Schema $schema, int $batchSize, Backend $backend): Iterator; + public function batches( + Schema $schema, + int $batchSize, + Backend $backend, + ?int $sampleSize = null, + ?Schema $known = null, + ): Iterator; } diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonReadBytes.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonReadBytes.php deleted file mode 100644 index b5f2a5dcf5..0000000000 --- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonReadBytes.php +++ /dev/null @@ -1,29 +0,0 @@ - - */ - private int $total = 0; - - /** - * @param int<0, max> $bytes - */ - public function add(int $bytes): void - { - $this->total += $bytes; - } - - /** - * @return int<0, max> - */ - public function total(): int - { - return $this->total; - } -} diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonReadProgress.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonReadProgress.php new file mode 100644 index 0000000000..a7d5479399 --- /dev/null +++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonReadProgress.php @@ -0,0 +1,66 @@ + + */ + private int $total = 0; + + /** + * @param int<0, max> $bytes + */ + public function add(int $bytes): void + { + $this->total += $bytes; + } + + /** + * The 1-based line of the record read last; null unless the read started at the file's first byte. + */ + public function lastLine(): ?int + { + return $this->lastLine; + } + + /** + * The byte offset of the line read last, or the 1-based number of the document element read last. + */ + public function lastOffset(): int + { + return $this->lastOffset; + } + + /** + * The text of the line read last; null for a document element. + */ + public function lastRecord(): ?string + { + return $this->lastRecord; + } + + public function recordAt(int $offset, ?int $line, ?string $record): void + { + $this->lastOffset = $offset; + $this->lastLine = $line; + $this->lastRecord = $record; + } + + /** + * @return int<0, max> + */ + public function total(): int + { + return $this->total; + } +} diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/PhpJsonOpenSource.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/PhpJsonOpenSource.php index 4a3b9e17ad..0071fed5ac 100644 --- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/PhpJsonOpenSource.php +++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/PhpJsonOpenSource.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Adapter\JSON\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Extractor\File\FileSplit; use Flow\ETL\IO\Extractor\File\SourceFile; use Flow\ETL\Values\Column\Backend; @@ -21,25 +22,23 @@ public function __construct( ) {} /** - * @return Generator + * @return Generator */ - public function batches(Schema $schema, int $batchSize, Backend $backend): Generator - { - $batch = new RecordBatch($schema, $backend, $batchSize); - - foreach ($this->reader->sample($this->source, window: $this->window) as $values) { - $rows = $batch->add($values); - - if ($rows !== null) { - yield $rows; - } + public function batches( + Schema $schema, + int $batchSize, + Backend $backend, + ?int $sampleSize = null, + ?Schema $known = null, + ): Generator { + $batch = new RecordBatch($schema, $backend, $batchSize, $this->source->uri(), $sampleSize, $known ?? $schema); + $read = new JsonReadProgress(); + + foreach ($this->reader->sample($this->source, $read, $this->window) as $values) { + yield from $batch->add($values, $read->lastOffset(), $read->lastLine(), $read->lastRecord()); } - $rest = $batch->flush(); - - if ($rest !== null) { - yield $rest; - } + yield from $batch->flush(); } /** diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Inference/JsonFileSample.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Inference/JsonFileSample.php index 811687e7ee..f5f334ccf0 100644 --- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Inference/JsonFileSample.php +++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Inference/JsonFileSample.php @@ -5,7 +5,7 @@ namespace Flow\ETL\Adapter\JSON\IO\Inference; use Flow\ETL\Adapter\JSON\IO\Extractor\JsonFileReader; -use Flow\ETL\Adapter\JSON\IO\Extractor\JsonReadBytes; +use Flow\ETL\Adapter\JSON\IO\Extractor\JsonReadProgress; use Flow\ETL\IO\Extractor\File\FileSplit; use Flow\ETL\IO\Extractor\File\SourceFile; use Generator; @@ -52,7 +52,7 @@ public function complete(): bool */ public function getIterator(): Generator { - $read = new JsonReadBytes(); + $read = new JsonReadProgress(); try { foreach ($this->reader->sample($this->source, $read, $this->window) as $row) { diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Context/JsonFixtureContext.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Context/JsonFixtureContext.php index 2d4afb9d36..2de15719f0 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Context/JsonFixtureContext.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Context/JsonFixtureContext.php @@ -10,6 +10,8 @@ use Flow\ETL\Adapter\JSON\IO\Extractor\JsonOpenSource; use Flow\ETL\Adapter\JSON\IO\Inference\JsonSniffer; use Flow\ETL\Adapter\JSON\IO\JsonFraming; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; +use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\SchemaMismatchException; use Flow\ETL\IO\Extractor\File\SourceFile; use Flow\ETL\IO\Inference\SchemaInference; @@ -129,8 +131,8 @@ public static function open( } /** - * The rows of every batch (of the first $batches when given) as one string, or the refusal's class, message, row - * and cause - consumes and closes the source. + * The rows of every batch (of the first $batches when given) and every corrupted record as one string, or the + * refusal's class, message, row and cause - consumes and closes the source. * * @param int<1, max> $batchSize * @param null|int<1, max> $batches @@ -141,12 +143,22 @@ public static function outcome( int $batchSize, Backend $backend = new PhpBackend(), ?int $batches = null, + ?int $sampleSize = null, ): string { $read = []; try { - foreach ($open->batches($schema, $batchSize, $backend) as $batch) { - $read[] = $batch->toArray(); + foreach ($open->batches($schema, $batchSize, $backend, $sampleSize) as $batch) { + $read[] = $batch instanceof CorruptedRecordException + ? ['corrupted' => [ + $batch->uri, + $batch->offset, + $batch->lineNumber, + $batch->reason, + $batch->record, + $batch->fields, + ]] + : $batch->toArray(); if (count($read) === $batches) { break; @@ -154,6 +166,8 @@ public static function outcome( } return serialize($read); + } catch (InferredSchemaException $e) { + return $e::class . ': ' . $e->getMessage(); } catch (SchemaMismatchException $e) { return ( $e::class diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonExtractorTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonExtractorTest.php index 164aa070aa..3e709ce27b 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonExtractorTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonExtractorTest.php @@ -60,6 +60,19 @@ final class JsonExtractorTest extends FlowTestCase { + public function test_a_key_first_seen_after_the_inference_sample_is_a_schema_error(): void + { + $this->expectException(InferredSchemaException::class); + $this->expectExceptionMessage('"c"'); + + ExtractedRows::of(from_json( + path('memory://source.json'), + filesystem: MemoryFiles::with([ + 'memory://source.json' => '[{"a":1,"b":2},{"a":3,"b":4},{"a":5,"b":6,"c":7}]', + ]), + )->inferSchema(infer_schema()->sampleSize(2))); + } + public function test_an_array_lines_file_reads_back_the_rows_it_was_written_with(): void { $memory = memory_filesystem(); @@ -83,13 +96,15 @@ public function test_an_empty_array_lines_document_reads_no_rows(): void static::assertSame( [], iterator_to_array( - (new JsonExtractor( - path('memory://framing/empty.json'), - MemoryFiles::with(['memory://framing/empty.json' => "[\n\n]"]), - JsonFraming::ArrayLines, - )) - ->withSchema(schema(int_schema('id'))) - ->extract(flow_context(config())->backend(), new Criteria()), + FlowTestCase::extracted( + (new JsonExtractor( + path('memory://framing/empty.json'), + MemoryFiles::with(['memory://framing/empty.json' => "[\n\n]"]), + JsonFraming::ArrayLines, + ))->withSchema(schema(int_schema('id'))), + flow_context(config())->backend(), + new Criteria(), + ), false, ), ); @@ -127,7 +142,7 @@ public function test_an_empty_glob_with_a_declared_schema_answers_it_and_extract static::assertEquals(schema(int_schema('id')), $extractor->schema()); static::assertSame( [], - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()), false), + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false), ); } @@ -165,7 +180,7 @@ public function test_an_empty_lines_glob_with_a_declared_schema_answers_it_and_e static::assertEquals(schema(int_schema('id')), $extractor->schema()); static::assertSame( [], - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()), false), + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false), ); } @@ -271,7 +286,7 @@ public function test_extracting_json_from_local_file_string_uri(): void $total = 0; - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { foreach ($rows->toArray() as $row) { static::assertSame( [ @@ -351,9 +366,11 @@ public function test_schema_is_the_declared_one(): void public function test_signal_stop(): void { - $generator = from_json(path(__DIR__ . '/../../../Fixtures/timezones.json')) - ->withBatchSize(100) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_json(path(__DIR__ . '/../../../Fixtures/timezones.json'))->withBatchSize(100), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->next(); @@ -370,7 +387,11 @@ public function test_every_batch_carries_the_inferred_schema(): void $expected = $extractor->schema(); $sizes = []; - foreach ($extractor->withBatchSize(3)->extract(flow_context()->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor->withBatchSize(3), + flow_context()->backend(), + new Criteria(), + ) as $rows) { static::assertTrue($rows->schema()->isSame($expected)); $sizes[] = $rows->count(); } @@ -389,7 +410,7 @@ public function test_the_inferred_schema_is_memoised_across_schema_and_extract() $extractor = from_json(JsonFixtureContext::path('five_rows.json'), filesystem: $counting); $extractor->schema(); - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())); static::assertSame(2, $counting->readFromCalls, 'sample + read'); static::assertSame(2, $counting->closedStreams()); @@ -405,7 +426,7 @@ public function test_a_declared_schema_opens_each_file_once(): void )->withSchema(schema(int_schema('id'))); $extractor->schema(); - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())); static::assertSame(1, $counting->readFromCalls); } @@ -515,7 +536,7 @@ public function test_sample_size_bounds_the_inferred_schema(): void public function test_a_value_past_the_sample_is_reported(): void { - $this->expectException(SchemaMismatchException::class); + $this->expectException(InferredSchemaException::class); df(config()) ->read(from_json(JsonFixtureContext::path('misfit.json'))->inferSchema(infer_schema()->sampleSize(3))) @@ -538,18 +559,16 @@ public function test_files_to_sniff_bounds_the_sources_opened(): void static::assertSame(1, $counting->readFromCalls); } - public function test_a_key_first_seen_past_the_sample_is_dropped(): void + public function test_a_key_first_seen_past_the_sample_is_a_schema_error(): void { - static::assertSame( - [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => 'b'], ['id' => 3, 'name' => 'c']], - df(config()) - ->read( - from_json(JsonFixtureContext::path('extra_key/*.json')) - ->inferSchema(infer_schema()->filesToSniff(1)), - ) - ->fetch() - ->toArray(), - ); + $this->expectException(InferredSchemaException::class); + $this->expectExceptionMessage('column "extra" is not declared by the schema'); + + df(config()) + ->read( + from_json(JsonFixtureContext::path('extra_key/*.json'))->inferSchema(infer_schema()->filesToSniff(1)), + ) + ->fetch(); } public function test_union_by_name_reads_diverging_columns_as_one_wider_schema(): void @@ -591,7 +610,10 @@ public function test_an_empty_document_refuses_to_infer_a_schema_and_extracts_no static::assertSame( [], - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria()), false), + iterator_to_array( + FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()), + false, + ), ); $this->expectException(InferredSchemaException::class); @@ -668,21 +690,29 @@ public function test_a_stream_is_closed_when_the_read_stops_early(string $mode): } if ($mode === 'limit') { - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria(limit: 2))); + iterator_to_array(FlowTestCase::extracted( + $extractor, + flow_context(config())->backend(), + new Criteria(limit: 2), + )); } if ($mode === 'stop') { - $generator = $extractor->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()); static::assertTrue($generator->valid()); $generator->send(Signal::STOP); } if ($mode === 'hydrator-throws') { try { - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted( + $extractor, + flow_context(config())->backend(), + new Criteria(), + )); static::fail('expected ' . SchemaMismatchException::class); - } catch (SchemaMismatchException) { + } catch (InferredSchemaException) { } } @@ -843,7 +873,8 @@ public function test_extract_builds_through_the_config_backend(): void $backend = new SpyBackend(); iterator_to_array( - from_json(__DIR__ . '/../../../Fixtures/timezones.json')->extract( + FlowTestCase::extracted( + from_json(__DIR__ . '/../../../Fixtures/timezones.json'), flow_context(config_builder()->backend($backend)->build())->backend(), new Criteria(), ), @@ -900,7 +931,8 @@ public function test_extract_split_stops_on_the_stop_signal(): void { $extractor = from_json(__DIR__ . '/../../../Fixtures/timezones.json'); $extractor->withBatchSize(1); - $generator = $extractor->extract( + $generator = FlowTestCase::extracted( + $extractor, flow_context()->backend(), (new Criteria())->withSplit($extractor->splits(new Criteria(), 1)[0]), ); diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonExtractorTypedColumnsTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonExtractorTypedColumnsTest.php index 980aca78f5..e431acb23d 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonExtractorTypedColumnsTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonExtractorTypedColumnsTest.php @@ -28,7 +28,11 @@ public function test_fast_path_hydrates_typed_columns_with_missing_column_null_a $actual = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { foreach ($rows->toArray() as $row) { $actual[] = $row; } @@ -53,7 +57,11 @@ public function test_fast_path_reads_partitioned_files_with_partition_value_from $actual = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { foreach ($rows->toArray() as $row) { $actual[] = $row; } @@ -79,7 +87,11 @@ public function test_appends_partition_columns_absent_from_the_schema(): void $actual = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { foreach ($rows->toArray() as $row) { $actual[] = $row; } diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonFileBatchesTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonFileBatchesTest.php index cf6ef5294c..244f0331c8 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonFileBatchesTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonFileBatchesTest.php @@ -14,6 +14,7 @@ use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Column\PhpBackend; +use Flow\ETL\Values\Rows; use Flow\ETL\Values\RustIterator; use Flow\Filesystem\Local\NativeLocalFilesystem; use Generator; @@ -54,6 +55,7 @@ public function test_close_closes_the_one_stream_the_source_opened(JsonFraming $ $open = JsonFixtureContext::open($format, JsonFixtureContext::source($fixture), $counting); foreach ($open->batches(schema(int_schema('id')), 2, new PhpBackend()) as $rows) { + static::assertInstanceOf(Rows::class, $rows); static::assertCount(2, $rows); break; diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonLinesExtractorTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonLinesExtractorTest.php index 0ba80b51b8..ba2ed6654f 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonLinesExtractorTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonLinesExtractorTest.php @@ -7,6 +7,7 @@ use Closure; use Flow\ETL\Adapter\JSON\IO\Extractor\JsonExtractor; use Flow\ETL\Adapter\JSON\Tests\Context\JsonFixtureContext; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; use Flow\ETL\Foundation\Exception\SchemaMismatchException; @@ -36,6 +37,7 @@ use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\df; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\infer_schema; use function Flow\ETL\DSL\int_schema; @@ -43,6 +45,7 @@ use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\schema_to_ascii; use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\to_array; use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; use function Flow\Filesystem\DSL\path_real; @@ -55,6 +58,41 @@ final class JsonLinesExtractorTest extends FlowTestCase { + public function test_a_key_first_seen_after_the_inference_sample_is_a_schema_error(): void + { + $this->expectException(InferredSchemaException::class); + $this->expectExceptionMessage('"c"'); + + ExtractedRows::of(from_json_lines( + path('memory://source.jsonl'), + MemoryFiles::with([ + 'memory://source.jsonl' => "{\"a\":1,\"b\":2}\n{\"a\":3,\"b\":4}\n{\"a\":5,\"b\":6,\"c\":7}\n", + ]), + )->inferSchema(infer_schema()->sampleSize(2))); + } + + public function test_a_value_that_does_not_cast_to_the_declared_type_is_a_corrupted_record(): void + { + try { + ExtractedRows::of(from_json_lines( + path('memory://source.jsonl'), + MemoryFiles::with([ + 'memory://source.jsonl' => "{\"id\":1,\"amount\":10.5}\n{\"id\":2,\"amount\":\"N/A\"}\n", + ]), + )->withSchema(schema(int_schema('id'), float_schema('amount', true)))); + static::fail('Expected the corrupted record to be refused'); + } catch (CorruptedRecordException $e) { + static::assertSame(['memory://source.jsonl', 23, 2, '{"id":2,"amount":"N/A"}', null], [ + $e->uri, + $e->offset, + $e->lineNumber, + $e->record, + $e->fields, + ]); + static::assertStringContainsString('"amount"', $e->reason); + } + } + public function test_extracting_jsonl_from_local_file_stream(): void { $rows = data_frame(config_builder()) @@ -158,7 +196,7 @@ public function test_extracting_jsonl_from_local_file_string_uri(): void $total = 0; - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { foreach ($rows->toArray() as $row) { static::assertSame( [ @@ -238,9 +276,11 @@ public function test_schema_is_the_declared_one(): void public function test_signal_stop(): void { - $generator = from_json_lines(path(__DIR__ . '/../../../Fixtures/timezones.jsonl')) - ->withBatchSize(100) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_json_lines(path(__DIR__ . '/../../../Fixtures/timezones.jsonl'))->withBatchSize(100), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->next(); @@ -257,7 +297,11 @@ public function test_every_batch_carries_the_inferred_schema(): void $expected = $extractor->schema(); $sizes = []; - foreach ($extractor->withBatchSize(3)->extract(flow_context()->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor->withBatchSize(3), + flow_context()->backend(), + new Criteria(), + ) as $rows) { static::assertTrue($rows->schema()->isSame($expected)); $sizes[] = $rows->count(); } @@ -276,7 +320,7 @@ public function test_the_inferred_schema_is_memoised_across_schema_and_extract() $extractor = from_json_lines(JsonFixtureContext::path('five_rows.jsonl'), filesystem: $counting); $extractor->schema(); - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())); static::assertSame(2, $counting->readFromCalls, 'sample + read'); static::assertSame(2, $counting->closedStreams()); @@ -292,7 +336,7 @@ public function test_a_declared_schema_opens_each_file_once(): void )->withSchema(schema(int_schema('id'))); $extractor->schema(); - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())); static::assertSame(1, $counting->readFromCalls); } @@ -402,7 +446,7 @@ public function test_sample_size_bounds_the_inferred_schema(): void public function test_a_value_past_the_sample_is_reported(): void { - $this->expectException(SchemaMismatchException::class); + $this->expectException(InferredSchemaException::class); df(config()) ->read( @@ -427,18 +471,17 @@ public function test_files_to_sniff_bounds_the_sources_opened(): void static::assertSame(1, $counting->readFromCalls); } - public function test_a_key_first_seen_past_the_sample_is_dropped(): void + public function test_a_key_first_seen_past_the_sample_is_a_schema_error(): void { - static::assertSame( - [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => 'b'], ['id' => 3, 'name' => 'c']], - df(config()) - ->read( - from_json_lines(JsonFixtureContext::path('extra_key/*.jsonl')) - ->inferSchema(infer_schema()->filesToSniff(1)), - ) - ->fetch() - ->toArray(), - ); + $this->expectException(InferredSchemaException::class); + $this->expectExceptionMessage('column "extra" is not declared by the schema'); + + df(config()) + ->read( + from_json_lines(JsonFixtureContext::path('extra_key/*.jsonl')) + ->inferSchema(infer_schema()->filesToSniff(1)), + ) + ->fetch(); } public function test_union_by_name_reads_diverging_columns_as_one_wider_schema(): void @@ -477,7 +520,10 @@ public function test_an_empty_document_refuses_to_infer_a_schema_and_extracts_no static::assertSame( [], - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria()), false), + iterator_to_array( + FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()), + false, + ), ); $this->expectException(InferredSchemaException::class); @@ -571,21 +617,29 @@ public function test_a_stream_is_closed_when_the_read_stops_early(string $mode): } if ($mode === 'limit') { - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria(limit: 2))); + iterator_to_array(FlowTestCase::extracted( + $extractor, + flow_context(config())->backend(), + new Criteria(limit: 2), + )); } if ($mode === 'stop') { - $generator = $extractor->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()); static::assertTrue($generator->valid()); $generator->send(Signal::STOP); } if ($mode === 'hydrator-throws') { try { - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted( + $extractor, + flow_context(config())->backend(), + new Criteria(), + )); static::fail('expected ' . SchemaMismatchException::class); - } catch (SchemaMismatchException) { + } catch (InferredSchemaException) { } } @@ -706,7 +760,7 @@ public function test_extract_samples_like_schema_does(): void { $extractor = from_json_lines(JsonFixtureContext::path('five_rows.jsonl')); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()), false); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false); static::assertEquals(Cardinality::exact(5), $extractor->statistics()->rows); } @@ -778,7 +832,8 @@ public function test_extract_builds_through_the_config_backend(): void $backend = new SpyBackend(); iterator_to_array( - from_json_lines(__DIR__ . '/../../../Fixtures/timezones.jsonl')->extract( + FlowTestCase::extracted( + from_json_lines(__DIR__ . '/../../../Fixtures/timezones.jsonl'), flow_context(config_builder()->backend($backend)->build())->backend(), new Criteria(), ), @@ -792,7 +847,8 @@ public function test_extract_split_stops_on_the_stop_signal(): void { $extractor = from_json_lines(__DIR__ . '/../../../Fixtures/timezones.jsonl'); $extractor->withBatchSize(1); - $generator = $extractor->extract( + $generator = FlowTestCase::extracted( + $extractor, flow_context()->backend(), (new Criteria())->withSplit($extractor->splits(new Criteria(), 1)[0]), ); @@ -852,4 +908,35 @@ public function test_a_pushed_path_filter_prunes_the_files_splits_are_cut_from() $extractor->splits((new Criteria())->withPathFilter(new RejectingFilter()), 1), ); } + + public function test_corrupted_records_are_routed_to_the_reject_loader(): void + { + $rejects = []; + + $rows = df() + ->read( + from_json_lines( + path('memory://source.jsonl'), + MemoryFiles::with([ + 'memory://source.jsonl' => "{\"id\":1}\n{\"id\":\"x\"}\n{\"id\":3}\n", + ]), + ) + ->withSchema(schema(int_schema('id'))) + ->withCorruptedRecords(to_array($rejects)), + ) + ->fetch(); + + static::assertSame([1, 3], $rows->reduceToArray('id')); + static::assertSame( + [[ + 'uri' => 'memory://source.jsonl', + 'offset' => 9, + 'line' => 2, + 'reason' => 'column "id": could not convert \'x\' (string) to integer', + 'record' => '{"id":"x"}', + 'fields' => null, + ]], + $rejects, + ); + } } diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonLinesExtractorTypedColumnsTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonLinesExtractorTypedColumnsTest.php index d54aaa1e91..317f9e1da5 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonLinesExtractorTypedColumnsTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonLinesExtractorTypedColumnsTest.php @@ -25,7 +25,11 @@ public function test_hydrates_typed_columns_with_missing_column_null(): void $actual = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { foreach ($rows->toArray() as $row) { $actual[] = $row; } diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonSplitReadTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonSplitReadTest.php index 89efab3671..81803e436c 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonSplitReadTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonSplitReadTest.php @@ -73,7 +73,8 @@ public function test_the_lines_reader_reads_a_file_by_windows_as_it_reads_it_who $rows = []; foreach ($splits as $split) { - foreach (DescribeRoundTrip::rebuilt($extractor->reader())->extract( + foreach (FlowTestCase::extracted( + DescribeRoundTrip::rebuilt($extractor->reader()), flow_context()->backend(), (new Criteria())->withSplit(type_instance_of(FileSplit::class)->assert($split)), ) as $batch) { diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/PhpJsonOpenSourceTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/PhpJsonOpenSourceTest.php index 220aea799f..7829473fa5 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/PhpJsonOpenSourceTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/PhpJsonOpenSourceTest.php @@ -7,7 +7,7 @@ use Flow\ETL\Adapter\JSON\IO\Extractor\PhpJsonOpenSource; use Flow\ETL\Adapter\JSON\IO\JsonFraming; use Flow\ETL\Adapter\JSON\Tests\Context\JsonFixtureContext; -use Flow\ETL\Foundation\Exception\SchemaMismatchException; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Column\PhpBackend; @@ -32,25 +32,32 @@ public function test_batches_are_sized_and_the_tail_is_shorter(): void false, ); - static::assertSame([2, 2, 1], array_map(static fn(Rows $rows): int => $rows->count(), $batches)); - static::assertSame([['id' => 5]], $batches[2]->toArray()); + static::assertContainsOnlyInstancesOf(Rows::class, $batches); + static::assertSame( + [[['id' => 1], ['id' => 2]], [['id' => 3], ['id' => 4]], [['id' => 5]]], + array_map(static fn(CorruptedRecordException|Rows $rows): array => ( + $rows instanceof Rows ? $rows->toArray() : [$rows->getMessage()] + ), $batches), + ); } - public function test_a_refusal_carries_its_row_in_the_whole_source(): void + public function test_under_a_declared_schema_a_refused_element_is_a_corrupted_record_numbered_in_the_document(): void { $batches = (new PhpJsonOpenSource( JsonFixtureContext::reader(JsonFraming::Array), JsonFixtureContext::source('misfit.json'), ))->batches(schema(int_schema('id')), 2, new PhpBackend()); - static::assertSame([['id' => 1], ['id' => 2]], $batches->current()->toArray()); + $rows = $batches->current(); + + static::assertInstanceOf(Rows::class, $rows); + static::assertSame([['id' => 1], ['id' => 2]], $rows->toArray()); - try { - $batches->next(); - static::fail('the second batch holds "n/a"'); - } catch (SchemaMismatchException $e) { - static::assertSame(3, $e->rowIndex); - } + $batches->next(); + $corrupted = $batches->current(); + + static::assertInstanceOf(CorruptedRecordException::class, $corrupted); + static::assertSame([4, null], [$corrupted->offset, $corrupted->lineNumber]); } #[TestWith([JsonFraming::Array, 'five_rows.json'])] @@ -63,7 +70,10 @@ public function test_batches_close_the_stream_when_abandoned(JsonFraming $format JsonFixtureContext::source($fixture), ))->batches(schema(int_schema('id')), 2, new PhpBackend()); - static::assertSame(2, $batches->current()->count()); + $rows = $batches->current(); + + static::assertInstanceOf(Rows::class, $rows); + static::assertSame(2, $rows->count()); unset($batches); static::assertSame(1, $counting->readFromCalls); diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/RustJsonOpenSourceTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/RustJsonOpenSourceTest.php index 9dc1e53c9c..bf0b81e68c 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/RustJsonOpenSourceTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/RustJsonOpenSourceTest.php @@ -4,13 +4,18 @@ namespace Flow\ETL\Adapter\JSON\Tests\Integration\IO\Extractor; +use Flow\ETL\Adapter\JSON\IO\Extractor\JsonFileBatches; use Flow\ETL\Adapter\JSON\IO\Extractor\PhpJsonOpenSource; use Flow\ETL\Adapter\JSON\IO\JsonFraming; use Flow\ETL\Adapter\JSON\Tests\Context\JsonFixtureContext; use Flow\ETL\Foundation\Exception\SchemaMismatchException; +use Flow\ETL\IO\Extractor\File\FileSplit; +use Flow\ETL\IO\Extractor\File\SourceFile; use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Column\PhpBackend; +use Flow\ETL\Values\Rows; +use Flow\ETL\Values\Schema; use Flow\Filesystem\Local\NativeLocalFilesystem; use Generator; use PHPUnit\Framework\Attributes\DataProvider; @@ -19,6 +24,8 @@ use function Flow\ETL\DSL\infer_schema; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\schema; +use function Flow\Filesystem\DSL\memory_filesystem; +use function Flow\Filesystem\DSL\path; use function in_array; use function str_starts_with; @@ -68,16 +75,84 @@ public function test_both_lanes_read_a_fixture_identically(string $fixture): voi ); } - public function test_a_value_past_the_sample_is_refused_as_the_php_lane_refuses_it(): void + /** + * @return Generator + */ + public static function refusal_cases(): Generator { - $schema = JsonFixtureContext::infer('misfit.json', infer_schema()->sampleSize(3)->build()); - $source = JsonFixtureContext::source('misfit.json'); + $id = schema(int_schema('id')); + + yield 'array element refused under a declared schema' => [ + JsonFraming::Array, + '[{"id":1},{"id":2},{"id":3},{"id":"n/a"},{"id":5}]', + $id, + null, + ]; + yield 'array element refused past the sample' => [JsonFraming::Array, '[{"id":1},{"id":"n/a"}]', $id, 1]; + yield 'lines refused and lacking a column under a declared schema' => [ + JsonFraming::Lines, + "{\"id\":1}\r\n\n{\"id\":\"x\"}\n{\"other\":2}\n{\"id\":4}", + $id, + null, + ]; + yield 'line refused past the sample' => [JsonFraming::Lines, "{\"id\":1}\n{\"id\":\"x\"}\n", $id, 1]; + yield 'line with a key past the sample' => [JsonFraming::Lines, "{\"id\":1}\n\n{\"id\":2,\"c\":3}\n", $id, 1]; + } + + #[DataProvider('refusal_cases')] + public function test_both_lanes_refuse_a_record_identically( + JsonFraming $format, + string $content, + Schema $schema, + ?int $sampleSize, + ): void { + $filesystem = memory_filesystem(); + $filesystem->writeTo(path('memory://records.json'))->append($content)->close(); + $source = new SourceFile(path('memory://records.json')); + + $native = JsonFixtureContext::outcome( + JsonFixtureContext::open($format, $source, $filesystem), + $schema, + 2, + sampleSize: $sampleSize, + ); + + static::assertStringNotContainsString(SchemaMismatchException::class, $native); + static::assertSame( + JsonFixtureContext::outcome( + new PhpJsonOpenSource(JsonFixtureContext::reader($format, $filesystem), $source), + $schema, + 2, + sampleSize: $sampleSize, + ), + $native, + ); + } - $native = JsonFixtureContext::outcome(JsonFixtureContext::open(JsonFraming::Array, $source), $schema, 2); + public function test_a_window_past_the_first_byte_names_a_refused_line_by_offset_alone(): void + { + $filesystem = memory_filesystem(); + $filesystem + ->writeTo(path('memory://records.jsonl')) + ->append("{\"id\":1}\n{\"id\":2}\n{\"id\":\"x\"}\n") + ->close(); + $source = new SourceFile(path('memory://records.jsonl')); + $split = new FileSplit(path('memory://records.jsonl'), 9, null); + $reader = JsonFixtureContext::reader(JsonFraming::Lines, $filesystem); + + $native = JsonFixtureContext::outcome( + (new JsonFileBatches($filesystem, $reader))->open($source, $split), + schema(int_schema('id')), + 2, + ); - static::assertStringStartsWith(SchemaMismatchException::class . ':', $native); + static::assertStringContainsString('i:1;i:18;i:2;N;', $native); static::assertSame( - JsonFixtureContext::outcome(new PhpJsonOpenSource(JsonFixtureContext::reader(), $source), $schema, 2), + JsonFixtureContext::outcome( + new PhpJsonOpenSource($reader, $source, $reader->window($split)), + schema(int_schema('id')), + 2, + ), $native, ); } @@ -88,6 +163,8 @@ public function test_under_the_php_backend_no_native_column_survives(): void try { foreach ($open->batches(JsonFixtureContext::infer('parity_people.jsonl'), 2, new PhpBackend()) as $batch) { + static::assertInstanceOf(Rows::class, $batch); + foreach ($batch->columns() as $column) { static::assertNotSame('Flow\ETL\Values\Column\RustColumn', $column::class); } @@ -103,6 +180,7 @@ public function test_close_after_a_partial_read_closes_the_stream(): void $open = JsonFixtureContext::open(JsonFraming::Array, JsonFixtureContext::source('five_rows.json'), $counting); foreach ($open->batches(schema(int_schema('id')), 2, new PhpBackend()) as $rows) { + static::assertInstanceOf(Rows::class, $rows); static::assertSame([['id' => 1], ['id' => 2]], $rows->toArray()); break; diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonFileBatchesTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonFileBatchesTest.php index 0d85260148..52a80be98f 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonFileBatchesTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonFileBatchesTest.php @@ -7,6 +7,7 @@ use Flow\ETL\Adapter\JSON\IO\Extractor\JsonFileBatches; use Flow\ETL\Adapter\JSON\IO\Extractor\JsonFileReader; use Flow\ETL\Adapter\JSON\IO\JsonFraming; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\File\ReadWindow; use Flow\ETL\IO\Extractor\File\SourceFile; @@ -44,7 +45,9 @@ public function test_a_document_yields_its_rows_consumes_no_offset_and_closes_it static::assertSame( [['id' => 1], ['id' => 2], ['id' => 3]], array_merge(...array_map( - static fn(Rows $rows): array => $rows->toArray(), + static fn(CorruptedRecordException|Rows $rows): array => ( + $rows instanceof Rows ? $rows->toArray() : [$rows->getMessage()] + ), iterator_to_array($file, false), )), ); @@ -59,7 +62,9 @@ public function test_lines_yield_one_row_per_line(): void static::assertSame( [[['id' => 1], ['id' => 2]]], array_map( - static fn(Rows $rows): array => $rows->toArray(), + static fn(CorruptedRecordException|Rows $rows): array => ( + $rows instanceof Rows ? $rows->toArray() : [$rows->getMessage()] + ), iterator_to_array( (new JsonFileBatches( $filesystem, diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonReadBytesTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonReadBytesTest.php deleted file mode 100644 index 149aa89d50..0000000000 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonReadBytesTest.php +++ /dev/null @@ -1,25 +0,0 @@ -total()); - } - - public function test_it_sums_what_was_added(): void - { - $read = new JsonReadBytes(); - $read->add(9); - $read->add(10); - - static::assertSame(19, $read->total()); - } -} diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonReadProgressTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonReadProgressTest.php new file mode 100644 index 0000000000..c65e1f7298 --- /dev/null +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonReadProgressTest.php @@ -0,0 +1,41 @@ +total()); + } + + public function test_it_sums_what_was_added(): void + { + $read = new JsonReadProgress(); + $read->add(9); + $read->add(10); + + static::assertSame(19, $read->total()); + } + + public function test_the_last_record_is_the_one_recorded_last(): void + { + $read = new JsonReadProgress(); + $read->recordAt(0, 1, '{"id":1}'); + $read->recordAt(9, 2, '{"id":2}'); + + static::assertSame([9, 2, '{"id":2}'], [$read->lastOffset(), $read->lastLine(), $read->lastRecord()]); + } + + public function test_nothing_recorded_is_offset_zero_without_a_line_or_a_record(): void + { + $read = new JsonReadProgress(); + + static::assertSame([0, null, null], [$read->lastOffset(), $read->lastLine(), $read->lastRecord()]); + } +} diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Extractor/ParquetExtractor.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Extractor/ParquetExtractor.php index 428291ee43..a2e56c8efe 100644 --- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Extractor/ParquetExtractor.php +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Extractor/ParquetExtractor.php @@ -6,6 +6,7 @@ use Flow\ETL\Adapter\Parquet\IO\Extractor\Pushdown\FilterTranslator; use Flow\ETL\Adapter\Parquet\Values\Schema\SchemaConverter; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; use Flow\ETL\Functions\ScalarFunction; @@ -123,7 +124,7 @@ public function isRepeatable(): bool } /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/Extractor/ParquetExtractorFooterTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/Extractor/ParquetExtractorFooterTest.php index 6565a03470..e51bad76c8 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/Extractor/ParquetExtractorFooterTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/Extractor/ParquetExtractorFooterTest.php @@ -59,7 +59,7 @@ public function test_every_file_is_opened_once_even_after_schema(): void $extractor = from_parquet(path(ParquetSourceFileContext::pagination() . '/*.parquet'), filesystem: $filesystem); $extractor->schema(); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria())); static::assertSame([1, 1, 1, 1, 1], array_values(ParquetFilesContext::opensPerFile($filesystem))); static::assertSame(5, $filesystem->closedStreams()); @@ -75,7 +75,7 @@ public function test_changing_the_columns_closes_the_kept_file_and_opens_it_once static::assertSame(1, $filesystem->closedStreams()); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria())); static::assertSame([2, 1, 1, 1, 1], array_values(ParquetFilesContext::opensPerFile($filesystem))); static::assertSame(6, $filesystem->closedStreams()); @@ -86,8 +86,8 @@ public function test_a_second_extract_opens_every_file_once_more(): void $filesystem = new CountingFilesystem(new NativeLocalFilesystem()); $extractor = from_parquet(path(ParquetSourceFileContext::pagination() . '/*.parquet'), filesystem: $filesystem); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria())); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria())); static::assertSame([2, 2, 2, 2, 2], array_values(ParquetFilesContext::opensPerFile($filesystem))); static::assertSame(10, $filesystem->closedStreams()); @@ -120,12 +120,13 @@ public function test_a_path_filter_excluding_the_first_file_closes_the_kept_one( 500, array_sum(array_map( static fn(Rows $batch): int => $batch->count(), - iterator_to_array( - $extractor->extract(flow_context()->backend(), new Criteria(pathFilter: new KeepPaths([path( + iterator_to_array(FlowTestCase::extracted( + $extractor, + flow_context()->backend(), + new Criteria(pathFilter: new KeepPaths([path( ParquetSourceFileContext::pagination() . '/02_500.parquet', - )]))), - false, - ), + )])), + ), false), )), ); static::assertSame( diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/Extractor/ParquetExtractorTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/Extractor/ParquetExtractorTest.php index ac6f2ded74..85a65eb33b 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/Extractor/ParquetExtractorTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/Extractor/ParquetExtractorTest.php @@ -19,6 +19,7 @@ use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\Double\RecordingFilesystem; use Flow\ETL\Tests\FlowIntegrationTestCase; +use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Cardinality; use Flow\ETL\Values\Schema\Definition; use Flow\Filesystem\Local\NativeLocalFilesystem; @@ -201,7 +202,11 @@ public function test_a_read_whose_path_filter_rejects_every_file_closes_the_file $extractor->schema(); iterator_to_array( - $extractor->extract(flow_context(config())->backend(), new Criteria(pathFilter: new RejectingFilter())), + FlowTestCase::extracted( + $extractor, + flow_context(config())->backend(), + new Criteria(pathFilter: new RejectingFilter()), + ), false, ); @@ -266,7 +271,11 @@ public function test_limit(): void $extractedRows = 0; - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria(limit: 2)) as $batch) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config())->backend(), + new Criteria(limit: 2), + ) as $batch) { $extractedRows += $batch->count(); } @@ -282,8 +291,8 @@ public function test_calling_schema_does_not_change_extraction(): void $warm->withMetadataColumns(true); static::assertEquals( - iterator_to_array($cold->extract(flow_context(config())->backend(), new Criteria())), - iterator_to_array($warm->extract(flow_context(config())->backend(), new Criteria())), + iterator_to_array(FlowTestCase::extracted($cold, flow_context(config())->backend(), new Criteria())), + iterator_to_array(FlowTestCase::extracted($warm, flow_context(config())->backend(), new Criteria())), ); } @@ -294,7 +303,7 @@ public function test_extract_yields_the_metadata_column_schema_promises(): void columns: ['email'], )->withMetadataColumns(true); - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $batch) { static::assertSame($extractor->schema()->names(), $batch->schema()->names()); break; @@ -391,7 +400,7 @@ public function test_extract_fills_partition_columns_from_the_path(): void $extractor = from_parquet(path(__DIR__ . '/../../../Fixtures/Pagination/partitioned/date=2024-01-01/*.parquet')); - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { static::assertSame(['id', 'name', 'date'], array_keys($rows->toArray()[0])); static::assertSame('2024-01-01', $rows->column('date')->value(0)); static::assertEquals($extractor->schema(), $rows->schema()); @@ -453,7 +462,7 @@ public function test_declared_partition_types_reach_the_rows(): void $extractor = from_parquet(path(__DIR__ . '/../../../Fixtures/Pagination/partitioned/date=2024-01-01/*.parquet')) ->partitionTypes(partition_types(date: type_datetime())); - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { static::assertEquals($extractor->schema(), $rows->schema()); static::assertEquals(new DateTimeImmutable('2024-01-01 00:00:00 UTC'), $rows->column('date')->value(0)); @@ -467,10 +476,14 @@ public function test_extract_closes_the_reader_when_the_generator_is_abandoned() { $filesystem = new CountingFilesystem(new NativeLocalFilesystem()); - $generator = from_parquet( - path(__DIR__ . '/../../../Fixtures/Pagination/partitioned/*/*.parquet'), - filesystem: $filesystem, - )->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_parquet( + path(__DIR__ . '/../../../Fixtures/Pagination/partitioned/*/*.parquet'), + filesystem: $filesystem, + ), + flow_context(config())->backend(), + new Criteria(), + ); $generator->current(); unset($generator); @@ -539,10 +552,14 @@ public function test_extract_closes_every_reader_it_opens(): void { $filesystem = new CountingFilesystem(new NativeLocalFilesystem()); - foreach (from_parquet( - path(__DIR__ . '/../../../Fixtures/Pagination/partitioned/*/*.parquet'), - filesystem: $filesystem, - )->extract(flow_context(config())->backend(), new Criteria()) as $_rows) { + foreach (FlowTestCase::extracted( + from_parquet( + path(__DIR__ . '/../../../Fixtures/Pagination/partitioned/*/*.parquet'), + filesystem: $filesystem, + ), + flow_context(config())->backend(), + new Criteria(), + ) as $_rows) { } static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); @@ -552,10 +569,14 @@ public function test_extract_closes_the_reader_it_skips_for_the_offset(): void { $filesystem = new CountingFilesystem(new NativeLocalFilesystem()); - foreach (from_parquet( - path(__DIR__ . '/../../../Fixtures/Pagination/partitioned/*/*.parquet'), - filesystem: $filesystem, - )->extract(flow_context(config())->backend(), new Criteria(offset: 2500)) as $_rows) { + foreach (FlowTestCase::extracted( + from_parquet( + path(__DIR__ . '/../../../Fixtures/Pagination/partitioned/*/*.parquet'), + filesystem: $filesystem, + ), + flow_context(config())->backend(), + new Criteria(offset: 2500), + ) as $_rows) { } static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); @@ -565,10 +586,14 @@ public function test_extract_closes_the_reader_when_the_pipeline_stops(): void { $filesystem = new CountingFilesystem(new NativeLocalFilesystem()); - $generator = from_parquet( - path(__DIR__ . '/../../../Fixtures/Pagination/partitioned/*/*.parquet'), - filesystem: $filesystem, - )->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_parquet( + path(__DIR__ . '/../../../Fixtures/Pagination/partitioned/*/*.parquet'), + filesystem: $filesystem, + ), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->send(Signal::STOP); @@ -631,7 +656,8 @@ public function test_a_path_filter_narrows_the_read_but_not_the_schema(): void static::assertSame( [], - iterator_to_array($extractor->extract( + iterator_to_array(FlowTestCase::extracted( + $extractor, flow_context(config())->backend(), new Criteria(pathFilter: new RejectingFilter()), )), @@ -642,8 +668,11 @@ public function test_a_path_filter_narrows_the_read_but_not_the_schema(): void public function test_signal_stop(): void { - $generator = from_parquet(path(__DIR__ . '/../../../Fixtures/Pagination/*.parquet')) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_parquet(path(__DIR__ . '/../../../Fixtures/Pagination/*.parquet')), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->next(); @@ -654,9 +683,11 @@ public function test_signal_stop(): void public function test_signal_stop_on_the_first_file_tail_batch_skips_the_remaining_files(): void { - $generator = from_parquet(path(__DIR__ . '/../../../Fixtures/Pagination/*.parquet')) - ->withBatchSize(1500) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_parquet(path(__DIR__ . '/../../../Fixtures/Pagination/*.parquet'))->withBatchSize(1500), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->send(Signal::STOP); @@ -676,9 +707,11 @@ public function test_the_second_file_is_asked_for_the_remainder_of_the_limit(): // 01_1000.parquet fills the first batch; 02_500.parquet must be read up to the 200 rows still // wanted, not up to the full limit again $batches = iterator_to_array( - from_parquet(path(__DIR__ . '/../../../Fixtures/Pagination/*.parquet')) - ->withBatchSize(1000) - ->extract(flow_context(config())->backend(), new Criteria(limit: 1200)), + FlowTestCase::extracted( + from_parquet(path(__DIR__ . '/../../../Fixtures/Pagination/*.parquet'))->withBatchSize(1000), + flow_context(config())->backend(), + new Criteria(limit: 1200), + ), false, ); @@ -906,7 +939,8 @@ public function test_extract_split_stops_on_the_stop_signal(): void ParquetFilesContext::writeRowGroups($filesystem, 'memory://split/a.parquet', 300); $extractor = from_parquet('memory://split/a.parquet', filesystem: $filesystem); $extractor->withBatchSize(1); - $generator = $extractor->extract( + $generator = FlowTestCase::extracted( + $extractor, flow_context()->backend(), (new Criteria())->withSplit($extractor->splits(new Criteria(), 100_000)[0]), ); @@ -924,12 +958,11 @@ public function test_extract_split_stops_on_the_stop_signal_of_a_whole_source_sp { $filesystem = memory_filesystem(); ParquetFilesContext::writeRowGroups($filesystem, 'memory://split/a.parquet', 300); - $generator = from_parquet('memory://split/a.parquet', filesystem: $filesystem) - ->withBatchSize(1) - ->extract( - flow_context()->backend(), - (new Criteria())->withSplit(new FileSplit(path('memory://split/a.parquet'), 0, null)), - ); + $generator = FlowTestCase::extracted( + from_parquet('memory://split/a.parquet', filesystem: $filesystem)->withBatchSize(1), + flow_context()->backend(), + (new Criteria())->withSplit(new FileSplit(path('memory://split/a.parquet'), 0, null)), + ); $totalRows = 0; foreach ($generator as $rows) { @@ -948,7 +981,8 @@ public function test_extract_split_reads_no_more_than_the_limit(): void $extractor->withBatchSize(10); $total = 0; - foreach ($extractor->extract( + foreach (FlowTestCase::extracted( + $extractor, flow_context()->backend(), (new Criteria(25))->withSplit($extractor->splits(new Criteria(), 100_000)[0]), ) as $rows) { diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PaginationTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PaginationTest.php index 99d45e53d2..bde49b242c 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PaginationTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PaginationTest.php @@ -27,8 +27,11 @@ public function test_multifile_pagination_from_beginning(): void { $extractedRows = 0; - foreach (from_parquet(path_real(__DIR__ . '/../Fixtures/Pagination/*.parquet')) - ->extract(flow_context(config())->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted( + from_parquet(path_real(__DIR__ . '/../Fixtures/Pagination/*.parquet')), + flow_context(config())->backend(), + new Criteria(), + ) as $batch) { $extractedRows += $batch->count(); } @@ -39,8 +42,11 @@ public function test_multifile_pagination_from_middle(): void { $extractedRows = 0; - foreach (from_parquet(path_real(__DIR__ . '/../Fixtures/Pagination/*.parquet')) - ->extract(flow_context(config())->backend(), new Criteria(offset: 2500)) as $batch) { + foreach (FlowTestCase::extracted( + from_parquet(path_real(__DIR__ . '/../Fixtures/Pagination/*.parquet')), + flow_context(config())->backend(), + new Criteria(offset: 2500), + ) as $batch) { $extractedRows += $batch->count(); } @@ -51,8 +57,11 @@ public function test_multifile_pagination_from_middle_partitioned(): void { $extractedRows = 0; - foreach (from_parquet(path_real(__DIR__ . '/../Fixtures/Pagination/partitioned/date=*/*.parquet')) - ->extract(flow_context(config())->backend(), new Criteria(offset: 2500)) as $batch) { + foreach (FlowTestCase::extracted( + from_parquet(path_real(__DIR__ . '/../Fixtures/Pagination/partitioned/date=*/*.parquet')), + flow_context(config())->backend(), + new Criteria(offset: 2500), + ) as $batch) { $extractedRows += $batch->count(); } @@ -63,8 +72,11 @@ public function test_multifile_pagination_from_offset_bigger_than_total_rows(): { $extractedRows = 0; - foreach (from_parquet(path_real(__DIR__ . '/../Fixtures/Pagination/*.parquet')) - ->extract(flow_context(config())->backend(), new Criteria(offset: 10_000)) as $batch) { + foreach (FlowTestCase::extracted( + from_parquet(path_real(__DIR__ . '/../Fixtures/Pagination/*.parquet')), + flow_context(config())->backend(), + new Criteria(offset: 10_000), + ) as $batch) { $extractedRows += $batch->count(); } @@ -75,11 +87,14 @@ public function test_reading_file_from_given_offset(): void { $extractedRows = 0; - foreach (from_parquet(path_real(__DIR__ . '/../Fixtures/orders_1k.parquet')) - ->extract(flow_context(config())->backend(), new Criteria(offset: (new Reader()) + foreach (FlowTestCase::extracted( + from_parquet(path_real(__DIR__ . '/../Fixtures/orders_1k.parquet')), + flow_context(config())->backend(), + new Criteria(offset: (new Reader()) ->read(__DIR__ . '/../Fixtures/orders_1k.parquet') ->metadata() - ->rowsNumber() - 100)) as $batch) { + ->rowsNumber() - 100), + ) as $batch) { $extractedRows += $batch->count(); } @@ -100,7 +115,10 @@ public function test_an_offset_spanning_two_files_and_a_limit_inside_the_third_r $extractor = from_parquet('memory://window/*.parquet', filesystem: $filesystem); $read = static fn(): array => array_merge(...array_map( static fn(Rows $rows): array => $rows->column('id')->values(), - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria(limit: 4, offset: 4)), false), + iterator_to_array( + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria(limit: 4, offset: 4)), + false, + ), )); static::assertSame([5, 6, 7, 8], $read()); diff --git a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/IO/Extractor/TextExtractor.php b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/IO/Extractor/TextExtractor.php index 3fb88db17f..dde8fdf151 100644 --- a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/IO/Extractor/TextExtractor.php +++ b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/IO/Extractor/TextExtractor.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Adapter\Text\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; use Flow\ETL\Functions\ScalarFunction; use Flow\ETL\IO\Extractor; @@ -74,7 +75,7 @@ public function isRepeatable(): bool } /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Extractor/TextExtractorTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Extractor/TextExtractorTest.php index 6de927a2c9..83576c4199 100644 --- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Extractor/TextExtractorTest.php +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Extractor/TextExtractorTest.php @@ -96,8 +96,11 @@ public function test_schema_describes_a_single_text_column(): void public function test_signal_stop(): void { - $generator = from_text(path_real(__DIR__ . '/../../../Fixtures/orders_flow.csv')) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_text(path_real(__DIR__ . '/../../../Fixtures/orders_flow.csv')), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->next(); @@ -110,9 +113,11 @@ public function test_signal_stop(): void public function test_signal_stop_on_the_first_file_tail_batch_skips_the_remaining_files(): void { - $generator = from_text(__DIR__ . '/../../../Fixtures/cross_stream/*/data.txt') - ->withBatchSize(10) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_text(__DIR__ . '/../../../Fixtures/cross_stream/*/data.txt')->withBatchSize(10), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->send(Signal::STOP); @@ -147,11 +152,13 @@ public function test_a_limited_read_closes_its_stream(): void { $filesystem = new RecordingFilesystem(native_local_filesystem()); - iterator_to_array( - (new TextExtractor(path_real(__DIR__ . '/../../../Fixtures/parity_lines.txt'), $filesystem)) - ->withBatchSize(1) - ->extract(flow_context(config())->backend(), new Criteria(limit: 1)), - ); + iterator_to_array(FlowTestCase::extracted( + (new TextExtractor(path_real(__DIR__ . '/../../../Fixtures/parity_lines.txt'), $filesystem))->withBatchSize( + 1, + ), + flow_context(config())->backend(), + new Criteria(limit: 1), + )); static::assertContains('closeSource', $filesystem->calls); } @@ -159,9 +166,13 @@ public function test_a_limited_read_closes_its_stream(): void public function test_a_stopped_read_closes_its_stream(): void { $filesystem = new RecordingFilesystem(native_local_filesystem()); - $generator = (new TextExtractor(path_real(__DIR__ . '/../../../Fixtures/parity_lines.txt'), $filesystem)) - ->withBatchSize(1) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + (new TextExtractor(path_real(__DIR__ . '/../../../Fixtures/parity_lines.txt'), $filesystem))->withBatchSize( + 1, + ), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); @@ -214,8 +225,11 @@ public function test_extract_builds_through_the_config_backend(): void $backend = new SpyBackend(); iterator_to_array( - from_text(path_real(__DIR__ . '/../../../Fixtures/orders_flow.csv')) - ->extract(flow_context(config_builder()->backend($backend)->build())->backend(), new Criteria()), + FlowTestCase::extracted( + from_text(path_real(__DIR__ . '/../../../Fixtures/orders_flow.csv')), + flow_context(config_builder()->backend($backend)->build())->backend(), + new Criteria(), + ), false, ); @@ -287,7 +301,8 @@ public function test_extract_split_stops_on_the_stop_signal(): void { $extractor = from_text(__DIR__ . '/../../../Fixtures/orders_1k.csv'); $extractor->withBatchSize(1); - $generator = $extractor->extract( + $generator = FlowTestCase::extracted( + $extractor, flow_context()->backend(), (new Criteria())->withSplit($extractor->splits(new Criteria(), 1)[0]), ); diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Extractor/TextExtractorTypedColumnsTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Extractor/TextExtractorTypedColumnsTest.php index df69271aa1..b009649365 100644 --- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Extractor/TextExtractorTypedColumnsTest.php +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Extractor/TextExtractorTypedColumnsTest.php @@ -21,7 +21,11 @@ public function test_reads_each_line_into_a_string_text_column(): void $actual = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { static::assertEquals(type_string(), $rows->schema()->get('text')->type()); foreach ($rows->toArray() as $row) { @@ -39,7 +43,11 @@ public function test_appends_input_file_uri_when_metadata_columns_are_enabled(): $actual = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $rows) { foreach ($rows->toArray() as $row) { $actual[] = $row; } diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/IO/Extractor/XMLParserExtractor.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/IO/Extractor/XMLParserExtractor.php index b88b3901f7..1d81a53be2 100644 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/IO/Extractor/XMLParserExtractor.php +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/IO/Extractor/XMLParserExtractor.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Adapter\XML\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; use Flow\ETL\Functions\ScalarFunction; use Flow\ETL\IO\Extractor; @@ -95,7 +96,7 @@ public function isRepeatable(): bool } /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLExtractorTypedColumnsTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLExtractorTypedColumnsTest.php index af308d833d..1b696d70e1 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLExtractorTypedColumnsTest.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLExtractorTypedColumnsTest.php @@ -27,7 +27,11 @@ public function test_reads_each_node_into_a_typed_xml_column(): void $rows = []; - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $batch) { static::assertEquals(type_xml(), $batch->schema()->get('node')->type()); $rows = [...$rows, ...$batch->toArray()]; @@ -42,7 +46,11 @@ public function test_appends_input_file_uri_when_metadata_columns_are_enabled(): ->withXMLNodePath('root/items/item') ->withMetadataColumns(true); - foreach ($extractor->extract(flow_context(config_builder()->build())->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config_builder()->build())->backend(), + new Criteria(), + ) as $batch) { static::assertSame([$path->uri()], array_values(array_unique($batch->column('_input_file_uri')->values()))); } } @@ -51,9 +59,11 @@ public function test_an_xml_node_column_holds_the_utf8_document_of_each_node(): { $physicals = []; - foreach (from_xml(path_real(__DIR__ . '/../../../Fixtures/simple_items.xml')) - ->withXMLNodePath('root/items/item') - ->extract(flow_context()->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted( + from_xml(path_real(__DIR__ . '/../../../Fixtures/simple_items.xml'))->withXMLNodePath('root/items/item'), + flow_context()->backend(), + new Criteria(), + ) as $batch) { $physicals = [...$physicals, ...$batch->column('node')->physicals()]; } @@ -69,10 +79,13 @@ public function test_a_string_node_column_holds_each_node_text(): void { $rows = []; - foreach (from_xml(path_real(__DIR__ . '/../../../Fixtures/simple_items.xml')) - ->withXMLNodePath('root/items/item') - ->withSchema(schema(str_schema('node'))) - ->extract(flow_context()->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted( + from_xml(path_real(__DIR__ . '/../../../Fixtures/simple_items.xml')) + ->withXMLNodePath('root/items/item') + ->withSchema(schema(str_schema('node'))), + flow_context()->backend(), + new Criteria(), + ) as $batch) { $rows = [...$rows, ...$batch->toArray()]; } @@ -87,10 +100,13 @@ public function test_an_xml_node_column_beside_other_columns_reads_the_node_docu { $nodes = []; - foreach (from_xml(path_real(__DIR__ . '/../../../Fixtures/simple_items.xml')) - ->withXMLNodePath('root/items/item') - ->withSchema(schema(xml_schema('node'), str_schema('missing', nullable: true))) - ->extract(flow_context()->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted( + from_xml(path_real(__DIR__ . '/../../../Fixtures/simple_items.xml')) + ->withXMLNodePath('root/items/item') + ->withSchema(schema(xml_schema('node'), str_schema('missing', nullable: true))), + flow_context()->backend(), + new Criteria(), + ) as $batch) { $nodes = [...$nodes, ...$batch->column('node')->values()]; } diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLParserExtractorTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLParserExtractorTest.php index 8e0de19d14..d556bf52c6 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLParserExtractorTest.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLParserExtractorTest.php @@ -20,6 +20,7 @@ use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\Double\UnsizedFilesystem; use Flow\ETL\Tests\FlowIntegrationTestCase; +use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Cardinality; use Flow\Filesystem\Tests\Double\RejectingFilter; @@ -363,9 +364,11 @@ public function test_a_schema_without_the_node_column_is_refused(): void public function test_signal_stop(): void { - $generator = from_xml(path_real(__DIR__ . '/../../../Fixtures/flow_orders.xml')) - ->withXMLNodePath('root/row') - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_xml(path_real(__DIR__ . '/../../../Fixtures/flow_orders.xml'))->withXMLNodePath('root/row'), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->next(); @@ -378,9 +381,11 @@ public function test_signal_stop(): void public function test_signal_stop_on_the_first_file_tail_batch_skips_the_remaining_files(): void { - $generator = from_xml(__DIR__ . '/../../../Fixtures/cross_stream/*/file.xml', 'root/item') - ->withBatchSize(10) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_xml(__DIR__ . '/../../../Fixtures/cross_stream/*/file.xml', 'root/item')->withBatchSize(10), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); $generator->send(Signal::STOP); @@ -409,7 +414,11 @@ public function test_a_limited_read_closes_its_stream(): void ->withXMLNodePath('root/items/item') ->withBatchSize(1); - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria(limit: 1)) as $_rows) { + foreach (FlowTestCase::extracted( + $extractor, + flow_context(config())->backend(), + new Criteria(limit: 1), + ) as $_rows) { } static::assertContains('closeSource', $filesystem->calls); @@ -444,7 +453,7 @@ public function test_a_read_stopped_at_a_batch_leaves_nothing_behind_for_the_nex { $extractor = from_xml(__DIR__ . '/../../../Fixtures/one_line_items.xml', 'root/item')->withBatchSize(1); - $extractor->extract(flow_context(config())->backend(), new Criteria())->send(Signal::STOP); + FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())->send(Signal::STOP); static::assertCount(10, ExtractedRows::of($extractor)); } @@ -456,7 +465,7 @@ public function test_a_read_stopped_inside_an_element_leaves_nothing_behind_for_ ->withBatchSize(1) ->withBufferSize(35); - $extractor->extract(flow_context(config())->backend(), new Criteria())->send(Signal::STOP); + FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())->send(Signal::STOP); static::assertEquals( ExtractedRows::of(from_xml(__DIR__ . '/../../../Fixtures/one_line_items.xml', 'root/item')), @@ -467,10 +476,13 @@ public function test_a_read_stopped_inside_an_element_leaves_nothing_behind_for_ public function test_a_stopped_read_closes_its_stream(): void { $filesystem = new RecordingFilesystem(native_local_filesystem()); - $generator = (new XMLParserExtractor(path_real(__DIR__ . '/../../../Fixtures/simple_items.xml'), $filesystem)) - ->withXMLNodePath('root/items/item') - ->withBatchSize(1) - ->extract(flow_context(config())->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + (new XMLParserExtractor(path_real(__DIR__ . '/../../../Fixtures/simple_items.xml'), $filesystem)) + ->withXMLNodePath('root/items/item') + ->withBatchSize(1), + flow_context(config())->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); @@ -523,9 +535,11 @@ public function test_extract_builds_through_the_config_backend(): void $backend = new SpyBackend(); iterator_to_array( - from_xml(path_real(__DIR__ . '/../../../Fixtures/one_line_items.xml')) - ->withXMLNodePath('root/item') - ->extract(flow_context(config_builder()->backend($backend)->build())->backend(), new Criteria()), + FlowTestCase::extracted( + from_xml(path_real(__DIR__ . '/../../../Fixtures/one_line_items.xml'))->withXMLNodePath('root/item'), + flow_context(config_builder()->backend($backend)->build())->backend(), + new Criteria(), + ), false, ); @@ -560,7 +574,8 @@ public function test_extract_split_stops_on_the_stop_signal(): void { $extractor = from_xml(__DIR__ . '/../../../Fixtures/flow_orders.xml', 'root/row'); $extractor->withBatchSize(1); - $generator = $extractor->extract( + $generator = FlowTestCase::extracted( + $extractor, flow_context()->backend(), (new Criteria())->withSplit($extractor->splits(new Criteria(), 1)[0]), ); diff --git a/src/bridge/filesystem/async-aws/tests/Flow/Filesystem/Bridge/AsyncAWS/Tests/Unit/AsyncAWSS3DestinationStreamTest.php b/src/bridge/filesystem/async-aws/tests/Flow/Filesystem/Bridge/AsyncAWS/Tests/Unit/AsyncAWSS3DestinationStreamTest.php index 75b382a9ca..fb464b6bb0 100644 --- a/src/bridge/filesystem/async-aws/tests/Flow/Filesystem/Bridge/AsyncAWS/Tests/Unit/AsyncAWSS3DestinationStreamTest.php +++ b/src/bridge/filesystem/async-aws/tests/Flow/Filesystem/Bridge/AsyncAWS/Tests/Unit/AsyncAWSS3DestinationStreamTest.php @@ -10,7 +10,7 @@ use Flow\Filesystem\Exception\RuntimeException; use PHPUnit\Framework\TestCase; -use function array_key_last; +use function end; use function Flow\Filesystem\DSL\path; final class AsyncAWSS3DestinationStreamTest extends TestCase @@ -34,7 +34,7 @@ public function test_exclusive_stream_aborts_and_asks_to_create_again_on_a_confl static::assertStringContainsString('call create() again', $e->getMessage()); } - static::assertSame('DELETE uploadId=upload-1', $calls[array_key_last($calls)]); + static::assertSame('DELETE uploadId=upload-1', end($calls)); } public function test_exclusive_stream_aborts_and_refuses_an_existing_key(): void @@ -53,6 +53,6 @@ public function test_exclusive_stream_aborts_and_refuses_an_existing_key(): void } catch (FileExistsException) { } - static::assertSame('DELETE uploadId=upload-1', $calls[array_key_last($calls)]); + static::assertSame('DELETE uploadId=upload-1', end($calls)); } } diff --git a/src/core/etl/src/Flow/ETL/Foundation/Exception/ColumnMismatchException.php b/src/core/etl/src/Flow/ETL/Foundation/Exception/ColumnMismatchException.php index 4c0c184820..3c65dab734 100644 --- a/src/core/etl/src/Flow/ETL/Foundation/Exception/ColumnMismatchException.php +++ b/src/core/etl/src/Flow/ETL/Foundation/Exception/ColumnMismatchException.php @@ -27,7 +27,7 @@ private function __construct( public readonly string $detail, ?Throwable $previous = null, ) { - parent::__construct('Row does not match its schema: column "' . $column . '"' . $detail, 0, $previous); + parent::__construct('Row does not match its schema: ' . $this->reason(), 0, $previous); } public static function missingColumn(string $column): self @@ -94,4 +94,9 @@ private static function describe(mixed $value): string default => get_debug_type($value), }; } + + public function reason(): string + { + return 'column "' . $this->column . '"' . $this->detail; + } } diff --git a/src/core/etl/src/Flow/ETL/Foundation/Exception/CorruptedRecordException.php b/src/core/etl/src/Flow/ETL/Foundation/Exception/CorruptedRecordException.php new file mode 100644 index 0000000000..f5ef2eb94d --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Foundation/Exception/CorruptedRecordException.php @@ -0,0 +1,178 @@ + + * } + */ +final class CorruptedRecordException extends RuntimeException +{ + private const int SHOWN_RECORD_LENGTH = 120; + + /** + * @param null|string $uri null for a source that is not a file + * @param int $offset the record's byte offset in a text file, its 1-based row number in a sheet or an array + * @param null|list $fields the cells a grid reader split the record into; null for keyed formats + */ + public function __construct( + public readonly ?string $uri, + public readonly int $offset, + public readonly ?int $lineNumber, + public readonly string $reason, + public readonly string $record, + public readonly ?array $fields = null, + ?Throwable $previous = null, + ) { + parent::__construct( + sprintf( + 'Corrupted record at offset %d%s of %s: %s: "%s". Route corrupted records to a loader with ' + . '->withCorruptedRecords($loader).', + $offset, + $lineNumber === null ? '' : ", line {$lineNumber},", + $uri ?? 'the source', + $reason, + mb_strlen($record) > self::SHOWN_RECORD_LENGTH + ? mb_substr($record, 0, self::SHOWN_RECORD_LENGTH) . '…' + : $record, + ), + 0, + $previous, + ); + } + + /** + * The inverse of toRecord(). + * + * @param array $record the shape of toRecord() + * + * @throws InvalidArgumentException + */ + public static function fromRecord(array $record, ?Throwable $previous = null): self + { + $shape = type_structure([ + 'uri' => type_optional(type_string()), + 'offset' => type_integer(), + 'line' => type_optional(type_integer()), + 'reason' => type_string(), + 'record' => type_string(), + 'fields' => type_optional(type_list(type_string())), + ]); + + if (!$shape->isValid($record)) { + throw new InvalidArgumentException( + 'A corrupted record must hold uri (?string), offset (int), line (?int), reason (string), record (string) ' + . 'and fields (?list)', + ); + } + + return new self( + $record['uri'], + $record['offset'], + $record['line'], + $record['reason'], + $record['record'], + $record['fields'], + $previous, + ); + } + + /** + * The row a routed corrupted record becomes. + * + * @return CorruptedRecordShape + */ + public function toRecord(): array + { + return [ + 'uri' => $this->uri, + 'offset' => $this->offset, + 'line' => $this->lineNumber, + 'reason' => $this->reason, + 'record' => $this->record, + 'fields' => $this->fields, + ]; + } + + /** + * A grid record: its cells in their string form, and its text the source's own when it holds one, else the cells + * as a JSON list. + * + * @param array $cells + */ + public static function fromCells( + ?string $uri, + int $offset, + ?int $lineNumber, + string $reason, + array $cells, + ?string $record = null, + ): self { + $fields = array_values(array_map(self::cell(...), $cells)); + + return new self($uri, $offset, $lineNumber, $reason, $record ?? self::json($fields), $fields); + } + + /** + * A keyed record: its text the source's own when it holds one, else the record as a JSON object. + * + * @param array $values + */ + public static function fromValues( + ?string $uri, + int $offset, + ?int $lineNumber, + string $reason, + array $values, + ?string $record = null, + ): self { + return new self($uri, $offset, $lineNumber, $reason, $record ?? self::json($values)); + } + + private static function cell(mixed $cell): string + { + return match (true) { + $cell === null => '', + $cell instanceof DateTimeInterface => $cell->format(DateTimeInterface::ATOM), + is_scalar($cell), $cell instanceof Stringable => (string) $cell, + default => self::json($cell), + }; + } + + private static function json(mixed $value): string + { + return (string) json_encode( + $value, + JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE, + ); + } +} diff --git a/src/core/etl/src/Flow/ETL/Foundation/Exception/InferredSchemaException.php b/src/core/etl/src/Flow/ETL/Foundation/Exception/InferredSchemaException.php index 6cac819ee1..c7b33c5580 100644 --- a/src/core/etl/src/Flow/ETL/Foundation/Exception/InferredSchemaException.php +++ b/src/core/etl/src/Flow/ETL/Foundation/Exception/InferredSchemaException.php @@ -62,6 +62,33 @@ public static function nothingToInferFrom(string $source, string $reason): self )); } + /** + * A file's record that does not fit a schema inferred from a sample - the sample can span files, so no position + * in this file places the record inside it. + */ + public static function outsideTheSample( + string $uri, + int $offset, + ?int $line, + int $sampleSize, + SchemaMismatchException $mismatch, + ): self { + return new self( + sprintf( + 'Record at offset %d%s of %s does not fit the schema inferred from a sample of %s rows: %s. ' + . 'Infer from more rows and files with ->inferSchema(infer_schema()->sampleSize(...)->filesToSniff(...)), ' + . 'or declare the schema with ->withSchema(...).', + $offset, + $line === null ? '' : ", line {$line},", + $uri, + $sampleSize === -1 ? 'all' : (string) $sampleSize, + $mismatch->cause->reason(), + ), + 0, + $mismatch, + ); + } + /** * The refusal to raise for $mismatch, a row at its 0-based position in the whole source: past a bounded sample it * becomes the way out, within the sample (or with no inference behind the schema) it stays the gate's own. @@ -79,14 +106,13 @@ public static function whenPastTheSample( return new self( sprintf( - 'Row %d of %s does not fit the schema inferred from its first %d rows: column "%s"%s. Infer from ' + 'Row %d of %s does not fit the schema inferred from its first %d rows: %s. Infer from ' . 'every row with ->inferSchema(infer_schema()->sampleSize(-1)), or declare the schema with ' . '->withSchema(...).', $mismatch->rowIndex, $source, $sampleSize, - $mismatch->cause->column, - $mismatch->cause->detail, + $mismatch->cause->reason(), ), 0, $mismatch, diff --git a/src/core/etl/src/Flow/ETL/Foundation/Exception/SchemaValidationException.php b/src/core/etl/src/Flow/ETL/Foundation/Exception/SchemaValidationException.php index 1ce67e9856..022bf4b821 100644 --- a/src/core/etl/src/Flow/ETL/Foundation/Exception/SchemaValidationException.php +++ b/src/core/etl/src/Flow/ETL/Foundation/Exception/SchemaValidationException.php @@ -4,10 +4,29 @@ namespace Flow\ETL\Foundation\Exception; +use function array_map; +use function implode; +use function sprintf; + final class SchemaValidationException extends RuntimeException { public function __construct(string $mismatches) { parent::__construct("Schema validation failed: \n" . $mismatches); } + + /** + * @param list $missing the declared columns $header lacks + * @param list $header the columns the source carries + */ + public static function declaredColumnsMissing(string $source, array $missing, array $header): self + { + return new self(sprintf( + 'Columns %s declared by the schema are not in the header of %s (%s). Correct their names, or remove them ' + . 'from ->withSchema(...).', + implode(', ', array_map(static fn(string $column): string => '"' . $column . '"', $missing)), + $source, + implode(', ', $header), + )); + } } diff --git a/src/core/etl/src/Flow/ETL/Functions/ToDateTime.php b/src/core/etl/src/Flow/ETL/Functions/ToDateTime.php index ca45b4cffb..3a4c65577d 100644 --- a/src/core/etl/src/Flow/ETL/Functions/ToDateTime.php +++ b/src/core/etl/src/Flow/ETL/Functions/ToDateTime.php @@ -145,8 +145,14 @@ public function eval(Rows $rows, Backend $backend): Column $format = $formats[$i]; $timeZone = $timeZones[$i]; - if ($value === null || $format === null || $timeZone === null) { - throw new InvalidArgumentException('ToDateTime function requires non-null values'); + if ($value === null) { + $results[] = null; + + continue; + } + + if ($format === null || $timeZone === null) { + throw new InvalidArgumentException('ToDateTime function requires a non-null format and time zone'); } if (is_object($value)) { diff --git a/src/core/etl/src/Flow/ETL/IO/CorruptedRecordsRouter.php b/src/core/etl/src/Flow/ETL/IO/CorruptedRecordsRouter.php new file mode 100644 index 0000000000..4e5f19be9b --- /dev/null +++ b/src/core/etl/src/Flow/ETL/IO/CorruptedRecordsRouter.php @@ -0,0 +1,15 @@ +conjuncts; only a FileExtractor ever receives any. The limit and the path filter are hints: the plan * still enforces the limit, and the predicate of every path filter is also one of the conjuncts or stays in a * Filter node, so a source that reads or lists more is still correct. Batches built on another backend than - * $backend are adopted by the runtime. + * $backend are adopted by the runtime. A record that cannot become a row is yielded as its + * CorruptedRecordException, in source order; the runtime routes or refuses it. * - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator; diff --git a/src/core/etl/src/Flow/ETL/IO/Extractor/ArrayExtractor.php b/src/core/etl/src/Flow/ETL/IO/Extractor/ArrayExtractor.php index 94a5234496..6241a3eda8 100644 --- a/src/core/etl/src/Flow/ETL/IO/Extractor/ArrayExtractor.php +++ b/src/core/etl/src/Flow/ETL/IO/Extractor/ArrayExtractor.php @@ -4,19 +4,23 @@ namespace Flow\ETL\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidLogicException; use Flow\ETL\Foundation\Exception\SchemaMismatchException; +use Flow\ETL\IO\CorruptedRecordsRouter; use Flow\ETL\IO\Extractor; use Flow\ETL\IO\Extractor\Memory\InMemoryRows; use Flow\ETL\IO\Extractor\Memory\SpilledRows; use Flow\ETL\IO\Inference\SchemaInference; use Flow\ETL\IO\Inference\SchemaInferenceBuilder; use Flow\ETL\IO\Inference\SchemaInferrer; +use Flow\ETL\IO\Loader; use Flow\ETL\Storage\Memory\Memory; use Flow\ETL\Values\BatchSize; use Flow\ETL\Values\Cardinality; use Flow\ETL\Values\Column\Backend; +use Flow\ETL\Values\Rows; use Flow\ETL\Values\Rows\RecordBatch; use Flow\ETL\Values\Schema; use Flow\Filesystem\Filesystem; @@ -27,8 +31,10 @@ use function count; use function is_array; -final class ArrayExtractor implements BatchableExtractor, Extractor, InfersSchema, Refreshable +final class ArrayExtractor implements BatchableExtractor, Extractor, InfersSchema, Refreshable, CorruptedRecordsRouter { + private ?Loader $corruptedRecordsLoader = null; + private BatchSize $batchSize; /** @@ -87,36 +93,36 @@ public function isRepeatable(): bool return true; } + public function corruptedRecordsLoader(): ?Loader + { + return $this->corruptedRecordsLoader; + } + /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { // a one-shot dataset is typed from every row it spills, so only an in-place dataset's bounded sample can be outrun $inPlace = $this->inPlace(); $sample = $this->schema === null && $inPlace !== null ? $this->inference : null; - $batches = new RecordBatch($this->describe(), $backend, $this->batchSize()); + $schema = $this->describe(); + $batches = new RecordBatch( + $schema, + $backend, + $this->batchSize(), + sampleSize: $this->schema === null ? $sample->sampleSize ?? -1 : null, + known: $schema, + ); try { - foreach ($this->source?->rows() ?? $inPlace ?? $this->dataset as $record) { - $rows = $batches->add($record); - - if ($rows === null) { - continue; - } - + foreach ($batches->of($this->source?->rows() ?? $inPlace ?? $this->dataset) as $rows) { $signal = yield $rows; if ($signal === Signal::STOP) { return; } } - - $rest = $batches->flush(); - - if ($rest !== null) { - yield $rest; - } } catch (SchemaMismatchException $mismatch) { throw InferredSchemaException::whenPastTheSample( $this->memory === null ? 'from_array()' : 'from_memory()', @@ -227,6 +233,13 @@ public function withBatchSize(int $batchSize): static return $this; } + public function withCorruptedRecords(?Loader $loader): static + { + $this->corruptedRecordsLoader = $loader; + + return $this; + } + public function withSchema(Schema $schema): static { $this->schema = $schema; diff --git a/src/core/etl/src/Flow/ETL/IO/Extractor/CacheExtractor.php b/src/core/etl/src/Flow/ETL/IO/Extractor/CacheExtractor.php index 6f7480517e..c173640a2e 100644 --- a/src/core/etl/src/Flow/ETL/IO/Extractor/CacheExtractor.php +++ b/src/core/etl/src/Flow/ETL/IO/Extractor/CacheExtractor.php @@ -4,6 +4,7 @@ namespace Flow\ETL\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidLogicException; use Flow\ETL\Foundation\Exception\SchemaNotDerivableException; @@ -38,7 +39,7 @@ public function __construct( ) {} /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { @@ -56,8 +57,11 @@ public function extract(Backend $backend, Criteria $criteria): Generator if (!$cache->has($this->id)) { if ($this->fallbackExtractor !== null) { + // a corrupted record passes through: the cache's read has no reject loader, so it is refused foreach ($this->fallbackExtractor->extract($backend, new Criteria()) as $rows) { - $signal = yield $declared === null ? $rows : $rows->matchTo($declared, $backend); + $signal = yield $declared === null || !$rows instanceof Rows + ? $rows + : $rows->matchTo($declared, $backend); if ($signal === Signal::STOP) { return; diff --git a/src/core/etl/src/Flow/ETL/IO/Extractor/ChainExtractor.php b/src/core/etl/src/Flow/ETL/IO/Extractor/ChainExtractor.php index 79d8223e23..d8ad0ea134 100644 --- a/src/core/etl/src/Flow/ETL/IO/Extractor/ChainExtractor.php +++ b/src/core/etl/src/Flow/ETL/IO/Extractor/ChainExtractor.php @@ -4,6 +4,7 @@ namespace Flow\ETL\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\IO\Extractor; use Flow\ETL\Values\Cardinality; @@ -40,7 +41,7 @@ public function isRepeatable(): bool } /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { @@ -52,8 +53,13 @@ public function extract(Backend $backend, Criteria $criteria): Generator foreach ($this->extractors as $extractor) { foreach ($extractor->extract($backend, new Criteria()) as $rows) { - $schema ??= $this->schema(); - $signal = yield $rows->matchTo($schema, $backend); + // a corrupted record passes through: the chain's read has no reject loader, so it is refused + if (!$rows instanceof Rows) { + $signal = yield $rows; + } else { + $schema ??= $this->schema(); + $signal = yield $rows->matchTo($schema, $backend); + } if ($signal === Signal::STOP) { return; diff --git a/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileBatches.php b/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileBatches.php index e8db880718..7e5760daed 100644 --- a/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileBatches.php +++ b/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileBatches.php @@ -5,6 +5,7 @@ namespace Flow\ETL\IO\Extractor\File; use Flow\ETL\Foundation\Describe\Describable; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\Values\Column\Backend; use Flow\ETL\Values\Rows; @@ -24,9 +25,10 @@ interface FileBatches extends Describable * FileReadLoop counts. FileReadLoop applies the conjuncts to every row returned; a format given them when built may * skip what they prove cannot match. * - * A file's own resources are released by the generator (finally); those that span files go at close(). + * A file's own resources are released by the generator (finally); those that span files go at close(). A corrupted + * record is handed up as its CorruptedRecordException, in record order, never thrown. * - * @return Generator + * @return Generator */ public function batches( FileRead $read, diff --git a/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileRead.php b/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileRead.php index 578f5a54c9..59a02eadf2 100644 --- a/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileRead.php +++ b/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileRead.php @@ -6,6 +6,7 @@ use Flow\ETL\Foundation\Describe\ConfiguredFilesystem; use Flow\ETL\Foundation\Describe\Rebuild; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\Signal; use Flow\ETL\Values\Column\Backend; @@ -63,7 +64,7 @@ public function normalize(): array /** * The files $criteria reads, through $batches. * - * @return Generator + * @return Generator */ public function scan(Backend $backend, Criteria $criteria, FileBatches $batches): Generator { diff --git a/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileReadLoop.php b/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileReadLoop.php index 6b50e76360..b16a09ec27 100644 --- a/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileReadLoop.php +++ b/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileReadLoop.php @@ -4,6 +4,7 @@ namespace Flow\ETL\IO\Extractor\File; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Functions\Evaluation\Filter; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\Signal; @@ -21,7 +22,7 @@ public function __construct( /** * @param list $sources * - * @return Generator + * @return Generator */ public function read(array $sources, FileBatches $batches, Backend $backend, ReadWindow $window): Generator { @@ -51,6 +52,17 @@ public function read(array $sources, FileBatches $batches, Backend $backend, Rea ); foreach ($file as $batch) { + // a corrupted record is no row: it passes through, outside the filter and the window + if ($batch instanceof CorruptedRecordException) { + $signal = yield $batch; + + if ($signal === Signal::STOP) { + return; + } + + continue; + } + $filtered = $filter->filter($constants->fillRows($batch, $schema, $backend), $backend); $rows = $skipping->skip($filtered, $skipped); $skipped += $filtered->count() - $rows->count(); diff --git a/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileSplitReader.php b/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileSplitReader.php index 6634579cb2..87e19b563a 100644 --- a/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileSplitReader.php +++ b/src/core/etl/src/Flow/ETL/IO/Extractor/File/FileSplitReader.php @@ -7,19 +7,25 @@ use Flow\ETL\Foundation\Describe\Describable; use Flow\ETL\Foundation\Describe\Description; use Flow\ETL\Foundation\Describe\Rebuild; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; +use Flow\ETL\IO\CorruptedRecordsRouter; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\Reader; +use Flow\ETL\IO\Extractor\Signal; +use Flow\ETL\IO\Loader; use Flow\ETL\Values\Column\Backend; +use Flow\ETL\Values\Rows; use Generator; use SensitiveParameter; use function Flow\Types\DSL\type_array; -final readonly class FileSplitReader implements Describable, Reader +final readonly class FileSplitReader implements Describable, Reader, CorruptedRecordsRouter { public function __construct( private FileRead $read, private FileBatches $batches, + private ?Loader $corruptedRecordsLoader = null, ) {} public static function fromDescription(#[SensitiveParameter] array $data, Rebuild $rebuild): static @@ -27,17 +33,32 @@ public static function fromDescription(#[SensitiveParameter] array $data, Rebuil return new self( FileRead::fromArray(type_array()->assert($data['read'] ?? null), $rebuild), $rebuild->of($data['batches'] ?? null, FileBatches::class), + ($data['corrupted_records_loader'] ?? null) === null + ? null + : $rebuild->of($data['corrupted_records_loader'], Loader::class), ); } + public function corruptedRecordsLoader(): ?Loader + { + return $this->corruptedRecordsLoader; + } + + /** + * Null when the reject loader does not describe: such a read runs only in this process. + */ public function describe(): ?Description { return Description::compose(self::class, [ 'read' => $this->read->normalize(), 'batches' => $this->batches, + 'corrupted_records_loader' => $this->corruptedRecordsLoader, ]); } + /** + * @return Generator + */ public function extract(Backend $backend, Criteria $criteria): Generator { try { @@ -46,4 +67,9 @@ public function extract(Backend $backend, Criteria $criteria): Generator $this->batches->close(); } } + + public function withCorruptedRecords(?Loader $loader): static + { + return new self($this->read, $this->batches, $loader); + } } diff --git a/src/core/etl/src/Flow/ETL/IO/Extractor/Grid/RecordDecoder.php b/src/core/etl/src/Flow/ETL/IO/Extractor/Grid/RecordDecoder.php index 559905ec19..6da577be72 100644 --- a/src/core/etl/src/Flow/ETL/IO/Extractor/Grid/RecordDecoder.php +++ b/src/core/etl/src/Flow/ETL/IO/Extractor/Grid/RecordDecoder.php @@ -9,6 +9,7 @@ use function array_fill; use function array_values; use function count; +use function sprintf; /** * Decodes the records of one file or stream: the header it resolves from the first record is that source's. @@ -29,7 +30,7 @@ final class RecordDecoder public function __construct( private readonly bool $withHeader, private readonly bool $emptyToNull, - private readonly bool $dropExtraColumns = true, + private readonly bool $missingColumnsAsNull = false, ?array $headers = null, ) { if ($headers !== null && $withHeader) { @@ -40,56 +41,52 @@ public function __construct( } /** - * Records before the header is resolved that hold no cell are skipped. A cell a record lacks is null; a present - * '' cell is null under emptyToNull. + * An extra empty cell ('' or null) loses nothing, so only a non-empty one makes a record corrupted. A missing cell + * makes it corrupted unless missingColumnsAsNull; a present '' cell is null under emptyToNull. * - * @param list> $batch + * @param array $cells * - * @throws InvalidArgumentException a record with more cells than headers, unless extra columns are dropped - * - * @return list> + * @return null|array|string null for a record that is no row (the header, an empty record), why + * the record is corrupted, or the row's values */ - public function decode(array $batch): array + public function decode(array $cells): array|string|null { - $decoded = []; - - foreach ($batch as $record) { - $cells = array_values($record); + $cells = array_values($cells); - if ($this->headers === null) { - if ($cells === []) { - continue; - } + if ($cells === [] || $cells === [null]) { + return null; + } - $this->headers = (new HeaderNames())->of( - $this->withHeader ? $cells : array_fill(0, count($cells), null), - ); + if ($this->headers === null) { + $this->headers = (new HeaderNames())->of($this->withHeader ? $cells : array_fill(0, count($cells), null)); - if ($this->withHeader) { - continue; - } + if ($this->withHeader) { + return null; } + } - if (!$this->dropExtraColumns && count($cells) > count($this->headers)) { - throw InvalidArgumentException::because( - 'Row has more columns (%d) than headers (%d)', - count($cells), - count($this->headers), - ); - } + $headers = count($this->headers); + $fields = count($cells); - $values = []; + $corrupted = $fields < $headers && !$this->missingColumnsAsNull; - foreach ($this->headers as $index => $name) { - // @mago-ignore analysis:mixed-assignment - $cell = $cells[$index] ?? null; - $values[$name] = $this->emptyToNull && $cell === '' ? null : $cell; - } + for ($index = $headers; !$corrupted && $index < $fields; $index++) { + $corrupted = $cells[$index] !== null && $cells[$index] !== ''; + } + + if ($corrupted) { + return sprintf('%d fields, the header has %d', $fields, $headers); + } + + $values = []; - $decoded[] = $values; + foreach ($this->headers as $index => $name) { + // @mago-ignore analysis:mixed-assignment + $cell = $cells[$index] ?? null; + $values[$name] = $this->emptyToNull && $cell === '' ? null : $cell; } - return $decoded; + return $values; } /** diff --git a/src/core/etl/src/Flow/ETL/IO/Extractor/Reader.php b/src/core/etl/src/Flow/ETL/IO/Extractor/Reader.php index 00482eb4e2..12fb0d9659 100644 --- a/src/core/etl/src/Flow/ETL/IO/Extractor/Reader.php +++ b/src/core/etl/src/Flow/ETL/IO/Extractor/Reader.php @@ -4,6 +4,7 @@ namespace Flow\ETL\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Values\Column\Backend; use Flow\ETL\Values\Rows; use Generator; @@ -12,9 +13,9 @@ interface Reader { /** * The read of a source with every state resolved (schema, options, inference): the whole plan scan in-process, - * one split of it in a worker. + * one split of it in a worker. A record that cannot become a row is yielded as its CorruptedRecordException. * - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator; } diff --git a/src/core/etl/src/Flow/ETL/IO/Extractor/SequenceExtractor.php b/src/core/etl/src/Flow/ETL/IO/Extractor/SequenceExtractor.php index e77cc6fb2e..07edac57e9 100644 --- a/src/core/etl/src/Flow/ETL/IO/Extractor/SequenceExtractor.php +++ b/src/core/etl/src/Flow/ETL/IO/Extractor/SequenceExtractor.php @@ -4,14 +4,17 @@ namespace Flow\ETL\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\SchemaMismatchException; +use Flow\ETL\IO\CorruptedRecordsRouter; use Flow\ETL\IO\Extractor; use Flow\ETL\IO\Extractor\Memory\InMemoryRows; use Flow\ETL\IO\Extractor\SequenceGenerator\SequenceGenerator; use Flow\ETL\IO\Inference\SchemaInference; use Flow\ETL\IO\Inference\SchemaInferenceBuilder; use Flow\ETL\IO\Inference\SchemaInferrer; +use Flow\ETL\IO\Loader; use Flow\ETL\Values\BatchSize; use Flow\ETL\Values\Column\Backend; use Flow\ETL\Values\Rows; @@ -19,8 +22,10 @@ use Flow\ETL\Values\Schema; use Generator; -final class SequenceExtractor implements BatchableExtractor, Extractor, InfersSchema +final class SequenceExtractor implements BatchableExtractor, Extractor, InfersSchema, CorruptedRecordsRouter { + private ?Loader $corruptedRecordsLoader = null; + private BatchSize $batchSize; private ?Schema $derivedSchema = null; @@ -52,40 +57,50 @@ public function isRepeatable(): bool return true; } + public function corruptedRecordsLoader(): ?Loader + { + return $this->corruptedRecordsLoader; + } + /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { $sample = $this->schema === null ? $this->inference : null; - $batches = new RecordBatch($this->describe(), $backend, $this->batchSize()); + $schema = $this->describe(); + $batches = new RecordBatch( + $schema, + $backend, + $this->batchSize(), + sampleSize: $sample?->sampleSize, + known: $schema, + ); try { - /** @var mixed $item */ - foreach ($this->generator->generate() as $item) { - $rows = $batches->add([$this->columnName => $item]); - - if ($rows === null) { - continue; - } - + foreach ($batches->of($this->records()) as $rows) { $signal = yield $rows; if ($signal === Signal::STOP) { return; } } - - $rest = $batches->flush(); - - if ($rest !== null) { - yield $rest; - } } catch (SchemaMismatchException $mismatch) { throw InferredSchemaException::whenPastTheSample('the sequence', $sample?->sampleSize, $mismatch); } } + /** + * @return Generator> + */ + private function records(): Generator + { + /** @var mixed $item */ + foreach ($this->generator->generate() as $item) { + yield [$this->columnName => $item]; + } + } + public function inferSchema(SchemaInferenceBuilder $builder): static { $this->inference = $builder->build(); @@ -143,6 +158,13 @@ public function withBatchSize(int $batchSize): static return $this; } + public function withCorruptedRecords(?Loader $loader): static + { + $this->corruptedRecordsLoader = $loader; + + return $this; + } + public function withSchema(Schema $schema): static { $this->schema = $schema; diff --git a/src/core/etl/src/Flow/ETL/IO/Floe/FloeExtractor.php b/src/core/etl/src/Flow/ETL/IO/Floe/FloeExtractor.php index b66438ee1c..1e88aa76d7 100644 --- a/src/core/etl/src/Flow/ETL/IO/Floe/FloeExtractor.php +++ b/src/core/etl/src/Flow/ETL/IO/Floe/FloeExtractor.php @@ -4,6 +4,7 @@ namespace Flow\ETL\IO\Floe; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; use Flow\ETL\Functions\ScalarFunction; @@ -90,7 +91,7 @@ public function isRepeatable(): bool } /** - * @return Generator + * @return Generator */ public function extract(Backend $backend, Criteria $criteria): Generator { diff --git a/src/core/etl/src/Flow/ETL/Operators/Transformer/JoinEachRowsTransformer.php b/src/core/etl/src/Flow/ETL/Operators/Transformer/JoinEachRowsTransformer.php index 14622ca18d..e5f7b83fd7 100644 --- a/src/core/etl/src/Flow/ETL/Operators/Transformer/JoinEachRowsTransformer.php +++ b/src/core/etl/src/Flow/ETL/Operators/Transformer/JoinEachRowsTransformer.php @@ -83,7 +83,13 @@ public function spanAttributes(): array public function transform(Rows $rows, Backend $backend): Rows { $right = $this->factory->from($rows); - $batches = iterator_to_array($right->extract($backend, new Criteria()), false); + $batches = []; + + // a join has no reject loader: a corrupted record of the right side is refused + foreach ($right->extract($backend, new Criteria()) as $batch) { + $batches[] = $batch instanceof Rows ? $batch : throw $batch; + } + $rightRows = $batches === [] ? Rows::empty($right->schema(), $backend) : Rows::concat($batches[0]->schema(), $backend, ...$batches); diff --git a/src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/ExtractionError.php b/src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/ExtractionError.php index 2ea1ab3ae6..8b85c9f880 100644 --- a/src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/ExtractionError.php +++ b/src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/ExtractionError.php @@ -5,12 +5,13 @@ namespace Flow\ETL\Runtime\ErrorHandler; use Flow\ETL\IO\Extractor; +use Flow\ETL\IO\Extractor\Reader; use Throwable; final readonly class ExtractionError { public function __construct( public Throwable $cause, - public Extractor $extractor, + public Extractor|Reader $extractor, ) {} } diff --git a/src/core/etl/src/Flow/ETL/Runtime/Executor/AdoptedBatches.php b/src/core/etl/src/Flow/ETL/Runtime/Executor/AdoptedBatches.php index 156a60afb2..e24739b594 100644 --- a/src/core/etl/src/Flow/ETL/Runtime/Executor/AdoptedBatches.php +++ b/src/core/etl/src/Flow/ETL/Runtime/Executor/AdoptedBatches.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Runtime\Executor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Extractor\Signal; use Flow\ETL\Values\Column\Backend; use Flow\ETL\Values\Rows; @@ -16,16 +17,20 @@ public function __construct( ) {} /** - * Every batch of $batches in the backend's storage, a stop forwarded to $batches. + * Every batch of $batches in the backend's storage, a corrupted record as it is, a stop forwarded to $batches. * - * @param Generator $batches what an extractor yields + * @template T of CorruptedRecordException|Rows * - * @return Generator + * @param Generator $batches what an extractor yields + * + * @return Generator */ public function of(Generator $batches): Generator { foreach ($batches as $batch) { - $signal = yield $batch->adoptedBy($this->backend); + /** @var T $adopted Rows adopted are Rows: T is kept */ + $adopted = $batch instanceof Rows ? $batch->adoptedBy($this->backend) : $batch; + $signal = yield $adopted; if ($signal === Signal::STOP) { $batches->send(Signal::STOP); diff --git a/src/core/etl/src/Flow/ETL/Runtime/Executor/Executor.php b/src/core/etl/src/Flow/ETL/Runtime/Executor/Executor.php index 608875f86f..ab00912f66 100644 --- a/src/core/etl/src/Flow/ETL/Runtime/Executor/Executor.php +++ b/src/core/etl/src/Flow/ETL/Runtime/Executor/Executor.php @@ -211,9 +211,10 @@ private function run( : $runner->steps($stage->steps, $generator, $context, $position === 0 ? $source : null, $sources); } - // a foreach, never `yield from`: the re-yield must not forward the consumer's sent signal + // a foreach, never `yield from`: the re-yield must not forward the consumer's sent signal; every stage routes + // or refuses a corrupted record, so one left here had no stage to meet foreach ($generator as $rows) { - yield $rows; + yield $rows instanceof Rows ? $rows : throw $rows; } } } diff --git a/src/core/etl/src/Flow/ETL/Runtime/Executor/SourceCount.php b/src/core/etl/src/Flow/ETL/Runtime/Executor/SourceCount.php index e6a39d40ce..e3fee6899f 100644 --- a/src/core/etl/src/Flow/ETL/Runtime/Executor/SourceCount.php +++ b/src/core/etl/src/Flow/ETL/Runtime/Executor/SourceCount.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Runtime\Executor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Extractor\Signal; use Flow\ETL\Values\Rows; use Generator; @@ -23,16 +24,20 @@ public function opened(bool $narrowed): void } /** - * Counts $batches into this read, forwards a STOP, closed() once exhausted. + * Counts the rows of $batches into this read, a corrupted record none, forwards a STOP, closed() once exhausted. * - * @param Generator $batches + * @template T of CorruptedRecordException|Rows * - * @return Generator + * @param Generator $batches + * + * @return Generator */ public function count(Generator $batches): Generator { foreach ($batches as $batch) { - $this->counted($batch->count()); + if ($batch instanceof Rows) { + $this->counted($batch->count()); + } $signal = yield $batch; diff --git a/src/core/etl/src/Flow/ETL/Runtime/Executor/SourceCounts.php b/src/core/etl/src/Flow/ETL/Runtime/Executor/SourceCounts.php index 54a3c3bca0..2c443d9ec8 100644 --- a/src/core/etl/src/Flow/ETL/Runtime/Executor/SourceCounts.php +++ b/src/core/etl/src/Flow/ETL/Runtime/Executor/SourceCounts.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Runtime\Executor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Extractor; use Flow\ETL\IO\Extractor\Signal; use Flow\ETL\IO\Extractor\Statistics; @@ -42,10 +43,12 @@ public function __construct() * Passes $batches through unchanged. A stop is forwarded and ends the count: the consumer that sent it reads no * further batch, so that read never closes. * - * @param Generator $batches what $extractor yields + * @template T of CorruptedRecordException|Rows + * + * @param Generator $batches what $extractor yields * @param bool $narrowed a limit or a partition filter was pushed into this read * - * @return Generator + * @return Generator */ public function count(Extractor $extractor, Generator $batches, bool $narrowed): Generator { diff --git a/src/core/etl/src/Flow/ETL/Runtime/Executor/StepRunner.php b/src/core/etl/src/Flow/ETL/Runtime/Executor/StepRunner.php index 6b04ed4b45..1b7b14ec24 100644 --- a/src/core/etl/src/Flow/ETL/Runtime/Executor/StepRunner.php +++ b/src/core/etl/src/Flow/ETL/Runtime/Executor/StepRunner.php @@ -4,11 +4,15 @@ namespace Flow\ETL\Runtime\Executor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; +use Flow\ETL\IO\CorruptedRecordsRouter; use Flow\ETL\IO\Extractor; +use Flow\ETL\IO\Extractor\Reader; use Flow\ETL\IO\Extractor\Signal; use Flow\ETL\IO\Loader; use Flow\ETL\IO\Loader\Closure; use Flow\ETL\IO\Loader\Discardable; +use Flow\ETL\IO\Loader\File\FileLoader; use Flow\ETL\Operators\Processor; use Flow\ETL\Operators\Transformer; use Flow\ETL\Operators\Transformer\Limiting; @@ -22,10 +26,18 @@ use Flow\ETL\Runtime\ErrorHandler\TransformationError; use Flow\ETL\Runtime\FlowContext; use Flow\ETL\Values\Rows; +use Flow\ETL\Values\Rows\RowsBuilder; +use Flow\ETL\Values\Schema; +use Flow\ETL\Values\Schema\Definition\IntegerDefinition; +use Flow\ETL\Values\Schema\Definition\ListDefinition; +use Flow\ETL\Values\Schema\Definition\StringDefinition; use Generator; use Throwable; use function count; +use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_string; +use function sprintf; final readonly class StepRunner { @@ -39,8 +51,9 @@ public function __construct( * goes to run(), an empty run included, and one more after the breaker. A join reads the rows of its right plan; * a branch runs as the step that feeds it. * - * @param Generator $input - * @param null|Extractor $source the extractor $input comes from directly, so its failures reach the error handler + * @param Generator $input + * @param null|Extractor|Reader $source the source $input comes from directly, so its failures reach the error + * handler and its corrupted records its reject loader * @param null|SourceCounts $sources counts the rows the sources of a join's right plan yield * * @return Generator @@ -49,7 +62,7 @@ public function steps( PipelineSteps $steps, Generator $input, FlowContext $context, - ?Extractor $source = null, + Extractor|Reader|null $source = null, ?SourceCounts $sources = null, ): Generator { $run = []; @@ -87,18 +100,32 @@ public function steps( } /** - * $input through one run of loaders and transformers: drops empty batches, ends its loaders when the run ends. + * $input through one run of loaders and transformers: drops empty batches, ends its loaders when the run ends. A + * corrupted record goes to the source's reject loader, loaded and ended like the run's own loaders; without one it + * fails the run, whatever the error handler. * * @param list $steps one run between two processors - * @param Generator $input - * @param null|Extractor $source the extractor $input comes from directly, so its failures reach the error handler + * @param Generator $input + * @param null|Extractor|Reader $source the source $input comes from directly, so its failures reach the error + * handler and its corrupted records its reject loader * * @return Generator */ - public function run(array $steps, Generator $input, FlowContext $context, ?Extractor $source = null): Generator - { + public function run( + array $steps, + Generator $input, + FlowContext $context, + Extractor|Reader|null $source = null, + ): Generator { $loaders = []; $limits = []; + $rejects = $source instanceof CorruptedRecordsRouter ? $source->corruptedRecordsLoader() : null; + /** @var list> $corrupted */ + $corrupted = []; + $routed = 0; + if ($rejects !== null) { + $loaders[] = $rejects; + } foreach ($steps as $step) { if (!$step instanceof Transformer) { @@ -154,70 +181,62 @@ public function run(array $steps, Generator $input, FlowContext $context, ?Extra continue; } + if ($rows instanceof CorruptedRecordException) { + // the reader's option alone owns a corrupted record: no error handler skips it + if ($rejects === null) { + throw $rows; + } + + $corrupted[] = $rows->toRecord(); + $routed++; + + continue; + } + + // a batch's corrupted records come before its rows, so they are loaded together when the rows arrive + if ($rejects !== null && $corrupted !== []) { + $this->loadRejects($rejects, $corrupted, $context); + $corrupted = []; + } + // the batch is half transformed, so its columns no longer match its siblings - // skipping means emitting nothing, not emitting what the failed step had produced $skipped = false; $stop = false; foreach ($steps as $step) { - $error = null; - - if ($step instanceof Transformer || $step instanceof Loader) { - try { - if ($step instanceof Transformer) { - $rows = $this->telemetry->transform($step, $rows, $context); - } elseif ($rows->count()) { - $this->telemetry->load($step, $rows, $context); - } - } catch (Throwable $failure) { - $error = $step instanceof Transformer - ? new TransformationError($failure, $step, $rows) - : new LoadingError($failure, $step, $rows); - } - } else { - // a NestedLoader: what it throws is never offered here, see NestedLoader::load() - $error = $rows->count() ? $step->load($rows) : null; - } + if (!$step instanceof Transformer) { + $this->load($step, $rows, $context); - if ($error === null) { continue; } - if ( - $error instanceof LoadingError - ? $context->errorHandler()->onLoading($error) === LoadingAction::propagate - : $context->errorHandler()->onTransformation($error) === TransformationAction::propagate - ) { - $context - ->telemetry() - ->logger() - ->error('Error during ETL pipeline execution.', ['exception' => $error->cause]); + try { + $rows = $this->telemetry->transform($step, $rows, $context); + } catch (Throwable $failure) { + if ( + $context->errorHandler()->onTransformation(new TransformationError($failure, $step, $rows)) + === TransformationAction::propagate + ) { + $context + ->telemetry() + ->logger() + ->error('Error during ETL pipeline execution.', ['exception' => $failure]); - throw $error->cause; - } + throw $failure; + } - // a write that failed is skipped and the batch flows on, a rolled-back transaction included - if (!$step instanceof Transformer) { $context ->telemetry() ->logger() - ->debug('Skipping loader due to error during ETL pipeline execution.', [ - 'exception' => $error->cause, + ->debug('Skipping rows due to error during ETL pipeline execution.', [ + 'exception' => $failure, ]); - continue; - } - - $context - ->telemetry() - ->logger() - ->debug('Skipping rows due to error during ETL pipeline execution.', [ - 'exception' => $error->cause, - ]); - - $skipped = true; + $skipped = true; - break; + break; + } } // a limit trims the batch it is reached in, so the steps after it still ran, on the trimmed batch @@ -244,6 +263,10 @@ public function run(array $steps, Generator $input, FlowContext $context, ?Extra } } + if ($rejects !== null && $corrupted !== []) { + $this->loadRejects($rejects, $corrupted, $context); + } + $completed = true; } finally { $endings = $this->endLoaders($loaders, $context, $completed); @@ -253,6 +276,89 @@ public function run(array $steps, Generator $input, FlowContext $context, ?Extra if ($endings !== []) { throw $endings[0]; } + + if ($rejects !== null && $routed > 0) { + $context + ->telemetry() + ->logger() + ->warn(sprintf( + 'Routed %d corrupted records to %s.', + $routed, + $rejects instanceof FileLoader ? $rejects->destination()->uri() : $rejects::class, + )); + } + } + + /** + * @param non-empty-list> $records CorruptedRecordException::toRecord() of each + */ + private function loadRejects(Loader $rejects, array $records, FlowContext $context): void + { + $this->load( + $rejects, + (new RowsBuilder( + new Schema( + new StringDefinition('uri', nullable: true), + new IntegerDefinition('offset'), + new IntegerDefinition('line', nullable: true), + new StringDefinition('reason'), + new StringDefinition('record'), + new ListDefinition('fields', type_list(type_string()), nullable: true), + ), + $context->backend(), + )) + ->appendRecords($records) + ->finish(), + $context, + ); + } + + /** + * $rows into $loader; a failure the error handler skips is logged and the batch flows on, a rolled-back + * transaction included. + */ + private function load(Loader|NestedLoader $loader, Rows $rows, FlowContext $context): void + { + if (!$rows->count()) { + return; + } + + if ($loader instanceof Loader) { + try { + $this->telemetry->load($loader, $rows, $context); + + return; + } catch (Throwable $failure) { + $error = new LoadingError($failure, $loader, $rows); + } + } else { + // what a NestedLoader throws is never offered here, see NestedLoader::load() + $error = $loader->load($rows); + } + + if ($error === null) { + return; + } + + if ( + $error instanceof LoadingError + ? $context->errorHandler()->onLoading($error) === LoadingAction::propagate + : $context->errorHandler()->onTransformation($error) === TransformationAction::propagate + ) { + $context + ->telemetry() + ->logger() + ->error('Error during ETL pipeline execution.', ['exception' => $error->cause]); + + throw $error->cause; + } + + $context + ->telemetry() + ->logger() + ->debug('Skipping loader due to error during ETL pipeline execution.', [ + 'exception' => $error->cause, + ]); } /** diff --git a/src/core/etl/src/Flow/ETL/Values/Rows/RecordBatch.php b/src/core/etl/src/Flow/ETL/Values/Rows/RecordBatch.php index 026a1006fb..42e53e889c 100644 --- a/src/core/etl/src/Flow/ETL/Values/Rows/RecordBatch.php +++ b/src/core/etl/src/Flow/ETL/Values/Rows/RecordBatch.php @@ -4,54 +4,178 @@ namespace Flow\ETL\Values\Rows; +use Flow\ETL\Foundation\Exception\ColumnMismatchException; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; +use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\SchemaMismatchException; use Flow\ETL\Values\Column\Backend; +use Flow\ETL\Values\Column\ColumnName; use Flow\ETL\Values\Rows; use Flow\ETL\Values\Schema; +use Generator; +use function array_diff_key; +use function array_key_exists; +use function array_keys; use function count; +use function is_int; +use function usort; final class RecordBatch { + /** + * @var list> + */ + private array $fields = []; + + /** + * @var list + */ + private array $lines = []; + private int $offset = 0; + /** + * @var list + */ + private array $offsets = []; + + /** + * @var list + */ + private array $raws = []; + /** * @var list> */ private array $records = []; + /** + * @var list + */ + private array $refused = []; + + /** + * @var null|array + */ + private readonly ?array $known; + /** * @param int<1, max> $batchSize + * @param null|string $uri the file the records come from; null for a source that is not a file + * @param null|int $sampleSize the inference's sample size when the schema was inferred, -1 for every row; null for a + * declared schema + * @param null|Schema $known a keyed reader's (JSON, array, sequence) full schema, $schema's columns and those it + * leaves out; under an inferred schema a key it lacks was first seen past the sample. + * A grid reader passes none: its header is checked once per file */ public function __construct( private readonly Schema $schema, private readonly Backend $backend, private readonly int $batchSize, - ) {} + private readonly ?string $uri = null, + private readonly ?int $sampleSize = null, + ?Schema $known = null, + ) { + $this->known = $known?->definitions(); + } /** * @param array $record + * @param null|string $raw the record as the source holds it; null when the source holds no text of it + * @param null|array $fields the cells a grid reader split the record into, before keying them by header * - * @throws SchemaMismatchException at the refused row's position among every record added so far + * @throws InferredSchemaException|SchemaMismatchException a refused record under an inferred schema * - * @return null|Rows a full batch once $batchSize records are held + * @return list a full batch's refused records, then its rows; [] while the batch + * is filling */ - public function add(array $record): ?Rows - { + public function add( + array $record, + int $offset, + ?int $line = null, + ?string $raw = null, + ?array $fields = null, + ): array { $this->records[] = $record; + $this->offsets[] = $offset; + $this->lines[] = $line; + $this->raws[] = $raw; + $this->fields[] = $fields; + + return (count($this->records) + count($this->refused)) < $this->batchSize ? [] : $this->flush(); + } + + /** + * A record its reader refused before it became values: it takes its place in the batch, so the batch hands its + * refused records out in source order. + * + * @return list as add() + */ + public function refuse(CorruptedRecordException $corrupted): array + { + $this->refused[] = $corrupted; + + return (count($this->records) + count($this->refused)) < $this->batchSize ? [] : $this->flush(); + } + + /** + * $records added in order, numbered from 1, then flushed. + * + * @param iterable> $records + * + * @throws InferredSchemaException|SchemaMismatchException a refused record under an inferred schema + * + * @return Generator + */ + public function of(iterable $records): Generator + { + $number = 0; - return count($this->records) < $this->batchSize ? null : $this->flush(); + foreach ($records as $record) { + yield from $this->add($record, ++$number); + } + + yield from $this->flush(); } /** - * @throws SchemaMismatchException at the refused row's position among every record added so far + * Under a declared schema a record with a value that does not cast is a corrupted record, handed up and left out + * of the rows; under an inferred schema it is a schema error. + * + * @throws InferredSchemaException|SchemaMismatchException a refused record under an inferred schema * - * @return null|Rows the held records, null when none + * @return list the records refused, in the order they were added, then the rows of + * the rest; [] when none */ - public function flush(): ?Rows + public function flush(): array { + $known = $this->known; + $sampleSize = $this->sampleSize; + + if ($sampleSize !== null && $known !== null) { + $columnName = new ColumnName(); + + foreach ($this->records as $index => $record) { + foreach (array_keys(array_diff_key($record, $known)) as $key) { + // an int key is a positional column (ColumnName), or a numeric-string name PHP turned into an int + if (is_int($key) && array_key_exists($columnName->of($key), $known)) { + continue; + } + + throw $this->outsideTheSample( + new SchemaMismatchException($index, ColumnMismatchException::unexpectedColumn((string) $key)), + $sampleSize, + ); + } + } + } + if ($this->records === []) { - return null; + $refused = $this->refused; + $this->clear(); + + return $refused; } try { @@ -59,12 +183,109 @@ public function flush(): ?Rows ->appendRows($this->records) ->finish(); } catch (SchemaMismatchException $mismatch) { - throw new SchemaMismatchException($this->offset + $mismatch->rowIndex, $mismatch->cause); + if ($sampleSize !== null) { + throw $this->outsideTheSample($mismatch, $sampleSize); + } + + return $this->recordByRecord(); } + $handed = [...$this->refused, $rows]; $this->offset += count($this->records); + $this->clear(); + + return $handed; + } + + /** + * A file's refusal names the record; a source that is not a file is named by its reader, which places the refusal + * by its position among every record added. + */ + private function outsideTheSample( + SchemaMismatchException $mismatch, + int $sampleSize, + ): InferredSchemaException|SchemaMismatchException { + $index = $mismatch->rowIndex; + + return $this->uri === null + ? new SchemaMismatchException($this->offset + $index, $mismatch->cause) + : InferredSchemaException::outsideTheSample( + $this->uri, + $this->offsets[$index], + $this->lines[$index], + $sampleSize, + $mismatch, + ); + } + + private function clear(): void + { $this->records = []; + $this->offsets = []; + $this->lines = []; + $this->raws = []; + $this->fields = []; + $this->refused = []; + } + + private function corrupted(int $index, ColumnMismatchException $cause): CorruptedRecordException + { + return $this->fields[$index] === null + ? CorruptedRecordException::fromValues( + $this->uri, + $this->offsets[$index], + $this->lines[$index], + $cause->reason(), + $this->records[$index], + $this->raws[$index], + ) + : CorruptedRecordException::fromCells( + $this->uri, + $this->offsets[$index], + $this->lines[$index], + $cause->reason(), + $this->fields[$index], + $this->raws[$index], + ); + } + + /** + * A batch with a refused record, in one pass: each record is tried alone, then the accepted ones are built at once. + * A refused record leaves the values of its other columns in the trial builder, so a refusal starts a new one. The + * records refused here and by refuse() leave in source order. + * + * @return list + */ + private function recordByRecord(): array + { + $handed = $this->refused; + $accepted = []; + $trial = new RowsBuilder($this->schema, $this->backend); + + foreach ($this->records as $index => $record) { + try { + $trial->appendRows([$record]); + $accepted[] = $record; + } catch (SchemaMismatchException $mismatch) { + $handed[] = $this->corrupted($index, $mismatch->cause); + $trial = new RowsBuilder($this->schema, $this->backend); + } + } + + usort( + $handed, + static fn(CorruptedRecordException $a, CorruptedRecordException $b): int => $a->offset <=> $b->offset, + ); + + if ($accepted !== []) { + $handed[] = (new RowsBuilder($this->schema, $this->backend)) + ->appendRows($accepted) + ->finish(); + } + + $this->offset += count($this->records); + $this->clear(); - return $rows; + return $handed; } } diff --git a/src/core/etl/src/Flow/ETL/Values/Schema.php b/src/core/etl/src/Flow/ETL/Values/Schema.php index c444c7c8d6..36fa076e5b 100644 --- a/src/core/etl/src/Flow/ETL/Values/Schema.php +++ b/src/core/etl/src/Flow/ETL/Values/Schema.php @@ -15,6 +15,7 @@ use Flow\ETL\Values\Schema\SortingStrategy; use Flow\ETL\Values\Schema\SortingStrategy\AlphabeticalStrategy; +use function array_diff; use function array_key_exists; use function array_keys; use function array_map; @@ -413,6 +414,18 @@ public function prepend(Definition ...$definitions): self return new self(...$definitions, ...array_values($this->definitions)); } + /** + * The names of this schema's columns $header does not hold. + * + * @param list $header + * + * @return list + */ + public function missingFrom(array $header): array + { + return array_values(array_diff($this->names(), $header)); + } + /** * @return list */ diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php b/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php index 753d2769ff..0fa10c5268 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php @@ -24,7 +24,7 @@ public static function of( $context ??= flow_context(); $extracted = null; - foreach (FlowTestCase::extracted($extractor, $context, $criteria) as $batch) { + foreach (FlowTestCase::extracted($extractor, $context->backend(), $criteria) as $batch) { $extracted = $extracted === null ? $batch : Rows::concat($extracted->schema(), $context->backend(), $extracted, $batch); @@ -42,8 +42,8 @@ public static function interleaved(Extractor $extractor, ?FlowContext $context = { $context ??= flow_context(); $reads = [ - $extractor->extract($context->backend(), new Criteria()), - $extractor->extract($context->backend(), new Criteria()), + FlowTestCase::extracted($extractor, $context->backend(), new Criteria()), + FlowTestCase::extracted($extractor, $context->backend(), new Criteria()), ]; /** @var array{0: null|Rows, 1: null|Rows} $extracted */ $extracted = [null, null]; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/FileFormatContract.php b/src/core/etl/tests/Flow/ETL/Tests/Context/FileFormatContract.php index aa13911922..0dce5e22e2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/FileFormatContract.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/FileFormatContract.php @@ -14,6 +14,7 @@ use Flow\ETL\Planning\Optimizer\Optimizer; use Flow\ETL\Planning\Optimizer\Rule\PushFilterIntoSource; use Flow\ETL\Tests\Double\ThrowWhenRowMatches; +use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Rows; use Flow\Filesystem\Path; use PHPUnit\Framework\Assert; @@ -196,7 +197,8 @@ public function theSplitsReadEveryRowOnce(FileFormat $format): void $read = 0; foreach ($extractor->splits(new Criteria(), 1) as $split) { - foreach ($extractor->reader()->extract( + foreach (FlowTestCase::extracted( + $extractor->reader(), flow_context()->backend(), (new Criteria())->withSplit(type_instance_of(FileSplit::class)->assert($split)), ) as $rows) { diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/IdRows.php b/src/core/etl/tests/Flow/ETL/Tests/Context/IdRows.php index f3e74b9d06..39ad0f1c05 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/IdRows.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/IdRows.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Context; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Values\Rows; use Generator; @@ -33,7 +34,7 @@ public static function batches(array ...$batches): Generator } /** - * @param Generator $batches + * @param Generator $batches * * @return list */ @@ -42,7 +43,7 @@ public static function ids(Generator $batches): array $ids = []; foreach ($batches as $batch) { - $ids = array_merge($ids, $batch->reduceToArray('id')); + $ids = array_merge($ids, ($batch instanceof Rows ? $batch : throw $batch)->reduceToArray('id')); } return $ids; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/ReaderRoundTrip.php b/src/core/etl/tests/Flow/ETL/Tests/Context/ReaderRoundTrip.php index 31a07b8b8c..e75a14de51 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/ReaderRoundTrip.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/ReaderRoundTrip.php @@ -8,6 +8,7 @@ use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\Reader; use Flow\ETL\Runtime\FlowContext; +use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Path; use RuntimeException; @@ -29,7 +30,7 @@ public static function batches(Reader $reader, FlowContext $context, Criteria $c { $sizes = []; - foreach ($reader->extract($context->backend(), $criteria) as $rows) { + foreach (FlowTestCase::extracted($reader, $context->backend(), $criteria) as $rows) { $sizes[] = $rows->count(); } @@ -45,7 +46,7 @@ public static function extractorBatches(Extractor $extractor, FlowContext $conte { $sizes = []; - foreach ($extractor->extract($context->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, $context->backend(), new Criteria()) as $rows) { $sizes[] = $rows->count(); } @@ -61,7 +62,7 @@ public static function rows(Reader $reader, FlowContext $context, Criteria $crit { $rows = []; - foreach ($reader->extract($context->backend(), $criteria) as $batch) { + foreach (FlowTestCase::extracted($reader, $context->backend(), $criteria) as $batch) { foreach ($batch->toArray() as $row) { $rows[] = $row; } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/SplitCoverage.php b/src/core/etl/tests/Flow/ETL/Tests/Context/SplitCoverage.php index 918eed8c5b..07f85f84f5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/SplitCoverage.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/SplitCoverage.php @@ -9,6 +9,7 @@ use Flow\ETL\IO\Extractor\File\FileSplit; use Flow\ETL\IO\Extractor\SplittableExtractor; use Flow\ETL\Runtime\FlowContext; +use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Rows; use function Flow\ETL\DSL\rows; @@ -33,7 +34,8 @@ public static function rowsFromSplits( $extracted = null; foreach ($extractor->splits($criteria, $targetSplitBytes) as $split) { - foreach ($extractor->extract( + foreach (FlowTestCase::extracted( + $extractor, $context->backend(), $criteria->withSplit(type_instance_of(FileSplit::class)->assert($split)), ) as $batch) { @@ -65,7 +67,8 @@ public static function rowCountPerSplit( foreach ($extractor->splits($criteria, $targetSplitBytes) as $split) { $count = 0; - foreach ($extractor->extract( + foreach (FlowTestCase::extracted( + $extractor, $context->backend(), $criteria->withSplit(type_instance_of(FileSplit::class)->assert($split)), ) as $batch) { diff --git a/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php b/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php index ad320c11b3..42510e094e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php +++ b/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php @@ -7,8 +7,10 @@ use Flow\ETL\IO\Extractor; use Flow\ETL\IO\Extractor\BatchableExtractor; use Flow\ETL\IO\Extractor\Criteria; +use Flow\ETL\IO\Extractor\Reader; use Flow\ETL\IO\Extractor\Signal; use Flow\ETL\Runtime\FlowContext; +use Flow\ETL\Values\Column\Backend; use Flow\ETL\Values\Column\PhpBackend; use Flow\ETL\Values\Rows; use Generator; @@ -41,7 +43,7 @@ final public static function assertExtractorHonoursBatchContract( $collected = rows(schema()); $extractor = $factory()->withBatchSize($size); - foreach ($extractor->extract(flow_context()->backend(), new Criteria()) as $batch) { + foreach (self::extracted($extractor, flow_context()->backend()) as $batch) { static::assertLessThanOrEqual($extractor->batchSize(), $batch->count(), $message); $collected = $collected->isEmpty() ? $batch @@ -55,7 +57,7 @@ final public static function assertExtractorHonoursBatchContract( if ($expected->count() > 1) { $largest = 0; - foreach ($factory()->withBatchSize(1000)->extract(flow_context()->backend(), new Criteria()) as $batch) { + foreach (self::extracted($factory()->withBatchSize(1000), flow_context()->backend()) as $batch) { $largest = max($largest, $batch->count()); } @@ -74,11 +76,11 @@ final public static function assertExtractorHonoursBatchContract( $first = rows(schema()); $second = rows(schema()); - foreach ($extractor->extract(flow_context()->backend(), new Criteria()) as $batch) { + foreach (self::extracted($extractor, flow_context()->backend()) as $batch) { $first = $first->isEmpty() ? $batch : Rows::concat($first->schema(), new PhpBackend(), $first, $batch); } - foreach ($extractor->extract(flow_context()->backend(), new Criteria()) as $batch) { + foreach (self::extracted($extractor, flow_context()->backend()) as $batch) { $second = $second->isEmpty() ? $batch : Rows::concat($second->schema(), new PhpBackend(), $second, $batch); @@ -101,7 +103,7 @@ final public static function assertExtractorHonoursMaximum( $extractor = $factory($maximum)->withBatchSize(1000); $total = 0; - foreach ($extractor->extract(flow_context()->backend(), new Criteria()) as $batch) { + foreach (self::extracted($extractor, flow_context()->backend()) as $batch) { static::assertLessThanOrEqual(1000, $batch->count(), $message); $total += $batch->count(); } @@ -122,7 +124,7 @@ final public static function assertExtractedRowsAsArrayEquals( $flowContext ??= flow_context(); $extractedRows = rows(schema()); - foreach (self::extracted($extractor, $flowContext, $criteria) as $nextRows) { + foreach (self::extracted($extractor, $flowContext->backend(), $criteria) as $nextRows) { $extractedRows = $extractedRows->isEmpty() ? $nextRows : Rows::concat($extractedRows->schema(), new PhpBackend(), $extractedRows, $nextRows); @@ -141,7 +143,7 @@ final public static function assertExtractedRowsCount( $flowContext ??= flow_context(); $totalRows = 0; - foreach (self::extracted($extractor, $flowContext, $criteria) as $rows) { + foreach (self::extracted($extractor, $flowContext->backend(), $criteria) as $rows) { $totalRows += $rows->count(); } @@ -158,7 +160,7 @@ final public static function assertExtractedRowsEquals( $flowContext ??= flow_context(); $extractedRows = rows(schema()); - foreach (self::extracted($extractor, $flowContext, $criteria) as $nextRows) { + foreach (self::extracted($extractor, $flowContext->backend(), $criteria) as $nextRows) { $extractedRows = $extractedRows->isEmpty() ? $nextRows : Rows::concat($extractedRows->schema(), new PhpBackend(), $extractedRows, $nextRows); @@ -168,11 +170,27 @@ final public static function assertExtractedRowsEquals( } /** - * @return Generator + * The batches of $extractor, a stop forwarded to it; a corrupted record is refused, as a read without a reject + * loader refuses it. + * + * @return Generator */ - final public static function extracted(Extractor $extractor, FlowContext $context, Criteria $criteria): Generator - { - return $extractor->extract($context->backend(), $criteria); + final public static function extracted( + Extractor|Reader $extractor, + Backend $backend, + Criteria $criteria = new Criteria(), + ): Generator { + $batches = $extractor->extract($backend, $criteria); + + foreach ($batches as $batch) { + $signal = yield $batch instanceof Rows ? $batch : throw $batch; + + if ($signal === Signal::STOP) { + $batches->send(Signal::STOP); + + return; + } + } } public function repositoryRoot(): string diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ChainedBatchSizesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ChainedBatchSizesTest.php index cc084eebc3..b1d765d7a4 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ChainedBatchSizesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ChainedBatchSizesTest.php @@ -35,14 +35,15 @@ public function test_each_chained_source_keeps_its_own_batch_size(): void $firstRows = 0; - foreach ($first->extract(flow_context()->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($first, flow_context()->backend(), new Criteria()) as $rows) { $firstRows += $rows->count(); } $read = 0; $largestFromSecond = 0; - foreach (from_all($first->withBatchSize(10), $second->withBatchSize(500))->extract( + foreach (FlowTestCase::extracted( + from_all($first->withBatchSize(10), $second->withBatchSize(500)), flow_context()->backend(), new Criteria(), ) as $rows) { diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SchemaTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SchemaTest.php index 20b273b5c7..f8c75039ac 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SchemaTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SchemaTest.php @@ -4,9 +4,9 @@ namespace Flow\ETL\Tests\Integration\DataFrame; -use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum; use Flow\ETL\Tests\FlowIntegrationTestCase; +use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Rows; use function array_map; @@ -205,7 +205,7 @@ public function test_an_array_extractor_derives_its_schema_before_it_yields(): v // Row 1's nulls are TYPED nulls in known columns rather than untyped null columns a later // row contradicts. - $batches = iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria())); + $batches = iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend())); static::assertSame( [ diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Extractor/CacheExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Extractor/CacheExtractorTest.php index e109f21450..91d000f56d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Extractor/CacheExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Extractor/CacheExtractorTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Integration\IO\Extractor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\SchemaNotDerivableException; use Flow\ETL\IO\Extractor\Criteria; @@ -13,6 +14,7 @@ use Flow\ETL\Tests\Double\CountingExtractor; use Flow\ETL\Tests\Double\RepeatableExtractor; use Flow\ETL\Tests\FlowIntegrationTestCase; +use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\RowsMother; use Flow\ETL\Values\Cardinality; use Flow\ETL\Values\Column\PhpBackend; @@ -96,7 +98,11 @@ public function test_extracting_rows_from_cache(): void static::assertCount( 3, - iterator_to_array(from_cache('key')->boundTo($cache)->extract(flow_context()->backend(), new Criteria())), + iterator_to_array(FlowTestCase::extracted( + from_cache('key')->boundTo($cache), + flow_context()->backend(), + new Criteria(), + )), ); static::assertTrue($cache->has('key')); } @@ -109,12 +115,11 @@ public function test_extracting_rows_from_cache_with_clearing_cache_afterwards() static::assertCount( 2, - iterator_to_array( - from_cache('key') - ->withClearOnFinish(true) - ->boundTo($cache) - ->extract(flow_context()->backend(), new Criteria()), - ), + iterator_to_array(FlowTestCase::extracted( + from_cache('key')->withClearOnFinish(true)->boundTo($cache), + flow_context()->backend(), + new Criteria(), + )), ); static::assertFalse($cache->has('key')); } @@ -135,9 +140,11 @@ public function test_extracting_rows_from_filesystem_cache(): void ], schema(int_schema('id')))); $cache->close('key'); - $rows = iterator_to_array( - from_cache('key')->boundTo($cache)->extract(flow_context()->backend(), new Criteria()), - ); + $rows = iterator_to_array(FlowTestCase::extracted( + from_cache('key')->boundTo($cache), + flow_context()->backend(), + new Criteria(), + )); static::assertCount(1, $rows); static::assertEquals( @@ -152,10 +159,11 @@ public function test_stop_signal_stops_extraction_and_skips_clearing(): void $cache->append('key', array_to_rows([['id' => 1]], schema(int_schema('id')))); $cache->append('key', array_to_rows([['id' => 2]], schema(int_schema('id')))); - $generator = from_cache('key') - ->withClearOnFinish(true) - ->boundTo($cache) - ->extract(flow_context()->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_cache('key')->withClearOnFinish(true)->boundTo($cache), + flow_context()->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); @@ -167,10 +175,13 @@ public function test_stop_signal_stops_extraction_and_skips_clearing(): void public function test_stop_signal_stops_fallback_extractor(): void { - $generator = from_cache('missing') - ->withFallbackExtractor(from_array([['id' => 1], ['id' => 2]])) - ->boundTo(new MemoryRowsStorage()) - ->extract(flow_context()->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_cache('missing') + ->withFallbackExtractor(from_array([['id' => 1], ['id' => 2]])) + ->boundTo(new MemoryRowsStorage()), + flow_context()->backend(), + new Criteria(), + ); static::assertTrue($generator->valid()); @@ -255,8 +266,8 @@ public function test_a_missing_entry_reads_the_fallback_again_on_every_read_and_ $cache = new MemoryRowsStorage(); $extractor = from_cache('key')->withFallbackExtractor($fallback)->boundTo($cache); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()), false); - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()), false); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false); static::assertSame(2, $fallback->extractCalls); static::assertFalse($cache->has('key')); @@ -267,4 +278,19 @@ public function test_is_not_repeatable_when_its_fallback_is_not(): void static::assertFalse(from_cache('key')->withFallbackExtractor(new RepeatableExtractor(false))->isRepeatable()); static::assertTrue(from_cache('key')->withFallbackExtractor(new RepeatableExtractor(true))->isRepeatable()); } + + public function test_a_corrupted_record_of_the_fallback_passes_through_unchanged(): void + { + $batches = iterator_to_array( + from_cache( + 'missing', + from_array([['id' => 'x']], schema(int_schema('id'))), + cache: new MemoryRowsStorage(), + )->extract(flow_context()->backend(), new Criteria()), + false, + ); + + static::assertCount(1, $batches); + static::assertInstanceOf(CorruptedRecordException::class, $batches[0]); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Floe/FloeExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Floe/FloeExtractorTest.php index a3e7c93928..8fb3b1e1ed 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Floe/FloeExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Floe/FloeExtractorTest.php @@ -15,6 +15,7 @@ use Flow\ETL\Tests\Context\FloeStreamReaderContext; use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\FlowIntegrationTestCase; +use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Cardinality; use Flow\ETL\Values\Rows; @@ -350,7 +351,10 @@ public function test_an_offset_spanning_two_files_and_a_limit_inside_the_third_r $extractor = from_floe('memory://window/*.floe', filesystem: $filesystem); $read = static fn(): array => array_merge(...array_map( static fn(Rows $rows): array => $rows->column('id')->values(), - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria(limit: 4, offset: 4)), false), + iterator_to_array( + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria(limit: 4, offset: 4)), + false, + ), )); static::assertSame([5, 6, 7, 8], $read()); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Floe/FloeSplitReadTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Floe/FloeSplitReadTest.php index 0a64c84dbe..5da3906962 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Floe/FloeSplitReadTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Floe/FloeSplitReadTest.php @@ -65,7 +65,8 @@ public function test_a_sized_split_is_read_without_listing(): void array_sum(array_map( static fn($rows): int => $rows->count(), iterator_to_array( - $reader->extract( + FlowTestCase::extracted( + $reader, flow_context()->backend(), (new Criteria())->withSplit(new FileSplit($file, 0, null, $size)), ), @@ -86,7 +87,8 @@ public function test_an_unsized_split_lists_its_path(): void $reader = from_floe(path('memory://a.floe'), filesystem: $counting)->reader(); $listedBySchema = $counting->listCalls; - iterator_to_array($reader->extract( + iterator_to_array(FlowTestCase::extracted( + $reader, flow_context()->backend(), (new Criteria())->withSplit(new FileSplit(path('memory://a.floe'), 0, null)), )); @@ -126,12 +128,11 @@ public function test_the_limit_travels_with_the_split(): void static::assertSame( 3, iterator_to_array( - from_floe(path('memory://a.floe'), filesystem: $memory) - ->reader() - ->extract( - flow_context()->backend(), - (new Criteria(3))->withSplit(new FileSplit(path('memory://a.floe'), 0, null)), - ), + FlowTestCase::extracted( + from_floe(path('memory://a.floe'), filesystem: $memory)->reader(), + flow_context()->backend(), + (new Criteria(3))->withSplit(new FileSplit(path('memory://a.floe'), 0, null)), + ), false, )[0]->count(), ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/ColumnMismatchExceptionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/ColumnMismatchExceptionTest.php index fed807f5f9..fa1b45378e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/ColumnMismatchExceptionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/ColumnMismatchExceptionTest.php @@ -117,4 +117,12 @@ public function test_value_does_not_match(): void $exception->getMessage(), ); } + + public function test_the_reason_is_the_column_and_its_detail(): void + { + static::assertSame( + 'column "id": could not convert \'x\' (string) to integer', + ColumnMismatchException::valueDoesNotMatch('id', 'integer', 'x')->reason(), + ); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/CorruptedRecordExceptionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/CorruptedRecordExceptionTest.php new file mode 100644 index 0000000000..e64675e150 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/CorruptedRecordExceptionTest.php @@ -0,0 +1,113 @@ +withCorruptedRecords($loader).', + (new CorruptedRecordException('file:///a.csv', 12, 3, '3 fields, the header has 2', '1,a,x'))->getMessage(), + ); + } + + public function test_a_record_without_a_line_or_a_uri_and_too_long_to_show_whole(): void + { + static::assertSame( + 'Corrupted record at offset 4 of the source: reason: "' + . str_repeat('ü', 120) + . '…". Route corrupted ' + . 'records to a loader with ->withCorruptedRecords($loader).', + (new CorruptedRecordException(null, 4, null, 'reason', str_repeat('ü', 121)))->getMessage(), + ); + } + + public function test_cells_are_kept_as_strings_and_shown_as_a_json_list_without_the_source_text(): void + { + $corrupted = CorruptedRecordException::fromCells('file:///a.xlsx', 5, null, 'reason', [ + 2 => 'a/ü', + 3 => null, + 4 => new DateTimeImmutable('2026-10-31 12:00:00+00:00'), + 5 => 1.5, + 6 => false, + 7 => ['x'], + ]); + + static::assertSame(['a/ü', '', '2026-10-31T12:00:00+00:00', '1.5', '', '["x"]'], $corrupted->fields); + static::assertSame('["a/ü","","2026-10-31T12:00:00+00:00","1.5","","[\\"x\\"]"]', $corrupted->record); + } + + public function test_cells_keep_the_source_text_when_the_source_holds_one(): void + { + $corrupted = CorruptedRecordException::fromCells('file:///a.csv', 0, 1, 'reason', ['1', 'a'], '1,a'); + + static::assertSame(['1,a', ['1', 'a']], [$corrupted->record, $corrupted->fields]); + } + + public function test_values_are_shown_as_a_json_object_without_the_source_text(): void + { + $corrupted = CorruptedRecordException::fromValues('file:///a.json', 4, null, 'reason', [ + 'id' => 'n/a', + 'at' => 'ü/x', + ]); + + static::assertSame(['{"id":"n/a","at":"ü/x"}', null], [$corrupted->record, $corrupted->fields]); + } + + public function test_values_keep_the_source_text_when_the_source_holds_one(): void + { + static::assertSame( + '{"id": "x"}', + CorruptedRecordException::fromValues( + 'file:///a.jsonl', + 0, + 1, + 'reason', + ['id' => 'x'], + '{"id": "x"}', + )->record, + ); + } + + public function test_from_record_is_the_inverse_of_to_record_and_keeps_the_previous(): void + { + $previous = new RuntimeException('from a task'); + $corrupted = CorruptedRecordException::fromCells('file:///a.csv', 12, 3, 'reason', ['1', 'a', 'x'], '1,a,x'); + + $rebuilt = CorruptedRecordException::fromRecord($corrupted->toRecord(), $previous); + + static::assertSame($corrupted->toRecord(), $rebuilt->toRecord()); + static::assertSame($corrupted->getMessage(), $rebuilt->getMessage()); + static::assertSame($previous, $rebuilt->getPrevious()); + } + + public function test_from_record_round_trips_a_keyed_record_without_a_uri_or_a_line(): void + { + $corrupted = CorruptedRecordException::fromValues(null, 4, null, 'reason', ['id' => 'x']); + + static::assertSame( + $corrupted->toRecord(), + CorruptedRecordException::fromRecord($corrupted->toRecord())->toRecord(), + ); + static::assertNull(CorruptedRecordException::fromRecord($corrupted->toRecord())->getPrevious()); + } + + public function test_from_record_refuses_a_record_of_another_shape(): void + { + $this->expectException(InvalidArgumentException::class); + + CorruptedRecordException::fromRecord(['uri' => null, 'offset' => '4']); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/InferredSchemaExceptionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/InferredSchemaExceptionTest.php index 21a41cc7e4..290da9cc8a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/InferredSchemaExceptionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/InferredSchemaExceptionTest.php @@ -102,4 +102,34 @@ public function test_a_refusal_the_sample_does_not_explain_stays_the_gate_s_own( $mismatch, )); } + + public function test_a_file_record_outside_the_sample_names_its_uri_offset_line_and_both_ways_out(): void + { + $mismatch = new SchemaMismatchException(0, ColumnMismatchException::unexpectedColumn('c')); + $exception = InferredSchemaException::outsideTheSample('file:///a.jsonl', 23, 2, 1000, $mismatch); + + static::assertSame( + 'Record at offset 23, line 2, of file:///a.jsonl does not fit the schema inferred from a sample of 1000 rows: ' + . 'column "c" is not declared by the schema. Infer from more rows and files with ' + . '->inferSchema(infer_schema()->sampleSize(...)->filesToSniff(...)), or declare the schema with ->withSchema(...).', + $exception->getMessage(), + ); + static::assertSame($mismatch, $exception->getPrevious()); + } + + public function test_a_record_outside_the_sample_without_a_line_and_an_unbounded_sample(): void + { + $exception = InferredSchemaException::outsideTheSample( + 'file:///a.xlsx', + 7, + null, + -1, + new SchemaMismatchException(0, ColumnMismatchException::missingColumn('id')), + ); + + static::assertStringStartsWith( + 'Record at offset 7 of file:///a.xlsx does not fit the schema inferred from a sample of all rows', + $exception->getMessage(), + ); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/SchemaValidationExceptionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/SchemaValidationExceptionTest.php index 41c4336132..a8430f4d09 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/SchemaValidationExceptionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/SchemaValidationExceptionTest.php @@ -73,4 +73,18 @@ public function test_the_message_prefixes_the_mismatches(): void (new SchemaValidationException(" Unexpected Definitions: \n"))->getMessage(), ); } + + public function test_declared_columns_missing_names_them_the_source_its_header_and_the_fix(): void + { + static::assertSame( + "Schema validation failed: \n" + . 'Columns "id", "at" declared by the schema are not in the header of file:///a.csv (name, price). Correct ' + . 'their names, or remove them from ->withSchema(...).', + SchemaValidationException::declaredColumnsMissing( + 'file:///a.csv', + ['id', 'at'], + ['name', 'price'], + )->getMessage(), + ); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ToDateTimeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ToDateTimeTest.php index 2a7170bd30..c1c6bc0eef 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ToDateTimeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ToDateTimeTest.php @@ -7,6 +7,7 @@ use DateTime; use DateTimeImmutable; use DateTimeZone; +use Flow\ETL\Foundation\Exception\EvaluationException; use Flow\ETL\Foundation\Exception\InvalidArgumentException; use Flow\ETL\Functions\ToDateTime; use Flow\ETL\Tests\Context\DescribeRoundTrip; @@ -112,6 +113,37 @@ public function test_string_to_date_time(): void ); } + public function test_a_null_value_is_null(): void + { + static::assertNull((new FunctionContext(flow_context()))->eval( + to_date_time(ref('string'), 'Y-m-d H:i:s'), + [ + 'string' => null, + ], + schema(str_schema('string', nullable: true)), + )); + } + + public function test_a_null_format_is_refused(): void + { + try { + (new FunctionContext(flow_context()))->eval( + to_date_time(ref('string'), ref('format')), + [ + 'string' => '2020-01-01', + 'format' => null, + ], + schema(str_schema('string'), str_schema('format', nullable: true)), + ); + static::fail('Expected a null format to be refused'); + } catch (EvaluationException $e) { + static::assertSame( + 'ToDateTime function requires a non-null format and time zone', + $e->getPrevious()?->getMessage(), + ); + } + } + public function test_unparseable_string_to_date_time_is_null(): void { static::assertNull((new FunctionContext(flow_context()))->eval( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/ArrayExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/ArrayExtractorTest.php index 6d43352dc8..8cb5e6b77b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/ArrayExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/ArrayExtractorTest.php @@ -6,12 +6,14 @@ use ArrayIterator; use ArrayObject; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\InvalidLogicException; use Flow\ETL\IO\Extractor\ArrayExtractor; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\Statistics; use Flow\ETL\Storage\Memory\ArrayMemory; +use Flow\ETL\Tests\Context\ExtractedRows; use Flow\ETL\Tests\Double\CountingMemory; use Flow\ETL\Tests\Double\FixedTmpDirFilesystem; use Flow\ETL\Tests\Double\FreshRowsAggregate; @@ -32,6 +34,7 @@ use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\execution_context; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\from_memory; @@ -47,6 +50,38 @@ final class ArrayExtractorTest extends FlowTestCase { + public function test_a_key_first_seen_after_the_inference_sample_is_a_schema_error(): void + { + $this->expectException(InferredSchemaException::class); + $this->expectExceptionMessage('"c"'); + + ExtractedRows::of(from_array([ + ['a' => 1, 'b' => 2], + ['a' => 3, 'b' => 4], + ['a' => 5, 'b' => 6, 'c' => 7], + ])->inferSchema(infer_schema()->sampleSize(2))); + } + + public function test_a_value_that_does_not_cast_to_the_declared_type_is_a_corrupted_record(): void + { + try { + ExtractedRows::of(from_array( + [['id' => 1, 'amount' => 10.5], ['id' => 2, 'amount' => 'N/A']], + schema(int_schema('id'), float_schema('amount', true)), + )); + static::fail('Expected the corrupted record to be refused'); + } catch (CorruptedRecordException $e) { + static::assertSame([null, 2, null, '{"id":2,"amount":"N/A"}', null], [ + $e->uri, + $e->offset, + $e->lineNumber, + $e->record, + $e->fields, + ]); + static::assertStringContainsString('"amount"', $e->reason); + } + } + public function test_an_empty_source_refuses_to_infer_a_schema(): void { $this->expectException(InferredSchemaException::class); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/ChainExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/ChainExtractorTest.php index 52378df063..c7180cfd9b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/ChainExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/ChainExtractorTest.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Tests\Unit\IO\Extractor; use ArrayIterator; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\IO\Extractor; use Flow\ETL\IO\Extractor\BatchableExtractor; @@ -15,6 +16,7 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Cardinality; use Flow\ETL\Values\Column\Backend; +use Flow\ETL\Values\Rows; use Flow\ETL\Values\Schema; use Generator; @@ -50,9 +52,11 @@ public function test_an_empty_nested_chain_reads_nothing_under_a_declared_schema static::assertSame( [], iterator_to_array( - from_all(from_all(), from_all(from_array([]))) - ->withSchema(schema(int_schema('id'))) - ->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted( + from_all(from_all(), from_all(from_array([])))->withSchema(schema(int_schema('id'))), + flow_context()->backend(), + new Criteria(), + ), false, ), ); @@ -63,9 +67,11 @@ public function test_a_declared_chain_schema_reads_an_empty_child_without_asking static::assertSame( [], iterator_to_array( - from_all(from_array([])) - ->withSchema(schema(int_schema('id'))) - ->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted( + from_all(from_array([]))->withSchema(schema(int_schema('id'))), + flow_context()->backend(), + new Criteria(), + ), false, ), ); @@ -142,9 +148,11 @@ public function test_with_schema_does_not_leak_into_a_second_pipeline(): void $child = from_rows(array_to_rows([['id' => 1]], schema(int_schema('id')))); iterator_to_array( - from_all($child) - ->withSchema(schema(int_schema('id'), str_schema('name', nullable: true))) - ->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted( + from_all($child)->withSchema(schema(int_schema('id'), str_schema('name', nullable: true))), + flow_context()->backend(), + new Criteria(), + ), false, ); @@ -152,7 +160,7 @@ public function test_with_schema_does_not_leak_into_a_second_pipeline(): void static::assertSame( [['id' => 1]], iterator_to_array( - from_all($child)->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted(from_all($child), flow_context()->backend(), new Criteria()), false, )[0]->toArray(), ); @@ -213,4 +221,19 @@ public function test_an_empty_chain_declares_zero_rows_and_bytes_exactly(): void { static::assertEquals(new Statistics(Cardinality::exact(0), Cardinality::exact(0)), from_all()->statistics()); } + + public function test_a_childs_corrupted_record_passes_through_unchanged(): void + { + $batches = iterator_to_array( + from_all(from_array([['id' => 'x'], ['id' => 2]], schema(int_schema('id')))) + ->withSchema(schema(int_schema('id'))) + ->extract(flow_context()->backend(), new Criteria()), + false, + ); + + static::assertInstanceOf(CorruptedRecordException::class, $batches[0]); + static::assertSame(1, $batches[0]->offset); + static::assertInstanceOf(Rows::class, $batches[1]); + static::assertSame([['id' => 2]], $batches[1]->toArray()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileReadLoopTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileReadLoopTest.php index 7aeea607e3..153215f2f7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileReadLoopTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileReadLoopTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit\IO\Extractor\File; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\File\FileReadLoop; use Flow\ETL\IO\Extractor\File\ReadWindow; @@ -33,25 +34,15 @@ public function test_every_batch_carries_its_files_constants(): void { static::assertSame( [[['id' => 1, 'year' => '2023'], ['id' => 2, 'year' => '2023']], [['id' => 3, 'year' => '2024']]], - array_map( - static fn(Rows $batch): array => $batch->toArray(), - iterator_to_array((new FileReadLoop(FileReadMother::of( - schema(int_schema('id'), str_schema('year')), - FileColumnsContext::discovering(), - 10, - )))->read( - [ - new SourceFile(path('memory://orders/year=2023/a.csv')), - new SourceFile(path('memory://orders/year=2024/b.csv')), - ], - new InMemoryFileBatches([ - 'memory://orders/year=2023/a.csv' => [1, 2], - 'memory://orders/year=2024/b.csv' => [3], - ]), - flow_context()->backend(), - new ReadWindow(), - )), - ), + array_map(static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array((new FileReadLoop(FileReadMother::of(schema(int_schema('id'), str_schema('year')), FileColumnsContext::discovering(), 10)))->read([ + new SourceFile(path('memory://orders/year=2023/a.csv')), + new SourceFile(path('memory://orders/year=2024/b.csv')), + ], new InMemoryFileBatches([ + 'memory://orders/year=2023/a.csv' => [1, 2], + 'memory://orders/year=2024/b.csv' => [3], + ]), flow_context()->backend(), new ReadWindow()))), ); } @@ -68,7 +59,10 @@ public function test_stop_after_the_first_batch_ends_the_read(): void new ReadWindow(), ); - static::assertSame([['id' => 1]], $read->current()->toArray()); + $rows = $read->current(); + + static::assertInstanceOf(Rows::class, $rows); + static::assertSame([['id' => 1]], $rows->toArray()); $read->send(Signal::STOP); @@ -81,19 +75,9 @@ public function test_a_limit_inside_a_file_ends_the_read_and_is_pushed_down(): v static::assertSame( [[['id' => 1], ['id' => 2]]], - array_map( - static fn(Rows $batch): array => $batch->toArray(), - iterator_to_array((new FileReadLoop(FileReadMother::of( - schema(int_schema('id')), - FileColumnsContext::discovering(names: []), - 10, - )))->read( - [new SourceFile(path('memory://a.csv')), new SourceFile(path('memory://b.csv'))], - $batches, - flow_context()->backend(), - new ReadWindow(limit: 2), - )), - ), + array_map(static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array((new FileReadLoop(FileReadMother::of(schema(int_schema('id')), FileColumnsContext::discovering(names: []), 10)))->read([new SourceFile(path('memory://a.csv')), new SourceFile(path('memory://b.csv'))], $batches, flow_context()->backend(), new ReadWindow(limit: 2)))), ); static::assertEquals(['memory://a.csv' => [new ReadWindow(0, 2)]], $batches->windows); } @@ -128,23 +112,13 @@ public function test_the_offset_a_file_leaves_and_the_rows_still_wanted_reach_th static::assertSame( [[['id' => 4]], [['id' => 5]]], - array_map( - static fn(Rows $batch): array => $batch->toArray(), - iterator_to_array((new FileReadLoop(FileReadMother::of( - schema(int_schema('id')), - FileColumnsContext::discovering(names: []), - 10, - )))->read( - [ - new SourceFile(path('memory://a.csv')), - new SourceFile(path('memory://b.csv')), - new SourceFile(path('memory://c.csv')), - ], - $batches, - flow_context()->backend(), - new ReadWindow(offset: 3, limit: 2), - )), - ), + array_map(static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array((new FileReadLoop(FileReadMother::of(schema(int_schema('id')), FileColumnsContext::discovering(names: []), 10)))->read([ + new SourceFile(path('memory://a.csv')), + new SourceFile(path('memory://b.csv')), + new SourceFile(path('memory://c.csv')), + ], $batches, flow_context()->backend(), new ReadWindow(offset: 3, limit: 2)))), ); static::assertEquals( [ @@ -179,7 +153,9 @@ public function test_the_loop_slices_the_offset_a_format_cannot_skip_across_file static::assertSame( [[['id' => 4]], [['id' => 5]]], array_map( - static fn(Rows $batch): array => $batch->toArray(), + static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array( (new FileReadLoop(FileReadMother::of( schema(int_schema('id')), @@ -216,7 +192,9 @@ public function test_the_loop_skips_the_offset_and_trims_to_the_limit_for_a_form static::assertSame( [[['id' => 4], ['id' => 5]]], array_map( - static fn(Rows $batch): array => $batch->toArray(), + static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array( (new FileReadLoop(FileReadMother::of( schema(int_schema('id')), @@ -240,7 +218,9 @@ public function test_a_filter_drops_the_rows_it_does_not_match_and_skips_a_batch static::assertSame( [[['id' => 3], ['id' => 4]], [['id' => 5]]], array_map( - static fn(Rows $batch): array => $batch->toArray(), + static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array( (new FileReadLoop( FileReadMother::of(schema(int_schema('id')), FileColumnsContext::discovering(names: []), 2), @@ -267,7 +247,9 @@ public function test_an_offset_counts_the_rows_the_filter_keeps_even_for_a_forma static::assertSame( [[['id' => 5], ['id' => 6], ['id' => 7]]], array_map( - static fn(Rows $batch): array => $batch->toArray(), + static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array( (new FileReadLoop( FileReadMother::of(schema(int_schema('id')), FileColumnsContext::discovering(names: []), 10), @@ -295,7 +277,9 @@ public function test_a_filter_mixing_a_body_and_a_partition_column_sees_the_part static::assertSame( [[['id' => 4, 'year' => '2024']]], array_map( - static fn(Rows $batch): array => $batch->toArray(), + static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array( (new FileReadLoop( FileReadMother::of( @@ -332,7 +316,9 @@ public function test_the_offset_and_the_limit_count_filtered_rows(): void static::assertSame( [[['id' => 7], ['id' => 8]]], array_map( - static fn(Rows $batch): array => $batch->toArray(), + static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array( (new FileReadLoop( FileReadMother::of(schema(int_schema('id')), FileColumnsContext::discovering(names: []), 10), @@ -357,7 +343,9 @@ public function test_the_loop_skips_the_offset_after_the_filter_for_a_format_tha static::assertSame( [[['id' => 7], ['id' => 8]]], array_map( - static fn(Rows $batch): array => $batch->toArray(), + static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array( (new FileReadLoop( FileReadMother::of(schema(int_schema('id')), FileColumnsContext::discovering(names: []), 10), @@ -386,7 +374,9 @@ public function test_a_limit_beside_a_filter_counts_matching_rows_across_files_a static::assertSame( [[['id' => 2]], [['id' => 3]]], array_map( - static fn(Rows $batch): array => $batch->toArray(), + static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array( (new FileReadLoop( FileReadMother::of(schema(int_schema('id')), FileColumnsContext::discovering(names: []), 10), @@ -416,7 +406,9 @@ public function test_each_conjunct_of_the_filter_sees_only_the_rows_the_ones_bef static::assertSame( [[['id' => 2]]], array_map( - static fn(Rows $batch): array => $batch->toArray(), + static fn(CorruptedRecordException|Rows $batch): array => ( + $batch instanceof Rows ? $batch->toArray() : [$batch->getMessage()] + ), iterator_to_array( (new FileReadLoop( FileReadMother::of(schema(int_schema('id')), FileColumnsContext::discovering(names: []), 10), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileReadTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileReadTest.php index f489af7608..7a5e9c0e05 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileReadTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileReadTest.php @@ -15,6 +15,7 @@ use Flow\ETL\Tests\Double\Described; use Flow\ETL\Tests\Double\InMemoryUnskippableFileBatches; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Values\Rows; use Flow\Filesystem\Local\NativeLocalFilesystem; use function Flow\ETL\DSL\flow_context; @@ -44,21 +45,20 @@ public function test_it_normalizes_and_rebuilds_equal(): void public function test_a_read_skips_the_offset_of_its_criteria(): void { - static::assertSame( - [['id' => 3]], - (new FileRead( - new SourceFiles(MemoryFiles::with(['memory://a.csv' => 'abcd']), path('memory://a.csv')), - FileColumnsContext::discovering([]), - schema(int_schema('id')), - 10, - )) - ->scan( - flow_context()->backend(), - new Criteria(offset: 2), - new InMemoryUnskippableFileBatches(['memory://a.csv' => [1, 2, 3]]), - ) - ->current() - ->toArray(), - ); + $rows = (new FileRead( + new SourceFiles(MemoryFiles::with(['memory://a.csv' => 'abcd']), path('memory://a.csv')), + FileColumnsContext::discovering([]), + schema(int_schema('id')), + 10, + )) + ->scan( + flow_context()->backend(), + new Criteria(offset: 2), + new InMemoryUnskippableFileBatches(['memory://a.csv' => [1, 2, 3]]), + ) + ->current(); + + static::assertInstanceOf(Rows::class, $rows); + static::assertSame([['id' => 3]], $rows->toArray()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileSplitReaderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileSplitReaderTest.php index 845df937a5..e448cdb568 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileSplitReaderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileSplitReaderTest.php @@ -39,7 +39,7 @@ public function test_a_read_goes_through_the_batches_and_closes_them_at_its_end( [[['id' => 1], ['id' => 2]], [['id' => 3]]], array_map( static fn(Rows $rows): array => $rows->toArray(), - iterator_to_array($reader->extract(flow_context()->backend(), new Criteria()), false), + iterator_to_array(FlowTestCase::extracted($reader, flow_context()->backend(), new Criteria()), false), ), ); static::assertSame(1, $batches->closes); @@ -48,15 +48,19 @@ public function test_a_read_goes_through_the_batches_and_closes_them_at_its_end( public function test_a_read_stopped_early_still_closes_the_batches(): void { $batches = new InMemoryFileBatches(['memory://a.csv' => [1, 2, 3]]); - $read = (new FileSplitReader( - FileReadMother::of( - schema(int_schema('id')), - batchSize: 2, - filesystem: MemoryFiles::with(['memory://a.csv' => 'abc']), - path: path('memory://a.csv'), + $read = FlowTestCase::extracted( + new FileSplitReader( + FileReadMother::of( + schema(int_schema('id')), + batchSize: 2, + filesystem: MemoryFiles::with(['memory://a.csv' => 'abc']), + path: path('memory://a.csv'), + ), + $batches, ), - $batches, - ))->extract(flow_context()->backend(), new Criteria()); + flow_context()->backend(), + new Criteria(), + ); $read->current(); $read->send(Signal::STOP); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/Grid/RecordDecoderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/Grid/RecordDecoderTest.php index ba45eec8cf..472a6f0342 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/Grid/RecordDecoderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/Grid/RecordDecoderTest.php @@ -16,14 +16,14 @@ public function test_header_cells_are_trimmed_and_a_blank_one_is_named_by_its_po { $records = new RecordDecoder(withHeader: true, emptyToNull: true); - static::assertSame([], $records->decode([[' id ', '', null, 'name ']])); + static::assertNull($records->decode([' id ', '', null, 'name '])); static::assertSame(['id', 'e01', 'e02', 'name'], $records->headers()); } public function test_a_numeric_header_cell_is_its_string(): void { $records = new RecordDecoder(withHeader: true, emptyToNull: true); - $records->decode([[1, 2.5, true]]); + $records->decode([1, 2.5, true]); static::assertSame(['1', '2.5', '1'], $records->headers()); } @@ -31,7 +31,7 @@ public function test_a_numeric_header_cell_is_its_string(): void public function test_a_non_scalar_header_cell_is_named_by_its_position(): void { $records = new RecordDecoder(withHeader: true, emptyToNull: true); - $records->decode([[new DateTimeImmutable('2026-01-01'), 'name']]); + $records->decode([new DateTimeImmutable('2026-01-01'), 'name']); static::assertSame(['e00', 'name'], $records->headers()); } @@ -40,21 +40,22 @@ public function test_without_a_header_the_first_record_sizes_the_generated_names { $records = new RecordDecoder(withHeader: false, emptyToNull: true); - static::assertSame([['e00' => 'a', 'e01' => 'b']], $records->decode([['a', 'b']])); + static::assertSame(['e00' => 'a', 'e01' => 'b'], $records->decode(['a', 'b'])); static::assertSame(['e00', 'e01'], $records->headers()); } - #[TestWith([true])] - #[TestWith([false])] - public function test_a_cell_a_short_record_lacks_is_null(bool $emptyToNull): void + public function test_preset_headers_decode_the_first_record_as_a_row(): void + { + $records = new RecordDecoder(withHeader: false, emptyToNull: true, headers: ['id', 'name']); + + static::assertSame(['id' => '1', 'name' => 'a'], $records->decode(['1', 'a'])); + } + + public function test_preset_headers_refuse_a_header_record(): void { - static::assertSame( - [['id' => '1', 'name' => 'a', 'age' => null]], - (new RecordDecoder(withHeader: true, emptyToNull: $emptyToNull))->decode([ - ['id', 'name', 'age'], - ['1', 'a'], - ]), - ); + $this->expectException(InvalidArgumentException::class); + + new RecordDecoder(withHeader: true, emptyToNull: true, headers: ['id']); } #[TestWith([true, null])] @@ -63,320 +64,93 @@ public function test_a_present_empty_cell_is_null_only_under_empty_to_null( bool $emptyToNull, ?string $expected, ): void { - static::assertSame( - [['id' => '1', 'name' => $expected]], - (new RecordDecoder(withHeader: true, emptyToNull: $emptyToNull))->decode([['id', 'name'], ['1', '']]), - ); + $records = new RecordDecoder(withHeader: true, emptyToNull: $emptyToNull); + $records->decode(['id', 'name']); + + static::assertSame(['id' => '1', 'name' => $expected], $records->decode(['1', ''])); } public function test_a_non_string_cell_is_kept_as_read(): void { $at = new DateTimeImmutable('2026-01-01'); - - static::assertSame( - [['at' => $at, 'n' => 0]], - (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([['at', 'n'], [$at, 0]]), - ); - } - - public function test_extra_cells_are_dropped_by_default(): void - { - static::assertSame( - [['id' => '1']], - (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([['id'], ['1', 'extra']]), - ); - } - - public function test_extra_cells_are_refused_when_they_are_not_dropped(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Row has more columns (2) than headers (1)'); - - (new RecordDecoder(withHeader: true, emptyToNull: true, dropExtraColumns: false))->decode([ - ['id'], - ['1', 'extra'], - ]); - } - - public function test_leading_empty_records_are_skipped_before_the_header(): void - { $records = new RecordDecoder(withHeader: true, emptyToNull: true); + $records->decode(['at', 'n']); - static::assertSame([['id' => '1']], $records->decode([[], [], ['id'], ['1']])); - static::assertSame(['id'], $records->headers()); - } - - public function test_an_empty_batch_decodes_to_nothing(): void - { - static::assertSame([], (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([])); - } - - public function test_headers_are_null_before_the_first_decode(): void - { - static::assertNull((new RecordDecoder(withHeader: true, emptyToNull: true))->headers()); + static::assertSame(['at' => $at, 'n' => 0], $records->decode([$at, 0])); } - public function test_the_header_is_kept_between_batches(): void + public function test_a_record_with_a_non_empty_extra_cell_is_corrupted(): void { $records = new RecordDecoder(withHeader: true, emptyToNull: true); - $records->decode([['id']]); + $records->decode(['id']); - static::assertSame([['id' => '2']], $records->decode([['2']])); + static::assertSame('2 fields, the header has 1', $records->decode(['1', 'extra'])); } - public function test_decode_captures_headers_from_the_first_row_and_emits_no_row_for_it(): void + #[TestWith([''])] + #[TestWith([null])] + public function test_an_empty_extra_cell_loses_nothing(?string $extra): void { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - - static::assertSame([], $decoder->decode([['id', 'name']])); - static::assertSame(['id' => 1, 'name' => 'Norbert'], $decoder->decode([[1, 'Norbert']])[0]); - } - - public function test_decode_generates_auto_headers_when_header_is_disabled(): void - { - static::assertSame( - ['e00' => 1, 'e01' => 'Norbert'], - (new RecordDecoder(withHeader: false, emptyToNull: true))->decode([[1, 'Norbert']])[0], - ); - } - - public function test_headers_are_generated_when_header_is_disabled(): void - { - $decoder = new RecordDecoder(withHeader: false, emptyToNull: true); - - static::assertCount(1, $decoder->decode([[1, 2, 3]])); - static::assertSame(['e00', 'e01', 'e02'], $decoder->headers()); - } - - public function test_excel_headers_are_null_before_the_first_decode(): void - { - static::assertNull((new RecordDecoder(withHeader: true, emptyToNull: true))->headers()); - } - - public function test_headers_are_stringified_and_named_by_position_for_blank_cells(): void - { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - $decoder->decode([[1, null, 'x']]); - - static::assertSame(['1', 'e01', 'x'], $decoder->headers()); - } - - public function test_headers_are_the_header_row_after_decode(): void - { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - - static::assertSame([], $decoder->decode([['id', 'name']])); - static::assertSame(['id', 'name'], $decoder->headers()); - } - - public function test_decode_keeps_empty_cells_when_convert_empty_to_null_is_disabled(): void - { - static::assertSame( - ['id' => 1, 'name' => ''], - (new RecordDecoder(withHeader: true, emptyToNull: false))->decode([['id', 'name'], [1, '']])[0], - ); - } - - public function test_decode_maps_cells_to_headers(): void - { - static::assertSame( - ['id' => 1, 'name' => 'Norbert'], - (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([['id', 'name'], [1, 'Norbert']])[0], - ); - } - - public function test_decode_turns_empty_cells_into_null_by_default(): void - { - static::assertSame( - ['id' => 1, 'name' => null], - (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([['id', 'name'], [1, '']])[0], - ); - } - - public function test_header_names_follow_the_csv_rule(): void - { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - $decoder->decode([[' id ', '', 'name']]); - - static::assertSame(['id', 'e01', 'name'], $decoder->headers()); - } - - public function test_decodes_rows_with_first_row_as_headers(): void - { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - - $decoded = $decoder->decode([ - ['id', 'name'], - ['1', 'Norbert'], - ['2', 'Tomek'], - ]); - - static::assertSame( - [ - ['id' => '1', 'name' => 'Norbert'], - ['id' => '2', 'name' => 'Tomek'], - ], - $decoded, - ); - } - - public function test_skips_empty_rows_before_headers(): void - { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - - $decoded = $decoder->decode([ - [], - ['id', 'name'], - ['1', 'Norbert'], - ]); - - static::assertSame([['id' => '1', 'name' => 'Norbert']], $decoded); - } - - public function test_keeps_headers_between_decode_calls(): void - { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - - $decoder->decode([['id', 'name'], ['1', 'Norbert']]); - - static::assertSame([['id' => '2', 'name' => 'Tomek']], $decoder->decode([['2', 'Tomek']])); - } - - public function test_pads_shorter_rows_with_nulls(): void - { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - - $decoded = $decoder->decode([ - ['id', 'name'], - ['1'], - ]); - - static::assertSame([['id' => '1', 'name' => null]], $decoded); - } - - public function test_drops_extra_columns(): void - { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - - $decoded = $decoder->decode([ - ['id', 'name'], - ['1', 'Norbert', 'extra'], - ]); - - static::assertSame([['id' => '1', 'name' => 'Norbert']], $decoded); - } - - public function test_throws_on_extra_columns_when_drop_is_disabled(): void - { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true, dropExtraColumns: false); - - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Row has more columns (3) than headers (2)'); - - $decoder->decode([ - ['id', 'name'], - ['1', 'Norbert', 'extra'], - ]); - } - - public function test_generates_auto_headers_without_header_row(): void - { - $decoder = new RecordDecoder(withHeader: false, emptyToNull: true); - - $decoded = $decoder->decode([ - ['1', 'Norbert'], - ['2', 'Tomek'], - ]); - - static::assertSame( - [ - ['e00' => '1', 'e01' => 'Norbert'], - ['e00' => '2', 'e01' => 'Tomek'], - ], - $decoded, - ); - } - - public function test_empty_cells_become_null_by_default(): void - { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + $records = new RecordDecoder(withHeader: true, emptyToNull: true); + $records->decode(['id']); - static::assertSame([['a' => null, 'b' => 'x']], $decoder->decode([['a', 'b'], ['', 'x']])); + static::assertSame(['id' => '1'], $records->decode(['1', $extra])); } - public function test_empty_cells_stay_strings_when_asked(): void + public function test_a_record_with_a_missing_cell_is_corrupted(): void { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: false); - - static::assertSame([['a' => '', 'b' => 'x']], $decoder->decode([['a', 'b'], ['', 'x']])); - } + $records = new RecordDecoder(withHeader: true, emptyToNull: true); + $records->decode(['id', 'name', 'age']); - public function test_omitted_trailing_cells_are_null_regardless_of_empty_to_null(): void - { - static::assertSame( - [['a' => 'x', 'b' => null]], - (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([ - ['a', 'b'], - ['x'], - ]), - ); - static::assertSame( - [['a' => 'x', 'b' => null]], - (new RecordDecoder(withHeader: true, emptyToNull: false))->decode([['a', 'b'], ['x']]), - ); + static::assertSame('2 fields, the header has 3', $records->decode(['1', 'a'])); } - public function test_headers_are_the_consumed_header_row(): void + #[TestWith([true])] + #[TestWith([false])] + public function test_a_missing_cell_is_null_under_missing_columns_as_null(bool $emptyToNull): void { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - $decoder->decode([['a', 'b'], ['1', '2']]); + $records = new RecordDecoder(withHeader: true, emptyToNull: $emptyToNull, missingColumnsAsNull: true); + $records->decode(['id', 'name', 'age']); - static::assertSame(['a', 'b'], $decoder->headers()); + static::assertSame(['id' => '1', 'name' => 'a', 'age' => null], $records->decode(['1', 'a'])); } - public function test_headers_skip_a_leading_empty_row(): void + public function test_missing_columns_as_null_still_refuses_a_non_empty_extra_cell(): void { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - $decoder->decode([[], ['a', 'b']]); + $records = new RecordDecoder(withHeader: true, emptyToNull: true, missingColumnsAsNull: true); + $records->decode(['id']); - static::assertSame(['a', 'b'], $decoder->headers()); + static::assertSame('2 fields, the header has 1', $records->decode(['1', 'extra'])); } - public function test_headers_are_generated_without_a_header_row(): void + #[TestWith([[]])] + #[TestWith([[null]])] + public function test_an_empty_record_is_no_row_before_and_after_the_header(array $empty): void { - $decoder = new RecordDecoder(withHeader: false, emptyToNull: true); - $decoder->decode([['1', '2', '3']]); + $records = new RecordDecoder(withHeader: true, emptyToNull: true); - static::assertSame(['e00', 'e01', 'e02'], $decoder->headers()); + static::assertNull($records->decode($empty)); + static::assertNull($records->decode(['id'])); + static::assertNull($records->decode($empty)); + static::assertSame(['id' => '1'], $records->decode(['1'])); } - public function test_headers_are_strings_even_when_the_api_typed_them(): void + public function test_headers_are_null_before_the_first_decode(): void { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - $decoder->decode([[1, '2'], ['x', 'y']]); - - static::assertSame(['1', '2'], $decoder->headers()); + static::assertNull((new RecordDecoder(withHeader: true, emptyToNull: true))->headers()); } /** - * b88, pinned as it behaves rather than as it should: `1` and `true` both stringify to `'1'`, and - * array_combine() keeps the last value under a repeated key, so the row loses a column. Not specific to a - * typed header - `a,a` does the same in CSV - and not introduced by inference. + * b88, pinned as it behaves rather than as it should: `1` and `true` both stringify to `'1'`, and the later + * value wins under a repeated key, so the row loses a column. Not specific to a typed header - `a,a` does the same + * in CSV - and not introduced by inference. */ public function test_two_header_cells_with_the_same_name_collapse_the_row_to_one_column(): void { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - $decoded = $decoder->decode([[1, true], ['x', 'y']]); - - static::assertSame(['1', '1'], $decoder->headers()); - static::assertSame(['1' => 'y'], $decoded[0]); - } - - public function test_google_sheet_header_names_follow_the_csv_rule(): void - { - $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); - $decoder->decode([[' id ', '', 'name']]); + $records = new RecordDecoder(withHeader: true, emptyToNull: true); + $records->decode([1, true]); - static::assertSame(['id', 'e01', 'name'], $decoder->headers()); + static::assertSame(['1', '1'], $records->headers()); + static::assertSame(['1' => 'y'], $records->decode(['x', 'y'])); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/SequenceExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/SequenceExtractorTest.php index 7a68925ea8..f66ea2592d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/SequenceExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/SequenceExtractorTest.php @@ -16,9 +16,11 @@ use Flow\ETL\Tests\Mother\RowsMother; use Flow\ETL\Values\Cardinality; +use function array_map; use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\date_schema; +use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_sequence_date_period; @@ -28,6 +30,8 @@ use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\to_array; +use function Flow\ETL\DSL\uuid_schema; use function iterator_to_array; final class SequenceExtractorTest extends FlowTestCase @@ -67,12 +71,16 @@ public function test_an_empty_source_extracts_nothing_without_a_declared_schema( static::assertSame( [], iterator_to_array( - from_sequence_date_period( - 'id', - new DateTimeImmutable('2024-01-01'), - new DateInterval('P1D'), - new DateTimeImmutable('2024-01-01'), - )->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted( + from_sequence_date_period( + 'id', + new DateTimeImmutable('2024-01-01'), + new DateInterval('P1D'), + new DateTimeImmutable('2024-01-01'), + ), + flow_context()->backend(), + new Criteria(), + ), false, ), ); @@ -177,7 +185,7 @@ public function test_a_bounded_sample_types_only_its_prefix(): void $this->expectException(InferredSchemaException::class); $this->expectExceptionMessage('Row 1 of the sequence does not fit the schema inferred from its first 1 rows'); - iterator_to_array($extractor->extract(flow_context(config())->backend(), new Criteria())); + iterator_to_array(FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria())); } public function test_a_numeric_entry_name_yields_one_column(): void @@ -252,7 +260,7 @@ public function test_a_heterogeneous_sequence_is_extracted_with_one_shape(): voi { $extractor = new SequenceExtractor(new MixedSequenceGenerator(), 'code'); - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { static::assertEquals($extractor->schema(), $rows->schema()); } @@ -271,4 +279,27 @@ public function test_sequence_extractor_honours_the_batch_contract(): void RowsMother::sequentialIds(5), ); } + + public function test_corrupted_records_are_routed_to_the_reject_loader(): void + { + $rejects = []; + + $rows = df() + ->read( + from_sequence_number('id', 1, 3) + ->withSchema(schema(uuid_schema('id'))) + ->withCorruptedRecords(to_array($rejects)), + ) + ->fetch(); + + static::assertSame(0, $rows->count()); + static::assertSame( + [[null, 1, '{"id":1}'], [null, 2, '{"id":2}'], [null, 3, '{"id":3}']], + array_map(static fn(array $reject): array => [ + $reject['uri'], + $reject['offset'], + $reject['record'], + ], $rejects), + ); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeExtractorTest.php index 9fc3f93968..cf4c8aa2af 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeExtractorTest.php @@ -62,7 +62,8 @@ public function test_a_later_file_with_other_columns_throws_naming_both_files(): . ' |-- extra', ); - iterator_to_array(from_floe(path('memory://glob/*.floe'), filesystem: $memory)->extract( + iterator_to_array(FlowTestCase::extracted( + from_floe(path('memory://glob/*.floe'), filesystem: $memory), flow_context(config())->backend(), new Criteria(), )); @@ -79,7 +80,8 @@ public function test_a_later_file_with_another_column_type_throws(): void $this->expectException(InferredSchemaException::class); $this->expectExceptionMessage('|-- expected: id, given: id'); - iterator_to_array(from_floe(path('memory://glob/*.floe'), filesystem: $memory)->extract( + iterator_to_array(FlowTestCase::extracted( + from_floe(path('memory://glob/*.floe'), filesystem: $memory), flow_context(config())->backend(), new Criteria(), )); @@ -147,11 +149,11 @@ public function test_extract_adds_input_file_uri_when_configured(): void $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), $context->backend()); $loader->closure(); - $batches = iterator_to_array( - from_floe($path, filesystem: $memory) - ->withMetadataColumns(true) - ->extract($context->backend(), new Criteria()), - ); + $batches = iterator_to_array(FlowTestCase::extracted( + from_floe($path, filesystem: $memory)->withMetadataColumns(true), + $context->backend(), + new Criteria(), + )); static::assertSame($path->uri(), $batches[0]->column('_input_file_uri')->value(0)); } @@ -172,7 +174,8 @@ public function test_extract_applies_offset_within_a_file(): void $ids = []; - foreach (from_floe($path, filesystem: $memory)->extract( + foreach (FlowTestCase::extracted( + from_floe($path, filesystem: $memory), $context->backend(), new Criteria(offset: 1), ) as $batch) { @@ -202,7 +205,7 @@ public function test_extract_honors_limit(): void $ids = []; - foreach ($extractor->extract($context->backend(), new Criteria(limit: 2)) as $batch) { + foreach (FlowTestCase::extracted($extractor, $context->backend(), new Criteria(limit: 2)) as $batch) { foreach ($batch->toArray() as $extractedRow) { $ids[] = $extractedRow['id']; } @@ -223,7 +226,8 @@ public function test_extract_omits_input_file_uri_by_default(): void static::assertSame( ['id'], - array_keys(iterator_to_array(from_floe($path, filesystem: $memory)->extract( + array_keys(iterator_to_array(FlowTestCase::extracted( + from_floe($path, filesystem: $memory), $context->backend(), new Criteria(), ))[0]->values(0)), @@ -242,7 +246,11 @@ public function test_extract_reads_rows(): void $ids = []; - foreach (from_floe($path, filesystem: $memory)->extract($context->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted( + from_floe($path, filesystem: $memory), + $context->backend(), + new Criteria(), + ) as $batch) { foreach ($batch->toArray() as $extractedRow) { $ids[] = $extractedRow['id']; } @@ -267,7 +275,8 @@ public function test_extract_skips_whole_files_with_offset(): void $ids = []; - foreach (from_floe(path('memory://skip-files/*.floe'), filesystem: $memory)->extract( + foreach (FlowTestCase::extracted( + from_floe(path('memory://skip-files/*.floe'), filesystem: $memory), $context->backend(), new Criteria(offset: 2), ) as $batch) { @@ -289,7 +298,11 @@ public function test_extract_stops_on_stop_signal(): void $loader->load(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), $context->backend()); $loader->closure(); - $generator = from_floe($path, filesystem: $memory)->extract($context->backend(), new Criteria()); + $generator = FlowTestCase::extracted( + from_floe($path, filesystem: $memory), + $context->backend(), + new Criteria(), + ); $batches = 0; foreach ($generator as $_batch) { @@ -312,7 +325,7 @@ public function test_extract_yields_the_metadata_column_schema_promises(): void $extractor = from_floe($path, filesystem: $memory)->withMetadataColumns(true); - $batches = iterator_to_array($extractor->extract($context->backend(), new Criteria())); + $batches = iterator_to_array(FlowTestCase::extracted($extractor, $context->backend(), new Criteria())); static::assertSame($extractor->schema()->names(), array_keys($batches[0]->values(0))); static::assertSame($path->uri(), $batches[0]->column('_input_file_uri')->value(0)); @@ -340,7 +353,8 @@ public function test_extract_builds_through_the_config_backend(): void $backend = new SpyBackend(); - iterator_to_array(from_floe($path, filesystem: $memory)->extract( + iterator_to_array(FlowTestCase::extracted( + from_floe($path, filesystem: $memory), flow_context(config_builder()->backend($backend)->build())->backend(), new Criteria(), )); @@ -380,7 +394,7 @@ public function test_extract_fills_partition_columns_from_the_path(): void $extractor = from_floe(path('memory://parts/*/*.floe'), filesystem: $memory); $values = []; - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { foreach ($rows->toArray() as $row) { $values[] = $row; } @@ -439,7 +453,7 @@ public function test_declared_partition_types_reach_the_rows(): void partition_types(year: type_integer()), ); - foreach ($extractor->extract(flow_context(config())->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted($extractor, flow_context(config())->backend(), new Criteria()) as $rows) { static::assertEquals($extractor->schema(), $rows->schema()); static::assertSame(2024, $rows->column('year')->value(0)); @@ -454,7 +468,8 @@ public function test_extract_closes_the_reader_when_the_generator_is_abandoned() $counting = new CountingFilesystem($memory = memory_filesystem()); FloeFilesContext::writePartitionedFiles($memory); - $generator = from_floe(path('memory://parts/*/*.floe'), filesystem: $counting)->extract( + $generator = FlowTestCase::extracted( + from_floe(path('memory://parts/*/*.floe'), filesystem: $counting), flow_context(config())->backend(), new Criteria(), ); @@ -469,7 +484,8 @@ public function test_extract_closes_every_reader_it_opens(): void $counting = new CountingFilesystem($memory = memory_filesystem()); FloeFilesContext::writePartitionedFiles($memory); - foreach (from_floe(path('memory://parts/*/*.floe'), filesystem: $counting)->extract( + foreach (FlowTestCase::extracted( + from_floe(path('memory://parts/*/*.floe'), filesystem: $counting), flow_context(config())->backend(), new Criteria(), ) as $_rows) { @@ -483,7 +499,8 @@ public function test_extract_closes_the_reader_it_skips_for_the_offset(): void $counting = new CountingFilesystem($memory = memory_filesystem()); FloeFilesContext::writePartitionedFiles($memory); - foreach (from_floe(path('memory://parts/*/*.floe'), filesystem: $counting)->extract( + foreach (FlowTestCase::extracted( + from_floe(path('memory://parts/*/*.floe'), filesystem: $counting), flow_context(config())->backend(), new Criteria(offset: 1), ) as $_rows) { @@ -497,7 +514,8 @@ public function test_extract_closes_the_reader_when_the_pipeline_stops(): void $counting = new CountingFilesystem($memory = memory_filesystem()); FloeFilesContext::writePartitionedFiles($memory); - $generator = from_floe(path('memory://parts/*/*.floe'), filesystem: $counting)->extract( + $generator = FlowTestCase::extracted( + from_floe(path('memory://parts/*/*.floe'), filesystem: $counting), flow_context(config())->backend(), new Criteria(), ); @@ -518,7 +536,8 @@ public function test_a_path_filter_narrows_the_read_but_not_the_schema(): void static::assertSame( [], - iterator_to_array($extractor->extract( + iterator_to_array(FlowTestCase::extracted( + $extractor, flow_context(config())->backend(), new Criteria(pathFilter: new RejectingFilter()), )), @@ -568,9 +587,11 @@ public function test_reader_batch_follows_the_batch_size(): void $sizes = []; - foreach (from_floe($path, filesystem: $memory) - ->withBatchSize(7) - ->extract($context->backend(), new Criteria()) as $rows) { + foreach (FlowTestCase::extracted( + from_floe($path, filesystem: $memory)->withBatchSize(7), + $context->backend(), + new Criteria(), + ) as $rows) { $sizes[] = $rows->count(); } @@ -615,7 +636,8 @@ public function test_extract_split_reads_no_more_than_the_limit(): void $extractor->withBatchSize(10); $total = 0; - foreach ($extractor->extract( + foreach (FlowTestCase::extracted( + $extractor, flow_context()->backend(), (new Criteria(25))->withSplit($extractor->splits(new Criteria(), 100_000)[0]), ) as $rows) { @@ -631,7 +653,8 @@ public function test_extract_split_stops_on_the_stop_signal(): void FloeFilesContext::writeSections($memory, 'memory://split/a.floe', 3, 100); $extractor = from_floe(path('memory://split/a.floe'), filesystem: $memory); $extractor->withBatchSize(1); - $generator = $extractor->extract( + $generator = FlowTestCase::extracted( + $extractor, flow_context()->backend(), (new Criteria())->withSplit($extractor->splits(new Criteria(), 100_000)[0]), ); @@ -649,12 +672,11 @@ public function test_extract_split_stops_on_the_stop_signal_of_a_whole_source_sp { $memory = memory_filesystem(); FloeFilesContext::writeSections($memory, 'memory://split/a.floe', 3, 100); - $generator = from_floe(path('memory://split/a.floe'), filesystem: $memory) - ->withBatchSize(1) - ->extract( - flow_context()->backend(), - (new Criteria())->withSplit(new FileSplit(path('memory://split/a.floe'), 0, null)), - ); + $generator = FlowTestCase::extracted( + from_floe(path('memory://split/a.floe'), filesystem: $memory)->withBatchSize(1), + flow_context()->backend(), + (new Criteria())->withSplit(new FileSplit(path('memory://split/a.floe'), 0, null)), + ); $totalRows = 0; foreach ($generator as $rows) { diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeLoaderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeLoaderTest.php index 4a6d6aa73b..ab1a365e64 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeLoaderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeLoaderTest.php @@ -17,6 +17,7 @@ use Flow\ETL\Tests\Context\FloeFilesContext; use Flow\ETL\Tests\Context\ReaderRoundTrip; use Flow\ETL\Tests\Context\StepRoundTrip; +use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Column\Frame\Codec; use Flow\ETL\Values\Schema\Metadata; use PHPUnit\Framework\TestCase; @@ -52,7 +53,11 @@ public function test_closure_writes_a_readable_non_torn_file(): void $ids = []; - foreach (from_floe($path, filesystem: $memory)->extract($context->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted( + from_floe($path, filesystem: $memory), + $context->backend(), + new Criteria(), + ) as $batch) { foreach ($batch->toArray() as $extractedRow) { $ids[] = $extractedRow['id']; } @@ -153,7 +158,8 @@ public function test_partitioned_batches_write_one_file_per_partition(): void $ids = []; - foreach (from_floe(path('memory://parts/**/*.floe'), filesystem: $memory)->extract( + foreach (FlowTestCase::extracted( + from_floe(path('memory://parts/**/*.floe'), filesystem: $memory), $context->backend(), new Criteria(), ) as $batch) { @@ -181,7 +187,11 @@ public function test_repeated_loads_write_a_single_file(): void $ids = []; // reading the concrete (non-pattern) path proves all rows landed in one file - foreach (from_floe($path, filesystem: $memory)->extract($context->backend(), new Criteria()) as $batch) { + foreach (FlowTestCase::extracted( + from_floe($path, filesystem: $memory), + $context->backend(), + new Criteria(), + ) as $batch) { foreach ($batch->toArray() as $extractedRow) { $ids[] = $extractedRow['id']; } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Join/HashJoinTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Join/HashJoinTest.php index 4a9f276592..be80e57b63 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Join/HashJoinTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Join/HashJoinTest.php @@ -757,7 +757,7 @@ public function test_resident_join_forwards_stop_to_its_left_upstream(): void { $upstream = (new CountingExtractor(schema(int_schema('id')), RowsMother::sequentialIds(5)))->withBatchSize(1); $joined = HashJoinMother::resident(Expression::on(['id' => 'id']), JoinType::inner, batchSize: 1)->join( - $upstream->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($upstream, flow_context()->backend(), new Criteria()), (static function () { yield RowsMother::sequentialIds(5); })(), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/BatchingByProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/BatchingByProcessorTest.php index 19802abc6d..c0e0a08531 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/BatchingByProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/BatchingByProcessorTest.php @@ -37,10 +37,14 @@ public function test_a_later_batch_that_widens_the_shape_is_refused(): void iterator_to_array( (new BatchingByProcessor(ref('g')))->process( - (new VaryingBatchesExtractor( - array_to_rows([['g' => 1]], schema(int_schema('g'))), - array_to_rows([['g' => 1, 'extra' => 'x']], schema(int_schema('g'), str_schema('extra'))), - ))->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted( + new VaryingBatchesExtractor( + array_to_rows([['g' => 1]], schema(int_schema('g'))), + array_to_rows([['g' => 1, 'extra' => 'x']], schema(int_schema('g'), str_schema('extra'))), + ), + flow_context()->backend(), + new Criteria(), + ), flow_context()->backend(), ), false, @@ -51,7 +55,7 @@ public function test_batching_by_processor_forwards_stop_to_its_upstream(): void { $upstream = (new CountingExtractor(schema(int_schema('id')), RowsMother::sequentialIds(5)))->withBatchSize(1); $processed = (new BatchingByProcessor(ref('id')))->process( - $upstream->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($upstream, flow_context()->backend(), new Criteria()), flow_context()->backend(), ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/BatchingProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/BatchingProcessorTest.php index 287caad215..09d2a81f03 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/BatchingProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/BatchingProcessorTest.php @@ -54,10 +54,14 @@ public function test_a_later_batch_that_widens_the_shape_is_refused(): void iterator_to_array( (new BatchingProcessor(new BatchSize(2)))->process( - (new VaryingBatchesExtractor( - array_to_rows([['id' => 1]], schema(int_schema('id'))), - array_to_rows([['id' => 1, 'extra' => 'x']], schema(int_schema('id'), str_schema('extra'))), - ))->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted( + new VaryingBatchesExtractor( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 1, 'extra' => 'x']], schema(int_schema('id'), str_schema('extra'))), + ), + flow_context()->backend(), + new Criteria(), + ), flow_context()->backend(), ), false, @@ -68,7 +72,7 @@ public function test_batching_processor_forwards_stop_to_its_upstream(): void { $upstream = (new CountingExtractor(schema(int_schema('id')), RowsMother::sequentialIds(5)))->withBatchSize(1); $processed = (new BatchingProcessor(new BatchSize(1)))->process( - $upstream->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($upstream, flow_context()->backend(), new Criteria()), flow_context()->backend(), ); @@ -93,7 +97,11 @@ public function test_windows_built_from_many_child_batches_keep_counts_and_order $windows = iterator_to_array( (new BatchingProcessor(new BatchSize(1_000)))->process( - (new VaryingBatchesExtractor(...$children))->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted( + new VaryingBatchesExtractor(...$children), + flow_context()->backend(), + new Criteria(), + ), flow_context()->backend(), ), false, diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/CachingProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/CachingProcessorTest.php index ba39329b82..3666ace6c8 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/CachingProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/CachingProcessorTest.php @@ -29,7 +29,7 @@ public function test_a_stop_still_completes_the_cache(): void $cache = new MemoryRowsStorage(); $upstream = (new CountingExtractor(schema(int_schema('id')), RowsMother::sequentialIds(5)))->withBatchSize(1); $processed = (new CachingProcessor('stop-cache', $cache))->process( - $upstream->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($upstream, flow_context()->backend(), new Criteria()), flow_context()->backend(), ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/ConstrainedProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/ConstrainedProcessorTest.php index 5fab25e9d9..39d41faea3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/ConstrainedProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/ConstrainedProcessorTest.php @@ -30,7 +30,7 @@ public function test_constrained_processor_forwards_stop_to_its_upstream(): void { $upstream = (new CountingExtractor(schema(int_schema('id')), RowsMother::sequentialIds(5)))->withBatchSize(1); $processed = (new ConstrainedProcessor([]))->process( - $upstream->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($upstream, flow_context()->backend(), new Criteria()), flow_context()->backend(), ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/ExpandingProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/ExpandingProcessorTest.php index 61fcb1d3eb..73a84cc67e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/ExpandingProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/ExpandingProcessorTest.php @@ -109,12 +109,14 @@ public function test_bound_and_unbound_give_the_same_rows(): void public function test_stop_after_the_first_batch_is_forwarded_upstream(): void { - $upstream = (new CountingExtractor(ListColumnsMother::tagsSchema(), array_to_rows([ - ['id' => 'a', 'tags' => ['x', 'y']], - ['id' => 'b', 'tags' => ['z']], - ], ListColumnsMother::tagsSchema()))) - ->withBatchSize(1) - ->extract(flow_context()->backend(), new Criteria()); + $upstream = FlowTestCase::extracted( + (new CountingExtractor(ListColumnsMother::tagsSchema(), array_to_rows([ + ['id' => 'a', 'tags' => ['x', 'y']], + ['id' => 'b', 'tags' => ['z']], + ], ListColumnsMother::tagsSchema())))->withBatchSize(1), + flow_context()->backend(), + new Criteria(), + ); $processed = (new ExpandingProcessor( 'tag', ref('tags')->expand(), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/OffsetProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/OffsetProcessorTest.php index 6a1f4cefa0..cdeacec3ac 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/OffsetProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/OffsetProcessorTest.php @@ -25,7 +25,7 @@ public function test_offset_processor_forwards_stop_to_its_upstream(): void { $upstream = (new CountingExtractor(schema(int_schema('id')), RowsMother::sequentialIds(5)))->withBatchSize(1); $processed = (new OffsetProcessor(1))->process( - $upstream->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($upstream, flow_context()->backend(), new Criteria()), flow_context()->backend(), ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/JoinEachRowsTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/JoinEachRowsTransformerTest.php index d58c08fe16..08bdbf6dde 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/JoinEachRowsTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/JoinEachRowsTransformerTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit\Operators\Transformer; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Foundation\Exception\DataDependentSchemaException; use Flow\ETL\Foundation\Exception\SchemaDefinitionNotUniqueException; use Flow\ETL\Functions\Join\Expression; @@ -17,6 +18,7 @@ use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\join_on; @@ -674,4 +676,14 @@ public function test_right_join_without_prefix(): void ->toArray(), ); } + + public function test_a_corrupted_record_of_the_right_side_is_refused(): void + { + $this->expectException(CorruptedRecordException::class); + + JoinEachRowsTransformer::inner( + new StaticExtractorFactory(from_array([['code' => 'x']], schema(int_schema('code')))), + Expression::on(['id' => 'code']), + )->transform(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context(config())->backend()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/AdoptedBatchesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/AdoptedBatchesTest.php index 151533816d..2c11765908 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/AdoptedBatchesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/AdoptedBatchesTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit\Runtime\Executor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\IO\Extractor\Signal; use Flow\ETL\Runtime\Executor\AdoptedBatches; @@ -56,15 +57,34 @@ public function test_a_stop_reaches_the_source(): void array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))), ); $extractor->withBatchSize(1); - $adopted = (new AdoptedBatches(new PhpBackend()))->of($extractor->extract( + $adopted = (new AdoptedBatches(new PhpBackend()))->of(FlowTestCase::extracted( + $extractor, flow_context()->backend(), new Criteria(), )); - static::assertCount(1, $adopted->current()); + $batch = $adopted->current(); + + static::assertInstanceOf(Rows::class, $batch); + static::assertCount(1, $batch); $adopted->send(Signal::STOP); static::assertFalse($adopted->valid()); static::assertSame(1, $extractor->batchesYielded); } + + public function test_a_corrupted_record_passes_through_unchanged(): void + { + $corrupted = new CorruptedRecordException('file:///a.csv', 4, 2, 'reason', 'x'); + $rows = array_to_rows([['id' => 1]], schema(int_schema('id')), new PhpBackend()); + /** @var Generator $source */ + $source = (static function () use ($corrupted, $rows) { + yield $corrupted; + yield $rows; + })(); + + $adopted = iterator_to_array((new AdoptedBatches(new PhpBackend()))->of($source), false); + + static::assertSame([$corrupted, $rows], $adopted); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/SourceCountTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/SourceCountTest.php index 4665619c1d..a498ba24bd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/SourceCountTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/SourceCountTest.php @@ -4,10 +4,15 @@ namespace Flow\ETL\Tests\Unit\Runtime\Executor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Extractor\Signal; use Flow\ETL\Runtime\Executor\SourceCount; use Flow\ETL\Tests\Context\IdRows; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Values\Rows; +use Generator; + +use function iterator_to_array; final class SourceCountTest extends FlowTestCase { @@ -119,4 +124,22 @@ public function test_a_stop_ends_the_count_and_leaves_the_read_open(): void static::assertSame(2, $read->rows()); static::assertFalse($read->isComplete()); } + + public function test_a_corrupted_record_passes_through_and_counts_no_rows(): void + { + $read = new SourceCount(); + $read->opened(false); + $corrupted = new CorruptedRecordException('file:///a.csv', 4, 2, 'reason', 'x'); + /** @var Generator $batches */ + $batches = (static function () use ($corrupted) { + yield $corrupted; + yield from IdRows::batches([1, 2]); + })(); + + $counted = iterator_to_array($read->count($batches), false); + + static::assertSame($corrupted, $counted[0]); + static::assertSame(2, $read->rows()); + static::assertTrue($read->isComplete()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/SourceCountsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/SourceCountsTest.php index a8fb2281a3..6dfc680ccb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/SourceCountsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/SourceCountsTest.php @@ -12,6 +12,7 @@ use Flow\ETL\Tests\Double\CountingExtractor; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Cardinality; +use Flow\ETL\Values\Rows; use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; @@ -28,12 +29,16 @@ public function test_every_batch_passes_through_unchanged_and_is_counted(): void $sources = new SourceCounts(); $batches = iterator_to_array( - $sources->count($extractor, $extractor->extract(flow_context()->backend(), new Criteria()), false), + $sources->count( + $extractor, + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), + false, + ), false, ); static::assertEquals( - iterator_to_array($extractor->extract(flow_context()->backend(), new Criteria()), false), + iterator_to_array(FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false), $batches, ); static::assertEquals( @@ -49,7 +54,7 @@ public function test_an_empty_one_absorbs_every_read_with_what_its_source_declar $sources = new SourceCounts(); iterator_to_array($attempt->count( $extractor, - $extractor->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false, )); @@ -68,12 +73,12 @@ public function test_an_absorbed_read_of_a_source_already_counted_adds_its_rows( $sources = new SourceCounts(); iterator_to_array($sources->count( $extractor, - $extractor->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false, )); iterator_to_array($attempt->count( $extractor, - $extractor->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false, )); @@ -93,7 +98,7 @@ public function test_an_absorbed_open_or_narrowed_read_leaves_the_source_incompl $attempt->open($open, false)->counted(1); iterator_to_array($attempt->count( $narrowed, - $narrowed->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($narrowed, flow_context()->backend(), new Criteria()), true, )); @@ -111,9 +116,16 @@ public function test_a_stop_reaches_the_source_and_ends_the_count(): void ); $extractor->withBatchSize(1); $sources = new SourceCounts(); - $counted = $sources->count($extractor, $extractor->extract(flow_context()->backend(), new Criteria()), false); + $counted = $sources->count( + $extractor, + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), + false, + ); - static::assertCount(1, $counted->current()); + $batch = $counted->current(); + + static::assertInstanceOf(Rows::class, $batch); + static::assertCount(1, $batch); $counted->send(Signal::STOP); static::assertFalse($counted->valid()); @@ -129,7 +141,7 @@ public function test_a_narrowed_read_is_not_complete(): void iterator_to_array($sources->count( $extractor, - $extractor->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), true, )); @@ -144,12 +156,12 @@ public function test_one_narrowed_read_makes_the_source_incomplete(): void iterator_to_array($sources->count( $extractor, - $extractor->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false, )); iterator_to_array($sources->count( $extractor, - $extractor->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), true, )); @@ -160,8 +172,16 @@ public function test_interleaved_reads_of_one_source_add_up(): void { $extractor = from_array([['id' => 1], ['id' => 2]])->withBatchSize(1); $sources = new SourceCounts(); - $first = $sources->count($extractor, $extractor->extract(flow_context()->backend(), new Criteria()), false); - $second = $sources->count($extractor, $extractor->extract(flow_context()->backend(), new Criteria()), false); + $first = $sources->count( + $extractor, + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), + false, + ); + $second = $sources->count( + $extractor, + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), + false, + ); $first->current(); iterator_to_array($second); @@ -181,12 +201,12 @@ public function test_a_source_read_twice_is_listed_once_with_both_counts(): void iterator_to_array($sources->count( $extractor, - $extractor->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false, )); iterator_to_array($sources->count( $extractor, - $extractor->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false, )); @@ -200,8 +220,16 @@ public function test_sources_are_listed_in_the_order_they_were_first_read(): voi $second = from_array([['id' => 1], ['id' => 2]]); $sources = new SourceCounts(); - iterator_to_array($sources->count($second, $second->extract(flow_context()->backend(), new Criteria()), false)); - iterator_to_array($sources->count($first, $first->extract(flow_context()->backend(), new Criteria()), false)); + iterator_to_array($sources->count( + $second, + FlowTestCase::extracted($second, flow_context()->backend(), new Criteria()), + false, + )); + iterator_to_array($sources->count( + $first, + FlowTestCase::extracted($first, flow_context()->backend(), new Criteria()), + false, + )); static::assertSame([2, 1], [$sources->statistics()[0]->rows, $sources->statistics()[1]->rows]); } @@ -213,7 +241,7 @@ public function test_a_source_that_yields_nothing_is_listed_with_zero_rows(): vo iterator_to_array($sources->count( $extractor, - $extractor->extract(flow_context()->backend(), new Criteria()), + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), false, )); @@ -225,7 +253,11 @@ public function test_a_source_never_read_is_not_listed(): void $extractor = from_array([['id' => 1]]); $sources = new SourceCounts(); - $sources->count($extractor, $extractor->extract(flow_context()->backend(), new Criteria()), false); + $sources->count( + $extractor, + FlowTestCase::extracted($extractor, flow_context()->backend(), new Criteria()), + false, + ); static::assertSame([], $sources->statistics()); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/StepRunnerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/StepRunnerTest.php index 7ae9a37885..b2b47522dc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/StepRunnerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/StepRunnerTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit\Runtime\Executor; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\IO\Extractor\Criteria; use Flow\ETL\Operators\Join\CrossJoin; use Flow\ETL\Operators\Processor\BatchingProcessor; @@ -14,7 +15,9 @@ use Flow\ETL\Operators\Transformer\UntilTransformer; use Flow\ETL\Planning\Physical\JoinStep; use Flow\ETL\Planning\Physical\PipelineSteps; +use Flow\ETL\Runtime\ErrorHandler; use Flow\ETL\Runtime\ErrorHandler\ExtractionError; +use Flow\ETL\Runtime\ErrorHandler\IgnoreError; use Flow\ETL\Runtime\ErrorHandler\LoadingError; use Flow\ETL\Runtime\ErrorHandler\SkipRows; use Flow\ETL\Runtime\ErrorHandler\ThrowError; @@ -35,6 +38,7 @@ use Flow\ETL\Values\Rows; use Flow\Telemetry\Logger\LogEntry; use Generator; +use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; use function array_map; @@ -43,6 +47,7 @@ use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; @@ -468,4 +473,163 @@ public function test_a_loader_before_a_processor_is_closed_before_the_processor_ static::assertSame(1, $spy->closureCount); static::assertSame([['id' => 1], ['id' => 2], ['id' => 3]], $steps->current()->toArray()); } + + public function test_a_corrupted_record_is_loaded_into_the_sources_reject_loader_before_its_batch_rows(): void + { + $rejects = new SpyLoader(); + $loader = new SpyLoader(); + $extractor = from_array([['id' => 1], ['id' => 'x'], ['id' => 3]], schema(int_schema('id'))) + ->withBatchSize(2) + ->withCorruptedRecords($rejects); + + iterator_to_array((new StepRunner())->steps( + new PipelineSteps([$loader]), + $extractor->extract(flow_context()->backend(), new Criteria()), + flow_context(config()), + $extractor, + )); + + static::assertSame([['id' => 1], ['id' => 3]], $loader->loadedRowsToArray()); + static::assertSame( + [[ + 'uri' => null, + 'offset' => 2, + 'line' => null, + 'reason' => 'column "id": could not convert \'x\' (string) to integer', + 'record' => '{"id":"x"}', + 'fields' => null, + ]], + $rejects->loadedRowsToArray(), + ); + static::assertSame(1, $rejects->closureCount); + } + + public function test_corrupted_records_left_after_the_last_rows_are_loaded_before_the_reject_loader_closes(): void + { + $rejects = new RecordingLoader(); + $extractor = from_array([['id' => 'x'], ['id' => 'y']], schema(int_schema('id'))) + ->withBatchSize(10) + ->withCorruptedRecords($rejects); + + static::assertSame( + [], + iterator_to_array((new StepRunner())->steps( + new PipelineSteps(), + $extractor->extract(flow_context()->backend(), new Criteria()), + flow_context(config()), + $extractor, + )), + ); + static::assertSame(['load#1(2)', 'closure'], $rejects->log); + } + + public function test_without_a_reject_loader_a_corrupted_record_is_thrown_as_an_extraction_error(): void + { + $extractor = from_array([['id' => 'x']], schema(int_schema('id'))); + + $this->expectException(CorruptedRecordException::class); + + iterator_to_array((new StepRunner())->steps( + new PipelineSteps(), + $extractor->extract(flow_context()->backend(), new Criteria()), + flow_context(config())->withErrorHandler(new ThrowError()), + $extractor, + )); + } + + /** + * @return Generator + */ + public static function skipping_error_handlers(): Generator + { + yield 'skip rows' => [new SkipRows()]; + yield 'ignore error' => [new IgnoreError()]; + } + + #[DataProvider('skipping_error_handlers')] + public function test_without_a_reject_loader_a_corrupted_record_fails_the_run_whatever_the_error_handler(ErrorHandler $decides): void + { + $handler = new RecordingErrorHandler($decides); + $extractor = from_array([['id' => 'x'], ['id' => 2]], schema(int_schema('id'))); + $thrown = null; + + try { + iterator_to_array((new StepRunner())->steps( + new PipelineSteps(), + $extractor->extract(flow_context()->backend(), new Criteria()), + flow_context(config())->withErrorHandler($handler), + $extractor, + )); + } catch (CorruptedRecordException $e) { + $thrown = $e; + } + + static::assertInstanceOf(CorruptedRecordException::class, $thrown); + static::assertSame([], $handler->errors); + } + + public function test_a_failed_run_discards_the_reject_loader(): void + { + $rejects = new RecordingLoader(); + $extractor = from_array([['id' => 'x'], ['id' => 2]], schema(int_schema('id'))) + ->withBatchSize(2) + ->withCorruptedRecords($rejects); + + try { + iterator_to_array((new StepRunner())->steps( + new PipelineSteps([new RecordingLoader(new RuntimeException('write failed'))]), + $extractor->extract(flow_context()->backend(), new Criteria()), + flow_context(config())->withErrorHandler(new ThrowError()), + $extractor, + )); + static::fail('Expected the write to fail the run'); + } catch (RuntimeException) { + } + + static::assertSame(['load#1(1)', 'discard'], $rejects->log); + } + + public function test_a_failing_reject_loader_reaches_the_loading_handler(): void + { + $handler = new RecordingErrorHandler(); + $rejects = new RecordingLoader(new RuntimeException('rejects failed')); + $extractor = from_array([['id' => 'x'], ['id' => 2]], schema(int_schema('id'))) + ->withBatchSize(2) + ->withCorruptedRecords($rejects); + + static::assertCount( + 1, + iterator_to_array((new StepRunner())->steps( + new PipelineSteps(), + $extractor->extract(flow_context()->backend(), new Criteria()), + flow_context(config())->withErrorHandler($handler), + $extractor, + )), + ); + static::assertCount(1, $handler->errors); + static::assertInstanceOf(LoadingError::class, $handler->errors[0]); + static::assertSame($rejects, $handler->errors[0]->loader); + } + + public function test_a_run_that_routed_records_logs_how_many_and_where(): void + { + $telemetry = new MemoryTelemetryContext(); + $extractor = from_array([ + ['id' => 'x'], + ['id' => 2], + ['id' => 'y'], + ], schema(int_schema('id')))->withCorruptedRecords(new SpyLoader()); + + iterator_to_array((new StepRunner())->steps( + new PipelineSteps(), + $extractor->extract(flow_context()->backend(), new Criteria()), + $telemetry->flowContext, + $extractor, + )); + + static::assertCount( + 1, + $telemetry->logs->entriesContaining('Routed 2 corrupted records to ' . SpyLoader::class . '.'), + ); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Rows/RecordBatchTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Rows/RecordBatchTest.php index 58257ee1f6..1889ef2915 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Rows/RecordBatchTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Rows/RecordBatchTest.php @@ -4,13 +4,20 @@ namespace Flow\ETL\Tests\Unit\Values\Rows; +use DateTimeImmutable; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; +use Flow\ETL\Foundation\Exception\InferredSchemaException; use Flow\ETL\Foundation\Exception\SchemaMismatchException; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Values\Column\PhpBackend; +use Flow\ETL\Values\Rows; use Flow\ETL\Values\Rows\RecordBatch; +use function array_map; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function iterator_to_array; final class RecordBatchTest extends FlowTestCase { @@ -18,28 +25,262 @@ public function test_a_batch_is_handed_out_once_it_is_full_and_the_rest_on_flush { $batch = new RecordBatch(schema(int_schema('id')), new PhpBackend(), 2); - static::assertNull($batch->add(['id' => 1])); - static::assertSame([['id' => 1], ['id' => 2]], $batch->add(['id' => 2])?->toArray()); - static::assertNull($batch->add(['id' => 3])); - static::assertSame([['id' => 3]], $batch->flush()?->toArray()); - static::assertNull($batch->flush()); + static::assertSame([], $batch->add(['id' => 1], 1)); + $full = $batch->add(['id' => 2], 2); + static::assertSame([], $batch->add(['id' => 3], 3)); + $rest = $batch->flush(); + + static::assertSame( + [[['id' => 1], ['id' => 2]]], + array_map(static fn(CorruptedRecordException|Rows $rows): array => ( + $rows instanceof Rows ? $rows->toArray() : [$rows->getMessage()] + ), $full), + ); + static::assertSame( + [[['id' => 3]]], + array_map(static fn(CorruptedRecordException|Rows $rows): array => ( + $rows instanceof Rows ? $rows->toArray() : [$rows->getMessage()] + ), $rest), + ); + static::assertSame([], $batch->flush()); } public function test_nothing_added_flushes_nothing(): void { - static::assertNull((new RecordBatch(schema(int_schema('id')), new PhpBackend(), 1))->flush()); + static::assertSame([], (new RecordBatch(schema(int_schema('id')), new PhpBackend(), 1))->flush()); } - public function test_a_refused_record_is_reported_at_its_position_among_every_record_added(): void + public function test_under_a_declared_schema_a_value_that_does_not_cast_is_a_corrupted_record(): void + { + $batch = new RecordBatch(schema(int_schema('id')), new PhpBackend(), 3, 'file:///orders.csv'); + $batch->add(['id' => '1'], 3, 2, '1', ['1']); + $batch->add(['id' => 'x'], 5, 3, 'x', ['x']); + + [$corrupted, $rows] = $batch->add(['id' => '3'], 7, 4, '3', ['3']); + + static::assertInstanceOf(CorruptedRecordException::class, $corrupted); + static::assertSame(['file:///orders.csv', 5, 3, 'x', ['x']], [ + $corrupted->uri, + $corrupted->offset, + $corrupted->lineNumber, + $corrupted->record, + $corrupted->fields, + ]); + static::assertStringStartsWith('column "id"', $corrupted->reason); + static::assertInstanceOf(Rows::class, $rows); + static::assertSame([['id' => 1], ['id' => 3]], $rows->toArray()); + } + + public function test_every_corrupted_record_of_a_batch_comes_before_its_rows(): void + { + $batch = new RecordBatch(schema(int_schema('id')), new PhpBackend(), 3); + $batch->add(['id' => 'x'], 1); + $batch->add(['id' => '2'], 2); + $handed = $batch->add(['id' => 'y'], 3); + + static::assertSame( + [1, 3, [['id' => 2]]], + array_map(static fn(CorruptedRecordException|Rows $item): array|int => $item instanceof Rows + ? $item->toArray() + : $item->offset, $handed), + ); + } + + public function test_every_refused_record_of_a_batch_is_handed_out_in_order_with_its_offset_and_the_rest_built(): void + { + $batch = new RecordBatch(schema(int_schema('id'), str_schema('name')), new PhpBackend(), 6, 'file:///a.csv'); + $batch->add(['id' => '1', 'name' => 'a'], 10, 2); + $batch->add(['id' => '1'], 20, 3); + $batch->add(['id' => 'x', 'name' => 'b'], 30, 4); + $batch->add(['id' => '4', 'name' => 'c'], 40, 5); + $batch->add(['id' => 'y'], 50, 6); + + static::assertSame( + [ + [20, 3, 'column "name" declared by the schema is missing from the row'], + [30, 4, 'column "id": could not convert \'x\' (string) to integer'], + [50, 6, 'column "id": could not convert \'y\' (string) to integer'], + [['id' => 1, 'name' => 'a'], ['id' => 4, 'name' => 'c'], ['id' => 6, 'name' => 'd']], + ], + array_map( + static fn(CorruptedRecordException|Rows $item): array => ( + $item instanceof Rows ? $item->toArray() : [$item->offset, $item->lineNumber, $item->reason] + ), + $batch->add(['id' => '6', 'name' => 'd'], 60, 7), + ), + ); + } + + public function test_a_batch_of_corrupted_records_only_hands_out_no_rows(): void { $batch = new RecordBatch(schema(int_schema('id')), new PhpBackend(), 2); - $batch->add(['id' => 1]); - $batch->add(['id' => 2]); - $batch->add(['id' => 3]); + $batch->add(['id' => 'x'], 1); + + static::assertCount(1, $batch->flush()); + static::assertSame([], $batch->flush()); + } + + public function test_a_record_without_source_text_is_its_json(): void + { + $batch = new RecordBatch(schema(int_schema('id'), str_schema('name')), new PhpBackend(), 1); + + [$corrupted] = $batch->add(['id' => 'x', 'name' => 'a/ü'], 2); + + static::assertInstanceOf(CorruptedRecordException::class, $corrupted); + static::assertSame(['{"id":"x","name":"a/ü"}', null, null], [ + $corrupted->record, + $corrupted->fields, + $corrupted->uri, + ]); + } + + public function test_the_fields_of_a_grid_record_are_their_string_form(): void + { + $batch = new RecordBatch( + schema(int_schema('id'), str_schema('at'), str_schema('a', true)), + new PhpBackend(), + 1, + ); + $at = new DateTimeImmutable('2026-10-31 12:00:00+00:00'); + + [$corrupted] = $batch->add(['id' => 'x', 'at' => 'n', 'a' => null], 4, fields: ['x', $at, null, 1.5, true]); + + static::assertInstanceOf(CorruptedRecordException::class, $corrupted); + static::assertSame(['x', '2026-10-31T12:00:00+00:00', '', '1.5', '1'], $corrupted->fields); + static::assertSame('["x","2026-10-31T12:00:00+00:00","","1.5","1"]', $corrupted->record); + } + + public function test_under_an_inferred_schema_a_refused_record_past_the_sample_is_a_schema_error(): void + { + $batch = new RecordBatch(schema(int_schema('id')), new PhpBackend(), 2, 'file:///orders.csv', 2); + $batch->add(['id' => 1], 1); + $batch->add(['id' => 2], 2); + $batch->add(['id' => 3], 3); + + $this->expectException(InferredSchemaException::class); + $this->expectExceptionMessage( + 'Record at offset 4 of file:///orders.csv does not fit the schema inferred from a sample of 2 rows', + ); + + $batch->add(['id' => 'x'], 4); + } + + public function test_under_an_inferred_schema_without_a_uri_the_refusal_is_left_to_the_reader(): void + { + $batch = new RecordBatch(schema(int_schema('id')), new PhpBackend(), 2, sampleSize: 2); + $batch->add(['id' => 1], 1); + $batch->add(['id' => 2], 2); + $batch->add(['id' => 3], 3); $this->expectException(SchemaMismatchException::class); $this->expectExceptionMessage('column "id" (row 3)'); - $batch->add(['id' => 'x']); + $batch->add(['id' => 'x'], 4); + } + + public function test_under_an_inferred_schema_a_keyed_record_refuses_a_key_the_schema_lacks(): void + { + $batch = new RecordBatch( + schema(int_schema('a')), + new PhpBackend(), + 1, + sampleSize: 1, + known: schema(int_schema('a')), + ); + + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('column "c" (row 0) is not declared by the schema'); + + $batch->add(['a' => 1, 'c' => 2], 1); + } + + public function test_under_an_inferred_schema_a_keyed_record_keeps_a_key_the_known_schema_declares(): void + { + $batch = new RecordBatch( + schema(int_schema('a')), + new PhpBackend(), + 1, + 'file:///group=1/data.jsonl', + 1, + schema(int_schema('a'), str_schema('group')), + ); + + [$rows] = $batch->add(['a' => 1, 'group' => '2'], 0, 1); + + static::assertInstanceOf(Rows::class, $rows); + static::assertSame([['a' => 1]], $rows->toArray()); + } + + public function test_under_an_inferred_schema_a_keyed_record_key_past_the_sample_names_the_record(): void + { + $batch = new RecordBatch( + schema(int_schema('a')), + new PhpBackend(), + 1, + 'file:///b.jsonl', + 1000, + schema(int_schema('a')), + ); + + $this->expectException(InferredSchemaException::class); + $this->expectExceptionMessage( + 'Record at offset 23, line 2, of file:///b.jsonl does not fit the schema inferred from a sample of 1000 rows: ' + . 'column "c"', + ); + + $batch->add(['a' => 1, 'c' => 2], 23, 2); + } + + public function test_under_an_inferred_schema_a_grid_record_keeps_a_partition_column_its_header_carries(): void + { + $batch = new RecordBatch(schema(int_schema('a')), new PhpBackend(), 1, 'file:///group=1/a.csv', 1); + + [$rows] = $batch->add(['a' => '1', 'group' => '2'], 4, 2, '1,2', ['1', '2']); + + static::assertInstanceOf(Rows::class, $rows); + static::assertSame([['a' => 1]], $rows->toArray()); + } + + public function test_a_refused_record_takes_its_slot_and_leaves_in_source_order_with_the_cast_refusals(): void + { + $batch = new RecordBatch(schema(int_schema('id')), new PhpBackend(), 4); + $batch->add(['id' => 'x'], 10); + $batch->refuse(new CorruptedRecordException(null, 20, null, '3 fields, the header has 1', 'a,b,c')); + $batch->add(['id' => 'y'], 30); + + static::assertSame( + [10, 20, 30, [['id' => 4]]], + array_map( + static fn(CorruptedRecordException|Rows $item): array|int => $item instanceof Rows + ? $item->toArray() + : $item->offset, + $batch->add(['id' => 4], 40), + ), + ); + } + + public function test_a_batch_of_refused_records_only_flushes_them(): void + { + $batch = new RecordBatch(schema(int_schema('id')), new PhpBackend(), 10); + $refused = new CorruptedRecordException(null, 1, null, 'reason', 'x'); + $batch->refuse($refused); + + static::assertSame([$refused], $batch->flush()); + static::assertSame([], $batch->flush()); + } + + public function test_of_numbers_the_records_from_one_and_flushes_the_rest(): void + { + $batch = new RecordBatch(schema(int_schema('id')), new PhpBackend(), 2); + + static::assertSame( + [2, [['id' => 1]], [['id' => 3], ['id' => 4]]], + array_map( + static fn(CorruptedRecordException|Rows $item): array|int => $item instanceof Rows + ? $item->toArray() + : $item->offset, + iterator_to_array($batch->of([['id' => 1], ['id' => 'x'], ['id' => 3], ['id' => 4]]), false), + ), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Values/SchemaTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Values/SchemaTest.php index a36933c55a..ee6a13e4b0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Values/SchemaTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Values/SchemaTest.php @@ -961,4 +961,13 @@ public function test_put_replaces_a_column_of_the_same_name_in_place(): void schema(int_schema('a'), int_schema('b'), int_schema('c'))->put(str_schema('b')), ); } + + public function test_missing_from_names_the_columns_a_header_lacks_in_schema_order(): void + { + static::assertSame( + ['id', 'at'], + schema(int_schema('id'), str_schema('name'), str_schema('at'))->missingFrom(['name', 'extra']), + ); + static::assertSame([], schema(int_schema('id'))->missingFrom(['id'])); + } } diff --git a/src/extension/flow-php-ext/crates/flow-csv-tokenizer/src/lib.rs b/src/extension/flow-php-ext/crates/flow-csv-tokenizer/src/lib.rs index ad5458a114..8dd138a81b 100644 --- a/src/extension/flow-php-ext/crates/flow-csv-tokenizer/src/lib.rs +++ b/src/extension/flow-php-ext/crates/flow-csv-tokenizer/src/lib.rs @@ -6,7 +6,7 @@ use std::io; use std::ops::Range; -use memchr::{memchr, memchr2}; +use memchr::{memchr, memchr2, memchr_iter}; pub struct Dialect { pub separator: u8, @@ -59,6 +59,8 @@ enum Span { pub struct Record { /// Where the record begins in the tokenizer's buffer. start: usize, + /// Where it ends, its line ending excluded. + end: usize, fields: Vec, unescaped: Vec, } @@ -75,6 +77,11 @@ impl Record { self.fields.len().max(1) } + /// An empty line: no field at all. + pub fn is_empty(&self) -> bool { + self.fields.is_empty() + } + fn clear(&mut self) { self.fields.clear(); self.unescaped.clear(); @@ -136,6 +143,9 @@ pub struct Tokenizer { /// The BOM the first record loses once its end is found, counted in its bytes. bom: usize, last_record_bytes: usize, + /// The newlines before `counted`, the buffer position `line()` last counted to, compacted bytes included. + newlines: u64, + counted: usize, } impl Tokenizer { @@ -153,6 +163,8 @@ impl Tokenizer { first_record: true, bom: 0, last_record_bytes: 0, + newlines: 0, + counted: 0, } } @@ -160,6 +172,8 @@ impl Tokenizer { /// are no longer valid. fn compact(&mut self) { if self.record_start > 0 { + self.count_newlines_to(self.record_start); + self.counted -= self.record_start; self.buffer.drain(..self.record_start); self.scanned -= self.record_start; self.record_start = 0; @@ -204,6 +218,26 @@ impl Tokenizer { } } + /// The 1-based line `record` starts on among every byte fed; valid until the next `feed()`. Each byte is counted + /// once: records only move forward. + pub fn line(&mut self, record: &Record) -> u64 { + self.count_newlines_to(record.start); + + 1 + self.newlines + } + + fn count_newlines_to(&mut self, position: usize) { + if position > self.counted { + self.newlines += memchr_iter(b'\n', &self.buffer[self.counted..position]).count() as u64; + self.counted = position; + } + } + + /// The record's bytes as the stream holds them, its line ending excluded; valid until the next `feed()`. + pub fn text<'a>(&'a self, record: &'a Record) -> &'a [u8] { + &self.buffer[record.start..record.end.max(record.start)] + } + pub fn next(&mut self, record: &mut Record) -> bool { if self.strip_bom && !self.find_bom() { return false; @@ -390,6 +424,7 @@ impl Tokenizer { limit -= 1; } + open.end = limit; let limit = limit - base; match self.scan { @@ -673,4 +708,50 @@ mod tests { assert!(matches!(tokenizer.field(&record, 0), Field::Value(b"a"))); assert!(matches!(tokenizer.field(&record, 1), Field::Missing)); } + + #[test] + fn text_is_the_record_as_read_without_its_line_ending_and_a_blank_line_is_empty() { + let mut tokenizer = Tokenizer::new(comma(), true); + let mut record = Record::default(); + let mut seen = Vec::new(); + + tokenizer.feed(b"\xEF\xBB\xBFa,\"b\r\nc\"\r\n\r\n4,5"); + tokenizer.finish(); + + while tokenizer.next(&mut record) { + seen.push((tokenizer.text(&record).to_vec(), record.is_empty())); + } + + assert_eq!( + seen, + vec![ + (b"a,\"b\r\nc\"".to_vec(), false), + (b"".to_vec(), true), + (b"4,5".to_vec(), false), + ] + ); + } + + #[test] + fn a_line_counts_every_newline_fed_before_the_record_compacted_or_not() { + let mut tokenizer = Tokenizer::new(comma(), true); + let mut record = Record::default(); + let mut lines = Vec::new(); + + for chunk in [&b"\xEF\xBB\xBFa,\"b\r\nc\"\r\n\r"[..], b"\n4,5\n", b"6,7"] { + tokenizer.feed(chunk); + + while tokenizer.next(&mut record) { + lines.push(tokenizer.line(&record)); + } + } + + tokenizer.finish(); + + while tokenizer.next(&mut record) { + lines.push(tokenizer.line(&record)); + } + + assert_eq!(lines, vec![1, 3, 4, 5]); + } } diff --git a/src/extension/flow-php-ext/crates/flow-json-records/src/lib.rs b/src/extension/flow-php-ext/crates/flow-json-records/src/lib.rs index a997a7fcb5..1a95420a3b 100644 --- a/src/extension/flow-php-ext/crates/flow-json-records/src/lib.rs +++ b/src/extension/flow-php-ext/crates/flow-json-records/src/lib.rs @@ -7,7 +7,7 @@ use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; use std::fmt; -use flow_batch_frame::key::Key; +use flow_batch_frame::key::{array_key_index, Key}; use flow_batch_frame::kind::Field; use serde::de::{self, DeserializeSeed, Deserializer as _, IgnoredAny, MapAccess, SeqAccess, Visitor}; use serde::Deserialize; @@ -39,6 +39,8 @@ pub struct Record { pub start: usize, pub end: usize, pub ordinal: usize, + /// Where the record begins among every byte fed, a BOM before the first line included. + pub offset: u64, } /// The document scanner: where the element being framed stands. @@ -144,6 +146,10 @@ pub struct Framing { /// The next line (1-based) or element (0-based). ordinal: usize, finished: bool, + /// The bytes fed before `input`'s first. + drained: u64, + /// The length of the BOM skipped before the first line. + bom_len: usize, } impl Framing { @@ -158,12 +164,15 @@ impl Framing { bom: true, ordinal: usize::from(lines), finished: false, + drained: 0, + bom_len: 0, } } pub fn feed(&mut self, chunk: &[u8]) { if self.start > 0 && self.start * 2 >= self.input.len() { self.input.drain(..self.start); + self.drained += self.start as u64; self.start = 0; } @@ -195,6 +204,7 @@ impl Framing { if pending.starts_with(BOM) { self.start += BOM.len(); + self.bom_len = BOM.len(); } self.bom = false; @@ -220,10 +230,13 @@ impl Framing { }); } + let first_line = self.lines && ordinal == 1; + bounds.push(Record { start: records.len(), end: records.len() + bytes.len(), ordinal, + offset: self.drained + (start - if first_line { self.bom_len } else { 0 }) as u64, }); records.extend_from_slice(bytes); @@ -459,6 +472,8 @@ impl<'de> DeserializeSeed<'de> for Text { pub struct MemberColumns { names: HashMap, usize>, positions: Vec>, + /// The names a record may hold, when a member outside them is to be reported. + known: Option>>, } impl MemberColumns { @@ -480,6 +495,23 @@ impl MemberColumns { columns } + + /// `map_record()` reports a record's first member none of `names` names: as a key, or as the positional name + /// (`e00`, `e01`, ...) of an int key. + pub fn knowing(mut self, names: impl Iterator>) -> Self { + self.known = Some(names.collect()); + + self + } + + fn unknown(&self, name: &[u8]) -> bool { + let Some(known) = &self.known else { + return false; + }; + + !known.contains(name) + && array_key_index(name).is_none_or(|index| !known.contains(format!("e{index:02}").as_bytes())) + } } /// A record's members, each stored as the span of its value in `slots[column]`. @@ -487,10 +519,15 @@ struct Members<'a> { columns: &'a MemberColumns, slots: &'a mut [Option<(usize, usize)>], base: usize, + unknown: &'a mut Option, } impl Members<'_> { - fn store(&mut self, column: Option, value: &RawValue) { + fn store(&mut self, name: &[u8], column: Option, value: &RawValue) { + if self.unknown.is_none() && self.columns.unknown(name) { + *self.unknown = Some(String::from_utf8_lossy(name).into_owned()); + } + if let Some(column) = column { let start = value.get().as_ptr() as usize - self.base; @@ -510,7 +547,7 @@ impl<'de> Visitor<'de> for Members<'_> { while let Some(name) = map.next_key_seed(Text)? { let value: &'de RawValue = map.next_value()?; - self.store(self.columns.names.get(name.as_bytes()).copied(), value); + self.store(name.as_bytes(), self.columns.names.get(name.as_bytes()).copied(), value); } Ok(()) @@ -520,7 +557,11 @@ impl<'de> Visitor<'de> for Members<'_> { let mut position = 0; while let Some(value) = seq.next_element::<&'de RawValue>()? { - self.store(self.columns.positions.get(position).copied().flatten(), value); + self.store( + position.to_string().as_bytes(), + self.columns.positions.get(position).copied().flatten(), + value, + ); position += 1; } @@ -528,22 +569,30 @@ impl<'de> Visitor<'de> for Members<'_> { } } -/// Maps one record's members onto `slots`, spans relative to `base` (the address of the batch's first byte). +/// Maps one record's members onto `slots`, spans relative to `base` (the address of the batch's first byte); the first +/// member outside the columns' known names, when they know any. pub fn map_record( record: &[u8], columns: &MemberColumns, slots: &mut [Option<(usize, usize)>], base: usize, -) -> Result<(), RecordError> { +) -> Result, RecordError> { let malformed = |e: serde_json::Error| RecordError::Malformed(e.to_string()); if let Some(b'{' | b'[') = record.iter().find(|byte| !is_whitespace(byte)) { + let mut unknown = None; let mut deserializer = deserializer(record); deserializer - .deserialize_any(Members { columns, slots, base }) + .deserialize_any(Members { + columns, + slots, + base, + unknown: &mut unknown, + }) .map_err(malformed)?; + deserializer.end().map_err(malformed)?; - return deserializer.end().map_err(malformed); + return Ok(unknown); } let mut deserializer = deserializer(record); @@ -962,4 +1011,45 @@ mod tests { assert_eq!(deduplicated(&object(&members)), expected, "{size} keys and k1 again"); } } + + #[test] + fn a_line_offset_counts_every_byte_fed_and_the_first_line_its_bom() { + let mut framing = Framing::new(true); + let mut records = Vec::new(); + let mut bounds: Vec = Vec::new(); + + for chunk in [&b"\xEF\xBB\xBF{\"a\":1}\n\n{\"a\":"[..], b"2}\r\n", b"{\"a\":3}"] { + framing.feed(chunk); + + while framing.next(&mut records, &mut bounds).unwrap() {} + } + + framing.finish(); + + while framing.next(&mut records, &mut bounds).unwrap() {} + + assert_eq!( + bounds + .iter() + .map(|record| (record.offset, record.ordinal)) + .collect::>(), + vec![(0, 1), (12, 3), (21, 4)], + ); + } + + #[test] + fn a_member_outside_the_known_names_is_reported_first_one_wins() { + let known = |names: &[&str]| columns(&["a"]).knowing(names.iter().map(|name| name.as_bytes().to_vec())); + let unknown = |record: &str, columns: &MemberColumns| { + let mut slots = vec![None; 1]; + + map_record(record.as_bytes(), columns, &mut slots, record.as_ptr() as usize).unwrap() + }; + + assert_eq!(unknown(r#"{"a":1,"c":2,"d":3}"#, &known(&["a"])), Some("c".to_string())); + assert_eq!(unknown(r#"{"a":1,"group":2}"#, &known(&["a", "group"])), None); + assert_eq!(unknown(r#"[1,2]"#, &known(&["a", "e01"])), Some("0".to_string())); + assert_eq!(unknown(r#"{"5":1}"#, &known(&["e05"])), None); + assert_eq!(unknown(r#"{"a":1,"c":2}"#, &columns(&["a"])), None); + } } diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/IO/Extractor/RustCSVOpenSource.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/IO/Extractor/RustCSVOpenSource.php index 8a2d021f17..4adf42ba06 100644 --- a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/IO/Extractor/RustCSVOpenSource.php +++ b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/IO/Extractor/RustCSVOpenSource.php @@ -35,6 +35,7 @@ public function __construct( bool $removeBOM, ?int $charactersReadInLine = null, ?array $headers = null, + bool $missingColumnsAsNull = false, ) { throw new RuntimeException('flow_php extension is not loaded'); } @@ -42,7 +43,7 @@ public function __construct( /** * @return RustIterator */ - public function batches(Schema $schema, int $batchSize, Backend $backend): RustIterator + public function batches(Schema $schema, int $batchSize, Backend $backend, ?int $sampleSize = null): RustIterator { throw new RuntimeException('flow_php extension is not loaded'); } diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/JSON/IO/Extractor/RustJsonOpenSource.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/JSON/IO/Extractor/RustJsonOpenSource.php index af519f8275..6ac5d598a5 100644 --- a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/JSON/IO/Extractor/RustJsonOpenSource.php +++ b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/JSON/IO/Extractor/RustJsonOpenSource.php @@ -32,8 +32,13 @@ public function __construct(SourceStream $stream, bool $lines, string $uri, stri /** * @return RustIterator */ - public function batches(Schema $schema, int $batchSize, Backend $backend): RustIterator - { + public function batches( + Schema $schema, + int $batchSize, + Backend $backend, + ?int $sampleSize = null, + ?Schema $known = null, + ): RustIterator { throw new RuntimeException('flow_php extension is not loaded'); } diff --git a/src/extension/flow-php-ext/src/batch_columns.rs b/src/extension/flow-php-ext/src/batch_columns.rs index 603de032b2..beb300d3ec 100644 --- a/src/extension/flow-php-ext/src/batch_columns.rs +++ b/src/extension/flow-php-ext/src/batch_columns.rs @@ -1,6 +1,5 @@ -//! A batch of native columns, one builder per schema definition, with `RowsBuilder::appendRows()`'s refusals: each -//! column keeps its first, the lowest refused row wins (then the earliest definition), and a NOT NULL column a row -//! lacks is refused only when nothing else was. The readers differ only in where a cell comes from. +//! A batch of native columns, one builder per schema definition. The readers differ only in where a cell comes from +//! and decide what a refused record becomes. use std::rc::Rc; @@ -11,12 +10,12 @@ use flow_batch_frame::builder::KindBuilder; use flow_batch_frame::key::Key; use flow_batch_frame::kind::Kind; -use crate::builder::{definition_nullable, schema_mismatch}; +use crate::builder::definition_nullable; use crate::cast::{ bool_from_str, float_from_str, integer_from_str, json_gate, parse_iso_date, parse_iso_datetime, CastKind, }; use crate::column::RustColumn; -use crate::ctx::{self, call_method, call_static, ht_for_each, ht_insert, ht_insert_long, zval_long}; +use crate::ctx::{self, call_method, ht_for_each, ht_insert, ht_insert_long}; use crate::date_check::iso_instant_micros; use crate::exception::ext_exception; use crate::json_check::json_valid; @@ -30,10 +29,6 @@ pub struct BatchColumn { pub nullable: bool, pub plan: Rc, pub values: KindBuilder, - /// The first refused row and its `ColumnMismatchException`. - pub refusal: Option<(usize, Zval)>, - /// The first row lacking this NOT NULL column. - pub absent: Option, } /// The batch being built, and the schema and batch size it is built for. @@ -43,8 +38,6 @@ pub struct BatchColumns { pub batch_size: usize, pub columns: Vec, pub rows: usize, - /// Rows already handed out from this source: a refusal names its row in the whole source, as `RecordBatch` does. - emitted: usize, } /// A PHP `$batchSize` as a row count: refused unless greater than 0. @@ -86,8 +79,6 @@ impl BatchColumns { nullable: definition_nullable(definition)?, values: KindBuilder::new(&plan.kind), plan, - refusal: None, - absent: None, }); Ok(()) @@ -99,7 +90,6 @@ impl BatchColumns { batch_size, columns, rows: 0, - emitted: 0, }) } @@ -127,10 +117,7 @@ impl BatchColumns { ))); } _ => { - let emitted = pending.as_ref().map_or(0, |state| state.emitted); - let mut state = Self::new(schema, json, batch_size)?; - state.emitted = emitted; - *pending = Some(state); + *pending = Some(Self::new(schema, json, batch_size)?); rebuilt = true; } } @@ -157,55 +144,19 @@ impl BatchColumns { for column in &mut self.columns { column.values.truncate(0); - column.refusal = None; - column.absent = None; } } - /// The batch's columns, or its refusal at its row in the whole source: the lowest refused row (ties: the earliest - /// definition), else the lowest absent row (ties: the earliest definition) as `missingColumn`. The state is reset - /// either way. + /// The batch's columns; the state is reset either way. pub fn finish(&mut self) -> Result { let batch = self.build(); - if batch.is_ok() { - self.emitted += self.rows; - } - self.reset(); batch } fn build(&mut self) -> Result { - // min_by_key() keeps the first of equal rows: the earliest definition - if let Some((row, cause)) = self - .columns - .iter() - .filter_map(|column| column.refusal.as_ref()) - .min_by_key(|(row, _)| *row) - { - return Err(schema_mismatch( - zval_long((self.emitted + *row) as i64), - cause.shallow_clone(), - )); - } - - if let Some((row, column)) = self - .columns - .iter() - .filter_map(|column| Some((column.absent?, column))) - .min_by_key(|(row, _)| *row) - { - let cause = call_static( - "Flow\\ETL\\Foundation\\Exception\\ColumnMismatchException", - "missingColumn", - &mut [call_method(&column.definition, "name", &mut [])?], - )?; - - return Err(schema_mismatch(zval_long((self.emitted + row) as i64), cause)); - } - let mut columns = ZendHashTable::with_capacity(self.columns.len() as u32); for column in &mut self.columns { diff --git a/src/extension/flow-php-ext/src/corrupted.rs b/src/extension/flow-php-ext/src/corrupted.rs new file mode 100644 index 0000000000..ebd9d33e91 --- /dev/null +++ b/src/extension/flow-php-ext/src/corrupted.rs @@ -0,0 +1,131 @@ +//! `RecordBatch`'s outcomes for a record a reader cannot hand out as a row: a `CorruptedRecordException` under a +//! declared schema, an `InferredSchemaException` under an inferred one. + +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::{ZendHashTable, Zval}; + +use crate::builder::throw; +use crate::ctx::{call_method, call_static, construct, find_class, null_zval, zval_long, zval_str}; +use crate::exception::ext_exception; + +const COLUMN_MISMATCH: &str = "Flow\\ETL\\Foundation\\Exception\\ColumnMismatchException"; +const CORRUPTED_RECORD: &str = "Flow\\ETL\\Foundation\\Exception\\CorruptedRecordException"; +const INFERRED_SCHEMA: &str = "Flow\\ETL\\Foundation\\Exception\\InferredSchemaException"; +const SCHEMA_MISMATCH: &str = "Flow\\ETL\\Foundation\\Exception\\SchemaMismatchException"; + +/// A record's `uri, offset, line` as the exceptions take them. +fn located(uri: &Zval, offset: u64, line: Option) -> [Zval; 3] { + [ + uri.shallow_clone(), + zval_long(offset as i64), + line.map_or_else(null_zval, |line| zval_long(line as i64)), + ] +} + +/// `ColumnMismatchException::missingColumn()` of `definition`. +pub fn missing_column(definition: &Zval) -> Result { + call_static( + COLUMN_MISMATCH, + "missingColumn", + &mut [call_method(definition, "name", &mut [])?], + ) +} + +/// `ColumnMismatchException::unexpectedColumn()` of `name`. +pub fn unexpected_column(name: &str) -> Result { + call_static(COLUMN_MISMATCH, "unexpectedColumn", &mut [zval_str(name.as_bytes())]) +} + +/// `ColumnMismatchException::reason()`. +pub fn reason_of(cause: &Zval) -> Result { + Ok(call_method(cause, "reason", &mut [])?.string().unwrap_or_default()) +} + +/// `CorruptedRecordException::fromCells()`: a grid record's fields and its text. +pub fn from_cells( + uri: &Zval, + offset: u64, + line: Option, + reason: &str, + fields: &[&[u8]], + record: &[u8], +) -> Result { + let mut cells = ZendHashTable::new(); + + for field in fields { + cells + .push(zval_str(field)) + .map_err(|e| ext_exception(format!("flow_php failed to collect a record's fields: {e:?}")))?; + } + + let mut cells_zv = Zval::new(); + cells_zv.set_hashtable(cells); + let [uri, offset, line] = located(uri, offset, line); + + call_static( + CORRUPTED_RECORD, + "fromCells", + &mut [ + uri, + offset, + line, + zval_str(reason.as_bytes()), + cells_zv, + zval_str(record), + ], + ) +} + +/// `CorruptedRecordException::fromValues()`: a keyed record's values, its text when the source holds one. +pub fn from_values( + uri: &Zval, + offset: u64, + line: Option, + reason: &str, + values: Zval, + record: Option<&[u8]>, +) -> Result { + let [uri, offset, line] = located(uri, offset, line); + + call_static( + CORRUPTED_RECORD, + "fromValues", + &mut [ + uri, + offset, + line, + zval_str(reason.as_bytes()), + values, + record.map_or_else(null_zval, zval_str), + ], + ) +} + +/// `InferredSchemaException::outsideTheSample()` for the record at `row` of its batch, refused by `cause`. +pub fn outside_the_sample( + uri: &Zval, + offset: u64, + line: Option, + sample_size: i64, + row: usize, + cause: Zval, +) -> PhpException { + let exception = find_class(SCHEMA_MISMATCH) + .and_then(|ce| construct(ce, &mut [zval_long(row as i64), cause])) + .and_then(|mut mismatch| { + let mut mismatch_zv = Zval::new(); + mismatch_zv.set_object(&mut mismatch); + let [uri, offset, line] = located(uri, offset, line); + + call_static( + INFERRED_SCHEMA, + "outsideTheSample", + &mut [uri, offset, line, zval_long(sample_size), mismatch_zv], + ) + }); + + match exception { + Ok(exception) => throw(exception), + Err(e) => e, + } +} diff --git a/src/extension/flow-php-ext/src/csv/columns.rs b/src/extension/flow-php-ext/src/csv/columns.rs index b9f0ed0808..71b0efa341 100644 --- a/src/extension/flow-php-ext/src/csv/columns.rs +++ b/src/extension/flow-php-ext/src/csv/columns.rs @@ -1,54 +1,61 @@ -//! `RustCSVOpenSource::batches()`: CSV cells straight into native columns through `BatchColumns`; a NOT NULL -//! column absent from the header is absent from every row. +//! `RustCSVOpenSource::batches()`: CSV cells straight into native columns through `BatchColumns`, with +//! `RecordBatch`'s outcomes: a record whose field count differs from the header, and under a declared schema a record +//! with a refused cell, takes its place in the batch and is handed out before its rows, in source order; under an +//! inferred schema a refused cell is a schema error. + +use std::collections::VecDeque; use ext_php_rs::exception::PhpException; use ext_php_rs::types::Zval; use crate::batch_columns::{native_text, BatchColumn, BatchColumns, HeldBatch}; use crate::builder::{is_types_exception, php_lane, value_does_not_match, Refusal}; +use crate::corrupted::{from_cells, missing_column, outside_the_sample, reason_of}; use crate::csv::CsvReader; use crate::ctx::{null_zval, transparent_exception, zval_str}; use crate::physical::append_physical; +use crate::source::Fetched; -/// The batch being built, and the field of the header each of its columns reads. +/// The batch being built, the field of the header each of its columns reads, and what it hands out before it. #[derive(Default)] pub struct CsvColumns { batch: Option, /// `None`: the header has no column of this name. fields: Vec>, + /// The records of the batch being built that were refused, in source order. + refused: Vec, + /// A finished batch's refused records, then the batch itself. + ready: VecDeque, + held: Option, } -/// One cell into its column: a NOT NULL null or a PHP-lane `Flow\Types` refusal becomes the column's refusal, any -/// other exception escapes as thrown. -fn append_cell(column: &mut BatchColumn, row: usize, cell: Option<&[u8]>) -> Result<(), PhpException> { +/// One cell into its column; the cell's `ColumnMismatchException` when it is refused. A PHP-lane exception that is not +/// a `Flow\Types` refusal escapes as thrown. +fn append_cell(column: &mut BatchColumn, cell: Option<&[u8]>) -> Result, PhpException> { let Some(bytes) = cell else { if !column.nullable { - column.refusal = Some((row, value_does_not_match(&column.definition, null_zval(), None)?)); - - return Ok(()); + return value_does_not_match(&column.definition, null_zval(), None).map(Some); } column.values.append_null(); - return Ok(()); + return Ok(None); }; if native_text(&mut column.values, &column.plan.kind, &column.plan.cast, bytes)? { - return Ok(()); + return Ok(None); } let value = zval_str(bytes); match php_lane(&column.plan, &value) { - Ok(physical) => append_physical(&mut column.values, &column.plan.kind, &physical), + Ok(physical) => append_physical(&mut column.values, &column.plan.kind, &physical).map(|()| None), Err(Refusal::Cast(mut exception)) => { if !is_types_exception(&exception)? { return Err(transparent_exception(&mut exception)); } - column.refusal = Some((row, value_does_not_match(&column.definition, value, None)?)); - - Ok(()) + value_does_not_match(&column.definition, value, None).map(Some) } Err(Refusal::Physical(mut exception)) => { if !is_types_exception(&exception)? { @@ -57,19 +64,41 @@ fn append_cell(column: &mut BatchColumn, row: usize, cell: Option<&[u8]>) -> Res let mut reason = Zval::new(); reason.set_object(&mut exception); - column.refusal = Some((row, value_does_not_match(&column.definition, value, Some(reason))?)); - Ok(()) + value_does_not_match(&column.definition, value, Some(reason)).map(Some) } } } +/// `CorruptedRecordException::fromCells()` of the reader's current record. +fn corrupted(reader: &mut CsvReader, uri: &Zval, reason: &str) -> Result { + from_cells( + uri, + reader.record_offset(), + reader.record_line(), + reason, + &reader.record_fields(), + reader.record_text(), + ) +} + +/// The next batch or corrupted record of `reader`, none while more input is needed. pub fn next_columns( reader: &mut CsvReader, pending: &mut CsvColumns, schema: &Zval, batch_size: usize, -) -> Result, PhpException> { + uri: &Zval, + sample_size: Option, +) -> Result, PhpException> { + if let Some(record) = pending.ready.pop_front() { + return Ok(Some(Fetched::Corrupted(record))); + } + + if let Some(batch) = pending.held.take() { + return Ok(Some(Fetched::Batch(batch))); + } + if reader.headers().is_empty() { return Ok(None); } @@ -84,17 +113,26 @@ pub fn next_columns( .collect(); } - while state.rows < state.batch_size && reader.next_record() { - let row = state.rows; + // a refused record fills its place in the batch, as RecordBatch counts it + while state.rows + pending.refused.len() < state.batch_size { + match reader.next_record() { + None => break, + Some(Err(reason)) => { + pending.refused.push(corrupted(reader, uri, &reason)?); - for (column, field) in state.columns.iter_mut().zip(&pending.fields) { - if column.refusal.is_some() { continue; } + Some(Ok(())) => {} + } + + let row = state.rows; + let mut absence = None; + let mut refusal = None; + for (column, field) in state.columns.iter_mut().zip(&pending.fields) { let Some(field) = field else { - if !column.nullable { - column.absent = column.absent.or(Some(row)); + if !column.nullable && absence.is_none() { + absence = Some(missing_column(&column.definition)?); } column.values.append_null(); @@ -102,19 +140,63 @@ pub fn next_columns( continue; }; - if let Err(e) = append_cell(column, row, reader.cell(*field)) { - state.reset(); + match append_cell(column, reader.cell(*field)) { + Ok(None) => {} + Ok(Some(cause)) => { + refusal = Some(cause); + + break; + } + Err(e) => { + state.reset(); - return Err(e); + return Err(e); + } } } - state.rows += 1; + // a refused value wins over an absence, as RowsBuilder::appendRows() decides + let Some(cause) = refusal.or(absence) else { + state.rows += 1; + + continue; + }; + + for column in &mut state.columns { + column.values.truncate(row); + } + + if let Some(sample_size) = sample_size { + state.reset(); + pending.refused.clear(); + + return Err(outside_the_sample( + uri, + reader.record_offset(), + reader.record_line(), + sample_size, + row, + cause, + )); + } + + let reason = reason_of(&cause)?; + pending.refused.push(corrupted(reader, uri, &reason)?); } - if state.rows == 0 || (state.rows < state.batch_size && !reader.is_finished()) { + if state.rows + pending.refused.len() < state.batch_size && !reader.is_finished() { return Ok(None); } - state.finish().map(Some) + pending.ready.extend(pending.refused.drain(..)); + + if state.rows > 0 { + pending.held = Some(state.finish()?); + } + + if let Some(record) = pending.ready.pop_front() { + return Ok(Some(Fetched::Corrupted(record))); + } + + Ok(pending.held.take().map(Fetched::Batch)) } diff --git a/src/extension/flow-php-ext/src/csv/mod.rs b/src/extension/flow-php-ext/src/csv/mod.rs index c3beb5c41c..b944a1ce98 100644 --- a/src/extension/flow-php-ext/src/csv/mod.rs +++ b/src/extension/flow-php-ext/src/csv/mod.rs @@ -57,17 +57,28 @@ struct Cell { pub struct CsvReader { tokenizer: Tokenizer, + record: Record, + /// The first record sized the generated header and is still to be handed out as a row. + first_row_pending: bool, + /// The byte the next record starts at in the source: the window's start plus every record read since. + position: u64, + /// Lines are counted only when the read began at the source's first byte. + from_first_byte: bool, + record_offset: u64, + /// The bytes, as read, of every row handed out: line endings included, the header excluded. + consumed_bytes: u64, + consumed_rows: u64, with_header: bool, empty_to_null: bool, + missing_columns_as_null: bool, headers: Option>, cells: Vec, - record: Record, - first_row_pending: bool, - consumed_bytes: u64, finished: bool, } impl CsvReader { + /// `start` is the source byte the stream begins at: a window's start, else 0. + #[allow(clippy::too_many_arguments)] pub fn new( separator: &[u8], enclosure: &[u8], @@ -76,6 +87,8 @@ impl CsvReader { empty_to_null: bool, remove_bom: bool, headers: Option>>, + missing_columns_as_null: bool, + start: u64, ) -> Result { let [separator] = separator else { return Err(ext_exception("flow_php CSV separator must be exactly one byte")); @@ -104,13 +117,18 @@ impl CsvReader { }, remove_bom, ), + record: Record::default(), + first_row_pending: false, + position: start, + from_first_byte: start == 0, + record_offset: start, + consumed_bytes: 0, + consumed_rows: 0, with_header, empty_to_null, + missing_columns_as_null, headers: None, cells: Vec::new(), - record: Record::default(), - first_row_pending: false, - consumed_bytes: 0, finished: false, }; @@ -156,17 +174,25 @@ impl CsvReader { .map(|cell| cell.field_index) } - /// Advances to the next complete row, counting its bytes; `false` when none is buffered. - pub fn next_record(&mut self) -> bool { + /// Advances to the next record that is not an empty line, counting a row's bytes: `RecordDecoder::decode()`'s + /// outcome, the reason a corrupted record is refused, `None` when no complete record is buffered. + pub fn next_record(&mut self) -> Option> { self.resolve_headers(); - self.headers.is_some() - && next_row( - &mut self.first_row_pending, - &mut self.tokenizer, - &mut self.record, - &mut self.consumed_bytes, - ) + let headers = self.headers.as_ref()?.len(); + + if !self.next_nonempty() { + return None; + } + + if let Some(reason) = self.corruption(headers) { + return Some(Err(reason)); + } + + self.consumed_bytes += self.tokenizer.last_record_bytes() as u64; + self.consumed_rows += 1; + + Some(Ok(())) } /// The current row's cell of `field`, as `Records::decode()` shapes it; `None` is null. @@ -174,12 +200,42 @@ impl CsvReader { cell_value(&self.tokenizer, &self.record, field, self.empty_to_null) } + /// The current record's byte offset in the source. + pub fn record_offset(&self) -> u64 { + self.record_offset + } + + /// The current record's line; `None` when the read began past the source's first byte. + pub fn record_line(&mut self) -> Option { + self.from_first_byte.then(|| self.tokenizer.line(&self.record)) + } + + /// The current record as the source holds it, its line ending excluded. + pub fn record_text(&self) -> &[u8] { + self.tokenizer.text(&self.record) + } + + /// The current record's fields as split, `''` for an empty one. + pub fn record_fields(&self) -> Vec<&[u8]> { + (0..self.record.field_count()) + .map(|index| match self.tokenizer.field(&self.record, index) { + Field::Value(value) => value, + Field::Null | Field::Missing => b"", + }) + .collect() + } + /// The bytes, as read, of every row `next` and `fold` produced so far: line endings included, the header /// record excluded. pub fn consumed_bytes(&self) -> u64 { self.consumed_bytes } + /// How many rows `next_record` handed out, a refused cell's included. + pub fn consumed_rows(&self) -> u64 { + self.consumed_rows + } + /// The resolved header, `[]` until the first record is available. pub fn headers(&mut self) -> Vec<&[u8]> { self.resolve_headers(); @@ -190,26 +246,20 @@ impl CsvReader { .unwrap_or_default() } - /// Up to `batch_size` rows keyed by header; an empty list when no complete record is buffered. + /// Up to `batch_size` rows keyed by header, corrupted records left out; an empty list when no complete record is + /// buffered. pub fn next(&mut self, batch_size: usize) -> Result, PhpException> { - self.resolve_headers(); - let mut batch = ZendHashTable::new(); - let Some(headers) = self.headers.as_ref() else { - return Ok(batch); - }; - while batch.len() < batch_size { - if !next_row( - &mut self.first_row_pending, - &mut self.tokenizer, - &mut self.record, - &mut self.consumed_bytes, - ) { - break; + match self.next_record() { + None => break, + Some(Err(_)) => continue, + Some(Ok(())) => {} } + let headers = self.headers.as_ref().expect("resolved by next_record()"); + batch .push(row_values(headers, &self.tokenizer, &self.record, self.empty_to_null)) .map_err(|e| ext_exception(format!("flow_php failed to collect CSV row values: {e:?}")))?; @@ -218,26 +268,23 @@ impl CsvReader { Ok(batch) } - /// Folds up to `limit` rows (all when `None`) into `fold`; returns how many it folded - fewer only when no - /// further complete record is buffered. - pub fn fold(&mut self, fold: &mut fold::Fold, limit: Option) -> Result { - self.resolve_headers(); - - let Some(headers) = self.headers.as_ref() else { - return Ok(0); - }; - + /// Folds up to `limit` rows (all when `None`), corrupted records left out, into `fold`; fewer only when no further + /// complete record is buffered. + pub fn fold(&mut self, fold: &mut fold::Fold, limit: Option) -> Result<(), PhpException> { let mut folded = 0; let mut positions = vec![None; self.cells.len()]; while limit.is_none_or(|limit| folded < limit) { - if !next_row( - &mut self.first_row_pending, - &mut self.tokenizer, - &mut self.record, - &mut self.consumed_bytes, - ) { - break; + match self.next_record() { + None => break, + Some(Err(_)) => continue, + Some(Ok(())) => {} + } + + let headers = self.headers.as_ref().expect("resolved by next_record()"); + + if positions.len() != self.cells.len() { + positions = vec![None; self.cells.len()]; } for (cell, position) in self.cells.iter().zip(positions.iter_mut()) { @@ -252,16 +299,67 @@ impl CsvReader { folded += 1; } - Ok(folded) + Ok(()) + } + + /// The tokenizer's next record, its offset taken; `false` when none is buffered. + fn advance(&mut self) -> bool { + if !self.tokenizer.next(&mut self.record) { + return false; + } + + self.record_offset = self.position; + self.position += self.tokenizer.last_record_bytes() as u64; + + true + } + + /// The next record that is not an empty line: the pending first one when auto headers were sized by it. + fn next_nonempty(&mut self) -> bool { + if std::mem::take(&mut self.first_row_pending) { + return true; + } + + while self.advance() { + if !self.record.is_empty() { + return true; + } + } + + false } - /// `Records::decode()`'s first record: the header when `withHeader`, else `e00, e01, ...` sized by - /// the first record, which then stays pending as the first row. + /// `RecordDecoder::decode()`'s verdict on the current record against a header of `headers` names: why it is + /// corrupted, or `None`. An extra empty field loses nothing; a missing one is null only under + /// `missing_columns_as_null`. + fn corruption(&self, headers: usize) -> Option { + let fields = self.record.field_count(); + let extra = (headers..fields).any(|index| match self.tokenizer.field(&self.record, index) { + Field::Value(value) => !value.is_empty(), + Field::Null | Field::Missing => false, + }); + + (extra || (fields < headers && !self.missing_columns_as_null)) + .then(|| format!("{fields} fields, the header has {headers}")) + } + + /// `Records::decode()`'s first record that is not an empty line: the header when `withHeader`, else + /// `e00, e01, ...` sized by that record, which then stays pending as the first row. fn resolve_headers(&mut self) { - if self.headers.is_some() || !self.tokenizer.next(&mut self.record) { + if self.headers.is_some() { return; } + loop { + if !self.advance() { + return; + } + + if !self.record.is_empty() { + break; + } + } + let (tokenizer, record) = (&self.tokenizer, &self.record); self.headers = Some(if self.with_header { @@ -309,23 +407,6 @@ impl CsvReader { } } -/// The first row is pending when auto headers were sized by it; otherwise the next complete record. Either way it is -/// the tokenizer's last record, so its bytes are added to `consumed_bytes`. -fn next_row( - first_row_pending: &mut bool, - tokenizer: &mut Tokenizer, - record: &mut Record, - consumed_bytes: &mut u64, -) -> bool { - if !(std::mem::take(first_row_pending) || tokenizer.next(record)) { - return false; - } - - *consumed_bytes += tokenizer.last_record_bytes() as u64; - - true -} - /// `Records::decode()` for one cell: a cell past the record's end is null, a present `''` is null under `emptyToNull`; /// `None` is null. fn cell_value<'a>(tokenizer: &'a Tokenizer, record: &'a Record, index: usize, empty_to_null: bool) -> Option<&'a [u8]> { diff --git a/src/extension/flow-php-ext/src/csv/source.rs b/src/extension/flow-php-ext/src/csv/source.rs index fb91999e97..40924e0ae0 100644 --- a/src/extension/flow-php-ext/src/csv/source.rs +++ b/src/extension/flow-php-ext/src/csv/source.rs @@ -18,7 +18,7 @@ use ext_php_rs::flags::{ClassFlags, DataType}; use ext_php_rs::prelude::*; use ext_php_rs::types::{ZendHashTable, Zval}; -use crate::batch_columns::{batch_size_of, HeldBatch}; +use crate::batch_columns::batch_size_of; use crate::csv::columns::{next_columns, CsvColumns}; use crate::csv::fold::{string_values, Fold}; use crate::csv::CsvReader; @@ -27,12 +27,11 @@ use crate::exception::ext_exception; use crate::interfaces::csv_open_source_ce; use crate::iterator::RustIterator; use crate::render::runtime; -use crate::source::{self, reading, Fed, CHUNK}; +use crate::source::{self, reading, window_start, Fed, Fetched, CHUNK, RANGE_SOURCE_STREAM}; const CLASS: &str = "RustCSVOpenSource"; const RECORDS_BATCH: usize = 1000; const NATIVE_LOCAL_SOURCE_STREAM: &str = "Flow\\Filesystem\\Stream\\NativeLocalSourceStream"; -const RANGE_SOURCE_STREAM: &str = "Flow\\Filesystem\\Stream\\RangeSourceStream"; /// One pread of a local file (DuckDB's CSV buffers are 2 MB): about 512 per GiB. `CHUNK` stays the PHP arm's. const NATIVE_CHUNK: usize = 2 << 20; const STRING_TYPE_NARROWER: &str = "Flow\\Types\\Type\\Native\\String\\StringTypeNarrower"; @@ -164,14 +163,11 @@ impl Shared { let mut reading = reading(&self.reading, CLASS)?; let result = f(&mut reading); self.produced_bytes.set(reading.reader.consumed_bytes()); + self.produced_rows.set(reading.reader.consumed_rows()); result } - fn produced(&self, rows: u64) { - self.produced_rows.set(self.produced_rows.get() + rows); - } - /// `$stream->iterate($chunkSize)`, the generator one consuming method feeds from; `None` on the native arm, which /// reads the file itself. fn chunks(&self) -> PhpResult> { @@ -247,19 +243,22 @@ struct CsvFeed { chunks: Option, schema: Zval, batch_size: usize, + uri: Zval, + sample_size: Option, } impl Fed for CsvFeed { - fn next_batch(&self) -> PhpResult> { - let batch = self - .shared - .with(|reading| next_columns(&mut reading.reader, &mut reading.columns, &self.schema, self.batch_size))?; - - if let Some(batch) = &batch { - self.shared.produced(batch.count as u64); - } - - Ok(batch) + fn next_batch(&self) -> PhpResult> { + self.shared.with(|reading| { + next_columns( + &mut reading.reader, + &mut reading.columns, + &self.schema, + self.batch_size, + &self.uri, + self.sample_size, + ) + }) } fn feed(&self) -> PhpResult { @@ -296,7 +295,6 @@ impl RustCSVOpenSource { let batch = shared.with(|reading| reading.reader.next(RECORDS_BATCH))?; if !batch.is_empty() { - shared.produced(batch.len() as u64); pending.extend(batch.values().map(Zval::shallow_clone)); continue; @@ -327,6 +325,7 @@ impl RustCSVOpenSource { removeBOM: bool, charactersReadInLine: Option, headers: Option<&ZendHashTable>, + missingColumnsAsNull: Option, ) -> PhpResult { // a local file is read by Rust and ignores the read length, as NativeLocalSourceStream::readLines() does on // the PHP path; for S3/Azure it is the range-request size - and from_csv() defaults it to 10 MB @@ -343,6 +342,8 @@ impl RustCSVOpenSource { emptyToNull, removeBOM, headers.map(string_values), + missingColumnsAsNull.unwrap_or(false), + window_start(stream)?, )?, columns: CsvColumns::default(), }), @@ -382,14 +383,22 @@ impl RustCSVOpenSource { self.headers() } - /// Exactly `$batchSize` rows keyed and ordered by `$schema` (the last may be shorter), every column adopted by - /// `$backend`; refusals are `RowsBuilder::appendRows()`'s, row indexes relative to the source. - pub fn batches(&self, schema: &Zval, batchSize: i64, backend: &Zval) -> PhpResult { + /// `PhpCSVOpenSource::batches()`: `$batchSize` records keyed and ordered by `$schema` at a time, every column + /// adopted by `$backend`, corrupted records handed out as they are read. + pub fn batches( + &self, + schema: &Zval, + batchSize: i64, + backend: &Zval, + sampleSize: Option, + ) -> PhpResult { let fed = CsvFeed { shared: Rc::clone(&self.shared), chunks: self.shared.chunks()?, schema: schema.shallow_clone(), batch_size: batch_size_of(batchSize, "CSV")?, + uri: call_method(&call_method(&self.shared.stream, "path", &mut [])?, "uri", &mut [])?, + sample_size: sampleSize, }; source::batches(fed, schema, backend) @@ -451,8 +460,7 @@ impl RustCSVOpenSource { while self.shared.feed(&chunks)? { let limit = remaining(&fold); - let folded = self.shared.with(|reading| reading.reader.fold(&mut fold, limit))?; - self.shared.produced(folded); + self.shared.with(|reading| reading.reader.fold(&mut fold, limit))?; if rowBudget != -1 && fold.rows() >= rowBudget as u64 { break; @@ -462,8 +470,7 @@ impl RustCSVOpenSource { if rowBudget == -1 || fold.rows() < rowBudget as u64 { self.shared.finish()?; let limit = remaining(&fold); - let folded = self.shared.with(|reading| reading.reader.fold(&mut fold, limit))?; - self.shared.produced(folded); + self.shared.with(|reading| reading.reader.fold(&mut fold, limit))?; } let mut types = ZendHashTable::new(); diff --git a/src/extension/flow-php-ext/src/interfaces.rs b/src/extension/flow-php-ext/src/interfaces.rs index 228d37eec6..041024fb4e 100644 --- a/src/extension/flow-php-ext/src/interfaces.rs +++ b/src/extension/flow-php-ext/src/interfaces.rs @@ -323,9 +323,11 @@ pub fn register() -> Result<()> { Arg::new("schema", SCHEMA), Arg::new("batchSize", DataType::Long), Arg::new("backend", BACKEND), + Arg::new("sampleSize", DataType::Long).allow_null().default("null"), ], ITERATOR, - ), + ) + .required_args(3), method("headers", vec![], DataType::Array), method("records", vec![], ITERATOR), method( @@ -350,9 +352,12 @@ pub fn register() -> Result<()> { Arg::new("schema", SCHEMA), Arg::new("batchSize", DataType::Long), Arg::new("backend", BACKEND), + Arg::new("sampleSize", DataType::Long).allow_null().default("null"), + Arg::new("known", SCHEMA).allow_null().default("null"), ], ITERATOR, - ), + ) + .required_args(3), ], )?; interface( diff --git a/src/extension/flow-php-ext/src/json/read.rs b/src/extension/flow-php-ext/src/json/read.rs index 4d25c18533..f00dd33ad5 100644 --- a/src/extension/flow-php-ext/src/json/read.rs +++ b/src/extension/flow-php-ext/src/json/read.rs @@ -3,11 +3,12 @@ //! refusals. use std::borrow::Cow; +use std::collections::VecDeque; use std::fmt; use ext_php_rs::exception::PhpException; use ext_php_rs::prelude::*; -use ext_php_rs::types::Zval; +use ext_php_rs::types::{ZendHashTable, Zval}; use flow_batch_frame::builder::KindBuilder; use flow_batch_frame::key::array_key_index; use flow_batch_frame::kind::Kind; @@ -18,13 +19,15 @@ use flow_json_records::{ use serde::de::{self, Deserializer as _, IgnoredAny, SeqAccess, Visitor}; use serde_json::value::RawValue; -use crate::batch_columns::{native_text, BatchColumns, HeldBatch}; +use crate::batch_columns::{native_text, BatchColumns}; use crate::builder::{append_native, append_one, native_leaf, Append}; use crate::cast::{CastKind, MapKeyKind}; +use crate::corrupted::{from_values, missing_column, outside_the_sample, reason_of, unexpected_column}; use crate::ctx::{self, null_zval, zval_str}; use crate::exception::ext_exception; use crate::physical::overflow; use crate::render::runtime; +use crate::source::Fetched; fn number_zval(text: &str) -> Zval { let mut zval = Zval::new(); @@ -222,12 +225,16 @@ struct JsonColumns { pub struct JsonReader { uri: String, + /// The byte of its file the stream begins at: a window's start, else 0. + start: u64, framing: Framing, /// The framed records of the pending batch, copied out of the input. records: Vec, bounds: Vec, slots: Vec>, columns: JsonColumns, + /// A read batch's corrupted records, then the batch itself. + ready: VecDeque, } /// `Malformed JSON in "" at line N: …` (lines, 1-based) or `at element N` (a document, 0-based). @@ -238,6 +245,91 @@ fn malformed(uri: &str, lines: bool, ordinal: usize, message: &str) -> PhpExcept )) } +/// A line by its byte offset in the file (and its line when the read began at the file's first byte), an array +/// element by its 1-based number. +fn located(start: u64, lines: bool, record: &Record) -> (u64, Option) { + match lines { + true => (start + record.offset, (start == 0).then_some(record.ordinal as u64)), + false => (record.ordinal as u64 + 1, None), + } +} + +/// A record of a batch refused: by a value (the earliest definition wins) or, ranked after every refused value, by a +/// NOT NULL column it lacks - RowsBuilder's order. +struct Refused { + row: usize, + absent: bool, + cause: Zval, +} + +/// Each record's members into the batch's columns, the rows of `skip` (ascending) left out; the refusals, one per +/// refused record, ascending by row. +fn fill( + state: &mut BatchColumns, + records: &[u8], + slots: &[Option<(usize, usize)>], + rows: usize, + skip: &[usize], +) -> PhpResult> { + let width = state.columns.len(); + let mut refusals: Vec = Vec::new(); + + for (position, column) in state.columns.iter_mut().enumerate() { + for row in (0..rows).filter(|row| skip.binary_search(row).is_err()) { + let Some((start, end)) = slots[row * width + position] else { + if !column.nullable { + refusals.push(Refused { + row, + absent: true, + cause: missing_column(&column.definition)?, + }); + } + + column.values.append_null(); + + continue; + }; + // SAFETY: every record of the batch passed map_record(), whose serde parse checked the UTF-8 of each + // member value; the span is one of those values, so it starts and ends on character boundaries + let text = unsafe { std::str::from_utf8_unchecked(&records[start..end]) }; + + let value = if text == "null" { + null_zval() + } else { + let len = column.values.len(); + + if into_column(&mut column.values, &column.plan.kind, &column.plan.cast, text)? { + continue; + } + + column.values.truncate(len); + + decode(text)? + }; + + if let Append::Refused { cause, .. } = append_one( + &mut column.values, + &column.plan, + &column.definition, + column.nullable, + &value, + )? { + refusals.push(Refused { + row, + absent: false, + cause, + }); + } + } + } + + // stable: within a row and kind, the earliest definition stays first + refusals.sort_by_key(|refused| (refused.row, refused.absent)); + refusals.dedup_by_key(|refused| refused.row); + + Ok(refusals) +} + impl JsonReader { fn clear(&mut self) { self.records.clear(); @@ -248,16 +340,23 @@ impl JsonReader { } } - fn read(&mut self, schema: &Zval, batch_size: usize) -> PhpResult> { + /// Reads the next batch into `ready`: its corrupted records, then its rows. + fn read(&mut self, schema: &Zval, batch_size: usize, sample_size: Option, known: &[Vec]) -> PhpResult<()> { let (state, rebuilt) = BatchColumns::prepare(&mut self.columns.batch, schema, batch_size, "JSON")?; if rebuilt { - self.columns.members = MemberColumns::of(state.columns.iter().map(|column| &column.key)); + let members = MemberColumns::of(state.columns.iter().map(|column| &column.key)); + + self.columns.members = match sample_size { + Some(_) => members.knowing(known.iter().cloned()), + None => members, + }; } let batch_size = state.batch_size; let lines = self.framing.lines(); + // a refused record fills its place in the batch, as RecordBatch counts it while self.bounds.len() < batch_size { match self.framing.next(&mut self.records, &mut self.bounds) { Ok(true) => {} @@ -275,14 +374,14 @@ impl JsonReader { } let state = self.columns.batch.as_mut().expect("prepared above"); - state.rows = self.bounds.len(); if self.bounds.is_empty() || (self.bounds.len() < state.batch_size && !self.framing.finished()) { - return Ok(None); + return Ok(()); } let width = state.columns.len(); let base = self.records.as_ptr() as usize; + let mut unknown = None; self.slots.clear(); self.slots.resize(self.bounds.len() * width, None); @@ -296,7 +395,10 @@ impl JsonReader { slots, base, ) { - Ok(()) => {} + Ok(None) => {} + Ok(Some(name)) => { + unknown = unknown.or(Some((row, name))); + } Err(RecordError::Malformed(message)) => { return Err(malformed(&self.uri, lines, record.ordinal, &message)) } @@ -309,58 +411,87 @@ impl JsonReader { } } - for (position, column) in state.columns.iter_mut().enumerate() { - for row in 0..self.bounds.len() { - if column.refusal.is_some() { - break; - } + let uri = || zval_str(self.uri.as_bytes()); - let Some((start, end)) = self.slots[row * width + position] else { - if !column.nullable { - column.absent = column.absent.or(Some(row)); - } + if let (Some((row, name)), Some(sample_size)) = (unknown, sample_size) { + let (offset, line) = located(self.start, lines, &self.bounds[row]); - column.values.append_null(); + return Err(outside_the_sample( + &uri(), + offset, + line, + sample_size, + row, + unexpected_column(&name)?, + )); + } - continue; - }; - // SAFETY: every record of the batch passed map_record(), whose serde parse checked the UTF-8 of each - // member value; the span is one of those values, so it starts and ends on character boundaries - let text = unsafe { std::str::from_utf8_unchecked(&self.records[start..end]) }; - - let value = if text == "null" { - null_zval() - } else { - let len = column.values.len(); - - if into_column(&mut column.values, &column.plan.kind, &column.plan.cast, text)? { - continue; - } + let rows = self.bounds.len(); + let refusals = fill(state, &self.records, &self.slots, rows, &[])?; + + if !refusals.is_empty() { + if let Some(sample_size) = sample_size { + let first = refusals.iter().find(|refused| !refused.absent).unwrap_or(&refusals[0]); + let (offset, line) = located(self.start, lines, &self.bounds[first.row]); + + return Err(outside_the_sample( + &uri(), + offset, + line, + sample_size, + first.row, + first.cause.shallow_clone(), + )); + } - column.values.truncate(len); + for refused in &refusals { + let record = &self.bounds[refused.row]; + let text = &self.records[record.start..record.end]; + let (offset, line) = located(self.start, lines, record); + let reason = reason_of(&refused.cause)?; + + // a line is its own text; an array element, which the source holds no text of alone, is its values + self.ready.push_back(Fetched::Corrupted(match lines { + true => from_values(&uri(), offset, line, &reason, empty_array(), Some(text))?, + false => from_values( + &uri(), + offset, + line, + &reason, + // SAFETY: map_record() parsed the record, checking its UTF-8 + decode(unsafe { std::str::from_utf8_unchecked(text) })?, + None, + )?, + })); + } - decode(text)? - }; + let skip: Vec = refusals.iter().map(|refused| refused.row).collect(); - if let Append::Refused { cause, .. } = append_one( - &mut column.values, - &column.plan, - &column.definition, - column.nullable, - &value, - )? { - column.refusal = Some((row, cause)); - } - } + state.reset(); + fill(state, &self.records, &self.slots, rows, &skip)?; + state.rows = rows - skip.len(); + } else { + state.rows = rows; + } + + if state.rows > 0 { + self.ready.push_back(Fetched::Batch(state.finish()?)); } self.records.clear(); self.bounds.clear(); - state.finish().map(Some) + Ok(()) } } +fn empty_array() -> Zval { + let mut array = Zval::new(); + array.set_hashtable(ZendHashTable::new()); + + array +} + /// `json_decode($text, true)`: PHP's own value of a JSON value `into_column()` did not cast. fn decode(text: &str) -> Result { let mut associative = Zval::new(); @@ -380,15 +511,17 @@ fn decode(text: &str) -> Result { } impl JsonReader { - /// `uri` names the file in refusals. - pub fn new(lines: bool, uri: &[u8]) -> Self { + /// `uri` names the file in malformed-input refusals; `start` is the byte of its file the stream begins at. + pub fn new(lines: bool, uri: &[u8], start: u64) -> Self { Self { uri: String::from_utf8_lossy(uri).into_owned(), + start, framing: Framing::new(lines), records: Vec::new(), bounds: Vec::new(), slots: Vec::new(), columns: JsonColumns::default(), + ready: VecDeque::new(), } } @@ -401,15 +534,24 @@ impl JsonReader { self.framing.finish(); } - /// Exactly `batch_size` rows keyed and ordered by `schema` while that many are buffered, the remainder after - /// `finish()`, else none. Refusals are `RowsBuilder::appendRows()`'s, row indexes relative to the source. - pub fn next_columns(&mut self, schema: &Zval, batch_size: usize) -> PhpResult> { - let batch = self.read(schema, batch_size); - - if batch.is_err() { - self.clear(); + /// Exactly `batch_size` records keyed and ordered by `schema` while that many are buffered, the remainder after + /// `finish()`, else none: a batch's corrupted records first, then its rows. Under an inferred schema a refused + /// record is an `InferredSchemaException`. + pub fn next_columns( + &mut self, + schema: &Zval, + batch_size: usize, + sample_size: Option, + known: &[Vec], + ) -> PhpResult> { + if self.ready.is_empty() { + if let Err(e) = self.read(schema, batch_size, sample_size, known) { + self.clear(); + + return Err(e); + } } - batch + Ok(self.ready.pop_front()) } } diff --git a/src/extension/flow-php-ext/src/json/source.rs b/src/extension/flow-php-ext/src/json/source.rs index ab6e00e1a8..dfc696003e 100644 --- a/src/extension/flow-php-ext/src/json/source.rs +++ b/src/extension/flow-php-ext/src/json/source.rs @@ -9,13 +9,13 @@ use ext_php_rs::flags::ClassFlags; use ext_php_rs::prelude::*; use ext_php_rs::types::Zval; -use crate::batch_columns::{batch_size_of, HeldBatch}; -use crate::ctx::{call_method, zval_long}; +use crate::batch_columns::batch_size_of; +use crate::ctx::{call_method, ht_for_each, zval_long}; use crate::exception::ext_exception; use crate::interfaces::json_open_source_ce; use crate::iterator::RustIterator; use crate::json::read::JsonReader; -use crate::source::{self, reading, Fed, CHUNK}; +use crate::source::{self, reading, window_start, Fed, Fetched, CHUNK}; const CLASS: &str = "RustJsonOpenSource"; @@ -32,11 +32,14 @@ struct JsonFeed { shared: Rc, schema: Zval, batch_size: usize, + sample_size: Option, + /// The names a record may hold under an inferred schema. + known: Vec>, } impl Fed for JsonFeed { - fn next_batch(&self) -> PhpResult> { - reading(&self.shared.reader, CLASS)?.next_columns(&self.schema, self.batch_size) + fn next_batch(&self) -> PhpResult> { + reading(&self.shared.reader, CLASS)?.next_columns(&self.schema, self.batch_size, self.sample_size, &self.known) } /// `SourceStream::read()` runs with no borrow held: the stream may call back into the source. @@ -88,24 +91,54 @@ pub struct RustJsonOpenSource { impl RustJsonOpenSource { /// `$head`: the chunk the opener already read at `$offset` ('' for none); the bytes before it are JSON whitespace /// after an optional BOM, which the reader would skip. `$uri` names the file in refusals. - pub fn __construct(stream: &Zval, lines: bool, uri: BinarySlice, head: BinarySlice, offset: i64) -> Self { - Self { + pub fn __construct( + stream: &Zval, + lines: bool, + uri: BinarySlice, + head: BinarySlice, + offset: i64, + ) -> PhpResult { + Ok(Self { shared: Rc::new(Shared { stream: stream.shallow_clone(), - reader: RefCell::new(JsonReader::new(lines, &uri)), + reader: RefCell::new(JsonReader::new(lines, &uri, window_start(stream)?)), head: RefCell::new(head.to_vec()), offset: Cell::new(offset), }), - } + }) } - /// Exactly `$batchSize` rows keyed and ordered by `$schema` (the last may be shorter), every column adopted by - /// `$backend`; refusals are `RowsBuilder::appendRows()`'s, row indexes relative to the source. - pub fn batches(&self, schema: &Zval, batchSize: i64, backend: &Zval) -> PhpResult { + /// `PhpJsonOpenSource::batches()`: `$batchSize` records keyed and ordered by `$schema` at a time, every column + /// adopted by `$backend`, a batch's corrupted records handed out before its rows. + pub fn batches( + &self, + schema: &Zval, + batchSize: i64, + backend: &Zval, + sampleSize: Option, + known: Option<&Zval>, + ) -> PhpResult { + let mut names = Vec::new(); + + if sampleSize.is_some() { + let definitions = call_method(known.unwrap_or(schema), "definitions", &mut [])?; + let definitions = definitions + .array() + .ok_or_else(|| ext_exception("flow_php expected Schema::definitions() to return an array"))?; + + ht_for_each(definitions, |name, index, _| { + names.push(name.map_or_else(|| index.to_string().into_bytes(), |name| name.as_bytes().to_vec())); + + Ok(()) + })?; + } + let fed = JsonFeed { shared: Rc::clone(&self.shared), schema: schema.shallow_clone(), batch_size: batch_size_of(batchSize, "JSON")?, + sample_size: sampleSize, + known: names, }; source::batches(fed, schema, backend) diff --git a/src/extension/flow-php-ext/src/lib.rs b/src/extension/flow-php-ext/src/lib.rs index fa6a73c08e..f983248b06 100644 --- a/src/extension/flow-php-ext/src/lib.rs +++ b/src/extension/flow-php-ext/src/lib.rs @@ -5,6 +5,7 @@ mod batch_columns; mod builder; mod cast; mod column; +mod corrupted; mod csv; mod ctx; mod date_check; diff --git a/src/extension/flow-php-ext/src/source.rs b/src/extension/flow-php-ext/src/source.rs index 62bf7fc391..ae5b86b2e1 100644 --- a/src/extension/flow-php-ext/src/source.rs +++ b/src/extension/flow-php-ext/src/source.rs @@ -8,6 +8,8 @@ use ext_php_rs::types::Zval; use crate::backend::{adopt_batch, Adoption, Definitions}; use crate::batch_columns::HeldBatch; +use crate::ctx::call_method; +use crate::exception::ext_exception; use crate::iterator::RustIterator; use crate::render::runtime; @@ -15,10 +17,35 @@ use crate::render::runtime; /// and narrow files. `JsonFileBatches::CHUNK` is its PHP copy. pub const CHUNK: i64 = 1 << 15; +/// What a reader hands out next. +pub enum Fetched { + Batch(HeldBatch), + /// A `CorruptedRecordException`. + Corrupted(Zval), +} + +pub const RANGE_SOURCE_STREAM: &str = "Flow\\Filesystem\\Stream\\RangeSourceStream"; + +/// The byte of its file `stream` begins at: a `RangeSourceStream`'s start, else 0. +pub fn window_start(stream: &Zval) -> PhpResult { + let is_window = matches!( + (stream.object(), ext_php_rs::zend::ClassEntry::try_find(RANGE_SOURCE_STREAM)), + (Some(object), Some(ce)) if object.instance_of(ce) + ); + + if !is_window { + return Ok(0); + } + + u64::try_from(call_method(stream, "start", &mut [])?.long().unwrap_or(-1)) + .map_err(|_| ext_exception("flow_php expected RangeSourceStream::start() to be an int >= 0")) +} + /// A reader a stream is fed into, one consuming call at a time. pub trait Fed { - /// The next full batch of what was fed (the remainder once finished), none while more input is needed. - fn next_batch(&self) -> PhpResult>; + /// The next full batch of what was fed (the remainder once finished) or corrupted record, none while more input + /// is needed. + fn next_batch(&self) -> PhpResult>; /// The next chunk of the stream fed in; false at its end. fn feed(&self) -> PhpResult; @@ -26,14 +53,16 @@ pub trait Fed { fn finish(&self) -> PhpResult<()>; } -/// The batches of `fed`, each adopted by `backend` into `Rows` of `schema`. +/// The batches of `fed`, each adopted by `backend` into `Rows` of `schema`, and its corrupted records as they are. pub fn batches(fed: impl Fed + 'static, schema: &Zval, backend: &Zval) -> PhpResult { let adoption = Adoption::new(Definitions::of(schema)?, schema, backend); let mut finished = false; Ok(RustIterator::new(Box::new(move || loop { - if let Some(batch) = fed.next_batch()? { - return adopt_batch(&adoption, batch).map(Some); + match fed.next_batch()? { + Some(Fetched::Batch(batch)) => return adopt_batch(&adoption, batch).map(Some), + Some(Fetched::Corrupted(record)) => return Ok(Some(record)), + None => {} } if finished { diff --git a/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt b/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt index ed70e73138..373a9375c4 100644 --- a/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt +++ b/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt @@ -14,7 +14,6 @@ $cases = [ 'space_before_enclosure' => '2261222c2020226222', 'junk_after_close' => '612c226222782c63', 'unterminated' => '22756e7465726d696e617465642c61', - 'blank' => '', 'trailing_sep' => '612c622c', ]; @@ -25,6 +24,9 @@ foreach ($cases as $case => $hex) { echo $case, ': ', $fields === $expected ? 'identical' : 'FAIL ' . json_encode(array_map(static fn(?string $f): ?string => $f === null ? null : bin2hex($f), $fields)), "\n"; } + +// a blank line is no record, as RecordDecoder::decode() skips str_getcsv('')'s [null] +echo 'blank: ', csv_native_rows("\n", ',', '"', '\\', false, false, false)[1] === [] ? 'no row' : 'FAIL', "\n"; ?> --EXPECT-- plain: identical @@ -34,5 +36,5 @@ doubled_quote: identical space_before_enclosure: identical junk_after_close: identical unterminated: identical -blank: identical trailing_sep: identical +blank: no row diff --git a/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt b/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt index 32474af62a..b50da3f30e 100644 --- a/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt +++ b/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt @@ -1,5 +1,5 @@ --TEST-- -native CSV rows pad and truncate to the header like Records, under both emptyToNull settings +native CSV leaves out ragged records like RecordDecoder, under both emptyToNull settings --SKIPIF-- --FILE-- @@ -19,7 +19,7 @@ foreach ([true, false] as $emptyToNull) { [$headers, $rows] = csv_native_rows($raw, ',', '"', '\\'); var_dump($headers); -var_dump($rows[0]); +var_dump($rows); ?> --EXPECT-- emptyToNull=1 withHeader=1: identical @@ -38,13 +38,5 @@ array(5) { [4]=> string(2) "id" } -array(4) { - ["id"]=> - NULL - ["name"]=> - NULL - ["e02"]=> - NULL - [5]=> - NULL +array(0) { } diff --git a/src/extension/flow-php-ext/tests/phpt/064_csv_next_columns_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/064_csv_next_columns_no_leaks.phpt index 96bb712070..0a8e11c253 100644 --- a/src/extension/flow-php-ext/tests/phpt/064_csv_next_columns_no_leaks.phpt +++ b/src/extension/flow-php-ext/tests/phpt/064_csv_next_columns_no_leaks.phpt @@ -20,12 +20,18 @@ $cycle = static function () use ($raw, $schema, $refusing): void { foreach ((new RustCSVOpenSource(new MemorySourceStream($raw), ',', '"', '\\', true, true, true, 5))->batches($schema, 2, new RustBackend()) as $batch) { } + foreach ((new RustCSVOpenSource(new MemorySourceStream("name\nx\n"), ',', '"', '\\', true, true, true))->batches($refusing, 2, new RustBackend()) as $batch) { + if (!$batch instanceof Flow\ETL\Foundation\Exception\CorruptedRecordException) { + throw new LogicException('a refused cell was accepted'); + } + } + try { - foreach ((new RustCSVOpenSource(new MemorySourceStream("name\nx\n"), ',', '"', '\\', true, true, true))->batches($refusing, 2, new RustBackend()) as $batch) { + foreach ((new RustCSVOpenSource(new MemorySourceStream("name\nx\n"), ',', '"', '\\', true, true, true))->batches($refusing, 2, new RustBackend(), 1) as $batch) { } - throw new LogicException('a refused cell was accepted'); - } catch (Flow\ETL\Foundation\Exception\SchemaMismatchException) { + throw new LogicException('a refused cell past the sample was accepted'); + } catch (Flow\ETL\Foundation\Exception\InferredSchemaException) { } }; diff --git a/src/extension/flow-php-ext/tests/phpt/086_json_read_parity.phpt b/src/extension/flow-php-ext/tests/phpt/086_json_read_parity.phpt index 207c9531c2..72a5dd0741 100644 --- a/src/extension/flow-php-ext/tests/phpt/086_json_read_parity.phpt +++ b/src/extension/flow-php-ext/tests/phpt/086_json_read_parity.phpt @@ -10,6 +10,7 @@ use Flow\ETL\Adapter\JSON\IO\Extractor\JsonFileReader; use Flow\ETL\Adapter\JSON\IO\JsonFraming; use Flow\ETL\Adapter\JSON\IO\Extractor\PhpJsonOpenSource; use Flow\ETL\Adapter\JSON\IO\Extractor\RustJsonOpenSource; +use Flow\ETL\Foundation\Exception\CorruptedRecordException; use Flow\ETL\Values\Column\{RustBackend, RustColumn, PhpBackend}; use Flow\ETL\IO\Extractor\File\SourceFile; use Flow\ETL\Values\{Rows, Schema}; @@ -117,6 +118,12 @@ $frames = static function (iterable $batches, object $backend): array { $frames = []; foreach ($batches as $batch) { + if ($batch instanceof CorruptedRecordException) { + $frames[] = 'corrupted ' . json_encode([$batch->uri, $batch->offset, $batch->lineNumber, $batch->reason, $batch->record]); + + continue; + } + foreach ($batch->columns() as $column) { if ($backend instanceof PhpBackend && $column instanceof RustColumn) { return ['a RustColumn survived PhpBackend']; @@ -192,7 +199,7 @@ for ($t = 0; $t < 400; $t++) { } $identical++; - $refused += refused($php) ? 1 : 0; + $refused += refused($php) || str_contains($php, 'corrupted ') ? 1 : 0; $nulls += str_contains($raw, 'null') ? 1 : 0; } } diff --git a/src/extension/flow-php-ext/tests/phpt/087_json_read_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/087_json_read_no_leaks.phpt index a0d47b3da1..76487be12a 100644 --- a/src/extension/flow-php-ext/tests/phpt/087_json_read_no_leaks.phpt +++ b/src/extension/flow-php-ext/tests/phpt/087_json_read_no_leaks.phpt @@ -8,7 +8,7 @@ require __DIR__ . '/bootstrap.php'; use Flow\ETL\Adapter\JSON\IO\Extractor\RustJsonOpenSource; use Flow\ETL\Values\Column\RustBackend; -use Flow\ETL\Foundation\Exception\{RuntimeException, SchemaMismatchException}; +use Flow\ETL\Foundation\Exception\{CorruptedRecordException, InferredSchemaException, RuntimeException}; use Flow\Filesystem\Stream\StringSourceStream; use function Flow\ETL\DSL\{int_schema, list_schema, map_schema, schema, str_schema, structure_schema}; @@ -27,20 +27,20 @@ $schema = schema( ])), ); $valid = '{"id":1,"name":"aé","tags":["x","y"],"scores":{"a":1,"a":2},"owner":{"id":1}}'; -$read = static function (string $raw, bool $lines) use ($backend, $schema): void { +$read = static function (string $raw, bool $lines, ?int $sampleSize = null) use ($backend, $schema): void { $open = new RustJsonOpenSource(new StringSourceStream(path('memory://leak.json'), $raw), $lines, 'memory://leak.json', '', 0); try { - foreach ($open->batches($schema, 2, $backend) as $batch) { - $batch->toArray(); + foreach ($open->batches($schema, 2, $backend, $sampleSize) as $batch) { + $batch instanceof CorruptedRecordException ? $batch->getMessage() : $batch->toArray(); } } finally { $open->close(); } }; -$refused = static function (string $raw, bool $lines, string $class) use ($read): void { +$refused = static function (string $raw, bool $lines, string $class, ?int $sampleSize = null) use ($read): void { try { - $read($raw, $lines); + $read($raw, $lines, $sampleSize); throw new LogicException('a refused read was accepted'); } catch (Throwable $e) { if (!$e instanceof $class) { @@ -52,8 +52,10 @@ $cycle = static function () use ($read, $refused, $valid): void { $read("[{$valid},{$valid},\n{$valid}]", false); $read("{$valid}\n{$valid}\n\n{$valid}", true); $read('[{"id":"2","tags":"[\"json\"]","scores":[],"owner":{"id":"3","note":null}}]', false); - $refused('[' . $valid . ',{"id":"x","tags":[],"scores":{},"owner":{"id":1}}]', false, SchemaMismatchException::class); - $refused('{"id":1,"tags":[1,[2]],"scores":{},"owner":{"note":"n"}}', true, SchemaMismatchException::class); + $read('[' . $valid . ',{"id":"x","tags":[],"scores":{},"owner":{"id":1}}]', false); + $read('{"id":1,"tags":[1,[2]],"scores":{},"owner":{"note":"n"}}', true); + $refused('[' . $valid . ',{"id":"x","tags":[],"scores":{},"owner":{"id":1}}]', false, InferredSchemaException::class, 1); + $refused($valid . "\n" . '{"id":1,"tags":[],"scores":{},"owner":{"id":1},"c":2}', true, InferredSchemaException::class, 1); $refused($valid . "\n" . $valid . '}', true, RuntimeException::class); $refused('[' . $valid . ',' . $valid, false, RuntimeException::class); $refused($valid . "\n5", true, RuntimeException::class); diff --git a/src/extension/flow-php-ext/tests/phpt/bootstrap.php b/src/extension/flow-php-ext/tests/phpt/bootstrap.php index b6f52c41d8..9d040a048e 100644 --- a/src/extension/flow-php-ext/tests/phpt/bootstrap.php +++ b/src/extension/flow-php-ext/tests/phpt/bootstrap.php @@ -400,7 +400,9 @@ function csv_php_rows( foreach ($lines->readLines( new Flow\Filesystem\Stream\StringSourceStream(Flow\Filesystem\DSL\path('memory://phpt.csv'), $raw), ) as $line) { - foreach ($decoder->decode([$line]) as $values) { + $values = $decoder->decode($decoder->cells($line)); + + if (is_array($values)) { $rows[] = $values; } } @@ -647,8 +649,8 @@ function csv_narrow_corpus(): array } /** - * RustCSVOpenSource::sniff() of one CSV column "v" (no names seeded), one row per value - a null value is an empty cell, - * read as null with $emptyToNull. + * RustCSVOpenSource::sniff() of one CSV column "v" (no names seeded), one row per value - a null value is a quoted empty + * cell (a blank line is no record), read as null with $emptyToNull. * * @param list $values */ @@ -657,7 +659,7 @@ function rust_sniff_column(array $values, Flow\Types\Type\TypeNarrower $typer, b $csv = "v\n"; foreach ($values as $value) { - $csv .= ($value === null ? '' : '"' . str_replace('"', '""', $value) . '"') . "\n"; + $csv .= '"' . str_replace('"', '""', $value ?? '') . '"' . "\n"; } return (new RustCSVOpenSource( diff --git a/web/landing/content/examples/topics/reading/corrupted_records_routing/_meta.yaml b/web/landing/content/examples/topics/reading/corrupted_records_routing/_meta.yaml new file mode 100644 index 0000000000..6d6ded0400 --- /dev/null +++ b/web/landing/content/examples/topics/reading/corrupted_records_routing/_meta.yaml @@ -0,0 +1,2 @@ +priority: 16 +hidden: false diff --git a/web/landing/content/examples/topics/reading/corrupted_records_routing/code.php b/web/landing/content/examples/topics/reading/corrupted_records_routing/code.php new file mode 100644 index 0000000000..f691a8cae1 --- /dev/null +++ b/web/landing/content/examples/topics/reading/corrupted_records_routing/code.php @@ -0,0 +1,26 @@ +read(from_csv(__DIR__ . '/orders.csv') + ->withSchema(schema(int_schema('id'), float_schema('amount'))) + ->withCorruptedRecords(to_json_lines(__DIR__ . '/rejects.jsonl')->saveMode(overwrite()))) + ->write(to_output()) + ->run(); + +data_frame() + ->read(from_json_lines(__DIR__ . '/rejects.jsonl')) + ->groupBy(ref('reason')) + ->aggregate(count(ref('offset'))) + ->write(to_output(truncate: false)) + ->run(); diff --git a/web/landing/content/examples/topics/reading/corrupted_records_routing/description.md b/web/landing/content/examples/topics/reading/corrupted_records_routing/description.md new file mode 100644 index 0000000000..383619cfe3 --- /dev/null +++ b/web/landing/content/examples/topics/reading/corrupted_records_routing/description.md @@ -0,0 +1,2 @@ +`withCorruptedRecords()` sends every corrupted record to a loader as a row (`uri`, `offset`, `line`, `reason`, +`record`, `fields`) and the read goes on. The clean rows are written, the rejects are read back and grouped by reason. diff --git a/web/landing/content/examples/topics/reading/corrupted_records_routing/documentation.md b/web/landing/content/examples/topics/reading/corrupted_records_routing/documentation.md new file mode 100644 index 0000000000..24114b74bb --- /dev/null +++ b/web/landing/content/examples/topics/reading/corrupted_records_routing/documentation.md @@ -0,0 +1,4 @@ +A routed record is one row with the same schema for every reader, so rejects can be reported and repaired in Flow. + +- [Corrupted Records](/documentation/components/core/corrupted-records) +- [JSON Adapter](/documentation/components/adapters/json) diff --git a/web/landing/content/examples/topics/reading/csv_cells/code.php b/web/landing/content/examples/topics/reading/csv_cells/code.php index 718fe9d18b..6a4b85b202 100644 --- a/web/landing/content/examples/topics/reading/csv_cells/code.php +++ b/web/landing/content/examples/topics/reading/csv_cells/code.php @@ -11,6 +11,6 @@ file_put_contents(__DIR__ . '/input.csv', " id ,,name\n1,a,Norbert\n2,,Jane\n3\n"); // json_encode() keeps an empty string and a null apart, a table would not -foreach (data_frame()->read(from_csv(__DIR__ . '/input.csv', empty_to_null: false))->fetch()->toArray() as $row) { +foreach (data_frame()->read(from_csv(__DIR__ . '/input.csv', empty_to_null: false)->withMissingColumnsAsNull(true))->fetch()->toArray() as $row) { echo json_encode($row) . "\n"; } diff --git a/web/landing/content/examples/topics/reading/csv_cells/description.md b/web/landing/content/examples/topics/reading/csv_cells/description.md index 07037ecc72..2a7d88e29a 100644 --- a/web/landing/content/examples/topics/reading/csv_cells/description.md +++ b/web/landing/content/examples/topics/reading/csv_cells/description.md @@ -1,2 +1,3 @@ -Header names are trimmed, and a blank header cell is named `e` + its position (`e01`). A cell missing -from a short line is `null`; `empty_to_null` only decides what a present empty cell becomes. +Header names are trimmed, and a blank header cell is named `e` + its position (`e01`). A short line is padded with +`null` under `withMissingColumnsAsNull(true)`, else it is a corrupted record; `empty_to_null` only decides what a present +empty cell becomes. diff --git a/web/landing/templates/documentation/navigation_left.html.twig b/web/landing/templates/documentation/navigation_left.html.twig index c169d62b77..5bccfdba86 100644 --- a/web/landing/templates/documentation/navigation_left.html.twig +++ b/web/landing/templates/documentation/navigation_left.html.twig @@ -70,6 +70,7 @@
  • {{ nav.nav_link('/documentation/components/core/schema', 'Schema', currentPath) }}
  • {{ nav.nav_link('/documentation/components/core/display', 'Display', currentPath) }}
  • {{ nav.nav_link('/documentation/components/core/error-handling', 'Error Handling', currentPath) }}
    • +
  • {{ nav.nav_link('/documentation/components/core/corrupted-records', 'Corrupted Records', currentPath) }}
    • {{ nav.nav_link('/documentation/components/cli/docs', 'CLI', currentPath) }}