From be4d07d6259910b9a8e72149389daf9d178a6a58 Mon Sep 17 00:00:00 2001 From: Norbert Orzechowicz Date: Fri, 30 Oct 2026 15:00:19 -0500 Subject: [PATCH 1/2] refactor(flow-php/etl): splittable sources and the etl layer model - splittable file sources, one schema inference contract, a launcher seam on Config - every namespace sits in one layer root (Foundation, Values, Storage, Kernels, Functions, IO, Operators, Planning, Runtime, API), guarded per layer, 0 upward dependencies - adapters follow the same layers (IO\Extractor, IO\Loader, IO\Inference, Values\Schema, Foundation\Exception) - one mechanism per operation: branch vocabulary, FileWriter, one FileCommit protocol, NestedLoader, Launch, BucketingAlgorithmBuilder - correctness: one count semantic, stateful steps never split, limit stop without an exception, readers skip hidden paths - tests mirror the layers, shared contracts run per package, new Architecture documentation page --- .env.dist | 2 +- .../rust-extension-toolchain/action.yml | 1 + .github/workflows/job-arrow-extension.yml | 12 + .github/workflows/job-flow-php-extension.yml | 14 +- .github/workflows/job-tests.yml | 4 +- .nix/php/lib/benchmark.ini.dist | 9 + .nix/pkgs/flow-php/package.nix | 15 + benchmarks/reference/duckdb/bench.sh | 25 + benchmarks/reference/polars/bench.py | 43 + benchmarks/reference/spark/bench.py | 43 + benchmarks/src/BenchmarkConfig.php | 2 +- benchmarks/src/Datasets/FixtureFormat.php | 2 +- benchmarks/src/Joining/JoinEachScenario.php | 2 +- .../Joining/JoinOrdersWithSellersScenario.php | 4 +- .../src/Joining/SellersDataFrameFactory.php | 24 - .../src/Joining/SellersExtractorFactory.php | 23 + .../src/Partitioning/PartitionCardinality.php | 4 +- .../Partitioning/PartitionedReadScenario.php | 2 +- benchmarks/src/Pipeline/CanonicalPipeline.php | 18 +- benchmarks/src/Pipeline/FetchScenario.php | 2 +- benchmarks/src/Pipeline/InMemoryOrders.php | 2 +- .../src/Pipeline/NodePipelineScenario.php | 2 +- benchmarks/src/Pipeline/OrdersSchema.php | 2 +- benchmarks/src/Pipeline/PlanDepthScenario.php | 2 +- benchmarks/src/Pipeline/ReadScenario.php | 2 +- benchmarks/src/Pipeline/SourceExtractor.php | 4 +- benchmarks/src/Pipeline/StageScenario.php | 2 +- .../src/Schema/FooterSchemaScenario.php | 2 +- benchmarks/src/Schema/RowsSchemaScenario.php | 2 +- .../src/Schema/SchemaInferenceScenario.php | 2 +- .../NestedTransformationScenario.php | 8 +- benchmarks/src/Transformation/NoopLoader.php | 22 +- .../src/Transformation/SortByCreatedAt.php | 4 +- benchmarks/src/Window/WindowPartitioning.php | 4 +- benchmarks/suites/Joining/JoinBench.php | 10 +- .../NestedTransformationBench.php | 16 +- .../Pipeline/NodePipelineScenarioTest.php | 2 +- .../Tests/Unit/Pipeline/OrdersSchemaTest.php | 4 +- bin/docs.php | 6 +- compose.yml.dist | 2 +- composer.json | 2 + composer.lock | 4 +- documentation/adrs/extension-points.md | 4 +- documentation/architecture.md | 205 ++ documentation/components/adapters/doctrine.md | 24 +- documentation/components/adapters/excel.md | 17 +- documentation/components/adapters/http.md | 6 +- documentation/components/adapters/logger.md | 2 +- .../components/adapters/postgresql.md | 44 +- documentation/components/adapters/seal.md | 2 +- documentation/components/adapters/xml.md | 2 +- .../bridges/filesystem-async-aws-bridge.md | 15 +- .../bridges/filesystem-azure-bridge.md | 18 + .../bridges/filesystem-sftp-bridge.md | 20 +- .../bridges/openapi-specification-bridge.md | 2 +- .../bridges/symfony-http-foundation-bridge.md | 55 +- .../bridges/symfony-postgresql-bundle.md | 17 +- documentation/components/cli/docs.md | 28 +- documentation/components/core/aggregations.md | 101 +- .../components/core/batch-processing.md | 13 +- .../components/core/building-blocks.md | 48 +- documentation/components/core/caching.md | 79 +- .../components/core/column-backend.md | 18 +- documentation/components/core/constraints.md | 3 +- documentation/components/core/core.md | 115 +- .../components/core/data-retrieval.md | 2 +- .../components/core/error-handling.md | 21 +- documentation/components/core/filter.md | 29 +- documentation/components/core/floe.md | 52 +- documentation/components/core/group-by.md | 12 +- documentation/components/core/join.md | 39 +- documentation/components/core/limit.md | 12 +- documentation/components/core/partitioning.md | 19 +- documentation/components/core/save-mode.md | 30 +- documentation/components/core/schema.md | 10 +- documentation/components/core/sort.md | 44 +- documentation/components/core/statistics.md | 83 +- documentation/components/core/telemetry.md | 22 +- .../components/core/transformations.md | 62 +- .../components/core/window-functions.md | 16 +- .../components/extensions/flow-php-ext.md | 42 +- documentation/components/libs/parquet.md | 10 +- .../libs/postgresql/index-query-builder.md | 7 +- .../components/libs/telemetry/contracts.md | 58 +- documentation/contributing/benchmarks.md | 47 +- documentation/contributing/nix.md | 41 + documentation/contributing/rust.md | 62 +- documentation/contributing/wasm.md | 4 +- documentation/installation/docker.md | 4 +- documentation/introduction.md | 4 +- documentation/quick-start.md | 6 +- documentation/upgrading.md | 1980 ++++++++++++- mago.toml | 442 ++- phpunit.xml.dist | 8 +- shell.nix | 17 +- .../Adapter/Avro/FlixTech/AvroExtractor.php | 80 - .../ETL/Adapter/Avro/FlixTech/AvroLoader.php | 51 - .../Avro/IO/Extractor/AvroExtractor.php | 87 + .../ETL/Adapter/Avro/IO/Loader/AvroLoader.php | 46 + .../src/Flow/ETL/Adapter/Avro/functions.php | 6 +- .../Avro/Tests/Integration/AvroTest.php | 5 +- .../Extractor}/AvroExtractorTest.php | 4 +- .../ETL/Adapter/ChartJS/ChartJSLoader.php | 147 - .../Adapter/ChartJS/{ => IO/Loader}/Chart.php | 6 +- .../{ => IO/Loader}/Chart/BarChart.php | 14 +- .../{ => IO/Loader}/Chart/LineChart.php | 14 +- .../{ => IO/Loader}/Chart/PieChart.php | 10 +- .../ChartJS/IO/Loader/ChartJSLoader.php | 134 + .../Flow/ETL/Adapter/ChartJS/functions.php | 12 +- .../Tests/Context/DescribableFixtures.php | 53 + .../{ => IO/Loader}/ChartJSLoaderTest.php | 12 +- .../ChartJS/Tests/Unit/Chart/BarChartTest.php | 138 - .../Tests/Unit/DescribableRoundTripTest.php | 75 + .../Unit/IO/Loader/Chart/BarChartTest.php | 138 + .../{ => IO/Loader}/Chart/LineChartTest.php | 2 +- .../{ => IO/Loader}/Chart/PieChartTest.php | 2 +- .../ETL/Adapter/CSV/AdaptiveCSVEncoder.php | 40 - .../ETL/Adapter/CSV/AdaptiveCSVOpenSource.php | 114 - .../src/Flow/ETL/Adapter/CSV/CSVDialect.php | 14 - .../src/Flow/ETL/Adapter/CSV/CSVExtractor.php | 300 -- .../Flow/ETL/Adapter/CSV/CSVFileBatches.php | 50 - .../Flow/ETL/Adapter/CSV/CSVFileReader.php | 82 - .../Flow/ETL/Adapter/CSV/CSVFileSample.php | 76 - .../src/Flow/ETL/Adapter/CSV/CSVHeader.php | 19 - .../src/Flow/ETL/Adapter/CSV/CSVLoader.php | 188 -- .../src/Flow/ETL/Adapter/CSV/CSVOpenSink.php | 40 - .../Flow/ETL/Adapter/CSV/CSVOpenSource.php | 80 - .../Flow/ETL/Adapter/CSV/CSVReadOptions.php | 131 - .../Flow/ETL/Adapter/CSV/CSVSampledFiles.php | 44 - .../Flow/ETL/Adapter/CSV/CSVSampledRows.php | 40 - .../src/Flow/ETL/Adapter/CSV/CSVSamples.php | 63 - .../Flow/ETL/Adapter/CSV/CSVWriteOptions.php | 34 - .../Flow/ETL/Adapter/CSV/Detector/Options.php | 109 - .../CSV/Exception/CantDetectCSVOptions.php | 9 - .../Exception/CantDetectCSVOptions.php | 9 + .../Flow/ETL/Adapter/CSV/IO/CSVDialect.php | 64 + .../CSV/{ => IO/Extractor}/CSVDecoder.php | 10 +- .../CSV/{ => IO/Extractor}/CSVDetector.php | 10 +- .../Adapter/CSV/IO/Extractor/CSVDialects.php | 52 + .../{ => IO/Extractor}/CSVEnclosureScan.php | 2 +- .../Adapter/CSV/IO/Extractor/CSVExtractor.php | 356 +++ .../CSV/IO/Extractor/CSVFileBatches.php | 84 + .../CSV/{ => IO/Extractor}/CSVLineReader.php | 2 +- .../CSV/IO/Extractor/CSVOpenSource.php | 79 + .../CSV/IO/Extractor/CSVReadOptions.php | 165 ++ .../{ => IO/Extractor}/CSVRecordBoundary.php | 2 +- .../CSV/IO/Extractor/CSVRecordResync.php | 219 ++ .../CSV/IO/Extractor/CSVSourceOpener.php | 229 ++ .../{ => IO/Extractor}/Detector/Option.php | 16 +- .../CSV/IO/Extractor/Detector/Options.php | 109 + .../CSV/IO/Extractor/PhpCSVOpenSource.php | 114 + .../Adapter/CSV/IO/Inference/CSVSniffer.php | 170 ++ .../CSV/{ => IO/Loader}/CSVEncoder.php | 4 +- .../Adapter/CSV/IO/Loader/CSVFileWriter.php | 40 + .../ETL/Adapter/CSV/IO/Loader/CSVLoader.php | 286 ++ .../Adapter/CSV/IO/Loader/CSVWriteOptions.php | 22 + .../CSV/{ => IO/Loader}/PhpCSVEncoder.php | 12 +- .../Flow/ETL/Adapter/CSV/PhpCSVOpenSource.php | 121 - .../src/Flow/ETL/Adapter/CSV/functions.php | 9 +- .../CSV/Tests/Context/CSVFileFormat.php | 33 + .../CSV/Tests/Context/CSVFixtureContext.php | 150 +- .../CSV/Tests/Context/DescribableFixtures.php | 80 + .../Double/LengthCapturingFilesystem.php | 23 + .../Integration/AdaptiveCSVOpenSourceTest.php | 176 -- .../Tests/Integration/CSVExtractorTest.php | 1372 --------- .../Tests/Integration/CSVFileReaderTest.php | 177 -- .../Tests/Integration/CSVFileSampleTest.php | 127 - .../Tests/Integration/CSVLineReaderTest.php | 107 - .../CSV/Tests/Integration/CSVSamplesTest.php | 68 - .../Adapter/CSV/Tests/Integration/CSVTest.php | 18 +- .../{ => IO/Extractor}/CSVDetectorTest.php | 22 +- .../CSVExtractorBatchContractTest.php | 4 +- .../IO/Extractor/CSVExtractorTest.php | 1793 ++++++++++++ .../CSVExtractorTypedColumnsTest.php | 17 +- .../IO/Extractor/CSVLineReaderTest.php | 105 + .../IO/Extractor/CSVSourceOpenerTest.php | 417 +++ .../IO/Extractor/CSVSplitReadTest.php | 121 + .../Extractor}/PhpCSVOpenSourceTest.php | 19 +- .../Extractor}/RustCSVOpenSourceTest.php | 76 +- .../Integration/IO/FileFormatContractTest.php | 205 ++ .../IO/Inference/CSVSnifferTest.php | 301 ++ .../CSV/Tests/Mother/CSVEncoderMother.php | 16 +- .../CSV/Tests/Unit/AdaptiveCSVEncoderTest.php | 67 - .../Adapter/CSV/Tests/Unit/CSVDialectTest.php | 33 - .../CSV/Tests/Unit/CSVExtractorTest.php | 210 -- .../CSV/Tests/Unit/CSVFileBatchesTest.php | 64 - .../Adapter/CSV/Tests/Unit/CSVHeaderTest.php | 27 - .../CSV/Tests/Unit/CSVLineReaderTest.php | 422 --- .../Adapter/CSV/Tests/Unit/CSVLoaderTest.php | 158 -- .../CSV/Tests/Unit/CSVOpenSinkTest.php | 47 - .../CSV/Tests/Unit/CSVSampledFilesTest.php | 48 - .../CSV/Tests/Unit/CSVSampledRowsTest.php | 34 - .../Tests/Unit/DescribableRoundTripTest.php | 75 + .../CSV/Tests/Unit/Detector/OptionTest.php | 37 - .../CSV/Tests/Unit/IO/CSVDialectTest.php | 45 + .../{ => IO/Extractor}/CSVDecoderTest.php | 4 +- .../Unit/IO/Extractor/CSVDialectsTest.php | 45 + .../Extractor}/CSVEnclosureScanTest.php | 4 +- .../Unit/IO/Extractor/CSVExtractorTest.php | 215 ++ .../Unit/IO/Extractor/CSVFileBatchesTest.php | 157 ++ .../Unit/IO/Extractor/CSVLineReaderTest.php | 422 +++ .../{ => IO/Extractor}/CSVReadOptionsTest.php | 26 +- .../Extractor}/CSVRecordBoundaryTest.php | 4 +- .../Unit/IO/Extractor/CSVRecordResyncTest.php | 223 ++ .../Unit/IO/Extractor/Detector/OptionTest.php | 62 + .../MetadataColumnsExtractorTest.php | 42 + .../Unit/IO/Loader/CSVFileWriterTest.php | 47 + .../Tests/Unit/IO/Loader/CSVLoaderTest.php | 339 +++ .../{ => IO/Loader}/CSVWriteOptionsTest.php | 6 +- .../{ => IO/Loader}/PhpCSVEncoderTest.php | 8 +- .../{ => IO/Loader}/RustCSVEncoderTest.php | 8 +- .../Adapter/Doctrine/DbalDataFrameFactory.php | 104 - .../Doctrine/DbalLimitOffsetExtractor.php | 238 -- .../Flow/ETL/Adapter/Doctrine/DbalLoader.php | 160 -- .../Doctrine/Explain/ExplainedRows.php | 64 - .../Exception}/ProbeRefusal.php | 2 +- .../IO/Extractor/DbalExtractorFactory.php | 102 + .../Extractor}/DbalKeySetExtractor.php | 71 +- .../IO/Extractor/DbalLimitOffsetExtractor.php | 238 ++ .../{ => IO/Extractor}/DbalQueryExtractor.php | 62 +- .../{ => IO/Extractor}/DbalResultSchema.php | 7 +- .../{ => IO/Extractor}/DescribeQuery.php | 2 +- .../IO/Extractor/Explain/ExplainedRows.php | 60 + .../Extractor}/Explain/MySqlExplainedRows.php | 2 +- .../Explain/PostgreSqlExplainedRows.php | 2 +- .../{ => IO/Extractor}/LiteralParameter.php | 4 +- .../{ => IO/Extractor}/MysqliPlaceholders.php | 2 +- .../Extractor}/MysqliResultColumns.php | 5 +- .../{ => IO/Extractor}/MysqliTypesMap.php | 2 +- .../{ => IO/Extractor}/NativePlaceholders.php | 2 +- .../{ => IO/Extractor}/NativeTypes.php | 2 +- .../Adapter/Doctrine/IO/Extractor/Order.php | 11 + .../Doctrine/{ => IO/Extractor}/OrderBy.php | 2 +- .../Doctrine/{ => IO/Extractor}/Pages.php | 2 +- .../Doctrine/IO/Extractor/Pagination/Key.php | 27 + .../IO/Extractor/Pagination/KeySet.php | 32 + .../IO/Extractor/Pagination/KeySetPage.php | 64 + .../IO/Extractor/Pagination/KeyValues.php | 99 + .../IO/Extractor/Pagination/Order.php | 11 + .../Doctrine/IO/Extractor/Parameter.php | 62 + .../{ => IO/Extractor}/ParametersSet.php | 2 +- .../Extractor}/PdoSqliteColumnNames.php | 3 +- .../{ => IO/Extractor}/PgSqlResultColumns.php | 3 +- .../{ => IO/Extractor}/PgSqlTypesMap.php | 2 +- .../{ => IO/Extractor}/PlaceholderDialect.php | 2 +- .../Extractor}/PlaceholderRewriter.php | 2 +- .../Extractor}/PostgreSqlPlaceholders.php | 2 +- .../{ => IO/Extractor}/QueryParameter.php | 4 +- .../Doctrine/IO/Extractor/ResultColumn.php | 16 + .../{ => IO/Extractor}/RewrittenSql.php | 2 +- .../{ => IO/Extractor}/Sqlite3ColumnNames.php | 3 +- .../{ => IO/Extractor}/SqliteResultSchema.php | 7 +- .../{ => IO/Extractor}/SqliteTypes.php | 2 +- .../Doctrine/{ => IO/Extractor}/Table.php | 2 +- .../{ => IO/Extractor}/TypedColumns.php | 17 +- .../Doctrine/{ => IO/Loader}/DbalEncoder.php | 10 +- .../Adapter/Doctrine/IO/Loader/DbalLoader.php | 194 ++ .../{ => IO/Loader}/DbalTransaction.php | 4 +- .../{ => IO/Loader}/IsolationLevelRestore.php | 2 +- .../src/Flow/ETL/Adapter/Doctrine/Order.php | 11 - .../ETL/Adapter/Doctrine/Pagination/Key.php | 27 - .../Adapter/Doctrine/Pagination/KeySet.php | 32 - .../Doctrine/Pagination/KeySetPage.php | 64 - .../Adapter/Doctrine/Pagination/KeyValues.php | 99 - .../ETL/Adapter/Doctrine/Pagination/Order.php | 11 - .../Flow/ETL/Adapter/Doctrine/Parameter.php | 62 - .../ETL/Adapter/Doctrine/ResultColumn.php | 16 - .../ETL/Adapter/Doctrine/SchemaConverter.php | 331 --- .../Flow/ETL/Adapter/Doctrine/TypesMap.php | 190 -- .../{ => Values/Schema}/DbalMetadata.php | 4 +- .../Values/Schema/SchemaConverter.php | 331 +++ .../Doctrine/Values/Schema/TypesMap.php | 190 ++ .../Flow/ETL/Adapter/Doctrine/functions.php | 59 +- .../Tests/Context/DatabaseContext.php | 3 + .../Tests/Context/DescribableFixtures.php | 44 + .../Doctrine/Tests/Context/KeySetSqlite.php | 2 +- .../{Context => Double}/CommitCounter.php | 2 +- .../InsertQueryCounter.php | 2 +- .../SelectQueryCounter.php | 2 +- .../Tests/Double/TransactionSpyLoader.php | 26 +- .../Integration/DbalDataFrameFactoryTest.php | 175 -- .../Integration/DbalKeySetExtractorTest.php | 116 - .../DbalLimitOffsetExtractorTest.php | 94 - .../Tests/Integration/DbalLoaderTest.php | 107 - .../MySQLDbalLimitOffsetExtractorTest.php | 22 +- .../Dialects/MySQLDbalLoaderTest.php | 38 +- .../Dialects/MySQLDbalQueryExtractorTest.php | 22 +- .../Dialects/MySQLKeySetExtractorTest.php | 154 +- .../Dialects/MySQLSchemaDerivationTest.php | 26 +- .../Dialects/MySQLTransactionSinkTest.php | 201 -- .../Dialects/MySQLTransactionalTest.php | 201 ++ .../PosgreSQLDbalLimitOffsetExtractorTest.php | 185 +- .../Dialects/PostgreSQLDbalLoaderTest.php | 44 +- .../PostgreSQLDbalQueryExtractorTest.php | 66 +- .../PostgreSQLKeySetExtractorTest.php | 185 +- .../PostgreSQLSchemaDerivationTest.php | 26 +- .../PostgreSQLTransactionSinkTest.php | 201 -- .../Dialects/PostgreSQLTransactionalTest.php | 201 ++ .../SqliteDbalLimitOffsetExtractorTest.php | 57 +- .../Dialects/SqliteDbalLoaderTest.php | 44 +- .../Dialects/SqliteDbalQueryExtractorTest.php | 22 +- .../Dialects/SqliteKeySetExtractorTest.php | 154 +- .../Dialects/SqliteSchemaDerivationTest.php | 7 +- .../Dialects/SqliteTransactionSinkTest.php | 201 -- .../Dialects/SqliteTransactionalTest.php | 201 ++ .../Integration/Explain/ExplainedRowsTest.php | 77 - .../IO/Extractor/DbalExtractorFactoryTest.php | 173 ++ .../IO/Extractor/DbalKeySetExtractorTest.php | 116 + .../DbalLimitOffsetExtractorTest.php | 94 + .../Extractor/Explain/ExplainedRowsTest.php | 55 + .../Explain/MySqlExplainedRowsTest.php | 4 +- .../Explain/PostgreSqlExplainedRowsTest.php | 4 +- .../Integration/IO/Loader/DbalLoaderTest.php | 127 + .../Doctrine/Tests/IntegrationTestCase.php | 6 +- .../Tests/Unit/DbalKeySetExtractorTest.php | 418 --- .../Unit/DbalLimitOffsetExtractorTest.php | 310 -- .../Tests/Unit/DbalTransactionSinkTest.php | 203 -- .../Tests/Unit/DescribableRoundTripTest.php | 75 + .../Exception}/ProbeRefusalTest.php | 4 +- .../DbalExtractorBatchContractTest.php | 8 +- .../IO/Extractor/DbalKeySetExtractorTest.php | 412 +++ .../DbalLimitOffsetExtractorTest.php | 300 ++ .../Extractor}/DbalQueryExtractorTest.php | 52 +- .../Extractor}/DbalResultSchemaTest.php | 15 +- .../{ => IO/Extractor}/DescribeQueryTest.php | 4 +- .../MetadataColumnsExtractorTest.php | 42 + .../{ => IO/Extractor}/MysqliTypesMapTest.php | 4 +- .../Extractor}/NativePlaceholdersTest.php | 6 +- .../Unit/{ => IO/Extractor}/PagesTest.php | 4 +- .../Extractor/Pagination/KeySetPageTest.php | 113 + .../Extractor}/Pagination/KeySetTest.php | 6 +- .../IO/Extractor/Pagination/KeyValuesTest.php | 82 + .../{ => IO/Extractor}/PgSqlTypesMapTest.php | 4 +- .../Extractor}/SqliteResultSchemaTest.php | 21 +- .../{ => IO/Extractor}/TypedColumnsTest.php | 16 +- .../Unit/{ => IO/Loader}/DbalEncoderTest.php | 4 +- .../Tests/Unit/IO/Loader/DbalLoaderTest.php | 39 + .../{ => IO/Loader}/DbalTransactionTest.php | 4 +- .../Unit/IO/Loader/DbalTransactionalTest.php | 203 ++ .../Loader}/IsolationLevelRestoreTest.php | 4 +- .../Tests/Unit/Pagination/KeySetPageTest.php | 113 - .../Tests/Unit/Pagination/KeyValuesTest.php | 82 - .../Tests/Unit/SchemaConverterTest.php | 300 -- .../Doctrine/Tests/Unit/TypesMapTest.php | 371 --- .../{ => Values/Schema}/DbalMetadataTest.php | 6 +- .../Values/Schema/SchemaConverterTest.php | 300 ++ .../Tests/Unit/Values/Schema/TypesMapTest.php | 371 +++ src/adapter/etl-adapter-excel/composer.json | 2 + .../Flow/ETL/Adapter/Excel/DSL/functions.php | 8 +- .../Flow/ETL/Adapter/Excel/ExcelExtractor.php | 318 --- .../ETL/Adapter/Excel/ExcelFileBatches.php | 90 - .../Flow/ETL/Adapter/Excel/ExcelHeader.php | 19 - .../Flow/ETL/Adapter/Excel/ExcelLoader.php | 367 --- .../ETL/Adapter/Excel/ExcelReadOptions.php | 82 - .../Flow/ETL/Adapter/Excel/ExcelReader.php | 12 - .../Flow/ETL/Adapter/Excel/ExcelWriter.php | 11 - .../Excel/Function/IsValidExcelSheetName.php | 68 - .../Excel/Functions/IsValidExcelSheetName.php | 101 + .../SheetNameAssertion.php | 4 +- .../Excel/IO/Extractor/ExcelExtractor.php | 369 +++ .../Excel/IO/Extractor/ExcelFileBatches.php | 109 + .../Extractor}/ExcelFormatDetector.php | 9 +- .../Excel/IO/Extractor/ExcelReadOptions.php | 93 + .../Excel/IO/Extractor/ExcelReader.php | 12 + .../{ => IO/Extractor}/Sheet/OpenSheet.php | 4 +- .../{ => IO/Extractor}/Sheet/SheetCells.php | 7 +- .../Extractor}/Sheet/SheetsManager.php | 5 +- .../Extractor}/Sheet/XlsxSheetRows.php | 2 +- .../Excel/IO/Extractor/WorkbookReader.php | 87 + .../{ => IO/Extractor}/WorkbookSheet.php | 4 +- .../{ => IO/Inference}/CellTypeNarrower.php | 4 +- .../Excel/IO/Inference/ExcelSniffer.php | 169 ++ .../Inference}/WorkbookSheetSample.php | 17 +- .../Excel/{ => IO/Loader}/CellStyler.php | 4 +- .../Excel/{ => IO/Loader}/ExcelEncoder.php | 16 +- .../Adapter/Excel/IO/Loader/ExcelLoader.php | 519 ++++ .../Adapter/Excel/IO/Loader/ExcelWriter.php | 11 + .../Excel/{ => IO/Loader}/WorkbookManager.php | 2 +- .../Flow/ETL/Adapter/Excel/WorkbookReader.php | 53 - .../ETL/Adapter/Excel/WorkbookSampler.php | 107 - .../Tests/Context/DescribableFixtures.php | 80 + .../Excel/Tests/Context/ExcelFileFormat.php | 33 + .../Tests/Context/ExcelFixtureContext.php | 66 +- .../Excel/Tests/Context/OpenSheetCells.php | 2 +- .../Tests/Double/RecordingCellStyler.php | 4 +- .../Tests/Integration/ExcelExtractorTest.php | 1036 ------- .../ExcelExtractorTypedColumnsTest.php | 46 - .../Integration/ExcelFormatDetectorTest.php | 67 - .../Tests/Integration/ExcelLoaderTest.php | 635 ----- .../ExcelExtractorBatchContractTest.php | 4 +- .../IO/Extractor/ExcelExtractorTest.php | 1127 ++++++++ .../ExcelExtractorTypedColumnsTest.php | 47 + .../IO/Extractor/ExcelFormatDetectorTest.php | 67 + .../IO/Extractor/ExcelSplitReadTest.php | 58 + .../Extractor}/Sheet/OpenSheetTest.php | 14 +- .../IO/Extractor/Sheet/SheetCellsTest.php | 30 + .../Extractor}/Sheet/XlsxSheetRowsTest.php | 4 +- .../IO/Extractor/WorkbookReaderTest.php | 78 + .../{ => IO/Extractor}/WorkbookSheetTest.php | 8 +- .../Integration/IO/FileFormatContractTest.php | 163 ++ .../IO/Inference/ExcelSnifferTest.php | 425 +++ .../Integration/IO/Loader/ExcelLoaderTest.php | 687 +++++ .../Integration/Sheet/SheetCellsTest.php | 30 - .../Tests/Integration/WorkbookReaderTest.php | 38 - .../Tests/Integration/WorkbookSamplerTest.php | 388 --- .../Tests/Unit/DescribableRoundTripTest.php | 75 + .../Excel/Tests/Unit/ExcelExtractorTest.php | 82 - .../Excel/Tests/Unit/ExcelFileBatchesTest.php | 102 - .../Tests/Unit/ExcelFormatDetectorTest.php | 27 - .../Excel/Tests/Unit/ExcelHeaderTest.php | 25 - .../Excel/Tests/Unit/ExcelReadOptionsTest.php | 80 - .../Functions/AllFunctionsContractTest.php | 121 + .../IsValidExcelSheetNameTest.php | 2 +- .../SheetNameAssertionTest.php | 6 +- .../Unit/IO/Extractor/ExcelExtractorTest.php | 82 + .../IO/Extractor/ExcelFileBatchesTest.php | 107 + .../IO/Extractor/ExcelFormatDetectorTest.php | 34 + .../IO/Extractor/ExcelReadOptionsTest.php | 82 + .../MetadataColumnsExtractorTest.php | 42 + .../IO/Extractor/Sheet/SheetCellsTest.php | 38 + .../Inference}/CellTypeNarrowerTest.php | 6 +- .../Inference}/WorkbookSheetSampleTest.php | 9 +- .../Unit/{ => IO/Loader}/ExcelEncoderTest.php | 4 +- .../Tests/Unit/IO/Loader/ExcelLoaderTest.php | 61 + .../Excel/Tests/Unit/Sheet/SheetCellsTest.php | 70 - .../Flow/ETL/Adapter/GoogleSheet/Columns.php | 45 - .../GoogleSheet/GoogleSheetExtractor.php | 287 -- .../GoogleSheet/GoogleSheetSampler.php | 87 - .../GoogleSheet/IO/Extractor/Columns.php | 45 + .../IO/Extractor/GoogleSheetExtractor.php | 309 ++ .../Extractor}/GoogleSheetReadOptions.php | 6 +- .../{ => IO/Extractor}/GoogleSheetReader.php | 3 +- .../{ => IO/Extractor}/SheetRange.php | 4 +- .../{ => IO/Inference}/GoogleSheetSample.php | 2 +- .../IO/Inference/GoogleSheetSniffer.php | 112 + .../ETL/Adapter/GoogleSheet/functions.php | 2 + .../Context/GoogleSheetFixtureContext.php | 35 +- .../{ => Context}/GoogleSheetsContext.php | 4 +- .../Integration/GoogleSheetExtractorTest.php | 393 --- .../IO/Extractor/GoogleSheetExtractorTest.php | 398 +++ .../GoogleSheetExtractorTypedColumnsTest.php | 19 +- .../GoogleSheet/Tests/Unit/ColumnsTest.php | 79 - .../Tests/Unit/GoogleSheetExtractorTest.php | 643 ----- .../Tests/Unit/GoogleSheetSamplerTest.php | 77 - .../Tests/Unit/IO/Extractor/ColumnsTest.php | 79 + .../GoogleSheetExtractorBatchContractTest.php | 4 +- .../IO/Extractor/GoogleSheetExtractorTest.php | 632 +++++ .../Extractor}/GoogleSheetReadOptionsTest.php | 8 +- .../Extractor}/GoogleSheetReaderTest.php | 6 +- .../MetadataColumnsExtractorTest.php | 42 + .../{ => IO/Extractor}/SheetRangeTest.php | 8 +- .../Inference}/GoogleSheetSampleTest.php | 4 +- .../IO/Inference/GoogleSheetSnifferTest.php | 73 + src/adapter/etl-adapter-http/composer.json | 1 + .../Flow/ETL/Adapter/Http/DSL/functions.php | 67 +- .../Extractor}/ContentTypeDetector.php | 2 +- .../Http/{ => IO/Extractor}/HttpDecoder.php | 4 +- .../Http/{ => IO/Extractor}/HttpExchange.php | 2 +- .../Extractor}/NextRequestFactory.php | 2 +- .../Extractor}/Pagination/DecodedResponse.php | 2 +- .../IO/Extractor/Pagination/InjectInto.php | 13 + .../Pagination/LinkHeaderParser.php | 2 +- .../Extractor}/Pagination/PageState.php | 2 +- .../Pagination/PaginationStrategy.php | 2 +- .../Extractor}/Pagination/Paginator.php | 4 +- .../Pagination/RelativeUriResolver.php | 2 +- .../Pagination/RequestComparator.php | 2 +- .../Extractor}/Pagination/RequestOption.php | 4 +- .../IO/Extractor/Pagination/ResultCounter.php | 10 + .../Http/IO/Extractor/Pagination/StopWhen.php | 10 + .../Extractor/Pagination/StopWhen/AllOf.php | 33 + .../Extractor/Pagination/StopWhen/AnyOf.php | 33 + .../Pagination/StopWhen/EmptyPath.php | 26 + .../Pagination/StopWhen/EmptyResponseBody.php | 19 + .../Pagination/StopWhen/FlagFalse.php | 26 + .../Pagination/StopWhen/FlagTrue.php | 26 + .../Pagination/StopWhen/MaxPages.php | 21 + .../Pagination/StopWhen/MaxResults.php | 21 + .../Pagination/StopWhen/NeverStop.php | 17 + .../Pagination/StopWhen/PathMissing.php | 27 + .../Pagination/StopWhen/TotalReached.php | 33 + .../Pagination/Strategy/CursorFromBody.php | 45 + .../Pagination/Strategy/LastRecordCursor.php | 12 +- .../Pagination/Strategy/LinkHeader.php | 39 + .../Pagination/Strategy/NextUrlFromBody.php | 45 + .../Pagination/Strategy/OffsetLimit.php | 47 + .../Pagination/Strategy/PageNumber.php | 12 +- .../PsrHttpClientDynamicExtractor.php | 136 + .../PsrHttpClientPaginatedExtractor.php | 142 + .../PsrHttpClientStaticExtractor.php | 172 ++ .../Http/{ => IO/Extractor}/ResponseType.php | 2 +- .../Adapter/Http/Pagination/InjectInto.php | 13 - .../Adapter/Http/Pagination/ResultCounter.php | 10 - .../ETL/Adapter/Http/Pagination/StopWhen.php | 23 - .../Http/Pagination/StopWhen/AllOf.php | 33 - .../Http/Pagination/StopWhen/AnyOf.php | 33 - .../Http/Pagination/StopWhen/EmptyPath.php | 26 - .../Pagination/StopWhen/EmptyResponseBody.php | 19 - .../Http/Pagination/StopWhen/FlagFalse.php | 26 - .../Http/Pagination/StopWhen/FlagTrue.php | 26 - .../Http/Pagination/StopWhen/MaxPages.php | 21 - .../Http/Pagination/StopWhen/MaxResults.php | 21 - .../Http/Pagination/StopWhen/NeverStop.php | 17 - .../Http/Pagination/StopWhen/PathMissing.php | 27 - .../Http/Pagination/StopWhen/TotalReached.php | 33 - .../Pagination/Strategy/CursorFromBody.php | 45 - .../Http/Pagination/Strategy/LinkHeader.php | 39 - .../Pagination/Strategy/NextUrlFromBody.php | 45 - .../Http/Pagination/Strategy/OffsetLimit.php | 47 - .../Http/PsrHttpClientDynamicExtractor.php | 135 - .../Http/PsrHttpClientPaginatedExtractor.php | 140 - .../Http/PsrHttpClientStaticExtractor.php | 153 - .../HTTP/Tests/Double/FixedStrategy.php | 27 - .../Unit/Pagination/StopWhen/AllOfTest.php | 32 - .../Unit/Pagination/StopWhen/AnyOfTest.php | 32 - .../StopWhen/EmptyResponseBodyTest.php | 25 - .../Unit/Pagination/StopWhen/MaxPagesTest.php | 29 - .../Pagination/StopWhen/MaxResultsTest.php | 29 - .../Pagination/StopWhen/NeverStopTest.php | 21 - .../Http/Tests/Double/FixedStrategy.php | 27 + .../Tests/Double/NumberedRequestFactory.php | 4 +- .../Tests/Fixtures/flow-php.json | 0 .../Tests/Fixtures/norberttech.json | 0 .../Tests/Fixtures/tomaszhanc.json | 0 .../PsrHttpClientDynamicExtractorTest.php | 49 +- ...srHttpClientExtractorBatchContractTest.php | 72 +- .../PsrHttpClientPaginatedExtractorTest.php | 41 +- .../PsrHttpClientStaticExtractorTest.php | 54 +- .../Tests/Mother/PaginationMother.php | 4 +- .../IO/Extractor}/ContentTypeDetectorTest.php | 6 +- .../Unit/IO/Extractor}/HttpDecoderTest.php | 8 +- .../MetadataColumnsExtractorTest.php | 42 + .../Pagination/LinkHeaderParserTest.php | 4 +- .../Extractor}/Pagination/PageStateTest.php | 4 +- .../Extractor}/Pagination/PaginatorTest.php | 12 +- .../Pagination/RelativeUriResolverTest.php | 4 +- .../Pagination/RequestComparatorTest.php | 4 +- .../Pagination/RequestOptionTest.php | 4 +- .../Pagination/StopWhen/AllOfTest.php | 34 + .../Pagination/StopWhen/AnyOfTest.php | 34 + .../Pagination/StopWhen/EmptyPathTest.php | 8 +- .../StopWhen/EmptyResponseBodyTest.php | 25 + .../Pagination/StopWhen/FlagFalseTest.php | 8 +- .../Pagination/StopWhen/FlagTrueTest.php | 8 +- .../Pagination/StopWhen/MaxPagesTest.php | 29 + .../Pagination/StopWhen/MaxResultsTest.php | 29 + .../Pagination/StopWhen/NeverStopTest.php | 21 + .../Pagination/StopWhen/PathMissingTest.php | 8 +- .../Pagination/StopWhen/TotalReachedTest.php | 8 +- .../Strategy/CursorFromBodyTest.php | 10 +- .../Strategy/LastRecordCursorTest.php | 10 +- .../Pagination/Strategy/LinkHeaderTest.php | 8 +- .../Strategy/NextUrlFromBodyTest.php | 8 +- .../Pagination/Strategy/OffsetLimitTest.php | 10 +- .../Pagination/Strategy/PageNumberTest.php | 10 +- .../ETL/Adapter/JSON/AdaptiveJsonEncoder.php | 42 - .../Adapter/JSON/AdaptiveJsonOpenSource.php | 99 - .../Exception/CircularReferenceException.php | 4 +- .../UnresolvableReferenceException.php | 17 + .../Exception/UnsupportedKeywordException.php | 4 +- .../Extractor}/JsonArrayElements.php | 2 +- .../JSON/IO/Extractor/JsonExtractor.php | 280 ++ .../JSON/IO/Extractor/JsonFileBatches.php | 150 + .../JSON/IO/Extractor/JsonFileReader.php | 236 ++ .../JSON/IO/Extractor/JsonLineResync.php | 48 + .../JSON/IO/Extractor/JsonOpenSource.php | 34 + .../Extractor}/JsonReadBytes.php | 2 +- .../JSON/IO/Extractor/PhpJsonOpenSource.php | 49 + .../JSON/IO/Inference/JsonFileSample.php | 67 + .../Adapter/JSON/IO/Inference/JsonSniffer.php | 119 + .../ETL/Adapter/JSON/{ => IO}/JsonFraming.php | 2 +- .../JSON/{ => IO/Loader}/JsonEncoder.php | 4 +- .../Adapter/JSON/IO/Loader/JsonFileWriter.php | 47 + .../ETL/Adapter/JSON/IO/Loader/JsonLoader.php | 258 ++ .../JSON/{ => IO/Loader}/PhpJsonEncoder.php | 23 +- .../Schema/JsonSchema/ReferenceResolver.php | 274 ++ .../JSON/IO/Schema/SchemaConverter.php | 1056 +++++++ .../JSON/JSONMachine/JsonExtractor.php | 239 -- .../JSON/JSONMachine/JsonFileBatches.php | 48 - .../JSON/JSONMachine/JsonFileReader.php | 211 -- .../JSON/JSONMachine/JsonFileSample.php | 54 - .../Adapter/JSON/JSONMachine/JsonFormat.php | 15 - .../JSON/JSONMachine/JsonSampledFiles.php | 52 - .../JSON/JSONMachine/JsonSampledRows.php | 40 - .../Adapter/JSON/JSONMachine/JsonSamples.php | 63 - .../src/Flow/ETL/Adapter/JSON/JsonLoader.php | 156 - .../Flow/ETL/Adapter/JSON/JsonOpenSink.php | 46 - .../Flow/ETL/Adapter/JSON/JsonOpenSource.php | 34 - .../UnresolvableReferenceException.php | 17 - .../JSON/JsonSchema/ReferenceResolver.php | 273 -- .../JSON/JsonSchema/ResolvedReference.php | 21 - .../ETL/Adapter/JSON/PhpJsonOpenSource.php | 38 - .../Flow/ETL/Adapter/JSON/SchemaConverter.php | 1057 ------- .../Schema}/JsonSchema/JsonSchemaMetadata.php | 2 +- .../Schema/JsonSchema/ResolvedReference.php | 21 + .../src/Flow/ETL/Adapter/JSON/functions.php | 16 +- .../Tests/Context/DescribableFixtures.php | 71 + .../JSON/Tests/Context/JsonFileFormat.php | 33 + .../JSON/Tests/Context/JsonFixtureContext.php | 88 +- .../Tests/Context/JsonLinesFileFormat.php | 33 + .../AdaptiveJsonOpenSourceTest.php | 167 -- .../JsonExtractorBatchContractTest.php | 4 +- .../Extractor}/JsonExtractorTest.php | 219 +- .../JsonExtractorTypedColumnsTest.php | 98 + .../IO/Extractor/JsonFileBatchesTest.php | 168 ++ .../IO/Extractor/JsonFileReaderTest.php | 317 +++ .../Extractor}/JsonLinesExtractorTest.php | 191 +- .../JsonLinesExtractorTypedColumnsTest.php | 9 +- .../IO/Extractor/JsonSplitReadTest.php | 103 + .../IO/Extractor/PhpJsonOpenSourceTest.php | 94 + .../Extractor}/RustJsonOpenSourceTest.php | 18 +- .../Integration/IO/FileFormatContractTest.php | 156 + .../IO/Inference/JsonFileSampleTest.php | 89 + .../IO/Inference/JsonSnifferTest.php | 221 ++ .../Schema}/JsonSchemaConversionTest.php | 8 +- .../JsonExtractorTypedColumnsTest.php | 97 - .../JSONMachine/JsonFileReaderTest.php | 352 --- .../JSONMachine/JsonFileSampleTest.php | 90 - .../JSONMachine/JsonSamplesTest.php | 81 - .../JSON/Tests/Integration/JsonLinesTest.php | 20 +- .../JSON/Tests/Integration/JsonTest.php | 28 - .../Integration/PhpJsonOpenSourceTest.php | 49 - .../Tests/Integration/RustJsonRefusalTest.php | 52 +- .../JSON/Tests/Mother/JsonEncoderMother.php | 11 +- .../Tests/Unit/AdaptiveJsonEncoderTest.php | 69 - .../Tests/Unit/DescribableRoundTripTest.php | 75 + .../Extractor}/JsonArrayElementsTest.php | 4 +- .../Unit/IO/Extractor/JsonFileBatchesTest.php | 80 + .../Unit/IO/Extractor/JsonLineResyncTest.php | 75 + .../Extractor}/JsonReadBytesTest.php | 4 +- .../MetadataColumnsExtractorTest.php | 42 + .../Tests/Unit/{ => IO}/JsonFramingTest.php | 4 +- .../Unit/IO/Loader/JsonFileWriterTest.php | 79 + .../{ => IO/Loader}/JsonLinesLoaderTest.php | 32 +- .../Tests/Unit/IO/Loader/JsonLoaderTest.php | 370 +++ .../{ => IO/Loader}/PhpJsonEncoderTest.php | 10 +- .../{ => IO/Loader}/RustJsonEncoderTest.php | 8 +- .../JsonSchema/ReferenceResolverTest.php | 270 ++ .../Unit/IO/Schema/SchemaConverterTest.php | 1033 +++++++ .../Unit/JSONMachine/JsonSampledFilesTest.php | 53 - .../Unit/JSONMachine/JsonSampledRowsTest.php | 34 - .../JSON/Tests/Unit/JsonFileBatchesTest.php | 65 - .../JSON/Tests/Unit/JsonLoaderTest.php | 147 - .../JSON/Tests/Unit/JsonOpenSinkTest.php | 79 - .../Unit/JsonSchema/ReferenceResolverTest.php | 270 -- .../JSON/Tests/Unit/SchemaConverterTest.php | 1065 ------- src/adapter/etl-adapter-logger/composer.json | 3 + .../{Logger => IO/Loader}/DumpLogger.php | 2 +- .../Logger/IO/Loader/PsrLoggerLoader.php | 45 + .../ETL/Adapter/Logger/PsrLoggerLoader.php | 43 - .../Tests/Context/DescribableFixtures.php | 52 + .../Tests/Unit/DescribableRoundTripTest.php | 75 + .../{Logger => IO/Loader}/DumpLoggerTest.php | 4 +- .../{ => IO/Loader}/PsrLoggerLoaderTest.php | 8 +- .../Parquet/AdaptiveParquetOpenSink.php | 57 - .../Parquet/AdaptiveParquetOpenSource.php | 38 - .../Parquet/IO/ConfiguredParquetEngine.php | 75 + .../Parquet/IO/Extractor/ParquetExtractor.php | 391 +++ .../IO/Extractor/ParquetFileBatches.php | 183 ++ .../{ => IO/Extractor}/ParquetOpenSource.php | 8 +- .../IO/Extractor/ParquetSourceFile.php | 132 + .../Extractor}/ParquetSourceFileOpener.php | 5 +- .../IO/Extractor/PhpParquetOpenSource.php | 47 + .../Extractor}/Pushdown/FilterTranslator.php | 137 +- .../Pushdown/TranslatedConjunct.php | 4 +- .../Loader}/ElementsValueConverter.php | 2 +- .../Loader}/JsonValueConverter.php | 2 +- .../{ => IO/Loader}/ParquetEncoder.php | 8 +- .../Parquet/IO/Loader/ParquetFileWriter.php | 30 + .../IO/Loader/ParquetFileWriterFactory.php | 64 + .../Parquet/IO/Loader/ParquetLoader.php | 258 ++ .../IO/Loader/ParquetSchemaConformance.php | 87 + .../Loader}/StructValueConverter.php | 2 +- .../Loader}/UuidValueConverter.php | 2 +- .../Parquet/IO/Loader/ValueConverter.php | 10 + .../Loader}/ValueConverters.php | 2 +- .../ETL/Adapter/Parquet/ParquetExtractor.php | 356 --- .../Adapter/Parquet/ParquetFileBatches.php | 136 - .../ETL/Adapter/Parquet/ParquetFileSink.php | 33 - .../ETL/Adapter/Parquet/ParquetFileSinks.php | 68 - .../ETL/Adapter/Parquet/ParquetLoader.php | 150 - .../ETL/Adapter/Parquet/ParquetOpenSink.php | 20 - .../Parquet/ParquetSchemaConformance.php | 62 - .../Adapter/Parquet/ParquetSchemaPassFile.php | 37 - .../ETL/Adapter/Parquet/ParquetSourceFile.php | 81 - .../ETL/Adapter/Parquet/ParquetStoredType.php | 35 - .../Adapter/Parquet/PhpParquetOpenSink.php | 26 - .../Adapter/Parquet/PhpParquetOpenSource.php | 46 - .../ETL/Adapter/Parquet/SchemaConverter.php | 337 --- .../Parquet/ValueConverter/ValueConverter.php | 10 - .../Parquet/Values/Schema/SchemaConverter.php | 332 +++ .../Flow/ETL/Adapter/Parquet/functions.php | 15 +- .../Tests/Context/DescribableFixtures.php | 64 + .../Tests/Context/ParquetFileFormat.php | 33 + .../Tests/Context/ParquetFilesContext.php | 54 +- .../Context/ParquetSourceFileContext.php | 17 +- .../Double/ReadPlanRecordingParquetEngine.php | 104 + .../Tests/Double/RecordingParquetEngine.php | 13 + .../Tests/Double/RefusingParquetEngine.php | 14 + .../Fixtures/EdgeCases/interval.parquet | Bin .../Fixtures/EdgeCases/unsigned.parquet | Bin .../Fixtures/Pagination/01_1000.parquet | Bin .../Fixtures/Pagination/02_500.parquet | Bin .../Fixtures/Pagination/03_350.parquet | Bin .../Fixtures/Pagination/04_2000.parquet | Bin .../Fixtures/Pagination/05_15.parquet | Bin .../date=2024-01-01/01_1000.parquet | Bin .../date=2024-02-01/02_500.parquet | Bin .../date=2024-02-01/03_350.parquet | Bin .../date=2024-12-01/04_2000.parquet | Bin .../partitioned/date=2025-01-01/05_15.parquet | Bin .../Fixtures/orders_1k.parquet | Bin .../Fixtures/xml-node-before-utf8.parquet | Bin .../ParquetExtractorBatchContractTest.php | 6 +- .../Extractor/ParquetExtractorFooterTest.php | 216 ++ .../IO/Extractor/ParquetExtractorTest.php | 1052 +++++++ .../IO/Extractor/ParquetSplitReadTest.php | 91 + .../Integration/IO/FileFormatContractTest.php | 134 + .../Tests/Integration/PaginationTest.php | 41 +- .../ParquetExtractorFooterTest.php | 200 -- .../Integration/ParquetExtractorTest.php | 802 ------ .../Parquet/Tests/Integration/ParquetTest.php | 6 +- .../Tests/Integration/PushdownParityTest.php | 37 +- .../Parquet/Tests/Mother/BodySchemaMother.php | 2 +- .../Unit/AdaptiveParquetOpenSinkTest.php | 149 - .../Unit/AdaptiveParquetOpenSourceTest.php | 86 - .../Tests/Unit/DescribableRoundTripTest.php | 75 + .../Unit/IO/ConfiguredParquetEngineTest.php | 61 + .../MetadataColumnsExtractorTest.php | 42 + .../IO/Extractor/ParquetExtractorTest.php | 187 ++ .../IO/Extractor/ParquetFileBatchesTest.php | 375 +++ .../ParquetSourceFileOpenerTest.php | 8 +- .../IO/Extractor/ParquetSourceFileTest.php | 140 + .../Extractor}/PhpParquetOpenSourceTest.php | 6 +- .../Pushdown/FilterTranslatorTest.php | 10 +- .../Pushdown/TranslatedConjunctTest.php | 4 +- .../Extractor}/RustParquetOpenSourceTest.php | 10 +- .../Loader}/ElementsValueConverterTest.php | 8 +- .../Loader}/JsonValueConverterTest.php | 4 +- .../{ => IO/Loader}/ParquetEncoderTest.php | 4 +- .../Loader/ParquetFileWriterFactoryTest.php | 223 ++ .../Unit/IO/Loader/ParquetFileWriterTest.php | 78 + .../Unit/IO/Loader/ParquetLoaderTest.php | 265 ++ .../Loader/ParquetSchemaConformanceTest.php | 153 + .../IO/Loader/RustParquetFileWriterTest.php | 91 + .../Loader}/StructValueConverterTest.php | 6 +- .../Loader}/UuidValueConverterTest.php | 4 +- .../Loader}/ValueConvertersTest.php | 12 +- .../Tests/Unit/ParquetExtractorTest.php | 211 -- .../Tests/Unit/ParquetFileBatchesTest.php | 362 --- .../Tests/Unit/ParquetFileSinkTest.php | 54 - .../Tests/Unit/ParquetFileSinksTest.php | 81 - .../Parquet/Tests/Unit/ParquetLoaderTest.php | 56 - .../Unit/ParquetSchemaConformanceTest.php | 111 - .../Tests/Unit/ParquetSourceFileTest.php | 73 - .../Tests/Unit/ParquetStoredTypeTest.php | 54 - .../Tests/Unit/PhpParquetOpenSinkTest.php | 49 - .../Tests/Unit/RustParquetOpenSinkTest.php | 87 - .../Schema}/FlowToParquetSchemaTest.php | 16 +- .../Schema}/ParquetToFlowSchemaTest.php | 18 +- .../PostgreSql/Exception/RuntimeException.php | 9 - .../ETL/Adapter/PostgreSql/ExplainedRows.php | 32 - .../Exception/TypeMappingException.php | 4 +- .../PostgreSql/IO/Extractor/ExplainedRows.php | 32 + .../IO/Extractor/Pagination/Key.php | 38 + .../IO/Extractor/Pagination/KeySet.php | 37 + .../IO/Extractor/Pagination/KeySetPage.php | 63 + .../IO/Extractor/Pagination/KeyValues.php | 98 + .../IO/Extractor/Pagination/Order.php | 21 + .../Extractor}/PostgreSqlCursorExtractor.php | 61 +- .../Extractor/PostgreSqlKeySetExtractor.php | 204 ++ .../PostgreSqlLimitOffsetExtractor.php | 198 ++ .../{ => IO/Extractor}/ReadQuery.php | 8 +- .../{ => IO/Extractor}/ResultSchema.php | 13 +- .../Loader}/EnumConverter.php | 2 +- .../Loader}/HTMLConverter.php | 2 +- .../Loader}/LoaderOptions/DeleteOptions.php | 2 +- .../Loader}/LoaderOptions/InsertOptions.php | 2 +- .../Loader}/LoaderOptions/UpdateOptions.php | 2 +- .../PostgreSql/IO/Loader/Operation.php | 12 + .../{ => IO/Loader}/PostgreSqlEncoder.php | 10 +- .../{ => IO/Loader}/PostgreSqlLoader.php | 64 +- .../{ => IO/Loader}/PostgreSqlTransaction.php | 4 +- .../QueryBuilder/DeleteQueryBuilder.php | 62 + .../QueryBuilder/InsertQueryBuilder.php | 21 +- .../QueryBuilder/UpdateQueryBuilder.php | 83 + .../Loader}/XMLConverter.php | 2 +- .../Flow/ETL/Adapter/PostgreSql/Operation.php | 12 - .../ETL/Adapter/PostgreSql/Pagination/Key.php | 38 - .../Adapter/PostgreSql/Pagination/KeySet.php | 37 - .../PostgreSql/Pagination/KeySetPage.php | 63 - .../PostgreSql/Pagination/KeyValues.php | 98 - .../Adapter/PostgreSql/Pagination/Order.php | 21 - .../PostgreSql/PostgreSqlKeySetExtractor.php | 189 -- .../PostgreSqlLimitOffsetExtractor.php | 179 -- .../QueryBuilder/DeleteQueryBuilder.php | 60 - .../QueryBuilder/UpdateQueryBuilder.php | 77 - .../Schema/SortingStrategy/TypeStrategy.php | 60 - .../Adapter/PostgreSql/SchemaConverter.php | 325 --- .../Flow/ETL/Adapter/PostgreSql/TypesMap.php | 253 -- .../Schema}/PostgreSqlMetadata.php | 6 +- .../Values/Schema/SchemaConverter.php | 324 +++ .../Schema/SortingStrategy/TypeStrategy.php | 59 + .../PostgreSql/Values/Schema/TypesMap.php | 253 ++ .../Flow/ETL/Adapter/PostgreSql/functions.php | 35 +- .../Tests/Context/DescribableFixtures.php | 47 + .../Tests/Double/SortedRowsClient.php | 2 +- .../Tests/Double/TransactionSpyLoader.php | 26 +- .../PostgreSqlCursorExtractorTest.php | 400 +++ .../PostgreSqlExtractorBatchContractTest.php | 109 + .../PostgreSqlKeySetExtractorTest.php | 519 ++++ .../PostgreSqlLimitOffsetExtractorTest.php | 243 ++ .../Loader/PostgreSqlLoaderAllTypesTest.php | 306 ++ .../IO/Loader/PostgreSqlLoaderTest.php | 248 ++ ...stgreSqlCursorExtractorIntegrationTest.php | 401 --- ...lExtractorBatchContractIntegrationTest.php | 109 - ...stgreSqlKeySetExtractorIntegrationTest.php | 514 ---- ...SqlLimitOffsetExtractorIntegrationTest.php | 243 -- ...ostgreSqlLoaderAllTypesIntegrationTest.php | 306 -- .../PostgreSqlLoaderIntegrationTest.php | 248 -- .../PostgreSqlTransactionSinkTest.php | 360 --- .../PostgreSqlTransactionalTest.php | 364 +++ .../SchemaConverterIntegrationTest.php | 138 - .../Values/Schema/SchemaConverterTest.php | 138 + .../Tests/Mother/WideRowsMother.php | 2 +- .../Tests/Unit/DescribableRoundTripTest.php | 75 + .../Tests/Unit/ExplainedRowsTest.php | 41 - .../Unit/IO/Extractor/ExplainedRowsTest.php | 41 + .../MetadataColumnsExtractorTest.php | 42 + .../Extractor/Pagination/KeySetPageTest.php | 109 + .../IO/Extractor/Pagination/KeyValuesTest.php | 90 + .../Extractor/Pagination}/PaginationTest.php | 10 +- .../PostgreSqlCursorExtractorTest.php | 512 ++++ .../PostgreSqlKeySetExtractorTest.php | 693 +++++ .../PostgreSqlLimitOffsetExtractorTest.php | 390 +++ .../Unit/{ => IO/Extractor}/ReadQueryTest.php | 6 +- .../{ => IO/Extractor}/ResultSchemaTest.php | 47 +- .../Loader}/EnumConverterTest.php | 4 +- .../Loader}/HTMLConverterTest.php | 4 +- .../{ => IO/Loader}/PostgreSqlEncoderTest.php | 4 +- .../Unit/IO/Loader/PostgreSqlLoaderTest.php | 169 ++ .../Loader}/PostgreSqlTransactionTest.php | 4 +- .../QueryBuilder/DeleteQueryBuilderTest.php | 10 +- .../QueryBuilder/InsertQueryBuilderTest.php | 10 +- .../QueryBuilder/UpdateQueryBuilderTest.php | 10 +- .../Loader}/XMLConverterTest.php | 4 +- .../PostgreSql/Tests/Unit/KeySetPageTest.php | 109 - .../PostgreSql/Tests/Unit/KeyValuesTest.php | 90 - .../Unit/PostgreSqlCursorExtractorTest.php | 475 ---- .../Unit/PostgreSqlKeySetExtractorTest.php | 702 ----- .../PostgreSqlLimitOffsetExtractorTest.php | 389 --- .../Tests/Unit/PostgreSqlLoaderTest.php | 169 -- .../Tests/Unit/SchemaConverterTest.php | 437 --- .../PostgreSql/Tests/Unit/TypesMapTest.php | 415 --- .../Schema}/PostgreSqlMetadataTest.php | 6 +- .../Values/Schema/SchemaConverterTest.php | 437 +++ .../SortingStrategy/TypeStrategyTest.php | 6 +- .../Tests/Unit/Values/Schema/TypesMapTest.php | 415 +++ .../ETL/Adapter/Seal/IO/Loader/Operation.php | 11 + .../Seal/{ => IO/Loader}/SealEncoder.php | 16 +- .../ETL/Adapter/Seal/IO/Loader/SealLoader.php | 120 + .../src/Flow/ETL/Adapter/Seal/Operation.php | 11 - .../Flow/ETL/Adapter/Seal/SchemaConverter.php | 239 -- .../src/Flow/ETL/Adapter/Seal/SealLoader.php | 118 - .../Seal/Values/Schema/SchemaConverter.php | 238 ++ .../Seal/{ => Values/Schema}/SealMetadata.php | 4 +- .../src/Flow/ETL/Adapter/Seal/functions.php | 5 +- .../Tests/Context/DescribableFixtures.php | 52 + .../Integration/IO/Loader/SealLoaderTest.php | 147 + .../Seal/Tests/Integration/SealLoaderTest.php | 147 - .../Tests/Unit/DescribableRoundTripTest.php | 75 + .../Unit/{ => IO/Loader}/SealEncoderTest.php | 4 +- .../Tests/Unit/IO/Loader/SealLoaderTest.php | 78 + .../Seal/Tests/Unit/SchemaConverterTest.php | 157 -- .../Seal/Tests/Unit/SealLoaderTest.php | 78 - .../Values/Schema/SchemaConverterTest.php | 157 ++ .../Text/IO/Extractor/TextExtractor.php | 169 ++ .../Text/IO/Extractor/TextFileBatches.php | 86 + .../Text/{ => IO/Loader}/TextEncoder.php | 10 +- .../Adapter/Text/IO/Loader/TextFileWriter.php | 27 + .../ETL/Adapter/Text/IO/Loader/TextLoader.php | 193 ++ .../Flow/ETL/Adapter/Text/TextExtractor.php | 134 - .../Flow/ETL/Adapter/Text/TextFileBatches.php | 75 - .../src/Flow/ETL/Adapter/Text/TextLoader.php | 126 - .../Flow/ETL/Adapter/Text/TextOpenSink.php | 27 - .../src/Flow/ETL/Adapter/Text/functions.php | 2 + .../Tests/Context/DescribableFixtures.php | 50 + .../Text/Tests/Context/TextFileFormat.php | 33 + .../TextExtractorBatchContractTest.php | 6 +- .../IO/Extractor/TextExtractorTest.php | 337 +++ .../TextExtractorTypedColumnsTest.php | 57 + .../IO/Extractor/TextSplitReadTest.php | 40 + .../Integration/IO/FileFormatContractTest.php | 134 + .../Integration/IO/Loader/TextLoaderTest.php | 177 ++ .../Tests/Integration/TextExtractorTest.php | 277 -- .../TextExtractorTypedColumnsTest.php | 55 - .../Text/Tests/Integration/TextLoaderTest.php | 74 - .../Text/Tests/Integration/TextTest.php | 6 +- .../Tests/Unit/DescribableRoundTripTest.php | 75 + .../MetadataColumnsExtractorTest.php | 42 + .../Unit/IO/Extractor/TextFileBatchesTest.php | 71 + .../Unit/{ => IO/Loader}/TextEncoderTest.php | 8 +- .../Unit/IO/Loader/TextFileWriterTest.php | 32 + .../Text/Tests/Unit/TextFileBatchesTest.php | 58 - .../Text/Tests/Unit/TextOpenSinkTest.php | 32 - .../Adapter/XML/Abstraction/XMLNodeType.php | 11 - .../{ => IO/Extractor}/RootlessDocument.php | 2 +- .../XML/IO/Extractor/XMLFileBatches.php | 91 + .../XML/{ => IO/Extractor}/XMLNodeBatches.php | 16 +- .../XML/{ => IO/Extractor}/XMLNodeCursor.php | 4 +- .../XML/{ => IO/Extractor}/XMLNodes.php | 4 +- .../XML/IO/Extractor/XMLParserExtractor.php | 207 ++ .../Loader}/XMLAttribute.php | 4 +- .../XML/{ => IO/Loader}/XMLEncoder.php | 21 +- .../Adapter/XML/IO/Loader/XMLFileWriter.php | 40 + .../ETL/Adapter/XML/IO/Loader/XMLLoader.php | 340 +++ .../{Abstraction => IO/Loader}/XMLNode.php | 4 +- .../ETL/Adapter/XML/IO/Loader/XMLNodeType.php | 11 + .../XML/{ => IO/Loader}/XMLValueNodes.php | 8 +- .../Adapter/XML/{ => IO/Loader}/XMLWriter.php | 4 +- .../IO/Loader/XMLWriter/DOMDocumentWriter.php | 88 + .../Loader}/XMLWriter/StringXMLWriter.php | 22 +- .../Flow/ETL/Adapter/XML/Loader/XMLLoader.php | 241 -- .../Flow/ETL/Adapter/XML/XMLFileBatches.php | 64 - .../src/Flow/ETL/Adapter/XML/XMLOpenSink.php | 40 - .../ETL/Adapter/XML/XMLParserExtractor.php | 172 -- .../XML/XMLWriter/DOMDocumentWriter.php | 74 - .../src/Flow/ETL/Adapter/XML/functions.php | 6 +- .../XML/Tests/Context/DescribableFixtures.php | 55 + .../XML/Tests/Context/SerializedNodes.php | 2 +- .../XML/Tests/Context/XMLFileFormat.php | 33 + .../XMLExtractorBatchContractTest.php | 28 + .../XMLExtractorTypedColumnsTest.php | 105 + .../IO/Extractor/XMLParserExtractorTest.php | 610 ++++ .../IO/Extractor/XMLParserSplitReadTest.php | 50 + .../Integration/IO/FileFormatContractTest.php | 151 + .../Integration/IO/Loader/XMLLoaderTest.php | 163 ++ .../Integration/Loader/XMLLoaderTest.php | 163 -- .../XMLExtractorBatchContractTest.php | 29 - .../XMLExtractorTypedColumnsTest.php | 103 - .../Integration/XMLParserExtractorTest.php | 550 ---- .../Tests/Unit/DescribableRoundTripTest.php | 75 + .../MetadataColumnsExtractorTest.php | 42 + .../Extractor}/RootlessDocumentTest.php | 4 +- .../Unit/IO/Extractor/XMLFileBatchesTest.php | 73 + .../Unit/IO/Extractor/XMLNodeBatchesTest.php | 92 + .../{ => IO/Extractor}/XMLNodeCursorTest.php | 6 +- .../Unit/{ => IO/Extractor}/XMLNodesTest.php | 8 +- .../Unit/{ => IO/Loader}/XMLEncoderTest.php | 20 +- .../Unit/IO/Loader/XMLFileWriterTest.php | 47 + .../Tests/Unit/IO/Loader/XMLLoaderTest.php | 280 ++ .../Loader}/XMLNodeTest.php | 10 +- .../{ => IO/Loader}/XMLValueNodesTest.php | 12 +- .../XMLWriter/DOMDocumentWriterTest.php | 157 ++ .../Loader}/XMLWriter/StringXMLWriterTest.php | 32 +- .../XML/Tests/Unit/XMLFileBatchesTest.php | 64 - .../Adapter/XML/Tests/Unit/XMLLoaderTest.php | 107 - .../XML/Tests/Unit/XMLNodeBatchesTest.php | 88 - .../XML/Tests/Unit/XMLOpenSinkTest.php | 47 - .../Unit/XMLWriter/DOMDocumentWriterTest.php | 144 - .../Bridge/AsyncAWS/AsyncAWSS3Filesystem.php | 145 +- .../Bridge/AsyncAWS/ConfiguredS3Client.php | 35 + .../Bridge/AsyncAWS/DSL/functions.php | 14 +- .../Bridge/AsyncAWS/LiveS3Client.php | 24 + .../Filesystem/Bridge/AsyncAWS/Options.php | 49 +- .../Bridge/AsyncAWS/S3ClientSource.php | 18 + .../Tests/Unit/AsyncAWSS3FilesystemTest.php | 46 + .../Tests/Unit/ConfiguredS3ClientTest.php | 32 + .../AsyncAWS/Tests/Unit/LiveS3ClientTest.php | 21 + .../AsyncAWS/Tests/Unit/OptionsTest.php | 44 + .../Bridge/Azure/AzureBlobFilesystem.php | 64 +- .../Bridge/Azure/BlobServiceSource.php | 20 + .../Bridge/Azure/ConfiguredBlobService.php | 129 + .../Filesystem/Bridge/Azure/DSL/functions.php | 15 +- .../Bridge/Azure/LiveBlobService.php | 24 + .../Flow/Filesystem/Bridge/Azure/Options.php | 98 +- .../Tests/Unit/AzureBlobFilesystemTest.php | 29 + .../Tests/Unit/ConfiguredBlobServiceTest.php | 101 + .../Azure/Tests/Unit/LiveBlobServiceTest.php | 21 + .../Bridge/Azure/Tests/Unit/OptionsTest.php | 60 + .../Bridge/SFTP/ConfiguredSFTPClient.php | 119 + .../Filesystem/Bridge/SFTP/DSL/functions.php | 19 +- .../Filesystem/Bridge/SFTP/LiveSFTPClient.php | 24 + .../Flow/Filesystem/Bridge/SFTP/Options.php | 51 +- .../Bridge/SFTP/SFTPClientSource.php | 20 + .../Filesystem/Bridge/SFTP/SFTPCredential.php | 17 + .../Filesystem/Bridge/SFTP/SFTPFilesystem.php | 66 +- .../Filesystem/Bridge/SFTP/SFTPPassword.php | 29 + .../Filesystem/Bridge/SFTP/SFTPPrivateKey.php | 43 + .../Tests/Unit/ConfiguredSFTPClientTest.php | 101 + .../SFTP/Tests/Unit/LiveSFTPClientTest.php | 21 + .../Bridge/SFTP/Tests/Unit/OptionsTest.php | 32 + .../SFTP/Tests/Unit/SFTPFilesystemTest.php | 29 + .../SFTP/Tests/Unit/SFTPPasswordTest.php | 16 + .../SFTP/Tests/Unit/SFTPPrivateKeyTest.php | 34 + .../OpenAPI/Specification/DSL/functions.php | 2 +- .../Specification/OpenAPIConverter.php | 30 +- .../OpenAPI/Specification/OpenAPIMetadata.php | 2 +- .../Tests/Unit/FlowToOpenAPIConverterTest.php | 391 +-- .../Tests/Unit/OpenAPIConverterTest.php | 40 +- .../Unit/OpenAPIMetadataIntegrationTest.php | 277 +- .../Tests/Unit/OpenAPIMetadataTest.php | 10 +- .../Tests/Unit/OpenAPIToFlowConverterTest.php | 123 +- .../Factory/AsyncAwsS3FilesystemFactory.php | 12 +- .../Factory/AzureBlobFilesystemFactory.php | 70 +- .../Filesystem/FstabBuilder.php | 2 +- .../AsyncAwsS3FilesystemFactoryTest.php | 45 + .../AzureBlobFilesystemFactoryTest.php | 53 + .../Tests/Unit/Double/FailingMvFilesystem.php | 14 + .../Symfony/HttpFoundation/DataStream.php | 31 +- .../Bridge/Symfony/HttpFoundation/Output.php | 2 +- .../HttpFoundation/Output/CSVOutput.php | 2 +- .../HttpFoundation/Output/JsonOutput.php | 2 +- .../HttpFoundation/Output/ParquetOutput.php | 4 +- .../HttpFoundation/Output/XMLOutput.php | 6 +- .../Response/FlowBufferedResponse.php | 27 +- .../Response/FlowStreamedResponse.php | 25 +- .../Symfony/HttpFoundation/StreamClosure.php | 2 +- .../Symfony/HttpFoundation/functions.php | 9 +- .../Tests/Context/ResponseContext.php | 21 + .../Integration/FlowBufferedResponseTest.php | 22 + .../Integration/FlowStreamedResponseTest.php | 211 +- .../Tests/Unit/DataStreamTest.php | 32 + .../Tests/Integration/CollectorMetrics.php | 2 +- .../CLI/Command/DatabaseTableListCommand.php | 13 +- .../Command/DatabaseTableSchemaCommand.php | 17 +- .../Flow/CLI/Command/FileAnalyzeCommand.php | 44 +- .../Flow/CLI/Command/FileConvertCommand.php | 57 +- .../src/Flow/CLI/Command/FileReadCommand.php | 43 +- .../Flow/CLI/Command/FileRowsCountCommand.php | 32 +- .../Flow/CLI/Command/FileSchemaCommand.php | 43 +- .../Flow/CLI/Command/PipelineRunCommand.php | 15 +- .../Flow/CLI/Command/SchemaFormatCommand.php | 11 +- .../Flow/CLI/Command/Traits/CSVOptions.php | 57 - .../Flow/CLI/Command/Traits/ConfigOptions.php | 22 - .../src/Flow/CLI/Command/Traits/DBOptions.php | 24 - .../Flow/CLI/Command/Traits/ExcelOptions.php | 29 - .../Flow/CLI/Command/Traits/JSONOptions.php | 45 - .../CLI/Command/Traits/ParquetOptions.php | 21 - .../Flow/CLI/Command/Traits/XMLOptions.php | 54 - .../Flow/CLI/Factory/CSVExtractorFactory.php | 2 +- .../src/Flow/CLI/Factory/CSVLoaderFactory.php | 2 +- .../CLI/Factory/ExcelExtractorFactory.php | 9 +- .../src/Flow/CLI/Factory/ExtractorFactory.php | 2 +- .../Flow/CLI/Factory/JsonExtractorFactory.php | 2 +- .../Flow/CLI/Factory/JsonLoaderFactory.php | 2 +- .../src/Flow/CLI/Factory/LoaderFactory.php | 6 +- .../CLI/Factory/ParquetExtractorFactory.php | 9 +- .../Flow/CLI/Factory/ParquetLoaderFactory.php | 2 +- .../Flow/CLI/Factory/XMLExtractorFactory.php | 2 +- .../src/Flow/CLI/Factory/XMLLoaderFactory.php | 2 +- .../CLI/Formatter/PipelineReportFormatter.php | 10 +- .../src/Flow/CLI/Loader/ProgressBarLoader.php | 22 +- .../src/Flow/CLI/Options/CSVInputOptions.php | 31 + .../src/Flow/CLI/Options/CSVOutputOptions.php | 37 + src/cli/src/Flow/CLI/Options/ConfigOption.php | 16 +- .../Flow/CLI/Options/DbConnectionOptions.php | 24 + .../Flow/CLI/Options/ExcelInputOptions.php | 26 + .../src/Flow/CLI/Options/JSONInputOptions.php | 28 + .../Flow/CLI/Options/JSONOutputOptions.php | 28 + .../Flow/CLI/Options/ParquetInputOptions.php | 21 + .../SchemaInferenceOptions.php | 12 +- .../Traits => Options}/StatisticsOptions.php | 6 +- .../src/Flow/CLI/Options/XMLInputOptions.php | 21 + .../src/Flow/CLI/Options/XMLOutputOptions.php | 44 + src/cli/src/Flow/CLI/PipelineFactory.php | 6 +- .../CLI/Tests/Context/DescribableFixtures.php | 50 + .../tests/Flow/CLI}/Tests/Fixtures/empty.php | 0 .../tests/Flow/CLI/Tests/Fixtures/empty.txt} | 0 .../CLI}/Tests/Fixtures/with-dataframe.php | 0 .../CLI}/Tests/Fixtures/without-dataframe.php | 0 .../DatabaseTableListCommandTest.php | 6 +- .../DatabaseTableSchemaCommandTest.php | 6 +- .../Integration/FileAnalyzeCommandTest.php | 36 +- .../Tests/Integration/FileReadCommandTest.php | 66 +- .../Integration/FileSchemaCommandTest.php | 53 +- .../Integration/Options/ConfigOptionTest.php | 2 +- .../Integration/PipelineRunCommandTest.php | 10 +- .../Integration/SchemaFormatCommandTest.php | 12 +- .../Tests/Unit/DescribableRoundTripTest.php | 69 + .../Unit/Loader/ProgressBarLoaderTest.php | 8 +- .../CLI}/Tests/Unit/PipelineFactoryTest.php | 6 +- .../etl/src/Flow/Calculator/Calculator.php | 186 -- .../Flow/Calculator/Exception/Exception.php | 9 - .../etl/src/Flow/Calculator/RunningSum.php | 74 - src/core/etl/src/Flow/ETL/API/Branch.php | 14 + .../Bucketing/BucketingConfigBuilder.php | 124 + .../API/Config/Cache/CacheConfigBuilder.php | 64 + .../src/Flow/ETL/API/Config/ConfigBuilder.php | 260 ++ .../ETL/API/Config/Join/HashJoinBuilder.php | 86 + .../API/Config/Sort/ExternalSortBuilder.php | 77 + .../ETL/API/Config/Sort/MemorySortBuilder.php | 18 + src/core/etl/src/Flow/ETL/API/DataFrame.php | 848 ++++++ .../etl/src/Flow/ETL/API/FilteredBranch.php | 21 + .../etl/src/Flow/ETL/API/GroupedDataFrame.php | 70 + .../etl/src/Flow/ETL/API/Transactional.php | 43 + .../src/Flow/ETL/{ => API}/Transformation.php | 2 +- .../ETL/API/Transformation/AddRowIndex.php | 23 + .../Flow/ETL/API/Transformation/BatchBy.php | 33 + .../Flow/ETL/API/Transformation/BatchSize.php | 21 + .../src/Flow/ETL/API/Transformation/Drop.php | 25 + .../src/Flow/ETL/API/Transformation/Limit.php | 20 + .../ETL/API/Transformation/MaskColumns.php | 33 + .../Flow/ETL/API/Transformation/Select.php | 25 + .../Flow/ETL/{ => API}/Transformations.php | 2 +- .../src/Flow/ETL/API/TransformedBranch.php | 26 + src/core/etl/src/Flow/ETL/BoundStep.php | 13 - .../src/Flow/ETL/Bucketing/BucketsStorage.php | 25 - .../src/Flow/ETL/Bucketing/HashBucketing.php | 96 - .../etl/src/Flow/ETL/Bucketing/Hasher.php | 15 - .../src/Flow/ETL/Bucketing/KeyGrouping.php | 61 - .../etl/src/Flow/ETL/Bucketing/KeyValues.php | 53 - .../src/Flow/ETL/Bucketing/NativeHasher.php | 73 - .../ETL/Bucketing/ResidentBucketsStorage.php | 7 - .../Flow/ETL/Bucketing/SingleBucketHasher.php | 16 - .../Bucketing/Storage/FilesystemBuckets.php | 141 - .../ETL/Bucketing/Storage/MemoryBuckets.php | 42 - .../ETL/Bucketing/Storage/PSRCacheBuckets.php | 98 - .../ETL/Bucketing/Storage/SpillingBuckets.php | 161 -- src/core/etl/src/Flow/ETL/Cache.php | 28 - .../etl/src/Flow/ETL/Cache/CacheIndex.php | 105 - .../ETL/Cache/Implementation/ApcuCache.php | 127 - .../Cache/Implementation/FilesystemCache.php | 105 - .../Cache/Implementation/InMemoryCache.php | 77 - .../Cache/Implementation/PSRSimpleCache.php | 94 - .../Cache/Implementation/TraceableCache.php | 149 - .../src/Flow/ETL/Column/AdaptiveBackend.php | 43 - src/core/etl/src/Flow/ETL/Column/Backend.php | 53 - src/core/etl/src/Flow/ETL/Column/Column.php | 80 - .../etl/src/Flow/ETL/Column/Layout/Layout.php | 22 - .../src/Flow/ETL/Column/Layout/LayoutFor.php | 66 - .../ETL/Column/Php/CastingColumnBuilder.php | 201 -- .../etl/src/Flow/ETL/Column/PhpBackend.php | 101 - .../ETL/Column/Physical/DaysFromCivil.php | 24 - .../Column/Physical/PhysicalBuilderFor.php | 56 - .../Flow/ETL/Column/Physical/PhysicalFor.php | 250 -- .../Flow/ETL/Column/Physical/PhysicalKind.php | 100 - src/core/etl/src/Flow/ETL/Config.php | 119 - .../ETL/Config/Bucketing/BucketingConfig.php | 20 - .../Bucketing/BucketingConfigBuilder.php | 88 - .../src/Flow/ETL/Config/Cache/CacheConfig.php | 18 - .../ETL/Config/Cache/CacheConfigBuilder.php | 69 - .../etl/src/Flow/ETL/Config/ConfigBuilder.php | 247 -- .../Grouping/GroupByAlgorithmBuilder.php | 13 - .../Config/Grouping/HashGroupByBuilder.php | 74 - .../ETL/Config/Grouping/HashGroupByConfig.php | 16 - .../Flow/ETL/Config/Join/HashJoinBuilder.php | 75 - .../Flow/ETL/Config/Join/HashJoinConfig.php | 16 - .../ETL/Config/Join/JoinAlgorithmBuilder.php | 13 - .../etl/src/Flow/ETL/Config/MemoryLimit.php | 41 - .../Repartition/HashRepartitionBuilder.php | 74 - .../Repartition/HashRepartitionConfig.php | 16 - .../RepartitionAlgorithmBuilder.php | 13 - .../ETL/Config/Sort/ExternalSortBuilder.php | 87 - .../ETL/Config/Sort/ExternalSortConfig.php | 23 - .../ETL/Config/Sort/MemorySortBuilder.php | 16 - .../Flow/ETL/Config/Sort/MemorySortConfig.php | 7 - .../ETL/Config/Sort/SortAlgorithmBuilder.php | 13 - .../Flow/ETL/Config/Telemetry/SpanStack.php | 38 - .../ETL/Config/Telemetry/TelemetryContext.php | 422 --- .../ETL/Constraint/SortedByConstraint.php | 102 - .../Flow/ETL/Constraint/UniqueConstraint.php | 79 - .../UniqueConstraint/InMemoryStorage.php | 35 - .../Constraint/UniqueConstraint/Storage.php | 16 - src/core/etl/src/Flow/ETL/DSL/functions.php | 605 ++-- src/core/etl/src/Flow/ETL/DataFrame.php | 884 ------ .../Flow/ETL/DataFrame/GroupedDataFrame.php | 34 - .../etl/src/Flow/ETL/DataFrameFactory.php | 10 - .../Flow/ETL/Dataset/Memory/Configuration.php | 59 - .../etl/src/Flow/ETL/Dataset/Statistics.php | 24 - .../Flow/ETL/Dataset/Statistics/Column.php | 275 -- .../Flow/ETL/Dataset/Statistics/Columns.php | 61 - .../Dataset/Statistics/DistinctCounter.php | 43 - src/core/etl/src/Flow/ETL/ErrorHandler.php | 21 - .../ETL/ErrorHandler/ExtractionAction.php | 11 - .../Flow/ETL/ErrorHandler/ExtractionError.php | 16 - .../src/Flow/ETL/ErrorHandler/IgnoreError.php | 25 - .../Flow/ETL/ErrorHandler/LoadingAction.php | 11 - .../Flow/ETL/ErrorHandler/LoadingError.php | 18 - .../src/Flow/ETL/ErrorHandler/SkipRows.php | 25 - .../src/Flow/ETL/ErrorHandler/ThrowError.php | 25 - .../ETL/ErrorHandler/TransformationAction.php | 11 - .../ETL/ErrorHandler/TransformationError.php | 18 - .../ETL/Exception/ColumnMismatchException.php | 117 - .../Exception/DuplicatedColumnsException.php | 7 - .../etl/src/Flow/ETL/Exception/Exception.php | 9 - .../ETL/Exception/InferredSchemaException.php | 93 - .../Exception/InvalidArgumentException.php | 15 - .../ETL/Exception/InvalidLogicException.php | 94 - .../src/Flow/ETL/Exception/JoinException.php | 7 - .../ETL/Exception/KeyNotInCacheException.php | 19 - .../ETL/Exception/LimitReachedException.php | 21 - .../src/Flow/ETL/Exception/OffsetOverflow.php | 7 - .../Flow/ETL/Exception/RuntimeException.php | 7 - .../SchemaDefinitionNotUniqueException.php | 7 - .../Exception/SchemaValidationException.php | 40 - .../src/Flow/ETL/Exception/SinkFailure.php | 20 - .../ETL/Exception/TransactionRolledBack.php | 22 - .../ETL/Exception/TypeAssertionException.php | 7 - src/core/etl/src/Flow/ETL/Executor.php | 172 -- .../etl/src/Flow/ETL/Executor/Described.php | 25 - .../src/Flow/ETL/Executor/PhysicalPlan.php | 23 - .../etl/src/Flow/ETL/Executor/Pipeline.php | 61 - src/core/etl/src/Flow/ETL/Executor/Raw.php | 30 - .../etl/src/Flow/ETL/Executor/Segment.php | 303 -- .../etl/src/Flow/ETL/Executor/Segments.php | 51 - .../etl/src/Flow/ETL/Executor/SinkFeed.php | 107 - .../etl/src/Flow/ETL/Executor/SinkOffers.php | 60 - .../etl/src/Flow/ETL/Executor/SinkRun.php | 75 - .../etl/src/Flow/ETL/Executor/SourceRead.php | 43 - .../etl/src/Flow/ETL/Executor/SourceRows.php | 80 - .../Flow/ETL/Executor/TransactionRollback.php | 28 - .../Flow/ETL/Executor/TransactionalSinks.php | 95 - src/core/etl/src/Flow/ETL/Extractor.php | 64 - .../src/Flow/ETL/Extractor/ArrayExtractor.php | 192 -- .../Flow/ETL/Extractor/BatchByExtractor.php | 96 - .../src/Flow/ETL/Extractor/BatchExtractor.php | 99 - .../Flow/ETL/Extractor/BatchableExtractor.php | 27 - .../etl/src/Flow/ETL/Extractor/Batches.php | 34 - .../src/Flow/ETL/Extractor/CacheExtractor.php | 173 -- .../src/Flow/ETL/Extractor/ChainExtractor.php | 100 - .../ETL/Extractor/CollectingExtractor.php | 73 - .../Flow/ETL/Extractor/DataFrameExtractor.php | 104 - .../Extractor/File/DeclaresPartitionTypes.php | 22 - .../Flow/ETL/Extractor/File/DerivedSchema.php | 35 - .../Flow/ETL/Extractor/File/FileBatches.php | 40 - .../Flow/ETL/Extractor/File/FileColumns.php | 110 - .../Flow/ETL/Extractor/File/FileExtractor.php | 38 - .../ETL/Extractor/File/FileMetadataRow.php | 27 - .../Flow/ETL/Extractor/File/FileReadLoop.php | 100 - .../Flow/ETL/Extractor/File/FileReading.php | 41 - .../src/Flow/ETL/Extractor/File/FileScan.php | 77 - .../ETL/Extractor/File/FooterStatistics.php | 60 - .../ETL/Extractor/File/InferredColumns.php | 54 - .../Flow/ETL/Extractor/File/ListedFileRow.php | 15 - .../Flow/ETL/Extractor/File/ListedFiles.php | 58 - .../ETL/Extractor/File/ListingPartitions.php | 29 - .../Flow/ETL/Extractor/File/ListingScan.php | 79 - .../ETL/Extractor/File/MetadataColumns.php | 17 - .../ETL/Extractor/File/PartitionColumns.php | 83 - .../ETL/Extractor/File/PartitionTypes.php | 59 - .../Flow/ETL/Extractor/File/PathFiltering.php | 94 - .../ETL/Extractor/File/PathPartitionsRow.php | 25 - .../Flow/ETL/Extractor/File/ReadWindow.php | 29 - .../src/Flow/ETL/Extractor/File/Scanning.php | 24 - .../ETL/Extractor/File/SelfDescribingFile.php | 22 - .../Flow/ETL/Extractor/File/SourceFile.php | 36 - .../src/Flow/ETL/Extractor/FilesExtractor.php | 124 - .../src/Flow/ETL/Extractor/InfersSchema.php | 12 - .../ETL/Extractor/Memory/InMemoryRows.php | 57 - .../ETL/Extractor/Memory/InferredRows.php | 58 - .../Flow/ETL/Extractor/MemoryExtractor.php | 132 - .../ETL/Extractor/OverridingExtractor.php | 21 - .../ETL/Extractor/PathPartitionsExtractor.php | 120 - .../src/Flow/ETL/Extractor/Repeatability.php | 55 - .../ETL/Extractor/RewindableExtractor.php | 16 - .../Flow/ETL/Extractor/SequenceExtractor.php | 138 - .../SequenceGenerator/SequenceGenerator.php | 18 - .../etl/src/Flow/ETL/Extractor/Signal.php | 10 - .../etl/src/Flow/ETL/Extractor/Statistics.php | 20 - .../etl/src/Flow/ETL/Filesystem/SaveMode.php | 28 - .../ETL/Filesystem/ScalarFunctionFilter.php | 89 - src/core/etl/src/Flow/ETL/Flow.php | 25 +- src/core/etl/src/Flow/ETL/FlowContext.php | 73 - src/core/etl/src/Flow/ETL/Formatter.php | 10 - .../Flow/ETL/Formatter/ASCII/ASCIIBody.php | 59 - .../Flow/ETL/Formatter/ASCII/ASCIIHeaders.php | 49 - .../Flow/ETL/Formatter/ASCII/ASCIITable.php | 24 - .../etl/src/Flow/ETL/Formatter/ASCII/Body.php | 59 - .../src/Flow/ETL/Formatter/ASCII/Headers.php | 30 - .../ETL/Formatter/AsciiTableFormatter.php | 21 - .../ArrayComparison/ArrayComparison.php | 2 +- .../ArrayComparison/ArraySortByKey.php | 2 +- .../ETL/Foundation/Calculator/Calculator.php | 192 ++ .../Calculator/Exception/Exception.php | 9 + .../Exception/InvalidExponentException.php | 2 +- .../Exception/InvalidScaleException.php | 2 +- .../Exception/NonNumericValueException.php | 2 +- .../Exception/RoundingNecessaryException.php | 2 +- .../Calculator/NumberNormalizer.php | 6 +- .../Foundation}/Calculator/Rounding.php | 2 +- .../ETL/Foundation/Calculator/RunningSum.php | 176 ++ .../{ => ETL/Foundation}/Clock/FakeClock.php | 2 +- .../Clock}/HighResolutionTime.php | 2 +- .../Foundation}/Clock/SystemClock.php | 2 +- .../Describe/ConfiguredFilesystem.php | 56 + .../ETL/Foundation/Describe/Describable.php | 20 + .../ETL/Foundation/Describe/Description.php | 276 ++ .../src/Flow/ETL/Foundation/Describe/Json.php | 55 + .../Flow/ETL/Foundation/Describe/Rebuild.php | 93 + .../Exception/ColumnMismatchException.php | 97 + .../ConstraintViolationException.php | 2 +- .../DataDependentSchemaException.php | 2 +- .../Exception/DuplicatedColumnsException.php | 7 + .../Exception/EvaluationException.php | 2 +- .../ETL/Foundation/Exception/Exception.php | 9 + .../Exception/InferredSchemaException.php | 95 + .../Exception/InvalidArgumentException.php | 33 + .../Exception/InvalidFileFormatException.php | 2 +- .../Exception/InvalidLogicException.php | 89 + .../Foundation/Exception/JoinException.php | 7 + .../Exception/MissingDependencyException.php | 2 +- .../Exception/MissingExtensionException.php | 2 +- .../Foundation/Exception/OffsetOverflow.php | 7 + .../Exception/OffsetReachedException.php | 2 +- .../Exception/RemoteStepException.php | 32 + .../Exception/RequiredPHPVersionException.php | 2 +- .../Foundation/Exception/RuntimeException.php | 7 + .../SchemaDefinitionNotFoundException.php | 4 +- .../SchemaDefinitionNotUniqueException.php | 7 + .../Exception/SchemaMismatchException.php | 2 +- .../Exception/SchemaNotDerivableException.php | 2 +- .../Exception/SchemaValidationException.php | 13 + .../Exception/TypeAssertionException.php | 7 + .../UnsupportedUnionTypeException.php | 17 +- .../ETL/{ => Foundation}/FlowPhpExtension.php | 6 +- .../NativePHPRandomValueGenerator.php | 43 + .../{ => Foundation}/RandomValueGenerator.php | 2 +- .../{Schema => Foundation}/SimilarNames.php | 2 +- .../Foundation/Telemetry/SpanAttributes.php | 13 + .../Telemetry/TelemetryAttributes.php | 4 +- .../Flow/ETL/Function/AggregatingFunction.php | 49 - src/core/etl/src/Flow/ETL/Function/All.php | 134 - src/core/etl/src/Flow/ETL/Function/Any.php | 134 - src/core/etl/src/Flow/ETL/Function/Append.php | 87 - .../etl/src/Flow/ETL/Function/ArrayExpand.php | 110 - .../ETL/Function/ArrayExpand/ArrayExpand.php | 14 - .../etl/src/Flow/ETL/Function/ArrayFilter.php | 94 - .../etl/src/Flow/ETL/Function/ArrayKeep.php | 94 - .../src/Flow/ETL/Function/ArrayKeyRename.php | 74 - .../etl/src/Flow/ETL/Function/ArrayKeys.php | 97 - .../ETL/Function/ArrayKeysStyleConvert.php | 148 - .../etl/src/Flow/ETL/Function/ArrayMerge.php | 105 - .../src/Flow/ETL/Function/ArrayPathExists.php | 101 - .../src/Flow/ETL/Function/ArrayReverse.php | 89 - .../etl/src/Flow/ETL/Function/ArraySort.php | 139 - .../src/Flow/ETL/Function/ArraySort/Sort.php | 35 - .../etl/src/Flow/ETL/Function/ArrayUnpack.php | 92 - .../etl/src/Flow/ETL/Function/ArrayValues.php | 94 - src/core/etl/src/Flow/ETL/Function/Ascii.php | 76 - .../etl/src/Flow/ETL/Function/Average.php | 161 -- .../etl/src/Flow/ETL/Function/Between.php | 132 - .../src/Flow/ETL/Function/BinaryLength.php | 76 - .../src/Flow/ETL/Function/CallUserFunc.php | 127 - .../etl/src/Flow/ETL/Function/Capitalize.php | 88 - src/core/etl/src/Flow/ETL/Function/Cast.php | 156 - src/core/etl/src/Flow/ETL/Function/Chunk.php | 92 - .../etl/src/Flow/ETL/Function/Coalesce.php | 103 - .../src/Flow/ETL/Function/CodePointLength.php | 76 - .../Flow/ETL/Function/CollapseWhitespace.php | 76 - .../etl/src/Flow/ETL/Function/Collect.php | 104 - .../src/Flow/ETL/Function/CollectUnique.php | 109 - .../etl/src/Flow/ETL/Function/Combine.php | 123 - src/core/etl/src/Flow/ETL/Function/Concat.php | 93 - .../Flow/ETL/Function/ConcatWithSeparator.php | 111 - .../etl/src/Flow/ETL/Function/Contains.php | 93 - src/core/etl/src/Flow/ETL/Function/Count.php | 162 -- .../ETL/Function/DOMElementAttributeValue.php | 134 - .../Function/DOMElementAttributesCount.php | 88 - .../ETL/Function/DOMElementNamespaceValue.php | 143 - .../ETL/Function/DOMElementNextSibling.php | 101 - .../Flow/ETL/Function/DOMElementParent.php | 101 - .../Function/DOMElementPreviousSibling.php | 101 - .../src/Flow/ETL/Function/DOMElementValue.php | 113 - .../src/Flow/ETL/Function/DateTimeFormat.php | 113 - .../etl/src/Flow/ETL/Function/DenseRank.php | 100 - src/core/etl/src/Flow/ETL/Function/Divide.php | 96 - .../etl/src/Flow/ETL/Function/EndsWith.php | 83 - .../etl/src/Flow/ETL/Function/EnsureEnd.php | 87 - .../etl/src/Flow/ETL/Function/EnsureStart.php | 87 - .../etl/src/Flow/ETL/Function/EnumName.php | 77 - .../etl/src/Flow/ETL/Function/EnumValue.php | 93 - src/core/etl/src/Flow/ETL/Function/Equals.php | 133 - .../Flow/ETL/Function/Evaluation/Filter.php | 103 - .../ETL/Function/Evaluation/ResultColumn.php | 166 -- .../ETL/Function/Evaluation/Selection.php | 52 - src/core/etl/src/Flow/ETL/Function/Exists.php | 82 - src/core/etl/src/Flow/ETL/Function/First.php | 107 - .../Flow/ETL/Function/FrameAccumulating.php | 18 - .../etl/src/Flow/ETL/Function/GreaterThan.php | 110 - .../Flow/ETL/Function/GreaterThanEqual.php | 110 - .../etl/src/Flow/ETL/Function/Greatest.php | 106 - .../Flow/ETL/Function/HTMLQuerySelector.php | 99 - .../ETL/Function/HTMLQuerySelectorAll.php | 119 - src/core/etl/src/Flow/ETL/Function/Hash.php | 90 - .../etl/src/Flow/ETL/Function/IndexOf.php | 103 - .../etl/src/Flow/ETL/Function/IndexOfLast.php | 103 - .../etl/src/Flow/ETL/Function/IsEmpty.php | 78 - src/core/etl/src/Flow/ETL/Function/IsIn.php | 179 -- .../etl/src/Flow/ETL/Function/IsNotNull.php | 73 - .../src/Flow/ETL/Function/IsNotNumeric.php | 79 - src/core/etl/src/Flow/ETL/Function/IsNull.php | 73 - .../etl/src/Flow/ETL/Function/IsNumeric.php | 79 - src/core/etl/src/Flow/ETL/Function/IsType.php | 92 - src/core/etl/src/Flow/ETL/Function/IsUtf8.php | 78 - .../etl/src/Flow/ETL/Function/JsonDecode.php | 117 - .../etl/src/Flow/ETL/Function/JsonEncode.php | 123 - src/core/etl/src/Flow/ETL/Function/Last.php | 107 - src/core/etl/src/Flow/ETL/Function/Least.php | 106 - .../etl/src/Flow/ETL/Function/LessThan.php | 110 - .../src/Flow/ETL/Function/LessThanEqual.php | 110 - .../etl/src/Flow/ETL/Function/ListSelect.php | 146 - .../etl/src/Flow/ETL/Function/Literal.php | 109 - .../etl/src/Flow/ETL/Function/MatchCases.php | 162 -- .../Function/MatchCases/MatchCondition.php | 92 - src/core/etl/src/Flow/ETL/Function/Max.php | 115 - src/core/etl/src/Flow/ETL/Function/Min.php | 115 - src/core/etl/src/Flow/ETL/Function/Minus.php | 112 - src/core/etl/src/Flow/ETL/Function/Mod.php | 85 - .../src/Flow/ETL/Function/ModifyDateTime.php | 94 - .../etl/src/Flow/ETL/Function/Multiply.php | 112 - src/core/etl/src/Flow/ETL/Function/Not.php | 61 - .../etl/src/Flow/ETL/Function/NotEquals.php | 87 - .../etl/src/Flow/ETL/Function/NotSame.php | 87 - src/core/etl/src/Flow/ETL/Function/Now.php | 88 - .../src/Flow/ETL/Function/NumberFormat.php | 102 - src/core/etl/src/Flow/ETL/Function/OnEach.php | 187 -- .../etl/src/Flow/ETL/Function/Optional.php | 61 - .../etl/src/Flow/ETL/Function/Parameter.php | 349 --- .../Flow/ETL/Function/PartitionRanking.php | 22 - src/core/etl/src/Flow/ETL/Function/Plus.php | 112 - src/core/etl/src/Flow/ETL/Function/Power.php | 116 - .../etl/src/Flow/ETL/Function/Prepend.php | 87 - .../src/Flow/ETL/Function/RandomString.php | 84 - src/core/etl/src/Flow/ETL/Function/Rank.php | 100 - .../Flow/ETL/Function/ReferenceResolver.php | 79 - src/core/etl/src/Flow/ETL/Function/Regex.php | 118 - .../etl/src/Flow/ETL/Function/RegexAll.php | 128 - .../etl/src/Flow/ETL/Function/RegexMatch.php | 104 - .../src/Flow/ETL/Function/RegexMatchAll.php | 105 - .../src/Flow/ETL/Function/RegexReplace.php | 120 - src/core/etl/src/Flow/ETL/Function/Repeat.php | 85 - .../ETL/Function/ResolvesFromChildren.php | 33 - .../etl/src/Flow/ETL/Function/Reverse.php | 76 - src/core/etl/src/Flow/ETL/Function/Round.php | 92 - .../etl/src/Flow/ETL/Function/RowNumber.php | 71 - src/core/etl/src/Flow/ETL/Function/Same.php | 87 - .../etl/src/Flow/ETL/Function/Sanitize.php | 103 - .../src/Flow/ETL/Function/ScalarFunction.php | 30 - .../Function/ScalarFunction/ExpandResults.php | 12 - .../Function/ScalarFunction/UnpackResults.php | 12 - src/core/etl/src/Flow/ETL/Function/Size.php | 88 - src/core/etl/src/Flow/ETL/Function/Slug.php | 98 - src/core/etl/src/Flow/ETL/Function/Split.php | 100 - .../etl/src/Flow/ETL/Function/Sprintf.php | 103 - .../etl/src/Flow/ETL/Function/StartsWith.php | 83 - src/core/etl/src/Flow/ETL/Function/StrPad.php | 97 - .../etl/src/Flow/ETL/Function/StrReplace.php | 102 - .../etl/src/Flow/ETL/Function/StringAfter.php | 88 - .../src/Flow/ETL/Function/StringAfterLast.php | 88 - .../src/Flow/ETL/Function/StringAggregate.php | 119 - .../src/Flow/ETL/Function/StringBefore.php | 88 - .../Flow/ETL/Function/StringBeforeLast.php | 88 - .../Flow/ETL/Function/StringContainsAny.php | 96 - .../src/Flow/ETL/Function/StringEqualsTo.php | 83 - .../etl/src/Flow/ETL/Function/StringFold.php | 76 - .../etl/src/Flow/ETL/Function/StringMatch.php | 94 - .../src/Flow/ETL/Function/StringMatchAll.php | 99 - .../src/Flow/ETL/Function/StringNormalize.php | 82 - .../etl/src/Flow/ETL/Function/StringStyle.php | 95 - .../etl/src/Flow/ETL/Function/StringTitle.php | 83 - .../etl/src/Flow/ETL/Function/StringWidth.php | 76 - .../etl/src/Flow/ETL/Function/Structure.php | 114 - .../src/Flow/ETL/Function/StructureSelect.php | 129 - src/core/etl/src/Flow/ETL/Function/Sum.php | 159 -- src/core/etl/src/Flow/ETL/Function/ToDate.php | 121 - .../etl/src/Flow/ETL/Function/ToDateTime.php | 158 -- .../etl/src/Flow/ETL/Function/ToLower.php | 76 - .../etl/src/Flow/ETL/Function/ToTimeZone.php | 100 - .../etl/src/Flow/ETL/Function/ToUpper.php | 76 - src/core/etl/src/Flow/ETL/Function/Trim.php | 101 - .../etl/src/Flow/ETL/Function/Trim/Type.php | 12 - .../etl/src/Flow/ETL/Function/Truncate.php | 94 - src/core/etl/src/Flow/ETL/Function/Ulid.php | 107 - .../src/Flow/ETL/Function/UnicodeLength.php | 76 - src/core/etl/src/Flow/ETL/Function/Uuid.php | 145 - .../etl/src/Flow/ETL/Function/ValueOrder.php | 54 - src/core/etl/src/Flow/ETL/Function/When.php | 122 - .../src/Flow/ETL/Function/WindowFunction.php | 31 - .../etl/src/Flow/ETL/Function/Wordwrap.php | 107 - src/core/etl/src/Flow/ETL/Function/XPath.php | 124 - .../ETL/Functions/AggregatingFunction.php | 66 + src/core/etl/src/Flow/ETL/Functions/All.php | 169 ++ src/core/etl/src/Flow/ETL/Functions/Any.php | 169 ++ .../etl/src/Flow/ETL/Functions/Append.php | 105 + .../src/Flow/ETL/Functions/ArrayExpand.php | 132 + .../ETL/Functions/ArrayExpand/ArrayExpand.php | 14 + .../src/Flow/ETL/Functions/ArrayFilter.php | 112 + .../ETL/{Function => Functions}/ArrayGet.php | 39 +- .../ArrayGetCollection.php | 54 +- .../etl/src/Flow/ETL/Functions/ArrayKeep.php | 112 + .../src/Flow/ETL/Functions/ArrayKeyRename.php | 96 + .../etl/src/Flow/ETL/Functions/ArrayKeys.php | 111 + .../ETL/Functions/ArrayKeysStyleConvert.php | 171 ++ .../etl/src/Flow/ETL/Functions/ArrayMerge.php | 123 + .../ArrayMergeCollection.php | 40 +- .../Flow/ETL/Functions/ArrayPathExists.php | 119 + .../src/Flow/ETL/Functions/ArrayReverse.php | 108 + .../etl/src/Flow/ETL/Functions/ArraySort.php | 165 ++ .../src/Flow/ETL/Functions/ArraySort/Sort.php | 35 + .../src/Flow/ETL/Functions/ArrayUnpack.php | 108 + .../src/Flow/ETL/Functions/ArrayValues.php | 108 + src/core/etl/src/Flow/ETL/Functions/Ascii.php | 90 + .../etl/src/Flow/ETL/Functions/Average.php | 262 ++ .../etl/src/Flow/ETL/Functions/Between.php | 156 + .../Between/Boundary.php | 4 +- .../src/Flow/ETL/Functions/BinaryLength.php | 90 + .../src/Flow/ETL/Functions/CallUserFunc.php | 151 + .../etl/src/Flow/ETL/Functions/Capitalize.php | 102 + src/core/etl/src/Flow/ETL/Functions/Cast.php | 175 ++ src/core/etl/src/Flow/ETL/Functions/Chunk.php | 108 + .../etl/src/Flow/ETL/Functions/Coalesce.php | 118 + .../Flow/ETL/Functions/CodePointLength.php | 90 + .../Flow/ETL/Functions/CollapseWhitespace.php | 90 + .../etl/src/Flow/ETL/Functions/Collect.php | 148 + .../src/Flow/ETL/Functions/CollectUnique.php | 150 + .../etl/src/Flow/ETL/Functions/Combine.php | 141 + .../etl/src/Flow/ETL/Functions/Comparison.php | 190 ++ .../ComparisonFunction.php | 2 +- .../etl/src/Flow/ETL/Functions/Concat.php | 122 + .../ETL/Functions/ConcatWithSeparator.php | 127 + .../ETL/{Function => Functions}/Conjuncts.php | 2 +- .../etl/src/Flow/ETL/Functions/Contains.php | 111 + src/core/etl/src/Flow/ETL/Functions/Count.php | 219 ++ .../Functions/DOMElementAttributeValue.php | 152 + .../Functions/DOMElementAttributesCount.php | 102 + .../Functions/DOMElementNamespaceValue.php | 161 ++ .../ETL/Functions/DOMElementNextSibling.php | 115 + .../Flow/ETL/Functions/DOMElementParent.php | 115 + .../Functions/DOMElementPreviousSibling.php | 115 + .../Flow/ETL/Functions/DOMElementValue.php | 127 + .../src/Flow/ETL/Functions/DateTimeFormat.php | 131 + .../etl/src/Flow/ETL/Functions/DenseRank.php | 115 + .../etl/src/Flow/ETL/Functions/Divide.php | 119 + .../etl/src/Flow/ETL/Functions/EndsWith.php | 101 + .../etl/src/Flow/ETL/Functions/EnsureEnd.php | 105 + .../src/Flow/ETL/Functions/EnsureStart.php | 105 + .../etl/src/Flow/ETL/Functions/EnumName.php | 91 + .../etl/src/Flow/ETL/Functions/EnumValue.php | 107 + .../Flow/ETL/Functions/Evaluation/Filter.php | 99 + .../ETL/Functions/Evaluation/ResultColumn.php | 172 ++ .../Evaluation/ReturnTypes.php | 6 +- .../ETL/Functions/Evaluation/Selection.php | 52 + .../Evaluation/TolerantEvaluation.php | 22 +- .../etl/src/Flow/ETL/Functions/Exists.php | 97 + .../ExpandingFunctions.php | 6 +- src/core/etl/src/Flow/ETL/Functions/First.php | 150 + .../Flow/ETL/Functions/FrameAccumulating.php | 18 + .../{Function => Functions}/FunctionTree.php | 2 +- .../etl/src/Flow/ETL/Functions/Greatest.php | 116 + .../Flow/ETL/Functions/HTMLQuerySelector.php | 117 + .../ETL/Functions/HTMLQuerySelectorAll.php | 137 + src/core/etl/src/Flow/ETL/Functions/Hash.php | 110 + .../etl/src/Flow/ETL/Functions/IndexOf.php | 125 + .../src/Flow/ETL/Functions/IndexOfLast.php | 125 + .../etl/src/Flow/ETL/Functions/IsEmpty.php | 92 + src/core/etl/src/Flow/ETL/Functions/IsIn.php | 124 + .../etl/src/Flow/ETL/Functions/IsNotNull.php | 84 + .../src/Flow/ETL/Functions/IsNotNumeric.php | 93 + .../etl/src/Flow/ETL/Functions/IsNull.php | 87 + .../etl/src/Flow/ETL/Functions/IsNumeric.php | 93 + .../etl/src/Flow/ETL/Functions/IsType.php | 119 + .../etl/src/Flow/ETL/Functions/IsUtf8.php | 92 + .../src/Flow/ETL/Functions/Join/Condition.php | 29 + .../Flow/ETL/Functions/Join/Condition/All.php | 103 + .../Flow/ETL/Functions/Join/Condition/Any.php | 103 + .../ETL/Functions/Join/Condition/Equal.php | 80 + .../Functions/Join/Condition/Identical.php | 79 + .../Flow/ETL/Functions/Join/Expression.php | 221 ++ .../etl/src/Flow/ETL/Functions/JsonDecode.php | 135 + .../etl/src/Flow/ETL/Functions/JsonEncode.php | 142 + src/core/etl/src/Flow/ETL/Functions/Last.php | 149 + src/core/etl/src/Flow/ETL/Functions/Least.php | 116 + .../{Function => Functions}/ListFunctions.php | 4 +- .../etl/src/Flow/ETL/Functions/ListSelect.php | 179 ++ .../etl/src/Flow/ETL/Functions/Literal.php | 127 + .../src/Flow/ETL/Functions/LiteralValue.php | 249 ++ .../etl/src/Flow/ETL/Functions/MatchCases.php | 179 ++ .../Functions/MatchCases/MatchCondition.php | 129 + src/core/etl/src/Flow/ETL/Functions/Max.php | 160 ++ src/core/etl/src/Flow/ETL/Functions/Min.php | 160 ++ src/core/etl/src/Flow/ETL/Functions/Minus.php | 135 + src/core/etl/src/Flow/ETL/Functions/Mod.php | 104 + .../src/Flow/ETL/Functions/ModifyDateTime.php | 112 + .../etl/src/Flow/ETL/Functions/Multiply.php | 135 + src/core/etl/src/Flow/ETL/Functions/Not.php | 76 + src/core/etl/src/Flow/ETL/Functions/Now.php | 102 + .../src/Flow/ETL/Functions/NumberFormat.php | 124 + .../etl/src/Flow/ETL/Functions/OnEach.php | 204 ++ .../OnEachElementSchema.php | 12 +- .../etl/src/Flow/ETL/Functions/Optional.php | 76 + .../etl/src/Flow/ETL/Functions/Parameter.php | 343 +++ src/core/etl/src/Flow/ETL/Functions/Plus.php | 135 + src/core/etl/src/Flow/ETL/Functions/Power.php | 139 + .../etl/src/Flow/ETL/Functions/Prepend.php | 105 + .../src/Flow/ETL/Functions/RandomString.php | 113 + src/core/etl/src/Flow/ETL/Functions/Rank.php | 115 + .../ETL/Functions/Reference/Reference.php | 30 + .../Reference}/References.php | 32 +- .../Functions/Reference/ResolvedReference.php | 146 + .../Reference/UnresolvedReference.php | 172 ++ .../ReferenceRename.php | 6 +- .../Flow/ETL/Functions/ReferenceResolver.php | 121 + .../ReferencedColumns.php | 6 +- src/core/etl/src/Flow/ETL/Functions/Regex.php | 141 + .../etl/src/Flow/ETL/Functions/RegexAll.php | 151 + .../etl/src/Flow/ETL/Functions/RegexMatch.php | 126 + .../src/Flow/ETL/Functions/RegexMatchAll.php | 127 + .../src/Flow/ETL/Functions/RegexReplace.php | 142 + .../etl/src/Flow/ETL/Functions/Repeat.php | 103 + .../etl/src/Flow/ETL/Functions/Reverse.php | 90 + src/core/etl/src/Flow/ETL/Functions/Round.php | 112 + .../etl/src/Flow/ETL/Functions/RowNumber.php | 96 + .../etl/src/Flow/ETL/Functions/Sanitize.php | 125 + .../src/Flow/ETL/Functions/ScalarFunction.php | 31 + .../ScalarFunction/ExpandResults.php | 12 + .../ScalarFunction/UnpackResults.php | 12 + .../ScalarFunctionChain.php | 99 +- src/core/etl/src/Flow/ETL/Functions/Size.php | 102 + src/core/etl/src/Flow/ETL/Functions/Slug.php | 120 + src/core/etl/src/Flow/ETL/Functions/Split.php | 120 + .../etl/src/Flow/ETL/Functions/Sprintf.php | 121 + .../etl/src/Flow/ETL/Functions/StartsWith.php | 101 + .../etl/src/Flow/ETL/Functions/StrPad.php | 119 + .../etl/src/Flow/ETL/Functions/StrReplace.php | 123 + .../{ => Functions}/String/StringStyles.php | 4 +- .../src/Flow/ETL/Functions/StringAfter.php | 108 + .../Flow/ETL/Functions/StringAfterLast.php | 108 + .../Flow/ETL/Functions/StringAggregate.php | 179 ++ .../src/Flow/ETL/Functions/StringBefore.php | 108 + .../Flow/ETL/Functions/StringBeforeLast.php | 108 + .../Flow/ETL/Functions/StringContainsAny.php | 114 + .../src/Flow/ETL/Functions/StringEqualsTo.php | 101 + .../etl/src/Flow/ETL/Functions/StringFold.php | 90 + .../src/Flow/ETL/Functions/StringMatch.php | 112 + .../src/Flow/ETL/Functions/StringMatchAll.php | 117 + .../Flow/ETL/Functions/StringNormalize.php | 100 + .../src/Flow/ETL/Functions/StringStyle.php | 113 + .../src/Flow/ETL/Functions/StringTitle.php | 101 + .../src/Flow/ETL/Functions/StringWidth.php | 90 + .../etl/src/Flow/ETL/Functions/Structure.php | 127 + .../StructureFunctions.php | 4 +- .../Flow/ETL/Functions/StructureSelect.php | 162 ++ .../StructureValues.php | 4 +- .../StyleConverter/ArrayKeyConverter.php | 2 +- src/core/etl/src/Flow/ETL/Functions/Sum.php | 251 ++ .../TemporalStringCoercion.php | 4 +- .../etl/src/Flow/ETL/Functions/ToDate.php | 141 + .../etl/src/Flow/ETL/Functions/ToDateTime.php | 186 ++ .../ToDateTime/PatternCoverage.php | 14 +- .../etl/src/Flow/ETL/Functions/ToLower.php | 90 + .../etl/src/Flow/ETL/Functions/ToTimeZone.php | 119 + .../etl/src/Flow/ETL/Functions/ToUpper.php | 90 + src/core/etl/src/Flow/ETL/Functions/Trim.php | 121 + .../etl/src/Flow/ETL/Functions/Trim/Type.php | 12 + .../etl/src/Flow/ETL/Functions/Truncate.php | 114 + src/core/etl/src/Flow/ETL/Functions/Ulid.php | 121 + .../src/Flow/ETL/Functions/UnicodeLength.php | 90 + src/core/etl/src/Flow/ETL/Functions/Uuid.php | 160 ++ src/core/etl/src/Flow/ETL/Functions/When.php | 142 + .../etl/src/Flow/ETL/Functions/Window.php | 105 + .../Flow/ETL/Functions/Window/BoundWindow.php | 22 + .../Functions/Window/FrameAccumulation.php | 69 + .../Flow/ETL/Functions/Window/FrameBound.php | 70 + .../{ => Functions}/Window/FrameBoundType.php | 2 +- .../Flow/ETL/Functions/Window/PeerFrame.php | 77 + .../Flow/ETL/Functions/Window/RowsFrame.php | 49 + .../Functions/Window/WholePartitionFrame.php | 28 + .../Flow/ETL/Functions/Window/WindowFrame.php | 19 + .../src/Flow/ETL/Functions/WindowFunction.php | 37 + .../etl/src/Flow/ETL/Functions/WithColumn.php | 13 + .../etl/src/Flow/ETL/Functions/Wordwrap.php | 129 + src/core/etl/src/Flow/ETL/Functions/XPath.php | 141 + src/core/etl/src/Flow/ETL/GroupBy.php | 199 -- .../src/Flow/ETL/GroupBy/AggregatedGroups.php | 113 - .../etl/src/Flow/ETL/GroupBy/Aggregators.php | 128 - .../etl/src/Flow/ETL/GroupBy/GroupBySteps.php | 72 - src/core/etl/src/Flow/ETL/GroupBy/Pivot.php | 15 - .../src/Flow/ETL/GroupBy/PivotAggregation.php | 73 - .../etl/src/Flow/ETL/GroupBy/PivotSchema.php | 46 - .../etl/src/Flow/ETL/GroupBy/PivotShape.php | 49 - .../etl/src/Flow/ETL/GroupBy/PivotValues.php | 17 - .../etl/src/Flow/ETL/GroupBy/PivotedTable.php | 113 - .../etl/src/Flow/ETL/Hash/NativePHPHash.php | 45 - src/core/etl/src/Flow/ETL/Hash/PlainText.php | 13 - src/core/etl/src/Flow/ETL/IO/Extractor.php | 78 + .../Flow/ETL/IO/Extractor/ArrayExtractor.php | 244 ++ .../ETL/IO/Extractor/BatchableExtractor.php | 26 + .../Flow/ETL/IO/Extractor/CacheExtractor.php | 195 ++ .../Flow/ETL/IO/Extractor/ChainExtractor.php | 110 + .../src/Flow/ETL/IO/Extractor/Criteria.php | 139 + .../ETL/IO/Extractor/ExtractorFactory.php | 16 + .../ETL/{ => IO}/Extractor/FeedExtractor.php | 21 +- .../Flow/ETL/IO/Extractor/File/ByteUnit.php | 51 + .../IO/Extractor/File/DerivedFromFiles.php | 94 + .../ETL/IO/Extractor/File/DerivedSchema.php | 57 + .../ETL/IO/Extractor/File/FileBatches.php | 45 + .../ETL/IO/Extractor/File/FileColumns.php | 166 ++ .../IO/Extractor/File/FileColumnsSource.php | 29 + .../{ => IO}/Extractor/File/FileConstants.php | 8 +- .../ETL/IO/Extractor/File/FileExtractor.php | 33 + .../ETL/IO/Extractor/File/FileMetadataRow.php | 43 + .../Flow/ETL/IO/Extractor/File/FileRead.php | 77 + .../ETL/IO/Extractor/File/FileReadLoop.php | 78 + .../Flow/ETL/IO/Extractor/File/FileSplit.php | 167 ++ .../ETL/IO/Extractor/File/FileSplitReader.php | 49 + .../IO/Extractor/File/FooterStatistics.php | 59 + .../ETL/IO/Extractor/File/InferredColumns.php | 84 + .../ETL/IO/Extractor/File/ListedFileRow.php | 21 + .../ETL/IO/Extractor/File/ListingScan.php | 73 + .../File/MetadataColumnsExtractor.php | 2 +- .../File/OffsetSkippingFileBatches.php | 2 +- .../ETL/IO/Extractor/File/PartitionTypes.php | 92 + .../IO/Extractor/File/PathPartitionsRow.php | 38 + .../Flow/ETL/IO/Extractor/File/ReadWindow.php | 59 + .../Flow/ETL/IO/Extractor/File/RowRange.php | 34 + .../IO/Extractor/File/SelfDescribingFile.php | 30 + .../Flow/ETL/IO/Extractor/File/SourceFile.php | 59 + .../ETL/IO/Extractor/File/SourceFiles.php | 204 ++ .../ETL/IO/Extractor/FileListingExtractor.php | 126 + .../Flow/ETL/IO/Extractor/Grid/GridHeader.php | 42 + .../{ => IO}/Extractor/Grid/HeaderNames.php | 2 +- .../{ => IO}/Extractor/Grid/RecordDecoder.php | 19 +- .../Flow/ETL/IO/Extractor/InfersSchema.php | 12 + .../ETL/IO/Extractor/Memory/InMemoryRows.php | 75 + .../{ => IO}/Extractor/Memory/SpillState.php | 2 +- .../{ => IO}/Extractor/Memory/SpilledRows.php | 48 +- .../etl/src/Flow/ETL/IO/Extractor/Reader.php | 20 + .../src/Flow/ETL/IO/Extractor/Refreshable.php | 14 + .../ETL/{ => IO}/Extractor/RowsExtractor.php | 22 +- .../ETL/IO/Extractor/SequenceExtractor.php | 152 + .../DatePeriodSequenceGenerator.php | 4 +- .../NumberSequenceGenerator.php | 4 +- .../SequenceGenerator/SequenceGenerator.php | 18 + .../etl/src/Flow/ETL/IO/Extractor/Signal.php | 10 + .../etl/src/Flow/ETL/IO/Extractor/Split.php | 21 + .../ETL/IO/Extractor/SplittableExtractor.php | 30 + .../src/Flow/ETL/IO/Extractor/Statistics.php | 20 + .../src/Flow/ETL/IO/Filesystem/SaveMode.php | 84 + .../IO/Filesystem/ScalarFunctionFilter.php | 66 + .../Flow/ETL/IO/Filesystem/VisibleFiles.php | 58 + .../src/Flow/ETL/IO/Floe/FloeExtractor.php | 238 ++ .../src/Flow/ETL/IO/Floe/FloeFileBatches.php | 128 + .../src/Flow/ETL/IO/Floe/FloeFileWriter.php | 26 + .../ETL/IO/Floe/FloeFileWriterFactory.php | 45 + .../etl/src/Flow/ETL/IO/Floe/FloeLoader.php | 214 ++ .../src/Flow/ETL/IO/Floe/FloeSourceFile.php | 68 + src/core/etl/src/Flow/ETL/IO/Formatter.php | 12 + .../{ => IO}/Formatter/ASCII/ASCIIValue.php | 2 +- .../ETL/IO/Formatter/AsciiTableFormatter.php | 88 + .../Formatter/Row}/ASCIISchemaFormatter.php | 33 +- .../Schema}/InlineSchemaFormatter.php | 5 +- .../Formatter/Schema}/JsonSchemaFormatter.php | 5 +- .../Schema}/PHPFormatter/TypeFormatter.php | 2 +- .../Schema}/PHPFormatter/ValueFormatter.php | 4 +- .../Formatter/Schema/PHPSchemaFormatter.php | 263 ++ .../IO/Formatter/Schema/SchemaFormatter.php | 12 + .../src/Flow/ETL/IO/Inference/ColumnTypes.php | 137 + .../src/Flow/ETL/IO/Inference/Inferred.php | 15 + .../Inference/InferredTypes.php | 37 +- .../etl/src/Flow/ETL/IO/Inference/Sample.php | 66 + .../Flow/ETL/IO/Inference/SampledUnits.php | 70 + .../Flow/ETL/IO/Inference/SchemaInference.php | 81 + .../IO/Inference/SchemaInferenceBuilder.php | 114 + .../Flow/ETL/IO/Inference/SchemaInferrer.php | 91 + .../Flow/ETL/IO/Inference/SniffedColumns.php | 109 + .../etl/src/Flow/ETL/IO/Inference/Sniffer.php | 45 + .../etl/src/Flow/ETL/IO/Inference/Sniffs.php | 17 + .../{Schema => IO}/Inference/TypeFloor.php | 2 +- src/core/etl/src/Flow/ETL/IO/Loader.php | 14 + .../src/Flow/ETL/IO/Loader/ArrayLoader.php | 40 + .../etl/src/Flow/ETL/IO/Loader/Closure.php | 14 + .../src/Flow/ETL/IO/Loader/Discardable.php | 14 + .../Flow/ETL/IO/Loader/File/FileCommit.php | 382 +++ .../Flow/ETL/IO/Loader/File/FileLoader.php | 40 + .../src/Flow/ETL/IO/Loader/File/FileWrite.php | 122 + .../Flow/ETL/IO/Loader/File/FileWriter.php | 20 + .../ETL/IO/Loader/File/FileWriterFactory.php | 20 + .../etl/src/Flow/ETL/IO/Loader/File/Part.php | 202 ++ .../ETL/IO/Loader/File/PartitionRouter.php | 87 + .../src/Flow/ETL/IO/Loader/MemoryLoader.php | 37 + .../src/Flow/ETL/IO/Loader/Partitioning.php | 63 + .../src/Flow/ETL/IO/Loader/RowPartitions.php | 41 + .../ETL/IO/Loader/SchemaValidationLoader.php | 55 + .../src/Flow/ETL/IO/Loader/StreamLoader.php | 160 ++ .../{ => IO}/Loader/StreamLoader/Output.php | 2 +- .../Flow/ETL/IO/Loader/StreamLoader/Type.php | 13 + src/core/etl/src/Flow/ETL/IO/Transaction.php | 19 + src/core/etl/src/Flow/ETL/Join/Comparison.php | 28 - .../etl/src/Flow/ETL/Join/Comparison/All.php | 82 - .../etl/src/Flow/ETL/Join/Comparison/Any.php | 82 - .../src/Flow/ETL/Join/Comparison/Equal.php | 71 - .../Flow/ETL/Join/Comparison/Identical.php | 60 - src/core/etl/src/Flow/ETL/Join/Expression.php | 104 - .../src/Flow/ETL/Join/HashJoin/HashTable.php | 63 - .../src/Flow/ETL/Join/HashJoin/JoinKeys.php | 90 - .../src/Flow/ETL/Join/HashJoin/JoinSide.php | 31 - .../etl/src/Flow/ETL/Join/HashJoin/Joiner.php | 363 --- .../Flow/ETL/Join/HashJoin/NullRowBuilder.php | 31 - .../src/Flow/ETL/Join/HashJoin/RowMerger.php | 160 -- src/core/etl/src/Flow/ETL/Join/Join.php | 13 - src/core/etl/src/Flow/ETL/Join/JoinSchema.php | 53 - src/core/etl/src/Flow/ETL/Join/JoinShape.php | 56 - src/core/etl/src/Flow/ETL/Join/JoinSteps.php | 47 - src/core/etl/src/Flow/ETL/Join/RowsJoin.php | 94 - .../ETL/{ => Kernels}/Bucketing/Bucket.php | 2 +- .../ETL/{ => Kernels}/Bucketing/BucketRun.php | 6 +- .../ETL/{ => Kernels}/Bucketing/Buckets.php | 9 +- .../ETL/Kernels/Bucketing/HashBucketing.php | 92 + .../Flow/ETL/Kernels/Bucketing/KeyEncoder.php | 95 + .../ETL/Kernels/Bucketing/KeyGrouping.php | 71 + .../Flow/ETL/Kernels/Bucketing/SeenKeys.php | 48 + .../ETL/Kernels/Comparison/ValueEquality.php | 254 ++ .../ETL/Kernels/Comparison/ValueOrder.php | 175 ++ .../Flow/ETL/{ => Kernels}/Hash/Algorithm.php | 2 +- .../Flow/ETL/Kernels/Hash/NativePHPHash.php | 62 + .../src/Flow/ETL/Kernels/Hash/PlainText.php | 28 + .../Flow/ETL/Kernels/HashJoin/HashTable.php | 63 + .../Flow/ETL/Kernels/HashJoin/JoinKeys.php | 56 + .../Flow/ETL/Kernels/HashJoin/JoinSide.php | 41 + .../src/Flow/ETL/Kernels/HashJoin/Joiner.php | 209 ++ .../ETL/Kernels/HashJoin/NullKeyFilter.php | 79 + .../ETL/Kernels/HashJoin/PairPredicate.php | 18 + .../src/Flow/ETL/Kernels/HashJoin/Role.php | 49 + .../Flow/ETL/Kernels/HashJoin/RowMerger.php | 50 + .../src/Flow/ETL/Kernels/Join/JoinSchema.php | 70 + .../Flow/ETL/Kernels/Sort/ExternalSort.php | 221 ++ .../ETL/Kernels/Sort/ExternalSortConfig.php | 45 + .../ETL/Kernels/Sort/MemorySortConfig.php | 23 + .../{ => Kernels}/Sort/Merge/BucketCursor.php | 12 +- .../Flow/ETL/Kernels/Sort/Merge/KWayMerge.php | 136 + .../src/Flow/ETL/Kernels/Sort/RowOrder.php | 251 ++ .../src/Flow/ETL/Kernels/Sort/SortColumn.php | 47 + .../etl/src/Flow/ETL/Kernels/Sort/SortKey.php | 55 + src/core/etl/src/Flow/ETL/Loader.php | 10 - .../etl/src/Flow/ETL/Loader/ArrayLoader.php | 38 - src/core/etl/src/Flow/ETL/Loader/Closure.php | 16 - .../etl/src/Flow/ETL/Loader/Discardable.php | 16 - .../src/Flow/ETL/Loader/File/FileLoader.php | 15 - .../etl/src/Flow/ETL/Loader/File/FileSink.php | 14 - .../src/Flow/ETL/Loader/File/FileSinks.php | 18 - .../Flow/ETL/Loader/File/FileWriteFrame.php | 114 - .../src/Flow/ETL/Loader/File/FilesSink.php | 168 -- .../Flow/ETL/Loader/File/PartitionRouter.php | 92 - .../etl/src/Flow/ETL/Loader/MemoryLoader.php | 35 - .../etl/src/Flow/ETL/Loader/Partitioning.php | 39 - .../Flow/ETL/Loader/PartitioningLoader.php | 12 - .../etl/src/Flow/ETL/Loader/RowPartitions.php | 37 - .../ETL/Loader/SchemaValidationLoader.php | 43 - .../etl/src/Flow/ETL/Loader/StreamLoader.php | 152 - .../src/Flow/ETL/Loader/StreamLoader/Type.php | 13 - .../ETL/NativePHPRandomValueGenerator.php | 28 - .../etl/src/Flow/ETL/Operators/BoundStep.php | 23 + .../Flow/ETL/{ => Operators}/Constraint.php | 4 +- .../Constraint/SortedByConstraint.php | 118 + .../Operators/Constraint/UniqueConstraint.php | 84 + .../etl/src/Flow/ETL/Operators/GroupBy.php | 218 ++ .../Operators/GroupBy/AggregatedGroups.php | 103 + .../ETL/Operators/GroupBy/Aggregators.php | 130 + .../GroupBy/DeclaredPivotValues.php | 9 +- .../GroupBy/DiscoveredPivotValues.php | 27 +- .../ETL/{ => Operators}/GroupBy/Group.php | 2 +- .../{ => Operators}/GroupBy/GroupByShape.php | 8 +- .../ETL/{ => Operators}/GroupBy/GroupKey.php | 2 +- .../ETL/Operators/GroupBy/MergedAggregate.php | 113 + .../Operators/GroupBy/PartialAggregate.php | 107 + .../src/Flow/ETL/Operators/GroupBy/Pivot.php | 15 + .../Flow/ETL/Operators/GroupBy/PivotShape.php | 43 + .../ETL/Operators/GroupBy/PivotValues.php | 19 + .../ETL/Operators/GroupBy/PivotedTable.php | 120 + .../src/Flow/ETL/Operators/Join/CrossJoin.php | 83 + .../src/Flow/ETL/Operators/Join/HashJoin.php | 331 +++ .../ETL/Operators/Join/HashJoinConfig.php | 41 + .../etl/src/Flow/ETL/Operators/Processor.php | 43 + .../Processor/BatchingByProcessor.php | 141 + .../Operators/Processor/BatchingProcessor.php | 95 + .../Processor/BoundExpansion.php | 57 +- .../Operators/Processor/CachingProcessor.php | 100 + .../Processor/CollectingProcessor.php | 69 + .../Processor/ConstrainedProcessor.php | 114 + .../Operators/Processor/CountingProcessor.php | 64 + .../Operators/Processor/DistinctProcessor.php | 111 + .../ETL/Operators/Processor/Distributable.php | 15 + .../ETL/Operators/Processor/Distribution.php | 53 + .../Processor/ExpandingProcessor.php | 124 + .../Processor/GroupByAggregationProcessor.php | 274 ++ .../Processor/MemorySortProcessor.php | 96 + .../Processor/MergeSortProcessor.php | 90 + .../Operators/Processor/OffsetProcessor.php | 113 + .../Operators/Processor/PivotProcessor.php | 122 + .../Processor/RepartitionProcessor.php | 151 + .../Processor}/RequiredColumns.php | 2 +- .../ETL/Operators/Processor/TopNProcessor.php | 169 ++ .../ETL/Operators/Processor/VoidProcessor.php | 58 + .../Operators/Processor/WindowProcessor.php | 158 ++ .../src/Flow/ETL/Operators/Transformer.php | 23 + .../Transformer/AddRowIndex/StartFrom.php | 11 + .../Transformer/AddRowIndexTransformer.php | 74 + .../CollectReferencesTransformer.php | 46 + .../Operators/Transformer/DerivedColumns.php | 53 + .../Transformer/DropColumnsTransformer.php | 46 + .../Transformer/DuplicateRowTransformer.php | 178 ++ .../{ => Operators}/Transformer/Expansion.php | 33 +- .../Transformer/JoinEachRowsTransformer.php | 108 + .../Transformer/LimitTransformer.php | 56 + .../ETL/Operators/Transformer/Limiting.php | 12 + .../Transformer/PruneColumnsTransformer.php | 67 + .../Rename/RenameCaseColumnStrategy.php | 49 + .../Rename/RenameColumnStrategy.php | 4 +- .../Rename/RenameMapColumnStrategy.php | 41 + .../Rename/RenameReplaceColumnStrategy.php | 61 + .../Transformer/RenameColumnTransformer.php | 52 + .../RenameEachColumnTransformer.php | 79 + .../ScalarFunctionFilterTransformer.php | 54 + .../Transformer/ScalarFunctionTransformer.php | 179 ++ .../Transformer/SelectColumnsTransformer.php | 82 + .../Transformer/SerializeTransformer.php | 87 + .../Transformer/SerializedPayloadDecoder.php | 86 + .../ETL/Operators/Transformer/Stateful.php | 17 + .../Transformer/UnpackedColumns.php | 18 +- .../Transformer/UnserializeTransformer.php | 92 + .../Transformer/UntilTransformer.php | 84 + .../etl/src/Flow/ETL/Optimizer/FilterWalk.php | 78 - .../etl/src/Flow/ETL/Optimizer/JoinSides.php | 35 - .../etl/src/Flow/ETL/Optimizer/LimitWalk.php | 50 - .../src/Flow/ETL/Optimizer/ProjectionWalk.php | 37 - src/core/etl/src/Flow/ETL/Optimizer/Rule.php | 19 - .../Flow/ETL/Optimizer/Rule/CombineLimits.php | 38 - .../Optimizer/Rule/CombineSortAndLimit.php | 25 - .../Optimizer/Rule/CountFromStatistics.php | 37 - .../Optimizer/Rule/PushFilterIntoSource.php | 108 - .../Optimizer/Rule/PushLimitIntoSource.php | 39 - .../Rule/PushProjectionIntoExpand.php | 51 - .../src/Flow/ETL/Optimizer/TopNRewrite.php | 33 - src/core/etl/src/Flow/ETL/Plan.php | 49 - src/core/etl/src/Flow/ETL/Plan/Explain.php | 38 - .../src/Flow/ETL/Plan/Explain/Branches.php | 29 - .../src/Flow/ETL/Plan/Explain/Condition.php | 66 - .../etl/src/Flow/ETL/Plan/Explain/Details.php | 305 -- .../src/Flow/ETL/Plan/Explain/Expression.php | 184 -- .../etl/src/Flow/ETL/Plan/Explain/Layout.php | 10 - .../etl/src/Flow/ETL/Plan/Explain/Outline.php | 84 - .../Flow/ETL/Plan/Explain/PhysicalOutline.php | 111 - .../src/Flow/ETL/Plan/Explain/StepDetails.php | 254 -- .../src/Flow/ETL/Plan/Explain/TreeLayout.php | 45 - src/core/etl/src/Flow/ETL/Plan/Format.php | 28 - .../etl/src/Flow/ETL/Plan/LogicalPlan.php | 126 - .../etl/src/Flow/ETL/Plan/Materialization.php | 11 - src/core/etl/src/Flow/ETL/Plan/Node.php | 62 - .../etl/src/Flow/ETL/Plan/Node/Aggregate.php | 58 - src/core/etl/src/Flow/ETL/Plan/Node/Batch.php | 63 - .../etl/src/Flow/ETL/Plan/Node/BatchBy.php | 65 - src/core/etl/src/Flow/ETL/Plan/Node/Cache.php | 63 - .../etl/src/Flow/ETL/Plan/Node/Collect.php | 51 - .../src/Flow/ETL/Plan/Node/CollectRefs.php | 56 - .../etl/src/Flow/ETL/Plan/Node/Constrain.php | 59 - src/core/etl/src/Flow/ETL/Plan/Node/Count.php | 51 - .../etl/src/Flow/ETL/Plan/Node/CrossJoin.php | 73 - .../etl/src/Flow/ETL/Plan/Node/Discard.php | 51 - .../etl/src/Flow/ETL/Plan/Node/Distinct.php | 66 - src/core/etl/src/Flow/ETL/Plan/Node/Drop.php | 57 - .../etl/src/Flow/ETL/Plan/Node/Filter.php | 53 - src/core/etl/src/Flow/ETL/Plan/Node/Join.php | 80 - .../etl/src/Flow/ETL/Plan/Node/JoinEach.php | 60 - src/core/etl/src/Flow/ETL/Plan/Node/Limit.php | 60 - src/core/etl/src/Flow/ETL/Plan/Node/Read.php | 118 - .../etl/src/Flow/ETL/Plan/Node/Rename.php | 53 - .../etl/src/Flow/ETL/Plan/Node/RenameEach.php | 63 - .../src/Flow/ETL/Plan/Node/Repartition.php | 56 - .../etl/src/Flow/ETL/Plan/Node/Select.php | 57 - src/core/etl/src/Flow/ETL/Plan/Node/Sort.php | 58 - src/core/etl/src/Flow/ETL/Plan/Node/TopN.php | 70 - .../src/Flow/ETL/Plan/Node/Transaction.php | 79 - src/core/etl/src/Flow/ETL/Plan/Node/Until.php | 53 - .../etl/src/Flow/ETL/Plan/Node/Validate.php | 58 - .../src/Flow/ETL/Plan/Node/WindowColumn.php | 66 - .../etl/src/Flow/ETL/Plan/Node/WithColumn.php | 67 - src/core/etl/src/Flow/ETL/Plan/Node/Write.php | 56 - .../etl/src/Flow/ETL/Plan/Redefinition.php | 12 - src/core/etl/src/Flow/ETL/Plan/Sinks.php | 47 - .../etl/src/Flow/ETL/Plan/Transparency.php | 11 - src/core/etl/src/Flow/ETL/Planner.php | 110 - .../src/Flow/ETL/Planner/NodeTranslator.php | 131 - .../src/Flow/ETL/Planner/PipelineSplit.php | 97 - .../etl/src/Flow/ETL/Planner/PlannedNode.php | 26 - .../src/Flow/ETL/Planner/SinkAttachment.php | 295 -- .../src/Flow/ETL/Planner/SinkFeedFactory.php | 61 - .../{Plan => Planning}/Explain/BoxLayout.php | 2 +- .../Flow/ETL/Planning/Explain/Condition.php | 69 + .../ETL/{Plan => Planning}/Explain/Entry.php | 2 +- .../Flow/ETL/Planning/Explain/Expression.php | 174 ++ .../{Plan => Planning}/Explain/FlowLayout.php | 4 +- .../src/Flow/ETL/Planning/Explain/Layout.php | 10 + .../ETL/Planning/Explain/LogicalOutline.php | 85 + .../Flow/ETL/Planning/Explain/NodeDetails.php | 296 ++ .../ETL/Planning/Explain/PhysicalOutline.php | 103 + .../Explain/StatisticsLine.php | 6 +- .../Flow/ETL/Planning/Explain/StepDetails.php | 223 ++ .../Flow/ETL/Planning/Explain/TreeLayout.php | 65 + .../src/Flow/ETL/Planning/Logical/Explain.php | 41 + .../src/Flow/ETL/Planning/Logical/Format.php | 28 + .../Flow/ETL/Planning/Logical/LogicalPlan.php | 175 ++ .../src/Flow/ETL/Planning/Logical/Node.php | 51 + .../ETL/Planning/Logical/Node/Aggregate.php | 51 + .../Flow/ETL/Planning/Logical/Node/Batch.php | 47 + .../ETL/Planning/Logical/Node/BatchBy.php | 59 + .../Node/BucketingAlgorithmBuilder.php | 14 + .../Flow/ETL/Planning/Logical/Node/Cache.php | 57 + .../ETL/Planning/Logical/Node/Collect.php | 45 + .../ETL/Planning/Logical/Node/CollectRefs.php | 50 + .../ETL/Planning/Logical/Node/Constrain.php | 53 + .../Flow/ETL/Planning/Logical/Node/Count.php | 45 + .../ETL/Planning/Logical/Node/CrossJoin.php | 67 + .../ETL/Planning/Logical/Node/Discard.php | 45 + .../ETL/Planning/Logical/Node/Distinct.php | 60 + .../Flow/ETL/Planning/Logical/Node/Drop.php | 51 + .../Logical}/Node/DuplicateRow.php | 18 +- .../Logical}/Node/ExpandColumn.php | 24 +- .../Flow/ETL/Planning/Logical/Node/Filter.php | 47 + .../Flow/ETL/Planning/Logical/Node/Join.php | 71 + .../Logical/Node/JoinAlgorithmBuilder.php | 14 + .../ETL/Planning/Logical/Node/JoinEach.php | 54 + .../Logical}/Node/JoinsFrame.php | 4 +- .../Flow/ETL/Planning/Logical/Node/Limit.php | 54 + .../Logical}/Node/Offset.php | 18 +- .../Logical}/Node/Outputs.php | 28 +- .../Flow/ETL/Planning/Logical/Node/Read.php | 123 + .../Flow/ETL/Planning/Logical/Node/Rename.php | 47 + .../ETL/Planning/Logical/Node/RenameEach.php | 57 + .../ETL/Planning/Logical/Node/Repartition.php | 50 + .../Logical}/Node/Result.php | 16 +- .../Flow/ETL/Planning/Logical/Node/Select.php | 51 + .../Flow/ETL/Planning/Logical/Node/Sort.php | 52 + .../Logical/Node/SortAlgorithmBuilder.php | 15 + .../Flow/ETL/Planning/Logical/Node/TopN.php | 64 + .../ETL/Planning/Logical/Node/Transaction.php | 73 + .../Logical}/Node/Transform.php | 18 +- .../Flow/ETL/Planning/Logical/Node/Until.php | 47 + .../ETL/Planning/Logical/Node/Validate.php | 52 + .../Planning/Logical/Node/WindowColumn.php | 60 + .../ETL/Planning/Logical/Node/WithColumn.php | 61 + .../Flow/ETL/Planning/Logical/Node/Write.php | 50 + .../src/Flow/ETL/Planning/Logical/Plan.php | 40 + .../{Plan => Planning/Logical}/Redefined.php | 2 +- .../ETL/Planning/Logical/Redefinition.php | 12 + .../Logical}/ReplaceNodes.php | 4 +- .../{Plan => Planning/Logical}/Rewrite.php | 2 +- .../{Plan => Planning/Logical}/RowCount.php | 2 +- .../ETL/{Plan => Planning/Logical}/Stage.php | 2 +- .../Logical}/TransformUp.php | 2 +- .../ETL/Planning/Logical/Transparency.php | 11 + .../{Plan => Planning/Logical}/Trigger.php | 6 +- .../src/Flow/ETL/Planning/Logical/Writes.php | 47 + .../Flow/ETL/Planning/Optimizer/JoinSides.php | 34 + .../{ => Planning/Optimizer}/Optimizer.php | 31 +- .../src/Flow/ETL/Planning/Optimizer/Rule.php | 19 + .../Planning/Optimizer/Rule/CombineLimits.php | 38 + .../Optimizer/Rule/CombineSortAndLimit.php | 42 + .../Optimizer/Rule/CountFromStatistics.php | 33 + .../Optimizer/Rule/PushFilterIntoSource.php | 179 ++ .../Optimizer/Rule/PushLimitIntoSource.php | 77 + .../Optimizer/Rule/PushOffsetIntoSource.php | 73 + .../Rule/PushProjectionIntoExpand.php | 75 + .../ETL/Planning/Physical/BranchPipeline.php | 46 + .../Flow/ETL/Planning/Physical/JoinStep.php | 16 + .../ETL/Planning/Physical/NodeTranslator.php | 194 ++ .../ETL/Planning/Physical/PhysicalPlan.php | 98 + .../Flow/ETL/Planning/Physical/Pipeline.php | 41 + .../ETL/Planning/Physical/PipelineSteps.php | 70 + .../ETL/Planning/Physical/PlannedNode.php | 26 + .../Physical}/PlannedNodes.php | 16 +- .../Flow/ETL/Planning/Physical/Planner.php | 235 ++ .../Planning/Physical/TransactionGroup.php | 19 + .../ETL/Planning/Physical/WritePlanner.php | 299 ++ src/core/etl/src/Flow/ETL/Processor.php | 32 - .../ETL/Processor/BatchingByProcessor.php | 120 - .../Flow/ETL/Processor/BatchingProcessor.php | 79 - .../Flow/ETL/Processor/CachingProcessor.php | 72 - .../ETL/Processor/CollectingProcessor.php | 47 - .../ETL/Processor/ConstrainedProcessor.php | 94 - .../Flow/ETL/Processor/CountingProcessor.php | 49 - .../Flow/ETL/Processor/ExpandingProcessor.php | 87 - .../Processor/GroupByAggregationProcessor.php | 185 -- .../Flow/ETL/Processor/HashJoinProcessor.php | 351 --- .../ETL/Processor/MemorySortProcessor.php | 58 - .../Flow/ETL/Processor/MergeSortProcessor.php | 29 - .../Flow/ETL/Processor/OffsetProcessor.php | 87 - .../src/Flow/ETL/Processor/PivotProcessor.php | 66 - .../ETL/Processor/RepartitionProcessor.php | 106 - .../src/Flow/ETL/Processor/TopNProcessor.php | 108 - .../src/Flow/ETL/Processor/VoidProcessor.php | 36 - .../Flow/ETL/Processor/WindowProcessor.php | 196 -- .../Flow/ETL/Repartition/RepartitionSteps.php | 50 - src/core/etl/src/Flow/ETL/Row/Reference.php | 28 - .../src/Flow/ETL/Row/ResolvedReference.php | 128 - .../src/Flow/ETL/Row/TypedValueComparator.php | 73 - .../src/Flow/ETL/Row/UnresolvedReference.php | 163 -- src/core/etl/src/Flow/ETL/Rows/RowHashes.php | 58 - src/core/etl/src/Flow/ETL/RowsFactory.php | 13 - .../src/Flow/ETL/{ => Runtime}/Analyze.php | 2 +- .../etl/src/Flow/ETL/Runtime/CacheConfig.php | 18 + src/core/etl/src/Flow/ETL/Runtime/Config.php | 159 ++ .../etl/src/Flow/ETL/Runtime/ErrorHandler.php | 22 + .../Runtime/ErrorHandler/ExtractionAction.php | 11 + .../Runtime/ErrorHandler/ExtractionError.php | 16 + .../ETL/Runtime/ErrorHandler/IgnoreError.php | 38 + .../Runtime/ErrorHandler/LoadingAction.php | 11 + .../ETL/Runtime/ErrorHandler/LoadingError.php | 18 + .../ETL/Runtime/ErrorHandler/SkipRows.php | 38 + .../ETL/Runtime/ErrorHandler/ThrowError.php | 38 + .../ErrorHandler/TransformationAction.php | 11 + .../ErrorHandler/TransformationError.php | 18 + .../{ => Runtime}/Executor/AdoptedBatches.php | 8 +- .../Flow/ETL/Runtime/Executor/BranchFiber.php | 80 + .../ETL/Runtime/Executor/BranchLoader.php | 101 + .../Flow/ETL/Runtime/Executor/Executor.php | 219 ++ .../ETL/Runtime/Executor/NestedLoader.php | 24 + .../ETL/Runtime/Executor/OfferedFailure.php | 78 + .../Flow/ETL/Runtime/Executor/SourceCount.php | 81 + .../ETL/Runtime/Executor/SourceCounts.php | 107 + .../Executor/StatisticsCollector.php | 39 +- .../Flow/ETL/Runtime/Executor/StepRunner.php | 311 ++ .../ETL/Runtime/Executor/StepTelemetry.php | 111 + .../Runtime/Executor/TransactionLoader.php | 122 + .../Runtime/Executor/WriteLoaderFactory.php | 104 + .../etl/src/Flow/ETL/Runtime/FlowContext.php | 60 + .../src/Flow/ETL/Runtime/Launcher/Launch.php | 24 + .../Flow/ETL/Runtime/Launcher/Launcher.php | 28 + .../etl/src/Flow/ETL/Runtime/MemoryLimit.php | 44 + .../{Dataset => Runtime/Metrics}/Report.php | 4 +- .../Metrics}/SourceStatistics.php | 4 +- .../Flow/ETL/Runtime/Metrics/Statistics.php | 24 + .../ETL/Runtime/Metrics/Statistics/Column.php | 234 ++ .../Runtime/Metrics/Statistics/Columns.php | 61 + .../Metrics}/Statistics/ExecutionTime.php | 5 +- .../Telemetry/SpanScope.php | 4 +- .../Flow/ETL/Runtime/Telemetry/SpanStack.php | 49 + .../Flow/ETL/Runtime/Telemetry/StepKind.php | 41 + .../Telemetry/TelemetryConfig.php | 2 +- .../Runtime/Telemetry/TelemetryContext.php | 367 +++ .../Telemetry/TelemetryOptions.php | 2 +- .../Telemetry/TraceableRowsStorage.php | 119 + src/core/etl/src/Flow/ETL/Schema.php | 642 ----- .../Schema/Definition/BooleanDefinition.php | 161 -- .../Flow/ETL/Schema/Definition/CommonType.php | 37 - .../ETL/Schema/Definition/DateDefinition.php | 170 -- .../Schema/Definition/DateTimeDefinition.php | 169 -- .../ETL/Schema/Definition/EnumDefinition.php | 191 -- .../ETL/Schema/Definition/FloatDefinition.php | 169 -- .../ETL/Schema/Definition/HTMLDefinition.php | 162 -- .../Definition/HTMLElementDefinition.php | 162 -- .../Schema/Definition/IntegerDefinition.php | 169 -- .../ETL/Schema/Definition/JsonDefinition.php | 162 -- .../ETL/Schema/Definition/ListDefinition.php | 199 -- .../ETL/Schema/Definition/MapDefinition.php | 206 -- .../ETL/Schema/Definition/NullDefinition.php | 144 - .../Schema/Definition/StringDefinition.php | 154 - .../Schema/Definition/StructureDefinition.php | 217 -- .../ETL/Schema/Definition/TimeDefinition.php | 165 -- .../Schema/Definition/TimeZoneDefinition.php | 161 -- .../ETL/Schema/Definition/UuidDefinition.php | 162 -- .../Flow/ETL/Schema/Definition/ValueMatch.php | 22 - .../ETL/Schema/Definition/XMLDefinition.php | 162 -- .../Definition/XMLElementDefinition.php | 161 -- .../Schema/Formatter/PHPSchemaFormatter.php | 274 -- .../Flow/ETL/Schema/Inference/ColumnTypes.php | 144 - .../ETL/Schema/Inference/SchemaInference.php | 28 - .../Inference/SchemaInferenceBuilder.php | 86 - .../ETL/Schema/Inference/SchemaInferrer.php | 85 - .../ETL/Schema/Inference/SchemaSampler.php | 17 - .../Schema/Inference/SniffsColumnTypes.php | 27 - .../src/Flow/ETL/Schema/SchemaFormatter.php | 12 - .../SortingStrategy/AlphabeticalStrategy.php | 29 - .../Schema/SortingStrategy/TypeStrategy.php | 30 - .../Schema/Validator/EvolvingValidator.php | 64 - .../Schema/Validator/SelectiveValidator.php | 41 - .../ETL/Schema/Validator/StrictValidator.php | 48 - src/core/etl/src/Flow/ETL/SchemaValidator.php | 12 - src/core/etl/src/Flow/ETL/Sink.php | 14 - src/core/etl/src/Flow/ETL/Sink/Branched.php | 32 - .../etl/src/Flow/ETL/Sink/Transactional.php | 50 - .../etl/src/Flow/ETL/Sink/Transformed.php | 29 - .../etl/src/Flow/ETL/Sort/ExternalSort.php | 203 -- .../etl/src/Flow/ETL/Sort/Merge/KWayMerge.php | 123 - src/core/etl/src/Flow/ETL/Sort/RowOrder.php | 191 -- src/core/etl/src/Flow/ETL/Sort/SortKey.php | 23 - src/core/etl/src/Flow/ETL/Sort/SortSteps.php | 56 - .../src/Flow/ETL/Storage/ApcuRowsStorage.php | 140 + .../src/Flow/ETL/Storage/BucketingConfig.php | 67 + .../ETL/Storage/FilesystemRowsStorage.php | 252 ++ .../Exception/IncompatibleSchemaException.php | 9 + .../src/Flow/ETL/Storage/Floe/FloeFiles.php | 98 + .../{ => ETL/Storage}/Floe/FloeMerger.php | 81 +- .../Flow/ETL/Storage/Floe/FloeSerializer.php | 97 + .../ETL/Storage/Floe/FloeStreamReader.php | 325 +++ .../ETL/Storage/Floe/FloeStreamWriter.php | 258 ++ .../Flow/{ => ETL/Storage}/Floe/Footer.php | 14 +- .../{ => ETL/Storage}/Floe/FooterLocation.php | 2 +- .../{ => ETL/Storage}/Floe/FooterReader.php | 5 +- .../ETL/Storage/Floe/FooterStatistics.php | 64 + .../etl/src/Flow/ETL/Storage/Floe/Format.php | 102 + .../{ => ETL/Storage}/Floe/FrameWriter.php | 2 +- .../etl/src/Flow/ETL/Storage/Floe/Options.php | 31 + .../Flow/{ => ETL/Storage}/Floe/Section.php | 4 +- .../Storage}/Floe/SplittingFrameEncoder.php | 16 +- .../ETL/{ => Storage}/Memory/ArrayMemory.php | 4 +- .../Flow/ETL/{ => Storage}/Memory/Memory.php | 2 +- .../Flow/ETL/Storage/MemoryRowsStorage.php | 91 + .../src/Flow/ETL/Storage/PsrRowsStorage.php | 139 + .../Flow/ETL/Storage/ResidentRowsStorage.php | 7 + .../etl/src/Flow/ETL/Storage/RowsStorage.php | 42 + .../Serializer/CompressingSerializer.php | 98 + .../Exception/SerializationException.php | 9 + .../ETL/Storage/Serializer/Serializer.php | 23 + .../Flow/ETL/Storage/SpillingRowsStorage.php | 167 ++ src/core/etl/src/Flow/ETL/Transaction.php | 19 - .../Flow/ETL/Transformation/AddRowIndex.php | 23 - .../Transformation/AddRowIndex/StartFrom.php | 11 - .../src/Flow/ETL/Transformation/BatchBy.php | 33 - .../src/Flow/ETL/Transformation/BatchSize.php | 37 - .../etl/src/Flow/ETL/Transformation/Drop.php | 25 - .../etl/src/Flow/ETL/Transformation/Limit.php | 20 - .../Flow/ETL/Transformation/MaskColumns.php | 34 - .../src/Flow/ETL/Transformation/Select.php | 25 - src/core/etl/src/Flow/ETL/Transformer.php | 21 - .../Transformer/AddRowIndexTransformer.php | 67 - .../CollectReferencesTransformer.php | 48 - .../Transformer/CrossJoinRowsTransformer.php | 62 - .../Flow/ETL/Transformer/DerivedColumns.php | 63 - .../Transformer/DropColumnsTransformer.php | 51 - .../ETL/Transformer/DropDuplicates/Hashes.php | 25 - .../Transformer/DropDuplicatesTransformer.php | 93 - .../Transformer/DuplicateRowTransformer.php | 177 -- .../Transformer/JoinEachRowsTransformer.php | 90 - .../Flow/ETL/Transformer/LimitTransformer.php | 68 - .../Transformer/PruneColumnsTransformer.php | 72 - .../Rename/RenameCaseColumnStrategy.php | 30 - .../Rename/RenameMapColumnStrategy.php | 25 - .../Rename/RenameReplaceColumnStrategy.php | 36 - .../Transformer/RenameColumnTransformer.php | 54 - .../RenameEachColumnTransformer.php | 74 - .../ScalarFunctionFilterTransformer.php | 55 - .../Transformer/ScalarFunctionTransformer.php | 165 -- .../Transformer/SelectColumnsTransformer.php | 51 - .../ETL/Transformer/SerializeTransformer.php | 85 - .../Transformer/SerializedPayloadDecoder.php | 76 - .../etl/src/Flow/ETL/Transformer/Stateful.php | 19 - .../Transformer/UnserializeTransformer.php | 84 - .../Flow/ETL/Transformer/UntilTransformer.php | 112 - .../etl/src/Flow/ETL/Values/BatchSize.php | 24 + .../src/Flow/ETL/{ => Values}/Cardinality.php | 4 +- .../ETL/Values/Column/AdaptiveBackend.php | 70 + .../src/Flow/ETL/Values/Column/Backend.php | 78 + .../etl/src/Flow/ETL/Values/Column/Column.php | 80 + .../ETL/{ => Values}/Column/ColumnBuilder.php | 6 +- .../ETL/{Row => Values/Column}/ColumnName.php | 2 +- .../{ => Values}/Column/ComparableValues.php | 67 +- .../{ => Values}/Column/DateFormatPattern.php | 2 +- .../Flow/ETL/{ => Values}/Column/Elements.php | 2 +- .../Values/Column/Frame}/Codec.php | 2 +- .../ETL/Values/Column/Frame/FloeException.php | 9 + .../ETL/Values/Column/Frame/FrameTooLarge.php | 7 + .../Values/Column/Frame}/NoopCodec.php | 4 +- .../Column/Layout/BooleanLayout.php | 4 +- .../Column/Layout/BufferLayout.php | 10 +- .../{ => Values}/Column/Layout/Buffers.php | 4 +- .../Column/Layout/FixedBinary16Layout.php | 4 +- .../Column/Layout/Float64Layout.php | 4 +- .../Column/Layout/Int32Layout.php | 4 +- .../Column/Layout/Int64Layout.php | 4 +- .../Flow/ETL/Values/Column/Layout/Layout.php | 22 + .../{ => Values}/Column/Layout/NullCount.php | 2 +- .../{ => Values}/Column/Layout/Offsets.php | 6 +- .../{ => Values}/Column/Layout/Utf8Layout.php | 4 +- .../{ => Values}/Column/Layout/Validity.php | 4 +- .../Column/Php/CastingColumnBuilder.php | 244 ++ .../{ => Values}/Column/Php/ColumnDecoder.php | 27 +- .../Column/Php/ConstantColumn.php | 15 +- .../{ => Values}/Column/Php/ListColumn.php | 12 +- .../Column/Php/ListColumnBuilder.php | 4 +- .../ETL/{ => Values}/Column/Php/MapColumn.php | 12 +- .../Column/Php/MapColumnBuilder.php | 4 +- .../Column/Php/NullColumnBuilder.php | 6 +- .../Column/Php/PhpColumnBuilder.php | 4 +- .../{ => Values}/Column/Php/ScalarColumn.php | 25 +- .../Column/Php/ScalarColumnBuilder.php | 6 +- .../{ => Values}/Column/Php/StructColumn.php | 10 +- .../Column/Php/StructColumnBuilder.php | 4 +- .../src/Flow/ETL/Values/Column/PhpBackend.php | 359 +++ .../Column/Physical/DatePhysical.php | 2 +- .../Column/Physical/DateTimePhysical.php | 2 +- .../Values/Column/Physical/DaysFromCivil.php | 28 + .../Column/Physical/ElementPosition.php | 2 +- .../Column/Physical/EnumPhysical.php | 2 +- .../Column/Physical/HtmlDocumentPhysical.php | 4 +- .../Column/Physical/HtmlElementPhysical.php | 4 +- .../Column/Physical/IdentityPhysical.php | 2 +- .../Column/Physical/JsonFactory.php | 2 +- .../Column/Physical/JsonPhysical.php | 2 +- .../Column/Physical/ListPhysical.php | 2 +- .../Column/Physical/MapPhysical.php | 2 +- .../Column/Physical/NullPhysical.php | 2 +- .../{ => Values}/Column/Physical/Physical.php | 2 +- .../Values/Column/Physical/PhysicalFor.php | 409 +++ .../Column/Physical/StructPhysical.php | 2 +- .../Column/Physical/TimePhysical.php | 2 +- .../Column/Physical/TimeZonePhysical.php | 2 +- .../Column/Physical/TimeZones.php | 2 +- .../Column/Physical/UuidPhysical.php | 2 +- .../Physical/XmlCharacterReferences.php | 8 +- .../Column/Physical/XmlDocumentPhysical.php | 4 +- .../Column/Physical/XmlElementPhysical.php | 4 +- .../Flow/ETL/{ => Values}/Column/Retype.php | 4 +- .../ETL/{ => Values}/Column/TextValues.php | 18 +- .../ETL/{ => Values}/Column/ValueColumn.php | 6 +- .../etl/src/Flow/ETL/Values/Join/JoinType.php | 36 + .../etl/src/Flow/ETL/Values/Join/Side.php | 11 + .../Memory/BoundedRead.php | 8 +- .../Memory/Consumption.php | 4 +- .../Memory/MemoryBudget.php | 4 +- .../ETL/{Dataset => Values}/Memory/Unit.php | 8 +- .../Flow/ETL/{Row => Values}/NullsOrder.php | 2 +- .../etl/src/Flow/ETL/{ => Values}/Rows.php | 197 +- .../ETL/{ => Values}/Rows/ColumnRetyping.php | 25 +- .../src/Flow/ETL/Values/Rows/RecordBatch.php | 70 + .../ETL/{ => Values}/Rows/RowsBuilder.php | 138 +- src/core/etl/src/Flow/ETL/Values/Schema.php | 625 ++++ .../ETL/{ => Values}/Schema/Definition.php | 5 +- .../Schema/Definition/BooleanDefinition.php | 130 + .../Schema/Definition/DateDefinition.php | 130 + .../Schema/Definition/DateTimeDefinition.php | 127 + .../Definition/ElementCompatibility.php | 9 +- .../Schema/Definition/EnumDefinition.php | 155 + .../Schema/Definition/FloatDefinition.php | 130 + .../Schema/Definition/HTMLDefinition.php | 131 + .../Definition/HTMLElementDefinition.php | 131 + .../Schema/Definition/IntegerDefinition.php | 130 + .../Schema/Definition/JsonDefinition.php | 131 + .../Schema/Definition/ListDefinition.php | 166 ++ .../Schema/Definition/MapDefinition.php | 173 ++ .../Schema/Definition/NullDefinition.php | 129 + .../Schema/Definition/StringDefinition.php | 134 + .../Schema/Definition/StructureDefinition.php | 180 ++ .../Schema/Definition/TimeDefinition.php | 130 + .../Schema/Definition/TimeZoneDefinition.php | 130 + .../Schema/Definition/TypeProjection.php | 7 +- .../Schema/Definition/UuidDefinition.php | 131 + .../Schema/Definition/XMLDefinition.php | 131 + .../Definition/XMLElementDefinition.php | 130 + .../Flow/ETL/Values/Schema/Definitions.php | 199 ++ .../Flow/ETL/{ => Values}/Schema/Metadata.php | 6 +- .../ETL/{ => Values}/Schema/SameSchemas.php | 4 +- .../{ => Values}/Schema/SortingStrategy.php | 2 +- .../SortingStrategy/AlphabeticalStrategy.php | 29 + .../SortingStrategy/CombinedStrategy.php | 6 +- .../SortingStrategy/MetadataStrategy.php | 12 +- .../Schema/SortingStrategy/TypeStrategy.php | 30 + .../TypeStrategy/TypePriorities.php | 4 +- .../Schema/Validator/EvolvingValidator.php | 78 + .../Schema/Validator/MismatchedDefinition.php | 4 +- .../Schema/Validator/SelectiveValidator.php | 55 + .../Schema/Validator/StrictValidator.php | 62 + .../Schema/Validator/ValidationContext.php | 10 +- .../src/Flow/ETL/Values/SchemaValidator.php | 12 + .../Flow/ETL/{Row => Values}/SortOrder.php | 2 +- src/core/etl/src/Flow/ETL/Window.php | 69 - .../Window/Accumulator/AverageAccumulator.php | 61 - .../Window/Accumulator/CountAccumulator.php | 43 - .../ETL/Window/Accumulator/SumAccumulator.php | 49 - .../etl/src/Flow/ETL/Window/BoundWindow.php | 22 - .../src/Flow/ETL/Window/FrameAccumulator.php | 18 - .../etl/src/Flow/ETL/Window/FrameBound.php | 44 - .../src/Flow/ETL/Window/PeerComparator.php | 94 - .../etl/src/Flow/ETL/Window/PeerFrame.php | 62 - .../etl/src/Flow/ETL/Window/RowsFrame.php | 32 - .../Flow/ETL/Window/WholePartitionFrame.php | 15 - .../etl/src/Flow/ETL/Window/WindowContext.php | 51 - .../etl/src/Flow/ETL/Window/WindowFrame.php | 18 - src/core/etl/src/Flow/ETL/WithColumn.php | 15 - src/core/etl/src/Flow/Floe/DSL/functions.php | 16 +- .../src/Flow/Floe/Exception/FloeException.php | 9 - .../src/Flow/Floe/Exception/FrameTooLarge.php | 7 - .../Exception/IncompatibleSchemaException.php | 7 - src/core/etl/src/Flow/Floe/FloeExtractor.php | 208 -- .../etl/src/Flow/Floe/FloeFileBatches.php | 83 - src/core/etl/src/Flow/Floe/FloeFileSink.php | 31 - src/core/etl/src/Flow/Floe/FloeFileSinks.php | 47 - src/core/etl/src/Flow/Floe/FloeLoader.php | 115 - src/core/etl/src/Flow/Floe/FloeReader.php | 29 - src/core/etl/src/Flow/Floe/FloeSerializer.php | 83 - src/core/etl/src/Flow/Floe/FloeSourceFile.php | 42 - .../etl/src/Flow/Floe/FloeStreamReader.php | 362 --- .../etl/src/Flow/Floe/FloeStreamWriter.php | 257 -- src/core/etl/src/Flow/Floe/FloeWriter.php | 111 - src/core/etl/src/Flow/Floe/Format.php | 102 - src/core/etl/src/Flow/Floe/FrameDecoder.php | 200 -- src/core/etl/src/Flow/Floe/FrameEncoder.php | 80 - src/core/etl/src/Flow/Floe/FrameReader.php | 106 - src/core/etl/src/Flow/Floe/Options.php | 30 - src/core/etl/src/Flow/Floe/Statistics.php | 64 - .../src/Flow/Serializer/Base64Serializer.php | 42 - .../Flow/Serializer/CompressingSerializer.php | 74 - .../etl/src/Flow/Serializer/DSL/functions.php | 4 +- .../Exception/SerializationException.php | 9 - .../Flow/Serializer/NativePHPSerializer.php | 45 - .../etl/src/Flow/Serializer/Serializer.php | 23 - .../Calculator/Tests/Unit/RunningSumTest.php | 130 - .../etl/tests/Flow/Clock/SystemClockTest.php | 32 - .../ETL/Tests/Comparator/ColumnComparator.php | 2 +- .../ETL/Tests/Comparator/RowsComparator.php | 2 +- .../ETL/Tests/Context/AggregatorFixtures.php | 132 + .../ETL/Tests/Context/AggregatorRoundTrip.php | 88 + .../Tests/Context/BucketsStorageContext.php | 31 - .../ETL/Tests/Context/CacheDirContext.php | 39 + .../{ => Context}/CommandOutputNormalizer.php | 12 +- .../ETL/Tests/Context/DescribableFixtures.php | 481 ++++ .../ETL/Tests/Context/DescribeRoundTrip.php | 37 + .../ETL/Tests/Context/ExecutedPipelines.php | 34 + .../Flow/ETL/Tests/Context/ExecutedPlan.php | 10 +- .../ETL/Tests/Context/ExecutedSegments.php | 29 - .../Tests/Context/ExpandColumnsContext.php | 8 +- .../Context/ExpandingProcessorContext.php | 10 +- .../ETL/Tests/Context/ExpansionContext.php | 8 +- .../Flow/ETL/Tests/Context/ExtractedRows.php | 28 +- .../ETL/Tests/Context/ExtractorClasses.php | 52 +- .../ETL/Tests/Context/FileColumnsContext.php | 9 +- .../ETL/Tests/Context/FileCommitContext.php | 57 + .../Flow/ETL/Tests/Context/FileFormat.php | 27 + .../ETL/Tests/Context/FileFormatContract.php | 364 +++ .../Flow/ETL/Tests/Context/FloeFileFormat.php | 32 + .../ETL/Tests/Context/FloeFilesContext.php | 180 ++ .../Tests/Context/FloeGoldenContext.php | 13 +- .../Tests/Context/FloeSchemaContext.php | 4 +- .../Tests/Context/FloeStreamReaderContext.php | 238 ++ .../Tests/Context/FrameContext.php | 2 +- .../ETL/Tests/Context/FunctionContext.php | 48 +- .../Flow/ETL/Tests/Context/GroupByContext.php | 75 +- .../Flow/ETL/Tests/Context/GroupedRows.php | 2 +- .../tests/Flow/ETL/Tests/Context/IdRows.php | 2 +- .../ETL/Tests/Context/KeyGroupingContext.php | 14 +- .../ETL/Tests/Context/LoaderEndingContext.php | 18 +- .../Tests/Context/MemoryTelemetryContext.php | 65 +- .../Tests/Context/MetadataColumnsContract.php | 35 + .../Tests/Context/PartitionRoutingContext.php | 8 +- .../ETL/Tests/Context/PipelineStepList.php | 44 + .../Flow/ETL/Tests/Context/PipelineSteps.php | 45 - .../Flow/ETL/Tests/Context/RankingContext.php | 4 +- .../ETL/Tests/Context/ReaderRoundTrip.php | 86 + .../ETL/Tests/Context/ReversedSniffing.php | 37 + .../Context/RowsSerializationContext.php | 31 - .../ETL/Tests/Context/RowsStorageContext.php | 31 + .../Tests/Context/ScalarFunctionClasses.php | 35 +- .../Tests/Context/ScalarFunctionContract.php | 183 ++ .../Tests/Context/ScalarFunctionFixtures.php | 788 +++--- .../Flow/ETL/Tests/Context/ScoredRows.php | 2 +- .../Flow/ETL/Tests/Context/ScratchBuckets.php | 40 + .../Context/SelfDescribingFilesContext.php | 8 +- .../Flow/ETL/Tests/Context/SortedRows.php | 29 + .../Flow/ETL/Tests/Context/SplitCoverage.php | 80 + .../Flow/ETL/Tests/Context/StepRoundTrip.php | 52 + .../ETL/Tests/Context/TextWriterBatches.php | 4 +- .../ETL/Tests/Context/WindowFrameContext.php | 4 +- .../Tests/Context/WindowProcessorContext.php | 12 +- .../Flow/ETL/Tests/Context/WrittenFiles.php | 68 + .../ETL/Tests/Double/AcceptingBackend.php | 23 +- .../Tests/Double/AcceptingColumnBuilder.php | 8 +- .../AddStampToStringColumnTransformer.php | 26 +- .../Double/BackendRecordingTransformer.php | 45 + .../Flow/ETL/Tests/Double/CallOrderLoader.php | 22 +- .../Tests/Double/CallbackTransformation.php | 4 +- .../Flow/ETL/Tests/Double/ChildlessNode.php | 14 +- .../Tests/Double/ClosingSpySourceStream.php | 2 +- .../Tests/Double/ClosureThrowingLoader.php | 30 +- .../{Floe => ETL}/Tests/Double/CodecStub.php | 4 +- .../ETL/Tests/Double/ConstantComparison.php | 20 +- .../Flow/ETL/Tests/Double/ConstantHasher.php | 22 - .../Double/ContextRecordingTransformer.php | 31 - .../ETL/Tests/Double/CountingAggregate.php | 102 + .../Flow/ETL/Tests/Double/CountingCache.php | 53 - .../ETL/Tests/Double/CountingExtractor.php | 52 +- .../ETL/Tests/Double/CountingFilesystem.php | 16 +- .../Double/CountingFrameAccumulating.php | 75 +- .../Tests/Double/CountingFrameAccumulator.php | 38 - .../Flow/ETL/Tests/Double/CountingHasher.php | 30 - .../Flow/ETL/Tests/Double/CountingMemory.php | 4 +- .../Tests/Double/CountingReturnsFunction.php | 30 +- .../ETL/Tests/Double/CountingWindowFrame.php | 18 +- .../ETL/Tests/Double/DeclaringExtractor.php | 20 +- .../tests/Flow/ETL/Tests/Double/Described.php | 33 + .../Flow/ETL/Tests/Double/EmptyExtractor.php | 16 +- .../Tests/Double/FailingOnValuesFunction.php | 40 +- .../Flow/ETL/Tests/Double/FakeExtractor.php | 22 +- .../Double/FakeRandomOrdersExtractor.php | 20 +- .../ETL/Tests/Double/FakeSchemaSampler.php | 32 - .../Tests/Double/FakeSelfDescribingFile.php | 15 +- .../Flow/ETL/Tests/Double/FakeSniffer.php | 64 + .../Double/FakeStaticOrdersExtractor.php | 27 +- .../Flow/ETL/Tests/Double/FanOutBranch.php | 32 + .../Flow/ETL/Tests/Double/FanOutSink.php | 32 - .../ETL/Tests/Double/FileReadingExtractor.php | 39 - .../Double/FixedRandomValueGenerator.php | 2 +- .../Tests/Double/FixedTmpDirFilesystem.php | 15 + .../ETL/Tests/Double/ForeignColumnStub.php | 2 +- .../Tests/Double/ForeignTypeDefinition.php | 11 +- .../ETL/Tests/Double/HoldingExtractor.php | 20 +- .../Tests/Double/IdRecordingRowsStorage.php | 66 + .../ETL/Tests/Double/InMemoryFileBatches.php | 42 +- .../Double/InMemoryUnskippableFileBatches.php | 36 +- .../Flow/ETL/Tests/Double/InlineLoader.php | 26 +- .../ETL/Tests/Double/LoadThenThrowLoader.php | 26 +- .../Tests/Double/MixedSequenceGenerator.php | 4 +- .../Tests/Double/NonStructureUnpackStub.php | 32 +- .../Flow/ETL/Tests/Double/NotDescribed.php | 27 + .../Tests/Double/OneRowBatchesExtractor.php | 43 +- .../Tests/Double/OverflowingColumnStub.php | 4 +- .../ETL/Tests/Double/PassThroughProcessor.php | 28 +- .../Tests/Double/PlanDrainingTransformer.php | 29 +- .../Tests/Double/PrefixingCodecStub.php | 4 +- .../ETL/Tests/Double/PresniffedSniffer.php | 49 + ...pagatingTransformationsSkippingLoaders.php | 28 +- .../Tests/Double/ReadsBackOnSecondBatch.php | 28 +- .../Tests/Double/RecordingBucketsStorage.php | 55 - .../Tests/Double/RecordingErrorHandler.php | 30 +- .../ETL/Tests/Double/RecordingExtractor.php | 20 +- .../Tests/Double/RecordingFileExtractor.php | 66 +- .../ETL/Tests/Double/RecordingFilesystem.php | 14 + .../ETL/Tests/Double/RecordingLauncher.php | 56 + .../Flow/ETL/Tests/Double/RecordingLoader.php | 37 +- .../ETL/Tests/Double/RecordingRowsStorage.php | 77 + .../Flow/ETL/Tests/Double/RecordingRule.php | 8 +- .../Double/RecordingSequenceGenerator.php | 4 +- .../Flow/ETL/Tests/Double/RecordingSink.php | 35 - .../Double/RecordingSplittableExtractor.php | 98 + .../ETL/Tests/Double/RecordingTransaction.php | 2 +- .../Flow/ETL/Tests/Double/RedefiningNode.php | 14 +- .../ETL/Tests/Double/RenameSelectRewrite.php | 6 +- .../ETL/Tests/Double/RepeatableExtractor.php | 42 +- .../Tests/Double/ReportedMemoryBackend.php | 23 +- .../Tests/Double/ReportsResolvedOperand.php | 36 +- .../ETL/Tests/Double/ReturnsSpyFunction.php | 37 +- .../ETL/Tests/Double/RowLessExtractor.php | 18 +- .../ETL/Tests/Double/SpineCopyingRule.php | 20 +- .../Flow/ETL/Tests/Double/SpyBackend.php | 41 +- .../tests/Flow/ETL/Tests/Double/SpyBranch.php | 18 + .../ETL/Tests/Double/SpyBucketsStorage.php | 89 - .../tests/Flow/ETL/Tests/Double/SpyColumn.php | 2 +- .../Flow/ETL/Tests/Double/SpyFileSinks.php | 68 - .../ETL/Tests/Double/SpyFileWriterFactory.php | 69 + .../tests/Flow/ETL/Tests/Double/SpyLoader.php | 39 +- .../Flow/ETL/Tests/Double/SpyRowsStorage.php | 99 + .../Flow/ETL/Tests/Double/SpySerializer.php | 8 +- .../tests/Flow/ETL/Tests/Double/SpySink.php | 18 - .../ETL/Tests/Double/SpySniffingSample.php | 50 - .../Flow/ETL/Tests/Double/SpyTransformer.php | 26 +- .../Tests/Double/StaticDataFrameFactory.php | 21 - .../Tests/Double/StaticExtractorFactory.php | 21 + .../Tests/Double/StopIgnoringExtractor.php | 41 +- .../ETL/Tests/Double/StopRecordingBatches.php | 45 + .../Flow/ETL/Tests/Double/StubLauncher.php | 58 + .../ETL/Tests/Double/ThrowWhenRowMatches.php | 26 +- .../ThrowingAfterFirstBatchExtractor.php | 16 +- .../Flow/ETL/Tests/Double/ThrowingLoader.php | 22 +- .../Double/ThrowingRemoveBucketsStorage.php | 41 - .../Double/ThrowingRemoveRowsStorage.php | 58 + .../ETL/Tests/Double/ThrowingRmFilesystem.php | 15 + .../ETL/Tests/Double/ThrowingTransformer.php | 26 +- .../Tests/Double/TrailingZerosCodecStub.php | 4 +- .../Double/UndescribableRowLessExtractor.php | 20 +- .../Flow/ETL/Tests/Double/UnenumerableSum.php | 75 +- .../ETL/Tests/Double/UnsizedFilesystem.php | 15 + .../Tests/Double/UnsizedSourceStream.php | 2 +- .../Tests/Double/UnsizedStreamFilesystem.php | 86 + .../Tests/Double/VaryingBatchesExtractor.php | 18 +- .../Double/WindowFunctionOverOperand.php | 59 +- .../CallUserFunc/StaticCalculator.php | 2 +- .../ETL/Tests/Fixtures/CustomDateTime.php | 9 - .../ETL/Tests/Fixtures/Enum/StringEnum.php | 12 + .../FileListExtractor/file_01.txt} | 0 .../FileListExtractor/file_02.json} | 0 .../FileListExtractor/file_03.md} | 0 .../FileListExtractor/nested/file_04.txt} | 0 .../FileListExtractor/nested/file_05.json} | 0 .../FileListExtractor/nested/file_06.md | 0 .../Floe}/pre-statistics/heterogeneous.floe | Bin .../ETL/Tests/Fixtures/Floe/regenerate.php | 9 + .../Fixtures/Floe}/v2/all-column-types.floe | Bin .../Tests/Fixtures/Floe}/v2/empty.floe | Bin .../Fixtures/Floe}/v2/heterogeneous.floe | Bin .../Fixtures/Floe}/xml-node-before-utf8.floe | Bin .../ETL/Tests/Fixtures/Join/AlwaysMeets.php | 22 +- .../Fixtures/ZeroExtension/data.0 | 0 .../year=2022/month=12/day=30/file.txt | 0 .../year=2022/month=12/day=31/file.txt | 0 .../year=2023/month=1/day=1/file.txt | 0 .../year=2023/month=1/day=2/file.txt | 0 .../year=2023/month=1/day=3/file.txt | 0 .../year=2023/month=1/day=4/file.txt | 0 .../year=2023/month=1/day=5/file.txt | 0 .../ETL/Tests/FlowIntegrationTestCase.php | 54 +- .../etl/tests/Flow/ETL/Tests/FlowTestCase.php | 62 +- .../Bucketing/BucketingConfigBuilderTest.php | 35 + .../API/Transformation/DropTest.php | 55 + .../API/Transformation/SelectTest.php | 55 + .../Bucketing/SpillingPathsTest.php | 26 +- .../Storage/FilesystemBucketsTest.php | 375 --- .../Bucketing/Storage/PSRCacheBucketsTest.php | 36 - .../Tests/Integration/Cache/CacheTestCase.php | 180 -- .../Integration/Cache/FilesystemCacheTest.php | 148 - .../Integration/Cache/InMemoryCacheTest.php | 16 - .../Cache/PSRSimpleFilesystemCacheTest.php | 67 - .../Integration/Cache/TraceableCacheTest.php | 55 - .../Bucketing/BucketingConfigBuilderTest.php | 35 - .../Integration/Context/WrittenFilesTest.php | 35 + .../DataFrame/AlgorithmOverrideTest.php | 48 +- .../DataFrame/AliasedReferenceTest.php | 86 + .../Integration/DataFrame/AnalyzeTest.php | 106 +- .../Integration/DataFrame/BranchingTest.php | 60 +- .../Tests/Integration/DataFrame/CacheTest.php | 145 +- .../DataFrame/ChainedBatchSizesTest.php | 47 +- .../DataFrame/ConfigBuilderTest.php | 115 +- .../DataFrame/ConfiguredBackendTest.php | 35 +- .../Integration/DataFrame/ConstraintTest.php | 18 +- .../Integration/DataFrame/DisplayTest.php | 437 +-- .../Integration/DataFrame/EmptySourceTest.php | 4 +- .../Integration/DataFrame/FilterTest.php | 99 +- .../DataFrame/Fixtures/Analyze/goldstock.csv | 2512 ----------------- .../year=2024/day=01/month=03/out.txt | 1 - .../Partitioning/overwrite/.gitignore | 2 - .../overwrite/date=2024-04-01/file.txt | 1 - .../overwrite/date=2024-04-02/file.txt | 1 - .../Partitioning/placeholders/.gitignore | 2 - .../rebind/region=eu/tier=gold/a.txt | 1 - .../Partitioning/rebind/region=us/b.txt | 1 - .../DataFrame/GroupByAggregationTest.php | 125 +- .../Integration/DataFrame/GroupByTest.php | 425 +-- .../Integration/DataFrame/JoinCrossTest.php | 126 +- .../Integration/DataFrame/JoinEachTest.php | 23 +- .../DataFrame/JoinEmptySideTest.php | 68 +- .../Tests/Integration/DataFrame/JoinTest.php | 581 ++-- .../Tests/Integration/DataFrame/LimitTest.php | 509 ++-- .../Tests/Integration/DataFrame/MathTest.php | 8 +- .../Tests/Integration/DataFrame/NaNTest.php | 134 + .../Integration/DataFrame/OffsetTest.php | 193 +- .../DataFrame/PartitioningTest.php | 187 +- .../Integration/DataFrame/PerRunPlanTest.php | 12 +- .../Integration/DataFrame/PlanBindTest.php | 82 +- .../Integration/DataFrame/RenameTest.php | 455 +-- .../Integration/DataFrame/SchemaTest.php | 50 +- .../Integration/DataFrame/SinglePlanTest.php | 79 +- .../SinkRootBlockingOperationsTest.php | 409 --- .../Integration/DataFrame/SinkRootTest.php | 820 ------ .../Tests/Integration/DataFrame/SortTest.php | 167 +- .../DataFrame/StructureSchemaMatchTest.php | 2 +- .../Integration/DataFrame/TelemetryTest.php | 564 +++- .../TemporalStringComparisonTest.php | 4 +- .../Integration/DataFrame/TransformTest.php | 4 +- ...ransformedBranchBlockingOperationsTest.php | 426 +++ .../DataFrame/TransformedBranchTest.php | 909 ++++++ .../Integration/DataFrame/TriggerPlanTest.php | 32 +- .../Integration/DataFrame/WithColumnTest.php | 26 - .../Dataset/SharedMemoryLimitTest.php | 62 - .../Extractor/CacheExtractorTest.php | 350 --- .../Extractor/FileExtractorScanTest.php | 155 - .../Extractor/FileSourceFilesystemTest.php | 284 -- .../year=2022/month=12/day=30/file.txt | 1 - .../year=2022/month=12/day=31/file.txt | 1 - .../year=2023/month=1/day=1/file.txt | 1 - .../year=2023/month=1/day=2/file.txt | 1 - .../year=2023/month=1/day=3/file.txt | 1 - .../year=2023/month=1/day=4/file.txt | 1 - .../year=2023/month=1/day=5/file.txt | 1 - .../Extractor/PathPartitionsExtractorTest.php | 265 -- .../Filesystem/SinkWritePropertiesTest.php | 175 -- .../Tests/Integration/Function/AllTest.php | 49 - .../Tests/Integration/Function/AnyTest.php | 49 - .../Tests/Integration/Function/AppendTest.php | 35 - .../Integration/Function/ArrayExpandTest.php | 379 --- .../Function/ArrayGetCollectionTest.php | 45 - .../Integration/Function/ArrayGetTest.php | 66 - .../Function/ArrayKeyRenameTest.php | 38 - .../Function/ArrayMergeCollectionTest.php | 45 - .../Integration/Function/ArrayMergeTest.php | 39 - .../Integration/Function/ArrayReverseTest.php | 37 - .../Integration/Function/ArraySortTest.php | 39 - .../Integration/Function/ArrayUnpackTest.php | 43 - .../Integration/Function/BetweenTest.php | 47 - .../Function/BinaryComparisonsTest.php | 69 - .../Integration/Function/BinaryLengthTest.php | 53 - .../Integration/Function/CallUserFuncTest.php | 42 - .../Integration/Function/CapitalizeTest.php | 34 - .../Tests/Integration/Function/CastTest.php | 98 - .../Tests/Integration/Function/ChunkTest.php | 67 - .../Function/CodePointLengthTest.php | 59 - .../Function/CollapseWhitespaceTest.php | 34 - .../Integration/Function/CombineTest.php | 39 - .../Tests/Integration/Function/ConcatTest.php | 82 - .../Function/ConcatWithSeparatorTest.php | 99 - .../Tests/Integration/Function/CountTest.php | 78 - .../Function/DOMElementAttributeValueTest.php | 74 - .../Function/DOMElementNextSiblingTest.php | 119 - .../Function/DOMElementParentTest.php | 48 - .../DOMElementPreviousSiblingTest.php | 101 - .../Function/DOMElementValueTest.php | 114 - .../Integration/Function/EnsureEndTest.php | 34 - .../Integration/Function/EnsureStartTest.php | 38 - .../Integration/Function/EnumNameTest.php | 69 - .../Integration/Function/EnumValueTest.php | 86 - .../Tests/Integration/Function/FirstTest.php | 88 - .../Function/HTMLQuerySelectorAllTest.php | 72 - .../Function/HTMLQuerySelectorTest.php | 82 - .../Tests/Integration/Function/HashTest.php | 65 - .../Integration/Function/IndexOfLastTest.php | 51 - .../Integration/Function/IsEmptyTest.php | 53 - .../Integration/Function/JsonDecodeTest.php | 35 - .../Integration/Function/JsonEncodeTest.php | 59 - .../Tests/Integration/Function/LastTest.php | 88 - .../Integration/Function/ListSelectTest.php | 80 - .../Integration/Function/MatchCasesTest.php | 61 - .../Function/ModifyDateTimeTest.php | 91 - .../Tests/Integration/Function/NotTest.php | 46 - .../Tests/Integration/Function/OnEachTest.php | 37 - .../Integration/Function/PrependTest.php | 35 - .../Integration/Function/RegexAllTest.php | 79 - .../Function/RegexMatchAllTest.php | 90 - .../Integration/Function/RegexMatchTest.php | 72 - .../Integration/Function/RegexReplaceTest.php | 72 - .../Tests/Integration/Function/RegexTest.php | 72 - .../Tests/Integration/Function/RepeatTest.php | 52 - .../Integration/Function/ReverseTest.php | 43 - .../Integration/Function/SanitizeTest.php | 72 - .../Tests/Integration/Function/SizeTest.php | 52 - .../Tests/Integration/Function/SplitTest.php | 72 - .../Integration/Function/SprintfTest.php | 72 - .../Tests/Integration/Function/StrPadTest.php | 34 - .../Integration/Function/StrReplaceTest.php | 34 - .../Function/StringContainsAnyTest.php | 53 - .../Function/StringEqualsToTest.php | 53 - .../Function/StringMatchAllTest.php | 39 - .../Integration/Function/StringMatchTest.php | 39 - .../Function/StringNormalizeTest.php | 51 - .../Integration/Function/StringWidthTest.php | 53 - .../Structure/StructureSelectTest.php | 83 - .../Function/Structure/StructureTest.php | 82 - .../Integration/Function/ToLowerTest.php | 34 - .../Integration/Function/ToUpperTest.php | 34 - .../Tests/Integration/Function/TrimTest.php | 89 - .../Integration/Function/TruncateTest.php | 46 - .../Function/UnicodeLengthTest.php | 58 - .../Tests/Integration/Function/WhenTest.php | 44 - .../Integration/Function/WordwrapTest.php | 38 - .../AddDynamicColumnsTest.php | 4 +- .../{Function => Functions}/AddJsonTest.php | 4 +- .../Tests/Integration/Functions/AllTest.php | 49 + .../Tests/Integration/Functions/AnyTest.php | 49 + .../Integration/Functions/AppendTest.php | 35 + .../Integration/Functions/ArrayExpandTest.php | 376 +++ .../Functions/ArrayGetCollectionTest.php | 45 + .../Integration/Functions/ArrayGetTest.php | 66 + .../Functions/ArrayKeyRenameTest.php | 38 + .../ArrayKeysStyleConvertTest.php | 9 +- .../Functions/ArrayMergeCollectionTest.php | 45 + .../Integration/Functions/ArrayMergeTest.php | 39 + .../ArrayPathExistsTest.php | 4 +- .../Functions/ArrayReverseTest.php | 37 + .../Integration/Functions/ArraySortTest.php | 39 + .../Integration/Functions/ArrayUnpackTest.php | 43 + .../Integration/Functions/BetweenTest.php | 47 + .../Functions/BinaryComparisonsTest.php | 69 + .../Functions/BinaryLengthTest.php | 53 + .../Functions/CallUserFuncTest.php | 42 + .../Integration/Functions/CapitalizeTest.php | 34 + .../Tests/Integration/Functions/CastTest.php | 98 + .../Tests/Integration/Functions/ChunkTest.php | 67 + .../Functions/CodePointLengthTest.php | 59 + .../Functions/CollapseWhitespaceTest.php | 34 + .../Integration/Functions/CombineTest.php | 39 + .../Integration/Functions/ConcatTest.php | 82 + .../Functions/ConcatWithSeparatorTest.php | 99 + .../{Function => Functions}/ContainsTest.php | 4 +- .../Tests/Integration/Functions/CountTest.php | 78 + .../DOMElementAttributeValueTest.php | 71 + .../Functions/DOMElementNextSiblingTest.php | 119 + .../Functions/DOMElementParentTest.php | 47 + .../DOMElementPreviousSiblingTest.php | 99 + .../Functions/DOMElementValueTest.php | 109 + .../{Function => Functions}/EndsWithTest.php | 4 +- .../Integration/Functions/EnsureEndTest.php | 34 + .../Integration/Functions/EnsureStartTest.php | 38 + .../Integration/Functions/EnumNameTest.php | 69 + .../Integration/Functions/EnumValueTest.php | 86 + .../Tests/Integration/Functions/FirstTest.php | 82 + .../Functions/HTMLQuerySelectorAllTest.php | 75 + .../Functions/HTMLQuerySelectorTest.php | 80 + .../Tests/Integration/Functions/HashTest.php | 65 + .../Integration/Functions/IndexOfLastTest.php | 51 + .../Integration/Functions/IsEmptyTest.php | 53 + .../Integration/Functions/JsonDecodeTest.php | 35 + .../Integration/Functions/JsonEncodeTest.php | 59 + .../Tests/Integration/Functions/LastTest.php | 82 + .../Integration/Functions/ListSelectTest.php | 87 + .../Integration/Functions/MatchCasesTest.php | 60 + .../Functions/ModifyDateTimeTest.php | 89 + .../Tests/Integration/Functions/NotTest.php | 46 + .../Integration/Functions/OnEachTest.php | 35 + .../Integration/Functions/PrependTest.php | 35 + .../Integration/Functions/RegexAllTest.php | 79 + .../Functions/RegexMatchAllTest.php | 90 + .../Integration/Functions/RegexMatchTest.php | 72 + .../Functions/RegexReplaceTest.php | 72 + .../Tests/Integration/Functions/RegexTest.php | 72 + .../Integration/Functions/RepeatTest.php | 52 + .../Integration/Functions/ReverseTest.php | 43 + .../Integration/Functions/SanitizeTest.php | 72 + .../Tests/Integration/Functions/SizeTest.php | 52 + .../Tests/Integration/Functions/SplitTest.php | 72 + .../Integration/Functions/SprintfTest.php | 72 + .../StartsWithTest.php | 4 +- .../Integration/Functions/StrPadTest.php | 34 + .../Integration/Functions/StrReplaceTest.php | 34 + .../Functions/StringContainsAnyTest.php | 53 + .../Functions/StringEqualsToTest.php | 53 + .../Functions/StringMatchAllTest.php | 39 + .../Integration/Functions/StringMatchTest.php | 39 + .../Functions/StringNormalizeTest.php | 51 + .../Integration/Functions/StringWidthTest.php | 53 + .../Functions/StructureSelectTest.php | 90 + .../Integration/Functions/StructureTest.php | 82 + .../Integration/Functions/ToLowerTest.php | 34 + .../Integration/Functions/ToUpperTest.php | 34 + .../Tests/Integration/Functions/TrimTest.php | 89 + .../Integration/Functions/TruncateTest.php | 46 + .../Functions/UnicodeLengthTest.php | 59 + .../UntypedIntermediateTest.php | 6 +- .../Tests/Integration/Functions/WhenTest.php | 44 + .../Integration/Functions/WordwrapTest.php | 38 + .../Extractor/ArrayExtractorSpillTest.php | 2 +- .../IO/Extractor/CacheExtractorTest.php | 270 ++ .../IO/Extractor/FileExtractorScanTest.php | 55 + .../IO/Extractor/FileListingExtractorTest.php | 267 ++ .../IO/Extractor/FileSourceFilesystemTest.php | 153 + .../Integration/IO/FileFormatContractTest.php | 134 + .../Filesystem/ConfigIsolationTest.php | 10 +- .../IO/Filesystem/FileWritePropertiesTest.php | 134 + .../IO/Floe}/FloeDataFrameTest.php | 4 +- .../Integration/IO/Floe/FloeExtractorTest.php | 431 +++ .../Integration/IO/Floe}/FloeMergeDSLTest.php | 6 +- .../Integration/IO/Floe/FloeSplitReadTest.php | 139 + .../Loader/File/FileCommit/FileCommitTest.php | 225 ++ .../NotPartitioned/AppendModeTest.php | 65 + .../ExceptionIfExistsModeTest.php | 54 + .../NotPartitioned/IgnoreModeTest.php | 56 + .../NotPartitioned/OverwriteModeTest.php | 64 + .../FileCommit/Partitioned/AppendModeTest.php | 91 + .../Partitioned/ExceptionIfExistsModeTest.php | 82 + .../FileCommit/Partitioned/IgnoreModeTest.php | 105 + .../Partitioned/OverwriteModeTest.php | 113 + .../OverwriteMultipleFilesTest.php | 135 + .../Partitioned/PartitionPlaceholdersTest.php | 169 ++ .../IO/Loader/PartitionedWriteTest.php | 181 ++ .../ExternalSortLifecycleTest.php | 6 +- .../ExternalSort/ExternalSortMergeTest.php | 20 +- .../ExternalSort/ExternalSortRunSizeTest.php | 80 + .../Loader/File/FilesSink/FilesSinkTest.php | 236 -- .../File/FilesSink/FilesSinkTestCase.php | 26 - .../NotPartitioned/AppendModeTest.php | 70 - .../ExceptionIfExistsModeTest.php | 57 - .../NotPartitioned/IgnoreModeTest.php | 59 - .../NotPartitioned/OverwriteModeTest.php | 70 - .../FilesSink/Partitioned/AppendModeTest.php | 96 - .../Partitioned/ExceptionIfExistsModeTest.php | 87 - .../FilesSink/Partitioned/IgnoreModeTest.php | 93 - .../Partitioned/OverwriteModeTest.php | 118 - .../OverwriteMultipleFilesTest.php | 140 - .../Partitioned/PartitionPlaceholdersTest.php | 174 -- .../Loader/PartitioningLoaderTest.php | 173 -- .../Monitoring/Memory/ConfigurationTest.php | 89 - .../Processor/BatchingProcessorTest.php | 130 + .../Processor/CollectingProcessorTest.php | 47 + .../Processor/MemorySortProcessorTest.php | 54 + .../Pipeline/BatchingPipelineTest.php | 126 - .../Pipeline/CollectingPipelineTest.php | 47 - .../Integration/Pipeline/PipelineTest.php | 24 +- .../Integration/Pipeline/SerialRunTest.php | 55 + .../Pipeline/SynchronousPipelineTest.php | 73 - .../{Config => Runtime}/MemoryLimitTest.php | 6 +- .../ExternalSort/ExternalSortRunSizeTest.php | 68 - .../Sort/MemorySort/MemorySortTest.php | 48 - .../Storage/FilesystemRowsStorageTest.php | 520 ++++ .../Storage/Floe}/FloeGoldenTest.php | 13 +- .../Storage/Floe/FloeHardBreakTest.php | 28 + .../Storage/Floe}/FloeStatisticsTest.php | 14 +- .../Storage/Floe/FloeStreamReaderTest.php | 146 + .../Storage/Floe}/FloeWriteContractTest.php | 47 +- .../Storage/PsrRowsStorageTest.php | 36 + .../Storage/SharedMemoryLimitTest.php | 64 + .../Integration/Transformation/DropTest.php | 55 - .../Integration/Transformation/SelectTest.php | 55 - .../Window/WindowFunctionsTest.php | 156 +- .../ETL/Tests/Mother/BranchLoaderMother.php | 69 + .../Flow/ETL/Tests/Mother/BucketMother.php | 2 +- .../Flow/ETL/Tests/Mother/ColumnMother.php | 6 +- .../ETL/Tests/Mother/ColumnTypesMother.php | 6 +- .../ETL/Tests/Mother/DefinitionMother.php | 2 +- .../Flow/ETL/Tests/Mother/EntryMother.php | 2 +- .../ETL/Tests/Mother/ExternalSortMother.php | 87 +- .../Flow/ETL/Tests/Mother/FileReadMother.php | 37 + .../Flow/ETL/Tests/Mother/FooterMother.php | 35 + .../GroupByAggregationProcessorMother.php | 32 +- .../Flow/ETL/Tests/Mother/HashJoinMother.php | 75 + .../Tests/Mother/HashJoinProcessorMother.php | 86 - .../Flow/ETL/Tests/Mother/JoinerMother.php | 27 + .../ETL/Tests/Mother/ListColumnsMother.php | 4 +- .../ETL/Tests/Mother/MarketRowsMother.php | 2 +- .../Flow/ETL/Tests/Mother/NodeMother.php | 34 +- .../ETL/Tests/Mother/NodeTranslatorMother.php | 29 + .../Tests/Mother/PartitionedSourceMother.php | 2 +- .../ETL/Tests/Mother/PhysicalPlanMother.php | 17 +- .../Flow/ETL/Tests/Mother/PipelinesMother.php | 84 + .../ETL/Tests/Mother/PivotedTableMother.php | 21 +- .../Mother/RepartitionProcessorMother.php | 35 +- .../Flow/ETL/Tests/Mother/RowsMother.php | 182 +- .../ETL/Tests/Mother/SinkAttachmentMother.php | 39 - .../Flow/ETL/Tests/Mother/SinkFeedMother.php | 68 - .../ETL/Tests/Mother/SortDatasetMother.php | 14 +- .../ETL/Tests/Mother/WindowContextMother.php | 25 - .../ETL/Tests/Mother/WritePlannerMother.php | 40 + .../Bucketing/BucketingConfigBuilderTest.php | 123 + .../API/Config/Join/HashJoinBuilderTest.php | 86 + .../Config/Sort/ExternalSortBuilderTest.php | 83 + .../Config/Sort/MemorySortBuilderTest.php | 6 +- .../Flow/ETL/Tests/Unit/API/DataFrameTest.php | 1664 +++++++++++ .../ETL/Tests/Unit/API/FilteredBranchTest.php | 76 + .../Tests/Unit/API/GroupedDataFrameTest.php | 93 + .../ETL/Tests/Unit/API/TransactionalTest.php | 261 ++ .../Transformation/AddRowIndexTest.php | 6 +- .../Unit/API/Transformation/BatchByTest.php | 55 + .../Unit/API/Transformation/BatchSizeTest.php | 31 + .../Unit/API/Transformation/LimitTest.php | 29 + .../API/Transformation/MaskColumnsTest.php | 38 + .../Unit/{ => API}/TransformationsTest.php | 6 +- .../Tests/Unit/API/TransformedBranchTest.php | 117 + .../Tests/Unit/Bucketing/BucketRunTest.php | 53 - .../Unit/Bucketing/HashBucketingTest.php | 227 -- .../Tests/Unit/Bucketing/KeyGroupingTest.php | 185 -- .../Tests/Unit/Bucketing/KeyValuesTest.php | 78 - .../Tests/Unit/Bucketing/NativeHasherTest.php | 90 - .../Unit/Bucketing/SingleBucketHasherTest.php | 21 - .../Bucketing/Storage/MemoryBucketsTest.php | 54 - .../Bucketing/Storage/PSRCacheBucketsTest.php | 137 - .../Bucketing/Storage/SpillingBucketsTest.php | 111 - .../ETL/Tests/Unit/Cache/CacheIndexTest.php | 129 - .../Cache/Implementation/ApcuCacheTest.php | 186 -- .../Implementation/TraceableCacheTest.php | 309 -- .../Tests/Unit/Column/AdaptiveBackendTest.php | 90 - .../Unit/Column/Layout/LayoutForTest.php | 89 - .../ETL/Tests/Unit/Column/PhpBackendTest.php | 628 ----- .../Physical/PhysicalBuilderForTest.php | 53 - .../Unit/Column/Physical/PhysicalForTest.php | 211 -- .../Unit/Column/Physical/PhysicalKindTest.php | 63 - .../Unit/Comparator/ColumnComparatorTest.php | 4 +- .../Unit/Comparator/RowsComparatorTest.php | 4 +- .../Bucketing/BucketingConfigBuilderTest.php | 75 - .../Grouping/HashGroupByBuilderTest.php | 43 - .../Unit/Config/Join/HashJoinBuilderTest.php | 66 - .../Config/Sort/ExternalSortBuilderTest.php | 83 - .../Unit/Config/Telemetry/SpanStackTest.php | 132 - .../Config/Telemetry/TelemetryContextTest.php | 1001 ------- .../Unit/Constraint/SortedConstraintTest.php | 337 --- .../Unit/Context/CacheDirContextTest.php | 47 + .../Unit/{Rows => DSL}/ArrayToRowsTest.php | 4 +- .../{ => DSL}/DateIntervalFunctionsTest.php | 2 +- .../Tests/Unit/DSL/DefinitionFromTypeTest.php | 14 +- .../ETL/Tests/Unit/DSL/InferSchemaTest.php | 2 +- .../Flow/ETL/Tests/Unit/DataFrameTest.php | 1459 ---------- .../Unit/Dataset/Memory/BoundedReadTest.php | 44 - .../Unit/Dataset/Memory/MemoryBudgetTest.php | 48 - .../Unit/Dataset/Statistics/ColumnsTest.php | 41 - .../Statistics/DistinctCounterTest.php | 41 - .../ETL/Tests/Unit/ETLErrorHandlingTest.php | 322 --- .../Tests/Unit/ErrorHandler/SkipRowsTest.php | 54 - .../Exception/ColumnMismatchExceptionTest.php | 97 - .../DataDependentSchemaExceptionTest.php | 33 - .../Exception/InferredSchemaExceptionTest.php | 77 - .../SchemaValidationExceptionTest.php | 64 - .../ETL/Tests/Unit/Executor/ClosureTest.php | 45 - .../ETL/Tests/Unit/Executor/DescribedTest.php | 30 - .../Tests/Unit/Executor/DiscardableTest.php | 145 - .../Unit/Executor/OffsetPipelineTest.php | 305 -- .../ETL/Tests/Unit/Executor/PipelineTest.php | 45 - .../Flow/ETL/Tests/Unit/Executor/RawTest.php | 55 - .../ETL/Tests/Unit/Executor/SegmentTest.php | 246 -- .../ETL/Tests/Unit/Executor/SegmentsTest.php | 199 -- .../ETL/Tests/Unit/Executor/SinkFeedTest.php | 230 -- .../Tests/Unit/Executor/SinkOffersTest.php | 87 - .../ETL/Tests/Unit/Executor/SinkRunTest.php | 142 - .../Tests/Unit/Executor/SourceReadTest.php | 63 - .../Tests/Unit/Executor/SourceRowsTest.php | 140 - .../Unit/Executor/TransactionRollbackTest.php | 38 - .../Unit/Executor/TransactionalSinksTest.php | 294 -- .../Executor/WindowFunctionPipelineTest.php | 156 - .../Flow/ETL/Tests/Unit/ExecutorTest.php | 522 ---- .../Unit/Extractor/ArrayExtractorTest.php | 434 --- .../Unit/Extractor/BatchByExtractorTest.php | 310 -- .../Unit/Extractor/BatchExtractorTest.php | 160 -- .../ETL/Tests/Unit/Extractor/BatchesTest.php | 47 - .../Unit/Extractor/ChainExtractorTest.php | 200 -- .../Extractor/CollectingExtractorTest.php | 77 - .../Unit/Extractor/DataFrameExtractorTest.php | 335 --- .../Unit/Extractor/File/DerivedSchemaTest.php | 37 - .../Unit/Extractor/File/FileColumnsTest.php | 205 -- .../Unit/Extractor/File/FileReadLoopTest.php | 342 --- .../Unit/Extractor/File/FileReadingTest.php | 121 - .../Unit/Extractor/File/FileScanTest.php | 91 - .../Extractor/File/FooterStatisticsTest.php | 55 - .../Extractor/File/InferredColumnsTest.php | 56 - .../Unit/Extractor/File/ListedFilesTest.php | 86 - .../Unit/Extractor/File/ListingScanTest.php | 154 - .../File/MetadataColumnsExtractorTest.php | 89 - .../Extractor/File/PartitionColumnsTest.php | 105 - .../Extractor/File/PartitionTypesTest.php | 85 - .../Unit/Extractor/File/ReadWindowTest.php | 36 - .../Unit/Extractor/File/SourceFileTest.php | 44 - .../Unit/Extractor/FilesExtractorTest.php | 224 -- .../Extractor/Memory/InMemoryRowsTest.php | 42 - .../Extractor/Memory/InferredRowsTest.php | 78 - .../Unit/Extractor/MemoryExtractorTest.php | 221 -- .../Unit/Extractor/RepeatabilityTest.php | 140 - .../Unit/Extractor/RowsExtractorTest.php | 96 - .../Tests/Unit/Extractor/StatisticsTest.php | 46 - .../Filesystem/ScalarFunctionFilterTest.php | 130 - .../Flow/ETL/Tests/Unit/FlowContextTest.php | 99 - .../Unit/FlowTestCaseBatchContractTest.php | 5 +- .../Unit/Formatter/ASCII/ASCIIBodyTest.php | 37 - .../Unit/Formatter/ASCII/ASCIIHeadersTest.php | 37 - .../Tests/Unit/Formatter/ASCIITableTest.php | 118 - .../ArrayComparison}/ArrayComparisonTest.php | 4 +- .../ArrayComparison}/ArraySortByKeyTest.php | 4 +- .../Foundation/Calculator}/CalculatorTest.php | 29 +- .../Calculator}/NumberNormalizerTest.php | 8 +- .../Foundation/Calculator/RunningSumTest.php | 225 ++ .../Clock}/HighResolutionTimeTest.php | 4 +- .../Unit/Foundation/Clock/SystemClockTest.php | 27 + .../Describe/ConfiguredFilesystemTest.php | 45 + .../Describe/DescribableRoundTripTest.php | 72 + .../Foundation/Describe/DescriptionTest.php | 192 ++ .../Unit/Foundation/Describe/JsonTest.php | 46 + .../Unit/Foundation/Describe/RebuildTest.php | 89 + .../Exception/ColumnMismatchExceptionTest.php | 120 + .../DataDependentSchemaExceptionTest.php | 33 + .../Exception/EvaluationExceptionTest.php | 6 +- .../Exception/InferredSchemaExceptionTest.php | 105 + .../InvalidArgumentExceptionTest.php | 28 + .../Exception/InvalidLogicExceptionTest.php | 4 +- .../SchemaDefinitionNotFoundExceptionTest.php | 4 +- .../Exception/SchemaMismatchExceptionTest.php | 11 +- .../SchemaNotDerivableExceptionTest.php | 4 +- .../SchemaValidationExceptionTest.php | 76 + .../{ => Foundation}/FlowPhpExtensionTest.php | 8 +- .../NativePHPRandomValueGeneratorTest.php | 4 +- .../SimilarNamesTest.php | 4 +- .../Telemetry/TelemetryAttributesTest.php | 4 +- ...llFunctionsDeclareTheirDeterminismTest.php | 85 - .../AllFunctionsDeclareTheirTypeTest.php | 262 -- ...ionsEvaluateOnTheConfiguredBackendTest.php | 113 - .../Flow/ETL/Tests/Unit/Function/AllTest.php | 110 - .../Flow/ETL/Tests/Unit/Function/AnyTest.php | 101 - .../ETL/Tests/Unit/Function/AppendTest.php | 88 - .../Tests/Unit/Function/ArrayExpandTest.php | 226 -- .../Unit/Function/ArrayGetCollectionTest.php | 247 -- .../ETL/Tests/Unit/Function/ArrayGetTest.php | 356 --- .../Unit/Function/ArrayKeyRenameTest.php | 153 - .../Function/ArrayMergeCollectionTest.php | 141 - .../Tests/Unit/Function/ArrayMergeTest.php | 83 - .../Tests/Unit/Function/ArrayReverseTest.php | 98 - .../ETL/Tests/Unit/Function/ArraySortTest.php | 301 -- .../Tests/Unit/Function/ArrayUnpackTest.php | 101 - .../Tests/Unit/Function/ArrayValuesTest.php | 89 - .../ETL/Tests/Unit/Function/AverageTest.php | 214 -- .../ETL/Tests/Unit/Function/BetweenTest.php | 201 -- .../Unit/Function/BinaryComparisonsTest.php | 729 ----- .../Tests/Unit/Function/BinaryLengthTest.php | 67 - .../Tests/Unit/Function/CallUserFuncTest.php | 150 - .../Tests/Unit/Function/CapitalizeTest.php | 25 - .../Flow/ETL/Tests/Unit/Function/CastTest.php | 368 --- .../ETL/Tests/Unit/Function/ChunkTest.php | 122 - .../Unit/Function/CodePointLengthTest.php | 47 - .../Unit/Function/CollapseWhitespaceTest.php | 101 - .../ETL/Tests/Unit/Function/CollectTest.php | 115 - .../Tests/Unit/Function/CollectUniqueTest.php | 105 - .../ETL/Tests/Unit/Function/CombineTest.php | 80 - .../ETL/Tests/Unit/Function/ConcatTest.php | 50 - .../Unit/Function/ConcatWithSeparatorTest.php | 43 - .../ETL/Tests/Unit/Function/CountTest.php | 211 -- .../Function/DOMElementAttributeValueTest.php | 90 - .../Unit/Function/DOMElementNamespaceTest.php | 86 - .../Unit/Function/DOMElementParentTest.php | 131 - .../Unit/Function/DOMElementValueTest.php | 87 - .../ETL/Tests/Unit/Function/DenseRankTest.php | 92 - .../ETL/Tests/Unit/Function/EnsureEndTest.php | 83 - .../Tests/Unit/Function/EnsureStartTest.php | 83 - .../ETL/Tests/Unit/Function/EnumNameTest.php | 75 - .../ETL/Tests/Unit/Function/EnumValueTest.php | 108 - .../ETL/Tests/Unit/Function/EqualsTest.php | 145 - .../Unit/Function/Evaluation/FilterTest.php | 110 - .../Function/Evaluation/SelectionTest.php | 76 - .../ETL/Tests/Unit/Function/FirstTest.php | 128 - .../Fixtures/CallUserFunc/Calculator.php | 18 - .../Unit/Function/GreaterThanEqualTest.php | 265 -- .../Tests/Unit/Function/GreaterThanTest.php | 275 -- .../Function/HTMLQuerySelectorAllTest.php | 77 - .../Unit/Function/HTMLQuerySelectorTest.php | 74 - .../Flow/ETL/Tests/Unit/Function/HashTest.php | 79 - .../Tests/Unit/Function/IndexOfLastTest.php | 69 - .../ETL/Tests/Unit/Function/IsEmptyTest.php | 52 - .../Tests/Unit/Function/JsonDecodeTest.php | 84 - .../Tests/Unit/Function/JsonEncodeTest.php | 71 - .../Flow/ETL/Tests/Unit/Function/LastTest.php | 145 - .../Tests/Unit/Function/LessThanEqualTest.php | 265 -- .../ETL/Tests/Unit/Function/LessThanTest.php | 265 -- .../Tests/Unit/Function/ListSelectTest.php | 155 - .../ETL/Tests/Unit/Function/LiteralTest.php | 96 - .../Tests/Unit/Function/MatchCasesTest.php | 114 - .../Flow/ETL/Tests/Unit/Function/MathTest.php | 97 - .../Flow/ETL/Tests/Unit/Function/MaxTest.php | 211 -- .../Flow/ETL/Tests/Unit/Function/MinTest.php | 220 -- .../Unit/Function/ModifyDateTimeTest.php | 102 - .../Flow/ETL/Tests/Unit/Function/NotTest.php | 95 - .../ETL/Tests/Unit/Function/OnEachTest.php | 340 --- .../Tests/Unit/Function/PregReplaceTest.php | 54 - .../ETL/Tests/Unit/Function/PrependTest.php | 74 - .../Flow/ETL/Tests/Unit/Function/RankTest.php | 90 - .../Unit/Function/ReferenceResolverTest.php | 209 -- .../ETL/Tests/Unit/Function/RegexAllTest.php | 58 - .../Tests/Unit/Function/RegexMatchAllTest.php | 65 - .../Tests/Unit/Function/RegexMatchTest.php | 51 - .../Tests/Unit/Function/RegexReplaceTest.php | 40 - .../ETL/Tests/Unit/Function/RegexTest.php | 52 - .../ETL/Tests/Unit/Function/RepeatTest.php | 93 - .../ETL/Tests/Unit/Function/ReverseTest.php | 47 - .../ETL/Tests/Unit/Function/RowNumberTest.php | 57 - .../Flow/ETL/Tests/Unit/Function/SameTest.php | 93 - .../ETL/Tests/Unit/Function/SanitizeTest.php | 67 - .../Flow/ETL/Tests/Unit/Function/SizeTest.php | 35 - .../ETL/Tests/Unit/Function/SplitTest.php | 41 - .../ETL/Tests/Unit/Function/SprintfTest.php | 38 - .../ETL/Tests/Unit/Function/StrPadTest.php | 27 - .../Tests/Unit/Function/StrReplaceTest.php | 34 - .../Unit/Function/StringAggregateTest.php | 160 -- .../Unit/Function/StringContainsAnyTest.php | 88 - .../Unit/Function/StringEqualsToTest.php | 70 - .../Unit/Function/StringMatchAllTest.php | 89 - .../Tests/Unit/Function/StringMatchTest.php | 91 - .../Unit/Function/StringNormalizeTest.php | 97 - .../Tests/Unit/Function/StringWidthTest.php | 56 - .../Unit/Function/StructureSelectTest.php | 111 - .../ETL/Tests/Unit/Function/StructureTest.php | 179 -- .../Flow/ETL/Tests/Unit/Function/SumTest.php | 296 -- .../ETL/Tests/Unit/Function/ToLowerTest.php | 21 - .../ETL/Tests/Unit/Function/ToUpperTest.php | 21 - .../Flow/ETL/Tests/Unit/Function/TrimTest.php | 44 - .../ETL/Tests/Unit/Function/TruncateTest.php | 138 - .../Tests/Unit/Function/UnicodeLengthTest.php | 47 - .../Tests/Unit/Function/ValueOrderTest.php | 81 - .../Flow/ETL/Tests/Unit/Function/WhenTest.php | 171 -- .../ETL/Tests/Unit/Function/WordwrapTest.php | 103 - .../AllAggregatesDeclareTheirTypeTest.php | 57 +- .../Functions/AllAggregatorsPartialTest.php | 127 + ...llFunctionsDeclareTheirDeterminismTest.php | 90 + .../AllFunctionsDeclareTheirTypeTest.php | 271 ++ .../Functions/AllFunctionsDescribeTest.php | 80 + ...ionsEvaluateOnTheConfiguredBackendTest.php | 46 + .../Flow/ETL/Tests/Unit/Functions/AllTest.php | 110 + .../Flow/ETL/Tests/Unit/Functions/AnyTest.php | 101 + .../ETL/Tests/Unit/Functions/AppendTest.php | 87 + .../Tests/Unit/Functions/ArrayExpandTest.php | 240 ++ .../ArrayFilterTest.php | 7 +- .../Unit/Functions/ArrayGetCollectionTest.php | 261 ++ .../ETL/Tests/Unit/Functions/ArrayGetTest.php | 385 +++ .../{Function => Functions}/ArrayKeepTest.php | 7 +- .../Unit/Functions/ArrayKeyRenameTest.php | 151 + .../ArrayKeysStyleConverterTest.php | 10 +- .../{Function => Functions}/ArrayKeysTest.php | 17 +- .../Functions/ArrayMergeCollectionTest.php | 145 + .../Tests/Unit/Functions/ArrayMergeTest.php | 83 + .../Tests/Unit/Functions/ArrayReverseTest.php | 116 + .../Tests/Unit/Functions/ArraySortTest.php | 329 +++ .../Tests/Unit/Functions/ArrayUnpackTest.php | 115 + .../Tests/Unit/Functions/ArrayValuesTest.php | 99 + .../{Function => Functions}/AsciiTest.php | 4 +- .../ETL/Tests/Unit/Functions/AverageTest.php | 274 ++ .../ETL/Tests/Unit/Functions/BetweenTest.php | 200 ++ .../Unit/Functions/BinaryComparisonsTest.php | 726 +++++ .../Tests/Unit/Functions/BinaryLengthTest.php | 65 + .../Tests/Unit/Functions/CallUserFuncTest.php | 164 ++ .../Tests/Unit/Functions/CapitalizeTest.php | 25 + .../ETL/Tests/Unit/Functions/CastTest.php | 383 +++ .../ETL/Tests/Unit/Functions/ChunkTest.php | 122 + .../{Function => Functions}/CoalesceTest.php | 25 +- .../Unit/Functions/CodePointLengthTest.php | 47 + .../Unit/Functions/CollapseWhitespaceTest.php | 101 + .../ETL/Tests/Unit/Functions/CollectTest.php | 135 + .../Unit/Functions/CollectUniqueTest.php | 121 + .../ETL/Tests/Unit/Functions/CombineTest.php | 80 + .../Tests/Unit/Functions/ComparisonTest.php | 1236 ++++++++ .../ETL/Tests/Unit/Functions/ConcatTest.php | 80 + .../Functions/ConcatWithSeparatorTest.php | 58 + .../{Function => Functions}/ConjunctsTest.php | 8 +- .../ETL/Tests/Unit/Functions/CountTest.php | 300 ++ .../DOMElementAttributeValueTest.php | 90 + .../DOMElementAttributesCountTest.php | 2 +- .../DOMElementNamespaceValueTest.php | 86 + .../Unit/Functions/DOMElementParentTest.php | 131 + .../Unit/Functions/DOMElementValueTest.php | 87 + .../DateTimeFormatTest.php | 6 +- .../Tests/Unit/Functions/DenseRankTest.php | 90 + .../Tests/Unit/Functions/EnsureEndTest.php | 83 + .../Tests/Unit/Functions/EnsureStartTest.php | 83 + .../ETL/Tests/Unit/Functions/EnumNameTest.php | 74 + .../Tests/Unit/Functions/EnumValueTest.php | 107 + .../Unit/Functions/Evaluation/FilterTest.php | 110 + .../Evaluation/ResultColumnTest.php | 22 +- .../Evaluation/ReturnTypesTest.php | 10 +- .../Functions/Evaluation/SelectionTest.php | 74 + .../Evaluation/TolerantEvaluationTest.php | 16 +- .../{Function => Functions}/ExistsTest.php | 17 +- .../ExpandingFunctionsTest.php | 8 +- .../ETL/Tests/Unit/Functions/FirstTest.php | 144 + .../{Function => Functions}/GreatestTest.php | 14 +- .../Functions/HTMLQuerySelectorAllTest.php | 99 + .../Unit/Functions/HTMLQuerySelectorTest.php | 96 + .../ETL/Tests/Unit/Functions/HashTest.php | 94 + .../Tests/Unit/Functions/IndexOfLastTest.php | 69 + .../{Function => Functions}/IndexOfTest.php | 4 +- .../ETL/Tests/Unit/Functions/IsEmptyTest.php | 52 + .../Unit/{Function => Functions}/IsInTest.php | 63 +- .../ETL/Tests/Unit/Functions/IsTypeTest.php | 29 + .../{Function => Functions}/IsUtf8Test.php | 2 +- .../Unit/Functions/Join/Condition/AllTest.php | 76 + .../Unit/Functions/Join/Condition/AnyTest.php | 76 + .../Functions/Join/Condition/EqualTest.php | 184 ++ .../Join/Condition}/IdenticalTest.php | 24 +- .../Unit/Functions/Join/ExpressionTest.php | 155 + .../Tests/Unit/Functions/JsonDecodeTest.php | 84 + .../Tests/Unit/Functions/JsonEncodeTest.php | 71 + .../ETL/Tests/Unit/Functions/LastTest.php | 161 ++ .../{Function => Functions}/LeastTest.php | 10 +- .../Tests/Unit/Functions/ListSelectTest.php | 169 ++ .../ETL/Tests/Unit/Functions/LiteralTest.php | 126 + .../Tests/Unit/Functions/LiteralValueTest.php | 131 + .../LogicalFunctionsTest.php | 2 +- .../Tests/Unit/Functions/MatchCasesTest.php | 129 + .../ETL/Tests/Unit/Functions/MathTest.php | 97 + .../Flow/ETL/Tests/Unit/Functions/MaxTest.php | 274 ++ .../Flow/ETL/Tests/Unit/Functions/MinTest.php | 287 ++ .../{Function => Functions}/MinusTest.php | 4 +- .../Unit/Functions/ModifyDateTimeTest.php | 102 + .../{Function => Functions}/MultiplyTest.php | 4 +- .../Flow/ETL/Tests/Unit/Functions/NotTest.php | 97 + .../Unit/{Function => Functions}/NowTest.php | 22 +- .../NumberFormatTest.php | 37 +- .../OnEachElementSchemaTest.php | 6 +- .../ETL/Tests/Unit/Functions/OnEachTest.php | 370 +++ .../{Function => Functions}/OptionalTest.php | 8 +- .../{Function => Functions}/ParameterTest.php | 124 +- .../Unit/{Function => Functions}/PlusTest.php | 4 +- .../{Function => Functions}/PowerTest.php | 6 +- .../ETL/Tests/Unit/Functions/PrependTest.php | 74 + .../Tests/Unit/Functions/RandomStringTest.php | 31 + .../ETL/Tests/Unit/Functions/RankTest.php | 90 + .../Functions/Reference/ReferencesTest.php | 54 + .../Reference/ResolvedReferenceTest.php | 57 + .../Reference/UnresolvedReferenceTest.php | 82 + .../ReferenceRenameTest.php | 4 +- .../Unit/Functions/ReferenceResolverTest.php | 258 ++ .../ReferencedColumnsTest.php | 4 +- .../ETL/Tests/Unit/Functions/RegexAllTest.php | 74 + .../Unit/Functions/RegexMatchAllTest.php | 71 + .../Tests/Unit/Functions/RegexMatchTest.php | 51 + .../Tests/Unit/Functions/RegexReplaceTest.php | 82 + .../ETL/Tests/Unit/Functions/RegexTest.php | 91 + .../ETL/Tests/Unit/Functions/RepeatTest.php | 93 + .../ETL/Tests/Unit/Functions/ReverseTest.php | 47 + .../{Function => Functions}/RoundTest.php | 4 +- .../Tests/Unit/Functions/RowNumberTest.php | 96 + .../ETL/Tests/Unit/Functions/SanitizeTest.php | 67 + .../ETL/Tests/Unit/Functions/SizeTest.php | 35 + .../Unit/{Function => Functions}/SlugTest.php | 4 +- .../ETL/Tests/Unit/Functions/SplitTest.php | 41 + .../ETL/Tests/Unit/Functions/SprintfTest.php | 50 + .../ETL/Tests/Unit/Functions/StrPadTest.php | 27 + .../Tests/Unit/Functions/StrReplaceTest.php | 34 + .../StringAfterLastTest.php | 4 +- .../StringAfterTest.php | 4 +- .../Unit/Functions/StringAggregateTest.php | 221 ++ .../StringBeforeLastTest.php | 4 +- .../StringBeforeTest.php | 4 +- .../Unit/Functions/StringContainsAnyTest.php | 87 + .../Unit/Functions/StringEqualsToTest.php | 70 + .../StringFoldTest.php | 4 +- .../Unit/Functions/StringMatchAllTest.php | 89 + .../Tests/Unit/Functions/StringMatchTest.php | 91 + .../Unit/Functions/StringNormalizeTest.php | 97 + .../StringStyleTest.php | 6 +- .../StringTitleTest.php | 4 +- .../Tests/Unit/Functions/StringWidthTest.php | 56 + .../Unit/Functions/StructureSelectTest.php | 125 + .../Tests/Unit/Functions/StructureTest.php | 210 ++ .../StructureValuesTest.php | 6 +- .../Flow/ETL/Tests/Unit/Functions/SumTest.php | 391 +++ .../TemporalStringCoercionTest.php | 10 +- .../{Function => Functions}/ToDateTest.php | 2 +- .../ToDateTime/PatternCoverageTest.php | 4 +- .../ToDateTimeTest.php | 19 +- .../ETL/Tests/Unit/Functions/ToLowerTest.php | 21 + .../ToTimeZoneTest.php | 19 +- .../ETL/Tests/Unit/Functions/ToUpperTest.php | 21 + .../ETL/Tests/Unit/Functions/TrimTest.php | 44 + .../ETL/Tests/Unit/Functions/TruncateTest.php | 138 + .../Unit/{Function => Functions}/UlidTest.php | 4 +- .../Unit/Functions/UnicodeLengthTest.php | 47 + .../Unit/{Function => Functions}/UuidTest.php | 22 +- .../ETL/Tests/Unit/Functions/WhenTest.php | 171 ++ .../Window/FrameAccumulationTest.php | 123 + .../{ => Functions}/Window/FrameBoundTest.php | 18 +- .../Unit/Functions/Window/PeerFrameTest.php | 127 + .../{ => Functions}/Window/RowsFrameTest.php | 12 +- .../Window/WholePartitionFrameTest.php | 42 + .../ETL/Tests/Unit/Functions/WindowTest.php | 118 + .../ETL/Tests/Unit/Functions/WordwrapTest.php | 103 + .../{Function => Functions}/XPathTest.php | 2 +- .../Unit/GroupBy/AggregatedGroupsTest.php | 132 - .../Tests/Unit/GroupBy/AggregatorsTest.php | 68 - .../Tests/Unit/GroupBy/GroupByShapeTest.php | 55 - .../Tests/Unit/GroupBy/GroupByStepsTest.php | 331 --- .../Unit/GroupBy/PivotAggregationTest.php | 100 - .../Tests/Unit/GroupBy/PivotSchemaTest.php | 90 - .../ETL/Tests/Unit/GroupBy/PivotShapeTest.php | 88 - .../Tests/Unit/GroupBy/PivotedTableTest.php | 143 - .../tests/Flow/ETL/Tests/Unit/GroupByTest.php | 489 ---- .../Unit/IO/Extractor/ArrayExtractorTest.php | 562 ++++ .../Unit/IO/Extractor/ChainExtractorTest.php | 216 ++ .../Tests/Unit/IO/Extractor/CriteriaTest.php | 182 ++ .../{ => IO}/Extractor/FeedExtractorTest.php | 24 +- .../Unit/IO/Extractor/File/ByteUnitTest.php | 39 + .../Extractor/File/DerivedFromFilesTest.php | 187 ++ .../IO/Extractor/File/DerivedSchemaTest.php | 45 + .../Extractor/File/FileColumnsSourceTest.php | 87 + .../IO/Extractor/File/FileColumnsTest.php | 283 ++ .../Extractor/File/FileConstantsTest.php | 6 +- .../Extractor/File/FileMetadataRowTest.php | 4 +- .../IO/Extractor/File/FileReadLoopTest.php | 438 +++ .../Unit/IO/Extractor/File/FileReadTest.php | 64 + .../IO/Extractor/File/FileSplitReaderTest.php | 82 + .../Unit/IO/Extractor/File/FileSplitTest.php | 272 ++ .../Extractor/File/FooterStatisticsTest.php | 46 + .../IO/Extractor/File/InferredColumnsTest.php | 74 + .../IO/Extractor/File/ListingScanTest.php | 187 ++ .../File/MetadataColumnsExtractorTest.php | 75 + .../IO/Extractor/File/PartitionTypesTest.php | 85 + .../Extractor/File/PathPartitionsRowTest.php | 4 +- .../Unit/IO/Extractor/File/ReadWindowTest.php | 87 + .../Unit/IO/Extractor/File/RowRangeTest.php | 35 + .../Unit/IO/Extractor/File/SourceFileTest.php | 56 + .../IO/Extractor/File/SourceFilesTest.php | 204 ++ .../IO/Extractor/FileListingExtractorTest.php | 273 ++ .../Unit/IO/Extractor/Grid/GridHeaderTest.php | 32 + .../Extractor/Grid/HeaderNamesTest.php | 4 +- .../Extractor/Grid/RecordDecoderTest.php | 6 +- .../IO/Extractor/Memory/InMemoryRowsTest.php | 42 + .../Extractor/Memory/SpilledRowsTest.php | 54 +- .../Unit/IO/Extractor/RowsExtractorTest.php | 96 + .../Extractor/SequenceExtractorTest.php | 21 +- .../DatePeriodSequenceGeneratorTest.php | 23 +- .../NumberSequenceGeneratorTest.php | 6 +- .../Unit/IO/Extractor/StatisticsTest.php | 46 + .../Filesystem/ScalarFunctionFilterTest.php | 205 ++ .../Unit/IO/Filesystem/VisibleFilesTest.php | 64 + .../Tests/Unit/IO/Floe/FloeExtractorTest.php | 715 +++++ .../Unit/IO/Floe/FloeFileBatchesTest.php | 139 + .../IO/Floe/FloeFileWriterFactoryTest.php | 57 + .../Tests/Unit/IO/Floe/FloeFileWriterTest.php | 60 + .../ETL/Tests/Unit/IO/Floe/FloeLoaderTest.php | 343 +++ .../Tests/Unit/IO/Floe/FloeSourceFileTest.php | 62 + .../Formatter/ASCII/ASCIIValueTest.php | 4 +- .../IO/Formatter/AsciiTableFormatterTest.php | 139 + .../Row}/ASCIISchemaFormatterTest.php | 18 +- .../Schema}/InlineSchemaFormatterTest.php | 4 +- .../Schema}/JsonSchemaFormatterTest.php | 6 +- .../Schema/PHPFormatter/TypeFormatterTest.php | 176 ++ .../PHPFormatter/ValueFormatterTest.php | 4 +- .../Schema}/PHPSchemaFormatterTest.php | 10 +- .../Unit/IO/Inference/ColumnTypesTest.php | 487 ++++ .../Tests/Unit/IO/Inference/InferredTest.php | 23 + .../Inference/InferredTypesTest.php | 25 +- .../Tests/Unit/IO/Inference/SampleTest.php | 52 + .../Unit/IO/Inference/SampledUnitsTest.php | 77 + .../Inference/SchemaInferenceBuilderTest.php | 15 +- .../Unit/IO/Inference/SchemaInferenceTest.php | 87 + .../Unit/IO/Inference/SchemaInferrerTest.php | 448 +++ .../Unit/IO/Inference/SniffedColumnsTest.php | 282 ++ .../Inference/TypeFloorTest.php | 38 +- .../Unit/{ => IO}/Loader/ArrayLoaderTest.php | 8 +- .../Unit/IO/Loader/File/FileCommitTest.php | 699 +++++ .../Unit/IO/Loader/File/FileWriteTest.php | 269 ++ .../Tests/Unit/IO/Loader/File/PartTest.php | 98 + .../IO/Loader/File/PartitionRouterTest.php | 137 + .../Unit/{ => IO}/Loader/MemoryLoaderTest.php | 6 +- .../Tests/Unit/IO/Loader/PartitioningTest.php | 60 + .../{ => IO}/Loader/RowPartitionsTest.php | 6 +- .../IO/Loader/SchemaValidationLoaderTest.php | 95 + .../Unit/{ => IO}/Loader/StreamLoaderTest.php | 38 +- .../Tests/Unit/Join/Comparison/AllTest.php | 64 - .../Tests/Unit/Join/Comparison/AnyTest.php | 64 - .../Tests/Unit/Join/Comparison/EqualTest.php | 125 - .../ETL/Tests/Unit/Join/ExpressionTest.php | 39 - .../Tests/Unit/Join/HashJoin/JoinKeysTest.php | 111 - .../Unit/Join/HashJoin/NullRowBuilderTest.php | 55 - .../Unit/Join/HashJoin/RowMergerTest.php | 187 -- .../ETL/Tests/Unit/Join/JoinSchemaTest.php | 144 - .../ETL/Tests/Unit/Join/JoinShapeTest.php | 131 - .../ETL/Tests/Unit/Join/JoinStepsTest.php | 53 - .../Flow/ETL/Tests/Unit/Join/RowsJoinTest.php | 711 ----- .../Unit/Kernels/Bucketing/BucketRunTest.php | 47 + .../{ => Kernels}/Bucketing/BucketTest.php | 4 +- .../{ => Kernels}/Bucketing/BucketsTest.php | 18 +- .../Kernels/Bucketing/HashBucketingTest.php | 258 ++ .../Unit/Kernels/Bucketing/KeyEncoderTest.php | 153 + .../Kernels/Bucketing/KeyGroupingTest.php | 231 ++ .../Unit/Kernels/Bucketing/SeenKeysTest.php | 43 + .../Kernels/Comparison/ValueEqualityTest.php | 139 + .../Kernels/Comparison/ValueOrderTest.php | 139 + .../{ => Kernels}/Hash/NativePHPHashTest.php | 9 +- .../HashJoin/HashTableTest.php | 4 +- .../Unit/Kernels/HashJoin/JoinKeysTest.php | 64 + .../Unit/Kernels/HashJoin/JoinSideTest.php | 34 + .../{Join => Kernels}/HashJoin/JoinerTest.php | 188 +- .../Kernels/HashJoin/NullKeyFilterTest.php | 64 + .../Tests/Unit/Kernels/HashJoin/RoleTest.php | 50 + .../Unit/Kernels/HashJoin/RowMergerTest.php | 57 + .../Unit/Kernels/Join/JoinSchemaTest.php | 248 ++ .../Kernels/Sort/ExternalSortConfigTest.php | 49 + .../Unit/Kernels/Sort/ExternalSortTest.php | 383 +++ .../Kernels/Sort/MemorySortConfigTest.php | 16 + .../Sort/Merge/BucketCursorTest.php | 32 +- .../Unit/Kernels/Sort/Merge/KWayMergeTest.php | 336 +++ .../Tests/Unit/Kernels/Sort/RowOrderTest.php | 346 +++ .../Tests/Unit/Kernels/Sort/SortKeyTest.php | 40 + .../Unit/Loader/File/FileWriteFrameTest.php | 156 - .../Tests/Unit/Loader/File/FilesSinkTest.php | 65 - .../Unit/Loader/File/PartitionRouterTest.php | 132 - .../Tests/Unit/Loader/PartitioningTest.php | 42 - .../Loader/SchemaValidationLoaderTest.php | 83 - .../Constraint/SortedByConstraintTest.php | 349 +++ .../Constraint/UniqueConstraintTest.php | 21 +- .../GroupBy/AggregatedGroupsTest.php | 136 + .../Operators/GroupBy/AggregatorsTest.php | 96 + .../GroupBy/DeclaredPivotValuesTest.php | 13 +- .../GroupBy/DiscoveredPivotValuesTest.php | 38 +- .../Operators/GroupBy/GroupByShapeTest.php | 56 + .../Operators/GroupBy/MergedAggregateTest.php | 94 + .../GroupBy/PartialAggregateTest.php | 82 + .../Unit/Operators/GroupBy/PivotShapeTest.php | 87 + .../Operators/GroupBy/PivotedTableTest.php | 155 + .../ETL/Tests/Unit/Operators/GroupByTest.php | 630 +++++ .../Unit/Operators/Join/CrossJoinTest.php | 248 ++ .../Operators/Join/HashJoinConfigTest.php | 61 + .../Unit/Operators/Join/HashJoinTest.php | 898 ++++++ .../Processor/BatchingByProcessorTest.php | 242 ++ .../Processor/BatchingProcessorTest.php | 181 ++ .../Processor/BoundExpansionTest.php | 34 +- .../Processor/CachingProcessorTest.php | 131 + .../Processor/CollectingProcessorTest.php | 97 + .../Processor/ConstrainedProcessorTest.php | 194 ++ .../Processor/CountingProcessorTest.php | 10 +- .../Processor/DistinctProcessorTest.php | 178 ++ .../Operators/Processor/DistributionTest.php | 69 + .../Processor/ExpandingProcessorTest.php | 73 +- .../GroupByAggregationProcessorTest.php | 597 ++++ .../Processor/MemorySortProcessorTest.php | 130 + .../Processor/MergeSortProcessorTest.php | 105 + .../Processor/OffsetProcessorTest.php | 270 ++ .../Processor/PivotProcessorTest.php | 322 +++ .../Processor/RepartitionProcessorTest.php | 220 ++ .../Processor}/RequiredColumnsTest.php | 4 +- .../Operators/Processor/TopNProcessorTest.php | 338 +++ .../Operators/Processor/VoidProcessorTest.php | 72 + .../Processor/WindowProcessorTest.php | 570 ++++ .../Unit/Operators/ProcessorBreaksTest.php | 89 + .../AddRowIndexTransformerTest.php | 99 + .../CollectReferencesTransformerTest.php | 60 + .../Transformer/DerivedColumnsTest.php | 46 +- .../DropColumnsTransformerTest.php | 94 + .../DuplicateRowTransformerTest.php | 319 +++ .../Transformer/ExpansionTest.php | 27 +- .../JoinEachRowsTransformerTest.php | 677 +++++ .../Transformer/LimitTransformerTest.php | 130 + .../PruneColumnsTransformerTest.php | 98 + .../Rename/RenameMapColumnStrategyTest.php | 2 +- .../RenameColumnTransformerTest.php | 149 + .../RenameEachColumnTransformerTest.php | 110 + .../ScalarFunctionFilterTransformerTest.php | 304 ++ .../ScalarFunctionTransformer/MathTest.php | 232 ++ .../ScalarFunctionTransformerTest.php | 371 +++ .../SelectColumnsTransformerTest.php | 166 ++ .../Transformer/SerializeTransformerTest.php | 128 + .../SerializedPayloadDecoderTest.php | 90 + .../Transformer/UnpackedColumnsTest.php | 13 +- .../UnserializeTransformerTest.php | 262 ++ .../Transformer/UntilTransformerTest.php | 181 ++ .../Tests/Unit/Optimizer/FilterWalkTest.php | 330 --- .../Tests/Unit/Optimizer/JoinSidesTest.php | 54 - .../Tests/Unit/Optimizer/LimitWalkTest.php | 72 - .../Unit/Optimizer/ProjectionWalkTest.php | 123 - .../Unit/Optimizer/Rule/CombineLimitsTest.php | 51 - .../Rule/CombineSortAndLimitTest.php | 62 - .../Rule/CountFromStatisticsTest.php | 223 -- .../Rule/PushFilterIntoSourceTest.php | 695 ----- .../Rule/PushLimitIntoSourceTest.php | 294 -- .../Rule/PushProjectionIntoExpandTest.php | 129 - .../Tests/Unit/Optimizer/TopNRewriteTest.php | 58 - .../Tests/Unit/Plan/Explain/BranchesTest.php | 33 - .../Tests/Unit/Plan/Explain/DetailsTest.php | 254 -- .../Unit/Plan/Explain/ExpressionTest.php | 139 - .../Unit/Plan/Explain/FlowLayoutTest.php | 100 - .../Tests/Unit/Plan/Explain/OutlineTest.php | 95 - .../Unit/Plan/Explain/PhysicalOutlineTest.php | 143 - .../Unit/Plan/Explain/StepDetailsTest.php | 245 -- .../Unit/Plan/Explain/TreeLayoutTest.php | 249 -- .../Flow/ETL/Tests/Unit/Plan/ExplainTest.php | 39 - .../ETL/Tests/Unit/Plan/LogicalPlanTest.php | 298 -- .../Tests/Unit/Plan/Node/AggregateTest.php | 55 - .../ETL/Tests/Unit/Plan/Node/BatchByTest.php | 64 - .../ETL/Tests/Unit/Plan/Node/BatchTest.php | 59 - .../ETL/Tests/Unit/Plan/Node/CacheTest.php | 53 - .../ETL/Tests/Unit/Plan/Node/CollectTest.php | 48 - .../ETL/Tests/Unit/Plan/Node/CountTest.php | 48 - .../Tests/Unit/Plan/Node/CrossJoinTest.php | 108 - .../ETL/Tests/Unit/Plan/Node/DropTest.php | 49 - .../ETL/Tests/Unit/Plan/Node/FilterTest.php | 53 - .../ETL/Tests/Unit/Plan/Node/JoinEachTest.php | 69 - .../ETL/Tests/Unit/Plan/Node/LimitTest.php | 58 - .../ETL/Tests/Unit/Plan/Node/ReadTest.php | 130 - .../ETL/Tests/Unit/Plan/Node/SelectTest.php | 49 - .../ETL/Tests/Unit/Plan/Node/SortTest.php | 56 - .../ETL/Tests/Unit/Plan/Node/TopNTest.php | 65 - .../Flow/ETL/Tests/Unit/Plan/SinksTest.php | 52 - .../Flow/ETL/Tests/Unit/Plan/TriggerTest.php | 122 - .../tests/Flow/ETL/Tests/Unit/PlanTest.php | 106 - .../Tests/Unit/Planner/NodeTranslatorTest.php | 576 ---- .../Tests/Unit/Planner/PipelineSplitTest.php | 701 ----- .../Tests/Unit/Planner/SinkAttachmentTest.php | 156 - .../Unit/Planner/SinkFeedFactoryTest.php | 94 - .../tests/Flow/ETL/Tests/Unit/PlannerTest.php | 577 ---- .../Explain/BoxLayoutTest.php | 78 +- .../Explain/ConditionTest.php | 12 +- .../{Plan => Planning}/Explain/EntryTest.php | 2 +- .../Unit/Planning/Explain/ExpressionTest.php | 139 + .../Unit/Planning/Explain/FlowLayoutTest.php | 100 + .../Planning/Explain/LogicalOutlineTest.php | 98 + .../Unit/Planning/Explain/NodeDetailsTest.php | 248 ++ .../Planning/Explain/PhysicalOutlineTest.php | 157 ++ .../Explain/StatisticsLineTest.php | 8 +- .../Unit/Planning/Explain/StepDetailsTest.php | 271 ++ .../Unit/Planning/Explain/TreeLayoutTest.php | 271 ++ .../Unit/Planning/Logical/ExplainTest.php | 42 + .../Unit/Planning/Logical/LogicalPlanTest.php | 383 +++ .../Planning/Logical/Node/AggregateTest.php | 54 + .../Planning/Logical/Node/BatchByTest.php | 62 + .../Unit/Planning/Logical/Node/BatchTest.php | 47 + .../Unit/Planning/Logical/Node/CacheTest.php | 50 + .../Logical}/Node/CollectRefsTest.php | 12 +- .../Planning/Logical/Node/CollectTest.php | 46 + .../Logical}/Node/ConstrainTest.php | 16 +- .../Unit/Planning/Logical/Node/CountTest.php | 46 + .../Planning/Logical/Node/CrossJoinTest.php | 103 + .../Logical}/Node/DiscardTest.php | 12 +- .../Logical}/Node/DistinctTest.php | 18 +- .../Unit/Planning/Logical/Node/DropTest.php | 47 + .../Logical}/Node/DuplicateRowTest.php | 17 +- .../Logical}/Node/ExpandColumnTest.php | 16 +- .../Unit/Planning/Logical/Node/FilterTest.php | 51 + .../Planning/Logical/Node/JoinEachTest.php | 65 + .../Logical}/Node/JoinTest.php | 47 +- .../Unit/Planning/Logical/Node/LimitTest.php | 56 + .../Logical}/Node/OffsetTest.php | 16 +- .../Logical}/Node/OutputsTest.php | 28 +- .../Unit/Planning/Logical/Node/ReadTest.php | 143 + .../Logical}/Node/RenameEachTest.php | 18 +- .../Logical}/Node/RenameTest.php | 12 +- .../Logical}/Node/RepartitionTest.php | 12 +- .../Logical}/Node/ResultTest.php | 12 +- .../Unit/Planning/Logical/Node/SelectTest.php | 47 + .../Unit/Planning/Logical/Node/SortTest.php | 52 + .../Unit/Planning/Logical/Node/TopNTest.php | 62 + .../Logical}/Node/TransactionTest.php | 32 +- .../Logical}/Node/TransformTest.php | 14 +- .../Logical}/Node/UntilTest.php | 12 +- .../Logical}/Node/ValidateTest.php | 19 +- .../Logical}/Node/WindowColumnTest.php | 12 +- .../Logical}/Node/WithColumnTest.php | 12 +- .../Logical}/Node/WriteTest.php | 19 +- .../Tests/Unit/Planning/Logical/PlanTest.php | 115 + .../Logical}/RedefinedTest.php | 4 +- .../Logical}/ReplaceNodesTest.php | 10 +- .../Logical}/TransformUpTest.php | 20 +- .../Unit/Planning/Logical/TriggerTest.php | 120 + .../Unit/Planning/Logical/WritesTest.php | 50 + .../Unit/Planning/Optimizer/JoinSidesTest.php | 56 + .../Optimizer}/OptimizerTest.php | 30 +- .../Optimizer/Rule/CombineLimitsTest.php | 53 + .../Rule/CombineSortAndLimitTest.php | 100 + .../Rule/CountFromStatisticsTest.php | 203 ++ .../Rule/PushFilterIntoSourceTest.php | 1116 ++++++++ .../Rule/PushLimitIntoSourceTest.php | 429 +++ .../Rule/PushOffsetIntoSourceTest.php | 129 + .../Rule/PushProjectionIntoExpandTest.php | 246 ++ .../Planning/Physical/NodeTranslatorTest.php | 821 ++++++ .../Planning/Physical/PhysicalPlanTest.php | 80 + .../Planning/Physical/PipelineStepsTest.php | 64 + .../Unit/Planning/Physical/PipelineTest.php | 47 + .../Physical}/PlannedNodeTest.php | 6 +- .../Physical}/PlannedNodesTest.php | 12 +- .../Unit/Planning/Physical/PlannerTest.php | 1337 +++++++++ .../Planning/Physical/WritePlannerTest.php | 206 ++ .../Processor/BatchingByProcessorTest.php | 194 -- .../Unit/Processor/BatchingProcessorTest.php | 137 - .../Unit/Processor/CachingProcessorTest.php | 137 - .../Processor/CollectingProcessorTest.php | 99 - .../Processor/ConstrainedProcessorTest.php | 186 -- .../GroupByAggregationProcessorTest.php | 334 --- .../Unit/Processor/HashJoinProcessorTest.php | 751 ----- .../Processor/MemorySortProcessorTest.php | 127 - .../Unit/Processor/MergeSortProcessorTest.php | 59 - .../Unit/Processor/OffsetProcessorTest.php | 129 - .../Unit/Processor/PivotProcessorTest.php | 207 -- .../Processor/RepartitionProcessorTest.php | 155 - .../Unit/Processor/TopNProcessorTest.php | 229 -- .../Unit/Processor/VoidProcessorTest.php | 73 - .../Unit/Processor/WindowProcessorTest.php | 489 ---- .../Unit/Repartition/RepartitionStepsTest.php | 41 - .../ETL/Tests/Unit/Row/ReferencesTest.php | 49 - .../Tests/Unit/Row/ResolvedReferenceTest.php | 52 - .../Unit/Row/TypedValueComparatorTest.php | 220 -- .../Unit/Row/UnresolvedReferenceTest.php | 87 - .../ETL/Tests/Unit/Rows/RowHashesTest.php | 151 - .../ETL/Tests/Unit/RowsSerializationTest.php | 242 -- .../Cache => Runtime}/CacheConfigTest.php | 4 +- .../ETL/Tests/Unit/Runtime/ConfigTest.php | 72 + .../ErrorHandler/ErrorHandlingTest.php | 393 +++ .../Runtime/ErrorHandler/IgnoreErrorTest.php | 29 + .../Runtime/ErrorHandler/SkipRowsTest.php | 69 + .../Runtime/ErrorHandler/ThrowErrorTest.php | 29 + .../Executor/AdoptedBatchesTest.php | 23 +- .../Unit/Runtime/Executor/BranchFiberTest.php | 128 + .../Runtime/Executor/BranchLoaderTest.php | 224 ++ .../Unit/Runtime/Executor/ClosureTest.php | 62 + .../Unit/Runtime/Executor/DiscardableTest.php | 143 + .../Unit/Runtime/Executor/ExecutorTest.php | 850 ++++++ .../Runtime/Executor/OfferedFailureTest.php | 121 + .../Unit/Runtime/Executor/SourceCountTest.php | 122 + .../Runtime/Executor/SourceCountsTest.php | 247 ++ .../Executor/StatisticsCollectorTest.php | 68 +- .../Unit/Runtime/Executor/StepRunnerTest.php | 471 ++++ .../Runtime/Executor/StepTelemetryTest.php | 277 ++ .../Executor/TransactionLoaderTest.php | 348 +++ .../Executor/WriteLoaderFactoryTest.php | 179 ++ .../Tests/Unit/Runtime/FlowContextTest.php | 84 + .../Metrics}/SourceStatisticsTest.php | 8 +- .../Metrics}/Statistics/ColumnTest.php | 42 +- .../Metrics/Statistics/ColumnsTest.php | 41 + .../Unit/Runtime/Telemetry/SpanStackTest.php | 144 + .../Telemetry/TelemetryContextTest.php | 768 +++++ .../Telemetry/TelemetryOptionsTest.php | 4 +- .../Telemetry/TraceableRowsStorageTest.php | 184 ++ .../Definition/BooleanDefinitionTest.php | 264 -- .../Unit/Schema/Definition/CommonTypeTest.php | 131 - .../Schema/Definition/DateDefinitionTest.php | 303 -- .../Schema/Definition/EnumDefinitionTest.php | 349 --- .../Schema/Definition/FloatDefinitionTest.php | 288 -- .../Schema/Definition/HTMLDefinitionTest.php | 269 -- .../Definition/HTMLElementDefinitionTest.php | 269 -- .../Definition/IntegerDefinitionTest.php | 289 -- .../Schema/Definition/JsonDefinitionTest.php | 275 -- .../Schema/Definition/ListDefinitionTest.php | 359 --- .../Schema/Definition/MapDefinitionTest.php | 366 --- .../Schema/Definition/TimeDefinitionTest.php | 297 -- .../Schema/Definition/UuidDefinitionTest.php | 265 -- .../Unit/Schema/Definition/ValueMatchTest.php | 59 - .../Schema/Definition/XMLDefinitionTest.php | 265 -- .../Definition/XMLElementDefinitionTest.php | 265 -- .../Unit/Schema/EvolvingValidatorTest.php | 143 - .../Schema/Formatter/Fixtures/StringEnum.php | 12 - .../PHPFormatter/FlowTypeFormatterTest.php | 176 -- .../Unit/Schema/Inference/ColumnTypesTest.php | 579 ---- .../Schema/Inference/SchemaInferenceTest.php | 38 - .../Schema/Inference/SchemaInferrerTest.php | 420 --- .../Flow/ETL/Tests/Unit/Sink/BranchedTest.php | 82 - .../ETL/Tests/Unit/Sink/TransactionalTest.php | 226 -- .../ETL/Tests/Unit/Sink/TransformedTest.php | 111 - .../ETL/Tests/Unit/Sort/ExternalSortTest.php | 311 -- .../Tests/Unit/Sort/Merge/KWayMergeTest.php | 319 --- .../Flow/ETL/Tests/Unit/Sort/RowOrderTest.php | 258 -- .../ETL/Tests/Unit/Sort/SortStepsTest.php | 149 - .../Unit/Storage/ApcuRowsStorageTest.php | 129 + .../Unit/Storage/BucketingConfigTest.php | 61 + .../Tests/Unit/Storage/Floe/FloeFilesTest.php | 1155 ++++++++ .../Unit/Storage/Floe}/FloeMergerTest.php | 30 +- .../Unit/Storage/Floe}/FloeSerializerTest.php | 29 +- .../Storage/Floe/FloeStreamReaderTest.php | 234 ++ .../Storage/Floe/FloeStreamWriterTest.php | 462 +++ .../Storage/Floe}/FloeValueRoundTripTest.php | 17 +- .../Unit/Storage/Floe}/FooterReaderTest.php | 32 +- .../Storage/Floe/FooterStatisticsTest.php | 51 + .../Tests/Unit/Storage/Floe}/FooterTest.php | 29 +- .../Tests/Unit/Storage/Floe}/FormatTest.php | 6 +- .../Unit/Storage/Floe}/FrameWriterTest.php | 9 +- .../Tests/Unit/Storage/Floe/OptionsTest.php | 52 + .../Tests/Unit/Storage/Floe}/SectionTest.php | 6 +- .../Floe/SplittingFrameEncoderTest.php | 24 + .../{ => Storage}/Memory/ArrayMemoryTest.php | 27 +- .../Unit/Storage/MemoryRowsStorageTest.php | 90 + .../Tests/Unit/Storage/PsrRowsStorageTest.php | 148 + .../AllSerializersKeepTheBackendTest.php | 58 + .../Serializer}/CompressingSerializerTest.php | 13 +- .../Unit/Storage/SpillingRowsStorageTest.php | 114 + .../Unit/Transformation/BatchSizeTest.php | 31 - .../Tests/Unit/Transformation/LimitTest.php | 29 - .../Unit/Transformation/MaskColumnTest.php | 38 - .../AddRowIndexTransformerTest.php | 80 - .../CollectReferencesTransformerTest.php | 38 - .../CrossJoinRowsTransformerTest.php | 64 - .../DropColumnsTransformerTest.php | 71 - .../DropDuplicatesTransformerTest.php | 112 - .../DuplicateRowTransformerTest.php | 320 --- .../JoinEachRowsTransformerTest.php | 137 - .../Unit/Transformer/LimitTransformerTest.php | 91 - .../PruneColumnsTransformerTest.php | 80 - .../RenameColumnTransformerTest.php | 128 - .../RenameEachColumnTransformerTest.php | 71 - .../ScalarFunctionFilterTransformerTest.php | 248 -- .../ScalarFunctionTransformer/MathTest.php | 230 -- .../ScalarFunctionTransformerTest.php | 346 --- .../SelectColumnsTransformerTest.php | 123 - .../Transformer/SerializeTransformerTest.php | 153 - .../SerializedPayloadDecoderTest.php | 92 - .../UnserializeTransformerTest.php | 285 -- .../Unit/Transformer/UntilTransformerTest.php | 155 - .../ETL/Tests/Unit/Values/BatchSizeTest.php | 33 + .../Unit/{ => Values}/CardinalityTest.php | 6 +- .../Values/Column/AdaptiveBackendTest.php | 121 + .../{ => Values}/Column/ColumnConcatTest.php | 10 +- .../{ => Values}/Column/ColumnEncodeTest.php | 6 +- .../Column/ColumnWithTypeTest.php | 4 +- .../Column/ComparableValuesTest.php | 47 +- .../Column/DateFormatPatternTest.php | 4 +- .../Unit/{ => Values}/Column/ElementsTest.php | 4 +- .../Values/Column/Frame}/NoopCodecTest.php | 4 +- .../Column/Layout/BooleanLayoutTest.php | 8 +- .../Column/Layout/BufferLayoutTest.php | 29 +- .../Column/Layout/BuffersTest.php | 6 +- .../Column/Layout/FixedBinary16LayoutTest.php | 8 +- .../Column/Layout/Float64LayoutTest.php | 8 +- .../Column/Layout/Int32LayoutTest.php | 8 +- .../Column/Layout/Int64LayoutTest.php | 8 +- .../Column/Layout/NullCountTest.php | 4 +- .../Column/Layout/OffsetsTest.php | 8 +- .../Column/Layout/Utf8LayoutTest.php | 8 +- .../Column/Layout/ValidityTest.php | 6 +- .../Column/Php/CastingColumnBuilderTest.php | 20 +- .../Column/Php/ColumnDecoderTest.php | 31 +- .../Column/Php/ConstantColumnTest.php | 38 +- .../Column/Php/ListColumnBuilderTest.php | 10 +- .../Column/Php/ListColumnTest.php | 6 +- .../Column/Php/MapColumnBuilderTest.php | 10 +- .../{ => Values}/Column/Php/MapColumnTest.php | 6 +- .../Column/Php/NullColumnBuilderTest.php | 6 +- .../Column/Php/ScalarColumnBuilderTest.php | 6 +- .../Column/Php/ScalarColumnTest.php | 6 +- .../Column/Php/StructColumnBuilderTest.php | 12 +- .../Column/Php/StructColumnTest.php | 4 +- .../Unit/Values/Column/PhpBackendTest.php | 961 +++++++ .../Column/Physical/DatePhysicalTest.php | 4 +- .../Column/Physical/DateTimePhysicalTest.php | 4 +- .../Column/Physical/DaysFromCivilTest.php | 4 +- .../Column/Physical/ElementPositionTest.php | 4 +- .../Column/Physical/EnumPhysicalTest.php | 4 +- .../Physical/HtmlDocumentPhysicalTest.php | 6 +- .../Physical/HtmlElementPhysicalTest.php | 6 +- .../Column/Physical/IdentityPhysicalTest.php | 4 +- .../Column/Physical/JsonFactoryTest.php | 4 +- .../Column/Physical/JsonPhysicalTest.php | 4 +- .../Column/Physical/ListPhysicalTest.php | 6 +- .../Column/Physical/MapPhysicalTest.php | 6 +- .../Column/Physical/NullPhysicalTest.php | 4 +- .../Column/Physical/PhysicalForTest.php | 336 +++ .../Column/Physical/StructPhysicalTest.php | 8 +- .../Column/Physical/TimePhysicalTest.php | 4 +- .../Column/Physical/TimeZonePhysicalTest.php | 4 +- .../Column/Physical/TimeZonesTest.php | 4 +- .../Column/Physical/UuidPhysicalTest.php | 4 +- .../Physical/XmlCharacterReferencesTest.php | 4 +- .../Physical/XmlDocumentPhysicalTest.php | 6 +- .../Physical/XmlElementPhysicalTest.php | 6 +- .../Unit/{ => Values}/Column/RetypeTest.php | 6 +- .../{ => Values}/Column/TextValuesTest.php | 11 +- .../{ => Values}/Column/ValueColumnTest.php | 8 +- .../Tests/Unit/Values/Join/JoinTypeTest.php | 39 + .../Unit/Values/Memory/BoundedReadTest.php | 40 + .../Memory/ConsumptionTest.php | 6 +- .../Unit/Values/Memory/MemoryBudgetTest.php | 48 + .../{Dataset => Values}/Memory/UnitTest.php | 4 +- .../Unit/{Row => Values}/NullsOrderTest.php | 6 +- .../{ => Values}/Rows/ColumnRetypingTest.php | 8 +- .../Unit/Values/Rows/RecordBatchTest.php | 45 + .../{ => Values}/Rows/RowsBuilderTest.php | 83 +- .../ETL/Tests/Unit/{ => Values}/RowsTest.php | 362 ++- .../Definition/BooleanDefinitionTest.php | 236 ++ .../Schema/Definition/DateDefinitionTest.php | 275 ++ .../Definition/DateTimeDefinitionTest.php | 90 +- .../Definition/ElementCompatibilityTest.php | 4 +- .../Schema/Definition/EnumDefinitionTest.php | 326 +++ .../Schema/Definition/FloatDefinitionTest.php | 260 ++ .../Schema/Definition/HTMLDefinitionTest.php | 245 ++ .../Definition/HTMLElementDefinitionTest.php | 244 ++ .../Definition/IntegerDefinitionTest.php | 273 ++ .../Schema/Definition/JsonDefinitionTest.php | 247 ++ .../Schema/Definition/ListDefinitionTest.php | 345 +++ .../Schema/Definition/MapDefinitionTest.php | 351 +++ .../Schema/Definition/MergeMatrixTest.php | 10 +- .../Schema/Definition/NullDefinitionTest.php | 26 +- .../Definition/StringDefinitionTest.php | 90 +- .../Definition/StructureDefinitionTest.php | 126 +- .../Schema/Definition/TimeDefinitionTest.php | 269 ++ .../Definition/TimeZoneDefinitionTest.php | 16 +- .../Schema/Definition/TypeProjectionTest.php | 60 +- .../Schema/Definition/UuidDefinitionTest.php | 239 ++ .../Schema/Definition/XMLDefinitionTest.php | 237 ++ .../Definition/XMLElementDefinitionTest.php | 240 ++ .../Schema/DefinitionCompatibilityTest.php | 4 +- .../{ => Values}/Schema/DefinitionTest.php | 31 +- .../Unit/{ => Values}/Schema/MetadataTest.php | 6 +- .../{ => Values}/Schema/SameSchemasTest.php | 4 +- .../Validator/EvolvingValidatorTest.php | 163 ++ .../Validator}/MismatchedDefinitionTest.php | 4 +- .../Validator}/SelectiveValidatorTest.php | 6 +- .../Schema/Validator}/StrictValidatorTest.php | 4 +- .../Validator}/ValidationContextTest.php | 50 +- .../{Schema => Values}/SchemaMergeTest.php | 2 +- .../Unit/{Schema => Values}/SchemaTest.php | 357 +-- .../Accumulator/AverageAccumulatorTest.php | 93 - .../Accumulator/CountAccumulatorTest.php | 74 - .../Window/Accumulator/SumAccumulatorTest.php | 110 - .../Tests/Unit/Window/PeerComparatorTest.php | 106 - .../ETL/Tests/Unit/Window/PeerFrameTest.php | 113 - .../Unit/Window/WholePartitionFrameTest.php | 34 - .../Tests/Unit/Window/WindowContextTest.php | 83 - .../tests/Flow/ETL/Tests/Unit/WindowTest.php | 101 - .../Floe/Tests/Context/FloeFilesContext.php | 91 - .../Tests/Context/FloeStreamReaderContext.php | 227 -- .../Floe/Tests/Double/UnsizedFilesystem.php | 72 - .../Flow/Floe/Tests/Fixtures/regenerate.php | 9 - .../Tests/Integration/FloeExtractorTest.php | 443 --- .../Tests/Integration/FloeHardBreakTest.php | 28 - .../Integration/FloeStreamReaderTest.php | 153 - .../Flow/Floe/Tests/Mother/FooterMother.php | 35 - .../Floe/Tests/Mother/FrameReaderMother.php | 23 - .../Flow/Floe/Tests/Mother/RowsMother.php | 176 -- .../Floe/Tests/Unit/{ => DSL}/FloeDSLTest.php | 8 +- .../Floe/Tests/Unit/FloeExtractorTest.php | 551 ---- .../Floe/Tests/Unit/FloeFileBatchesTest.php | 138 - .../Flow/Floe/Tests/Unit/FloeFileSinkTest.php | 53 - .../Floe/Tests/Unit/FloeFileSinksTest.php | 58 - .../Flow/Floe/Tests/Unit/FloeLoaderTest.php | 193 -- .../Flow/Floe/Tests/Unit/FloeReaderTest.php | 941 ------ .../Floe/Tests/Unit/FloeSourceFileTest.php | 60 - .../Floe/Tests/Unit/FloeStreamReaderTest.php | 194 -- .../Floe/Tests/Unit/FloeStreamWriterTest.php | 445 --- .../Flow/Floe/Tests/Unit/FloeWriterTest.php | 519 ---- .../Flow/Floe/Tests/Unit/FrameDecoderTest.php | 280 -- .../Flow/Floe/Tests/Unit/FrameEncoderTest.php | 104 - .../Flow/Floe/Tests/Unit/FrameReaderTest.php | 97 - .../Flow/Floe/Tests/Unit/OptionsTest.php | 52 - .../Tests/Unit/SplittingFrameEncoderTest.php | 23 - .../Flow/Floe/Tests/Unit/StatisticsTest.php | 51 - .../Tests/Unit/Base64SerializerTest.php | 71 - .../Tests/Unit/NativePHPSerializerTest.php | 71 - src/extension/arrow-ext/Cargo.toml | 4 + src/extension/arrow-ext/Makefile | 6 +- .../Flow/Parquet/Engine/RustParquetEngine.php | 18 + src/extension/arrow-ext/src/interfaces.rs | 34 +- src/extension/arrow-ext/src/lib.rs | 2 +- .../arrow-ext/src/parquet/canonical.rs | 50 +- src/extension/arrow-ext/src/parquet/engine.rs | 67 +- .../arrow-ext/src/parquet/lib_calls.rs | 3 + .../arrow-ext/src/parquet/library.rs | 117 +- .../arrow-ext/src/parquet/options.rs | 8 +- src/extension/arrow-ext/src/parquet/read.rs | 202 +- src/extension/arrow-ext/src/parquet/source.rs | 79 +- src/extension/arrow-ext/src/render.rs | 4 +- .../049_parquet_write_columns_parity.phpt | 6 +- .../054_abi_matches_flow_php_parquet.phpt | 2 +- .../phpt/057_parquet_statistics_options.phpt | 2 +- .../phpt/060_parquet_page_selection.phpt | 4 +- .../tests/phpt/064_parquet_row_filter.phpt | 4 +- .../phpt/068_parquet_schema_hint_read.phpt | 29 + .../phpt/069_parquet_native_file_parity.phpt | 59 + ..._parquet_writer_options_apply_on_open.phpt | 37 + .../arrow-ext/tests/phpt/bootstrap.php | 2 +- .../phpt/fixtures/arrow_schema_hints.parquet | Bin 0 -> 2263 bytes .../tests/phpt/fixtures/arrow_schema_hints.py | 28 + src/extension/flow-php-ext/.gitignore | 3 + src/extension/flow-php-ext/Cargo.lock | 44 + src/extension/flow-php-ext/Cargo.toml | 35 +- src/extension/flow-php-ext/Makefile | 6 +- .../crates/flow-arrow-c/Cargo.toml | 11 + .../crates/flow-arrow-c/src/lib.rs | 439 +++ .../crates/flow-batch-frame/.gitignore | 1 - .../crates/flow-batch-frame/Cargo.lock | 629 ----- .../crates/flow-batch-frame/Cargo.toml | 12 +- .../crates/flow-batch-frame/src/builder.rs | 761 +++++ .../crates/flow-batch-frame/src/column.rs | 132 +- .../crates/flow-batch-frame/src/error.rs | 10 + .../crates/flow-batch-frame/src/frame.rs | 790 ++++++ .../crates/flow-batch-frame/src/key.rs | 63 + .../crates/flow-batch-frame/src/kind.rs | 145 +- .../crates/flow-batch-frame/src/lib.rs | 10 +- .../crates/flow-column-text/Cargo.toml | 13 + .../flow-column-text/src}/date.rs | 103 +- .../crates/flow-column-text/src/float.rs | 258 ++ .../flow-column-text/src}/json.rs | 4 +- .../crates/flow-column-text/src/lib.rs | 100 + .../flow-column-text/src}/value.rs | 48 +- .../crates/flow-csv-tokenizer/Cargo.toml | 10 + .../crates/flow-csv-tokenizer/src/lib.rs | 676 +++++ .../crates/flow-json-records/Cargo.toml | 13 + .../crates/flow-json-records/src/lib.rs | 965 +++++++ .../crates/flow-numeric-string/Cargo.toml | 9 + .../crates/flow-numeric-string/src/lib.rs | 279 ++ .../{ => IO/Extractor}/RustCSVOpenSource.php | 20 +- .../Adapter/CSV/IO/Loader/RustCSVEncoder.php | 38 + .../Flow/ETL/Adapter/CSV/RustCSVEncoder.php | 45 - .../{ => IO/Extractor}/RustJsonOpenSource.php | 10 +- .../JSON/{ => IO/Loader}/RustJsonEncoder.php | 8 +- .../Extractor}/RustParquetOpenSource.php | 12 +- .../IO/Loader/RustParquetFileWriter.php | 48 + .../Adapter/Parquet/RustParquetOpenSink.php | 42 - .../php/Flow/ETL/Column/RustBackend.php | 42 - .../Flow/ETL/Values/Column/RustBackend.php | 69 + .../{ => Values}/Column/RustColumnsBatch.php | 4 +- .../Flow/ETL/{ => Values}/RustIterator.php | 4 +- src/extension/flow-php-ext/src/alloc.rs | 15 +- src/extension/flow-php-ext/src/arrow_c.rs | 341 +-- src/extension/flow-php-ext/src/backend.rs | 261 +- .../flow-php-ext/src/batch_columns.rs | 69 +- src/extension/flow-php-ext/src/builder.rs | 49 +- src/extension/flow-php-ext/src/cast.rs | 66 +- src/extension/flow-php-ext/src/column.rs | 12 +- src/extension/flow-php-ext/src/csv/fold.rs | 2 +- src/extension/flow-php-ext/src/csv/mod.rs | 65 +- src/extension/flow-php-ext/src/csv/source.rs | 232 +- .../flow-php-ext/src/csv/tokenizer.rs | 363 --- src/extension/flow-php-ext/src/csv/write.rs | 37 +- src/extension/flow-php-ext/src/ctx.rs | 13 +- src/extension/flow-php-ext/src/exception.rs | 2 + src/extension/flow-php-ext/src/frame.rs | 96 + src/extension/flow-php-ext/src/interfaces.rs | 107 +- src/extension/flow-php-ext/src/iterator.rs | 4 +- src/extension/flow-php-ext/src/json/read.rs | 972 +------ src/extension/flow-php-ext/src/json/source.rs | 10 +- src/extension/flow-php-ext/src/json/write.rs | 35 +- .../flow-php-ext/src/kind_builder.rs | 506 ---- src/extension/flow-php-ext/src/lib.rs | 8 +- src/extension/flow-php-ext/src/parquet/etl.rs | 89 +- src/extension/flow-php-ext/src/physical.rs | 2 +- src/extension/flow-php-ext/src/plan.rs | 77 +- src/extension/flow-php-ext/src/render.rs | 146 +- src/extension/flow-php-ext/src/source.rs | 14 +- src/extension/flow-php-ext/src/text/batch.rs | 5 +- src/extension/flow-php-ext/src/text/float.rs | 47 - src/extension/flow-php-ext/src/text/mod.rs | 93 +- src/extension/flow-php-ext/src/values.rs | 16 +- .../tests/phpt/001_extension_loaded.phpt | 41 +- .../tests/phpt/045_csv_no_leaks.phpt | 4 +- .../tests/phpt/046_csv_narrow_parity.phpt | 6 +- .../phpt/047_csv_narrow_candidate_gating.phpt | 2 +- .../tests/phpt/048_csv_widen_parity.phpt | 4 +- .../tests/phpt/049_csv_fold_no_leaks.phpt | 7 +- .../tests/phpt/050_csv_tokenizer_fuzz.phpt | 4 +- .../tests/phpt/051_csv_guards.phpt | 19 +- .../tests/phpt/053_csv_fold_json_parity.phpt | 2 +- .../phpt/056_native_constructors_refused.phpt | 28 +- .../tests/phpt/057_native_encode_parity.phpt | 24 +- .../tests/phpt/058_native_values_parity.phpt | 12 +- .../tests/phpt/059_native_cast_parity.phpt | 4 +- .../phpt/060_native_decode_refusals.phpt | 38 +- .../phpt/061_native_mixed_class_building.phpt | 15 +- .../tests/phpt/062_native_no_leaks.phpt | 11 +- .../phpt/063_csv_next_columns_parity.phpt | 12 +- .../phpt/064_csv_next_columns_no_leaks.phpt | 6 +- .../tests/phpt/065_type_plan_by_object.phpt | 2 +- .../066_iso_datetime_fast_path_parity.phpt | 6 +- .../067_adopt_refuses_untyped_column.phpt | 10 +- .../tests/phpt/068_parquet_read_parity.phpt | 13 +- .../tests/phpt/069_parquet_offset_limit.phpt | 27 +- .../phpt/070_parquet_write_roundtrip.phpt | 70 +- .../tests/phpt/071_parquet_no_leaks.phpt | 22 +- .../phpt/072_parquet_allocations_counted.phpt | 26 +- .../tests/phpt/077_text_float_parity.phpt | 38 +- .../phpt/078_text_json_string_parity.phpt | 10 +- .../phpt/079_text_date_format_parity.phpt | 12 +- .../tests/phpt/080_csv_write_parity.phpt | 24 +- .../tests/phpt/081_json_write_parity.phpt | 28 +- .../tests/phpt/082_write_no_leaks.phpt | 48 +- .../phpt/084_native_nested_cast_parity.phpt | 4 +- .../phpt/085_native_nested_cast_no_leaks.phpt | 6 +- .../tests/phpt/086_json_read_parity.phpt | 17 +- .../tests/phpt/087_json_read_no_leaks.phpt | 6 +- .../tests/phpt/088_json_read_streaming.phpt | 4 +- .../phpt/089_parquet_arrow_c_no_leaks.phpt | 24 +- .../090_parquet_schema_refused_at_open.phpt | 40 +- ...091_parquet_canonical_layout_contract.phpt | 24 +- .../phpt/092_failure_without_autoloader.phpt | 8 +- .../phpt/093_adapter_writers_contracts.phpt | 30 +- .../094_rust_iterator_generator_parity.phpt | 4 +- .../phpt/095_sources_reentrant_stream.phpt | 6 +- .../096_encoders_columns_left_to_php.phpt | 11 +- .../097_parquet_uint64_numeric_string.phpt | 36 +- .../tests/phpt/098_rust_append_physicals.phpt | 8 +- .../phpt/099_abi_matches_flow_php_etl.phpt | 6 +- .../tests/phpt/100_csv_preset_headers.phpt | 74 + .../phpt/101_numeric_grammar_parity.phpt | 114 + .../phpt/102_csv_native_file_parity.phpt | 116 + .../tests/phpt/103_frame_decode_parity.phpt | 176 ++ .../phpt/104_frame_decode_zero_copy.phpt | 90 + .../tests/phpt/105_frame_encode_parity.phpt | 113 + ...t_backend_describe_and_append_refusal.phpt | 33 + .../flow-php-ext/tests/phpt/bootstrap.php | 115 +- .../BlobService/CopyBlob/CopyBlobOptions.php | 13 +- .../CreateContainerOptions.php | 13 +- .../DeleteBlob/DeleteBlobOptions.php | 13 +- .../DeleteContainerOptions.php | 13 +- .../BlobService/GetBlob/GetBlobOptions.php | 13 +- .../GetBlobPropertiesOptions.php | 13 +- .../GetBlockBlobBlockListOptions.php | 13 +- .../GetContainerPropertiesOptions.php | 13 +- .../BlobService/ListBlobs/ListBlobOptions.php | 13 +- .../PutBlockBlob/PutBlockBlobOptions.php | 13 +- .../PutBlockBlobBlockOptions.php | 13 +- .../PutBlockBlobBlockListOptions.php | 13 +- .../Azure/SDK/Endpoints/UserAgentHeader.php | 20 - .../src/Flow/Filesystem/Filesystem.php | 14 + .../Filesystem/Local/MemoryFilesystem.php | 59 +- .../Local/NativeLocalFilesystem.php | 39 +- .../Filesystem/Local/StdOutFilesystem.php | 13 + .../filesystem/src/Flow/Filesystem/Path.php | 109 + .../Block/NativeLocalFileBlocksFactory.php | 5 + .../Stream/BlockCachedSourceStream.php | 117 + .../Stream/NativeLocalSourceStream.php | 10 +- .../Filesystem/Stream/RangeSourceStream.php | 159 ++ .../Telemetry/TraceableFilesystem.php | 16 + .../Tests/Context/OperatingSystem.php | 20 + .../Tests/Double/FailingCloseFilesystem.php | 14 + .../Tests/Double/FailingMvFilesystem.php | 97 + .../Tests/Double/FailingRmFilesystem.php | 14 + .../Double/FakeNativeLocalFilesystem.php | 19 + .../Flow/Filesystem/Tests/Double/Flavour.php | 11 + .../Double/ReadRecordingSourceStream.php | 63 + .../Tests/Double/ThrowingSourceFilesystem.php | 14 + .../Tests/Integration/FileListingTest.php | 6 +- .../Local/MemoryFilesystemTest.php | 132 +- .../Local/StdOutFilesystemTest.php | 16 + .../Integration/NativeLocalFilesystemTest.php | 97 +- .../NativeLocalSourceStreamTest.php | 19 + .../OS/Unix/NativeLocalFilesystemTest.php | 6 +- .../Tests/Integration/OS/Unix/PathTest.php | 12 +- .../Integration/OS/Unix/RealpathTest.php | 6 +- .../OS/Unix/Stream/BlocksUnixSpecificTest.php | 6 +- .../OS/Unix/Stream/BlocksUnixTest.php | 6 +- .../OS/Windows/NativeLocalFilesystemTest.php | 6 +- .../Tests/Integration/OS/Windows/PathTest.php | 12 +- .../Integration/OS/Windows/RealpathTest.php | 6 +- .../Stream/BlocksWindowsSpecificTest.php | 6 +- .../OS/Windows/Stream/BlocksWindowsTest.php | 6 +- .../Flow/Filesystem/Tests/OperatingSystem.php | 20 - .../Flow/Filesystem/Tests/Unit/PathTest.php | 72 + .../NativeLocalFileBlocksFactoryTest.php | 40 + .../Stream/BlockCachedSourceStreamTest.php | 85 + .../Unit/Stream/RangeSourceStreamTest.php | 150 + .../Telemetry/TraceableFilesystemTest.php | 21 + .../Command/ReadMetadataCommand.php | 4 +- .../Tests/Integration/ReadDDLCommandTest.php | 6 +- .../Tests/Integration/ReadDataCommandTest.php | 6 +- .../Tests/Integration/ReadMetadataTest.php | 33 +- .../ColumnData/ReadFlatColumnValues.php | 55 +- .../ColumnData/WriteFlatColumnValues.php | 141 - .../Parquet/Engine/AdaptiveParquetEngine.php | 40 +- .../Parquet/Engine/Arrow/OptionsConverter.php | 42 +- .../Engine/Arrow/ReadPlanConverter.php | 2 - .../Engine/Arrow/RustRowGroupChunks.php | 60 +- .../Flow/Parquet/Engine/ArrowExtension.php | 2 +- .../Flow/Parquet/Engine/PhpParquetEngine.php | 39 +- .../Flow/Parquet/Engine/PhpParquetFooter.php | 5 +- .../Parquet/Filter/Binding/BoundFilter.php | 111 +- .../Flow/Parquet/Filter/Binding/ExactLeaf.php | 25 + .../Parquet/Filter/Binding/InexactLeaf.php | 23 + .../Parquet/Filter/Binding/LeafResolver.php | 107 - .../Parquet/Filter/Binding/ResolvedLeaf.php | 26 - .../src/Flow/Parquet/Filter/ColumnRef.php | 2 +- .../Parquet/Filter/Evaluation/BatchTruth.php | 7 +- .../Filter/Evaluation/BloomEvaluator.php | 9 +- .../Filter/Evaluation/FilterEvaluator.php | 9 +- .../Filter/Evaluation/PageZoneMaps.php | 12 +- .../Filter/Evaluation/RowEvaluator.php | 7 +- .../Filter/Evaluation/RowGroupZoneMaps.php | 8 +- .../Filter/Evaluation/StatisticsEvaluator.php | 12 +- src/lib/parquet/src/Flow/Parquet/Options.php | 125 + .../src/Flow/Parquet/ParquetEngine.php | 13 + .../parquet/src/Flow/Parquet/ParquetFile.php | 4 +- .../ParquetFile/Index/StreamIndexes.php | 11 +- .../Parquet/ParquetFile/Page/ColumnData.php | 157 -- .../Page/Header/DataPageHeader.php | 5 +- .../Page/Header/DataPageHeaderV2.php | 9 +- .../src/Flow/Parquet/ParquetFile/RowGroup.php | 14 - .../ParquetFile/RowGroup/ColumnChunk.php | 8 +- .../ParquetFile/RowGroup/FlowColumnChunk.php | 14 - .../ParquetFile/RowGroup/StatisticsReader.php | 70 - .../Flow/Parquet/ParquetFile/Statistics.php | 28 +- .../src/Flow/Parquet/Reader/ChunkBuffer.php | 64 + .../Flow/Parquet/Reader/ColumnChunkReader.php | 47 +- .../Flow/Parquet/Reader/ColumnChunkViewer.php | 53 +- .../src/Flow/Parquet/Reader/ColumnReader.php | 10 +- .../src/Flow/Parquet/Reader/PageReader.php | 15 +- .../src/Flow/Parquet/Reader/ReadPlan.php | 58 + .../src/Flow/Parquet/Reader/ReadPlanner.php | 60 - .../src/Flow/Parquet/Reader/SelectedPages.php | 47 +- .../Parquet/Writer/ColumnChunkBuilders.php | 8 - .../src/Flow/Parquet/Writer/PageContainer.php | 5 - .../Flow/Parquet/Tests/Context/FilePlan.php | 5 +- .../Tests/Context/StatisticsParityRows.php | 8 +- .../Engine/RowGroupChunksParityTest.php | 12 + .../Integration/IO/SchemaReadingTest.php | 4 +- .../Tests/Integration/IO/WriterTest.php | 47 +- .../Parquet/Tests/Integration/OptionsTest.php | 58 + .../ColumnData/ReadFlatColumnValuesTest.php | 3 +- .../ColumnData/WriteFlatColumnValuesTest.php | 188 -- .../Unit/Engine/AdaptiveParquetEngineTest.php | 19 + .../Engine/Arrow/OptionsConverterTest.php | 24 +- .../Engine/Arrow/RustRowGroupChunksTest.php | 16 + .../Tests/Unit/Engine/ArrowExtensionTest.php | 2 +- .../Unit/Engine/PhpParquetEngineTest.php | 29 + .../Unit/Engine/PhpParquetFileWriterTest.php | 14 +- .../Unit/Engine/PhpParquetFooterTest.php | 13 + .../Unit/Engine/RustParquetEngineTest.php | 25 + .../Unit/Filter/Binding/BoundFilterTest.php | 148 +- .../Unit/Filter/Binding/LeafResolverTest.php | 149 - .../Page/Header/DataPageHeaderTest.php | 2 +- .../RowGroup/StatisticsReaderTest.php | 68 - .../Tests/Unit/ParquetFile/StatisticsTest.php | 67 + .../Tests/Unit/Reader/ChunkBufferTest.php | 44 + .../Unit/Reader/ColumnChunkViewerTest.php | 35 + .../Tests/Unit/Reader/ColumnReaderTest.php | 13 +- .../Tests/Unit/Reader/ReadPlanTest.php | 196 ++ .../Tests/Unit/Reader/ReadPlannerTest.php | 214 -- .../Tests/Unit/Reader/SelectedPagesTest.php | 24 +- .../FlatColumnChunkBuilderTest.php | 4 +- .../Unit/Writer/ColumnChunkBuildersTest.php | 15 +- .../Flow/PostgreSql/AST/Nodes/Statement.php | 13 - .../AST/Nodes/Statement/AlterStatement.php | 3 - .../AST/Nodes/Statement/AnalyzeStatement.php | 3 - .../AST/Nodes/Statement/CallStatement.php | 3 - .../Nodes/Statement/CheckpointStatement.php | 3 - .../AST/Nodes/Statement/CloseStatement.php | 3 - .../AST/Nodes/Statement/ClusterStatement.php | 3 - .../AST/Nodes/Statement/CommentStatement.php | 3 - .../AST/Nodes/Statement/CopyStatement.php | 3 - .../AST/Nodes/Statement/CreateStatement.php | 3 - .../Nodes/Statement/DeallocateStatement.php | 3 - .../AST/Nodes/Statement/DeclareStatement.php | 3 - .../AST/Nodes/Statement/DeleteStatement.php | 3 - .../AST/Nodes/Statement/DiscardStatement.php | 3 - .../AST/Nodes/Statement/DoStatement.php | 3 - .../AST/Nodes/Statement/DropStatement.php | 3 - .../AST/Nodes/Statement/ExecuteStatement.php | 3 - .../AST/Nodes/Statement/ExplainStatement.php | 3 - .../AST/Nodes/Statement/FetchStatement.php | 3 - .../AST/Nodes/Statement/GrantStatement.php | 3 - .../AST/Nodes/Statement/ImportStatement.php | 3 - .../AST/Nodes/Statement/IndexStatement.php | 3 - .../AST/Nodes/Statement/InsertStatement.php | 3 - .../AST/Nodes/Statement/ListenStatement.php | 3 - .../AST/Nodes/Statement/LoadStatement.php | 3 - .../AST/Nodes/Statement/LockStatement.php | 3 - .../AST/Nodes/Statement/MergeStatement.php | 3 - .../AST/Nodes/Statement/MoveStatement.php | 3 - .../AST/Nodes/Statement/NotifyStatement.php | 3 - .../AST/Nodes/Statement/PrepareStatement.php | 3 - .../AST/Nodes/Statement/ReassignStatement.php | 3 - .../AST/Nodes/Statement/RefreshStatement.php | 3 - .../AST/Nodes/Statement/ReindexStatement.php | 3 - .../AST/Nodes/Statement/ReleaseStatement.php | 3 - .../AST/Nodes/Statement/ResetStatement.php | 3 - .../AST/Nodes/Statement/RevokeStatement.php | 3 - .../AST/Nodes/Statement/RuleStatement.php | 3 - .../Nodes/Statement/SavepointStatement.php | 3 - .../Statement/SecurityLabelStatement.php | 3 - .../AST/Nodes/Statement/SelectStatement.php | 3 - .../AST/Nodes/Statement/SetStatement.php | 3 - .../AST/Nodes/Statement/ShowStatement.php | 3 - .../Nodes/Statement/TransactionStatement.php | 3 - .../AST/Nodes/Statement/TruncateStatement.php | 3 - .../AST/Nodes/Statement/UnknownStatement.php | 3 - .../AST/Nodes/Statement/UnlistenStatement.php | 3 - .../AST/Nodes/Statement/UpdateStatement.php | 3 - .../AST/Nodes/Statement/VacuumStatement.php | 3 - .../AST/Nodes/Statement/ValuesStatement.php | 3 - .../AST/Nodes/Statement/ViewStatement.php | 3 - .../PostgreSql/AST/Nodes/StatementTrait.php | 34 - .../PostgreSql/QueryBuilder/AstDeparser.php | 33 + .../Flow/PostgreSql/QueryBuilder/AstToSql.php | 46 - .../QueryBuilder/Copy/CopyFromBuilder.php | 9 +- .../QueryBuilder/Copy/CopyToBuilder.php | 9 +- .../Cursor/CloseCursorBuilder.php | 9 +- .../Cursor/DeclareCursorBuilder.php | 9 +- .../Cursor/FetchCursorBuilder.php | 9 +- .../QueryBuilder/Delete/DeleteBuilder.php | 9 +- .../QueryBuilder/Insert/InsertBuilder.php | 16 +- .../QueryBuilder/Listen/ListenBuilder.php | 9 +- .../QueryBuilder/Merge/MergeBuilder.php | 9 +- .../QueryBuilder/Notify/NotifyBuilder.php | 9 +- .../AlterSequence/AlterSequenceBuilder.php | 9 +- .../AlterSequenceLoggingBuilder.php | 9 +- .../AlterSequenceOwnerBuilder.php | 9 +- .../AlterSequenceSchemaBuilder.php | 9 +- .../AlterSequence/RenameSequenceBuilder.php | 9 +- .../Schema/AlterTable/AlterTableBuilder.php | 9 +- .../AlterTable/AlterTableLoggingBuilder.php | 9 +- .../AlterTable/AlterTableSchemaBuilder.php | 9 +- .../Schema/AlterTable/RenameTableBuilder.php | 9 +- .../CreateSequence/CreateSequenceBuilder.php | 9 +- .../Schema/CreateTable/CreateTableBuilder.php | 9 +- .../CreateTableAs/CreateTableAsBuilder.php | 9 +- .../Schema/Database/CreateDatabaseBuilder.php | 9 +- .../Schema/Database/DropDatabaseBuilder.php | 9 +- .../Schema/Domain/AlterDomainBuilder.php | 9 +- .../Schema/Domain/CreateDomainBuilder.php | 9 +- .../Schema/Domain/DropDomainBuilder.php | 9 +- .../DropSequence/DropSequenceBuilder.php | 9 +- .../Schema/DropTable/DropTableBuilder.php | 9 +- .../Extension/AlterExtensionBuilder.php | 9 +- .../Extension/CreateExtensionBuilder.php | 9 +- .../Schema/Extension/DropExtensionBuilder.php | 9 +- .../Schema/Function/AlterFunctionBuilder.php | 9 +- .../Schema/Function/AlterProcedureBuilder.php | 9 +- .../Schema/Function/CallBuilder.php | 9 +- .../Schema/Function/CreateFunctionBuilder.php | 9 +- .../Function/CreateProcedureBuilder.php | 9 +- .../Schema/Function/DoBuilder.php | 9 +- .../Schema/Function/DropFunctionBuilder.php | 9 +- .../Schema/Function/DropProcedureBuilder.php | 9 +- .../Schema/Grant/GrantBuilder.php | 9 +- .../Schema/Grant/GrantRoleBuilder.php | 9 +- .../Schema/Grant/RevokeBuilder.php | 9 +- .../Schema/Grant/RevokeRoleBuilder.php | 9 +- .../AlterTablespaceIndexBuilder.php | 9 +- .../Index/AlterIndex/RenameIndexBuilder.php | 9 +- .../Index/CreateIndex/CreateIndexBuilder.php | 9 +- .../Index/DropIndex/DropIndexBuilder.php | 9 +- .../Schema/Index/Reindex/ReindexBuilder.php | 9 +- .../Schema/Ownership/DropOwnedBuilder.php | 9 +- .../Schema/Ownership/ReassignOwnedBuilder.php | 9 +- .../Schema/Role/AlterRoleBuilder.php | 9 +- .../Schema/Role/AlterRoleRenameBuilder.php | 9 +- .../Schema/Role/CreateRoleBuilder.php | 9 +- .../Schema/Role/DropRoleBuilder.php | 9 +- .../Schema/Rule/CreateRuleBuilder.php | 9 +- .../Schema/Rule/DropRuleBuilder.php | 9 +- .../Schema/Schema/AlterSchemaOwnerBuilder.php | 9 +- .../Schema/AlterSchemaRenameBuilder.php | 9 +- .../Schema/Schema/CreateSchemaBuilder.php | 9 +- .../Schema/Schema/DropSchemaBuilder.php | 9 +- .../Schema/Session/ResetRoleBuilder.php | 9 +- .../Schema/Session/SetRoleBuilder.php | 9 +- .../Schema/Trigger/AlterTriggerBuilder.php | 9 +- .../Schema/Trigger/CreateTriggerBuilder.php | 9 +- .../Schema/Trigger/DropTriggerBuilder.php | 9 +- .../Schema/Truncate/TruncateBuilder.php | 9 +- .../Schema/Type/AlterEnumTypeBuilder.php | 9 +- .../Type/CreateCompositeTypeBuilder.php | 9 +- .../Schema/Type/CreateEnumTypeBuilder.php | 9 +- .../Schema/Type/CreateRangeTypeBuilder.php | 9 +- .../Schema/Type/DropTypeBuilder.php | 9 +- .../AlterMatViewOwnerBuilder.php | 9 +- .../AlterMatViewSchemaBuilder.php | 9 +- .../AlterMatViewTablespaceBuilder.php | 9 +- .../RenameMatViewBuilder.php | 9 +- .../View/AlterView/AlterViewOwnerBuilder.php | 9 +- .../View/AlterView/AlterViewSchemaBuilder.php | 9 +- .../View/AlterView/RenameViewBuilder.php | 9 +- .../CreateMaterializedViewBuilder.php | 9 +- .../View/CreateView/CreateViewBuilder.php | 9 +- .../DropMaterializedViewBuilder.php | 9 +- .../Schema/View/DropView/DropViewBuilder.php | 9 +- .../RefreshMaterializedViewBuilder.php | 9 +- .../QueryBuilder/Select/SelectBuilder.php | 9 +- .../QueryBuilder/Transaction/BeginBuilder.php | 9 +- .../Transaction/CommitBuilder.php | 9 +- .../PreparedTransactionBuilder.php | 9 +- .../Transaction/RollbackBuilder.php | 9 +- .../Transaction/SavepointBuilder.php | 9 +- .../Transaction/SetTransactionBuilder.php | 9 +- .../QueryBuilder/Unlisten/UnlistenBuilder.php | 9 +- .../QueryBuilder/Update/UpdateBuilder.php | 9 +- .../QueryBuilder/Utility/AnalyzeBuilder.php | 9 +- .../QueryBuilder/Utility/ClusterBuilder.php | 9 +- .../QueryBuilder/Utility/CommentBuilder.php | 9 +- .../QueryBuilder/Utility/DiscardBuilder.php | 9 +- .../QueryBuilder/Utility/ExplainBuilder.php | 9 +- .../QueryBuilder/Utility/LockBuilder.php | 9 +- .../QueryBuilder/Utility/VacuumBuilder.php | 9 +- .../Flow/PostgreSql/Schema/TableOptions.php | 2 +- .../Unit/AST/Nodes/StatementAssertTest.php | 49 - .../Unit/QueryBuilder/AstDeparserTest.php | 34 + .../src/Flow/Telemetry/ObjectExtractor.php | 10 +- .../Provider/Console/ConsoleOutput.php | 2 +- .../Tests/Unit/ObjectExtractorTest.php | 15 + .../Flow/Types/Type/Logical/OptionalType.php | 2 +- .../Type/Logical/PositiveIntegerType.php | 6 +- .../Documentation/Models/FunctionModel.php | 2 +- .../Flow/Documentation/Models/MethodModel.php | 2 +- .../Unit/Docs/DocumentMethodCallsTest.php | 19 +- .../Tests/Unit/Docs/InferredTypesTest.php | 2 +- .../codemirror/completions/dataframe.js | 232 +- .../assets/codemirror/completions/dsl.js | 237 +- .../assets/codemirror/completions/flow.js | 35 +- .../completions/scalarfunctionchain.js | 344 +-- web/landing/composer.lock | 22 +- .../topics/batching/column_backend/code.php | 2 +- .../examples/topics/caching/cache/code.php | 4 +- .../topics/columns/rename_style/code.php | 2 +- .../topics/inspecting/for_each/code.php | 2 +- .../examples/topics/joins/join/code.php | 4 +- .../examples/topics/joins/join/description.md | 2 +- .../examples/topics/joins/join_each/code.php | 31 +- .../topics/joins/join_each/description.md | 4 +- .../topics/joins/left_right_inner/code.php | 4 +- .../topics/reading/data_frame/_meta.yaml | 2 - .../topics/reading/data_frame/code.php | 27 - .../topics/reading/data_frame/description.md | 2 - .../reading/data_frame/documentation.md | 3 - .../topics/reading/dbal/limit_offset/code.php | 3 +- .../topics/reading/dbal/queries/code.php | 2 +- .../topics/reading/dbal/query/code.php | 2 +- .../examples/topics/schema/apply/code.php | 4 +- .../examples/topics/schema/display/code.php | 2 +- .../examples/topics/schema/inferring/code.php | 2 +- .../examples/topics/schema/validate/code.php | 4 +- .../topics/transformations/custom/code.php | 5 +- .../writing/postgresql/transaction/code.php | 2 +- .../postgresql/transaction/description.md | 6 +- web/landing/resources/api.json | 2 +- web/landing/resources/dsl.json | 2 +- .../GenerateDataFrameCompleterCommand.php | 2 +- .../Github/ContributorsRequestFactory.php | 2 +- .../Website/Service/FlowConfigFactory.php | 12 +- .../src/Flow/Website/Service/Github.php | 8 +- .../extractor-01.php | 2 +- .../extractor-04.php | 2 +- .../string-length-validation.php | 4 +- .../user-id-join-data-frame-factory.php | 2 +- .../validation-transformation.php | 4 +- .../templates/documentation/dsl.html.twig | 2 +- .../documentation/navigation_left.html.twig | 1 + .../Functional/PlaygroundBranchSinkTest.php | 46 - .../Tests/Functional/PlaygroundBranchTest.php | 46 + 4374 files changed, 200437 insertions(+), 172155 deletions(-) create mode 100644 .nix/php/lib/benchmark.ini.dist create mode 100755 benchmarks/reference/duckdb/bench.sh create mode 100644 benchmarks/reference/polars/bench.py create mode 100644 benchmarks/reference/spark/bench.py delete mode 100644 benchmarks/src/Joining/SellersDataFrameFactory.php create mode 100644 benchmarks/src/Joining/SellersExtractorFactory.php create mode 100644 documentation/architecture.md delete mode 100644 src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php delete mode 100644 src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroLoader.php create mode 100644 src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/IO/Extractor/AvroExtractor.php create mode 100644 src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/IO/Loader/AvroLoader.php rename src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Unit/{FlixTech => IO/Extractor}/AvroExtractorTest.php (82%) delete mode 100644 src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/ChartJSLoader.php rename src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/{ => IO/Loader}/Chart.php (79%) rename src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/{ => IO/Loader}/Chart/BarChart.php (89%) rename src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/{ => IO/Loader}/Chart/LineChart.php (89%) rename src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/{ => IO/Loader}/Chart/PieChart.php (92%) create mode 100644 src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/IO/Loader/ChartJSLoader.php create mode 100644 src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Context/DescribableFixtures.php rename src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/{ => IO/Loader}/ChartJSLoaderTest.php (98%) delete mode 100644 src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Unit/Chart/BarChartTest.php create mode 100644 src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Unit/DescribableRoundTripTest.php create mode 100644 src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Unit/IO/Loader/Chart/BarChartTest.php rename src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Unit/{ => IO/Loader}/Chart/LineChartTest.php (92%) rename src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Unit/{ => IO/Loader}/Chart/PieChartTest.php (95%) delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/AdaptiveCSVEncoder.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/AdaptiveCSVOpenSource.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVDialect.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileBatches.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileReader.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVFileSample.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVHeader.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLoader.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVOpenSink.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVOpenSource.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVReadOptions.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVSampledFiles.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVSampledRows.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVSamples.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVWriteOptions.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/Detector/Options.php delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/Exception/CantDetectCSVOptions.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/Foundation/Exception/CantDetectCSVOptions.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/CSVDialect.php rename src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/{ => IO/Extractor}/CSVDecoder.php (80%) rename src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/{ => IO/Extractor}/CSVDetector.php (80%) create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVDialects.php rename src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/{ => IO/Extractor}/CSVEnclosureScan.php (97%) create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVExtractor.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVFileBatches.php rename src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/{ => IO/Extractor}/CSVLineReader.php (98%) create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVOpenSource.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVReadOptions.php rename src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/{ => IO/Extractor}/CSVRecordBoundary.php (98%) create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVRecordResync.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/CSVSourceOpener.php rename src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/{ => IO/Extractor}/Detector/Option.php (80%) create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/Detector/Options.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Extractor/PhpCSVOpenSource.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Inference/CSVSniffer.php rename src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/{ => IO/Loader}/CSVEncoder.php (82%) create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Loader/CSVFileWriter.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Loader/CSVLoader.php create mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/IO/Loader/CSVWriteOptions.php rename src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/{ => IO/Loader}/PhpCSVEncoder.php (95%) delete mode 100644 src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/PhpCSVOpenSource.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/CSVFileFormat.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Context/DescribableFixtures.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/AdaptiveCSVOpenSourceTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileReaderTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVFileSampleTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVLineReaderTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVSamplesTest.php rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/{ => IO/Extractor}/CSVDetectorTest.php (82%) rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/{ => IO/Extractor}/CSVExtractorBatchContractTest.php (87%) create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVExtractorTest.php rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/{ => IO/Extractor}/CSVExtractorTypedColumnsTest.php (77%) create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVLineReaderTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVSourceOpenerTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Extractor/CSVSplitReadTest.php rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/{ => IO/Extractor}/PhpCSVOpenSourceTest.php (94%) rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/{ => IO/Extractor}/RustCSVOpenSourceTest.php (90%) create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/FileFormatContractTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/IO/Inference/CSVSnifferTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/AdaptiveCSVEncoderTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVDialectTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVExtractorTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVFileBatchesTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVHeaderTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVLineReaderTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVLoaderTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVOpenSinkTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVSampledFilesTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/CSVSampledRowsTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/DescribableRoundTripTest.php delete mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/Detector/OptionTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/CSVDialectTest.php rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/{ => IO/Extractor}/CSVDecoderTest.php (97%) create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVDialectsTest.php rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/{ => IO/Extractor}/CSVEnclosureScanTest.php (86%) create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVExtractorTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVFileBatchesTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVLineReaderTest.php rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/{ => IO/Extractor}/CSVReadOptionsTest.php (77%) rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/{ => IO/Extractor}/CSVRecordBoundaryTest.php (90%) create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/CSVRecordResyncTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/Detector/OptionTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Extractor/MetadataColumnsExtractorTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Loader/CSVFileWriterTest.php create mode 100644 src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/IO/Loader/CSVLoaderTest.php rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/{ => IO/Loader}/CSVWriteOptionsTest.php (89%) rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/{ => IO/Loader}/PhpCSVEncoderTest.php (98%) rename src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Unit/{ => IO/Loader}/RustCSVEncoderTest.php (94%) delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/DbalDataFrameFactory.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/DbalLimitOffsetExtractor.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/DbalLoader.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/Explain/ExplainedRows.php rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => Foundation/Exception}/ProbeRefusal.php (92%) create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Extractor/DbalExtractorFactory.php rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/DbalKeySetExtractor.php (82%) create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Extractor/DbalLimitOffsetExtractor.php rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/DbalQueryExtractor.php (78%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/DbalResultSchema.php (92%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/DescribeQuery.php (82%) create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Extractor/Explain/ExplainedRows.php rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/Explain/MySqlExplainedRows.php (95%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/Explain/PostgreSqlExplainedRows.php (96%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/LiteralParameter.php (93%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/MysqliPlaceholders.php (84%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/MysqliResultColumns.php (92%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/MysqliTypesMap.php (97%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/NativePlaceholders.php (90%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/NativeTypes.php (83%) create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Extractor/Order.php rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/OrderBy.php (78%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/Pages.php (85%) create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Extractor/Pagination/Key.php create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Extractor/Pagination/KeySet.php create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Extractor/Pagination/KeySetPage.php create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Extractor/Pagination/KeyValues.php create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Extractor/Pagination/Order.php create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Extractor/Parameter.php rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/ParametersSet.php (91%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/PdoSqliteColumnNames.php (91%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/PgSqlResultColumns.php (95%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/PgSqlTypesMap.php (95%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/PlaceholderDialect.php (88%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/PlaceholderRewriter.php (96%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/PostgreSqlPlaceholders.php (85%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/QueryParameter.php (82%) create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Extractor/ResultColumn.php rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/RewrittenSql.php (85%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/Sqlite3ColumnNames.php (90%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/SqliteResultSchema.php (78%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/SqliteTypes.php (91%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/Table.php (83%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Extractor}/TypedColumns.php (76%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Loader}/DbalEncoder.php (84%) create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/IO/Loader/DbalLoader.php rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Loader}/DbalTransaction.php (97%) rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => IO/Loader}/IsolationLevelRestore.php (93%) delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/Order.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/Pagination/Key.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/Pagination/KeySet.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/Pagination/KeySetPage.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/Pagination/KeyValues.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/Pagination/Order.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/Parameter.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/ResultColumn.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/SchemaConverter.php delete mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/TypesMap.php rename src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/{ => Values/Schema}/DbalMetadata.php (96%) create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/Values/Schema/SchemaConverter.php create mode 100644 src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/Values/Schema/TypesMap.php create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Context/DescribableFixtures.php rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/{Context => Double}/CommitCounter.php (87%) rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/{Context => Double}/InsertQueryCounter.php (95%) rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/{Context => Double}/SelectQueryCounter.php (95%) delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/DbalDataFrameFactoryTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/DbalKeySetExtractorTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/DbalLimitOffsetExtractorTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/DbalLoaderTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/Dialects/MySQLTransactionSinkTest.php create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/Dialects/MySQLTransactionalTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/Dialects/PostgreSQLTransactionSinkTest.php create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/Dialects/PostgreSQLTransactionalTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/Dialects/SqliteTransactionSinkTest.php create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/Dialects/SqliteTransactionalTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/Explain/ExplainedRowsTest.php create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/IO/Extractor/DbalExtractorFactoryTest.php create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/IO/Extractor/DbalKeySetExtractorTest.php create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/IO/Extractor/DbalLimitOffsetExtractorTest.php create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/IO/Extractor/Explain/ExplainedRowsTest.php rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/{ => IO/Extractor}/Explain/MySqlExplainedRowsTest.php (89%) rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/{ => IO/Extractor}/Explain/PostgreSqlExplainedRowsTest.php (86%) create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/IO/Loader/DbalLoaderTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalKeySetExtractorTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalLimitOffsetExtractorTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalTransactionSinkTest.php create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DescribableRoundTripTest.php rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => Foundation/Exception}/ProbeRefusalTest.php (88%) rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Extractor}/DbalExtractorBatchContractTest.php (92%) create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/IO/Extractor/DbalKeySetExtractorTest.php create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/IO/Extractor/DbalLimitOffsetExtractorTest.php rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Extractor}/DbalQueryExtractorTest.php (83%) rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Extractor}/DbalResultSchemaTest.php (89%) rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Extractor}/DescribeQueryTest.php (88%) create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/IO/Extractor/MetadataColumnsExtractorTest.php rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Extractor}/MysqliTypesMapTest.php (96%) rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Extractor}/NativePlaceholdersTest.php (95%) rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Extractor}/PagesTest.php (82%) create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/IO/Extractor/Pagination/KeySetPageTest.php rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Extractor}/Pagination/KeySetTest.php (81%) create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/IO/Extractor/Pagination/KeyValuesTest.php rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Extractor}/PgSqlTypesMapTest.php (95%) rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Extractor}/SqliteResultSchemaTest.php (84%) rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Extractor}/TypedColumnsTest.php (84%) rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Loader}/DbalEncoderTest.php (96%) create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/IO/Loader/DbalLoaderTest.php rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Loader}/DbalTransactionTest.php (97%) create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/IO/Loader/DbalTransactionalTest.php rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => IO/Loader}/IsolationLevelRestoreTest.php (89%) delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/Pagination/KeySetPageTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/Pagination/KeyValuesTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/SchemaConverterTest.php delete mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/TypesMapTest.php rename src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/{ => Values/Schema}/DbalMetadataTest.php (81%) create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/Values/Schema/SchemaConverterTest.php create mode 100644 src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/Values/Schema/TypesMapTest.php delete mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelExtractor.php delete mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelFileBatches.php delete mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelHeader.php delete mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelLoader.php delete mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelReadOptions.php delete mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelReader.php delete mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelWriter.php delete mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/Function/IsValidExcelSheetName.php create mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/Functions/IsValidExcelSheetName.php rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{Sheet => Functions}/SheetNameAssertion.php (83%) create mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelExtractor.php create mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelFileBatches.php rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{ => IO/Extractor}/ExcelFormatDetector.php (87%) create mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelReadOptions.php create mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/ExcelReader.php rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{ => IO/Extractor}/Sheet/OpenSheet.php (84%) rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{ => IO/Extractor}/Sheet/SheetCells.php (92%) rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{ => IO/Extractor}/Sheet/SheetsManager.php (82%) rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{ => IO/Extractor}/Sheet/XlsxSheetRows.php (99%) create mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Extractor/WorkbookReader.php rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{ => IO/Extractor}/WorkbookSheet.php (96%) rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{ => IO/Inference}/CellTypeNarrower.php (94%) create mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Inference/ExcelSniffer.php rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{ => IO/Inference}/WorkbookSheetSample.php (75%) rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{ => IO/Loader}/CellStyler.php (93%) rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{ => IO/Loader}/ExcelEncoder.php (91%) create mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Loader/ExcelLoader.php create mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/IO/Loader/ExcelWriter.php rename src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/{ => IO/Loader}/WorkbookManager.php (99%) delete mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/WorkbookReader.php delete mode 100644 src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/WorkbookSampler.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Context/DescribableFixtures.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Context/ExcelFileFormat.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelExtractorTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelExtractorTypedColumnsTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelFormatDetectorTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelLoaderTest.php rename src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/{ => IO/Extractor}/ExcelExtractorBatchContractTest.php (85%) create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelExtractorTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelExtractorTypedColumnsTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelFormatDetectorTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/ExcelSplitReadTest.php rename src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/{ => IO/Extractor}/Sheet/OpenSheetTest.php (79%) create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/Sheet/SheetCellsTest.php rename src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/{ => IO/Extractor}/Sheet/XlsxSheetRowsTest.php (98%) create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Extractor/WorkbookReaderTest.php rename src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/{ => IO/Extractor}/WorkbookSheetTest.php (96%) create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/FileFormatContractTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Inference/ExcelSnifferTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/IO/Loader/ExcelLoaderTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/Sheet/SheetCellsTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/WorkbookReaderTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/WorkbookSamplerTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/DescribableRoundTripTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/ExcelExtractorTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/ExcelFileBatchesTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/ExcelFormatDetectorTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/ExcelHeaderTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/ExcelReadOptionsTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/Functions/AllFunctionsContractTest.php rename src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/{Function => Functions}/IsValidExcelSheetNameTest.php (95%) rename src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/{Sheet => Functions}/SheetNameAssertionTest.php (90%) create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelExtractorTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelFileBatchesTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelFormatDetectorTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/ExcelReadOptionsTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/MetadataColumnsExtractorTest.php create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Extractor/Sheet/SheetCellsTest.php rename src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/{ => IO/Inference}/CellTypeNarrowerTest.php (93%) rename src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/{ => IO/Inference}/WorkbookSheetSampleTest.php (78%) rename src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/{ => IO/Loader}/ExcelEncoderTest.php (98%) create mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/IO/Loader/ExcelLoaderTest.php delete mode 100644 src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Unit/Sheet/SheetCellsTest.php delete mode 100644 src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/Columns.php delete mode 100644 src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/GoogleSheetExtractor.php delete mode 100644 src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/GoogleSheetSampler.php create mode 100644 src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/Columns.php create mode 100644 src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Extractor/GoogleSheetExtractor.php rename src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/{ => IO/Extractor}/GoogleSheetReadOptions.php (93%) rename src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/{ => IO/Extractor}/GoogleSheetReader.php (96%) rename src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/{ => IO/Extractor}/SheetRange.php (93%) rename src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/{ => IO/Inference}/GoogleSheetSample.php (93%) create mode 100644 src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/IO/Inference/GoogleSheetSniffer.php rename src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/{ => Context}/GoogleSheetsContext.php (94%) delete mode 100644 src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Integration/GoogleSheetExtractorTest.php create mode 100644 src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Integration/IO/Extractor/GoogleSheetExtractorTest.php rename src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Integration/{ => IO/Extractor}/GoogleSheetExtractorTypedColumnsTest.php (75%) delete mode 100644 src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/ColumnsTest.php delete mode 100644 src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/GoogleSheetExtractorTest.php delete mode 100644 src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/GoogleSheetSamplerTest.php create mode 100644 src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/ColumnsTest.php rename src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/{ => IO/Extractor}/GoogleSheetExtractorBatchContractTest.php (88%) create mode 100644 src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/GoogleSheetExtractorTest.php rename src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/{ => IO/Extractor}/GoogleSheetReadOptionsTest.php (94%) rename src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/{ => IO/Extractor}/GoogleSheetReaderTest.php (98%) create mode 100644 src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Extractor/MetadataColumnsExtractorTest.php rename src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/{ => IO/Extractor}/SheetRangeTest.php (92%) rename src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/{ => IO/Inference}/GoogleSheetSampleTest.php (79%) create mode 100644 src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/IO/Inference/GoogleSheetSnifferTest.php rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/ContentTypeDetector.php (96%) rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/HttpDecoder.php (97%) rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/HttpExchange.php (85%) rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{DynamicExtractor => IO/Extractor}/NextRequestFactory.php (82%) rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/Pagination/DecodedResponse.php (93%) create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/InjectInto.php rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/Pagination/LinkHeaderParser.php (96%) rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/Pagination/PageState.php (85%) rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/Pagination/PaginationStrategy.php (84%) rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/Pagination/Paginator.php (91%) rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/Pagination/RelativeUriResolver.php (98%) rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/Pagination/RequestComparator.php (93%) rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/Pagination/RequestOption.php (96%) create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/ResultCounter.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen/AllOf.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen/AnyOf.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen/EmptyPath.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen/EmptyResponseBody.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen/FlagFalse.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen/FlagTrue.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen/MaxPages.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen/MaxResults.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen/NeverStop.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen/PathMissing.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/StopWhen/TotalReached.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/Strategy/CursorFromBody.php rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/Pagination/Strategy/LastRecordCursor.php (79%) create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/Strategy/LinkHeader.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/Strategy/NextUrlFromBody.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/Pagination/Strategy/OffsetLimit.php rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/Pagination/Strategy/PageNumber.php (82%) create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/PsrHttpClientDynamicExtractor.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/PsrHttpClientPaginatedExtractor.php create mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/IO/Extractor/PsrHttpClientStaticExtractor.php rename src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/{ => IO/Extractor}/ResponseType.php (77%) delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/InjectInto.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/ResultCounter.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen/AllOf.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen/AnyOf.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen/EmptyPath.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen/EmptyResponseBody.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen/FlagFalse.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen/FlagTrue.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen/MaxPages.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen/MaxResults.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen/NeverStop.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen/PathMissing.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/StopWhen/TotalReached.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/Strategy/CursorFromBody.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/Strategy/LinkHeader.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/Strategy/NextUrlFromBody.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/Pagination/Strategy/OffsetLimit.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/PsrHttpClientDynamicExtractor.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/PsrHttpClientPaginatedExtractor.php delete mode 100644 src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/PsrHttpClientStaticExtractor.php delete mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/HTTP/Tests/Double/FixedStrategy.php delete mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/HTTP/Tests/Unit/Pagination/StopWhen/AllOfTest.php delete mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/HTTP/Tests/Unit/Pagination/StopWhen/AnyOfTest.php delete mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/HTTP/Tests/Unit/Pagination/StopWhen/EmptyResponseBodyTest.php delete mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/HTTP/Tests/Unit/Pagination/StopWhen/MaxPagesTest.php delete mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/HTTP/Tests/Unit/Pagination/StopWhen/MaxResultsTest.php delete mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/HTTP/Tests/Unit/Pagination/StopWhen/NeverStopTest.php create mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/Http/Tests/Double/FixedStrategy.php rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP => Http}/Tests/Double/NumberedRequestFactory.php (85%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP => Http}/Tests/Fixtures/flow-php.json (100%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP => Http}/Tests/Fixtures/norberttech.json (100%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP => Http}/Tests/Fixtures/tomaszhanc.json (100%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Integration => Http/Tests/Integration/IO/Extractor}/PsrHttpClientDynamicExtractorTest.php (84%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Integration => Http/Tests/Integration/IO/Extractor}/PsrHttpClientExtractorBatchContractTest.php (77%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Integration => Http/Tests/Integration/IO/Extractor}/PsrHttpClientPaginatedExtractorTest.php (93%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Integration => Http/Tests/Integration/IO/Extractor}/PsrHttpClientStaticExtractorTest.php (84%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP => Http}/Tests/Mother/PaginationMother.php (91%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/ContentTypeDetectorTest.php (92%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/HttpDecoderTest.php (95%) create mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/Http/Tests/Unit/IO/Extractor/MetadataColumnsExtractorTest.php rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/LinkHeaderParserTest.php (92%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/PageStateTest.php (76%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/PaginatorTest.php (82%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/RelativeUriResolverTest.php (91%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/RequestComparatorTest.php (91%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/RequestOptionTest.php (94%) create mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/Http/Tests/Unit/IO/Extractor/Pagination/StopWhen/AllOfTest.php create mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/Http/Tests/Unit/IO/Extractor/Pagination/StopWhen/AnyOfTest.php rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/StopWhen/EmptyPathTest.php (75%) create mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/Http/Tests/Unit/IO/Extractor/Pagination/StopWhen/EmptyResponseBodyTest.php rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/StopWhen/FlagFalseTest.php (75%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/StopWhen/FlagTrueTest.php (75%) create mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/Http/Tests/Unit/IO/Extractor/Pagination/StopWhen/MaxPagesTest.php create mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/Http/Tests/Unit/IO/Extractor/Pagination/StopWhen/MaxResultsTest.php create mode 100644 src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/Http/Tests/Unit/IO/Extractor/Pagination/StopWhen/NeverStopTest.php rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/StopWhen/PathMissingTest.php (79%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/StopWhen/TotalReachedTest.php (78%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/Strategy/CursorFromBodyTest.php (80%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/Strategy/LastRecordCursorTest.php (80%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/Strategy/LinkHeaderTest.php (81%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/Strategy/NextUrlFromBodyTest.php (79%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/Strategy/OffsetLimitTest.php (80%) rename src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/{HTTP/Tests/Unit => Http/Tests/Unit/IO/Extractor}/Pagination/Strategy/PageNumberTest.php (83%) delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/AdaptiveJsonEncoder.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/AdaptiveJsonOpenSource.php rename src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/{JsonSchema => Foundation}/Exception/CircularReferenceException.php (80%) create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/Foundation/Exception/UnresolvableReferenceException.php rename src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/{JsonSchema => Foundation}/Exception/UnsupportedKeywordException.php (78%) rename src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/{JSONMachine => IO/Extractor}/JsonArrayElements.php (99%) create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonExtractor.php create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonFileBatches.php create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonFileReader.php create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonLineResync.php create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/JsonOpenSource.php rename src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/{JSONMachine => IO/Extractor}/JsonReadBytes.php (89%) create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Extractor/PhpJsonOpenSource.php create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Inference/JsonFileSample.php create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Inference/JsonSniffer.php rename src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/{ => IO}/JsonFraming.php (96%) rename src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/{ => IO/Loader}/JsonEncoder.php (76%) create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Loader/JsonFileWriter.php create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Loader/JsonLoader.php rename src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/{ => IO/Loader}/PhpJsonEncoder.php (82%) create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Schema/JsonSchema/ReferenceResolver.php create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/IO/Schema/SchemaConverter.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonFileBatches.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonFileReader.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonFileSample.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonFormat.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonSampledFiles.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonSampledRows.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonSamples.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JsonLoader.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JsonOpenSink.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JsonOpenSource.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JsonSchema/Exception/UnresolvableReferenceException.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JsonSchema/ReferenceResolver.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JsonSchema/ResolvedReference.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/PhpJsonOpenSource.php delete mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/SchemaConverter.php rename src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/{ => Values/Schema}/JsonSchema/JsonSchemaMetadata.php (94%) create mode 100644 src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/Values/Schema/JsonSchema/ResolvedReference.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Context/DescribableFixtures.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Context/JsonFileFormat.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Context/JsonLinesFileFormat.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/AdaptiveJsonOpenSourceTest.php rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/{JSONMachine => IO/Extractor}/JsonExtractorBatchContractTest.php (88%) rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/{JSONMachine => IO/Extractor}/JsonExtractorTest.php (80%) create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonExtractorTypedColumnsTest.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonFileBatchesTest.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonFileReaderTest.php rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/{JSONMachine => IO/Extractor}/JsonLinesExtractorTest.php (80%) rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/{JSONMachine => IO/Extractor}/JsonLinesExtractorTypedColumnsTest.php (77%) create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/JsonSplitReadTest.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Extractor/PhpJsonOpenSourceTest.php rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/{ => IO/Extractor}/RustJsonOpenSourceTest.php (85%) create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/FileFormatContractTest.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Inference/JsonFileSampleTest.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/IO/Inference/JsonSnifferTest.php rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/{ => IO/Schema}/JsonSchemaConversionTest.php (88%) delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTypedColumnsTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonFileReaderTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonFileSampleTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonSamplesTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/PhpJsonOpenSourceTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/AdaptiveJsonEncoderTest.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/DescribableRoundTripTest.php rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/{JSONMachine => IO/Extractor}/JsonArrayElementsTest.php (96%) create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonFileBatchesTest.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/JsonLineResyncTest.php rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/{JSONMachine => IO/Extractor}/JsonReadBytesTest.php (80%) create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Extractor/MetadataColumnsExtractorTest.php rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/{ => IO}/JsonFramingTest.php (89%) create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Loader/JsonFileWriterTest.php rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/{ => IO/Loader}/JsonLinesLoaderTest.php (85%) create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Loader/JsonLoaderTest.php rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/{ => IO/Loader}/PhpJsonEncoderTest.php (98%) rename src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/{ => IO/Loader}/RustJsonEncoderTest.php (94%) create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Schema/JsonSchema/ReferenceResolverTest.php create mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/IO/Schema/SchemaConverterTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/JSONMachine/JsonSampledFilesTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/JSONMachine/JsonSampledRowsTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/JsonFileBatchesTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/JsonLoaderTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/JsonOpenSinkTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/JsonSchema/ReferenceResolverTest.php delete mode 100644 src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Unit/SchemaConverterTest.php rename src/adapter/etl-adapter-logger/src/Flow/ETL/Adapter/Logger/{Logger => IO/Loader}/DumpLogger.php (94%) create mode 100644 src/adapter/etl-adapter-logger/src/Flow/ETL/Adapter/Logger/IO/Loader/PsrLoggerLoader.php delete mode 100644 src/adapter/etl-adapter-logger/src/Flow/ETL/Adapter/Logger/PsrLoggerLoader.php create mode 100644 src/adapter/etl-adapter-logger/tests/Flow/ETL/Adapter/Logger/Tests/Context/DescribableFixtures.php create mode 100644 src/adapter/etl-adapter-logger/tests/Flow/ETL/Adapter/Logger/Tests/Unit/DescribableRoundTripTest.php rename src/adapter/etl-adapter-logger/tests/Flow/ETL/Adapter/Logger/Tests/Unit/{Logger => IO/Loader}/DumpLoggerTest.php (91%) rename src/adapter/etl-adapter-logger/tests/Flow/ETL/Adapter/Logger/Tests/Unit/{ => IO/Loader}/PsrLoggerLoaderTest.php (88%) delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/AdaptiveParquetOpenSink.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/AdaptiveParquetOpenSource.php create mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/ConfiguredParquetEngine.php create mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Extractor/ParquetExtractor.php create mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Extractor/ParquetFileBatches.php rename src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/{ => IO/Extractor}/ParquetOpenSource.php (80%) create mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Extractor/ParquetSourceFile.php rename src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/{ => IO/Extractor}/ParquetSourceFileOpener.php (88%) create mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Extractor/PhpParquetOpenSource.php rename src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/{ => IO/Extractor}/Pushdown/FilterTranslator.php (75%) rename src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/{ => IO/Extractor}/Pushdown/TranslatedConjunct.php (92%) rename src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/{ValueConverter => IO/Loader}/ElementsValueConverter.php (92%) rename src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/{ValueConverter => IO/Loader}/JsonValueConverter.php (85%) rename src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/{ => IO/Loader}/ParquetEncoder.php (86%) create mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Loader/ParquetFileWriter.php create mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Loader/ParquetFileWriterFactory.php create mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Loader/ParquetLoader.php create mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Loader/ParquetSchemaConformance.php rename src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/{ValueConverter => IO/Loader}/StructValueConverter.php (93%) rename src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/{ValueConverter => IO/Loader}/UuidValueConverter.php (85%) create mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/IO/Loader/ValueConverter.php rename src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/{ValueConverter => IO/Loader}/ValueConverters.php (96%) delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetFileBatches.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetFileSink.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetFileSinks.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetLoader.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetOpenSink.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetSchemaConformance.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetSchemaPassFile.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetSourceFile.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetStoredType.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/PhpParquetOpenSink.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/PhpParquetOpenSource.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php delete mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ValueConverter/ValueConverter.php create mode 100644 src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Values/Schema/SchemaConverter.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/DescribableFixtures.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/ParquetFileFormat.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Double/ReadPlanRecordingParquetEngine.php rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/EdgeCases/interval.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/EdgeCases/unsigned.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/Pagination/01_1000.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/Pagination/02_500.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/Pagination/03_350.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/Pagination/04_2000.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/Pagination/05_15.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/Pagination/partitioned/date=2024-01-01/01_1000.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/Pagination/partitioned/date=2024-02-01/02_500.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/Pagination/partitioned/date=2024-02-01/03_350.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/Pagination/partitioned/date=2024-12-01/04_2000.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/Pagination/partitioned/date=2025-01-01/05_15.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/orders_1k.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/{Integration => }/Fixtures/xml-node-before-utf8.parquet (100%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/{ => IO/Extractor}/ParquetExtractorBatchContractTest.php (82%) create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/Extractor/ParquetExtractorFooterTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/Extractor/ParquetExtractorTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/Extractor/ParquetSplitReadTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/IO/FileFormatContractTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorFooterTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/AdaptiveParquetOpenSinkTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/AdaptiveParquetOpenSourceTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/DescribableRoundTripTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/IO/ConfiguredParquetEngineTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/IO/Extractor/MetadataColumnsExtractorTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/IO/Extractor/ParquetExtractorTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/IO/Extractor/ParquetFileBatchesTest.php rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ => IO/Extractor}/ParquetSourceFileOpenerTest.php (88%) create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/IO/Extractor/ParquetSourceFileTest.php rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ => IO/Extractor}/PhpParquetOpenSourceTest.php (96%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ => IO/Extractor}/Pushdown/FilterTranslatorTest.php (97%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ => IO/Extractor}/Pushdown/TranslatedConjunctTest.php (91%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ => IO/Extractor}/RustParquetOpenSourceTest.php (95%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ValueConverter => IO/Loader}/ElementsValueConverterTest.php (80%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ValueConverter => IO/Loader}/JsonValueConverterTest.php (81%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ => IO/Loader}/ParquetEncoderTest.php (98%) create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/IO/Loader/ParquetFileWriterFactoryTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/IO/Loader/ParquetFileWriterTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/IO/Loader/ParquetLoaderTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/IO/Loader/ParquetSchemaConformanceTest.php create mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/IO/Loader/RustParquetFileWriterTest.php rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ValueConverter => IO/Loader}/StructValueConverterTest.php (79%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ValueConverter => IO/Loader}/UuidValueConverterTest.php (85%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ValueConverter => IO/Loader}/ValueConvertersTest.php (87%) delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetExtractorTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileSinkTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileSinksTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetLoaderTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSchemaConformanceTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetStoredTypeTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSinkTest.php delete mode 100644 src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSinkTest.php rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ => Values/Schema}/FlowToParquetSchemaTest.php (88%) rename src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/{ => Values/Schema}/ParquetToFlowSchemaTest.php (89%) delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/Exception/RuntimeException.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/ExplainedRows.php rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => Foundation}/Exception/TypeMappingException.php (90%) create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/IO/Extractor/ExplainedRows.php create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/IO/Extractor/Pagination/Key.php create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/IO/Extractor/Pagination/KeySet.php create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/IO/Extractor/Pagination/KeySetPage.php create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/IO/Extractor/Pagination/KeyValues.php create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/IO/Extractor/Pagination/Order.php rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => IO/Extractor}/PostgreSqlCursorExtractor.php (80%) create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/IO/Extractor/PostgreSqlKeySetExtractor.php create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/IO/Extractor/PostgreSqlLimitOffsetExtractor.php rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => IO/Extractor}/ReadQuery.php (95%) rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => IO/Extractor}/ResultSchema.php (84%) rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ValueConverter => IO/Loader}/EnumConverter.php (93%) rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ValueConverter => IO/Loader}/HTMLConverter.php (95%) rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => IO/Loader}/LoaderOptions/DeleteOptions.php (84%) rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => IO/Loader}/LoaderOptions/InsertOptions.php (97%) rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => IO/Loader}/LoaderOptions/UpdateOptions.php (84%) create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/IO/Loader/Operation.php rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => IO/Loader}/PostgreSqlEncoder.php (86%) rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => IO/Loader}/PostgreSqlLoader.php (76%) rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => IO/Loader}/PostgreSqlTransaction.php (95%) create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/IO/Loader/QueryBuilder/DeleteQueryBuilder.php rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => IO/Loader}/QueryBuilder/InsertQueryBuilder.php (86%) create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/IO/Loader/QueryBuilder/UpdateQueryBuilder.php rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ValueConverter => IO/Loader}/XMLConverter.php (94%) delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/Operation.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/Pagination/Key.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/Pagination/KeySet.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/Pagination/KeySetPage.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/Pagination/KeyValues.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/Pagination/Order.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlKeySetExtractor.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlLimitOffsetExtractor.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/DeleteQueryBuilder.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/UpdateQueryBuilder.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/Schema/SortingStrategy/TypeStrategy.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/SchemaConverter.php delete mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/TypesMap.php rename src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/{ => Values/Schema}/PostgreSqlMetadata.php (94%) create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/Values/Schema/SchemaConverter.php create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/Values/Schema/SortingStrategy/TypeStrategy.php create mode 100644 src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/Values/Schema/TypesMap.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Context/DescribableFixtures.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/IO/Extractor/PostgreSqlCursorExtractorTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/IO/Extractor/PostgreSqlExtractorBatchContractTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/IO/Extractor/PostgreSqlKeySetExtractorTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/IO/Extractor/PostgreSqlLimitOffsetExtractorTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/IO/Loader/PostgreSqlLoaderAllTypesTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/IO/Loader/PostgreSqlLoaderTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlCursorExtractorIntegrationTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlExtractorBatchContractIntegrationTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlKeySetExtractorIntegrationTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlLimitOffsetExtractorIntegrationTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlLoaderAllTypesIntegrationTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlLoaderIntegrationTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlTransactionSinkTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlTransactionalTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/SchemaConverterIntegrationTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/Values/Schema/SchemaConverterTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/DescribableRoundTripTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/ExplainedRowsTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/IO/Extractor/ExplainedRowsTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/IO/Extractor/MetadataColumnsExtractorTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/IO/Extractor/Pagination/KeySetPageTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/IO/Extractor/Pagination/KeyValuesTest.php rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ => IO/Extractor/Pagination}/PaginationTest.php (87%) create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/IO/Extractor/PostgreSqlCursorExtractorTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/IO/Extractor/PostgreSqlKeySetExtractorTest.php create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/IO/Extractor/PostgreSqlLimitOffsetExtractorTest.php rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ => IO/Extractor}/ReadQueryTest.php (97%) rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ => IO/Extractor}/ResultSchemaTest.php (89%) rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ValueConverter => IO/Loader}/EnumConverterTest.php (93%) rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ValueConverter => IO/Loader}/HTMLConverterTest.php (94%) rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ => IO/Loader}/PostgreSqlEncoderTest.php (95%) create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/IO/Loader/PostgreSqlLoaderTest.php rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ => IO/Loader}/PostgreSqlTransactionTest.php (95%) rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ => IO/Loader}/QueryBuilder/DeleteQueryBuilderTest.php (89%) rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ => IO/Loader}/QueryBuilder/InsertQueryBuilderTest.php (92%) rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ => IO/Loader}/QueryBuilder/UpdateQueryBuilderTest.php (91%) rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ValueConverter => IO/Loader}/XMLConverterTest.php (93%) delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/KeySetPageTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/KeyValuesTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLoaderTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/SchemaConverterTest.php delete mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/TypesMapTest.php rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ => Values/Schema}/PostgreSqlMetadataTest.php (96%) create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/Values/Schema/SchemaConverterTest.php rename src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/{ => Values}/Schema/SortingStrategy/TypeStrategyTest.php (86%) create mode 100644 src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/Values/Schema/TypesMapTest.php create mode 100644 src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/IO/Loader/Operation.php rename src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/{ => IO/Loader}/SealEncoder.php (91%) create mode 100644 src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/IO/Loader/SealLoader.php delete mode 100644 src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/Operation.php delete mode 100644 src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/SchemaConverter.php delete mode 100644 src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/SealLoader.php create mode 100644 src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/Values/Schema/SchemaConverter.php rename src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/{ => Values/Schema}/SealMetadata.php (93%) create mode 100644 src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Context/DescribableFixtures.php create mode 100644 src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Integration/IO/Loader/SealLoaderTest.php delete mode 100644 src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Integration/SealLoaderTest.php create mode 100644 src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/DescribableRoundTripTest.php rename src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/{ => IO/Loader}/SealEncoderTest.php (98%) create mode 100644 src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/IO/Loader/SealLoaderTest.php delete mode 100644 src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/SchemaConverterTest.php delete mode 100644 src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/SealLoaderTest.php create mode 100644 src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/Values/Schema/SchemaConverterTest.php create mode 100644 src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/IO/Extractor/TextExtractor.php create mode 100644 src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/IO/Extractor/TextFileBatches.php rename src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/{ => IO/Loader}/TextEncoder.php (88%) create mode 100644 src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/IO/Loader/TextFileWriter.php create mode 100644 src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/IO/Loader/TextLoader.php delete mode 100644 src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php delete mode 100644 src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextFileBatches.php delete mode 100644 src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextLoader.php delete mode 100644 src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextOpenSink.php create mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Context/DescribableFixtures.php create mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Context/TextFileFormat.php rename src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/{ => IO/Extractor}/TextExtractorBatchContractTest.php (80%) create mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Extractor/TextExtractorTest.php create mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Extractor/TextExtractorTypedColumnsTest.php create mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Extractor/TextSplitReadTest.php create mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/FileFormatContractTest.php create mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/IO/Loader/TextLoaderTest.php delete mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php delete mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTypedColumnsTest.php delete mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextLoaderTest.php create mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/DescribableRoundTripTest.php create mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/IO/Extractor/MetadataColumnsExtractorTest.php create mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/IO/Extractor/TextFileBatchesTest.php rename src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/{ => IO/Loader}/TextEncoderTest.php (94%) create mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/IO/Loader/TextFileWriterTest.php delete mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextFileBatchesTest.php delete mode 100644 src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextOpenSinkTest.php delete mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/Abstraction/XMLNodeType.php rename src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/{ => IO/Extractor}/RootlessDocument.php (98%) create mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/IO/Extractor/XMLFileBatches.php rename src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/{ => IO/Extractor}/XMLNodeBatches.php (92%) rename src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/{ => IO/Extractor}/XMLNodeCursor.php (97%) rename src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/{ => IO/Extractor}/XMLNodes.php (95%) create mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/IO/Extractor/XMLParserExtractor.php rename src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/{Abstraction => IO/Loader}/XMLAttribute.php (76%) rename src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/{ => IO/Loader}/XMLEncoder.php (83%) create mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/IO/Loader/XMLFileWriter.php create mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/IO/Loader/XMLLoader.php rename src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/{Abstraction => IO/Loader}/XMLNode.php (95%) create mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/IO/Loader/XMLNodeType.php rename src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/{ => IO/Loader}/XMLValueNodes.php (95%) rename src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/{ => IO/Loader}/XMLWriter.php (86%) create mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/IO/Loader/XMLWriter/DOMDocumentWriter.php rename src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/{ => IO/Loader}/XMLWriter/StringXMLWriter.php (90%) delete mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/Loader/XMLLoader.php delete mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLFileBatches.php delete mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLOpenSink.php delete mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php delete mode 100644 src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter/DOMDocumentWriter.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Context/DescribableFixtures.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Context/XMLFileFormat.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLExtractorBatchContractTest.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLExtractorTypedColumnsTest.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLParserExtractorTest.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Extractor/XMLParserSplitReadTest.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/FileFormatContractTest.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/IO/Loader/XMLLoaderTest.php delete mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/Loader/XMLLoaderTest.php delete mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLExtractorBatchContractTest.php delete mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLExtractorTypedColumnsTest.php delete mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/DescribableRoundTripTest.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/IO/Extractor/MetadataColumnsExtractorTest.php rename src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/{ => IO/Extractor}/RootlessDocumentTest.php (96%) create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/IO/Extractor/XMLFileBatchesTest.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/IO/Extractor/XMLNodeBatchesTest.php rename src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/{ => IO/Extractor}/XMLNodeCursorTest.php (96%) rename src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/{ => IO/Extractor}/XMLNodesTest.php (97%) rename src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/{ => IO/Loader}/XMLEncoderTest.php (94%) create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/IO/Loader/XMLFileWriterTest.php create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/IO/Loader/XMLLoaderTest.php rename src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/{Abstraction => IO/Loader}/XMLNodeTest.php (83%) rename src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/{ => IO/Loader}/XMLValueNodesTest.php (95%) create mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/IO/Loader/XMLWriter/DOMDocumentWriterTest.php rename src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/{ => IO/Loader}/XMLWriter/StringXMLWriterTest.php (87%) delete mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLFileBatchesTest.php delete mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLLoaderTest.php delete mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLNodeBatchesTest.php delete mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLOpenSinkTest.php delete mode 100644 src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLWriter/DOMDocumentWriterTest.php create mode 100644 src/bridge/filesystem/async-aws/src/Flow/Filesystem/Bridge/AsyncAWS/ConfiguredS3Client.php create mode 100644 src/bridge/filesystem/async-aws/src/Flow/Filesystem/Bridge/AsyncAWS/LiveS3Client.php create mode 100644 src/bridge/filesystem/async-aws/src/Flow/Filesystem/Bridge/AsyncAWS/S3ClientSource.php create mode 100644 src/bridge/filesystem/async-aws/tests/Flow/Filesystem/Bridge/AsyncAWS/Tests/Unit/ConfiguredS3ClientTest.php create mode 100644 src/bridge/filesystem/async-aws/tests/Flow/Filesystem/Bridge/AsyncAWS/Tests/Unit/LiveS3ClientTest.php create mode 100644 src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/BlobServiceSource.php create mode 100644 src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/ConfiguredBlobService.php create mode 100644 src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/LiveBlobService.php create mode 100644 src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Unit/ConfiguredBlobServiceTest.php create mode 100644 src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Unit/LiveBlobServiceTest.php create mode 100644 src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Unit/OptionsTest.php create mode 100644 src/bridge/filesystem/sftp/src/Flow/Filesystem/Bridge/SFTP/ConfiguredSFTPClient.php create mode 100644 src/bridge/filesystem/sftp/src/Flow/Filesystem/Bridge/SFTP/LiveSFTPClient.php create mode 100644 src/bridge/filesystem/sftp/src/Flow/Filesystem/Bridge/SFTP/SFTPClientSource.php create mode 100644 src/bridge/filesystem/sftp/src/Flow/Filesystem/Bridge/SFTP/SFTPCredential.php create mode 100644 src/bridge/filesystem/sftp/src/Flow/Filesystem/Bridge/SFTP/SFTPPassword.php create mode 100644 src/bridge/filesystem/sftp/src/Flow/Filesystem/Bridge/SFTP/SFTPPrivateKey.php create mode 100644 src/bridge/filesystem/sftp/tests/Flow/Filesystem/Bridge/SFTP/Tests/Unit/ConfiguredSFTPClientTest.php create mode 100644 src/bridge/filesystem/sftp/tests/Flow/Filesystem/Bridge/SFTP/Tests/Unit/LiveSFTPClientTest.php create mode 100644 src/bridge/filesystem/sftp/tests/Flow/Filesystem/Bridge/SFTP/Tests/Unit/SFTPPasswordTest.php create mode 100644 src/bridge/filesystem/sftp/tests/Flow/Filesystem/Bridge/SFTP/Tests/Unit/SFTPPrivateKeyTest.php create mode 100644 src/bridge/symfony/http-foundation/tests/Flow/Bridge/Symfony/HttpFoundation/Tests/Context/ResponseContext.php create mode 100644 src/bridge/symfony/http-foundation/tests/Flow/Bridge/Symfony/HttpFoundation/Tests/Unit/DataStreamTest.php delete mode 100644 src/cli/src/Flow/CLI/Command/Traits/CSVOptions.php delete mode 100644 src/cli/src/Flow/CLI/Command/Traits/ConfigOptions.php delete mode 100644 src/cli/src/Flow/CLI/Command/Traits/DBOptions.php delete mode 100644 src/cli/src/Flow/CLI/Command/Traits/ExcelOptions.php delete mode 100644 src/cli/src/Flow/CLI/Command/Traits/JSONOptions.php delete mode 100644 src/cli/src/Flow/CLI/Command/Traits/ParquetOptions.php delete mode 100644 src/cli/src/Flow/CLI/Command/Traits/XMLOptions.php create mode 100644 src/cli/src/Flow/CLI/Options/CSVInputOptions.php create mode 100644 src/cli/src/Flow/CLI/Options/CSVOutputOptions.php create mode 100644 src/cli/src/Flow/CLI/Options/DbConnectionOptions.php create mode 100644 src/cli/src/Flow/CLI/Options/ExcelInputOptions.php create mode 100644 src/cli/src/Flow/CLI/Options/JSONInputOptions.php create mode 100644 src/cli/src/Flow/CLI/Options/JSONOutputOptions.php create mode 100644 src/cli/src/Flow/CLI/Options/ParquetInputOptions.php rename src/cli/src/Flow/CLI/{Command/Traits => Options}/SchemaInferenceOptions.php (89%) rename src/cli/src/Flow/CLI/{Command/Traits => Options}/StatisticsOptions.php (86%) create mode 100644 src/cli/src/Flow/CLI/Options/XMLInputOptions.php create mode 100644 src/cli/src/Flow/CLI/Options/XMLOutputOptions.php create mode 100644 src/cli/tests/Flow/CLI/Tests/Context/DescribableFixtures.php rename src/{core/etl/tests/Flow/ETL => cli/tests/Flow/CLI}/Tests/Fixtures/empty.php (100%) rename src/{core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Fixtures/FileListExtractor/file_01.txt => cli/tests/Flow/CLI/Tests/Fixtures/empty.txt} (100%) rename src/{core/etl/tests/Flow/ETL => cli/tests/Flow/CLI}/Tests/Fixtures/with-dataframe.php (100%) rename src/{core/etl/tests/Flow/ETL => cli/tests/Flow/CLI}/Tests/Fixtures/without-dataframe.php (100%) create mode 100644 src/cli/tests/Flow/CLI/Tests/Unit/DescribableRoundTripTest.php rename src/{core/etl/tests/Flow/ETL => cli/tests/Flow/CLI}/Tests/Unit/PipelineFactoryTest.php (90%) delete mode 100644 src/core/etl/src/Flow/Calculator/Calculator.php delete mode 100644 src/core/etl/src/Flow/Calculator/Exception/Exception.php delete mode 100644 src/core/etl/src/Flow/Calculator/RunningSum.php create mode 100644 src/core/etl/src/Flow/ETL/API/Branch.php create mode 100644 src/core/etl/src/Flow/ETL/API/Config/Bucketing/BucketingConfigBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/API/Config/Cache/CacheConfigBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/API/Config/ConfigBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/API/Config/Join/HashJoinBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/API/Config/Sort/ExternalSortBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/API/Config/Sort/MemorySortBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/API/DataFrame.php create mode 100644 src/core/etl/src/Flow/ETL/API/FilteredBranch.php create mode 100644 src/core/etl/src/Flow/ETL/API/GroupedDataFrame.php create mode 100644 src/core/etl/src/Flow/ETL/API/Transactional.php rename src/core/etl/src/Flow/ETL/{ => API}/Transformation.php (84%) create mode 100644 src/core/etl/src/Flow/ETL/API/Transformation/AddRowIndex.php create mode 100644 src/core/etl/src/Flow/ETL/API/Transformation/BatchBy.php create mode 100644 src/core/etl/src/Flow/ETL/API/Transformation/BatchSize.php create mode 100644 src/core/etl/src/Flow/ETL/API/Transformation/Drop.php create mode 100644 src/core/etl/src/Flow/ETL/API/Transformation/Limit.php create mode 100644 src/core/etl/src/Flow/ETL/API/Transformation/MaskColumns.php create mode 100644 src/core/etl/src/Flow/ETL/API/Transformation/Select.php rename src/core/etl/src/Flow/ETL/{ => API}/Transformations.php (96%) create mode 100644 src/core/etl/src/Flow/ETL/API/TransformedBranch.php delete mode 100644 src/core/etl/src/Flow/ETL/BoundStep.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/BucketsStorage.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/HashBucketing.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/Hasher.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/KeyGrouping.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/KeyValues.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/NativeHasher.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/ResidentBucketsStorage.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/SingleBucketHasher.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/Storage/FilesystemBuckets.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/Storage/MemoryBuckets.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/Storage/PSRCacheBuckets.php delete mode 100644 src/core/etl/src/Flow/ETL/Bucketing/Storage/SpillingBuckets.php delete mode 100644 src/core/etl/src/Flow/ETL/Cache.php delete mode 100644 src/core/etl/src/Flow/ETL/Cache/CacheIndex.php delete mode 100644 src/core/etl/src/Flow/ETL/Cache/Implementation/ApcuCache.php delete mode 100644 src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php delete mode 100644 src/core/etl/src/Flow/ETL/Cache/Implementation/InMemoryCache.php delete mode 100644 src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php delete mode 100644 src/core/etl/src/Flow/ETL/Cache/Implementation/TraceableCache.php delete mode 100644 src/core/etl/src/Flow/ETL/Column/AdaptiveBackend.php delete mode 100644 src/core/etl/src/Flow/ETL/Column/Backend.php delete mode 100644 src/core/etl/src/Flow/ETL/Column/Column.php delete mode 100644 src/core/etl/src/Flow/ETL/Column/Layout/Layout.php delete mode 100644 src/core/etl/src/Flow/ETL/Column/Layout/LayoutFor.php delete mode 100644 src/core/etl/src/Flow/ETL/Column/Php/CastingColumnBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Column/PhpBackend.php delete mode 100644 src/core/etl/src/Flow/ETL/Column/Physical/DaysFromCivil.php delete mode 100644 src/core/etl/src/Flow/ETL/Column/Physical/PhysicalBuilderFor.php delete mode 100644 src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php delete mode 100644 src/core/etl/src/Flow/ETL/Column/Physical/PhysicalKind.php delete mode 100644 src/core/etl/src/Flow/ETL/Config.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Bucketing/BucketingConfig.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Bucketing/BucketingConfigBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Cache/CacheConfig.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Cache/CacheConfigBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Grouping/GroupByAlgorithmBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Grouping/HashGroupByBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Grouping/HashGroupByConfig.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Join/HashJoinBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Join/HashJoinConfig.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Join/JoinAlgorithmBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/MemoryLimit.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Repartition/HashRepartitionBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Repartition/HashRepartitionConfig.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Repartition/RepartitionAlgorithmBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Sort/ExternalSortBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Sort/ExternalSortConfig.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Sort/MemorySortBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Sort/MemorySortConfig.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Sort/SortAlgorithmBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Telemetry/SpanStack.php delete mode 100644 src/core/etl/src/Flow/ETL/Config/Telemetry/TelemetryContext.php delete mode 100644 src/core/etl/src/Flow/ETL/Constraint/SortedByConstraint.php delete mode 100644 src/core/etl/src/Flow/ETL/Constraint/UniqueConstraint.php delete mode 100644 src/core/etl/src/Flow/ETL/Constraint/UniqueConstraint/InMemoryStorage.php delete mode 100644 src/core/etl/src/Flow/ETL/Constraint/UniqueConstraint/Storage.php delete mode 100644 src/core/etl/src/Flow/ETL/DataFrame.php delete mode 100644 src/core/etl/src/Flow/ETL/DataFrame/GroupedDataFrame.php delete mode 100644 src/core/etl/src/Flow/ETL/DataFrameFactory.php delete mode 100644 src/core/etl/src/Flow/ETL/Dataset/Memory/Configuration.php delete mode 100644 src/core/etl/src/Flow/ETL/Dataset/Statistics.php delete mode 100644 src/core/etl/src/Flow/ETL/Dataset/Statistics/Column.php delete mode 100644 src/core/etl/src/Flow/ETL/Dataset/Statistics/Columns.php delete mode 100644 src/core/etl/src/Flow/ETL/Dataset/Statistics/DistinctCounter.php delete mode 100644 src/core/etl/src/Flow/ETL/ErrorHandler.php delete mode 100644 src/core/etl/src/Flow/ETL/ErrorHandler/ExtractionAction.php delete mode 100644 src/core/etl/src/Flow/ETL/ErrorHandler/ExtractionError.php delete mode 100644 src/core/etl/src/Flow/ETL/ErrorHandler/IgnoreError.php delete mode 100644 src/core/etl/src/Flow/ETL/ErrorHandler/LoadingAction.php delete mode 100644 src/core/etl/src/Flow/ETL/ErrorHandler/LoadingError.php delete mode 100644 src/core/etl/src/Flow/ETL/ErrorHandler/SkipRows.php delete mode 100644 src/core/etl/src/Flow/ETL/ErrorHandler/ThrowError.php delete mode 100644 src/core/etl/src/Flow/ETL/ErrorHandler/TransformationAction.php delete mode 100644 src/core/etl/src/Flow/ETL/ErrorHandler/TransformationError.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/ColumnMismatchException.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/DuplicatedColumnsException.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/Exception.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/InferredSchemaException.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/InvalidArgumentException.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/InvalidLogicException.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/JoinException.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/KeyNotInCacheException.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/LimitReachedException.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/OffsetOverflow.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/RuntimeException.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/SchemaDefinitionNotUniqueException.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/SchemaValidationException.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/SinkFailure.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/TransactionRolledBack.php delete mode 100644 src/core/etl/src/Flow/ETL/Exception/TypeAssertionException.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/Described.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/PhysicalPlan.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/Pipeline.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/Raw.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/Segment.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/Segments.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/SinkFeed.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/SinkOffers.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/SinkRun.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/SourceRead.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/SourceRows.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/TransactionRollback.php delete mode 100644 src/core/etl/src/Flow/ETL/Executor/TransactionalSinks.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/ArrayExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/BatchByExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/BatchExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/BatchableExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/Batches.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/CacheExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/ChainExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/CollectingExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/DataFrameExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/DeclaresPartitionTypes.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/DerivedSchema.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/FileColumns.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/FileMetadataRow.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/FileReading.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/FileScan.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/FooterStatistics.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/InferredColumns.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/ListedFileRow.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/ListedFiles.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/ListingPartitions.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/ListingScan.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/MetadataColumns.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/PartitionColumns.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/PartitionTypes.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/PathFiltering.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/PathPartitionsRow.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/ReadWindow.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/Scanning.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/SelfDescribingFile.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/File/SourceFile.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/InfersSchema.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/Memory/InMemoryRows.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/Memory/InferredRows.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/MemoryExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/OverridingExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/Repeatability.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/RewindableExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/SequenceExtractor.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/SequenceGenerator/SequenceGenerator.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/Signal.php delete mode 100644 src/core/etl/src/Flow/ETL/Extractor/Statistics.php delete mode 100644 src/core/etl/src/Flow/ETL/Filesystem/SaveMode.php delete mode 100644 src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php delete mode 100644 src/core/etl/src/Flow/ETL/FlowContext.php delete mode 100644 src/core/etl/src/Flow/ETL/Formatter.php delete mode 100644 src/core/etl/src/Flow/ETL/Formatter/ASCII/ASCIIBody.php delete mode 100644 src/core/etl/src/Flow/ETL/Formatter/ASCII/ASCIIHeaders.php delete mode 100644 src/core/etl/src/Flow/ETL/Formatter/ASCII/ASCIITable.php delete mode 100644 src/core/etl/src/Flow/ETL/Formatter/ASCII/Body.php delete mode 100644 src/core/etl/src/Flow/ETL/Formatter/ASCII/Headers.php delete mode 100644 src/core/etl/src/Flow/ETL/Formatter/AsciiTableFormatter.php rename src/core/etl/src/Flow/{ => ETL/Foundation}/ArrayComparison/ArrayComparison.php (97%) rename src/core/etl/src/Flow/{ => ETL/Foundation}/ArrayComparison/ArraySortByKey.php (92%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Calculator/Calculator.php create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Calculator/Exception/Exception.php rename src/core/etl/src/Flow/{ => ETL/Foundation}/Calculator/Exception/InvalidExponentException.php (83%) rename src/core/etl/src/Flow/{ => ETL/Foundation}/Calculator/Exception/InvalidScaleException.php (83%) rename src/core/etl/src/Flow/{ => ETL/Foundation}/Calculator/Exception/NonNumericValueException.php (82%) rename src/core/etl/src/Flow/{ => ETL/Foundation}/Calculator/Exception/RoundingNecessaryException.php (84%) rename src/core/etl/src/Flow/{ => ETL/Foundation}/Calculator/NumberNormalizer.php (92%) rename src/core/etl/src/Flow/{ => ETL/Foundation}/Calculator/Rounding.php (85%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Calculator/RunningSum.php rename src/core/etl/src/Flow/{ => ETL/Foundation}/Clock/FakeClock.php (93%) rename src/core/etl/src/Flow/ETL/{Dataset/Statistics => Foundation/Clock}/HighResolutionTime.php (97%) rename src/core/etl/src/Flow/{ => ETL/Foundation}/Clock/SystemClock.php (94%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Describe/ConfiguredFilesystem.php create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Describe/Describable.php create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Describe/Description.php create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Describe/Json.php create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Describe/Rebuild.php create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/ColumnMismatchException.php rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/ConstraintViolationException.php (90%) rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/DataDependentSchemaException.php (88%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/DuplicatedColumnsException.php rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/EvaluationException.php (95%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/Exception.php create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/InferredSchemaException.php create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/InvalidArgumentException.php rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/InvalidFileFormatException.php (91%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/InvalidLogicException.php create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/JoinException.php rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/MissingDependencyException.php (90%) rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/MissingExtensionException.php (91%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/OffsetOverflow.php rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/OffsetReachedException.php (89%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/RemoteStepException.php rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/RequiredPHPVersionException.php (90%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/RuntimeException.php rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/SchemaDefinitionNotFoundException.php (94%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/SchemaDefinitionNotUniqueException.php rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/SchemaMismatchException.php (94%) rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/SchemaNotDerivableException.php (97%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/SchemaValidationException.php create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Exception/TypeAssertionException.php rename src/core/etl/src/Flow/ETL/{ => Foundation}/Exception/UnsupportedUnionTypeException.php (79%) rename src/core/etl/src/Flow/ETL/{ => Foundation}/FlowPhpExtension.php (93%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/NativePHPRandomValueGenerator.php rename src/core/etl/src/Flow/ETL/{ => Foundation}/RandomValueGenerator.php (84%) rename src/core/etl/src/Flow/ETL/{Schema => Foundation}/SimilarNames.php (96%) create mode 100644 src/core/etl/src/Flow/ETL/Foundation/Telemetry/SpanAttributes.php rename src/core/etl/src/Flow/ETL/{Config => Foundation}/Telemetry/TelemetryAttributes.php (93%) delete mode 100644 src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/All.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Any.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Append.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayExpand.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayExpand/ArrayExpand.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayFilter.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayKeep.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayKeyRename.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayKeys.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayKeysStyleConvert.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayMerge.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayPathExists.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayReverse.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArraySort.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArraySort/Sort.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayUnpack.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ArrayValues.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Ascii.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Average.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Between.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/BinaryLength.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/CallUserFunc.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Capitalize.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Cast.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Chunk.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Coalesce.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/CodePointLength.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/CollapseWhitespace.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Collect.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/CollectUnique.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Combine.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Concat.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ConcatWithSeparator.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Contains.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Count.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/DOMElementAttributeValue.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/DOMElementAttributesCount.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/DOMElementNamespaceValue.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/DOMElementNextSibling.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/DOMElementParent.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/DOMElementPreviousSibling.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/DOMElementValue.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/DateTimeFormat.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/DenseRank.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Divide.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/EndsWith.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/EnsureEnd.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/EnsureStart.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/EnumName.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/EnumValue.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Equals.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Evaluation/Filter.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Evaluation/ResultColumn.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Evaluation/Selection.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Exists.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/First.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/FrameAccumulating.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/GreaterThan.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Greatest.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/HTMLQuerySelector.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/HTMLQuerySelectorAll.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Hash.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/IndexOf.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/IndexOfLast.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/IsEmpty.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/IsIn.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/IsNotNull.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/IsNotNumeric.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/IsNull.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/IsNumeric.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/IsType.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/IsUtf8.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/JsonDecode.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/JsonEncode.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Last.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Least.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/LessThan.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/LessThanEqual.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ListSelect.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Literal.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/MatchCases.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/MatchCases/MatchCondition.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Max.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Min.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Minus.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Mod.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ModifyDateTime.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Multiply.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Not.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/NotEquals.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/NotSame.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Now.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/NumberFormat.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/OnEach.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Optional.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Parameter.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/PartitionRanking.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Plus.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Power.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Prepend.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/RandomString.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Rank.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ReferenceResolver.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Regex.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/RegexAll.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/RegexMatch.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/RegexMatchAll.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/RegexReplace.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Repeat.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ResolvesFromChildren.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Reverse.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Round.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/RowNumber.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Same.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Sanitize.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ScalarFunction.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ScalarFunction/ExpandResults.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ScalarFunction/UnpackResults.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Size.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Slug.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Split.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Sprintf.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StartsWith.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StrPad.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StrReplace.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringAfter.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringAfterLast.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringAggregate.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringBefore.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringBeforeLast.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringContainsAny.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringEqualsTo.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringFold.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringMatch.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringMatchAll.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringNormalize.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringStyle.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringTitle.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StringWidth.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Structure.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/StructureSelect.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Sum.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ToDate.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ToDateTime.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ToLower.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ToTimeZone.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ToUpper.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Trim.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Trim/Type.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Truncate.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Ulid.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/UnicodeLength.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Uuid.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/ValueOrder.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/When.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/WindowFunction.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/Wordwrap.php delete mode 100644 src/core/etl/src/Flow/ETL/Function/XPath.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/AggregatingFunction.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/All.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Any.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Append.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayExpand.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayExpand/ArrayExpand.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayFilter.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/ArrayGet.php (78%) rename src/core/etl/src/Flow/ETL/{Function => Functions}/ArrayGetCollection.php (77%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayKeep.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayKeyRename.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayKeys.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayKeysStyleConvert.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayMerge.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/ArrayMergeCollection.php (75%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayPathExists.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayReverse.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArraySort.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArraySort/Sort.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayUnpack.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ArrayValues.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Ascii.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Average.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Between.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/Between/Boundary.php (94%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/BinaryLength.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/CallUserFunc.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Capitalize.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Cast.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Chunk.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Coalesce.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/CodePointLength.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/CollapseWhitespace.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Collect.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/CollectUnique.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Combine.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Comparison.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/ComparisonFunction.php (92%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/Concat.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ConcatWithSeparator.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/Conjuncts.php (94%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/Contains.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Count.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/DOMElementAttributeValue.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/DOMElementAttributesCount.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/DOMElementNamespaceValue.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/DOMElementNextSibling.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/DOMElementParent.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/DOMElementPreviousSibling.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/DOMElementValue.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/DateTimeFormat.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/DenseRank.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Divide.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/EndsWith.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/EnsureEnd.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/EnsureStart.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/EnumName.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/EnumValue.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Evaluation/Filter.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Evaluation/ResultColumn.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/Evaluation/ReturnTypes.php (87%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/Evaluation/Selection.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/Evaluation/TolerantEvaluation.php (84%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/Exists.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/ExpandingFunctions.php (90%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/First.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/FrameAccumulating.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/FunctionTree.php (97%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/Greatest.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/HTMLQuerySelector.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/HTMLQuerySelectorAll.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Hash.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/IndexOf.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/IndexOfLast.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/IsEmpty.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/IsIn.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/IsNotNull.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/IsNotNumeric.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/IsNull.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/IsNumeric.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/IsType.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/IsUtf8.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Join/Condition.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Join/Condition/All.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Join/Condition/Any.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Join/Condition/Equal.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Join/Condition/Identical.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Join/Expression.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/JsonDecode.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/JsonEncode.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Last.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Least.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/ListFunctions.php (79%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/ListSelect.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Literal.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/LiteralValue.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/MatchCases.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/MatchCases/MatchCondition.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Max.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Min.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Minus.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Mod.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ModifyDateTime.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Multiply.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Not.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Now.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/NumberFormat.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/OnEach.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/OnEachElementSchema.php (75%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/Optional.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Parameter.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Plus.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Power.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Prepend.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/RandomString.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Rank.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Reference/Reference.php rename src/core/etl/src/Flow/ETL/{Row => Functions/Reference}/References.php (76%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/Reference/ResolvedReference.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Reference/UnresolvedReference.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/ReferenceRename.php (90%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/ReferenceResolver.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/ReferencedColumns.php (84%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/Regex.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/RegexAll.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/RegexMatch.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/RegexMatchAll.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/RegexReplace.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Repeat.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Reverse.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Round.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/RowNumber.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Sanitize.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ScalarFunction.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ScalarFunction/ExpandResults.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ScalarFunction/UnpackResults.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/ScalarFunctionChain.php (89%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/Size.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Slug.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Split.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Sprintf.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StartsWith.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StrPad.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StrReplace.php rename src/core/etl/src/Flow/ETL/{ => Functions}/String/StringStyles.php (97%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringAfter.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringAfterLast.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringAggregate.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringBefore.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringBeforeLast.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringContainsAny.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringEqualsTo.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringFold.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringMatch.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringMatchAll.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringNormalize.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringStyle.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringTitle.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/StringWidth.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Structure.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/StructureFunctions.php (79%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/StructureSelect.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/StructureValues.php (91%) rename src/core/etl/src/Flow/ETL/{Function => Functions}/StyleConverter/ArrayKeyConverter.php (95%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/Sum.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/TemporalStringCoercion.php (96%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/ToDate.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ToDateTime.php rename src/core/etl/src/Flow/ETL/{Function => Functions}/ToDateTime/PatternCoverage.php (79%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/ToLower.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ToTimeZone.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/ToUpper.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Trim.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Trim/Type.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Truncate.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Ulid.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/UnicodeLength.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Uuid.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/When.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Window.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Window/BoundWindow.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Window/FrameAccumulation.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Window/FrameBound.php rename src/core/etl/src/Flow/ETL/{ => Functions}/Window/FrameBoundType.php (82%) create mode 100644 src/core/etl/src/Flow/ETL/Functions/Window/PeerFrame.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Window/RowsFrame.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Window/WholePartitionFrame.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Window/WindowFrame.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/WindowFunction.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/WithColumn.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/Wordwrap.php create mode 100644 src/core/etl/src/Flow/ETL/Functions/XPath.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/AggregatedGroups.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Aggregators.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/GroupBySteps.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/Pivot.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/PivotAggregation.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/PivotSchema.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/PivotShape.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/PivotValues.php delete mode 100644 src/core/etl/src/Flow/ETL/GroupBy/PivotedTable.php delete mode 100644 src/core/etl/src/Flow/ETL/Hash/NativePHPHash.php delete mode 100644 src/core/etl/src/Flow/ETL/Hash/PlainText.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/ArrayExtractor.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/BatchableExtractor.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/CacheExtractor.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/ChainExtractor.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/Criteria.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/ExtractorFactory.php rename src/core/etl/src/Flow/ETL/{ => IO}/Extractor/FeedExtractor.php (81%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/ByteUnit.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/DerivedFromFiles.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/DerivedSchema.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/FileBatches.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/FileColumns.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/FileColumnsSource.php rename src/core/etl/src/Flow/ETL/{ => IO}/Extractor/File/FileConstants.php (93%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/FileExtractor.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/FileMetadataRow.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/FileRead.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/FileReadLoop.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/FileSplit.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/FileSplitReader.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/FooterStatistics.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/InferredColumns.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/ListedFileRow.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/ListingScan.php rename src/core/etl/src/Flow/ETL/{ => IO}/Extractor/File/MetadataColumnsExtractor.php (79%) rename src/core/etl/src/Flow/ETL/{ => IO}/Extractor/File/OffsetSkippingFileBatches.php (86%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/PartitionTypes.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/PathPartitionsRow.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/ReadWindow.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/RowRange.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/SelfDescribingFile.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/SourceFile.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/File/SourceFiles.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/FileListingExtractor.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/Grid/GridHeader.php rename src/core/etl/src/Flow/ETL/{ => IO}/Extractor/Grid/HeaderNames.php (95%) rename src/core/etl/src/Flow/ETL/{ => IO}/Extractor/Grid/RecordDecoder.php (78%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/InfersSchema.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/Memory/InMemoryRows.php rename src/core/etl/src/Flow/ETL/{ => IO}/Extractor/Memory/SpillState.php (91%) rename src/core/etl/src/Flow/ETL/{ => IO}/Extractor/Memory/SpilledRows.php (86%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/Reader.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/Refreshable.php rename src/core/etl/src/Flow/ETL/{ => IO}/Extractor/RowsExtractor.php (79%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/SequenceExtractor.php rename src/core/etl/src/Flow/ETL/{ => IO}/Extractor/SequenceGenerator/DatePeriodSequenceGenerator.php (89%) rename src/core/etl/src/Flow/ETL/{ => IO}/Extractor/SequenceGenerator/NumberSequenceGenerator.php (94%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/SequenceGenerator/SequenceGenerator.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/Signal.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/Split.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/SplittableExtractor.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Extractor/Statistics.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Filesystem/SaveMode.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Filesystem/ScalarFunctionFilter.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Filesystem/VisibleFiles.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Floe/FloeExtractor.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Floe/FloeFileBatches.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Floe/FloeFileWriter.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Floe/FloeFileWriterFactory.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Floe/FloeLoader.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Floe/FloeSourceFile.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Formatter.php rename src/core/etl/src/Flow/ETL/{ => IO}/Formatter/ASCII/ASCIIValue.php (98%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Formatter/AsciiTableFormatter.php rename src/core/etl/src/Flow/ETL/{Row/Formatter => IO/Formatter/Row}/ASCIISchemaFormatter.php (83%) rename src/core/etl/src/Flow/ETL/{Schema/Formatter => IO/Formatter/Schema}/InlineSchemaFormatter.php (82%) rename src/core/etl/src/Flow/ETL/{Schema/Formatter => IO/Formatter/Schema}/JsonSchemaFormatter.php (84%) rename src/core/etl/src/Flow/ETL/{Schema/Formatter => IO/Formatter/Schema}/PHPFormatter/TypeFormatter.php (99%) rename src/core/etl/src/Flow/ETL/{Schema/Formatter => IO/Formatter/Schema}/PHPFormatter/ValueFormatter.php (93%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Formatter/Schema/PHPSchemaFormatter.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Formatter/Schema/SchemaFormatter.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Inference/ColumnTypes.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Inference/Inferred.php rename src/core/etl/src/Flow/ETL/{Schema => IO}/Inference/InferredTypes.php (79%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Inference/Sample.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Inference/SampledUnits.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Inference/SchemaInference.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Inference/SchemaInferenceBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Inference/SchemaInferrer.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Inference/SniffedColumns.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Inference/Sniffer.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Inference/Sniffs.php rename src/core/etl/src/Flow/ETL/{Schema => IO}/Inference/TypeFloor.php (98%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/ArrayLoader.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/Closure.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/Discardable.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/File/FileCommit.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/File/FileLoader.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/File/FileWrite.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/File/FileWriter.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/File/FileWriterFactory.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/File/Part.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/File/PartitionRouter.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/MemoryLoader.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/Partitioning.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/RowPartitions.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/SchemaValidationLoader.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/StreamLoader.php rename src/core/etl/src/Flow/ETL/{ => IO}/Loader/StreamLoader/Output.php (81%) create mode 100644 src/core/etl/src/Flow/ETL/IO/Loader/StreamLoader/Type.php create mode 100644 src/core/etl/src/Flow/ETL/IO/Transaction.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/Comparison.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/Comparison/All.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/Comparison/Any.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/Comparison/Equal.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/Comparison/Identical.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/Expression.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/HashJoin/HashTable.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/HashJoin/JoinKeys.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/HashJoin/JoinSide.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/HashJoin/Joiner.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/HashJoin/NullRowBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/HashJoin/RowMerger.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/Join.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/JoinSchema.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/JoinShape.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/JoinSteps.php delete mode 100644 src/core/etl/src/Flow/ETL/Join/RowsJoin.php rename src/core/etl/src/Flow/ETL/{ => Kernels}/Bucketing/Bucket.php (83%) rename src/core/etl/src/Flow/ETL/{ => Kernels}/Bucketing/BucketRun.php (83%) rename src/core/etl/src/Flow/ETL/{ => Kernels}/Bucketing/Buckets.php (83%) create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Bucketing/HashBucketing.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Bucketing/KeyEncoder.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Bucketing/KeyGrouping.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Bucketing/SeenKeys.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Comparison/ValueEquality.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Comparison/ValueOrder.php rename src/core/etl/src/Flow/ETL/{ => Kernels}/Hash/Algorithm.php (76%) create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Hash/NativePHPHash.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Hash/PlainText.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/HashJoin/HashTable.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/HashJoin/JoinKeys.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/HashJoin/JoinSide.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/HashJoin/Joiner.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/HashJoin/NullKeyFilter.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/HashJoin/PairPredicate.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/HashJoin/Role.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/HashJoin/RowMerger.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Join/JoinSchema.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Sort/ExternalSort.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Sort/ExternalSortConfig.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Sort/MemorySortConfig.php rename src/core/etl/src/Flow/ETL/{ => Kernels}/Sort/Merge/BucketCursor.php (91%) create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Sort/Merge/KWayMerge.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Sort/RowOrder.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Sort/SortColumn.php create mode 100644 src/core/etl/src/Flow/ETL/Kernels/Sort/SortKey.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/ArrayLoader.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/Closure.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/Discardable.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/File/FileLoader.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/File/FileSink.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/File/FileSinks.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/File/FileWriteFrame.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/File/FilesSink.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/File/PartitionRouter.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/MemoryLoader.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/Partitioning.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/PartitioningLoader.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/RowPartitions.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/SchemaValidationLoader.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/StreamLoader.php delete mode 100644 src/core/etl/src/Flow/ETL/Loader/StreamLoader/Type.php delete mode 100644 src/core/etl/src/Flow/ETL/NativePHPRandomValueGenerator.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/BoundStep.php rename src/core/etl/src/Flow/ETL/{ => Operators}/Constraint.php (84%) create mode 100644 src/core/etl/src/Flow/ETL/Operators/Constraint/SortedByConstraint.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Constraint/UniqueConstraint.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/GroupBy.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/GroupBy/AggregatedGroups.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/GroupBy/Aggregators.php rename src/core/etl/src/Flow/ETL/{ => Operators}/GroupBy/DeclaredPivotValues.php (79%) rename src/core/etl/src/Flow/ETL/{ => Operators}/GroupBy/DiscoveredPivotValues.php (75%) rename src/core/etl/src/Flow/ETL/{ => Operators}/GroupBy/Group.php (82%) rename src/core/etl/src/Flow/ETL/{ => Operators}/GroupBy/GroupByShape.php (75%) rename src/core/etl/src/Flow/ETL/{ => Operators}/GroupBy/GroupKey.php (95%) create mode 100644 src/core/etl/src/Flow/ETL/Operators/GroupBy/MergedAggregate.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/GroupBy/PartialAggregate.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/GroupBy/Pivot.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/GroupBy/PivotShape.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/GroupBy/PivotValues.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/GroupBy/PivotedTable.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Join/CrossJoin.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Join/HashJoin.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Join/HashJoinConfig.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/BatchingByProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/BatchingProcessor.php rename src/core/etl/src/Flow/ETL/{ => Operators}/Processor/BoundExpansion.php (76%) create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/CachingProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/CollectingProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/ConstrainedProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/CountingProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/DistinctProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/Distributable.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/Distribution.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/ExpandingProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/GroupByAggregationProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/MemorySortProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/MergeSortProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/OffsetProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/PivotProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/RepartitionProcessor.php rename src/core/etl/src/Flow/ETL/{Plan => Operators/Processor}/RequiredColumns.php (98%) create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/TopNProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/VoidProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Processor/WindowProcessor.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/AddRowIndex/StartFrom.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/AddRowIndexTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/CollectReferencesTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/DerivedColumns.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/DropColumnsTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/DuplicateRowTransformer.php rename src/core/etl/src/Flow/ETL/{ => Operators}/Transformer/Expansion.php (90%) create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/JoinEachRowsTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/LimitTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/Limiting.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/PruneColumnsTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/Rename/RenameCaseColumnStrategy.php rename src/core/etl/src/Flow/ETL/{ => Operators}/Transformer/Rename/RenameColumnStrategy.php (78%) create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/Rename/RenameMapColumnStrategy.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/Rename/RenameReplaceColumnStrategy.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/RenameColumnTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/RenameEachColumnTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/ScalarFunctionFilterTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/ScalarFunctionTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/SelectColumnsTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/SerializeTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/SerializedPayloadDecoder.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/Stateful.php rename src/core/etl/src/Flow/ETL/{ => Operators}/Transformer/UnpackedColumns.php (77%) create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/UnserializeTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Operators/Transformer/UntilTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/FilterWalk.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/JoinSides.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/LimitWalk.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/ProjectionWalk.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/Rule.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineLimits.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineSortAndLimit.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/Rule/PushLimitIntoSource.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/Rule/PushProjectionIntoExpand.php delete mode 100644 src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Explain.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Explain/Branches.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Explain/Condition.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Explain/Details.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Explain/Expression.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Explain/Layout.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Explain/Outline.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Explain/PhysicalOutline.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Explain/StepDetails.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Explain/TreeLayout.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Format.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/LogicalPlan.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Materialization.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Aggregate.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Batch.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/BatchBy.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Cache.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Collect.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/CollectRefs.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Constrain.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Count.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/CrossJoin.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Discard.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Distinct.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Drop.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Filter.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Join.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/JoinEach.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Limit.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Read.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Rename.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/RenameEach.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Repartition.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Select.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Sort.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/TopN.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Transaction.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Until.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Validate.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/WindowColumn.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/WithColumn.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Node/Write.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Redefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Sinks.php delete mode 100644 src/core/etl/src/Flow/ETL/Plan/Transparency.php delete mode 100644 src/core/etl/src/Flow/ETL/Planner.php delete mode 100644 src/core/etl/src/Flow/ETL/Planner/NodeTranslator.php delete mode 100644 src/core/etl/src/Flow/ETL/Planner/PipelineSplit.php delete mode 100644 src/core/etl/src/Flow/ETL/Planner/PlannedNode.php delete mode 100644 src/core/etl/src/Flow/ETL/Planner/SinkAttachment.php delete mode 100644 src/core/etl/src/Flow/ETL/Planner/SinkFeedFactory.php rename src/core/etl/src/Flow/ETL/{Plan => Planning}/Explain/BoxLayout.php (99%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Explain/Condition.php rename src/core/etl/src/Flow/ETL/{Plan => Planning}/Explain/Entry.php (97%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Explain/Expression.php rename src/core/etl/src/Flow/ETL/{Plan => Planning}/Explain/FlowLayout.php (97%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Explain/Layout.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Explain/LogicalOutline.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Explain/NodeDetails.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Explain/PhysicalOutline.php rename src/core/etl/src/Flow/ETL/{Plan => Planning}/Explain/StatisticsLine.php (93%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Explain/StepDetails.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Explain/TreeLayout.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Explain.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Format.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/LogicalPlan.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Aggregate.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Batch.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/BatchBy.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/BucketingAlgorithmBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Cache.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Collect.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/CollectRefs.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Constrain.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Count.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/CrossJoin.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Discard.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Distinct.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Drop.php rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/Node/DuplicateRow.php (75%) rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/Node/ExpandColumn.php (77%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Filter.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Join.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/JoinAlgorithmBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/JoinEach.php rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/Node/JoinsFrame.php (76%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Limit.php rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/Node/Offset.php (77%) rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/Node/Outputs.php (78%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Read.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Rename.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/RenameEach.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Repartition.php rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/Node/Result.php (76%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Select.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Sort.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/SortAlgorithmBuilder.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/TopN.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Transaction.php rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/Node/Transform.php (76%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Until.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Validate.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/WindowColumn.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/WithColumn.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Node/Write.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Plan.php rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/Redefined.php (98%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Redefinition.php rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/ReplaceNodes.php (95%) rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/Rewrite.php (92%) rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/RowCount.php (79%) rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/Stage.php (90%) rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/TransformUp.php (97%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Transparency.php rename src/core/etl/src/Flow/ETL/{Plan => Planning/Logical}/Trigger.php (82%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Logical/Writes.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Optimizer/JoinSides.php rename src/core/etl/src/Flow/ETL/{ => Planning/Optimizer}/Optimizer.php (75%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Optimizer/Rule.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Optimizer/Rule/CombineLimits.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Optimizer/Rule/CombineSortAndLimit.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Optimizer/Rule/CountFromStatistics.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Optimizer/Rule/PushFilterIntoSource.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Optimizer/Rule/PushLimitIntoSource.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Optimizer/Rule/PushOffsetIntoSource.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Optimizer/Rule/PushProjectionIntoExpand.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Physical/BranchPipeline.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Physical/JoinStep.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Physical/NodeTranslator.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Physical/PhysicalPlan.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Physical/Pipeline.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Physical/PipelineSteps.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Physical/PlannedNode.php rename src/core/etl/src/Flow/ETL/{Planner => Planning/Physical}/PlannedNodes.php (84%) create mode 100644 src/core/etl/src/Flow/ETL/Planning/Physical/Planner.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Physical/TransactionGroup.php create mode 100644 src/core/etl/src/Flow/ETL/Planning/Physical/WritePlanner.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/BatchingByProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/BatchingProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/CachingProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/CollectingProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/ConstrainedProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/CountingProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/ExpandingProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/GroupByAggregationProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/HashJoinProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/MemorySortProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/MergeSortProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/OffsetProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/PivotProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/RepartitionProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/TopNProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/VoidProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Processor/WindowProcessor.php delete mode 100644 src/core/etl/src/Flow/ETL/Repartition/RepartitionSteps.php delete mode 100644 src/core/etl/src/Flow/ETL/Row/Reference.php delete mode 100644 src/core/etl/src/Flow/ETL/Row/ResolvedReference.php delete mode 100644 src/core/etl/src/Flow/ETL/Row/TypedValueComparator.php delete mode 100644 src/core/etl/src/Flow/ETL/Row/UnresolvedReference.php delete mode 100644 src/core/etl/src/Flow/ETL/Rows/RowHashes.php delete mode 100644 src/core/etl/src/Flow/ETL/RowsFactory.php rename src/core/etl/src/Flow/ETL/{ => Runtime}/Analyze.php (97%) create mode 100644 src/core/etl/src/Flow/ETL/Runtime/CacheConfig.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Config.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/ErrorHandler.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/ExtractionAction.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/ExtractionError.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/IgnoreError.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/LoadingAction.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/LoadingError.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/SkipRows.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/ThrowError.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/TransformationAction.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/ErrorHandler/TransformationError.php rename src/core/etl/src/Flow/ETL/{ => Runtime}/Executor/AdoptedBatches.php (84%) create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Executor/BranchFiber.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Executor/BranchLoader.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Executor/Executor.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Executor/NestedLoader.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Executor/OfferedFailure.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Executor/SourceCount.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Executor/SourceCounts.php rename src/core/etl/src/Flow/ETL/{ => Runtime}/Executor/StatisticsCollector.php (77%) create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Executor/StepRunner.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Executor/StepTelemetry.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Executor/TransactionLoader.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Executor/WriteLoaderFactory.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/FlowContext.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Launcher/Launch.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Launcher/Launcher.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/MemoryLimit.php rename src/core/etl/src/Flow/ETL/{Dataset => Runtime/Metrics}/Report.php (91%) rename src/core/etl/src/Flow/ETL/{Dataset => Runtime/Metrics}/SourceStatistics.php (94%) create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Metrics/Statistics.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Metrics/Statistics/Column.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Metrics/Statistics/Columns.php rename src/core/etl/src/Flow/ETL/{Dataset => Runtime/Metrics}/Statistics/ExecutionTime.php (81%) rename src/core/etl/src/Flow/ETL/{Config => Runtime}/Telemetry/SpanScope.php (75%) create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Telemetry/SpanStack.php create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Telemetry/StepKind.php rename src/core/etl/src/Flow/ETL/{Config => Runtime}/Telemetry/TelemetryConfig.php (97%) create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Telemetry/TelemetryContext.php rename src/core/etl/src/Flow/ETL/{Config => Runtime}/Telemetry/TelemetryOptions.php (97%) create mode 100644 src/core/etl/src/Flow/ETL/Runtime/Telemetry/TraceableRowsStorage.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/BooleanDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/CommonType.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/DateDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/DateTimeDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/EnumDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/FloatDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/HTMLDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/HTMLElementDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/IntegerDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/JsonDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/ListDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/MapDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/NullDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/StringDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/StructureDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/TimeDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/TimeZoneDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/UuidDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/ValueMatch.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/XMLDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Definition/XMLElementDefinition.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Formatter/PHPSchemaFormatter.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInference.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferenceBuilder.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/SchemaFormatter.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/SortingStrategy/AlphabeticalStrategy.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/SortingStrategy/TypeStrategy.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Validator/EvolvingValidator.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Validator/SelectiveValidator.php delete mode 100644 src/core/etl/src/Flow/ETL/Schema/Validator/StrictValidator.php delete mode 100644 src/core/etl/src/Flow/ETL/SchemaValidator.php delete mode 100644 src/core/etl/src/Flow/ETL/Sink.php delete mode 100644 src/core/etl/src/Flow/ETL/Sink/Branched.php delete mode 100644 src/core/etl/src/Flow/ETL/Sink/Transactional.php delete mode 100644 src/core/etl/src/Flow/ETL/Sink/Transformed.php delete mode 100644 src/core/etl/src/Flow/ETL/Sort/ExternalSort.php delete mode 100644 src/core/etl/src/Flow/ETL/Sort/Merge/KWayMerge.php delete mode 100644 src/core/etl/src/Flow/ETL/Sort/RowOrder.php delete mode 100644 src/core/etl/src/Flow/ETL/Sort/SortKey.php delete mode 100644 src/core/etl/src/Flow/ETL/Sort/SortSteps.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/ApcuRowsStorage.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/BucketingConfig.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/FilesystemRowsStorage.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/Floe/Exception/IncompatibleSchemaException.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/Floe/FloeFiles.php rename src/core/etl/src/Flow/{ => ETL/Storage}/Floe/FloeMerger.php (75%) create mode 100644 src/core/etl/src/Flow/ETL/Storage/Floe/FloeSerializer.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/Floe/FloeStreamReader.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/Floe/FloeStreamWriter.php rename src/core/etl/src/Flow/{ => ETL/Storage}/Floe/Footer.php (93%) rename src/core/etl/src/Flow/{ => ETL/Storage}/Floe/FooterLocation.php (84%) rename src/core/etl/src/Flow/{ => ETL/Storage}/Floe/FooterReader.php (93%) create mode 100644 src/core/etl/src/Flow/ETL/Storage/Floe/FooterStatistics.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/Floe/Format.php rename src/core/etl/src/Flow/{ => ETL/Storage}/Floe/FrameWriter.php (98%) create mode 100644 src/core/etl/src/Flow/ETL/Storage/Floe/Options.php rename src/core/etl/src/Flow/{ => ETL/Storage}/Floe/Section.php (93%) rename src/core/etl/src/Flow/{ => ETL/Storage}/Floe/SplittingFrameEncoder.php (76%) rename src/core/etl/src/Flow/ETL/{ => Storage}/Memory/ArrayMemory.php (96%) rename src/core/etl/src/Flow/ETL/{ => Storage}/Memory/Memory.php (95%) create mode 100644 src/core/etl/src/Flow/ETL/Storage/MemoryRowsStorage.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/PsrRowsStorage.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/ResidentRowsStorage.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/RowsStorage.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/Serializer/CompressingSerializer.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/Serializer/Exception/SerializationException.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/Serializer/Serializer.php create mode 100644 src/core/etl/src/Flow/ETL/Storage/SpillingRowsStorage.php delete mode 100644 src/core/etl/src/Flow/ETL/Transaction.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformation/AddRowIndex.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformation/AddRowIndex/StartFrom.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformation/BatchBy.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformation/BatchSize.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformation/Drop.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformation/Limit.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformation/MaskColumns.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformation/Select.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/AddRowIndexTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/CollectReferencesTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/CrossJoinRowsTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/DerivedColumns.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/DropColumnsTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/DropDuplicates/Hashes.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/DropDuplicatesTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/DuplicateRowTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/JoinEachRowsTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/LimitTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/PruneColumnsTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/Rename/RenameCaseColumnStrategy.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/Rename/RenameMapColumnStrategy.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/Rename/RenameReplaceColumnStrategy.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/RenameColumnTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/RenameEachColumnTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/SelectColumnsTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/SerializeTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/SerializedPayloadDecoder.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/Stateful.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/UnserializeTransformer.php delete mode 100644 src/core/etl/src/Flow/ETL/Transformer/UntilTransformer.php create mode 100644 src/core/etl/src/Flow/ETL/Values/BatchSize.php rename src/core/etl/src/Flow/ETL/{ => Values}/Cardinality.php (97%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Column/AdaptiveBackend.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Column/Backend.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Column/Column.php rename src/core/etl/src/Flow/ETL/{ => Values}/Column/ColumnBuilder.php (91%) rename src/core/etl/src/Flow/ETL/{Row => Values/Column}/ColumnName.php (94%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/ComparableValues.php (80%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/DateFormatPattern.php (97%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Elements.php (98%) rename src/core/etl/src/Flow/{Floe => ETL/Values/Column/Frame}/Codec.php (91%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Column/Frame/FloeException.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Column/Frame/FrameTooLarge.php rename src/core/etl/src/Flow/{Floe/Codec => ETL/Values/Column/Frame}/NoopCodec.php (86%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Layout/BooleanLayout.php (93%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Layout/BufferLayout.php (93%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Layout/Buffers.php (86%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Layout/FixedBinary16Layout.php (92%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Layout/Float64Layout.php (92%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Layout/Int32Layout.php (93%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Layout/Int64Layout.php (92%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Column/Layout/Layout.php rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Layout/NullCount.php (90%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Layout/Offsets.php (92%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Layout/Utf8Layout.php (93%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Layout/Validity.php (96%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Column/Php/CastingColumnBuilder.php rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/ColumnDecoder.php (90%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/ConstantColumn.php (88%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/ListColumn.php (94%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/ListColumnBuilder.php (96%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/MapColumn.php (95%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/MapColumnBuilder.php (95%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/NullColumnBuilder.php (85%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/PhpColumnBuilder.php (87%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/ScalarColumn.php (84%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/ScalarColumnBuilder.php (91%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/StructColumn.php (97%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Php/StructColumnBuilder.php (96%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Column/PhpBackend.php rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/DatePhysical.php (97%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/DateTimePhysical.php (98%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Column/Physical/DaysFromCivil.php rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/ElementPosition.php (98%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/EnumPhysical.php (95%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/HtmlDocumentPhysical.php (92%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/HtmlElementPhysical.php (96%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/IdentityPhysical.php (89%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/JsonFactory.php (95%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/JsonPhysical.php (95%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/ListPhysical.php (96%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/MapPhysical.php (96%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/NullPhysical.php (91%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/Physical.php (91%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Column/Physical/PhysicalFor.php rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/StructPhysical.php (97%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/TimePhysical.php (98%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/TimeZonePhysical.php (95%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/TimeZones.php (87%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/UuidPhysical.php (95%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/XmlCharacterReferences.php (79%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/XmlDocumentPhysical.php (95%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Physical/XmlElementPhysical.php (96%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/Retype.php (97%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/TextValues.php (96%) rename src/core/etl/src/Flow/ETL/{ => Values}/Column/ValueColumn.php (94%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Join/JoinType.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Join/Side.php rename src/core/etl/src/Flow/ETL/{Dataset => Values}/Memory/BoundedRead.php (88%) rename src/core/etl/src/Flow/ETL/{Dataset => Values}/Memory/Consumption.php (95%) rename src/core/etl/src/Flow/ETL/{Dataset => Values}/Memory/MemoryBudget.php (91%) rename src/core/etl/src/Flow/ETL/{Dataset => Values}/Memory/Unit.php (93%) rename src/core/etl/src/Flow/ETL/{Row => Values}/NullsOrder.php (92%) rename src/core/etl/src/Flow/ETL/{ => Values}/Rows.php (78%) rename src/core/etl/src/Flow/ETL/{ => Values}/Rows/ColumnRetyping.php (88%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Rows/RecordBatch.php rename src/core/etl/src/Flow/ETL/{ => Values}/Rows/RowsBuilder.php (77%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema.php rename src/core/etl/src/Flow/ETL/{ => Values}/Schema/Definition.php (94%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/BooleanDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/DateDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/DateTimeDefinition.php rename src/core/etl/src/Flow/ETL/{ => Values}/Schema/Definition/ElementCompatibility.php (77%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/EnumDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/FloatDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/HTMLDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/HTMLElementDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/IntegerDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/JsonDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/ListDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/MapDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/NullDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/StringDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/StructureDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/TimeDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/TimeZoneDefinition.php rename src/core/etl/src/Flow/ETL/{ => Values}/Schema/Definition/TypeProjection.php (95%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/UuidDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/XMLDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definition/XMLElementDefinition.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Definitions.php rename src/core/etl/src/Flow/ETL/{ => Values}/Schema/Metadata.php (96%) rename src/core/etl/src/Flow/ETL/{ => Values}/Schema/SameSchemas.php (94%) rename src/core/etl/src/Flow/ETL/{ => Values}/Schema/SortingStrategy.php (87%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/SortingStrategy/AlphabeticalStrategy.php rename src/core/etl/src/Flow/ETL/{ => Values}/Schema/SortingStrategy/CombinedStrategy.php (81%) rename src/core/etl/src/Flow/ETL/{ => Values}/Schema/SortingStrategy/MetadataStrategy.php (77%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/SortingStrategy/TypeStrategy.php rename src/core/etl/src/Flow/ETL/{ => Values}/Schema/SortingStrategy/TypeStrategy/TypePriorities.php (94%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Validator/EvolvingValidator.php rename src/core/etl/src/Flow/ETL/{ => Values}/Schema/Validator/MismatchedDefinition.php (87%) create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Validator/SelectiveValidator.php create mode 100644 src/core/etl/src/Flow/ETL/Values/Schema/Validator/StrictValidator.php rename src/core/etl/src/Flow/ETL/{ => Values}/Schema/Validator/ValidationContext.php (91%) create mode 100644 src/core/etl/src/Flow/ETL/Values/SchemaValidator.php rename src/core/etl/src/Flow/ETL/{Row => Values}/SortOrder.php (75%) delete mode 100644 src/core/etl/src/Flow/ETL/Window.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/Accumulator/AverageAccumulator.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/Accumulator/CountAccumulator.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/Accumulator/SumAccumulator.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/BoundWindow.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/FrameAccumulator.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/FrameBound.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/PeerComparator.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/PeerFrame.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/RowsFrame.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/WholePartitionFrame.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/WindowContext.php delete mode 100644 src/core/etl/src/Flow/ETL/Window/WindowFrame.php delete mode 100644 src/core/etl/src/Flow/ETL/WithColumn.php delete mode 100644 src/core/etl/src/Flow/Floe/Exception/FloeException.php delete mode 100644 src/core/etl/src/Flow/Floe/Exception/FrameTooLarge.php delete mode 100644 src/core/etl/src/Flow/Floe/Exception/IncompatibleSchemaException.php delete mode 100644 src/core/etl/src/Flow/Floe/FloeExtractor.php delete mode 100644 src/core/etl/src/Flow/Floe/FloeFileBatches.php delete mode 100644 src/core/etl/src/Flow/Floe/FloeFileSink.php delete mode 100644 src/core/etl/src/Flow/Floe/FloeFileSinks.php delete mode 100644 src/core/etl/src/Flow/Floe/FloeLoader.php delete mode 100644 src/core/etl/src/Flow/Floe/FloeReader.php delete mode 100644 src/core/etl/src/Flow/Floe/FloeSerializer.php delete mode 100644 src/core/etl/src/Flow/Floe/FloeSourceFile.php delete mode 100644 src/core/etl/src/Flow/Floe/FloeStreamReader.php delete mode 100644 src/core/etl/src/Flow/Floe/FloeStreamWriter.php delete mode 100644 src/core/etl/src/Flow/Floe/FloeWriter.php delete mode 100644 src/core/etl/src/Flow/Floe/Format.php delete mode 100644 src/core/etl/src/Flow/Floe/FrameDecoder.php delete mode 100644 src/core/etl/src/Flow/Floe/FrameEncoder.php delete mode 100644 src/core/etl/src/Flow/Floe/FrameReader.php delete mode 100644 src/core/etl/src/Flow/Floe/Options.php delete mode 100644 src/core/etl/src/Flow/Floe/Statistics.php delete mode 100644 src/core/etl/src/Flow/Serializer/Base64Serializer.php delete mode 100644 src/core/etl/src/Flow/Serializer/CompressingSerializer.php delete mode 100644 src/core/etl/src/Flow/Serializer/Exception/SerializationException.php delete mode 100644 src/core/etl/src/Flow/Serializer/NativePHPSerializer.php delete mode 100644 src/core/etl/src/Flow/Serializer/Serializer.php delete mode 100644 src/core/etl/tests/Flow/Calculator/Tests/Unit/RunningSumTest.php delete mode 100644 src/core/etl/tests/Flow/Clock/SystemClockTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/AggregatorFixtures.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/AggregatorRoundTrip.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/BucketsStorageContext.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/CacheDirContext.php rename src/core/etl/tests/Flow/ETL/Tests/{ => Context}/CommandOutputNormalizer.php (82%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/DescribableFixtures.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/DescribeRoundTrip.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/ExecutedPipelines.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/ExecutedSegments.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/FileCommitContext.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/FileFormat.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/FileFormatContract.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/FloeFileFormat.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/FloeFilesContext.php rename src/core/etl/tests/Flow/{Floe => ETL}/Tests/Context/FloeGoldenContext.php (85%) rename src/core/etl/tests/Flow/{Floe => ETL}/Tests/Context/FloeSchemaContext.php (81%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/FloeStreamReaderContext.php rename src/core/etl/tests/Flow/{Floe => ETL}/Tests/Context/FrameContext.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/MetadataColumnsContract.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/PipelineStepList.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/PipelineSteps.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/ReaderRoundTrip.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/ReversedSniffing.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/RowsSerializationContext.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/RowsStorageContext.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/ScalarFunctionContract.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/ScratchBuckets.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/SortedRows.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/SplitCoverage.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/StepRoundTrip.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Context/WrittenFiles.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/BackendRecordingTransformer.php rename src/core/etl/tests/Flow/{Floe => ETL}/Tests/Double/ClosingSpySourceStream.php (97%) rename src/core/etl/tests/Flow/{Floe => ETL}/Tests/Double/CodecStub.php (84%) delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/ConstantHasher.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/ContextRecordingTransformer.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/CountingAggregate.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/CountingCache.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/CountingFrameAccumulator.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/CountingHasher.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/Described.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/FakeSchemaSampler.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/FakeSniffer.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/FanOutBranch.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/FanOutSink.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/FileReadingExtractor.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/IdRecordingRowsStorage.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/NotDescribed.php rename src/core/etl/tests/Flow/{Floe => ETL}/Tests/Double/OverflowingColumnStub.php (96%) rename src/core/etl/tests/Flow/{Floe => ETL}/Tests/Double/PrefixingCodecStub.php (88%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/PresniffedSniffer.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/RecordingBucketsStorage.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/RecordingLauncher.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/RecordingRowsStorage.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/RecordingSink.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/RecordingSplittableExtractor.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/SpyBranch.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/SpyBucketsStorage.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/SpyFileSinks.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/SpyFileWriterFactory.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/SpyRowsStorage.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/SpySink.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/StaticDataFrameFactory.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/StaticExtractorFactory.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/StopRecordingBatches.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/StubLauncher.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/ThrowingRemoveBucketsStorage.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/ThrowingRemoveRowsStorage.php rename src/core/etl/tests/Flow/{Floe => ETL}/Tests/Double/TrailingZerosCodecStub.php (90%) rename src/core/etl/tests/Flow/{Floe => ETL}/Tests/Double/UnsizedSourceStream.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Double/UnsizedStreamFilesystem.php rename src/core/etl/tests/Flow/ETL/Tests/{Unit/Function => }/Fixtures/CallUserFunc/StaticCalculator.php (83%) delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Fixtures/CustomDateTime.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Fixtures/Enum/StringEnum.php rename src/core/etl/tests/Flow/ETL/Tests/{Unit/Extractor/Fixtures/FileListExtractor/file_02.json => Fixtures/FileListExtractor/file_01.txt} (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Unit/Extractor/Fixtures/FileListExtractor/file_03.md => Fixtures/FileListExtractor/file_02.json} (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Unit/Extractor/Fixtures/FileListExtractor/nested/file_04.txt => Fixtures/FileListExtractor/file_03.md} (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Unit/Extractor/Fixtures/FileListExtractor/nested/file_05.json => Fixtures/FileListExtractor/nested/file_04.txt} (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Unit/Extractor/Fixtures/FileListExtractor/nested/file_06.md => Fixtures/FileListExtractor/nested/file_05.json} (100%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Fixtures/FileListExtractor/nested/file_06.md rename src/core/etl/tests/Flow/{Floe/Tests/Fixtures => ETL/Tests/Fixtures/Floe}/pre-statistics/heterogeneous.floe (100%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Fixtures/Floe/regenerate.php rename src/core/etl/tests/Flow/{Floe/Tests/Fixtures => ETL/Tests/Fixtures/Floe}/v2/all-column-types.floe (100%) rename src/core/etl/tests/Flow/{Floe/Tests/Fixtures => ETL/Tests/Fixtures/Floe}/v2/empty.floe (100%) rename src/core/etl/tests/Flow/{Floe/Tests/Fixtures => ETL/Tests/Fixtures/Floe}/v2/heterogeneous.floe (100%) rename src/core/etl/tests/Flow/{Floe/Tests/Fixtures => ETL/Tests/Fixtures/Floe}/xml-node-before-utf8.floe (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Unit/Extractor => }/Fixtures/ZeroExtension/data.0 (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Integration/DataFrame/Fixtures/Partitioning/multi_partition_pruning_test => Fixtures/multi_partitioned}/year=2022/month=12/day=30/file.txt (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Integration/DataFrame/Fixtures/Partitioning/multi_partition_pruning_test => Fixtures/multi_partitioned}/year=2022/month=12/day=31/file.txt (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Integration/DataFrame/Fixtures/Partitioning/multi_partition_pruning_test => Fixtures/multi_partitioned}/year=2023/month=1/day=1/file.txt (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Integration/DataFrame/Fixtures/Partitioning/multi_partition_pruning_test => Fixtures/multi_partitioned}/year=2023/month=1/day=2/file.txt (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Integration/DataFrame/Fixtures/Partitioning/multi_partition_pruning_test => Fixtures/multi_partitioned}/year=2023/month=1/day=3/file.txt (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Integration/DataFrame/Fixtures/Partitioning/multi_partition_pruning_test => Fixtures/multi_partitioned}/year=2023/month=1/day=4/file.txt (100%) rename src/core/etl/tests/Flow/ETL/Tests/{Integration/DataFrame/Fixtures/Partitioning/multi_partition_pruning_test => Fixtures/multi_partitioned}/year=2023/month=1/day=5/file.txt (100%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/API/Config/Bucketing/BucketingConfigBuilderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/API/Transformation/DropTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/API/Transformation/SelectTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/Storage/FilesystemBucketsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/Storage/PSRCacheBucketsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheTestCase.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/InMemoryCacheTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/TraceableCacheTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Config/Bucketing/BucketingConfigBuilderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Context/WrittenFilesTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AliasedReferenceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Analyze/goldstock.csv delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/declaration_order/year=2024/day=01/month=03/out.txt delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/.gitignore delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-01/file.txt delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-02/file.txt delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/placeholders/.gitignore delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/rebind/region=eu/tier=gold/a.txt delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/rebind/region=us/b.txt create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/NaNTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootBlockingOperationsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/TransformedBranchBlockingOperationsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/TransformedBranchTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Dataset/SharedMemoryLimitTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/CacheExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileExtractorScanTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileSourceFilesystemTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/Fixtures/multi_partitioned/year=2022/month=12/day=30/file.txt delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/Fixtures/multi_partitioned/year=2022/month=12/day=31/file.txt delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/Fixtures/multi_partitioned/year=2023/month=1/day=1/file.txt delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/Fixtures/multi_partitioned/year=2023/month=1/day=2/file.txt delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/Fixtures/multi_partitioned/year=2023/month=1/day=3/file.txt delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/Fixtures/multi_partitioned/year=2023/month=1/day=4/file.txt delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/Fixtures/multi_partitioned/year=2023/month=1/day=5/file.txt delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/SinkWritePropertiesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/AllTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/AnyTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/AppendTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayExpandTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayGetCollectionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayGetTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayKeyRenameTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayMergeCollectionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayMergeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayReverseTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArraySortTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayUnpackTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/BetweenTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/BinaryComparisonsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/BinaryLengthTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/CallUserFuncTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/CapitalizeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/CastTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ChunkTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/CodePointLengthTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/CollapseWhitespaceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/CombineTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ConcatTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ConcatWithSeparatorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/CountTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementAttributeValueTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementNextSiblingTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementParentTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementPreviousSiblingTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementValueTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnsureEndTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnsureStartTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnumNameTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnumValueTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/FirstTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorAllTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HashTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/IndexOfLastTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/IsEmptyTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/JsonDecodeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/JsonEncodeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/LastTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ListSelectTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/MatchCasesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ModifyDateTimeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/NotTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/OnEachTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/PrependTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/RegexAllTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/RegexMatchAllTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/RegexMatchTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/RegexReplaceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/RegexTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/RepeatTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ReverseTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/SanitizeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/SizeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/SplitTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/SprintfTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/StrPadTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/StrReplaceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/StringContainsAnyTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/StringEqualsToTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/StringMatchAllTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/StringMatchTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/StringNormalizeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/StringWidthTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/Structure/StructureSelectTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/Structure/StructureTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ToLowerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ToUpperTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/TrimTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/TruncateTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/UnicodeLengthTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/WhenTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/WordwrapTest.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/{Function => Functions}/AddDynamicColumnsTest.php (91%) rename src/core/etl/tests/Flow/ETL/Tests/Integration/{Function => Functions}/AddJsonTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/AllTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/AnyTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/AppendTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ArrayExpandTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ArrayGetCollectionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ArrayGetTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ArrayKeyRenameTest.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/{Function => Functions}/ArrayKeysStyleConvertTest.php (90%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ArrayMergeCollectionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ArrayMergeTest.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/{Function => Functions}/ArrayPathExistsTest.php (94%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ArrayReverseTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ArraySortTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ArrayUnpackTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/BetweenTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/BinaryComparisonsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/BinaryLengthTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/CallUserFuncTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/CapitalizeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/CastTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ChunkTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/CodePointLengthTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/CollapseWhitespaceTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/CombineTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ConcatTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ConcatWithSeparatorTest.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/{Function => Functions}/ContainsTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/CountTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/DOMElementAttributeValueTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/DOMElementNextSiblingTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/DOMElementParentTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/DOMElementPreviousSiblingTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/DOMElementValueTest.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/{Function => Functions}/EndsWithTest.php (95%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/EnsureEndTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/EnsureStartTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/EnumNameTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/EnumValueTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/FirstTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/HTMLQuerySelectorAllTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/HTMLQuerySelectorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/HashTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/IndexOfLastTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/IsEmptyTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/JsonDecodeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/JsonEncodeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/LastTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ListSelectTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/MatchCasesTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ModifyDateTimeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/NotTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/OnEachTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/PrependTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/RegexAllTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/RegexMatchAllTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/RegexMatchTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/RegexReplaceTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/RegexTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/RepeatTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ReverseTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/SanitizeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/SizeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/SplitTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/SprintfTest.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/{Function => Functions}/StartsWithTest.php (95%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/StrPadTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/StrReplaceTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/StringContainsAnyTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/StringEqualsToTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/StringMatchAllTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/StringMatchTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/StringNormalizeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/StringWidthTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/StructureSelectTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/StructureTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ToLowerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/ToUpperTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/TrimTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/TruncateTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/UnicodeLengthTest.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/{Function => Functions}/UntypedIntermediateTest.php (93%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/WhenTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Functions/WordwrapTest.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/{ => IO}/Extractor/ArrayExtractorSpillTest.php (98%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Extractor/CacheExtractorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Extractor/FileExtractorScanTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Extractor/FileListingExtractorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Extractor/FileSourceFilesystemTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/FileFormatContractTest.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/{ => IO}/Filesystem/ConfigIsolationTest.php (95%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Filesystem/FileWritePropertiesTest.php rename src/core/etl/tests/Flow/{Floe/Tests/Integration => ETL/Tests/Integration/IO/Floe}/FloeDataFrameTest.php (99%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Floe/FloeExtractorTest.php rename src/core/etl/tests/Flow/{Floe/Tests/Integration => ETL/Tests/Integration/IO/Floe}/FloeMergeDSLTest.php (92%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Floe/FloeSplitReadTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/File/FileCommit/FileCommitTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/File/FileCommit/NotPartitioned/AppendModeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/File/FileCommit/NotPartitioned/ExceptionIfExistsModeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/File/FileCommit/NotPartitioned/IgnoreModeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/File/FileCommit/NotPartitioned/OverwriteModeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/File/FileCommit/Partitioned/AppendModeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/File/FileCommit/Partitioned/ExceptionIfExistsModeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/File/FileCommit/Partitioned/IgnoreModeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/File/FileCommit/Partitioned/OverwriteModeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/File/FileCommit/Partitioned/OverwriteMultipleFilesTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/File/FileCommit/Partitioned/PartitionPlaceholdersTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/IO/Loader/PartitionedWriteTest.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/{ => Kernels}/Sort/ExternalSort/ExternalSortLifecycleTest.php (93%) rename src/core/etl/tests/Flow/ETL/Tests/Integration/{ => Kernels}/Sort/ExternalSort/ExternalSortMergeTest.php (87%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Kernels/Sort/ExternalSort/ExternalSortRunSizeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/FilesSinkTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/FilesSinkTestCase.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/AppendModeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/ExceptionIfExistsModeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/IgnoreModeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/OverwriteModeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/AppendModeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/ExceptionIfExistsModeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/IgnoreModeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/OverwriteModeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/OverwriteMultipleFilesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/PartitionPlaceholdersTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/PartitioningLoaderTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Monitoring/Memory/ConfigurationTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Operators/Processor/BatchingProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Operators/Processor/CollectingProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Operators/Processor/MemorySortProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/BatchingPipelineTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/CollectingPipelineTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/SerialRunTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/SynchronousPipelineTest.php rename src/core/etl/tests/Flow/ETL/Tests/Integration/{Config => Runtime}/MemoryLimitTest.php (89%) delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortRunSizeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/MemorySort/MemorySortTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Storage/FilesystemRowsStorageTest.php rename src/core/etl/tests/Flow/{Floe/Tests/Integration => ETL/Tests/Integration/Storage/Floe}/FloeGoldenTest.php (87%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Storage/Floe/FloeHardBreakTest.php rename src/core/etl/tests/Flow/{Floe/Tests/Integration => ETL/Tests/Integration/Storage/Floe}/FloeStatisticsTest.php (92%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Storage/Floe/FloeStreamReaderTest.php rename src/core/etl/tests/Flow/{Floe/Tests/Integration => ETL/Tests/Integration/Storage/Floe}/FloeWriteContractTest.php (82%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Storage/PsrRowsStorageTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Storage/SharedMemoryLimitTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Transformation/DropTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Transformation/SelectTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/BranchLoaderMother.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/FileReadMother.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/FooterMother.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/HashJoinMother.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/HashJoinProcessorMother.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/JoinerMother.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/NodeTranslatorMother.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/PipelinesMother.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/SinkAttachmentMother.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/SinkFeedMother.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/WindowContextMother.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Mother/WritePlannerMother.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/Config/Bucketing/BucketingConfigBuilderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/Config/Join/HashJoinBuilderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/Config/Sort/ExternalSortBuilderTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => API}/Config/Sort/MemorySortBuilderTest.php (76%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/DataFrameTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/FilteredBranchTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/GroupedDataFrameTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/TransactionalTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => API}/Transformation/AddRowIndexTest.php (94%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/Transformation/BatchByTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/Transformation/BatchSizeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/Transformation/LimitTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/Transformation/MaskColumnsTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => API}/TransformationsTest.php (88%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/API/TransformedBranchTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketRunTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/HashBucketingTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/KeyGroupingTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/KeyValuesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/NativeHasherTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/SingleBucketHasherTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/MemoryBucketsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/PSRCacheBucketsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/SpillingBucketsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/CacheIndexTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/ApcuCacheTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/TraceableCacheTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Column/AdaptiveBackendTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/LayoutForTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Column/PhpBackendTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalBuilderForTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalKindTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Bucketing/BucketingConfigBuilderTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/HashGroupByBuilderTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Join/HashJoinBuilderTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Sort/ExternalSortBuilderTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/SpanStackTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Constraint/SortedConstraintTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Context/CacheDirContextTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Rows => DSL}/ArrayToRowsTest.php (98%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => DSL}/DateIntervalFunctionsTest.php (99%) delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/BoundedReadTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/MemoryBudgetTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Statistics/ColumnsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Statistics/DistinctCounterTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/ETLErrorHandlingTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/ErrorHandler/SkipRowsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/ColumnMismatchExceptionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/DataDependentSchemaExceptionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/InferredSchemaExceptionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/SchemaValidationExceptionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/ClosureTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/DescribedTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/DiscardableTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/OffsetPipelineTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/PipelineTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/RawTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/SegmentTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/SegmentsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/SinkFeedTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/SinkOffersTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/SinkRunTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/SourceReadTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/SourceRowsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/TransactionRollbackTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/TransactionalSinksTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/WindowFunctionPipelineTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ArrayExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/BatchByExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/BatchExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/BatchesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ChainExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/CollectingExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/DerivedSchemaTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileColumnsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadingTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileScanTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FooterStatisticsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/InferredColumnsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ListedFilesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ListingScanTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/MetadataColumnsExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PartitionColumnsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PartitionTypesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ReadWindowTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/SourceFileTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Memory/InMemoryRowsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Memory/InferredRowsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/MemoryExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/RepeatabilityTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/RowsExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/StatisticsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/ScalarFunctionFilterTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/FlowContextTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCII/ASCIIBodyTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCII/ASCIIHeadersTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCIITableTest.php rename src/core/etl/tests/Flow/{ArrayComparison/Tests/Unit => ETL/Tests/Unit/Foundation/ArrayComparison}/ArrayComparisonTest.php (98%) rename src/core/etl/tests/Flow/{ArrayComparison/Tests/Unit => ETL/Tests/Unit/Foundation/ArrayComparison}/ArraySortByKeyTest.php (97%) rename src/core/etl/tests/Flow/{Calculator/Tests/Unit => ETL/Tests/Unit/Foundation/Calculator}/CalculatorTest.php (90%) rename src/core/etl/tests/Flow/{Calculator/Tests/Unit => ETL/Tests/Unit/Foundation/Calculator}/NumberNormalizerTest.php (87%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Calculator/RunningSumTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Dataset/Statistics => Foundation/Clock}/HighResolutionTimeTest.php (95%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Clock/SystemClockTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Describe/ConfiguredFilesystemTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Describe/DescribableRoundTripTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Describe/DescriptionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Describe/JsonTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Describe/RebuildTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/ColumnMismatchExceptionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/DataDependentSchemaExceptionTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Foundation}/Exception/EvaluationExceptionTest.php (88%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/InferredSchemaExceptionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/InvalidArgumentExceptionTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Foundation}/Exception/InvalidLogicExceptionTest.php (85%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Foundation}/Exception/SchemaDefinitionNotFoundExceptionTest.php (88%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Foundation}/Exception/SchemaMismatchExceptionTest.php (83%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Foundation}/Exception/SchemaNotDerivableExceptionTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Foundation/Exception/SchemaValidationExceptionTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Foundation}/FlowPhpExtensionTest.php (86%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Foundation}/NativePHPRandomValueGeneratorTest.php (96%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema => Foundation}/SimilarNamesTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Config => Foundation}/Telemetry/TelemetryAttributesTest.php (95%) delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AllFunctionsDeclareTheirDeterminismTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AllFunctionsDeclareTheirTypeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AllFunctionsEvaluateOnTheConfiguredBackendTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AllTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AnyTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AppendTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayExpandTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayGetCollectionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayGetTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeyRenameTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayMergeCollectionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayMergeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayReverseTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArraySortTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayUnpackTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayValuesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AverageTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BinaryComparisonsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BinaryLengthTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CallUserFuncTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CapitalizeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CastTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ChunkTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CodePointLengthTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollapseWhitespaceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollectTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollectUniqueTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CombineTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConcatTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConcatWithSeparatorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CountTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementAttributeValueTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementNamespaceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementParentTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementValueTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DenseRankTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnsureEndTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnsureStartTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnumNameTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnumValueTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EqualsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/FilterTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/SelectionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/FirstTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Fixtures/CallUserFunc/Calculator.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorAllTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HashTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IndexOfLastTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsEmptyTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/JsonDecodeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/JsonEncodeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LastTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ListSelectTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LiteralTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MatchCasesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MathTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MaxTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MinTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ModifyDateTimeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NotTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/OnEachTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PregReplaceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PrependTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RankTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ReferenceResolverTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexAllTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexMatchAllTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexMatchTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexReplaceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RepeatTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ReverseTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RowNumberTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SameTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SanitizeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SizeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SplitTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SprintfTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StrPadTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StrReplaceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAggregateTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringContainsAnyTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringEqualsToTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringMatchAllTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringMatchTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringNormalizeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringWidthTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureSelectTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SumTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToLowerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToUpperTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/TrimTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/TruncateTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UnicodeLengthTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ValueOrderTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/WhenTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/WordwrapTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/AllAggregatesDeclareTheirTypeTest.php (76%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/AllAggregatorsPartialTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/AllFunctionsDeclareTheirDeterminismTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/AllFunctionsDeclareTheirTypeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/AllFunctionsDescribeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/AllFunctionsEvaluateOnTheConfiguredBackendTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/AllTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/AnyTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/AppendTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ArrayExpandTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ArrayFilterTest.php (93%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ArrayGetCollectionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ArrayGetTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ArrayKeepTest.php (93%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ArrayKeyRenameTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ArrayKeysStyleConverterTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ArrayKeysTest.php (87%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ArrayMergeCollectionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ArrayMergeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ArrayReverseTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ArraySortTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ArrayUnpackTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ArrayValuesTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/AsciiTest.php (90%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/AverageTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/BetweenTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/BinaryComparisonsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/BinaryLengthTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/CallUserFuncTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/CapitalizeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/CastTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ChunkTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/CoalesceTest.php (77%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/CodePointLengthTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/CollapseWhitespaceTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/CollectTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/CollectUniqueTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/CombineTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ComparisonTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ConcatTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ConcatWithSeparatorTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ConjunctsTest.php (88%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/CountTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/DOMElementAttributeValueTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/DOMElementAttributesCountTest.php (99%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/DOMElementNamespaceValueTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/DOMElementParentTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/DOMElementValueTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/DateTimeFormatTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/DenseRankTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/EnsureEndTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/EnsureStartTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/EnumNameTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/EnumValueTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Evaluation/FilterTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/Evaluation/ResultColumnTest.php (90%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/Evaluation/ReturnTypesTest.php (87%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Evaluation/SelectionTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/Evaluation/TolerantEvaluationTest.php (89%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ExistsTest.php (89%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ExpandingFunctionsTest.php (94%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/FirstTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/GreatestTest.php (91%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/HTMLQuerySelectorAllTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/HTMLQuerySelectorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/HashTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/IndexOfLastTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/IndexOfTest.php (95%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/IsEmptyTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/IsInTest.php (78%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/IsTypeTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/IsUtf8Test.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Join/Condition/AllTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Join/Condition/AnyTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Join/Condition/EqualTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Join/Comparison => Functions/Join/Condition}/IdenticalTest.php (82%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Join/ExpressionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/JsonDecodeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/JsonEncodeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/LastTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/LeastTest.php (93%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ListSelectTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/LiteralTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/LiteralValueTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/LogicalFunctionsTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/MatchCasesTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/MathTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/MaxTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/MinTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/MinusTest.php (94%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ModifyDateTimeTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/MultiplyTest.php (94%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/NotTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/NowTest.php (75%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/NumberFormatTest.php (75%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/OnEachElementSchemaTest.php (95%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/OnEachTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/OptionalTest.php (93%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ParameterTest.php (85%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/PlusTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/PowerTest.php (94%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/PrependTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/RandomStringTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/RankTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Reference/ReferencesTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Reference/ResolvedReferenceTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Reference/UnresolvedReferenceTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ReferenceRenameTest.php (94%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ReferenceResolverTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ReferencedColumnsTest.php (92%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/RegexAllTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/RegexMatchAllTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/RegexMatchTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/RegexReplaceTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/RegexTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/RepeatTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ReverseTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/RoundTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/RowNumberTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/SanitizeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/SizeTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/SlugTest.php (93%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/SplitTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/SprintfTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/StrPadTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/StrReplaceTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/StringAfterLastTest.php (93%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/StringAfterTest.php (94%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/StringAggregateTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/StringBeforeLastTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/StringBeforeTest.php (95%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/StringContainsAnyTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/StringEqualsToTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/StringFoldTest.php (90%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/StringMatchAllTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/StringMatchTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/StringNormalizeTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/StringStyleTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/StringTitleTest.php (92%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/StringWidthTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/StructureSelectTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/StructureTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/StructureValuesTest.php (91%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/SumTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/TemporalStringCoercionTest.php (93%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ToDateTest.php (97%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ToDateTime/PatternCoverageTest.php (92%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ToDateTimeTest.php (89%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ToLowerTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/ToTimeZoneTest.php (80%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/ToUpperTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/TrimTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/TruncateTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/UlidTest.php (93%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/UnicodeLengthTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/UuidTest.php (78%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/WhenTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Window/FrameAccumulationTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Functions}/Window/FrameBoundTest.php (86%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Window/PeerFrameTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Functions}/Window/RowsFrameTest.php (89%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/Window/WholePartitionFrameTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/WindowTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Functions/WordwrapTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Function => Functions}/XPathTest.php (98%) delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/AggregatedGroupsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/AggregatorsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupByShapeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupByStepsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotAggregationTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotSchemaTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotShapeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotedTableTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/ArrayExtractorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/ChainExtractorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/CriteriaTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Extractor/FeedExtractorTest.php (87%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/ByteUnitTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/DerivedFromFilesTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/DerivedSchemaTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileColumnsSourceTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileColumnsTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Extractor/File/FileConstantsTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Extractor/File/FileMetadataRowTest.php (90%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileReadLoopTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileReadTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileSplitReaderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FileSplitTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/FooterStatisticsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/InferredColumnsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/ListingScanTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/MetadataColumnsExtractorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/PartitionTypesTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Extractor/File/PathPartitionsRowTest.php (89%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/ReadWindowTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/RowRangeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/SourceFileTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/File/SourceFilesTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/FileListingExtractorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/Grid/GridHeaderTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Extractor/Grid/HeaderNamesTest.php (86%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Extractor/Grid/RecordDecoderTest.php (98%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/Memory/InMemoryRowsTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Extractor/Memory/SpilledRowsTest.php (84%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/RowsExtractorTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Extractor/SequenceExtractorTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Extractor/SequenceGenerator/DatePeriodSequenceGeneratorTest.php (81%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Extractor/SequenceGenerator/NumberSequenceGeneratorTest.php (89%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Extractor/StatisticsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Filesystem/ScalarFunctionFilterTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Filesystem/VisibleFilesTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeExtractorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeFileBatchesTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeFileWriterFactoryTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeFileWriterTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeLoaderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Floe/FloeSourceFileTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Formatter/ASCII/ASCIIValueTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Formatter/AsciiTableFormatterTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema/Formatter => IO/Formatter/Row}/ASCIISchemaFormatterTest.php (93%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema/Formatter => IO/Formatter/Schema}/InlineSchemaFormatterTest.php (89%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema/Formatter => IO/Formatter/Schema}/JsonSchemaFormatterTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Formatter/Schema/PHPFormatter/TypeFormatterTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema/Formatter => IO/Formatter/Schema}/PHPFormatter/ValueFormatterTest.php (86%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema/Formatter => IO/Formatter/Schema}/PHPSchemaFormatterTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Inference/ColumnTypesTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Inference/InferredTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema => IO}/Inference/InferredTypesTest.php (82%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Inference/SampleTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Inference/SampledUnitsTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema => IO}/Inference/SchemaInferenceBuilderTest.php (88%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Inference/SchemaInferenceTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Inference/SchemaInferrerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Inference/SniffedColumnsTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema => IO}/Inference/TypeFloorTest.php (92%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Loader/ArrayLoaderTest.php (86%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Loader/File/FileCommitTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Loader/File/FileWriteTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Loader/File/PartTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Loader/File/PartitionRouterTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Loader/MemoryLoaderTest.php (82%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Loader/PartitioningTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Loader/RowPartitionsTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/IO/Loader/SchemaValidationLoaderTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => IO}/Loader/StreamLoaderTest.php (82%) delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/AllTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/AnyTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/EqualTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Join/ExpressionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/JoinKeysTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/NullRowBuilderTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/RowMergerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinSchemaTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinShapeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinStepsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Join/RowsJoinTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Bucketing/BucketRunTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Kernels}/Bucketing/BucketTest.php (81%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Kernels}/Bucketing/BucketsTest.php (85%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Bucketing/HashBucketingTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Bucketing/KeyEncoderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Bucketing/KeyGroupingTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Bucketing/SeenKeysTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Comparison/ValueEqualityTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Comparison/ValueOrderTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Kernels}/Hash/NativePHPHashTest.php (76%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Join => Kernels}/HashJoin/HashTableTest.php (95%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/HashJoin/JoinKeysTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/HashJoin/JoinSideTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Join => Kernels}/HashJoin/JoinerTest.php (81%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/HashJoin/NullKeyFilterTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/HashJoin/RoleTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/HashJoin/RowMergerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Join/JoinSchemaTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Sort/ExternalSortConfigTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Sort/ExternalSortTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Sort/MemorySortConfigTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Kernels}/Sort/Merge/BucketCursorTest.php (81%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Sort/Merge/KWayMergeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Sort/RowOrderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Kernels/Sort/SortKeyTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/FileWriteFrameTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/FilesSinkTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/PartitionRouterTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/PartitioningTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/SchemaValidationLoaderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Constraint/SortedByConstraintTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Operators}/Constraint/UniqueConstraintTest.php (83%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/GroupBy/AggregatedGroupsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/GroupBy/AggregatorsTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Operators}/GroupBy/DeclaredPivotValuesTest.php (77%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Operators}/GroupBy/DiscoveredPivotValuesTest.php (77%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/GroupBy/GroupByShapeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/GroupBy/MergedAggregateTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/GroupBy/PartialAggregateTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/GroupBy/PivotShapeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/GroupBy/PivotedTableTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/GroupByTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Join/CrossJoinTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Join/HashJoinConfigTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Join/HashJoinTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/BatchingByProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/BatchingProcessorTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Operators}/Processor/BoundExpansionTest.php (89%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/CachingProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/CollectingProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/ConstrainedProcessorTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Operators}/Processor/CountingProcessorTest.php (91%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/DistinctProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/DistributionTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Operators}/Processor/ExpandingProcessorTest.php (92%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/GroupByAggregationProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/MemorySortProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/MergeSortProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/OffsetProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/PivotProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/RepartitionProcessorTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Operators/Processor}/RequiredColumnsTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/TopNProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/VoidProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Processor/WindowProcessorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/ProcessorBreaksTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/AddRowIndexTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/CollectReferencesTransformerTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Operators}/Transformer/DerivedColumnsTest.php (76%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/DropColumnsTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/DuplicateRowTransformerTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Operators}/Transformer/ExpansionTest.php (90%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/JoinEachRowsTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/LimitTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/PruneColumnsTransformerTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Operators}/Transformer/Rename/RenameMapColumnStrategyTest.php (95%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/RenameColumnTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/RenameEachColumnTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/ScalarFunctionFilterTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/ScalarFunctionTransformer/MathTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/ScalarFunctionTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/SelectColumnsTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/SerializeTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/SerializedPayloadDecoderTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Operators}/Transformer/UnpackedColumnsTest.php (85%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/UnserializeTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Operators/Transformer/UntilTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/FilterWalkTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/JoinSidesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/LimitWalkTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/ProjectionWalkTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CombineLimitsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CombineSortAndLimitTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushLimitIntoSourceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushProjectionIntoExpandTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BranchesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/ExpressionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/OutlineTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/StepDetailsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ExplainTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/LogicalPlanTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/AggregateTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/BatchByTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/BatchTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/CacheTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/CollectTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/CountTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/CrossJoinTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/DropTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/FilterTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/JoinEachTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/LimitTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/ReadTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/SelectTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/SortTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/TopNTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/SinksTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/TriggerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/PlanTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/NodeTranslatorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/PipelineSplitTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/SinkAttachmentTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/SinkFeedFactoryTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning}/Explain/BoxLayoutTest.php (83%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning}/Explain/ConditionTest.php (85%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning}/Explain/EntryTest.php (95%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Explain/ExpressionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Explain/FlowLayoutTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Explain/LogicalOutlineTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Explain/NodeDetailsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Explain/PhysicalOutlineTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning}/Explain/StatisticsLineTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Explain/StepDetailsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Explain/TreeLayoutTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/ExplainTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/LogicalPlanTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/AggregateTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/BatchByTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/BatchTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/CacheTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/CollectRefsTest.php (82%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/CollectTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/ConstrainTest.php (81%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/CountTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/CrossJoinTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/DiscardTest.php (81%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/DistinctTest.php (77%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/DropTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/DuplicateRowTest.php (76%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/ExpandColumnTest.php (90%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/FilterTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/JoinEachTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/JoinTest.php (78%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/LimitTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/OffsetTest.php (82%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/OutputsTest.php (88%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/ReadTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/RenameEachTest.php (81%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/RenameTest.php (83%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/RepartitionTest.php (82%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/ResultTest.php (79%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/SelectTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/SortTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/Node/TopNTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/TransactionTest.php (80%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/TransformTest.php (80%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/UntilTest.php (83%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/ValidateTest.php (76%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/WindowColumnTest.php (85%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/WithColumnTest.php (90%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/Node/WriteTest.php (75%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/PlanTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/RedefinedTest.php (96%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/ReplaceNodesTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Plan => Planning/Logical}/TransformUpTest.php (89%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/TriggerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Logical/WritesTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Optimizer/JoinSidesTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Planning/Optimizer}/OptimizerTest.php (83%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Optimizer/Rule/CombineLimitsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Optimizer/Rule/CombineSortAndLimitTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Optimizer/Rule/CountFromStatisticsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Optimizer/Rule/PushFilterIntoSourceTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Optimizer/Rule/PushLimitIntoSourceTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Optimizer/Rule/PushOffsetIntoSourceTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Optimizer/Rule/PushProjectionIntoExpandTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Physical/NodeTranslatorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Physical/PhysicalPlanTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Physical/PipelineStepsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Physical/PipelineTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Planner => Planning/Physical}/PlannedNodeTest.php (81%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Planner => Planning/Physical}/PlannedNodesTest.php (85%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Physical/PlannerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Planning/Physical/WritePlannerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BatchingByProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BatchingProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CachingProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CollectingProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/ConstrainedProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByAggregationProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/HashJoinProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/MemorySortProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/MergeSortProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/OffsetProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/PivotProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/RepartitionProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/TopNProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/VoidProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/WindowProcessorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Repartition/RepartitionStepsTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Row/ReferencesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Row/ResolvedReferenceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Row/TypedValueComparatorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Row/UnresolvedReferenceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/RowHashesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/RowsSerializationTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Config/Cache => Runtime}/CacheConfigTest.php (87%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/ConfigTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/ErrorHandler/ErrorHandlingTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/ErrorHandler/IgnoreErrorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/ErrorHandler/SkipRowsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/ErrorHandler/ThrowErrorTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Runtime}/Executor/AdoptedBatchesTest.php (78%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/BranchFiberTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/BranchLoaderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/ClosureTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/DiscardableTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/ExecutorTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/OfferedFailureTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/SourceCountTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/SourceCountsTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Runtime}/Executor/StatisticsCollectorTest.php (77%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/StepRunnerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/StepTelemetryTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/TransactionLoaderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Executor/WriteLoaderFactoryTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/FlowContextTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Dataset => Runtime/Metrics}/SourceStatisticsTest.php (93%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Dataset => Runtime/Metrics}/Statistics/ColumnTest.php (88%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Metrics/Statistics/ColumnsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Telemetry/SpanStackTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Telemetry/TelemetryContextTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Config => Runtime}/Telemetry/TelemetryOptionsTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Runtime/Telemetry/TraceableRowsStorageTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/BooleanDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/CommonTypeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/DateDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/EnumDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/FloatDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/HTMLDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/HTMLElementDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/IntegerDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/JsonDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/ListDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/MapDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/TimeDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/UuidDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/ValueMatchTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/XMLDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/XMLElementDefinitionTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/EvolvingValidatorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/Fixtures/StringEnum.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/PHPFormatter/FlowTypeFormatterTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferenceTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/BranchedTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/TransactionalTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/TransformedTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/ExternalSortTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/KWayMergeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/RowOrderTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/SortStepsTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/ApcuRowsStorageTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/BucketingConfigTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/Floe/FloeFilesTest.php rename src/core/etl/tests/Flow/{Floe/Tests/Unit => ETL/Tests/Unit/Storage/Floe}/FloeMergerTest.php (95%) rename src/core/etl/tests/Flow/{Floe/Tests/Unit => ETL/Tests/Unit/Storage/Floe}/FloeSerializerTest.php (92%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/Floe/FloeStreamReaderTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/Floe/FloeStreamWriterTest.php rename src/core/etl/tests/Flow/{Floe/Tests/Unit => ETL/Tests/Unit/Storage/Floe}/FloeValueRoundTripTest.php (97%) rename src/core/etl/tests/Flow/{Floe/Tests/Unit => ETL/Tests/Unit/Storage/Floe}/FooterReaderTest.php (83%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/Floe/FooterStatisticsTest.php rename src/core/etl/tests/Flow/{Floe/Tests/Unit => ETL/Tests/Unit/Storage/Floe}/FooterTest.php (90%) rename src/core/etl/tests/Flow/{Floe/Tests/Unit => ETL/Tests/Unit/Storage/Floe}/FormatTest.php (96%) rename src/core/etl/tests/Flow/{Floe/Tests/Unit => ETL/Tests/Unit/Storage/Floe}/FrameWriterTest.php (94%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/Floe/OptionsTest.php rename src/core/etl/tests/Flow/{Floe/Tests/Unit => ETL/Tests/Unit/Storage/Floe}/SectionTest.php (89%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/Floe/SplittingFrameEncoderTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Storage}/Memory/ArrayMemoryTest.php (76%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/MemoryRowsStorageTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/PsrRowsStorageTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/Serializer/AllSerializersKeepTheBackendTest.php rename src/core/etl/tests/Flow/{Serializer/Tests/Unit => ETL/Tests/Unit/Storage/Serializer}/CompressingSerializerTest.php (85%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Storage/SpillingRowsStorageTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformation/BatchSizeTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformation/LimitTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformation/MaskColumnTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/AddRowIndexTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/CollectReferencesTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/CrossJoinRowsTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DropColumnsTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DropDuplicatesTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DuplicateRowTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/JoinEachRowsTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/LimitTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/PruneColumnsTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/RenameColumnTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/RenameEachColumnTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionTransformer/MathTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SelectColumnsTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializeTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializedPayloadDecoderTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnserializeTransformerTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UntilTransformerTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/BatchSizeTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/CardinalityTest.php (97%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Column/AdaptiveBackendTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/ColumnConcatTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/ColumnEncodeTest.php (97%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/ColumnWithTypeTest.php (98%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/ComparableValuesTest.php (80%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/DateFormatPatternTest.php (93%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/ElementsTest.php (94%) rename src/core/etl/tests/Flow/{Floe/Tests/Unit/Codec => ETL/Tests/Unit/Values/Column/Frame}/NoopCodecTest.php (85%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Layout/BooleanLayoutTest.php (87%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Layout/BufferLayoutTest.php (84%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Layout/BuffersTest.php (83%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Layout/FixedBinary16LayoutTest.php (83%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Layout/Float64LayoutTest.php (83%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Layout/Int32LayoutTest.php (85%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Layout/Int64LayoutTest.php (85%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Layout/NullCountTest.php (76%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Layout/OffsetsTest.php (90%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Layout/Utf8LayoutTest.php (83%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Layout/ValidityTest.php (93%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/CastingColumnBuilderTest.php (90%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/ColumnDecoderTest.php (82%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/ConstantColumnTest.php (79%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/ListColumnBuilderTest.php (87%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/ListColumnTest.php (95%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/MapColumnBuilderTest.php (81%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/MapColumnTest.php (96%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/NullColumnBuilderTest.php (80%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/ScalarColumnBuilderTest.php (91%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/ScalarColumnTest.php (93%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/StructColumnBuilderTest.php (88%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Php/StructColumnTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Column/PhpBackendTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/DatePhysicalTest.php (93%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/DateTimePhysicalTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/DaysFromCivilTest.php (86%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/ElementPositionTest.php (95%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/EnumPhysicalTest.php (83%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/HtmlDocumentPhysicalTest.php (86%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/HtmlElementPhysicalTest.php (95%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/IdentityPhysicalTest.php (80%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/JsonFactoryTest.php (87%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/JsonPhysicalTest.php (83%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/ListPhysicalTest.php (78%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/MapPhysicalTest.php (82%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/NullPhysicalTest.php (82%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Column/Physical/PhysicalForTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/StructPhysicalTest.php (82%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/TimePhysicalTest.php (95%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/TimeZonePhysicalTest.php (83%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/TimeZonesTest.php (79%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/UuidPhysicalTest.php (89%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/XmlCharacterReferencesTest.php (88%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/XmlDocumentPhysicalTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/Physical/XmlElementPhysicalTest.php (96%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/RetypeTest.php (97%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/TextValuesTest.php (98%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Column/ValueColumnTest.php (92%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Join/JoinTypeTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Memory/BoundedReadTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Dataset => Values}/Memory/ConsumptionTest.php (87%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Memory/MemoryBudgetTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Dataset => Values}/Memory/UnitTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Row => Values}/NullsOrderTest.php (78%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Rows/ColumnRetypingTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Rows/RecordBatchTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Rows/RowsBuilderTest.php (94%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/RowsTest.php (83%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/BooleanDefinitionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/DateDefinitionTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/Definition/DateTimeDefinitionTest.php (78%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/Definition/ElementCompatibilityTest.php (96%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/EnumDefinitionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/FloatDefinitionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/HTMLDefinitionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/HTMLElementDefinitionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/IntegerDefinitionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/JsonDefinitionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/ListDefinitionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/MapDefinitionTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/Definition/MergeMatrixTest.php (97%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/Definition/NullDefinitionTest.php (90%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/Definition/StringDefinitionTest.php (76%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/Definition/StructureDefinitionTest.php (86%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/TimeDefinitionTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/Definition/TimeZoneDefinitionTest.php (93%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/Definition/TypeProjectionTest.php (83%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/UuidDefinitionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/XMLDefinitionTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Definition/XMLElementDefinitionTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/DefinitionCompatibilityTest.php (99%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/DefinitionTest.php (90%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/MetadataTest.php (96%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{ => Values}/Schema/SameSchemasTest.php (89%) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Values/Schema/Validator/EvolvingValidatorTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema => Values/Schema/Validator}/MismatchedDefinitionTest.php (85%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema => Values/Schema/Validator}/SelectiveValidatorTest.php (97%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema => Values/Schema/Validator}/StrictValidatorTest.php (98%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema => Values/Schema/Validator}/ValidationContextTest.php (79%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema => Values}/SchemaMergeTest.php (99%) rename src/core/etl/tests/Flow/ETL/Tests/Unit/{Schema => Values}/SchemaTest.php (75%) delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/AverageAccumulatorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/CountAccumulatorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/SumAccumulatorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Window/PeerComparatorTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Window/PeerFrameTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Window/WholePartitionFrameTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Window/WindowContextTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/WindowTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Context/FloeFilesContext.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Context/FloeStreamReaderContext.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Double/UnsizedFilesystem.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Fixtures/regenerate.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Integration/FloeExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Integration/FloeHardBreakTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Integration/FloeStreamReaderTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Mother/FooterMother.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Mother/FrameReaderMother.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Mother/RowsMother.php rename src/core/etl/tests/Flow/Floe/Tests/Unit/{ => DSL}/FloeDSLTest.php (94%) delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileBatchesTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileSinkTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileSinksTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeLoaderTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeReaderTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSourceFileTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeStreamReaderTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeStreamWriterTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FloeWriterTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FrameDecoderTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FrameEncoderTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/FrameReaderTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/OptionsTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/SplittingFrameEncoderTest.php delete mode 100644 src/core/etl/tests/Flow/Floe/Tests/Unit/StatisticsTest.php delete mode 100644 src/core/etl/tests/Flow/Serializer/Tests/Unit/Base64SerializerTest.php delete mode 100644 src/core/etl/tests/Flow/Serializer/Tests/Unit/NativePHPSerializerTest.php create mode 100644 src/extension/arrow-ext/tests/phpt/068_parquet_schema_hint_read.phpt create mode 100644 src/extension/arrow-ext/tests/phpt/069_parquet_native_file_parity.phpt create mode 100644 src/extension/arrow-ext/tests/phpt/070_parquet_writer_options_apply_on_open.phpt create mode 100644 src/extension/arrow-ext/tests/phpt/fixtures/arrow_schema_hints.parquet create mode 100644 src/extension/arrow-ext/tests/phpt/fixtures/arrow_schema_hints.py create mode 100644 src/extension/flow-php-ext/crates/flow-arrow-c/Cargo.toml create mode 100644 src/extension/flow-php-ext/crates/flow-arrow-c/src/lib.rs delete mode 100644 src/extension/flow-php-ext/crates/flow-batch-frame/.gitignore delete mode 100644 src/extension/flow-php-ext/crates/flow-batch-frame/Cargo.lock create mode 100644 src/extension/flow-php-ext/crates/flow-batch-frame/src/builder.rs create mode 100644 src/extension/flow-php-ext/crates/flow-batch-frame/src/frame.rs create mode 100644 src/extension/flow-php-ext/crates/flow-batch-frame/src/key.rs create mode 100644 src/extension/flow-php-ext/crates/flow-column-text/Cargo.toml rename src/extension/flow-php-ext/{src/text => crates/flow-column-text/src}/date.rs (86%) create mode 100644 src/extension/flow-php-ext/crates/flow-column-text/src/float.rs rename src/extension/flow-php-ext/{src/text => crates/flow-column-text/src}/json.rs (98%) create mode 100644 src/extension/flow-php-ext/crates/flow-column-text/src/lib.rs rename src/extension/flow-php-ext/{src/text => crates/flow-column-text/src}/value.rs (92%) create mode 100644 src/extension/flow-php-ext/crates/flow-csv-tokenizer/Cargo.toml create mode 100644 src/extension/flow-php-ext/crates/flow-csv-tokenizer/src/lib.rs create mode 100644 src/extension/flow-php-ext/crates/flow-json-records/Cargo.toml create mode 100644 src/extension/flow-php-ext/crates/flow-json-records/src/lib.rs create mode 100644 src/extension/flow-php-ext/crates/flow-numeric-string/Cargo.toml create mode 100644 src/extension/flow-php-ext/crates/flow-numeric-string/src/lib.rs rename src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/{ => IO/Extractor}/RustCSVOpenSource.php (76%) create mode 100644 src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/IO/Loader/RustCSVEncoder.php delete mode 100644 src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVEncoder.php rename src/extension/flow-php-ext/php/Flow/ETL/Adapter/JSON/{ => IO/Extractor}/RustJsonOpenSource.php (85%) rename src/extension/flow-php-ext/php/Flow/ETL/Adapter/JSON/{ => IO/Loader}/RustJsonEncoder.php (76%) rename src/extension/flow-php-ext/php/Flow/ETL/Adapter/Parquet/{ => IO/Extractor}/RustParquetOpenSource.php (75%) create mode 100644 src/extension/flow-php-ext/php/Flow/ETL/Adapter/Parquet/IO/Loader/RustParquetFileWriter.php delete mode 100644 src/extension/flow-php-ext/php/Flow/ETL/Adapter/Parquet/RustParquetOpenSink.php delete mode 100644 src/extension/flow-php-ext/php/Flow/ETL/Column/RustBackend.php create mode 100644 src/extension/flow-php-ext/php/Flow/ETL/Values/Column/RustBackend.php rename src/extension/flow-php-ext/php/Flow/ETL/{ => Values}/Column/RustColumnsBatch.php (90%) rename src/extension/flow-php-ext/php/Flow/ETL/{ => Values}/RustIterator.php (85%) delete mode 100644 src/extension/flow-php-ext/src/csv/tokenizer.rs create mode 100644 src/extension/flow-php-ext/src/frame.rs delete mode 100644 src/extension/flow-php-ext/src/kind_builder.rs delete mode 100644 src/extension/flow-php-ext/src/text/float.rs create mode 100644 src/extension/flow-php-ext/tests/phpt/100_csv_preset_headers.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/101_numeric_grammar_parity.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/102_csv_native_file_parity.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/103_frame_decode_parity.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/104_frame_decode_zero_copy.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/105_frame_encode_parity.phpt create mode 100644 src/extension/flow-php-ext/tests/phpt/106_rust_backend_describe_and_append_refusal.phpt delete mode 100644 src/lib/azure-sdk/src/Flow/Azure/SDK/Endpoints/UserAgentHeader.php create mode 100644 src/lib/filesystem/src/Flow/Filesystem/Stream/BlockCachedSourceStream.php create mode 100644 src/lib/filesystem/src/Flow/Filesystem/Stream/RangeSourceStream.php create mode 100644 src/lib/filesystem/tests/Flow/Filesystem/Tests/Context/OperatingSystem.php create mode 100644 src/lib/filesystem/tests/Flow/Filesystem/Tests/Double/FailingMvFilesystem.php create mode 100644 src/lib/filesystem/tests/Flow/Filesystem/Tests/Double/Flavour.php create mode 100644 src/lib/filesystem/tests/Flow/Filesystem/Tests/Double/ReadRecordingSourceStream.php delete mode 100644 src/lib/filesystem/tests/Flow/Filesystem/Tests/OperatingSystem.php create mode 100644 src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Stream/Block/NativeLocalFileBlocksFactoryTest.php create mode 100644 src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Stream/BlockCachedSourceStreamTest.php create mode 100644 src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Stream/RangeSourceStreamTest.php create mode 100644 src/lib/parquet/src/Flow/Parquet/Filter/Binding/ExactLeaf.php create mode 100644 src/lib/parquet/src/Flow/Parquet/Filter/Binding/InexactLeaf.php delete mode 100644 src/lib/parquet/src/Flow/Parquet/Filter/Binding/LeafResolver.php delete mode 100644 src/lib/parquet/src/Flow/Parquet/Filter/Binding/ResolvedLeaf.php delete mode 100644 src/lib/parquet/src/Flow/Parquet/ParquetFile/Page/ColumnData.php delete mode 100644 src/lib/parquet/src/Flow/Parquet/ParquetFile/RowGroup/FlowColumnChunk.php delete mode 100644 src/lib/parquet/src/Flow/Parquet/ParquetFile/RowGroup/StatisticsReader.php create mode 100644 src/lib/parquet/src/Flow/Parquet/Reader/ChunkBuffer.php delete mode 100644 src/lib/parquet/src/Flow/Parquet/Reader/ReadPlanner.php delete mode 100644 src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Filter/Binding/LeafResolverTest.php delete mode 100644 src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/RowGroup/StatisticsReaderTest.php create mode 100644 src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/StatisticsTest.php create mode 100644 src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/ChunkBufferTest.php create mode 100644 src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/ColumnChunkViewerTest.php delete mode 100644 src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/ReadPlannerTest.php delete mode 100644 src/lib/postgresql/src/Flow/PostgreSql/AST/Nodes/StatementTrait.php create mode 100644 src/lib/postgresql/src/Flow/PostgreSql/QueryBuilder/AstDeparser.php delete mode 100644 src/lib/postgresql/src/Flow/PostgreSql/QueryBuilder/AstToSql.php delete mode 100644 src/lib/postgresql/tests/Flow/PostgreSql/Tests/Unit/AST/Nodes/StatementAssertTest.php create mode 100644 src/lib/postgresql/tests/Flow/PostgreSql/Tests/Unit/QueryBuilder/AstDeparserTest.php delete mode 100644 web/landing/content/examples/topics/reading/data_frame/_meta.yaml delete mode 100644 web/landing/content/examples/topics/reading/data_frame/code.php delete mode 100644 web/landing/content/examples/topics/reading/data_frame/description.md delete mode 100644 web/landing/content/examples/topics/reading/data_frame/documentation.md delete mode 100644 web/landing/tests/Flow/Website/Tests/Functional/PlaygroundBranchSinkTest.php create mode 100644 web/landing/tests/Flow/Website/Tests/Functional/PlaygroundBranchTest.php diff --git a/.env.dist b/.env.dist index 44bdb3748f..5fa9545235 100644 --- a/.env.dist +++ b/.env.dist @@ -12,7 +12,7 @@ SFTP_PORT=2222 SFTP_USER=flow SFTP_PASSWORD=flow PGSQL_DATABASE_URL=pgsql://postgres:postgres@127.0.0.1:5432/postgres?serverVersion=18&charset=utf8 -MYSQL_DATABASE_URL=mysql://mysql:mysql@127.0.0.1:3306/mysql +MYSQL_DATABASE_URL=mysql://mysql:mysql@127.0.0.1:3306/flow SQLITE_DATABASE_PATH=${FLOW_MONOREPO_PROJECT_ROOT}/var/db/sqlite/test.db ELASTICSEARCH_URL=http://127.0.0.1:9200 OTEL_COLLECTOR_HTTP_ENDPOINT=http://localhost:4318 diff --git a/.github/actions/rust-extension-toolchain/action.yml b/.github/actions/rust-extension-toolchain/action.yml index 06de1efb9d..3afed4526f 100644 --- a/.github/actions/rust-extension-toolchain/action.yml +++ b/.github/actions/rust-extension-toolchain/action.yml @@ -16,6 +16,7 @@ runs: uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable with: toolchain: stable + components: clippy, rustfmt - name: "Install build dependencies (Ubuntu)" if: runner.os == 'Linux' diff --git a/.github/workflows/job-arrow-extension.yml b/.github/workflows/job-arrow-extension.yml index 444a295efe..87e435f482 100644 --- a/.github/workflows/job-arrow-extension.yml +++ b/.github/workflows/job-arrow-extension.yml @@ -59,6 +59,16 @@ jobs: CLANG_PATH: ${{ steps.rust.outputs.clang-path }} run: make build + - name: Test arrow crate + working-directory: src/extension/arrow-ext + env: *rust-env + run: cargo test --locked --features arrow-validate + + - name: Lint arrow crate + working-directory: src/extension/arrow-ext + env: *rust-env + run: cargo fmt --check && cargo clippy --locked --all-targets -- -D warnings + - name: Install Composer Dependencies uses: ./.github/actions/composer-install with: @@ -72,7 +82,9 @@ jobs: - name: Install extension and verify working-directory: src/extension/arrow-ext + env: *rust-env run: | + make build EXT_DIR=$(php -r 'echo ini_get("extension_dir");') sudo cp ext/modules/arrow.so "$EXT_DIR/" echo "extension=arrow.so" | sudo tee -a "$(php -r 'echo php_ini_loaded_file();')" diff --git a/.github/workflows/job-flow-php-extension.yml b/.github/workflows/job-flow-php-extension.yml index 5e4508f1e3..c0b2f89539 100644 --- a/.github/workflows/job-flow-php-extension.yml +++ b/.github/workflows/job-flow-php-extension.yml @@ -51,7 +51,6 @@ jobs: key: php${{ matrix.php }} workspaces: | src/extension/flow-php-ext -> target - src/extension/flow-php-ext/crates/flow-batch-frame -> target src/extension/arrow-ext -> target - name: Build arrow extension @@ -66,10 +65,15 @@ jobs: env: *rust-env run: make build - - name: Test flow-batch-frame crate - working-directory: src/extension/flow-php-ext/crates/flow-batch-frame + - name: Test flow_php workspace + working-directory: src/extension/flow-php-ext env: *rust-env - run: cargo test --locked + run: cargo test --workspace --locked --features arrow-validate + + - name: Lint flow_php workspace + working-directory: src/extension/flow-php-ext + env: *rust-env + run: cargo fmt --all --check && cargo clippy --workspace --locked --all-targets -- -D warnings - name: Install Composer Dependencies uses: ./.github/actions/composer-install @@ -84,7 +88,9 @@ jobs: - name: Install extension and verify working-directory: src/extension/flow-php-ext + env: *rust-env run: | + make build EXT_DIR=$(php -r 'echo ini_get("extension_dir");') sudo cp ext/modules/flow_php.so "$EXT_DIR/" echo "extension=flow_php.so" | sudo tee -a "$(php -r 'echo php_ini_loaded_file();')" diff --git a/.github/workflows/job-tests.yml b/.github/workflows/job-tests.yml index 4bbed9e3fb..240f02e809 100644 --- a/.github/workflows/job-tests.yml +++ b/.github/workflows/job-tests.yml @@ -30,7 +30,7 @@ jobs: env: MYSQL_USER: mysql MYSQL_PASSWORD: mysql - MYSQL_DATABASE: mysql + MYSQL_DATABASE: flow MYSQL_ROOT_PASSWORD: root elasticsearch: image: elasticsearch:8.19.0 @@ -143,7 +143,7 @@ jobs: run: "just test --exclude-group grpc --log-junit ./var/phpunit/logs/junit.xml ${{ (matrix.php-version == '8.3' && matrix.dependencies == 'locked') && '--coverage-clover=./var/phpunit/coverage/clover/coverage.xml' || '' }}" env: &test-env PGSQL_DATABASE_URL: pgsql://postgres:postgres@127.0.0.1:${{ job.services.postgres.ports['5432'] }}/postgres?serverVersion=18&charset=utf8 - MYSQL_DATABASE_URL: mysql://mysql:mysql@127.0.0.1:${{ job.services.mysql.ports['3306'] }}/mysql + MYSQL_DATABASE_URL: mysql://mysql:mysql@127.0.0.1:${{ job.services.mysql.ports['3306'] }}/flow ELASTICSEARCH_URL: http://127.0.0.1:${{ job.services.elasticsearch.ports['9200'] }} SQLITE_DATABASE_URL: "sqlite:///:memory:" AZURITE_HOST: "localhost" diff --git a/.nix/php/lib/benchmark.ini.dist b/.nix/php/lib/benchmark.ini.dist new file mode 100644 index 0000000000..2396b2ae0d --- /dev/null +++ b/.nix/php/lib/benchmark.ini.dist @@ -0,0 +1,9 @@ +opcache.enable=1 +opcache.enable_cli=1 +opcache.jit=tracing +opcache.jit_buffer_size=128M +opcache.memory_consumption=256 +opcache.interned_strings_buffer=32 +opcache.max_accelerated_files=20000 +opcache.validate_timestamps=1 +zend.assertions=-1 diff --git a/.nix/pkgs/flow-php/package.nix b/.nix/pkgs/flow-php/package.nix index fd6c7db0a9..67b9940186 100644 --- a/.nix/pkgs/flow-php/package.nix +++ b/.nix/pkgs/flow-php/package.nix @@ -1,6 +1,7 @@ { php, fetchurl, + stdenv, php-snappy, php-lz4, php-brotli, @@ -11,6 +12,7 @@ with-pcov ? true, with-xdebug ? false, with-blackfire ? false, + with-php-benchmark ? false, with-pg-query-ext ? false, with-arrow-ext ? false, with-flow-php-ext ? false, @@ -87,5 +89,18 @@ flowPHP.buildEnv { then builtins.readFile ./../../php/lib/pcov.ini else builtins.readFile ./../../php/lib/pcov.ini.dist else "" + ) + + "\n" + + ( + if with-php-benchmark + then + ( + if builtins.pathExists ./../../php/lib/benchmark.ini + then builtins.readFile ./../../php/lib/benchmark.ini + else builtins.readFile ./../../php/lib/benchmark.ini.dist + ) + # macOS has no huge pages; OPcache warns on every process start, which lands in every worker's stderr. + + (if stdenv.hostPlatform.isLinux then "\nopcache.huge_code_pages=1\n" else "") + else "" ); } diff --git a/benchmarks/reference/duckdb/bench.sh b/benchmarks/reference/duckdb/bench.sh new file mode 100755 index 0000000000..7e7ff4483e --- /dev/null +++ b/benchmarks/reference/duckdb/bench.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# usage: bench.sh +# scan: count the rows that pass a predicate touching every column, so every column is decoded (a bare count(*) on +# Parquet is answered from the footer). +set -euo pipefail + +shape="$1"; input="$2"; output="$3"; threads="$4" + +case "$input" in + *.parquet) source="read_parquet('$input')" ;; + *) source="read_csv('$input', header = true)" ;; +esac + +scan="id % 2 >= 0 AND quantity % 2 >= 0 AND amount >= 0 AND email <> '' AND city <> '' AND country <> '' AND first_name <> '' AND last_name <> '' AND order_uuid IS NOT NULL AND created_at IS NOT NULL AND active IS NOT NULL" + +case "$shape" in + scan) query="SELECT count(*) FROM $source WHERE $scan;" ;; + to_parquet) query="COPY (SELECT * FROM $source) TO '$output/to_parquet.parquet' (FORMAT parquet);" ;; + group_by) query="COPY (SELECT country, sum(amount) AS amount_sum, count(id) AS id_count, avg(amount) AS amount_avg FROM $source GROUP BY country) TO '$output/group_by.parquet' (FORMAT parquet);" ;; + sort) query="COPY (SELECT * FROM $source ORDER BY amount DESC) TO '$output/sort.parquet' (FORMAT parquet);" ;; + *) echo "unknown shape: $shape" >&2; exit 2 ;; +esac + +mkdir -p "$output" +duckdb -c "SET threads TO $threads; $query" diff --git a/benchmarks/reference/polars/bench.py b/benchmarks/reference/polars/bench.py new file mode 100644 index 0000000000..296c4e3dc7 --- /dev/null +++ b/benchmarks/reference/polars/bench.py @@ -0,0 +1,43 @@ +# usage: POLARS_MAX_THREADS= python bench.py +# POLARS_MAX_THREADS is read once when polars is imported, so it has to come from the environment. +# scan: count the rows that pass a predicate touching every column, so every column is decoded (pl.len() alone is +# answered from the Parquet footer). +import os +import sys + +import polars as pl + +shape, source, output = sys.argv[1], sys.argv[2], sys.argv[3] +os.makedirs(output, exist_ok=True) + +frame = pl.scan_parquet(source) if source.endswith(".parquet") else pl.scan_csv(source, try_parse_dates=True) + +scan = ( + (pl.col("id") % 2 >= 0) + & (pl.col("quantity") % 2 >= 0) + & (pl.col("amount") >= 0) + & (pl.col("email") != "") + & (pl.col("city") != "") + & (pl.col("country") != "") + & (pl.col("first_name") != "") + & (pl.col("last_name") != "") + & pl.col("order_uuid").is_not_null() + & pl.col("created_at").is_not_null() + & pl.col("active").is_not_null() +) + +match shape: + case "scan": + print(frame.filter(scan).select(pl.len()).collect(engine="streaming").item()) + case "to_parquet": + frame.sink_parquet(f"{output}/to_parquet.parquet") + case "group_by": + frame.group_by("country").agg( + pl.col("amount").sum().alias("amount_sum"), + pl.col("id").count().alias("id_count"), + pl.col("amount").mean().alias("amount_avg"), + ).sink_parquet(f"{output}/group_by.parquet") + case "sort": + frame.sort("amount", descending=True).sink_parquet(f"{output}/sort.parquet") + case _: + sys.exit(f"unknown shape: {shape}") diff --git a/benchmarks/reference/spark/bench.py b/benchmarks/reference/spark/bench.py new file mode 100644 index 0000000000..f259fd73b5 --- /dev/null +++ b/benchmarks/reference/spark/bench.py @@ -0,0 +1,43 @@ +# usage: spark-submit --master local[] bench.py +# scan: count the rows that pass a predicate touching every column, so every column is decoded. +import sys + +from pyspark.sql import SparkSession +from pyspark.sql import functions as F + +shape, source, output = sys.argv[1], sys.argv[2], sys.argv[3] + +spark = SparkSession.builder.appName(f"flow-reference-{shape}").getOrCreate() +frame = spark.read.parquet(source) if source.endswith(".parquet") else spark.read.csv(source, header=True, inferSchema=True) + +scan = ( + (F.col("id") % 2 >= 0) + & (F.col("quantity") % 2 >= 0) + & (F.col("amount") >= 0) + & (F.col("email") != "") + & (F.col("city") != "") + & (F.col("country") != "") + & (F.col("first_name") != "") + & (F.col("last_name") != "") + & F.col("order_uuid").isNotNull() + & F.col("created_at").isNotNull() + & F.col("active").isNotNull() +) + +match shape: + case "scan": + print(frame.filter(scan).count()) + case "to_parquet": + frame.write.mode("overwrite").parquet(f"{output}/to_parquet.parquet") + case "group_by": + frame.groupBy("country").agg( + F.sum("amount").alias("amount_sum"), + F.count("id").alias("id_count"), + F.avg("amount").alias("amount_avg"), + ).write.mode("overwrite").parquet(f"{output}/group_by.parquet") + case "sort": + frame.orderBy(F.col("amount").desc()).write.mode("overwrite").parquet(f"{output}/sort.parquet") + case _: + sys.exit(f"unknown shape: {shape}") + +spark.stop() diff --git a/benchmarks/src/BenchmarkConfig.php b/benchmarks/src/BenchmarkConfig.php index f078d676ff..f70a9b0cf7 100644 --- a/benchmarks/src/BenchmarkConfig.php +++ b/benchmarks/src/BenchmarkConfig.php @@ -5,7 +5,7 @@ namespace Flow\Benchmarks; use Flow\Benchmarks\Datasets\Paths; -use Flow\ETL\Config\ConfigBuilder; +use Flow\ETL\API\Config\ConfigBuilder; use function Flow\ETL\DSL\config_builder; diff --git a/benchmarks/src/Datasets/FixtureFormat.php b/benchmarks/src/Datasets/FixtureFormat.php index 638838853f..ec901edc32 100644 --- a/benchmarks/src/Datasets/FixtureFormat.php +++ b/benchmarks/src/Datasets/FixtureFormat.php @@ -5,7 +5,7 @@ namespace Flow\Benchmarks\Datasets; /** - * Named FixtureFormat rather than Format so the short name does not collide with Flow\Floe\Format. + * Named FixtureFormat rather than Format so the short name does not collide with Flow\ETL\Storage\Floe\Format. */ enum FixtureFormat: string { diff --git a/benchmarks/src/Joining/JoinEachScenario.php b/benchmarks/src/Joining/JoinEachScenario.php index 55ead80482..354fb9baa6 100644 --- a/benchmarks/src/Joining/JoinEachScenario.php +++ b/benchmarks/src/Joining/JoinEachScenario.php @@ -25,7 +25,7 @@ public function run(): void ->read(from_floe(Datasets::orders($this->rows)->floe())) ->batchSize(1000) ->withColumn('seller_id', ref('seller_id')->cast(type_string())) - ->joinEach(new SellersDataFrameFactory(Datasets::sellers($this->rows)->parquet()), join_on([ + ->joinEach(new SellersExtractorFactory(Datasets::sellers($this->rows)->parquet()), join_on([ 'seller_id' => 'id', ], join_prefix: 'sellers_')) ->run(); diff --git a/benchmarks/src/Joining/JoinOrdersWithSellersScenario.php b/benchmarks/src/Joining/JoinOrdersWithSellersScenario.php index 3e44503240..68a714bf0d 100644 --- a/benchmarks/src/Joining/JoinOrdersWithSellersScenario.php +++ b/benchmarks/src/Joining/JoinOrdersWithSellersScenario.php @@ -6,7 +6,7 @@ use Flow\Benchmarks\BenchmarkConfig; use Flow\Benchmarks\Datasets\Datasets; -use Flow\ETL\Join\Join; +use Flow\ETL\Values\Join\JoinType; use function Flow\ETL\Adapter\Parquet\from_parquet; use function Flow\ETL\DSL\data_frame; @@ -18,7 +18,7 @@ final readonly class JoinOrdersWithSellersScenario { public function __construct( - private Join $type, + private JoinType $type, private int $rows, ) {} diff --git a/benchmarks/src/Joining/SellersDataFrameFactory.php b/benchmarks/src/Joining/SellersDataFrameFactory.php deleted file mode 100644 index 338b4cc4eb..0000000000 --- a/benchmarks/src/Joining/SellersDataFrameFactory.php +++ /dev/null @@ -1,24 +0,0 @@ -read(from_parquet($this->sellersPath)); - } -} diff --git a/benchmarks/src/Joining/SellersExtractorFactory.php b/benchmarks/src/Joining/SellersExtractorFactory.php new file mode 100644 index 0000000000..3240789d4b --- /dev/null +++ b/benchmarks/src/Joining/SellersExtractorFactory.php @@ -0,0 +1,23 @@ +sellersPath); + } +} diff --git a/benchmarks/src/Partitioning/PartitionCardinality.php b/benchmarks/src/Partitioning/PartitionCardinality.php index b6fb6f685c..289b0657d7 100644 --- a/benchmarks/src/Partitioning/PartitionCardinality.php +++ b/benchmarks/src/Partitioning/PartitionCardinality.php @@ -4,8 +4,8 @@ namespace Flow\Benchmarks\Partitioning; -use Flow\ETL\DataFrame; -use Flow\ETL\Row\Reference; +use Flow\ETL\API\DataFrame; +use Flow\ETL\Functions\Reference\Reference; use function Flow\ETL\DSL\ref; diff --git a/benchmarks/src/Partitioning/PartitionedReadScenario.php b/benchmarks/src/Partitioning/PartitionedReadScenario.php index 86ff133c75..d71af6ce37 100644 --- a/benchmarks/src/Partitioning/PartitionedReadScenario.php +++ b/benchmarks/src/Partitioning/PartitionedReadScenario.php @@ -5,7 +5,7 @@ namespace Flow\Benchmarks\Partitioning; use Flow\Benchmarks\BenchmarkConfig; -use Flow\ETL\Rows; +use Flow\ETL\Values\Rows; use function Flow\ETL\Adapter\CSV\from_csv; use function Flow\ETL\DSL\data_frame; diff --git a/benchmarks/src/Pipeline/CanonicalPipeline.php b/benchmarks/src/Pipeline/CanonicalPipeline.php index 3fc498193c..dd87a074d1 100644 --- a/benchmarks/src/Pipeline/CanonicalPipeline.php +++ b/benchmarks/src/Pipeline/CanonicalPipeline.php @@ -6,7 +6,7 @@ use Flow\Benchmarks\BenchmarkConfig; use Flow\Benchmarks\Datasets\Paths; -use Flow\ETL\Extractor; +use Flow\ETL\IO\Extractor; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\lit; @@ -19,23 +19,23 @@ * read leg. Every subject that claims to be comparable with the others must run through here. * * withColumn is the only verb that adds a column, so it forces schema widening at plan-bind time. - * filter is deliberately non-selective - created_at is never null - so the row count reaching the sink + * filter is deliberately non-selective - created_at is never null - so the row count reaching the loader * cannot move for a second reason. select re-converges every source onto the same five columns, so the - * sink does identical work whatever was read. Floe stores typed values, adding no stringification the + * loader does identical work whatever was read. Floe stores typed values, adding no stringification the * source had not already paid. * - * The sink path is unique per call because a fixed path re-run across iterations and revolutions would + * The output path is unique per call because a fixed path re-run across iterations and revolutions would * measure overwrite behaviour. */ final readonly class CanonicalPipeline { - private string $sink; + private string $output; public function __construct( private Extractor $extractor, string $label, ) { - $this->sink = Paths::var() . '/pipeline_' . $label . '_' . uniqid('', true) . '.floe'; + $this->output = Paths::var() . '/pipeline_' . $label . '_' . uniqid('', true) . '.floe'; } public function run(): void @@ -45,12 +45,12 @@ public function run(): void ->withColumn('contact', ref('customer')->concat(lit(' <'), ref('email'), lit('>'))) ->filter(ref('created_at')->isNotNull()) ->select('order_id', 'seller_id', 'created_at', 'customer', 'contact') - ->write(to_floe($this->sink)) + ->write(to_floe($this->output)) ->run(); } - public function sink(): string + public function output(): string { - return $this->sink; + return $this->output; } } diff --git a/benchmarks/src/Pipeline/FetchScenario.php b/benchmarks/src/Pipeline/FetchScenario.php index fb0e7a2e05..3f7ddecacd 100644 --- a/benchmarks/src/Pipeline/FetchScenario.php +++ b/benchmarks/src/Pipeline/FetchScenario.php @@ -5,7 +5,7 @@ namespace Flow\Benchmarks\Pipeline; use Flow\Benchmarks\BenchmarkConfig; -use Flow\ETL\Rows; +use Flow\ETL\Values\Rows; use function Flow\ETL\DSL\data_frame; diff --git a/benchmarks/src/Pipeline/InMemoryOrders.php b/benchmarks/src/Pipeline/InMemoryOrders.php index e35bf5bcc2..c37d52f034 100644 --- a/benchmarks/src/Pipeline/InMemoryOrders.php +++ b/benchmarks/src/Pipeline/InMemoryOrders.php @@ -35,7 +35,7 @@ public static function of(int $rows): array $data = []; foreach (data_frame()->read(from_parquet(Datasets::orders($rows)->parquet()))->get() as $batch) { - $names = $batch->schema()->references()->names(); + $names = $batch->schema()->names(); // Rows::toArray() is declared array; names() is array, so // recombining them is what makes the string keys ArrayMemory requires provable. diff --git a/benchmarks/src/Pipeline/NodePipelineScenario.php b/benchmarks/src/Pipeline/NodePipelineScenario.php index bacfc75d5e..52d3f97fe2 100644 --- a/benchmarks/src/Pipeline/NodePipelineScenario.php +++ b/benchmarks/src/Pipeline/NodePipelineScenario.php @@ -23,7 +23,7 @@ * XML and Text do not infer and cannot carry the orders schema, so they form their own family: the * same four verbs in the same order as the canonical shape, over one column. * - * These two numbers are comparable to each other and to nothing else. The sink is to_text because a + * These two numbers are comparable to each other and to nothing else. The loader is to_text because a * one-column string frame in Floe would measure Floe's footer rather than the pipeline. */ final readonly class NodePipelineScenario diff --git a/benchmarks/src/Pipeline/OrdersSchema.php b/benchmarks/src/Pipeline/OrdersSchema.php index ceb3591766..7ae69930ee 100644 --- a/benchmarks/src/Pipeline/OrdersSchema.php +++ b/benchmarks/src/Pipeline/OrdersSchema.php @@ -4,7 +4,7 @@ namespace Flow\Benchmarks\Pipeline; -use Flow\ETL\Schema; +use Flow\ETL\Values\Schema; use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\datetime_schema; diff --git a/benchmarks/src/Pipeline/PlanDepthScenario.php b/benchmarks/src/Pipeline/PlanDepthScenario.php index dabe473ce9..b584d6fdcf 100644 --- a/benchmarks/src/Pipeline/PlanDepthScenario.php +++ b/benchmarks/src/Pipeline/PlanDepthScenario.php @@ -5,7 +5,7 @@ namespace Flow\Benchmarks\Pipeline; use Flow\Benchmarks\BenchmarkConfig; -use Flow\ETL\Schema; +use Flow\ETL\Values\Schema; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\lit; diff --git a/benchmarks/src/Pipeline/ReadScenario.php b/benchmarks/src/Pipeline/ReadScenario.php index b942f960b2..cbc48a37c0 100644 --- a/benchmarks/src/Pipeline/ReadScenario.php +++ b/benchmarks/src/Pipeline/ReadScenario.php @@ -5,7 +5,7 @@ namespace Flow\Benchmarks\Pipeline; use Flow\Benchmarks\BenchmarkConfig; -use Flow\ETL\Extractor; +use Flow\ETL\IO\Extractor; use function Flow\ETL\DSL\data_frame; diff --git a/benchmarks/src/Pipeline/SourceExtractor.php b/benchmarks/src/Pipeline/SourceExtractor.php index 15bfccd716..ca3f89312b 100644 --- a/benchmarks/src/Pipeline/SourceExtractor.php +++ b/benchmarks/src/Pipeline/SourceExtractor.php @@ -5,8 +5,8 @@ namespace Flow\Benchmarks\Pipeline; use Flow\Benchmarks\Datasets\Datasets; -use Flow\ETL\Extractor; -use Flow\ETL\Memory\ArrayMemory; +use Flow\ETL\IO\Extractor; +use Flow\ETL\Storage\Memory\ArrayMemory; use function Flow\ETL\Adapter\CSV\from_csv; use function Flow\ETL\Adapter\Excel\DSL\from_excel; diff --git a/benchmarks/src/Pipeline/StageScenario.php b/benchmarks/src/Pipeline/StageScenario.php index 3928704301..2f43b54782 100644 --- a/benchmarks/src/Pipeline/StageScenario.php +++ b/benchmarks/src/Pipeline/StageScenario.php @@ -14,7 +14,7 @@ /** * Rungs 2 and 3 of the ladder. They project five columns of the same five types as rung 4, but rung * 4's fifth column is the concatenated contact string where these write email — so the r4 - r3 delta - * carries a small sink-payload difference on top of the transform. That residual is stated, not + * carries a small loader-payload difference on top of the transform. That residual is stated, not * hidden, and no delta is gate-able. * * Subtractability across the three bench classes holds only because every one of them uses stock diff --git a/benchmarks/src/Schema/FooterSchemaScenario.php b/benchmarks/src/Schema/FooterSchemaScenario.php index 9bde84a404..4a980bdf45 100644 --- a/benchmarks/src/Schema/FooterSchemaScenario.php +++ b/benchmarks/src/Schema/FooterSchemaScenario.php @@ -5,7 +5,7 @@ namespace Flow\Benchmarks\Schema; use Flow\Benchmarks\Datasets\Datasets; -use Flow\ETL\Schema; +use Flow\ETL\Values\Schema; use function Flow\ETL\DSL\data_frame; use function Flow\Floe\DSL\from_floe; diff --git a/benchmarks/src/Schema/RowsSchemaScenario.php b/benchmarks/src/Schema/RowsSchemaScenario.php index 52104ebcc3..f5886377cb 100644 --- a/benchmarks/src/Schema/RowsSchemaScenario.php +++ b/benchmarks/src/Schema/RowsSchemaScenario.php @@ -4,7 +4,7 @@ namespace Flow\Benchmarks\Schema; -use Flow\ETL\Rows; +use Flow\ETL\Values\Rows; use function array_key_exists; use function Flow\ETL\DSL\array_to_rows; diff --git a/benchmarks/src/Schema/SchemaInferenceScenario.php b/benchmarks/src/Schema/SchemaInferenceScenario.php index a2fe133269..58c818ac8a 100644 --- a/benchmarks/src/Schema/SchemaInferenceScenario.php +++ b/benchmarks/src/Schema/SchemaInferenceScenario.php @@ -5,7 +5,7 @@ namespace Flow\Benchmarks\Schema; use Flow\Benchmarks\Datasets\Datasets; -use Flow\ETL\Schema; +use Flow\ETL\Values\Schema; use function Flow\ETL\DSL\data_frame; use function Flow\Floe\DSL\from_floe; diff --git a/benchmarks/src/Transformation/NestedTransformationScenario.php b/benchmarks/src/Transformation/NestedTransformationScenario.php index 304c42f86b..418bc4f327 100644 --- a/benchmarks/src/Transformation/NestedTransformationScenario.php +++ b/benchmarks/src/Transformation/NestedTransformationScenario.php @@ -6,8 +6,8 @@ use Flow\Benchmarks\BenchmarkConfig; use Flow\Benchmarks\Datasets\Datasets; -use Flow\ETL\Loader; -use Flow\ETL\Sink; +use Flow\ETL\API\Branch; +use Flow\ETL\IO\Loader; use function Flow\ETL\DSL\data_frame; use function Flow\Floe\DSL\from_floe; @@ -16,7 +16,7 @@ { public function __construct( private int $rows, - private Loader|Sink $sink, + private Loader|Branch $write, ) {} public function run(): void @@ -24,7 +24,7 @@ public function run(): void data_frame(BenchmarkConfig::builder()) ->read(from_floe(Datasets::orders($this->rows)->floe())) ->batchSize(1000) - ->write($this->sink) + ->write($this->write) ->run(); } } diff --git a/benchmarks/src/Transformation/NoopLoader.php b/benchmarks/src/Transformation/NoopLoader.php index 56ea59ad3c..032d84db80 100644 --- a/benchmarks/src/Transformation/NoopLoader.php +++ b/benchmarks/src/Transformation/NoopLoader.php @@ -4,11 +4,25 @@ namespace Flow\Benchmarks\Transformation; -use Flow\ETL\FlowContext; -use Flow\ETL\Loader; -use Flow\ETL\Rows; +use Flow\ETL\Foundation\Describe\Description; +use Flow\ETL\Foundation\Describe\Rebuild; +use Flow\ETL\Foundation\Exception\InvalidArgumentException; +use Flow\ETL\IO\Loader; +use Flow\ETL\Values\Column\Backend; +use Flow\ETL\Values\Rows; +use SensitiveParameter; final readonly class NoopLoader implements Loader { - public function load(Rows $rows, FlowContext $context): void {} + public static function fromDescription(#[SensitiveParameter] array $data, Rebuild $rebuild): static + { + throw InvalidArgumentException::because('%s holds live state and is never described', self::class); + } + + public function describe(): ?Description + { + return null; + } + + public function load(Rows $rows, Backend $backend): void {} } diff --git a/benchmarks/src/Transformation/SortByCreatedAt.php b/benchmarks/src/Transformation/SortByCreatedAt.php index 2745e102e2..cb94a3b461 100644 --- a/benchmarks/src/Transformation/SortByCreatedAt.php +++ b/benchmarks/src/Transformation/SortByCreatedAt.php @@ -4,8 +4,8 @@ namespace Flow\Benchmarks\Transformation; -use Flow\ETL\DataFrame; -use Flow\ETL\Transformation; +use Flow\ETL\API\DataFrame; +use Flow\ETL\API\Transformation; use function Flow\ETL\DSL\ref; diff --git a/benchmarks/src/Window/WindowPartitioning.php b/benchmarks/src/Window/WindowPartitioning.php index 676c1b935f..3735983ae1 100644 --- a/benchmarks/src/Window/WindowPartitioning.php +++ b/benchmarks/src/Window/WindowPartitioning.php @@ -5,8 +5,8 @@ namespace Flow\Benchmarks\Window; use Flow\Benchmarks\Partitioning\PartitionCardinality; -use Flow\ETL\DataFrame; -use Flow\ETL\Row\Reference; +use Flow\ETL\API\DataFrame; +use Flow\ETL\Functions\Reference\Reference; /** * Both arms derive the key even though only the high-cardinality one partitions by it, so the pair diff --git a/benchmarks/suites/Joining/JoinBench.php b/benchmarks/suites/Joining/JoinBench.php index 6e83b6ca6f..42193b67cb 100644 --- a/benchmarks/suites/Joining/JoinBench.php +++ b/benchmarks/suites/Joining/JoinBench.php @@ -6,7 +6,7 @@ use Flow\Benchmarks\BenchmarkRows; use Flow\Benchmarks\Datasets\Datasets; -use Flow\ETL\Join\Join; +use Flow\ETL\Values\Join\JoinType; use Generator; use PhpBench\Attributes as Bench; @@ -23,28 +23,28 @@ public function warm(array $params): void #[Bench\Groups(['joining'])] public function bench_join_inner(array $params): void { - (new JoinOrdersWithSellersScenario(Join::inner, (int) $params['rows']))->run(); + (new JoinOrdersWithSellersScenario(JoinType::inner, (int) $params['rows']))->run(); } #[Bench\ParamProviders('rows')] #[Bench\Groups(['joining'])] public function bench_join_left(array $params): void { - (new JoinOrdersWithSellersScenario(Join::left, (int) $params['rows']))->run(); + (new JoinOrdersWithSellersScenario(JoinType::left, (int) $params['rows']))->run(); } #[Bench\ParamProviders('rows')] #[Bench\Groups(['joining'])] public function bench_join_left_anti(array $params): void { - (new JoinOrdersWithSellersScenario(Join::left_anti, (int) $params['rows']))->run(); + (new JoinOrdersWithSellersScenario(JoinType::left_anti, (int) $params['rows']))->run(); } #[Bench\ParamProviders('rows')] #[Bench\Groups(['joining'])] public function bench_join_right(array $params): void { - (new JoinOrdersWithSellersScenario(Join::right, (int) $params['rows']))->run(); + (new JoinOrdersWithSellersScenario(JoinType::right, (int) $params['rows']))->run(); } #[Bench\ParamProviders('rows')] diff --git a/benchmarks/suites/Transformation/NestedTransformationBench.php b/benchmarks/suites/Transformation/NestedTransformationBench.php index f194ecf5bc..fea63adddc 100644 --- a/benchmarks/suites/Transformation/NestedTransformationBench.php +++ b/benchmarks/suites/Transformation/NestedTransformationBench.php @@ -26,36 +26,36 @@ public function warm(array $params): void #[Bench\Groups(['transformation'])] public function bench_bare_branch(array $params): void { - $sink = to_branch(ref('order_id')->isNotNull(), new NoopLoader()); + $branch = to_branch(ref('order_id')->isNotNull(), new NoopLoader()); - (new NestedTransformationScenario((int) $params['rows'], $sink))->run(); + (new NestedTransformationScenario((int) $params['rows'], $branch))->run(); } #[Bench\ParamProviders('rows')] #[Bench\Groups(['transformation'])] public function bench_blocking_transformation(array $params): void { - $sink = to_transformation(new SortByCreatedAt(), new NoopLoader()); + $branch = to_transformation(new SortByCreatedAt(), new NoopLoader()); - (new NestedTransformationScenario((int) $params['rows'], $sink))->run(); + (new NestedTransformationScenario((int) $params['rows'], $branch))->run(); } #[Bench\ParamProviders('rows')] #[Bench\Groups(['transformation'])] public function bench_branch_with_transformation(array $params): void { - $sink = to_branch(ref('order_id')->isNotNull(), new NoopLoader())->withTransformation(new SortByCreatedAt()); + $branch = to_branch(ref('order_id')->isNotNull(), to_transformation(new SortByCreatedAt(), new NoopLoader())); - (new NestedTransformationScenario((int) $params['rows'], $sink))->run(); + (new NestedTransformationScenario((int) $params['rows'], $branch))->run(); } #[Bench\ParamProviders('rows')] #[Bench\Groups(['transformation'])] public function bench_streaming_transformation(array $params): void { - $sink = to_transformation(select('order_id'), new NoopLoader()); + $branch = to_transformation(select('order_id'), new NoopLoader()); - (new NestedTransformationScenario((int) $params['rows'], $sink))->run(); + (new NestedTransformationScenario((int) $params['rows'], $branch))->run(); } public function rows(): Generator diff --git a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/NodePipelineScenarioTest.php b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/NodePipelineScenarioTest.php index 5d89b5f794..4bc15d49d0 100644 --- a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/NodePipelineScenarioTest.php +++ b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/NodePipelineScenarioTest.php @@ -21,7 +21,7 @@ final class NodePipelineScenarioTest extends TestCase public const ROWS = 100; /** - * The sink is line-oriented but an XML node stringifies to many lines, so the row count is + * The loader is line-oriented but an XML node stringifies to many lines, so the row count is * counted by a per-row marker rather than by lines. */ public static function sources(): Generator diff --git a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/OrdersSchemaTest.php b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/OrdersSchemaTest.php index 424b2b1a87..543097ead8 100644 --- a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/OrdersSchemaTest.php +++ b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/OrdersSchemaTest.php @@ -29,7 +29,7 @@ public function test_every_column_is_nullable_on_every_source(): void { foreach (Source::cases() as $source) { foreach (OrdersSchema::of($source)->definitions() as $definition) { - static::assertTrue($definition->isNullable(), $source->value . '.' . $definition->reference()->name()); + static::assertTrue($definition->isNullable(), $source->value . '.' . $definition->name()); } } } @@ -40,7 +40,7 @@ public function test_the_five_projected_columns_exist_on_every_source(): void $names = []; foreach (OrdersSchema::of($source)->definitions() as $definition) { - $names[] = $definition->reference()->name(); + $names[] = $definition->name(); } foreach (['order_id', 'seller_id', 'created_at', 'customer', 'email'] as $projected) { diff --git a/bin/docs.php b/bin/docs.php index 0ac9d5ff8e..491a803cad 100755 --- a/bin/docs.php +++ b/bin/docs.php @@ -8,10 +8,10 @@ use Flow\Documentation\FunctionsExtractor; use Flow\Documentation\MethodCollector; use Flow\Documentation\MethodsExtractor; -use Flow\ETL\DataFrame; -use Flow\ETL\DataFrame\GroupedDataFrame; +use Flow\ETL\API\DataFrame; +use Flow\ETL\API\GroupedDataFrame; use Flow\ETL\Flow; -use Flow\ETL\Function\ScalarFunctionChain; +use Flow\ETL\Functions\ScalarFunctionChain; use Symfony\Component\Console\Application; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; diff --git a/compose.yml.dist b/compose.yml.dist index f7049a12d6..1255cce432 100644 --- a/compose.yml.dist +++ b/compose.yml.dist @@ -22,7 +22,7 @@ services: environment: MYSQL_USER: mysql MYSQL_PASSWORD: mysql - MYSQL_DATABASE: mysql + MYSQL_DATABASE: flow MYSQL_ROOT_PASSWORD: root networks: - flow-php diff --git a/composer.json b/composer.json index 1bca0798ee..6e58429f69 100644 --- a/composer.json +++ b/composer.json @@ -16,9 +16,11 @@ "ext-hash": "*", "ext-json": "*", "ext-pgsql": "*", + "ext-sodium": "*", "ext-xml": "*", "ext-xmlreader": "*", "ext-xmlwriter": "*", + "ext-zip": "*", "ext-zlib": "*", "composer-runtime-api": "^2.2", "async-aws/s3": "^2.6 || ^3.0", diff --git a/composer.lock b/composer.lock index b99c59ceaa..3300b09e7c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b0786ea593504405fc9a478542938c95", + "content-hash": "4f9cc5c1333e324aec853586183835bf", "packages": [ { "name": "async-aws/core", @@ -8313,9 +8313,11 @@ "ext-hash": "*", "ext-json": "*", "ext-pgsql": "*", + "ext-sodium": "*", "ext-xml": "*", "ext-xmlreader": "*", "ext-xmlwriter": "*", + "ext-zip": "*", "ext-zlib": "*", "composer-runtime-api": "^2.2" }, diff --git a/documentation/adrs/extension-points.md b/documentation/adrs/extension-points.md index e2ac85df15..2412e14825 100644 --- a/documentation/adrs/extension-points.md +++ b/documentation/adrs/extension-points.md @@ -12,11 +12,11 @@ Flow is a framework, and as all frameworks, it should provide extension points f How to find an extension point? -Easy, look for interfaces! For example take a look at `\Flow\ETL\Cache` which is +Easy, look for interfaces! For example take a look at `\Flow\ETL\Storage\RowsStorage` which is an interface with few implementations. Thanks to the interface, you can easily create your own implementation and use it in the framework. -Most of the extension points exposed through `\Flow\ETL\Config` where you can +Most of the extension points exposed through `\Flow\ETL\Runtime\Config` where you can replace a default implementations of various interfaces with those that better fit your needs. diff --git a/documentation/architecture.md b/documentation/architecture.md new file mode 100644 index 0000000000..c9918be1e7 --- /dev/null +++ b/documentation/architecture.md @@ -0,0 +1,205 @@ +# Architecture + +[TOC] + +This page shows how Flow is built and what happens to your data between `read()` and `run()`. It is an overview. +Each section links to the page with the details. + +## Layers + +Flow is built in layers. A layer uses only the layers below it. `mago guard` enforces this rule in CI. + +| Layer | Namespace | What lives there | Types you meet | +|------------|--------------------------|---------------------------------------------------------|---------------------------------------------------------------| +| DSL | `Flow\ETL\DSL` | the functions you call | `data_frame()`, `ref()`, `from_array()`, `to_output()` | +| API | `Flow\ETL\API` | the user-facing objects | `DataFrame`, `ConfigBuilder`, `Branch`, `Transactional` | +| Runtime | `Flow\ETL\Runtime` | the run: config, context, executor, telemetry | `Config`, `FlowContext`, `Executor` | +| Planning | `Flow\ETL\Planning` | plans, the optimizer, the physical planner, `explain()` | `LogicalPlan`, `Optimizer`, `PhysicalPlan` | +| Operators | `Flow\ETL\Operators` | the steps a pipeline runs | `Transformer`, `Processor`, `HashJoin` | +| IO | `Flow\ETL\IO` | reading, writing, schema inference, formatting | `Extractor`, `Loader`, `FileLoader`, `SaveMode` | +| Functions | `Flow\ETL\Functions` | expressions over columns | `ScalarFunction`, `AggregatingFunction`, `WindowFunction` | +| Kernels | `Flow\ETL\Kernels` | the algorithms: sort, hash join, bucketing, hashing | `ExternalSort`, `Joiner`, `HashBucketing` | +| Storage | `Flow\ETL\Storage` | where rows wait: cache, spill files, Floe encoding | `RowsStorage`, `FilesystemRowsStorage` | +| Values | `Flow\ETL\Values` | the data itself | `Rows`, `Schema`, `Column`, `Backend` | +| Foundation | `Flow\ETL\Foundation` | shared basics: describe/rebuild, exceptions, clock | `Describable`, `InvalidArgumentException` | + +`Flow\ETL\Flow` is the one class at the root. It is the object `data_frame()` returns. + +Adapters use the same layers inside their own namespace. The CSV reader is +`Flow\ETL\Adapter\CSV\IO\Extractor\CSVExtractor`, and the CSV writer is `Flow\ETL\Adapter\CSV\IO\Loader\CSVLoader`. +DSL functions keep their namespaces: `Flow\ETL\DSL`, `Flow\ETL\Adapter\CSV`, `Flow\Floe\DSL`. + +## What happens when you call run() + +```mermaid +flowchart LR + dsl["DSL calls"] --> logical["Logical plan"] + logical --> optimizer["Optimizer"] + optimizer --> planner["Physical planner"] + planner --> executor["Executor"] + executor --> loaders["Loaders"] +``` + +1. Each DSL call adds a node to a logical plan. Nothing reads data yet. +2. The optimizer rewrites the plan. It pushes filters, limits and offsets into the source. It turns a sort followed + by a limit into one top-N step. It answers `count()` from source statistics. +3. The physical planner picks an operator for each node. The sort, join, group-by and repartition algorithms come + from the config. It cuts the plan into pipelines at every step that needs all of its input. +4. The executor pulls batches from the extractor, runs them through the steps and hands them to the loaders. + +`explain()` prints each stage. This pipeline reads a CSV file, groups, sorts, limits and writes: + +```php +read(from_csv('/data/orders.csv')) + ->filter(ref('country')->equals('PL')) + ->groupBy('country') + ->aggregate(sum(ref('total'))) + ->sortBy(ref('total_sum')->desc()) + ->limit(10) + ->write(to_csv('/data/out.csv')->saveMode(overwrite())); + +$plan = $frame->explain(Trigger::run); + +echo $plan->toString(Stage::unoptimized); +echo $plan->toString(Stage::optimized); +echo $plan->toString(Stage::physical); +``` + +The logical plan, as written: + +```text +#6 Write +│ Loader: CSVLoader +└─ #5 Limit + │ Limit: 10 + └─ #4 Sort + │ Sort by: total_sum desc + └─ #3 Aggregate + │ Group by: country + │ Changes columns known only at run time + └─ #2 Filter + │ Condition: country = 'PL' + └─ #1 Read + Extractor: CSVExtractor + Source: file:///data/orders.csv +``` + +After the optimizer, `Sort` and `Limit` are one `TopN` node: + +```text +#5 Write +│ Loader: CSVLoader +└─ #4 TopN + │ Top: 10 + └─ #3 Aggregate + │ Group by: country + │ Changes columns known only at run time + └─ #2 Filter + │ Condition: country = 'PL' + └─ #1 Read + Extractor: CSVExtractor + Source: file:///data/orders.csv +``` + +The physical plan has three pipelines. Each pipeline ends at a processor or at the loader: + +```text +Physical plan +│ Columns: country, total_sum +└─ Pipeline #2 + │ Loader: CSVLoader + └─ Pipeline #1 + │ Processor: TopNProcessor + │ Top: 10 + │ Sort by: total_sum desc + └─ Pipeline #0 + Extractor: CSVExtractor + Statistics: rows exact 4 · size exact 49 B + Transformer: ScalarFunctionFilterTransformer + Transformer: PruneColumnsTransformer + Processor: GroupByAggregationProcessor + Group by: country + Aggregations: Sum + Memory: 1000 MB + Storage: FilesystemRowsStorage + Batch: 1000 +``` + +A limit goes into the source. The `Read` node carries it, so the extractor can stop after 2 rows: + +```text +#3 Result +│ Rows this plan hands out: to the trigger, or to the node reading it +└─ #2 Limit + │ Limit: 2 + └─ #1 Read + Extractor: CSVExtractor + Source: file:///data/orders.csv + Limit: 2 +``` + +The executor writes each loader's rows in one run. A branch (`to_branch()`) sends only the rows that match its +condition to its loader. A transaction (`to_dbal_transaction()`, `to_pgsql_transaction()`) commits its writes +together, or rolls all of them back. + +More: [triggers and explain()](/documentation/components/core/core.md), +[sort](/documentation/components/core/sort.md), [join](/documentation/components/core/join.md), +[group by](/documentation/components/core/group-by.md), +[statistics](/documentation/components/core/statistics.md). + +## Batches and columns + +Rows travel in batches. A batch is a `Rows` object: one `Column` per schema definition, never one PHP array per +row. Every step reads whole columns and returns whole columns. + +The backend decides where the columns live. `AdaptiveBackend` is the default. With the native +[flow_php extension](/documentation/components/extensions/flow-php-ext.md) loaded, it uses the extension. Without it, +it uses plain PHP arrays. Both give the same values. Set a backend on the config: + +```php +use Flow\ETL\Values\Column\PhpBackend; + +use function Flow\ETL\DSL\{config_builder, data_frame, from_array}; + +data_frame(config_builder()->backend(new PhpBackend())) + ->read(from_array([['id' => 1], ['id' => 2]])) + ->fetch(); +``` + +More: [column backend](/documentation/components/core/column-backend.md), +[batch processing](/documentation/components/core/batch-processing.md). + +## Writes + +Each file loader carries its own save mode: `exception_if_exists()` (the default), `append()`, `ignore()` or +`overwrite()`. + +```php +data_frame() + ->read(from_array([['id' => 1]])) + ->write(to_csv('/data/out.csv')->saveMode(overwrite())) + ->run(); +``` + +An overwrite writes its files to a temporary directory in the destination first: `.flow-tmp--`. The +commit moves the files into place, then removes the old parts. A failed commit can leave duplicates. It never loses +data. `append()`, `ignore()` and `exception_if_exists()` write in place. If the run fails, they remove the files +they created. + +Readers skip every path below the root you name that starts with `.` or `_`. They keep partition directories +(`country=PL`), `_metadata` and `_common_metadata`. Because of this rule, a reader never sees an unfinished +`.flow-tmp-` directory. + +More: [save mode](/documentation/components/core/save-mode.md), +[partitioning](/documentation/components/core/partitioning.md). diff --git a/documentation/components/adapters/doctrine.md b/documentation/components/adapters/doctrine.md index 592e5ea7fa..6c13beb1e6 100644 --- a/documentation/components/adapters/doctrine.md +++ b/documentation/components/adapters/doctrine.md @@ -87,7 +87,7 @@ A custom map replaces the default one, so spread `TypesMap::FLOW_TYPES` to keep ```php use Doctrine\DBAL\Types\TextType; -use Flow\ETL\Adapter\Doctrine\TypesMap; +use Flow\ETL\Adapter\Doctrine\Values\Schema\TypesMap; use Flow\Types\Type\Native\StringType; use function Flow\ETL\Adapter\Doctrine\to_dbal_table_insert; @@ -126,8 +126,8 @@ data_frame() ## Transactional Loading -`to_dbal_transaction()` writes one or more sinks inside transactions: each batch of rows is written in its own -transaction, and if any sink throws, the open transaction is rolled back: +`to_dbal_transaction()` writes one or more children inside transactions: each batch of rows is written in its own +transaction, and if any child throws, the open transaction is rolled back: ```php use function Flow\ETL\Adapter\Doctrine\{to_dbal_table_insert, to_dbal_transaction}; @@ -143,18 +143,18 @@ data_frame() ->run(); ``` -A plain loader child is a bare sink root; a `to_transformation(...)` child delivers inside the same transaction. +A plain loader child is written directly; a `to_transformation(...)` child delivers inside the same transaction. Every child's loader must use the same connection as the transaction - pass one live `Connection` to `to_dbal_transaction()` and to every child. A loader built from array params (like `to_dbal_table_insert($params, 'users')`) opens its own connection and escapes the transaction. -Sinks with blocking operations (`to_transformation()` / `to_branch(...)->withTransformation(...)` with `sortBy()`, +Branches with blocking operations (`to_transformation()`, also inside `to_branch()`, with `sortBy()`, `aggregate()`, `groupBy()->aggregate()`, `pivot()`, window functions, `collect()`, `join()` - see [transformations](../core/transformations.md)) buffer the stream and deliver it when the run ends; `to_dbal_transaction()` opens one final transaction around that delivery - the whole drained stream commits atomically, a failure during it rolls back and surfaces from `run()`. -Inside the transaction every sink runs throw-only: a failure rolls the batch back first, and only then the frame's +Inside the transaction every child runs throw-only: a failure rolls the batch back first, and only then the frame's `onError()` handler decides whether the run continues. To set the isolation level, build the transaction yourself; it applies to every transaction opened, including the @@ -162,8 +162,8 @@ final one, and the previous level is restored when each one ends: ```php use Doctrine\DBAL\TransactionIsolationLevel; -use Flow\ETL\Adapter\Doctrine\DbalTransaction; -use Flow\ETL\Sink\Transactional; +use Flow\ETL\Adapter\Doctrine\IO\Loader\DbalTransaction; +use Flow\ETL\API\Sink\Transactional; use function Flow\ETL\Adapter\Doctrine\to_dbal_table_insert; @@ -205,7 +205,7 @@ data_frame() ### Multiple Parametrized Queries ```php -use Flow\ETL\Adapter\Doctrine\ParametersSet; +use Flow\ETL\Adapter\Doctrine\IO\Extractor\ParametersSet; use function Flow\ETL\Adapter\Doctrine\from_dbal_queries; use function Flow\ETL\DSL\{data_frame, to_output}; @@ -266,12 +266,12 @@ data_frame() With `to_dbal_schema_table()` function we can convert any Flow Schema (which represents a dataset) to Doctrine DBAL Schema Table. -By providing metadata defined in `\Flow\ETL\Adapter\Doctrine\DbalMetadata` we can also add additional information to the schema, +By providing metadata defined in `\Flow\ETL\Adapter\Doctrine\Values\Schema\DbalMetadata` we can also add additional information to the schema, like length, primary key, index, precision, etc ```php -use Flow\ETL\Adapter\Doctrine\DbalMetadata; +use Flow\ETL\Adapter\Doctrine\Values\Schema\DbalMetadata; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\date_schema; @@ -370,7 +370,7 @@ public const array FLOW_TYPES = [ The `TypesMap` class provides bidirectional mapping between Flow types and Doctrine DBAL types, allowing for flexible type conversion in both directions: ```php -use Flow\ETL\Adapter\Doctrine\TypesMap; +use Flow\ETL\Adapter\Doctrine\Values\Schema\TypesMap; use Flow\Types\Type\Native\StringType; use Doctrine\DBAL\Types\TextType; diff --git a/documentation/components/adapters/excel.md b/documentation/components/adapters/excel.md index 4163f09f88..8cfbffd237 100644 --- a/documentation/components/adapters/excel.md +++ b/documentation/components/adapters/excel.md @@ -60,16 +60,13 @@ $rows = data_frame() read( - from_excel('path/to/your/excel.xlsx') - ->withHeader(false) - ->withOffset(5) - ) + ->read(from_excel('path/to/your/excel.xlsx')->withHeader(false)) + ->offset(4) ->fetch(); ``` -Row numbers count from 1 and include the header row, so `->withOffset(5)` on a sheet with a header skips rows 2-4 -and starts at row 5. +`offset()` counts data rows, so a sheet read without a header starts at row 5 here; with a header, `->offset(3)` +starts at row 5. ### Header and missing cells @@ -159,7 +156,7 @@ data_frame() ```php read($extractor) @@ -242,8 +239,8 @@ data_frame() ```php or()` / `->and()` +### Stop conditions - `http_stop_when_*`, composed with `http_stop_when_any_of()` / `http_stop_when_all_of()` ```php // path_missing · empty_path · flag_false · flag_true · total_reached · max_pages · max_results -stop_when: http_stop_when_flag_false('has_more')->or(http_stop_when_max_pages(1000)); +stop_when: http_stop_when_any_of(http_stop_when_flag_false('has_more'), http_stop_when_max_pages(1000)); ``` ## Extractor - PsrHttpClientStaticExtractor @@ -195,7 +195,7 @@ receives the previous response and returns the next request, or `null` to stop. ```php read(from_array([ @@ -405,7 +405,7 @@ Delete rows matching primary key values: ```php use function Flow\ETL\Adapter\PostgreSql\{pgsql_delete_options, to_pgsql_table}; -use Flow\ETL\Adapter\PostgreSql\Operation; +use Flow\ETL\Adapter\PostgreSql\IO\Loader\Operation; df() ->read(from_array([ @@ -422,8 +422,8 @@ df() ### Transactional Loading -`to_pgsql_transaction()` writes one or more sinks inside transactions: each batch of rows is written in its own -transaction, and if any sink throws, the open transaction is rolled back: +`to_pgsql_transaction()` writes one or more children inside transactions: each batch of rows is written in its own +transaction, and if any child throws, the open transaction is rolled back: ```php use function Flow\ETL\Adapter\PostgreSql\{to_pgsql_table, to_pgsql_transaction}; @@ -438,23 +438,23 @@ df() ->run(); ``` -Sinks with blocking operations (`to_transformation()` / `to_branch(...)->withTransformation(...)` with `sortBy()`, +Branches with blocking operations (`to_transformation()`, also inside `to_branch()`, with `sortBy()`, `aggregate()`, `groupBy()->aggregate()`, `pivot()`, window functions, `collect()`, `join()` - see [transformations](../core/transformations.md)) buffer the stream and deliver it when the run ends; `to_pgsql_transaction()` opens one final transaction around that delivery - the whole drained stream commits -atomically, a failure during it rolls back and surfaces from `run()`. A plain loader child is a bare sink root; a -`to_transformation(...)` child delivers inside the same transaction. Every sink's loader must use the same `Client` +atomically, a failure during it rolls back and surfaces from `run()`. A plain loader child is written directly; a +`to_transformation(...)` child delivers inside the same transaction. Every child's loader must use the same `Client` instance as the transaction - a loader holding its own `Client` escapes it. -Inside the transaction every sink runs throw-only: a failure rolls the batch back first, and only then the frame's +Inside the transaction every child runs throw-only: a failure rolls the batch back first, and only then the frame's `onError()` handler decides whether the run continues. To set the isolation level, build the transaction yourself; it applies to every transaction opened, including the final one: ```php -use Flow\ETL\Adapter\PostgreSql\PostgreSqlTransaction; -use Flow\ETL\Sink\Transactional; +use Flow\ETL\Adapter\PostgreSql\IO\Loader\PostgreSqlTransaction; +use Flow\ETL\API\Transactional; use Flow\PostgreSql\QueryBuilder\Transaction\IsolationLevel; use function Flow\ETL\Adapter\PostgreSql\to_pgsql_table; @@ -467,13 +467,13 @@ new Transactional( ## Loader DSL Functions Reference -| Function | Description | -|--------------------------------------------|--------------------------------------------------| -| `to_pgsql_table($client, $table)` | Create a PostgreSQL loader for a table | -| `to_pgsql_transaction($client, ...$sinks)` | Write sinks, every delivery inside a transaction | -| `pgsql_insert_options(...)` | Configure insert behavior (conflicts, upsert) | -| `pgsql_update_options($primaryKeys)` | Configure update behavior (primary key columns) | -| `pgsql_delete_options($primaryKeys)` | Configure delete behavior (primary key columns) | +| Function | Description | +|---------------------------------------------|-----------------------------------------------------| +| `to_pgsql_table($client, $table)` | Create a PostgreSQL loader for a table | +| `to_pgsql_transaction($client, ...$writes)` | Write children, every delivery inside a transaction | +| `pgsql_insert_options(...)` | Configure insert behavior (conflicts, upsert) | +| `pgsql_update_options($primaryKeys)` | Configure update behavior (primary key columns) | +| `pgsql_delete_options($primaryKeys)` | Configure delete behavior (primary key columns) | ## Schema Conversion @@ -493,7 +493,7 @@ and explicit type overrides - are driven by `PostgreSqlMetadata` entries attache table: ```php -use Flow\ETL\Adapter\PostgreSql\PostgreSqlMetadata; +use Flow\ETL\Adapter\PostgreSql\Values\Schema\PostgreSqlMetadata; use function Flow\ETL\Adapter\PostgreSql\to_pgsql_schema_table; use function Flow\ETL\DSL\{bool_schema, int_schema, json_schema, schema, str_schema}; @@ -526,7 +526,7 @@ $table = to_pgsql_schema_table($schema, 'users', 'analytics'); schema DSL helpers. Combine multiple entries with `merge()`: ```php -use Flow\ETL\Adapter\PostgreSql\PostgreSqlMetadata; +use Flow\ETL\Adapter\PostgreSql\Values\Schema\PostgreSqlMetadata; use Flow\PostgreSql\Schema\IdentityGeneration; use function Flow\ETL\DSL\{float_schema, int_schema, schema, str_schema}; @@ -562,7 +562,7 @@ order independently of schema field order - useful, for example, for a keyset-pa column must serve `ORDER BY`: ```php -use Flow\ETL\Adapter\PostgreSql\PostgreSqlMetadata; +use Flow\ETL\Adapter\PostgreSql\Values\Schema\PostgreSqlMetadata; use function Flow\ETL\Adapter\PostgreSql\to_pgsql_schema_table; use function Flow\ETL\DSL\{datetime_schema, int_schema, schema}; @@ -588,7 +588,7 @@ Because each index is tracked under its own metadata key, a single column can be `merge()` to attach more than one: ```php -use Flow\ETL\Adapter\PostgreSql\PostgreSqlMetadata; +use Flow\ETL\Adapter\PostgreSql\Values\Schema\PostgreSqlMetadata; use function Flow\ETL\DSL\{datetime_schema, int_schema, schema}; @@ -629,7 +629,7 @@ Both helpers accept an optional `TypesMap`. Its second constructor argument over column type mapping (the first argument keeps overriding the value-binding types used by the loader): ```php -use Flow\ETL\Adapter\PostgreSql\TypesMap; +use Flow\ETL\Adapter\PostgreSql\Values\Schema\TypesMap; use Flow\PostgreSql\QueryBuilder\Schema\ColumnType; use Flow\Types\Type\Native\StringType; diff --git a/documentation/components/adapters/seal.md b/documentation/components/adapters/seal.md index beb4c8b4cc..562cd49992 100644 --- a/documentation/components/adapters/seal.md +++ b/documentation/components/adapters/seal.md @@ -136,7 +136,7 @@ Each row becomes one document: field per index - name it with the `identifier` argument or mark a definition with `SealMetadata::identifier()`. ```php -use Flow\ETL\Adapter\Seal\SealMetadata; +use Flow\ETL\Adapter\Seal\Values\Schema\SealMetadata; use function Flow\ETL\Adapter\Seal\to_seal_schema; use function Flow\ETL\DSL\{datetime_schema, float_schema, int_schema, schema, str_schema}; diff --git a/documentation/components/adapters/xml.md b/documentation/components/adapters/xml.md index 7cf11cc085..cdaa30302f 100644 --- a/documentation/components/adapters/xml.md +++ b/documentation/components/adapters/xml.md @@ -124,7 +124,7 @@ own escaping. A writer that only builds nodes delegates to `write()`: $_ENV['AWS_S3_REGION'], - 'accessKeyId' => $_ENV['AWS_S3_KEY'], - 'accessKeySecret' => $_ENV['AWS_S3_SECRET'], - ]) -); +$aws = aws_s3_filesystem($_ENV['AWS_S3_BUCKET'], [ + 'region' => $_ENV['AWS_S3_REGION'], + 'accessKeyId' => $_ENV['AWS_S3_KEY'], + 'accessKeySecret' => $_ENV['AWS_S3_SECRET'], +]); $fstab = fstab($aws); ``` diff --git a/documentation/components/bridges/filesystem-azure-bridge.md b/documentation/components/bridges/filesystem-azure-bridge.md index e21c3e9b4d..b392ba7d6c 100644 --- a/documentation/components/bridges/filesystem-azure-bridge.md +++ b/documentation/components/bridges/filesystem-azure-bridge.md @@ -16,6 +16,24 @@ For detailed installation instructions, see the [installation page](/documentati ## Usage +The plain form takes the account, the container and the shared key and builds the service itself: + +```php + $_ENV['AZURE_ACCOUNT'], + 'container' => $_ENV['AZURE_CONTAINER'], + 'shared_key' => $_ENV['AZURE_SHARED_KEY'], + // 'url' => ['azurite' => ['host' => '127.0.0.1', 'port' => '10000', 'https' => false]], +]); +``` + +A filesystem built from a `BlobServiceInterface` you assembled yourself - your own HTTP client, factories or logger - +stays in this process and reads serially. + > [!NOTE] > Since the Azure SDK is not providing any http client or factories, you need to install them manually. > The following example uses the `php-http/discovery` package to find the factories in your project existing dependencies. diff --git a/documentation/components/bridges/filesystem-sftp-bridge.md b/documentation/components/bridges/filesystem-sftp-bridge.md index 71dc584578..9e274dd914 100644 --- a/documentation/components/bridges/filesystem-sftp-bridge.md +++ b/documentation/components/bridges/filesystem-sftp-bridge.md @@ -18,7 +18,25 @@ For detailed installation instructions, see the [installation page](/documentati ## Opening a connection -The bridge never handles credentials itself, it takes an already authenticated phpseclib client. +The plain form takes the host, the user and one credential, and logs in on first use: + +```php +use function Flow\Filesystem\Bridge\SFTP\DSL\sftp_filesystem; + +// password authentication +$sftp = sftp_filesystem(['host' => $_ENV['SFTP_HOST'], 'user' => $_ENV['SFTP_USER'], 'password' => $_ENV['SFTP_PASSWORD']]); + +// key based authentication, port 2222 +$sftp = sftp_filesystem([ + 'host' => $_ENV['SFTP_HOST'], + 'user' => $_ENV['SFTP_USER'], + 'private_key' => \file_get_contents($_ENV['SFTP_PRIVATE_KEY']), + 'passphrase' => $_ENV['SFTP_PASSPHRASE'] ?? null, + 'port' => 2222, +]); +``` + +A filesystem built from an already authenticated phpseclib `SFTP` client stays in this process and reads serially. `sftp_client()` covers the common cases and returns phpseclib's own `SFTP` object, so you can configure it further before handing it over. diff --git a/documentation/components/bridges/openapi-specification-bridge.md b/documentation/components/bridges/openapi-specification-bridge.md index dc60a9d23f..59c54d5893 100644 --- a/documentation/components/bridges/openapi-specification-bridge.md +++ b/documentation/components/bridges/openapi-specification-bridge.md @@ -219,7 +219,7 @@ The OpenAPI bridge uses a priority system when handling metadata: ```php read(from_parquet(__DIR__ . '/reports/orders.parquet')) + ->filter(ref('status')->equals(lit('paid'))) + ->select('id', 'total'); + + return http_stream_open($orders) + ->as('paid_orders.csv') + ->streamedResponse(http_csv_output()); + } +} +``` + ## Available Outputs - `Flow\Bridge\Symfony\HttpFoundation\Output\CSVOutput` - `http_csv_output()` - converts dataset to CSV format. @@ -89,7 +125,7 @@ To do that, FlowStreamedResponse allows passing a Transformation that will be ap ```php config(Config::builder()->analyze(analyze())) + ->config(config_builder()->analyze(analyze())) ->onComplete(http_on_complete(function (?Report $report) : void { if ($report === null) { return; @@ -169,10 +204,10 @@ The `analyze()` function supports additional options for collecting more detaile ```php ignore // Collect schema information along with basic statistics -->config(Config::builder()->analyze(analyze()->withSchema())) +->config(config_builder()->analyze(analyze()->withSchema())) // Collect everything -->config(Config::builder()->analyze(analyze()->withSchema()->withColumnStatistics())) +->config(config_builder()->analyze(analyze()->withSchema()->withColumnStatistics())) ``` ### Report Contents diff --git a/documentation/components/bridges/symfony-postgresql-bundle.md b/documentation/components/bridges/symfony-postgresql-bundle.md index ccafe40d64..b360c54143 100644 --- a/documentation/components/bridges/symfony-postgresql-bundle.md +++ b/documentation/components/bridges/symfony-postgresql-bundle.md @@ -312,17 +312,24 @@ Annotate a class implementing `CatalogProvider` with `#[AsCatalogProvider]` for namespace App\Database; use Flow\Bridge\Symfony\PostgreSqlBundle\Attribute\AsCatalogProvider; -use Flow\PostgreSql\Catalog\Catalog; -use Flow\PostgreSql\Catalog\CatalogProvider; +use Flow\PostgreSql\Schema\Catalog; +use Flow\PostgreSql\Schema\CatalogProvider; + +use function Flow\PostgreSql\DSL\{schema, schema_column_integer, schema_column_text, schema_table}; #[AsCatalogProvider] final class UsersCatalogProvider implements CatalogProvider { public function get(): Catalog { - return Catalog::create( - // Define your tables, columns, indexes... - ); + return new Catalog([ + schema('public', [ + schema_table('users', [ + schema_column_integer('id', nullable: false), + schema_column_text('email'), + ]), + ]), + ]); } } ``` diff --git a/documentation/components/cli/docs.md b/documentation/components/cli/docs.md index 67ef00422d..7e70f6f1f7 100644 --- a/documentation/components/cli/docs.md +++ b/documentation/components/cli/docs.md @@ -110,7 +110,7 @@ Options: --output-file-format=OUTPUT-FILE-FORMAT File format. When not set file format is guessed from output file path extension --output-overwrite[=OUTPUT-OVERWRITE] When set output file will be overwritten if exists --analyze[=ANALYZE] Collect processing statistics and print them. [default: false] - --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Config + --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Runtime\Config --input-json-pointer=INPUT-JSON-POINTER JSON Pointer to a subtree from which schema should be extracted --input-json-pointer-column-name When set, JSON Pointer will be used as a column name in the schema --output-json-date-time-format=OUTPUT-JSON-DATE-TIME-FORMAT Date time format used in JSON output @@ -128,7 +128,6 @@ Options: --output-csv-date-time-format=OUTPUT-CSV-DATE-TIME-FORMAT DateTime format for CSV output --input-excel-header[=INPUT-EXCEL-HEADER] When set, Excel header will be used as a schema --input-excel-sheet-name=INPUT-EXCEL-SHEET-NAME When set, Excel sheet name will be selected for reading - --input-excel-offset=INPUT-EXCEL-OFFSET Offset to start reading from --input-xml-node-path=INPUT-XML-NODE-PATH XML node path to a subtree from which schema should be extracted, for example /root/element This is not xpath, just a node names separated by slash --input-xml-buffer-size=INPUT-XML-BUFFER-SIZE XML buffer size in bytes --output-xml-root-element=OUTPUT-XML-ROOT-ELEMENT Root element name in XML output @@ -136,7 +135,6 @@ Options: --output-xml-attribute-prefix=OUTPUT-XML-ATTRIBUTE-PREFIX Attribute prefix in XML output --output-xml-date-time-format=OUTPUT-XML-DATE-TIME-FORMAT Date time format used in XML output --input-parquet-columns=INPUT-PARQUET-COLUMNS Columns to read from parquet file (multiple values allowed) - --input-parquet-offset=INPUT-PARQUET-OFFSET Offset to start reading from --schema-sample-size=SCHEMA-SAMPLE-SIZE Rows read to infer the schema before it is frozen; -1 reads every row. Ignored when the source declares its own schema. --schema-files-to-sniff=SCHEMA-FILES-TO-SNIFF Sources that yielded a row count against this bound; -1 sniffs every source (leading empty files are all opened). Ignored when the source declares its own schema. --schema-all-strings Infer every column as string instead of narrowing it. @@ -172,7 +170,7 @@ Options: --output-php Print schema as PHP code --output-table Print schema as ascii table --output-ascii Print schema as ascii list - --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Config + --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Runtime\Config --input-json-pointer=INPUT-JSON-POINTER JSON Pointer to a subtree from which schema should be extracted --input-json-pointer-column-name When set, JSON Pointer will be used as a column name in the schema --input-csv-header[=INPUT-CSV-HEADER] When set, CSV header will be used as a schema @@ -182,11 +180,9 @@ Options: --input-csv-escape=INPUT-CSV-ESCAPE CSV escape character --input-excel-header[=INPUT-EXCEL-HEADER] When set, Excel header will be used as a schema --input-excel-sheet-name=INPUT-EXCEL-SHEET-NAME When set, Excel sheet name will be selected for reading - --input-excel-offset=INPUT-EXCEL-OFFSET Offset to start reading from --input-xml-node-path=INPUT-XML-NODE-PATH XML node path to a subtree from which schema should be extracted, for example /root/element This is not xpath, just a node names separated by slash --input-xml-buffer-size=INPUT-XML-BUFFER-SIZE XML buffer size in bytes --input-parquet-columns=INPUT-PARQUET-COLUMNS Columns to read from parquet file (multiple values allowed) - --input-parquet-offset=INPUT-PARQUET-OFFSET Offset to start reading from --schema-sample-size=SCHEMA-SAMPLE-SIZE Rows read to infer the schema before it is frozen; -1 reads every row. Ignored when the source declares its own schema. --schema-files-to-sniff=SCHEMA-FILES-TO-SNIFF Sources that yielded a row count against this bound; -1 sniffs every source (leading empty files are all opened). Ignored when the source declares its own schema. --schema-all-strings Infer every column as string instead of narrowing it. @@ -241,7 +237,7 @@ Options: --input-file-batch-size=INPUT-FILE-BATCH-SIZE Number of rows processed in one batch after reading, when set to -1 whole dataset will be displayed at once [default: 1000] --input-file-limit=INPUT-FILE-LIMIT Limit number of rows read from the file. --input-file-offset=INPUT-FILE-OFFSET Number of rows to skip before starting to read data - --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Config + --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Runtime\Config --input-json-pointer=INPUT-JSON-POINTER JSON Pointer to a subtree from which schema should be extracted --input-json-pointer-column-name When set, JSON Pointer will be used as a column name in the schema --input-csv-header[=INPUT-CSV-HEADER] When set, CSV header will be used as a schema @@ -251,11 +247,9 @@ Options: --input-csv-escape=INPUT-CSV-ESCAPE CSV escape character --input-excel-header[=INPUT-EXCEL-HEADER] When set, Excel header will be used as a schema --input-excel-sheet-name=INPUT-EXCEL-SHEET-NAME When set, Excel sheet name will be selected for reading - --input-excel-offset=INPUT-EXCEL-OFFSET Offset to start reading from --input-xml-node-path=INPUT-XML-NODE-PATH XML node path to a subtree from which schema should be extracted, for example /root/element This is not xpath, just a node names separated by slash --input-xml-buffer-size=INPUT-XML-BUFFER-SIZE XML buffer size in bytes --input-parquet-columns=INPUT-PARQUET-COLUMNS Columns to read from parquet file (multiple values allowed) - --input-parquet-offset=INPUT-PARQUET-OFFSET Offset to start reading from --schema-sample-size=SCHEMA-SAMPLE-SIZE Rows read to infer the schema before it is frozen; -1 reads every row. Ignored when the source declares its own schema. --schema-files-to-sniff=SCHEMA-FILES-TO-SNIFF Sources that yielded a row count against this bound; -1 sniffs every source (leading empty files are all opened). Ignored when the source declares its own schema. --schema-all-strings Infer every column as string instead of narrowing it. @@ -293,7 +287,7 @@ Options: --input-file-offset=INPUT-FILE-OFFSET Number of rows to skip before starting to read data --output-truncate=OUTPUT-TRUNCATE Truncate output to given number of characters, when set to -1 output is not truncated at all [default: 20] --output-columns=OUTPUT-COLUMNS Columns to include in output, when not set all columns are displayed (multiple values allowed) - --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Config + --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Runtime\Config --input-json-pointer=INPUT-JSON-POINTER JSON Pointer to a subtree from which schema should be extracted --input-json-pointer-column-name When set, JSON Pointer will be used as a column name in the schema --input-csv-header[=INPUT-CSV-HEADER] When set, CSV header will be used as a schema @@ -303,11 +297,9 @@ Options: --input-csv-escape=INPUT-CSV-ESCAPE CSV escape character --input-excel-header[=INPUT-EXCEL-HEADER] When set, Excel header will be used as a schema --input-excel-sheet-name=INPUT-EXCEL-SHEET-NAME When set, Excel sheet name will be selected for reading - --input-excel-offset=INPUT-EXCEL-OFFSET Offset to start reading from --input-xml-node-path=INPUT-XML-NODE-PATH XML node path to a subtree from which schema should be extracted, for example /root/element This is not xpath, just a node names separated by slash --input-xml-buffer-size=INPUT-XML-BUFFER-SIZE XML buffer size in bytes --input-parquet-columns=INPUT-PARQUET-COLUMNS Columns to read from parquet file (multiple values allowed) - --input-parquet-offset=INPUT-PARQUET-OFFSET Offset to start reading from --schema-sample-size=SCHEMA-SAMPLE-SIZE Rows read to infer the schema before it is frozen; -1 reads every row. Ignored when the source declares its own schema. --schema-files-to-sniff=SCHEMA-FILES-TO-SNIFF Sources that yielded a row count against this bound; -1 sniffs every source (leading empty files are all opened). Ignored when the source declares its own schema. --schema-all-strings Infer every column as string instead of narrowing it. @@ -339,12 +331,11 @@ Options: --input-file-format=INPUT-FILE-FORMAT Source file format. When not set file format is guessed from source file path extension --input-file-limit=INPUT-FILE-LIMIT Limit number of rows that are going to be used to infer file schema, when not set whole file is analyzed --input-file-offset=INPUT-FILE-OFFSET Number of rows to skip before starting to read data - --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Config + --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Runtime\Config --input-json-pointer=INPUT-JSON-POINTER JSON Pointer to a subtree from which schema should be extracted --input-json-pointer-column-name When set, JSON Pointer will be used as a column name in the schema --input-excel-header[=INPUT-EXCEL-HEADER] When set, Excel header will be used as a schema --input-excel-sheet-name=INPUT-EXCEL-SHEET-NAME When set, Excel sheet name will be selected for reading - --input-excel-offset=INPUT-EXCEL-OFFSET Offset to start reading from --input-csv-header[=INPUT-CSV-HEADER] When set, CSV header will be used as a schema --input-csv-empty-to-null[=INPUT-CSV-EMPTY-TO-NULL] When set, empty CSV values will be treated as NULL values --input-csv-separator=INPUT-CSV-SEPARATOR CSV separator character @@ -353,7 +344,6 @@ Options: --input-xml-node-path=INPUT-XML-NODE-PATH XML node path to a subtree from which schema should be extracted, for example /root/element This is not xpath, just a node names separated by slash --input-xml-buffer-size=INPUT-XML-BUFFER-SIZE XML buffer size in bytes --input-parquet-columns=INPUT-PARQUET-COLUMNS Columns to read from parquet file (multiple values allowed) - --input-parquet-offset=INPUT-PARQUET-OFFSET Offset to start reading from -h, --help Display help for the given command. When no command is given display help for the list command --silent Do not output any message -q, --quiet Only errors are displayed. All other output is suppressed @@ -407,7 +397,7 @@ Arguments: Options: --analyze[=ANALYZE] Collect processing statistics and print them. [default: false] - --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Config + --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Runtime\Config --stats-schema[=STATS-SCHEMA] Prints schema of executed data transformation pipeline. [default: false] --stats-columns[=STATS-COLUMNS] Prints number of rows in dataset. [default: false] --stats-sources[=STATS-SOURCES] Prints the rows every source declared next to the rows it yielded. [default: false] @@ -458,7 +448,7 @@ Options: --output-table Print schema as ascii table --output-ascii Print schema as ascii list --db-column[=DB-COLUMN] Filter schema by column name(s) (multiple values allowed) - --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Config + --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Runtime\Config -c, --db-connection-file=DB-CONNECTION-FILE Path to file that returns and instance of \Doctrine\DBAL\Connection -h, --help Display help for the given command. When no command is given display help for the list command --silent Do not output any message @@ -485,7 +475,7 @@ Usage: Options: --db-namespace=DB-NAMESPACE List of namespaces for which this command will list tables, multiple values allowed. When not set, all tables from all namespaces are listed. (multiple values allowed) - --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Config + --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Runtime\Config -c, --db-connection-file=DB-CONNECTION-FILE Path to file that returns and instance of \Doctrine\DBAL\Connection -h, --help Display help for the given command. When no command is given display help for the list command --silent Do not output any message @@ -521,7 +511,7 @@ Options: --output-php Print schema as PHP code --output-table Print schema as ascii table --output-ascii Print schema as ascii list - --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Config + --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Runtime\Config -h, --help Display help for the given command. When no command is given display help for the list command --silent Do not output any message -q, --quiet Only errors are displayed. All other output is suppressed diff --git a/documentation/components/core/aggregations.md b/documentation/components/core/aggregations.md index 2ec311a3fb..fb70c15e5a 100644 --- a/documentation/components/core/aggregations.md +++ b/documentation/components/core/aggregations.md @@ -8,21 +8,90 @@ Each group created by `groupBy` function can be aggregated using one of the aggr ## Aggregation functions -- [`COUNT()`](/src/core/etl/src/Flow/ETL/Function/Count.php) -- [`AVERAGE()`](/src/core/etl/src/Flow/ETL/Function/Average.php) -- [`SUM()`](/src/core/etl/src/Flow/ETL/Function/Sum.php) -- [`MIN()`](/src/core/etl/src/Flow/ETL/Function/Min.php) -- [`MAX()`](/src/core/etl/src/Flow/ETL/Function/Max.php) -- [`COLLECT()`](/src/core/etl/src/Flow/ETL/Function/Collect.php) -- [`COLLECT_UNIQUE()`](/src/core/etl/src/Flow/ETL/Function/CollectUnique.php) -- [`FIRST()`](/src/core/etl/src/Flow/ETL/Function/First.php) -- [`LAST()`](/src/core/etl/src/Flow/ETL/Function/Last.php) -- [`STRING_AGG()`](/src/core/etl/src/Flow/ETL/Function/StringAggregate.php) - -All aggregation functions implement [`AggregatingFunction`](/src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php): -`aggregate(Rows $rows, array $indices, FlowContext $context)` folds the rows of one group (`$indices` of the batch) -into the aggregator's state, and `merge()` folds in the partial state of another aggregator of the same class, which -group by uses after it spilled. +- [`COUNT()`](/src/core/etl/src/Flow/ETL/Functions/Count.php) +- [`AVERAGE()`](/src/core/etl/src/Flow/ETL/Functions/Average.php) +- [`SUM()`](/src/core/etl/src/Flow/ETL/Functions/Sum.php) +- [`MIN()`](/src/core/etl/src/Flow/ETL/Functions/Min.php) +- [`MAX()`](/src/core/etl/src/Flow/ETL/Functions/Max.php) +- [`COLLECT()`](/src/core/etl/src/Flow/ETL/Functions/Collect.php) +- [`COLLECT_UNIQUE()`](/src/core/etl/src/Flow/ETL/Functions/CollectUnique.php) +- [`FIRST()`](/src/core/etl/src/Flow/ETL/Functions/First.php) +- [`LAST()`](/src/core/etl/src/Flow/ETL/Functions/Last.php) +- [`STRING_AGG()`](/src/core/etl/src/Flow/ETL/Functions/StringAggregate.php) + +All aggregation functions implement [`AggregatingFunction`](/src/core/etl/src/Flow/ETL/Functions/AggregatingFunction.php). +`aggregate(Rows $rows, array $indices, Backend $backend)` folds the rows of one group into the state. +`merge(array $partial, Backend $backend)` folds in the `partial()` of another aggregator of the same function. +Group by merges after it spills. + +## Custom aggregators + +A custom aggregator also describes itself (it is rebuilt in a worker from `describe()`) and exposes its state as a +partial, the plain values a worker hands back for the parent or a later task to merge: + +```php +of($data['ref'], Reference::class)); + } + + public function describe(): ?Description + { + return Description::compose(self::class, ['ref' => $this->ref]); + } + + public function aggregate(Rows $rows, array $indices, Backend $backend): void + { + $column = $rows->column($this->ref->to()); + + foreach ($indices as $index) { + $value = $column->value($index); + + if (is_string($value) && strlen($value) > strlen($this->longest ?? '')) { + $this->longest = $value; + } + } + } + + public function partialType(): StructureType + { + return type_structure(['longest' => type_optional(type_string())]); + } + + public function partial(): array + { + return ['longest' => $this->longest]; + } + + public function merge(array $partial, Backend $backend): void + { + $longest = $this->partialType()->assert($partial)['longest']; + + if ($longest !== null && strlen($longest) > strlen($this->longest ?? '')) { + $this->longest = $longest; + } + } + + public function withChildren(array $children): static + { + return new self($children[0]); // a fresh instance: no state crosses + } + + // children(), resolved(), deterministic(), references(), returns(), outputName(), value() as before +} +``` + +`partialType()` is valid once the aggregator is resolved; `partial()` of a fresh instance is the initial state, and +merging it is a no-op. Partials of two tasks merge in task order: an order-sensitive aggregator (`first()`, +`last()`, `collect()`) relies on it. A `describe()` returning `null` keeps the group by in the parent process. ## Float arithmetic @@ -43,7 +112,7 @@ df()->read(from_array([['a' => 0.1], ['a' => 0.2]]))->aggregate([sum(ref('a'), e ```php **Performance Tip**: Optimal batch size depends on your data and available memory. Larger batches reduce I/O > operations but increase memory usage. Start with 1000-5000 rows and adjust based on your specific use case. -### Source, re-slice, pipeline - three batch sizes +### Source and pipeline - two batch sizes -`withBatchSize()` bounds what a source **builds**; `batches()` re-slices what a source **already emitted** and -cannot lower its peak; `batchSize()` re-batches the pipeline **after** the source. +`withBatchSize()` bounds what a source **builds**; `batchSize()` re-batches the pipeline **after** the source. ```php read(from_all( - from_csv('orders.csv')->withBatchSize(500), // the source builds batches of at most 500 rows (default 100) - batches(from_json('orders.json'), 50), // re-sliced after the source built its own batches + from_csv('orders.csv')->withBatchSize(500), // the source builds batches of at most 500 rows (default 100) + from_json('orders.json')->withBatchSize(50), // the source builds batches of at most 50 rows )) - ->batchSize(1000) // re-batched after reading; the tip above is about this one + ->batchSize(1000) // re-batched after reading; the tip above is about this one ->write(to_output()) ->run(); ``` diff --git a/documentation/components/core/building-blocks.md b/documentation/components/core/building-blocks.md index 4a5431fe21..857d885955 100644 --- a/documentation/components/core/building-blocks.md +++ b/documentation/components/core/building-blocks.md @@ -3,8 +3,8 @@ [DOC_LINK:/documentation/components/core/core.md] Columns of the [Data Frame](/documentation/components/core/core.md) are described by the -[Schema](/src/core/etl/src/Flow/ETL/Schema.php) - it owns their names, order, types and nullability. -A batch of records is represented by the [Rows](/src/core/etl/src/Flow/ETL/Rows.php) class: one column per +[Schema](/src/core/etl/src/Flow/ETL/Values/Schema.php) - it owns their names, order, types and nullability. +A batch of records is represented by the [Rows](/src/core/etl/src/Flow/ETL/Values/Rows.php) class: one column per schema definition, and every `Rows` carries the one `Schema` that describes it. Let's look at the following example: @@ -36,31 +36,31 @@ Extractors yield `Rows`, transformers return new `Rows`, loaders write them. A r ## Column Types -Every column is described by a [Definition](/src/core/etl/src/Flow/ETL/Schema/Definition.php), built with +Every column is described by a [Definition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition.php), built with the matching `*_schema()` [DSL function](/src/core/etl/src/Flow/ETL/DSL/functions.php). A definition owns the column name, its [Flow Type](/documentation/components/libs/types.md), nullability and metadata. -| Column | DSL function | Definition | -|--------------|------------------------------------|-------------------------------------------------------------------------------------------------| -| Boolean | `bool_schema()` | [BooleanDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/BooleanDefinition.php) | -| Date | `date_schema()` | [DateDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/DateDefinition.php) | -| DateTime | `datetime_schema()` | [DateTimeDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/DateTimeDefinition.php) | -| Enum | `enum_schema()` | [EnumDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/EnumDefinition.php) | -| Float | `float_schema()` | [FloatDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/FloatDefinition.php) | -| HTML | `html_schema()` | [HTMLDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/HTMLDefinition.php) | -| HTML Element | `html_element_schema()` | [HTMLElementDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/HTMLElementDefinition.php) | -| Integer | `int_schema()`, `integer_schema()` | [IntegerDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/IntegerDefinition.php) | -| Json | `json_schema()` | [JsonDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/JsonDefinition.php) | -| List | `list_schema()` | [ListDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/ListDefinition.php) | -| Map | `map_schema()` | [MapDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/MapDefinition.php) | -| Null | `null_schema()` | [NullDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/NullDefinition.php) | -| String | `str_schema()`, `string_schema()` | [StringDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/StringDefinition.php) | -| Structure | `structure_schema()` | [StructureDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/StructureDefinition.php) | -| Time | `time_schema()` | [TimeDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/TimeDefinition.php) | -| Time Zone | `time_zone_schema()` | [TimeZoneDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/TimeZoneDefinition.php) | -| Uuid | `uuid_schema()` | [UuidDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/UuidDefinition.php) | -| XML | `xml_schema()` | [XMLDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/XMLDefinition.php) | -| XML Element | `xml_element_schema()` | [XMLElementDefinition](/src/core/etl/src/Flow/ETL/Schema/Definition/XMLElementDefinition.php) | +| Column | DSL function | Definition | +|--------------|------------------------------------|--------------------------------------------------------------------------------------------------------| +| Boolean | `bool_schema()` | [BooleanDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/BooleanDefinition.php) | +| Date | `date_schema()` | [DateDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/DateDefinition.php) | +| DateTime | `datetime_schema()` | [DateTimeDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/DateTimeDefinition.php) | +| Enum | `enum_schema()` | [EnumDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/EnumDefinition.php) | +| Float | `float_schema()` | [FloatDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/FloatDefinition.php) | +| HTML | `html_schema()` | [HTMLDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/HTMLDefinition.php) | +| HTML Element | `html_element_schema()` | [HTMLElementDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/HTMLElementDefinition.php) | +| Integer | `int_schema()`, `integer_schema()` | [IntegerDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/IntegerDefinition.php) | +| Json | `json_schema()` | [JsonDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/JsonDefinition.php) | +| List | `list_schema()` | [ListDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/ListDefinition.php) | +| Map | `map_schema()` | [MapDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/MapDefinition.php) | +| Null | `null_schema()` | [NullDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/NullDefinition.php) | +| String | `str_schema()`, `string_schema()` | [StringDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/StringDefinition.php) | +| Structure | `structure_schema()` | [StructureDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/StructureDefinition.php) | +| Time | `time_schema()` | [TimeDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/TimeDefinition.php) | +| Time Zone | `time_zone_schema()` | [TimeZoneDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/TimeZoneDefinition.php) | +| Uuid | `uuid_schema()` | [UuidDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/UuidDefinition.php) | +| XML | `xml_schema()` | [XMLDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/XMLDefinition.php) | +| XML Element | `xml_element_schema()` | [XMLElementDefinition](/src/core/etl/src/Flow/ETL/Values/Schema/Definition/XMLElementDefinition.php) | A `datetime` column has one time zone, UTC by default: `datetime_schema('at', zone: 'Europe/Warsaw')`. A value in any other zone is converted to it when the row is admitted, the instant unchanged. diff --git a/documentation/components/core/caching.md b/documentation/components/core/caching.md index 829084c6a0..d721e3bf99 100644 --- a/documentation/components/core/caching.md +++ b/documentation/components/core/caching.md @@ -4,13 +4,11 @@ ## Cache -The goal of cache is to serialize and save on disk (or in another location defined by Cache implementation) +The goal of cache is to serialize and save on disk (or in another location defined by the cache storage) already transformed dataset. -Cache runs the pipeline and saves each batch (`Rows`) into the cache, from where it can be read later. - -Each batch is stored under its own cache key, and the pipeline's cache id points at an index - a batch with `key` -and `rows` columns listing those chunk keys and their row counts in insertion order. +Cache runs the pipeline and appends each batch (`Rows`) under the cache id, from where it can be read later in the +same order. Another interesting use case for caching would be to share the dataset between multiple data processing pipelines. So instead of going to datasource multiple times and then repeating all transformations, only one ETL would @@ -45,11 +43,9 @@ data_frame() 3 rows ``` -By default, Flow is using Filesystem Cache, location of the cache storage can be adjusted through -the `FLOW_LOCAL_FILESYSTEM_CACHE_DIR` environment variable. - -To only move the cache directory, set it on the config - the default cache keeps the pipeline's serializer -and backend: +By default, Flow writes the cache as [Floe](/documentation/components/core/floe.md) files on the local filesystem, in +the pipeline's backend. The location can be adjusted through the `FLOW_LOCAL_FILESYSTEM_CACHE_DIR` environment +variable, or on the config: ```php cacheDir(__DIR__ . '/var/cache'); ``` -To use different cache implementation please use `ConfigBuilder` +To use a different cache, pass a `RowsStorage` to `ConfigBuilder`: ```php cache( - new PSRSimpleCache( + new PsrRowsStorage( new Psr16Cache(new ArrayAdapter()), new FloeSerializer(new AdaptiveBackend()), ) @@ -83,28 +79,18 @@ config_builder() The following implementations are available out of the box: -* [InMemory](/src/core/etl/src/Flow/ETL/Cache/Implementation/InMemoryCache.php) -* [LocalFilesystem](/src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php) -* [PSRSimpleCache](/src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php) -* [ApcuCache](/src/core/etl/src/Flow/ETL/Cache/Implementation/ApcuCache.php) - `new ApcuCache($serializer, $namespace)`, +* [MemoryRowsStorage](/src/core/etl/src/Flow/ETL/Storage/MemoryRowsStorage.php) +* [FilesystemRowsStorage](/src/core/etl/src/Flow/ETL/Storage/FilesystemRowsStorage.php) - the default, + `filesystem_cache($dir, $filesystem, $backend)` +* [PsrRowsStorage](/src/core/etl/src/Flow/ETL/Storage/PsrRowsStorage.php) - any + [psr/simple-cache implementation](https://packagist.org/providers/psr/simple-cache-implementation) +* [ApcuRowsStorage](/src/core/etl/src/Flow/ETL/Storage/ApcuRowsStorage.php) - `new ApcuRowsStorage($serializer, $namespace)`, requires `ext-apcu` -PSRSimpleCache makes possible to use any of -the [psr/simple-cache-implementation](https://packagist.org/providers/psr/simple-cache-implementation) -but it does not come with any out of the box. - -## Serialization +## Batches and Serialization -Persisting caches (`FilesystemCache`, `PSRSimpleCache`, `ApcuCache`) turn `Rows` into bytes through a -[`Serializer`](/src/core/etl/src/Flow/Serializer/Serializer.php). The default is -[`FloeSerializer`](/src/core/etl/src/Flow/Floe/FloeSerializer.php), which writes each batch as one -[Floe](/documentation/components/core/floe.md) BATCH frame and reads it back into the -[column backend](/documentation/components/core/column-backend.md) it was built with - -`new FloeSerializer($backend, $batchSize = 1000)`. - -`Cache::get()` returns one fully materialized `Rows` per key. The caching processor writes exactly -one batch per cache key, so each key holds a single batch bounded by `DataFrame::cache(cacheBatchSize:)` -(falling back to `DataFrame::batchSize()`): +`RowsStorage::get()` yields the cached batches one by one. Each one holds at most `DataFrame::cache(cacheBatchSize:)` +rows (falling back to `DataFrame::batchSize()`): ```php run(); ``` -A `Serializer` writes to and reads from streams, so the default `FilesystemCache` streams the payload -directly to and from the cache file. `PSRSimpleCache` materializes the payload string - its PSR-16 -backend stores string values - bounded by `cacheBatchSize`. - -The default `FilesystemCache` shares the serializer of the rest of the pipeline (so it decodes into the configured -backend). Build one with `filesystem_cache()` only for a custom serializer or filesystem - a cache you build takes -its serializer from you: - -```php -/flow-php-rows//.floe`, and +reads it back in the backend it was given. A file without a footer - a write cut short by a killed process - counts +as missing, so the next run writes it again. -`FilesystemCache` stores each entry in a file named after the cache key, with no extension, under hash-prefixed -subdirectories, and its schema the same way under `.schema` - the entry bytes are whatever the configured serializer -produced. \ No newline at end of file +`PsrRowsStorage` and `ApcuRowsStorage` store each batch as a string, made by the +[`Serializer`](/src/core/etl/src/Flow/ETL/Storage/Serializer/Serializer.php) they are given - +[`FloeSerializer`](/src/core/etl/src/Flow/ETL/Storage/Floe/FloeSerializer.php) writes it as one Floe BATCH frame +and reads it back into its [column backend](/documentation/components/core/column-backend.md). diff --git a/documentation/components/core/column-backend.md b/documentation/components/core/column-backend.md index 7ac580e44b..5f8c10fcd1 100644 --- a/documentation/components/core/column-backend.md +++ b/documentation/components/core/column-backend.md @@ -5,7 +5,7 @@ [TOC] Every batch (`Rows`) stores one column per schema definition. The -[Backend](/src/core/etl/src/Flow/ETL/Column/Backend.php) decides where those columns live and builds them. A data +[Backend](/src/core/etl/src/Flow/ETL/Values/Column/Backend.php) decides where those columns live and builds them. A data frame has exactly one backend, set on its config: ```php @@ -13,7 +13,7 @@ frame has exactly one backend, set on its config: declare(strict_types=1); -use Flow\ETL\Column\PhpBackend; +use Flow\ETL\Values\Column\PhpBackend; use function Flow\ETL\DSL\{config_builder, data_frame, from_array, lit, ref}; @@ -22,17 +22,17 @@ $rows = data_frame(config_builder()->backend(new PhpBackend())) ->withColumn('double', ref('id')->multiply(lit(2))) ->fetch(); -$rows->column('double')::class; // Flow\ETL\Column\Php\ScalarColumn +$rows->column('double')::class; // Flow\ETL\Values\Column\Php\ScalarColumn $rows->toArray(); // [['id' => 1, 'double' => 2], ['id' => 2, 'double' => 4]] ``` ## Backends -| Backend | Columns | Requires | -|--------------------------------------------------------------------------|-------------------------------|--------------------------------------------------------------------------------| -| [AdaptiveBackend](/src/core/etl/src/Flow/ETL/Column/AdaptiveBackend.php) | `RustBackend` or `PhpBackend` | nothing (the default) | -| [PhpBackend](/src/core/etl/src/Flow/ETL/Column/PhpBackend.php) | PHP arrays | nothing | -| `RustBackend` | Apache Arrow arrays | the [flow_php extension](/documentation/components/extensions/flow-php-ext.md) | +| Backend | Columns | Requires | +|---------------------------------------------------------------------------------|-------------------------------|--------------------------------------------------------------------------------| +| [AdaptiveBackend](/src/core/etl/src/Flow/ETL/Values/Column/AdaptiveBackend.php) | `RustBackend` or `PhpBackend` | nothing (the default) | +| [PhpBackend](/src/core/etl/src/Flow/ETL/Values/Column/PhpBackend.php) | PHP arrays | nothing | +| `RustBackend` | Apache Arrow arrays | the [flow_php extension](/documentation/components/extensions/flow-php-ext.md) | When `extension_loaded('flow_php')` is true, `AdaptiveBackend` picks `RustBackend`. Else it picks `PhpBackend`. Both produce the same values: `PhpBackend` is the reference implementation, the extension is an accelerator. Set @@ -55,7 +55,7 @@ transformer or function, or any `Backend` to the DSL: declare(strict_types=1); -use Flow\ETL\Column\PhpBackend; +use Flow\ETL\Values\Column\PhpBackend; use function Flow\ETL\DSL\{array_to_rows, int_schema, schema}; diff --git a/documentation/components/core/constraints.md b/documentation/components/core/constraints.md index 25d3a9e998..7ede6b8ee8 100644 --- a/documentation/components/core/constraints.md +++ b/documentation/components/core/constraints.md @@ -49,7 +49,8 @@ You can implement custom constraints by creating classes that implement the `Con ```php explain()->toString(); ```text Outputs -├─ #3 Result +├─ #4 Result │ │ Rows this plan hands out: to the trigger, or to the node reading it -│ └─ #2 Select +│ └─ #3 Select │ │ Keeps only: id, email -│ └─ #1 Read -│ Extractor: CSVExtractor -│ Source: file:///data/orders.csv -│ Filter: email IS NOT NULL -└─ #4 Write +│ └─ #2 Filter +│ │ Condition: email IS NOT NULL +│ └─ #1 Read +│ Extractor: CSVExtractor +│ Source: file:///data/orders.csv +└─ #5 Write │ Loader: JsonLoader - └─ #1 Read (shared) + └─ #2 Filter (shared) ``` ```php @@ -86,23 +87,23 @@ echo data_frame() ``` ```text -#2 Write +#3 Write │ Loader: JsonLoader -└─ #1 Read - Extractor: CSVExtractor - Source: file:///data/orders.csv - Filter: email IS NOT NULL +└─ #2 Filter + │ Condition: email IS NOT NULL + └─ #1 Read + Extractor: CSVExtractor + Source: file:///data/orders.csv ``` -`run()` takes no rows, so its plan carries only the sinks - the `Write` is the root and there is no `Result`. A +`run()` takes no rows, so its plan carries only the writes - the `Write` is the root and there is no `Result`. A `Result` comes back only when a verb follows the last `write()`: nothing would read that chain end, so one is put on it to pull the rows, and `run(analyze: ...)` still counts them. -`Outputs` lists everything the frame produces: `Result` is what a rows-returning trigger reads, `Write` is the sink. -A node several consumers read is printed once, and every other consumer points back at it by number - `#1 Read (shared)` is that same read, not a second one. A node keeps its number in every format. +`Outputs` lists everything the frame produces: `Result` is what a rows-returning trigger reads, `Write` is a write. +A node several consumers read is printed once, and every other consumer points back at it by number - `#2 Filter (shared)` is that same filter, not a second one. A node keeps its number in every format. A frame joined with `join()` / `crossJoin()` is part of the same tree: the join reads it the way it reads any other -input, numbered with the rest of the plan, and it runs with this frame's configuration. A frame read with -`from_data_frame()` is a `Read` of `DataFrameExtractor` and runs with its own. +input, numbered with the rest of the plan, and it runs with this frame's configuration. `toString()` takes the stage and the format to print: @@ -124,13 +125,14 @@ echo $dataFrame->explain()->toString(format: Format::flow); #1 Read │ Extractor: CSVExtractor │ Source: file:///data/orders.csv -│ Filter: email IS NOT NULL -├─ #2 Select -│ │ Keeps only: id, email -│ └─ #3 Result -│ Rows this plan hands out: to the trigger, or to the node reading it -└─ #4 Write - Loader: JsonLoader +└─ #2 Filter + │ Condition: email IS NOT NULL + ├─ #3 Select + │ │ Keeps only: id, email + │ └─ #4 Result + │ Rows this plan hands out: to the trigger, or to the node reading it + └─ #5 Write + Loader: JsonLoader ``` ```php @@ -142,30 +144,36 @@ echo $dataFrame->explain()->toString(format: Format::boxes); │ Outputs ├──────────────┠└─────────────┬─────────────┘ │ ┌─────────────┴─────────────â”┌─────────────┴─────────────┠-│ #3 Result ││ #4 Write │ +│ #4 Result ││ #5 Write │ │ ──────────────────── ││ ──────────────────── │ │ Rows this plan hands out: ││ Loader: JsonLoader │ │ to the trigger, or to the ││ │ │ node reading it ││ │ └─────────────┬─────────────┘└─────────────┬─────────────┘ ┌─────────────┴─────────────â”┌─────────────┴─────────────┠-│ #2 Select ││ #1 Read │ +│ #3 Select ││ #2 Filter │ │ ──────────────────── ││ (shared) │ │ Keeps only: id, email ││ │ └─────────────┬─────────────┘└───────────────────────────┘ ┌─────────────┴─────────────┠+│ #2 Filter │ +│ ──────────────────── │ +│ Condition: email IS NOT │ +│ NULL │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┠│ #1 Read │ │ ──────────────────── │ │ Extractor: CSVExtractor │ │ Source: │ │ file:///data/orders.csv │ -│ Filter: email IS NOT NULL │ └───────────────────────────┘ ``` -`Stage::physical` prints what the executor runs. A pipeline ends where a blocking step cuts it, so the pipeline a -step sits in is where rows stop flowing through, and every step lists the settings it was given - the algorithm's -storage among them, which the logical stages cannot know because the planner picks it. +`Stage::physical` prints what the executor runs. A pipeline ends at a step that needs every row before it emits one +(a sort, a group by, a join, `collect()`), so the pipeline a step sits in is where rows stop flowing through. Every +step lists the settings it was given - the algorithm's storage among them, which the logical stages cannot know +because the planner picks it. ```php $users = [['id' => 1, 'name' => 'Alice'], ['id' => 2, 'name' => 'Bob']]; @@ -173,7 +181,7 @@ $emails = [['id' => 1, 'email' => 'alice@example.com']]; echo data_frame() ->read(from_array($users)) - ->join(data_frame()->read(from_array($emails)), join_on(['id' => 'id'], join_prefix: 'joined_'), Join::left) + ->join(data_frame()->read(from_array($emails)), join_on(['id' => 'id'], join_prefix: 'joined_'), JoinType::left) ->collect() ->write(to_output(truncate: false)) ->explain()->toString(Stage::physical); @@ -182,24 +190,25 @@ echo data_frame() ```text Physical plan │ Columns: id, name, joined_id, joined_email -└─ Pipeline #1 - │ Processor: CollectingProcessor - │ Schema: declared +└─ Pipeline #2 │ Loader: StreamLoader - └─ Pipeline #0 - │ Extractor: ArrayExtractor - │ Statistics: rows exact 2 · size unknown - │ Processor: HashJoinProcessor - │ Join: left - │ On: id = id - │ Prefix: joined_ - │ Storage: FilesystemBuckets - │ Memory: 1000 MB - │ Buckets: 64 - │ Batch: 1000 - └─ Right side: Pipeline #0 - Extractor: ArrayExtractor - Statistics: rows exact 1 · size unknown + └─ Pipeline #1 + │ Processor: CollectingProcessor + │ Schema: declared + └─ Pipeline #0 + │ Extractor: ArrayExtractor + │ Statistics: rows exact 2 · size unknown + │ Join: HashJoin + │ Type: left + │ On: id = id + │ Prefix: joined_ + │ Storage: FilesystemRowsStorage + │ Memory: 1000 MB + │ Buckets: 64 + │ Batch: 1000 + └─ Right side: Pipeline #0 + Extractor: ArrayExtractor + Statistics: rows exact 1 · size unknown ``` `Memory: 1000 MB` is the default limit under `memory_limit = -1`. With a set `memory_limit` it is 70% of it, or @@ -276,13 +285,13 @@ Before a frame runs, the optimizer rewrites its plan. `Optimizer::default()` run | `PushFilterIntoSource` | a `filter()` on partition columns skips whole partition directories | | `CountFromStatistics` | `count()` over a source that knows its rows exactly reads that number, no rows | -Rules live in `Flow\ETL\Optimizer\Rule` and are configured through `config_builder()->optimizer()`: +Rules live in `Flow\ETL\Planning\Optimizer\Rule` and are configured through `config_builder()->optimizer()`: ```php optimizer(new Optimizer())); // the defaults without one rule data_frame(config_builder()->optimizer(Optimizer::default()->without(CombineSortAndLimit::class))); -// the defaults followed by your own Optimizer\Rule implementation +// the defaults followed by your own Rule implementation data_frame(config_builder()->optimizer(Optimizer::default()->with(new MyRule()))); ``` diff --git a/documentation/components/core/data-retrieval.md b/documentation/components/core/data-retrieval.md index 4344f242ea..08b108b64b 100644 --- a/documentation/components/core/data-retrieval.md +++ b/documentation/components/core/data-retrieval.md @@ -113,7 +113,7 @@ $dataFrame->forEach(function (Rows $rows) { ``` `forEach()` owns the streaming loop. `run(bool|Analyze $analyze = false)` takes no callback: it executes the frame's -sinks and returns a `Report` when asked to analyze. +writes and returns a `Report` when asked to analyze. ```php of($data['amount'], ScalarFunction::class), $rebuild->of($data['type'], ScalarFunction::class)); + } + + public function describe(): ?Description + { + return Description::compose(self::class, ['amount' => $this->amount, 'type' => $this->type]); + } + public function children(): array { return [$this->amount, $this->type]; @@ -67,17 +78,17 @@ final class HighValuePurchase implements ScalarFunction return type_boolean(); } - public function eval(Rows $rows, FlowContext $context): Column + public function eval(Rows $rows, Backend $backend): Column { - $amounts = (new Parameter($this->amount))->asNumbers($rows, $context); - $types = (new Parameter($this->type))->asStrings($rows, $context); + $amounts = (new Parameter($this->amount))->asNumbers($rows, $backend); + $types = (new Parameter($this->type))->asStrings($rows, $backend); $results = []; foreach ($amounts as $i => $amount) { $results[] = $amount > 1000 && $types[$i] === 'purchase'; } - return (new ResultColumn($context->backend()))->of($this, $results); + return (new ResultColumn($backend))->of($this, $results); } } diff --git a/documentation/components/core/floe.md b/documentation/components/core/floe.md index 4afa1566ee..453f9b296d 100644 --- a/documentation/components/core/floe.md +++ b/documentation/components/core/floe.md @@ -86,7 +86,7 @@ data_frame() ``` > Floe additionally supports appending into a **single** file through the low-level -> `Flow\Floe\FloeWriter::append()` API - appended batches must match the file's schema (a drifted +> `Flow\ETL\Storage\Floe\FloeFiles::append()` API - appended batches must match the file's schema (a drifted > batch throws `IncompatibleSchemaException`; schema evolution is only available through > `merge_floe()`). Before the check, the appending schema is rewritten into the file's declared > column and structure-field order, so a batch that carries the same columns or the same structure @@ -121,9 +121,9 @@ data_frame() ## Limit & Offset Pushdown -`limit()` stops reading early, and `withOffset()` skips whole file sections using the footer before +`limit()` stops reading early, and `offset()` skips whole file sections using the footer before reading, then passes over every BATCH frame whose row count lies inside the offset without decoding it, so -neither scans the full file: +neither scans the full file. A filter the source takes is applied first, and the offset then counts the rows it keeps: ```php read(from_floe(__DIR__ . '/output.floe')->withOffset(1_000)) + ->read(from_floe(__DIR__ . '/output.floe')) + ->offset(1_000) ->limit(100) ->run(); ``` @@ -150,34 +151,30 @@ use function Flow\Floe\DSL\from_floe; $schema = from_floe(__DIR__ . '/output.floe')->schema(); ``` -## Reading the First or Last Rows +## Reading a Range of Rows -The low-level reader pulls just the head or tail of a single `.floe` file without a DataFrame. `tail()` -reads the row count from the footer and seeks past the leading sections, so it decodes only from the -boundary section onward - the leading rows are never read: +The low-level reader reads a range of a single `.floe` file without a DataFrame. It seeks by the footer's sections, +so it decodes only from the section holding the first requested row onward: ```php read(path(__DIR__ . '/output.floe')); +$file = (new FloeFiles(native_local_filesystem(), new AdaptiveBackend()))->read(path(__DIR__ . '/output.floe')); -foreach ($file->head(300) as $rows) { - // first 300 rows; stops reading once 300 are yielded +foreach ($file->rows(batchSize: 1000, offset: 0, limit: 300) as $rows) { + // the first 300 rows } -foreach ($file->tail(300) as $rows) { - // last 300 rows; decoded from the boundary section onward +foreach ($file->rows(batchSize: 1000, offset: max(0, $file->totalRows() - 300)) as $rows) { + // the last 300 rows } -``` -Both yield `Rows` in batches of at most 1000 rows (override with the second argument) and return every row when -the file holds fewer than the requested count. For the first N through the DataFrame API use `->limit(N)` -(pushed into the extractor); `tail()` is a reader-level convenience because it needs the file's total -from the footer. +$file->close(); +``` ## Merging Files @@ -188,7 +185,7 @@ re-reads and re-writes every batch, coalescing same-schema runs into fewer secti ```php create(path(__DIR__ . '/orders.floe')); +$writer = (new FloeFiles(native_local_filesystem(), new PhpBackend()))->create(path(__DIR__ . '/orders.floe'), schema(int_schema('id'))); $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')), new PhpBackend())); $writer->write(array_to_rows([['id' => 'AB-1']], schema(str_schema('id')), new PhpBackend())); ``` diff --git a/documentation/components/core/group-by.md b/documentation/components/core/group-by.md index d6d3ddfa8f..d8ca8e9ffb 100644 --- a/documentation/components/core/group-by.md +++ b/documentation/components/core/group-by.md @@ -24,8 +24,8 @@ data_frame() +-------+----------+ | group | id_count | +-------+----------+ -| A | 3 | -| B | 2 | +| A | 3 | +| B | 2 | +-------+----------+ 2 rows ``` @@ -35,17 +35,17 @@ data_frame() ## Buckets Storage While the process stays under the memory limit, `groupBy()` aggregates in one pass and writes nothing. Past the limit it -partitions the rest of the stream by the hashed grouping key into buckets of a `BucketsStorage` - the same abstraction +partitions the rest of the stream by the hashed grouping key into buckets of a `RowsStorage` - the same abstraction used by [external sort](/documentation/components/core/sort.md) and [join](/documentation/components/core/join.md) - aggregates each bucket on its own and merges the groups aggregated before the limit into them. The default storage is -`FilesystemBuckets` (Floe files in the local cache directory); `MemoryBuckets` and `PSRCacheBuckets` (any PSR-16 +`FilesystemRowsStorage` (Floe files in the local cache directory); `MemoryRowsStorage` and `PsrRowsStorage` (any PSR-16 cache) are the alternatives. -`groupBy()` takes an optional trailing `GroupByAlgorithmBuilder`, so one operation can override the configured +`groupBy()` takes an optional trailing `BucketingAlgorithmBuilder`, so one operation can override the configured algorithm: ```php ignore -->groupBy([ref('country')], hash_group_by()->storage(new MemoryBuckets())) +->groupBy([ref('country')], hash_group_by()->storage(new MemoryRowsStorage())) ``` The algorithm is configured through `config_builder()->groupBy(hash_group_by())`. `memoryLimit()` defaults to the diff --git a/documentation/components/core/join.md b/documentation/components/core/join.md index 2878496772..541ded05fc 100644 --- a/documentation/components/core/join.md +++ b/documentation/components/core/join.md @@ -21,19 +21,19 @@ Joins each row from the left side with each row on the right side, creating `cou ### joinEach () - Streaming Join -The right side is built for each left batch by `DataFrameFactory::from(Rows $rows)`, useful for large right-side -datasets that don't fit in memory. +The right side is read for each left batch from the extractor `ExtractorFactory::from(Rows $rows)` returns, useful for +large right-side datasets that don't fit in memory. ## Join Types Flow PHP supports four join types with specific behaviors: -| Join Type | Description | -|----------------------------------------|--------------------------------------------------------------------------------------------| -| **Left Join** (`Join::left`) | Returns all rows from left DataFrame with matching rows from right (or NULL) - **Default** | -| **Inner Join** (`Join::inner`) | Returns only rows that exist in both DataFrames | -| **Right Join** (`Join::right`) | Returns all rows from right DataFrame with matching rows from left (or NULL) | -| **Left Anti Join** (`Join::left_anti`) | Returns rows from left DataFrame that have NO match in right DataFrame | +| Join Type | Description | +|--------------------------------------------|--------------------------------------------------------------------------------------------| +| **Left Join** (`JoinType::left`) | Returns all rows from left DataFrame with matching rows from right (or NULL) - **Default** | +| **Inner Join** (`JoinType::inner`) | Returns only rows that exist in both DataFrames | +| **Right Join** (`JoinType::right`) | Returns all rows from right DataFrame with matching rows from left (or NULL) | +| **Left Anti Join** (`JoinType::left_anti`) | Returns rows from left DataFrame that have NO match in right DataFrame | Joins follow SQL semantics - every matching pair of rows produces one output row, so a left row that matches multiple right rows is emitted multiple times. A `null` join key never matches anything, including another `null` - rows with @@ -49,21 +49,21 @@ right rows is emitted multiple times. A `null` join key never matches anything, While the process stays under the memory limit, `join()` holds the right DataFrame in memory and streams the left one through its hash table, so left rows keep their order. Past the limit both sides are partitioned by join key into -buckets of a `BucketsStorage` - the same abstraction used by external sort - and joined bucket pair by bucket pair: +buckets of a `RowsStorage` - the same abstraction used by external sort - and joined bucket pair by bucket pair: -| Buckets storage | Past the memory limit | -|-----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| `FilesystemBuckets` - **Default** | Both sides are spilled to disk (Floe files); the hash table is built from the smaller bucket of each pair. Left row order is not preserved. | -| `PSRCacheBuckets` | Like `FilesystemBuckets`, with buckets in any PSR-16 cache. | -| `MemoryBuckets` | Never partitions: the right DataFrame stays in memory whatever its size and left rows keep their order. | +| Buckets storage | Past the memory limit | +|---------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| +| `FilesystemRowsStorage` - **Default** | Both sides are spilled to disk (Floe files); the hash table is built from the smaller bucket of each pair. Left row order is not preserved. | +| `PsrRowsStorage` | Like `FilesystemRowsStorage`, with buckets in any PSR-16 cache. | +| `MemoryRowsStorage` | Never partitions: the right DataFrame stays in memory whatever its size and left rows keep their order. | `join()` takes an optional trailing `JoinAlgorithmBuilder`, so one join can override the configured algorithm: ```php ignore -->join($right, join_on(['id' => 'id']), Join::left, hash_join()->storage(new MemoryBuckets())) +->join($right, join_on(['id' => 'id']), JoinType::left, hash_join()->storage(new MemoryRowsStorage())) ``` -`joinEach()` does not take one - it builds a DataFrame per left batch, so a per-call algorithm would be rebuilt per +`joinEach()` does not take one - it reads one extractor per left batch, so a per-call algorithm would be rebuilt per batch too. The join algorithm is configured through `config_builder()->join(hash_join())` - all its options live on the @@ -80,6 +80,7 @@ data_frame( ->memoryLimit(Unit::fromMb(512)) // process memory past which rows go to the storage ->bucketsCount(64) // number of buckets ->batchSize(1000) // rows per bucket frame and per batch read back + ->broadcastLimit(Unit::fromMb(10)) // a right side under it is broadcast instead of hashed ) ) ->read(from_parquet('orders.parquet')) @@ -112,7 +113,7 @@ data_frame() ->join( data_frame()->read(from_array($internalProducts)), Expression::on(['id' => 'id']), - Join::left_anti + JoinType::left_anti ) ->write(to_output()) ->run(); @@ -122,9 +123,9 @@ Output: ```console +----+-----------+ -| id | sku | +| id | sku | +----+-----------+ -| 1 | PRODUCT01 | +| 1 | PRODUCT01 | +----+-----------+ 1 rows ``` diff --git a/documentation/components/core/limit.md b/documentation/components/core/limit.md index 2c4e89a24d..908b5d7960 100644 --- a/documentation/components/core/limit.md +++ b/documentation/components/core/limit.md @@ -43,20 +43,18 @@ the result of the expansion. ## Limit Push-Down When nothing between `read()` and `limit()` can change the number of rows, the optimizer hands the limit to the -extractor, so a source that supports it reads only what is needed (`offset()` in between adds the skipped rows). A -frame read with `from_data_frame()` passes the limit on to its own source the same way. +extractor, so a source that supports it reads only what is needed (`offset()` in between adds the skipped rows). ```php read(from_csv(__DIR__ . '/orders.csv'))->select('id', 'total'); +use function Flow\ETL\DSL\data_frame; data_frame() - ->read(from_data_frame($orders)) + ->read(from_csv(__DIR__ . '/orders.csv')) + ->select('id', 'total') ->offset(10) ->limit(5) - ->fetch(); // the CSV extractor is asked for 15 rows + ->fetch(); // the CSV extractor skips 10 rows and stops after 5 ``` diff --git a/documentation/components/core/partitioning.md b/documentation/components/core/partitioning.md index 92b844e7e2..17168695d2 100644 --- a/documentation/components/core/partitioning.md +++ b/documentation/components/core/partitioning.md @@ -118,7 +118,13 @@ data_frame() ``` A placeholder matches any file in that location and the partition value is taken from the file name -as-is. Appended files get randomized suffixes (`sales_a1b2c3.csv`), which become part of the value. +as-is. Appended files carry the run in their name (`sales-a1b2c3d4e5f6a7b8.csv`), which becomes part of the value. + +## The {part} placeholder + +`{part}` in a destination is not a partition placeholder: it is filled with the part number (`part-00003.parquet`). +It is reserved - a partition column named `part` together with a `{part}` destination throws when the plan is +built; rename the column. ## Choosing Partition Columns @@ -208,7 +214,7 @@ data_frame() ->run(); ``` -Adds files with randomized suffixes; repeated runs accumulate files. +Adds files named after the run (`sales-.csv`); repeated runs accumulate files. ### Ignore @@ -268,10 +274,11 @@ it from the file schema (Parquet, Floe), so `filter(ref('date')->between(lit($fr ### Partition Pruning -The optimizer moves a `filter()` into the file source, which returns only matching rows. A filter that reads only -partition columns skips whole directories; a filter on a body column reads every file and drops the rows it does not -match (Parquet skips row groups, see [filter pushdown](/documentation/components/adapters/parquet.md#filter-pushdown)). -A `limit()` after the filter reaches the source with it ([filter pushdown](/documentation/components/adapters/parquet.md#filter-pushdown)). A partition column renamed +The optimizer moves a `filter()` that reads only partition columns into the file source, which skips whole +directories. Every format does this. A partition filter after another filter also skips directories. Parquet also +takes a filter on a body column and skips row groups with it ([filter pushdown](/documentation/components/adapters/parquet.md#filter-pushdown)). +For other formats, a filter on a body column stays a `Filter` step. +If a filter moved into the source, a `limit()` after it reaches the source too ([filter pushdown](/documentation/components/adapters/parquet.md#filter-pushdown)). A partition column renamed with `rename()` or copied with `withColumn('day', ref('date'))` is still pruned; any other redefinition of it stops the push. A filter whose answer can change between two evaluations - `call()`, `now()`, `uuid()`, `to_date_time()` with a format that leaves fields out (`'Y-m-d'`) - still filters rows but prunes nothing. diff --git a/documentation/components/core/save-mode.md b/documentation/components/core/save-mode.md index 96a8aec599..7df94e82bb 100644 --- a/documentation/components/core/save-mode.md +++ b/documentation/components/core/save-mode.md @@ -72,21 +72,21 @@ Creates additional files in the same directory when destination already exists: ->run(); // First run creates: data.csv -// Second run creates: data_.csv (e.g., data_5f8a3b2c.csv) +// Second run creates: data-.csv (e.g., data-5f8a3b2c0d1e2f30.csv) // Read them all with a glob: from_csv(__DIR__ . '/data*.csv') ``` **How it works:** - If destination file doesn't exist: writes normally -- If destination file exists: generates a new file with randomized name in the same directory +- If destination file exists: writes `-.` beside it **File structure after multiple runs:** ``` output/ -├── orders.csv # First run -├── orders_5ea42a0310.csv # Second run -├── orders_ceadbdb4d1.csv # Third run -└── orders_2140bfc5fd.csv # Fourth run +├── orders.csv # First run +├── orders-5ea42a0310c3d4e5.csv # Second run +├── orders-ceadbdb4d1a2b3c4.csv # Third run +└── orders-2140bfc5fd6e7f80.csv # Fourth run ``` Reading `orders.csv` reads that one file; read `orders*.csv` to get every run. @@ -112,10 +112,11 @@ Removes all existing files at the destination and writes fresh data: ``` **Implementation details:** -- Files are written to temporary files with `._flow_php_tmp.` prefix -- After writing completes, existing files are removed -- Temporary files are renamed to final names -- For partitioned writes, removes all files in partition directories +- Files are written under `.flow-tmp--/` inside the destination directory; readers skip it +- After writing completes, they are moved to their final names, then the files they replace are removed +- Only files Flow named are replaced: `.`, `-.` and the `{part}` parts +- For partitioned writes, only the partitions that received rows are replaced +- A run without rows empties an unpartitioned destination **Use when:** - Regenerating reports or exports @@ -136,6 +137,9 @@ Silently skips writing if the destination already exists: // If file doesn't exist: data is written normally ``` +Ignore and ExceptionIfExists decide once for the whole destination: one existing file in any partition skips (or +refuses) the whole write. + **Use when:** - Idempotent pipelines where re-running should have no effect - Avoiding duplicate work in batch processing @@ -165,3 +169,9 @@ When using `SaveMode::Overwrite` with partitions: - All files within each affected partition directory are removed - Only partitions being written to are affected - Unrelated partitions remain untouched + +## Part destinations + +A destination holding `{part}` is written as `part-00000`. The save mode is checked over every existing part of the +destination (`orders-*.parquet`), never over one file: `ExceptionIfExists` and `Ignore` see any part, `Overwrite` +removes every stale part before the new ones are renamed in, and `Append` names every part with the run id. diff --git a/documentation/components/core/schema.md b/documentation/components/core/schema.md index 169a72b5e8..47fba17488 100644 --- a/documentation/components/core/schema.md +++ b/documentation/components/core/schema.md @@ -359,11 +359,11 @@ cannot be null. Flow PHP provides three built-in validation strategies: -- **[StrictValidator](/src/core/etl/src/Flow/ETL/Schema/Validator/StrictValidator.php)** - Rows must exactly match the +- **[StrictValidator](/src/core/etl/src/Flow/ETL/Values/Schema/Validator/StrictValidator.php)** - Rows must exactly match the schema; extra columns cause validation failure -- **[SelectiveValidator](/src/core/etl/src/Flow/ETL/Schema/Validator/SelectiveValidator.php)** - Only validates columns +- **[SelectiveValidator](/src/core/etl/src/Flow/ETL/Values/Schema/Validator/SelectiveValidator.php)** - Only validates columns defined in schema; ignores extra columns -- **[EvolvingValidator](/src/core/etl/src/Flow/ETL/Schema/Validator/EvolvingValidator.php)** - Allows missing and extra +- **[EvolvingValidator](/src/core/etl/src/Flow/ETL/Values/Schema/Validator/EvolvingValidator.php)** - Allows missing and extra columns as long as nullability permits it By default, DataFrame uses `StrictValidator`, but you can specify a different validator as the second parameter to @@ -377,8 +377,8 @@ expected schema declares them nullable too: ```php read(from_array([['code' => '9'], ['code' => null], ['code' => '10']])) @@ -39,7 +39,7 @@ data_frame() The sort algorithm is configured through `config_builder()->sort()` - all options live on the algorithm's own builder: - `external_sort()` - **default**; sorts in memory while the buffered rows stay under its memory limit. Past the limit - it spills each sorted run as a bucket through a `BucketsStorage` (the same abstraction used by + it spills each sorted run as a bucket through a `RowsStorage` (the same abstraction used by [join](/documentation/components/core/join.md) and [group by](/documentation/components/core/group-by.md)), then k-way merges the runs back into one sorted stream. The memory limit defaults to `FLOW_MAX_MEMORY` when set, otherwise 70% of PHP's `memory_limit` (1 GB when @@ -61,11 +61,11 @@ data_frame( ### External sort options -| Buckets storage | Behavior | -|-----------------------------------|----------------------------------------------------------------------------------------------| -| `FilesystemBuckets` - **Default** | Sorted runs are spilled to disk (Floe files). | -| `MemoryBuckets` | Runs are kept in memory - prefer `memory_sort()` instead, it sorts once and skips the merge. | -| `PSRCacheBuckets` | Runs are spilled into any PSR-16 cache. | +| Buckets storage | Behavior | +|---------------------------------------|----------------------------------------------------------------------------------------------| +| `FilesystemRowsStorage` - **Default** | Sorted runs are spilled to disk (Floe files). | +| `MemoryRowsStorage` | Runs are kept in memory - prefer `memory_sort()` instead, it sorts once and skips the merge. | +| `PsrRowsStorage` | Runs are spilled into any PSR-16 cache. | ```php run(); ``` -The storage implementation is swapped with `external_sort()->storage(BucketsStorage $storage)`, which covers +The storage implementation is swapped with `external_sort()->storage(RowsStorage $storage)`, which covers both phases - the sorted runs and the runs written while merging. `sortBy()` followed by `limit(n)` keeps only the top `n` rows while it reads. Past the same memory limit it hands @@ -100,23 +100,23 @@ config - everything else in the pipeline keeps the configured default: data_frame() ->read(from_parquet('orders.parquet')) - ->sortBy([ref('total')->desc()], external_sort()->storage(new MemoryBuckets())) + ->sortBy([ref('total')->desc()], external_sort()->storage(new MemoryRowsStorage())) ->write(to_output()) ->run(); ``` Passing `null` (or omitting it) uses the configured algorithm. -To keep merged runs somewhere else, add `external_sort()->mergeStorage(BucketsStorage $storage)`. It defaults +To keep merged runs somewhere else, add `external_sort()->mergeStorage(RowsStorage $storage)`. It defaults to the spill storage, so `storage()` alone stays sufficient: ```php external_sort() - ->storage(new MemoryBuckets()) // spill runs - ->mergeStorage(new FilesystemBuckets($fs, path('/tmp/merge'), new AdaptiveBackend())); // merged runs only + ->storage(new MemoryRowsStorage()) // spill runs + ->mergeStorage(new FilesystemRowsStorage($fs, path('/tmp/merge'), new AdaptiveBackend())); // merged runs only ``` -`FilesystemBuckets` reads runs back into the [column backend](/documentation/components/core/column-backend.md) it is given. +`FilesystemRowsStorage` reads runs back into the [column backend](/documentation/components/core/column-backend.md) it is given. ## Example @@ -138,15 +138,15 @@ Output: | id | +----+ | 10 | -| 9 | -| 8 | -| 7 | -| 6 | -| 5 | -| 4 | -| 3 | -| 2 | -| 1 | +| 9 | +| 8 | +| 7 | +| 6 | +| 5 | +| 4 | +| 3 | +| 2 | +| 1 | +----+ 10 rows ``` \ No newline at end of file diff --git a/documentation/components/core/statistics.md b/documentation/components/core/statistics.md index c51dfec7d3..94197ea9ab 100644 --- a/documentation/components/core/statistics.md +++ b/documentation/components/core/statistics.md @@ -13,8 +13,8 @@ use function Flow\ETL\Adapter\Parquet\from_parquet; $statistics = from_parquet('orders.parquet')->statistics(); -$statistics->rows; // Flow\ETL\Cardinality -$statistics->size; // Flow\ETL\Cardinality, bytes +$statistics->rows; // Flow\ETL\Values\Cardinality +$statistics->size; // Flow\ETL\Values\Cardinality, bytes ``` The answer describes the whole source, with no `limit()` or partition filter applied. @@ -26,7 +26,7 @@ A `Cardinality` carries a guarantee and a guess side by side: ```php isUnknown(); // true ## What each source declares -| Source | rows | size | -|------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------|------------------------------| -| `from_parquet()`, `from_floe()` | exact from the footers; an estimate when a glob's footers were not all read | exact, or estimate on a glob | -| `from_csv()`, `from_json_lines()` | unknown before the schema is sniffed; exact when the sniff read every file whole; else an estimate from the mean row size | exact listed bytes | -| `from_json()`, `from_excel()` | unknown before the schema is sniffed; exact when the sniff read every file whole; else unknown | exact listed bytes | -| `from_text()`, `from_xml()` | unknown | exact listed bytes | -| `files()` | exact listed file count | exact listed bytes | -| `from_path_partitions()` | exact listed file count | unknown | -| `from_array()`, `from_rows()`, `from_sequence_*()` | exact | unknown | -| `from_cache()` | exact from the cache index | unknown | -| `from_pgsql_*()`, `from_dbal_*()` | estimate from `EXPLAIN` (never `ANALYZE`), bounded by `withMaximum()` | unknown | -| `from_google_sheet()` | bounded by the sheet's grid size once the schema is sniffed | unknown | -| `from_all()`, `batches()`, `batched_by()` | what the wrapped extractors declare, merged | same | -| `from_avro()`, `from_*_http_*()`, `from_memory()`, `from_data_frame()` | unknown | unknown | +| Source | rows | size | +|---------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------|------------------------------| +| `from_parquet()`, `from_floe()` | exact from the footers; an estimate when a glob's footers were not all read | exact, or estimate on a glob | +| `from_csv()`, `from_json_lines()` | unknown before the schema is sniffed; exact when the sniff read every file whole; else an estimate from the mean row size | exact listed bytes | +| `from_json()`, `from_excel()` | unknown before the schema is sniffed; exact when the sniff read every file whole; else unknown | exact listed bytes | +| `from_text()`, `from_xml()` | unknown | exact listed bytes | +| `files()` | exact listed file count | exact listed bytes | +| `from_path_partitions()` | exact listed file count | unknown | +| `from_array()`, `from_rows()`, `from_memory()`, `from_sequence_*()` | exact | unknown | +| `from_cache()` | exact from the cache index | unknown | +| `from_pgsql_*()`, `from_dbal_*()` | estimate from `EXPLAIN` (never `ANALYZE`), bounded by `withMaximum()` | unknown | +| `from_google_sheet()` | bounded by the sheet's grid size once the schema is sniffed | unknown | +| `from_all()` | what the wrapped extractors declare, merged | same | +| `from_avro()`, `from_*_http_*()` | unknown | unknown | ## Where statistics are used @@ -71,9 +71,9 @@ Cardinality::unknown()->isUnknown(); // true ```php new ConfiguredFilesystem($this->filesystem), + 'options' => $this->options->normalize(), + ]); + } + + public static function fromDescription(#[\SensitiveParameter] array $data, Rebuild $rebuild): static + { + return new self( + $rebuild->of($data['filesystem'], ConfiguredFilesystem::class)->filesystem(), + ApiOptions::fromArray($data['options']), + ); + } +} +``` + +- a library object is not `Describable`: it exposes `normalize(): ?array` / `fromArray(array)` and core wraps it in an + adapter (`ConfiguredFilesystem`, `ConfiguredParquetEngine`); a filesystem's adapter is secret, so a description + nesting one is `sensitive()` and may only cross a ciphered exchange; +- a value object travels as `normalize(): array` / `fromArray(array)`, enums as `{enum, case}`, paths as + `Path::normalize()`; input is untrusted and validated on the way back in; +- `#[\SensitiveParameter]` on every constructor and `fromDescription()` argument that may hold a credential keeps it + out of stack traces; `null` from `describe()` means "this instance stays in-process" and is never an error. diff --git a/documentation/components/core/telemetry.md b/documentation/components/core/telemetry.md index f6f0771ff3..12119c8916 100644 --- a/documentation/components/core/telemetry.md +++ b/documentation/components/core/telemetry.md @@ -24,7 +24,7 @@ The simplest way to get started with telemetry is using console exporters for lo ```php withTelemetry($telemetry, telemetry_options( Flow PHP provides granular control over what telemetry data is collected through `telemetry_options()`: -| Option | Description | Performance Impact | -|-------------------------|----------------------------------------------|-----------------------------------------| -| `trace_loading` | Creates spans for each loader execution | Low | -| `trace_transformations` | Creates spans for each transformer execution | Medium (many transformers = many spans) | -| `collect_metrics` | Records counters and throughput metrics | Low | +| Option | Description | Performance Impact | +|-------------------------|-------------------------------------------------------------------------|-----------------------------------------| +| `trace_loading` | Creates spans for each loader execution | Low | +| `trace_transformations` | Creates spans for each transformer execution and each processor or join | Medium (many transformers = many spans) | +| `collect_metrics` | Records counters and throughput metrics | Low | ### Selective Configuration @@ -132,7 +132,7 @@ $logExporter = console_exporter(colors: true, maxLogBodyLength: 100); For production environments, send telemetry to an OTLP-compatible collector using the OTLP bridge: ```php -use Flow\Clock\SystemClock; +use Flow\ETL\Foundation\Clock\SystemClock; use function Flow\ETL\DSL\{config_builder, telemetry_options}; use function Flow\Telemetry\DSL\{ batching_log_processor, batching_metric_processor, batching_span_processor, @@ -218,6 +218,14 @@ When `trace_transformations` is enabled, child spans are created for each transf - Parent: DataFrame span - Status: OK on success, ERROR on failure +The same option creates one span for each processor and join (sort, group by, join, distinct): + +- Span name: short class name, for example `MergeSortProcessor`. The full name is in `flow.etl.operator.class`. +- Duration: wall time from the first batch requested to the last batch yielded, including the time the steps after it + spend on those batches. +- Parent: DataFrame span +- Status: ERROR on failure, OK in every other case (an early stop included) + ### Metrics When `collect_metrics` is enabled: diff --git a/documentation/components/core/transformations.md b/documentation/components/core/transformations.md index 71e04398c3..567ba318c1 100644 --- a/documentation/components/core/transformations.md +++ b/documentation/components/core/transformations.md @@ -21,27 +21,22 @@ interface Transformation ## Using Transformations -Transformations can be applied to DataFrames using two methods: - -- `with()` - Applies one or more transformations -- `transform()` - Alias for `with()`, provides semantic clarity - -Both methods accept `Transformation` objects directly or through convenient DSL functions. +`with()` applies a transformation to a DataFrame. It accepts `Transformation` objects directly or through DSL functions. ```php use function Flow\ETL\DSL\{df, from_array, select, drop, to_output}; -// Using with() +// Keeping columns df() ->read(from_array([['id' => 1, 'name' => 'Alice', 'temporary_column' => 'x']])) ->with(select('id', 'name')) ->write(to_output()) ->run(); -// Using transform() - identical behavior +// Dropping a column df() ->read(from_array([['id' => 1, 'name' => 'Alice', 'temporary_column' => 'x']])) - ->transform(drop('temporary_column')) + ->with(drop('temporary_column')) ->write(to_output()) ->run(); ``` @@ -116,13 +111,31 @@ df() ->run(); ``` +### Batch By + +Keep all rows with the same value of a column in one batch. The optional minimum size joins small groups into one batch. + +```php +use function Flow\ETL\DSL\{df, from_array, batch_by, to_output}; + +df() + ->read(from_array([ + ['order_id' => 1, 'item' => 'Widget'], + ['order_id' => 1, 'item' => 'Gadget'], + ['order_id' => 2, 'item' => 'Gizmo'], + ])) + ->with(batch_by('order_id', 2)) + ->write(to_output()) + ->run(); +``` + ### Add Row Index Add an index column to each row, useful for tracking row position or creating unique identifiers. ```php use function Flow\ETL\DSL\{df, from_array, add_row_index, to_output}; -use Flow\ETL\Transformation\AddRowIndex\StartFrom; +use Flow\ETL\Operators\Transformer\AddRowIndex\StartFrom; // Add default index starting from 0 df() @@ -144,7 +157,7 @@ df() ``` The index starts again on every run of the frame. A custom `Transformer` that keeps state between batches does the -same by implementing `Flow\ETL\Transformer\Stateful`: its `fresh()` returns a new instance in its constructed state, +same by implementing `Flow\ETL\Operators\Transformer\Stateful`: its `fresh()` returns a new instance in its constructed state, and every run uses that instance. ### Limit @@ -216,9 +229,9 @@ df() ->run(); ``` -## Transformations as Sinks +## Transformations as Branches -`to_transformation()` and `to_branch()` are sinks: each becomes its own root of the plan, fed the rows of the node it +`to_transformation()` and `to_branch()` are branches: each becomes its own root of the plan, fed the rows of the node it was written at, and planned and bound together with the rest of the frame. ```php @@ -232,21 +245,20 @@ df() ->run(); ``` -A branch filters first; `withTransformation()`, called before `write()`, transforms what passed the condition. Sinks -nest - wherever a sink takes a loader, it also takes another sink: +A branch filters first; a `to_transformation()` inside it transforms what passed the condition. Branches nest - wherever +a branch takes a loader, it also takes another branch or a transaction: ```php -to_branch(ref('active')->equals(lit(true)), to_csv('active.csv'))->withTransformation($sortById); to_branch(ref('active')->equals(lit(true)), to_transformation($sortById, to_csv('active.csv'))); ``` The `Transformation` runs once per run over the whole stream it is fed, so `limit()` and `add_row_index()` apply across the stream, not per batch. A triggering verb (`fetch()`, `count()`, `schema()`, ...) inside it executes the prefix plan, -as `from_data_frame()` executes a frame. `onError()` inside it throws `InvalidLogicException` at `write()` - set the +as a terminal verb does. `onError()` inside it throws `InvalidLogicException` at `write()` - set the error handler on the outer frame. ```php -use Flow\ETL\{DataFrame, Transformation}; +use Flow\ETL\API\{DataFrame, Transformation}; use function Flow\ETL\DSL\{df, from_array, ref, to_output, to_transformation}; @@ -266,7 +278,7 @@ df() ### Memory Cost Correctness is the same everywhere; what differs between operations is how much they hold, and they hold it inside the -sink. Three groups: +branch. Three groups: | Cost | Operations | |----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------| @@ -280,7 +292,7 @@ batch as it passes. They need the whole stream to answer correctly, not to accum ### Chunk Shape and Order -`batch_size()`, `batchBy()` and `repartition()` change only which rows are grouped into the `Rows` handed to the sink's +`batch_size()`, `batchBy()` and `repartition()` change only which rows are grouped into the `Rows` handed to the branch's loader. No row is lost or mis-assigned. `repartition()` and `join()` also change the **order** the rows arrive in: both group their output by key rather than @@ -290,16 +302,16 @@ emitting it in input order. `batchBy()` preserves input order and only cuts the arrives together, so it belongs in the first group above, alongside `sortBy()` and `join()`. Writing one directory per key is a separate thing, declared on the loader: `to_csv(...)->partitionBy('region')`. -To re-batch the pipeline itself rather than what reaches the sink's loader, call `$df->batchSize(...)` on the frame. +To re-batch the pipeline itself rather than what reaches the branch's loader, call `$df->batchSize(...)` on the frame. ### Failure Behaviour -A sink runs under the frame's `->onError(...)` handler, and each failure is offered to it exactly once: +A branch runs under the frame's `->onError(...)` handler, and each failure is offered to it exactly once: ```php df() ->read(from_array([['id' => 2], ['id' => 1]])) - ->onError(skip_rows_handler()) // a failing step inside the sink skips that batch + ->onError(skip_rows_handler()) // a failing step inside the branch skips that batch ->write(to_transformation($sortById, to_csv('sorted.csv'))) ->run(); ``` @@ -310,7 +322,7 @@ everything it had buffered. Under the default handler the failure propagates and A loader whose `closure()` fails is never offered to the handler: its own exception surfaces from `run()`. None of this is durability or atomicity: `closure()` both commits and closes, so a destination written up to the point -of failure can be left behind. For all-or-nothing batches wrap the sinks in a transaction - +of failure can be left behind. For all-or-nothing batches wrap the writes in a transaction - `to_dbal_transaction()`, `to_pgsql_transaction()`. ## Creating Custom Transformations @@ -318,7 +330,7 @@ of failure can be left behind. For all-or-nothing batches wrap the sinks in a tr You can create custom transformations by implementing the `Transformation` interface: ```php -use Flow\ETL\{DataFrame, Transformation}; +use Flow\ETL\API\{DataFrame, Transformation}; use function Flow\ETL\DSL\{df, from_array, ref, to_output}; diff --git a/documentation/components/core/window-functions.md b/documentation/components/core/window-functions.md index 626e81e036..5c437c9716 100644 --- a/documentation/components/core/window-functions.md +++ b/documentation/components/core/window-functions.md @@ -14,14 +14,14 @@ smaller partitions on which later window function will be applied. ### Window Functions: -- [`ROW_NUMBER()`](/src/core/etl/src/Flow/ETL/Function/RowNumber.php) -- [`RANK()`](/src/core/etl/src/Flow/ETL/Function/Rank.php) -- [`DENSE_RANK()`](/src/core/etl/src/Flow/ETL/Function/DenseRank.php) -- [`SUM`](/src/core/etl/src/Flow/ETL/Function/Sum.php) -- [`AVERAGE`](/src/core/etl/src/Flow/ETL/Function/Average.php) -- [`COUNT`](/src/core/etl/src/Flow/ETL/Function/Count.php) - -All window functions implement [`WindowFunction`](/src/core/etl/src/Flow/ETL/Function/WindowFunction.php) +- [`ROW_NUMBER()`](/src/core/etl/src/Flow/ETL/Functions/RowNumber.php) +- [`RANK()`](/src/core/etl/src/Flow/ETL/Functions/Rank.php) +- [`DENSE_RANK()`](/src/core/etl/src/Flow/ETL/Functions/DenseRank.php) +- [`SUM`](/src/core/etl/src/Flow/ETL/Functions/Sum.php) +- [`AVERAGE`](/src/core/etl/src/Flow/ETL/Functions/Average.php) +- [`COUNT`](/src/core/etl/src/Flow/ETL/Functions/Count.php) + +All window functions implement [`WindowFunction`](/src/core/etl/src/Flow/ETL/Functions/WindowFunction.php) interface. ### Example diff --git a/documentation/components/extensions/flow-php-ext.md b/documentation/components/extensions/flow-php-ext.md index 8cdfaf6576..e72eed817a 100644 --- a/documentation/components/extensions/flow-php-ext.md +++ b/documentation/components/extensions/flow-php-ext.md @@ -43,40 +43,41 @@ df() ->run(); ``` -With the extension loaded, `Flow\ETL\Column\AdaptiveBackend` - the default of `config_builder()->backend()` - picks the -extension's `RustBackend` and every batch column is a `Flow\ETL\Column\RustColumn` over an Apache Arrow array. How to +With the extension loaded, `Flow\ETL\Values\Column\AdaptiveBackend` - the default of `config_builder()->backend()` - picks the +extension's `RustBackend` and every batch column is a `Flow\ETL\Values\Column\RustColumn` over an Apache Arrow array. How to choose a backend and what it builds: [Column Backend](/documentation/components/core/column-backend.md). ```php builder(int_schema('id')); $builder->appendMany([1, '2', 3.0]); -$column = $builder->finish(); // Flow\ETL\Column\RustColumn +$column = $builder->finish(); // Flow\ETL\Values\Column\RustColumn $column->values(); // [1, 2, 3] (new RustBackend())->allocatedBytes(); // bytes held outside PHP's memory manager ``` Every contract with a Rust and a PHP implementation follows one pattern: the extension registers the interface, the -`Rust*` class implements it, `Php*` is the PHP implementation and `Adaptive*` picks one by `extension_loaded()`. - -| contract | Rust | PHP | picked by | -|----------------------------------------------|-------------------------|------------------------|-----------------------------| -| `Flow\ETL\Column\Backend` | `RustBackend` | `PhpBackend` | `AdaptiveBackend` | -| `Flow\ETL\Adapter\CSV\CSVOpenSource` | `RustCSVOpenSource` | `PhpCSVOpenSource` | `AdaptiveCSVOpenSource` | -| `Flow\ETL\Adapter\CSV\CSVEncoder` | `RustCSVEncoder` | `PhpCSVEncoder` | `AdaptiveCSVEncoder` | -| `Flow\ETL\Adapter\JSON\JsonOpenSource` | `RustJsonOpenSource` | `PhpJsonOpenSource` | `AdaptiveJsonOpenSource` | -| `Flow\ETL\Adapter\JSON\JsonEncoder` | `RustJsonEncoder` | `PhpJsonEncoder` | `AdaptiveJsonEncoder` | -| `Flow\ETL\Adapter\Parquet\ParquetOpenSource` | `RustParquetOpenSource` | `PhpParquetOpenSource` | `AdaptiveParquetOpenSource` | -| `Flow\ETL\Adapter\Parquet\ParquetOpenSink` | `RustParquetOpenSink` | `PhpParquetOpenSink` | `AdaptiveParquetOpenSink` | - -The extension also registers `Flow\ETL\Column\{Column, ColumnBuilder}` (`RustColumn`, `RustColumnBuilder`) and -`Flow\ETL\RustIterator`, the iterator its sources return. +`Rust*` class implements it, `Php*` is the PHP implementation and the class that builds it picks one by +`extension_loaded()`. + +| contract | Rust | PHP | picked by | +|-----------------------------------------------------------|-------------------------|------------------------|--------------------------------------| +| `Flow\ETL\Values\Column\Backend` | `RustBackend` | `PhpBackend` | `AdaptiveBackend` | +| `Flow\ETL\Adapter\CSV\IO\Extractor\CSVOpenSource` | `RustCSVOpenSource` | `PhpCSVOpenSource` | `CSVSourceOpener` | +| `Flow\ETL\Adapter\CSV\IO\Loader\CSVEncoder` | `RustCSVEncoder` | `PhpCSVEncoder` | `CSVLoader::writer()` | +| `Flow\ETL\Adapter\JSON\IO\Extractor\JsonOpenSource` | `RustJsonOpenSource` | `PhpJsonOpenSource` | `JsonFileBatches::open()` | +| `Flow\ETL\Adapter\JSON\IO\Loader\JsonEncoder` | `RustJsonEncoder` | `PhpJsonEncoder` | `JsonLoader::writer()` | +| `Flow\ETL\Adapter\Parquet\IO\Extractor\ParquetOpenSource` | `RustParquetOpenSource` | `PhpParquetOpenSource` | `ParquetSourceFile::open()` | +| `Flow\ETL\IO\Loader\File\FileWriter` | `RustParquetFileWriter` | `ParquetFileWriter` | `ParquetFileWriterFactory::writer()` | + +The extension also registers `Flow\ETL\Values\Column\{Column, ColumnBuilder}` (`RustColumn`, `RustColumnBuilder`) and +`Flow\ETL\Values\RustIterator`, the iterator its sources return. ## CSV and JSON writers @@ -129,7 +130,8 @@ use function Flow\ETL\Adapter\Parquet\{from_parquet, to_parquet}; use function Flow\ETL\DSL\df; df() - ->read(from_parquet(__DIR__ . '/orders.parquet')->withOffset(50_000)) // only the row groups from row 50 000 are decoded + ->read(from_parquet(__DIR__ . '/orders.parquet')) + ->offset(50_000) // only the row groups from row 50 000 are decoded ->write(to_parquet(__DIR__ . '/copy.parquet')) ->run(); @@ -144,6 +146,6 @@ imported batches are counted by `RustBackend::allocatedBytes()` for as long as t buffers (a row group's projected column chunks, decode buffers) are not, so a `MemoryBudget` does not see them. Read and write refusals are arrow-ext's `Flow\Parquet\Exception\*`, as with `RustParquetEngine`. -`RustParquetOpenSource` reads through arrow-ext's `Flow\Arrow\Parquet\RustBatchReader`, `RustParquetOpenSink` writes +`RustParquetOpenSource` reads through arrow-ext's `Flow\Arrow\Parquet\RustBatchReader`, `RustParquetFileWriter` writes through its `Flow\Parquet\Engine\RustParquetFileWriter`. diff --git a/documentation/components/libs/parquet.md b/documentation/components/libs/parquet.md index 55dbd7e508..bc7044e203 100644 --- a/documentation/components/libs/parquet.md +++ b/documentation/components/libs/parquet.md @@ -251,11 +251,11 @@ skip the same ones and read a kept page's bytes alone. A second `where()` narrow ```php use Flow\Parquet\Filter\Binding\BoundFilter; -use Flow\Parquet\Reader\ReadPlanner; +use Flow\Parquet\Reader\ReadPlan; use function Flow\Parquet\DSL\parquet_column; $filter = parquet_column('id')->greaterThan(100); -$plan = (new ReadPlanner())->plan( +$plan = ReadPlan::forFilter( $file->reader()->footer(), $file->indexes(), BoundFilter::bind($filter, $file->schema(), $file->options()), @@ -463,9 +463,9 @@ foreach ($metadata->rowGroups()->all() as $rowGroup) { $statistics?->minValue($column); // physical value: bytes for strings, unscaled int for decimal(10, 2) $statistics?->maxValue($column); - $statistics?->isMaxValueExact(); // false when the max was truncated and incremented - $statistics?->nullCount(); - $statistics?->nanCount(); // FLOAT, DOUBLE and FLOAT16 only + $statistics?->isMaxValueExact; // false when the max was truncated and incremented + $statistics?->nullCount; + $statistics?->nanCount; // FLOAT, DOUBLE and FLOAT16 only } $metadata->columnOrder($column); // ColumnOrder::TypeDefined; ColumnOrder::Absent in files written before it diff --git a/documentation/components/libs/postgresql/index-query-builder.md b/documentation/components/libs/postgresql/index-query-builder.md index f2aa8108ae..ec8a0074a6 100644 --- a/documentation/components/libs/postgresql/index-query-builder.md +++ b/documentation/components/libs/postgresql/index-query-builder.md @@ -239,15 +239,14 @@ Create an index on an expression: ```php index('idx_users_lower_email') ->on('users') - ->columns(index_expr(new RawExpression('lower(email)'))); + ->columns(index_expr(func('lower', [col('email')]))); echo $query->toSql(); -// CREATE INDEX idx_users_lower_email ON users ((lower(email))) +// CREATE INDEX idx_users_lower_email ON users (lower(email)) ``` ### Covering Index (INCLUDE) diff --git a/documentation/components/libs/telemetry/contracts.md b/documentation/components/libs/telemetry/contracts.md index 05b90efcb2..fdad14baf5 100644 --- a/documentation/components/libs/telemetry/contracts.md +++ b/documentation/components/libs/telemetry/contracts.md @@ -12,21 +12,21 @@ implementations. ### Transport -**Interface:** `Flow\Telemetry\Transport\Transport` +**Interface:** `Flow\Bridge\Telemetry\OTLP\Transport\Transport` -Sends serialized telemetry data to backends over the network. The signal type is inferred from the batch payload type -(`LogsBatch`, `MetricsBatch`, `TracesBatch`). Single method: +Sends telemetry to an OTLP backend. The signal type is `Signals::$type`: ```php ignore -public function send(LogsBatch|MetricsBatch|TracesBatch $batch) : void; +public function send(Signals $signal) : void; public function shutdown() : void; ``` -| Implementation | Package | Description | -|-----------------|----------------------------------|------------------------------------------| -| `VoidTransport` | `flow-php/telemetry` | No-op transport that discards all data | -| `GrpcTransport` | `flow-php/telemetry-otlp-bridge` | gRPC transport (requires ext-grpc) | -| `CurlTransport` | `flow-php/telemetry-otlp-bridge` | Async curl transport (requires ext-curl) | +| Implementation | Package | Description | +|----------------------|----------------------------------|------------------------------------------------------| +| `GrpcTransport` | `flow-php/telemetry-otlp-bridge` | Non-blocking gRPC transport (requires ext-grpc) | +| `CurlTransport` | `flow-php/telemetry-otlp-bridge` | Synchronous HTTP transport (requires ext-curl) | +| `AsyncCurlTransport` | `flow-php/telemetry-otlp-bridge` | Non-blocking HTTP transport (requires ext-curl) | +| `StreamTransport` | `flow-php/telemetry-otlp-bridge` | Writes JSONL to a file path or a `php://` stream | --- @@ -34,14 +34,12 @@ public function shutdown() : void; **Interface:** `Flow\Telemetry\Exporter\Exporter` -Exports telemetry batches to external observability backends. A single exporter handles all three signals via -`match(true)` instanceof dispatch on `LogsBatch | MetricsBatch | TracesBatch`. +Exports telemetry to external observability backends. A single exporter handles all three signals with a `match` on +`Signals::$type`. ```php ignore -public function export(LogsBatch|MetricsBatch|TracesBatch $batch) : bool; - -/** @return array */ -public function transports() : array; +public function export(Signals $signal) : bool; +public function shutdown() : void; ``` | Implementation | Package | Description | @@ -53,13 +51,15 @@ public function transports() : array; --- -### Batch value objects +### Signals -Three small immutable value objects in `Flow\Telemetry\Batch` carry the payload across Transport / Exporter boundaries: +`Flow\Telemetry\Signal\Signals` carries the payload of one signal type across Exporter and Transport boundaries: -- `LogsBatch(array $entries)` -- `MetricsBatch(array $metrics)` -- `TracesBatch(array $spans)` +- `Signals::logs(array)`, read with `allLogs()` +- `Signals::metrics(array)`, read with `allMetrics()` +- `Signals::traces(array)`, read with `allSpans()` + +`$type` is a `SignalType`: `LOGS`, `METRICS` or `TRACES`. --- @@ -239,26 +239,26 @@ All contracts are designed for extension. To implement a custom exporter: ```php $this->writeSpans($batch->spans), - $batch instanceof MetricsBatch => $this->writeMetrics($batch->metrics), - $batch instanceof LogsBatch => $this->writeLogs($batch->entries), + match ($signal->type) { + SignalType::TRACES => $this->writeSpans($signal->allSpans()), + SignalType::METRICS => $this->writeMetrics($signal->allMetrics()), + SignalType::LOGS => $this->writeLogs($signal->allLogs()), }; return true; } - public function transports() : array + public function shutdown() : void { - return [new VoidTransport()]; } // ... per-signal helpers diff --git a/documentation/contributing/benchmarks.md b/documentation/contributing/benchmarks.md index 277e2dcc4a..8cba2d4629 100644 --- a/documentation/contributing/benchmarks.md +++ b/documentation/contributing/benchmarks.md @@ -78,5 +78,50 @@ Blackfire without any phpbench overhead. Bootstrapping through `benchmarks/boots environment (wipes `var/`, exports the service defaults) so the profile matches the benchmark run: ```bash -blackfire run php -r 'require "benchmarks/bootstrap.php"; (new Flow\Benchmarks\Joining\JoinOrdersWithSellersScenario(Flow\ETL\Join\Join::left, 100000))->run();' +blackfire run php -r 'require "benchmarks/bootstrap.php"; (new Flow\Benchmarks\Joining\JoinOrdersWithSellersScenario(Flow\ETL\Values\Join\JoinType::left, 100000))->run();' ``` + +## Reference engines + +`benchmarks/reference/` holds one script per engine. Each script runs the same four shapes over one CSV or Parquet file: + +| shape | work | +|------------------|-----------------------------------------------------------------------| +| `scan` | count the rows that pass a predicate touching every column | +| `to_parquet` | write the rows as Parquet | +| `group_by` | group by `country` with `sum(amount)`, `count(id)` and `avg(amount)` | +| `sort` | sort by `amount` descending, write as Parquet | + +Every script takes ` `. The thread count is the knob of each engine. The engines come +from the nix shell flags `with-duckdb`, `with-polars` and `with-spark`, see the +[Nix page](/documentation/contributing/nix.md#reference-engines-duckdb-polars-spark). + +```bash +benchmarks/reference/duckdb/bench.sh group_by orders.csv var/reference 8 +POLARS_MAX_THREADS=8 python benchmarks/reference/polars/bench.py group_by orders.csv var/reference +spark-submit --master 'local[8]' benchmarks/reference/spark/bench.py group_by orders.csv var/reference +``` + +Rules for a comparable number: + +- Time with `hyperfine` and an explicit run count. Its default run count targets three seconds in total. + +```bash +hyperfine --runs 3 --warmup 1 --export-json var/reference/group_by.json 'benchmarks/reference/duckdb/bench.sh group_by orders.csv var/reference 8' +``` + +- Hot page cache is the default. Record one cold run per shape with a prepare command that drops the page cache. + +```bash +# macOS +hyperfine --runs 1 --prepare 'sudo purge' '...' +# Linux +hyperfine --runs 1 --prepare 'sync && echo 3 | sudo tee /proc/sys/vm/drop_caches' '...' +``` + +- Give every engine the same N. DuckDB takes `SET threads TO N`, Polars `POLARS_MAX_THREADS=N`, Spark `local[N]` + and Flow N workers. The Flow coordinator process is busy on top of its N workers. State that next to every table. +- Time Flow in a `--arg with-php-benchmark true` shell, without pcov or blackfire. +- Spark pays a JVM start on every submission. The warmup run absorbs it. +- DuckDB and Polars infer column types from the CSV. Spark runs with `inferSchema=True`, which costs a second pass + over the file. diff --git a/documentation/contributing/nix.md b/documentation/contributing/nix.md index df96da6d8c..c5b66e618b 100644 --- a/documentation/contributing/nix.md +++ b/documentation/contributing/nix.md @@ -89,6 +89,32 @@ nix-shell --arg with-blackfire true To configure blackfire, you can create a file `./.nix/php/lib/blackfire.ini` with your blackfire configuration. +## Benchmarking PHP + +The default shell runs PHP with development settings. OPcache is off for the CLI, which keeps the JIT off. +`zend.assertions` is `1`. pcov is loaded. To time Flow with production settings: + +```shell +nix-shell --arg with-php-benchmark true +``` + +The flag removes pcov, xdebug and blackfire. A shell that combines the flag with one of them fails at evaluation. +The flag appends `./.nix/php/lib/benchmark.ini.dist` to `php.ini`: + +```ini +opcache.enable=1 +opcache.enable_cli=1 +opcache.jit=tracing +opcache.jit_buffer_size=128M +opcache.memory_consumption=256 +opcache.interned_strings_buffer=32 +opcache.max_accelerated_files=20000 +opcache.validate_timestamps=1 +zend.assertions=-1 +``` + +On Linux the flag also sets `opcache.huge_code_pages=1`. To override a value, create `./.nix/php/lib/benchmark.ini`. + ## Changing PHP Versions To change the PHP version, you can run nix shell with `--arg php-version 8.3` flag: @@ -153,6 +179,21 @@ nix-shell --arg with-protoc true The `ext-protobuf` and `ext-grpc` PHP extensions are part of the default shell (`with-protobuf` / `with-grpc`, both default `true`); pass `false` to opt out. +### Reference Engines (DuckDB, Polars, Spark) + +Each engine has its own flag. Enable only the engines a comparison needs: + +```shell +nix-shell --arg with-duckdb true --arg with-polars true --arg with-spark true --arg with-php-benchmark true +``` + +- `with-duckdb` adds the `duckdb` CLI. +- `with-polars` adds a `python3` with `polars` and `pyarrow`. +- `with-spark` adds `spark-submit` with its JDK, without R. + +The scripts and the measurement rules are on the +[Benchmarks page](/documentation/contributing/benchmarks.md#reference-engines). + ### Combining Multiple Options You can combine multiple arguments: diff --git a/documentation/contributing/rust.md b/documentation/contributing/rust.md index cdd6bfeac5..e5c3a7ac7e 100644 --- a/documentation/contributing/rust.md +++ b/documentation/contributing/rust.md @@ -8,7 +8,7 @@ This document describes how to develop the two Rust PHP extensions in this monor | Extension | Package | What it provides | |----------------|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `arrow-ext` | `flow-php/arrow-ext` | Parquet reader and writer powered by the [Apache Arrow](https://arrow.apache.org/) Rust ecosystem, exposed as `Flow\Parquet\Engine\RustParquetEngine` | -| `flow-php-ext` | `flow-php/flow-php-ext` | The native column backend (`Flow\ETL\Column\RustBackend`, `RustColumn`, `RustColumnBuilder` over Apache Arrow arrays) and the Rust CSV / JSON / Parquet sources and encoders | +| `flow-php-ext` | `flow-php/flow-php-ext` | The native column backend (`Flow\ETL\Values\Column\RustBackend`, `RustColumn`, `RustColumnBuilder` over Apache Arrow arrays) and the Rust CSV / JSON / Parquet sources and encoders | Both are optional. The pure-PHP implementations in `flow-php/etl` and `flow-php/parquet` remain the behaviour reference; every contract has a `Php*`, a `Rust*` and an `Adaptive*` class that picks the Rust one when the extension @@ -71,7 +71,6 @@ src/extension/flow-php-ext/ │ ├── backend.rs # RustBackend │ ├── column.rs # RustColumn │ ├── builder.rs # RustColumnBuilder: native cast lanes, PHP lane for the rest -│ ├── kind_builder.rs # Arrow storage appended row by row │ ├── physical.rs # Arrow rows as physical and logical zvals │ ├── plan.rs # Per-type plan, cached by Type object and type JSON │ ├── render.rs # flow-batch-frame refusals as the PHP messages @@ -79,18 +78,23 @@ src/extension/flow-php-ext/ │ ├── json_check.rs # JSON validation shared by casting and CSV inference │ ├── batch_columns.rs # BatchColumns: one native builder per definition, shared by the CSV and JSON sources │ ├── source.rs # Shared stream loop of the Rust open sources -│ ├── csv/ # CSV tokenizer, RustCSVOpenSource, RustCSVEncoder, schema-inference fold -│ ├── json/ # RustJsonOpenSource, RustJsonEncoder -│ ├── text/ # Column values as the writers render them (TextValues) -│ ├── parquet/ # RustParquetOpenSource, RustParquetOpenSink -│ ├── arrow_c.rs # Batches across the Arrow C Data Interface (arrow-ext) +│ ├── csv/ # RustCSVOpenSource, RustCSVEncoder, schema-inference fold +│ ├── json/ # RustJsonOpenSource, RustJsonEncoder, JSON values into columns +│ ├── text/ # The PHP side of the text writers: Rows batches, DateTimeZone transitions +│ ├── parquet/ # RustParquetOpenSource, RustParquetFileWriter +│ ├── arrow_c.rs # RustColumnsBatch and the PHP side of the Arrow C Data hand-off (arrow-ext) │ ├── ctx.rs # Request-scoped context and engine helpers │ ├── globals.rs # Module globals, RINIT/RSHUTDOWN │ ├── alloc.rs # Counting global allocator (allocatedBytes()) │ ├── values.rs # Zval <-> PHP value helpers │ └── exception.rs # Exception mapping -├── crates/ -│ └── flow-batch-frame/ # Pure-Rust column buffers and BATCH frame bodies (no PHP types) +├── crates/ # The cargo workspace members: pure Rust, no PHP types, every #[test] lives here +│ ├── flow-batch-frame/ # Column buffers, builders, BATCH frame bodies, kinds and value shapes, column keys +│ ├── flow-column-text/ # Column values as the writers render them: floats, dates, JSON strings, uuids +│ ├── flow-json-records/ # JSON lines / array framing and member-to-column mapping +│ ├── flow-arrow-c/ # Batches across the Arrow C Data Interface, imported bytes counted +│ ├── flow-csv-tokenizer/ # CSV records and fields as php_fgetcsv() produces them +│ └── flow-numeric-string/ # PHP's is_numeric() grammar ├── php/ # PHP stubs for static analysis │ └── Flow/ ├── tests/ @@ -139,11 +143,11 @@ nix-shell --run "just test --testsuite=etl-unit" # flow-php- | `clean` | Remove build artifacts | | `rebuild` | Full clean + build | -The two PHPT runners differ in two ways: - -- `arrow-ext` runs each test with `php -n`, so `php.ini` is ignored and no other extension is loaded. - `flow-php-ext` does not, so the ambient extensions load alongside it. -- `flow-php-ext` honours `--SKIPIF--` blocks and reports a skipped count. `arrow-ext` ignores them. +The two PHPT runners are the same shell loop: each test runs with `php -d extension=` (not `php -n`, so +the ambient `php.ini` and its extensions load alongside), `--SKIPIF--` blocks are honoured and the skipped count is +reported, and the output is compared exactly (`--EXPECTF--` is accepted but `%s` is not expanded). `flow-php-ext` +additionally takes `ARROW_SO=` to pre-load arrow-ext, so the Parquet ETL phpts run instead of skipping: +`make test ARROW_SO=../arrow-ext/ext/modules/arrow.so`. ## Rebuilding after a source change @@ -157,17 +161,17 @@ Re-enter `nix-shell` to rebuild the derivation, or build and test the extension nix-shell --arg with-rust true --run "cd src/extension/flow-php-ext && make build && make test" ``` -This covers `crates/flow-batch-frame`, which the flow-php-ext build compiles from the same directory. +This covers every crate under `crates/`, which the flow-php-ext build compiles from the same workspace. ## Extension ABI Each extension registers an `int` constant with the version of the contract its PHP package expects. The PHP package holds the same number: -| Extension | Constant (Rust `lib.rs`) | PHP side | -|-----------|--------------------------|-------------------------------------------| -| flow_php | `FLOW_PHP_ABI` | `Flow\ETL\FlowPhpExtension::ABI` | -| arrow | `FLOW_ARROW_ABI` | `Flow\Parquet\Engine\ArrowExtension::ABI` | +| Extension | Constant (Rust `lib.rs`) | PHP side | +|-----------|--------------------------|---------------------------------------------| +| flow_php | `FLOW_PHP_ABI` | `Flow\ETL\Foundation\FlowPhpExtension::ABI` | +| arrow | `FLOW_ARROW_ABI` | `Flow\Parquet\Engine\ArrowExtension::ABI` | Bump both, in one change, whenever the contract changes: a class or interface the extension registers, or the schema, options or values that cross it. Every `Adaptive*` pick calls `available()`: an extension that is loaded with @@ -177,17 +181,25 @@ another ABI, or with none, is refused with a `RuntimeException` naming the exten The PHP interfaces an extension also registers are guarded by `interface_exists(X::class, false)`, so an extension that does not register one still gets the PHP declaration and fails at the ABI check, not with "Class not found". -## The flow-batch-frame crate +## The flow-php-ext workspace -`src/extension/flow-php-ext/crates/flow-batch-frame` holds Flow's batch layout in pure Rust: one column's buffers in the -canonical form of `Column::encode()` and back, and the BATCH frame body around them. It returns every refusal as a data -`Error` variant; `flow_php` renders each into the PHP message. flow-php-ext depends on it by `path`: +`src/extension/flow-php-ext` is a cargo workspace: the root package is the extension (PHP glue only: ext-php-rs classes, +Zval conversions, exception text), the members under `crates/` are pure Rust with no PHP types. Every `#[test]` lives in +a member, so the test binaries link without a PHP engine; a member that reaches a Zend symbol fails to link on Linux, +which CI runs for every push. Each member returns its refusals as data (`Error`, `Refusal`, `Malformed`); `flow_php` +renders each into the PHP message. `flow-batch-frame` holds Flow's batch layout: one column's buffers in the canonical +form of `Column::encode()` and back, the builders that append rows into it, and the BATCH frame body around them. ```bash -nix-shell --arg with-rust true --run "cd src/extension/flow-php-ext/crates/flow-batch-frame && cargo test" +nix-shell --arg with-rust true --run "cd src/extension/flow-php-ext && cargo test --workspace --features arrow-validate" +nix-shell --arg with-rust true --run "cd src/extension/flow-php-ext && cargo fmt --all --check && cargo clippy --workspace --all-targets -- -D warnings" ``` -arrow-ext does not depend on the crate. Its `canonical_type()` (`src/parquet/canonical.rs`) must produce the Arrow +`arrow-validate` turns on `arrow-data/force_validate`, which re-validates every array the builders hand over with +`build_unchecked`; `make test` builds the extension with it, `make build` without. The same feature and the same three +commands (without `--workspace`) apply to `src/extension/arrow-ext`. + +arrow-ext does not depend on the workspace. Its `canonical_type()` (`src/parquet/canonical.rs`) must produce the Arrow types of flow-batch-frame's `kind::data_type()`, pinned by flow-php-ext phpt 091. ## Releasing diff --git a/documentation/contributing/wasm.md b/documentation/contributing/wasm.md index 7b3e215ca6..47d5fccfcf 100644 --- a/documentation/contributing/wasm.md +++ b/documentation/contributing/wasm.md @@ -273,12 +273,12 @@ What is now true: - `pack()`/`unpack()` 64-bit format codes (`q`, `Q`, `J`, `P`) work. `Flow\Floe` therefore works, and so does **`sortBy()`** and everything else that spills runs to disk through - `Flow\ETL\Column\Layout\Int64Layout` (`pack('P*')`). Before this, every Floe write in the browser died with + `Flow\ETL\Values\Column\Layout\Int64Layout` (`pack('P*')`). Before this, every Floe write in the browser died with `ValueError: 64-bit format codes are not available for 32-bit versions of PHP`. - Large integers from Thrift/Parquet metadata no longer overflow to float. On the 32-bit build, reading a parquet file emitted hundreds of `Warning: The float 2147483648 is not representable as an int` from - `Flow\Parquet\Thrift\CompactProtocol` and `Flow\ETL\Bucketing\HashBucketing`; the 64-bit build + `Flow\Parquet\Thrift\CompactProtocol` and `Flow\ETL\Kernels\Bucketing\HashBucketing`; the 64-bit build emits none. - Hex constants like `0x80000000` are plain integers and need no `(int)` cast. - `DateTimePhysical` stores `getTimestamp() * 1_000_000 + µs`, which now represents dates past 2038-01-19. diff --git a/documentation/installation/docker.md b/documentation/installation/docker.md index 55c0432b43..80eeb52354 100644 --- a/documentation/installation/docker.md +++ b/documentation/installation/docker.md @@ -95,7 +95,7 @@ The backend is chosen per data frame, the Parquet engine per read: ```php read(...)` pulls rows from a source (file, API, database). - **Transform** - `->filter()`, `->withColumn()`, `->join()`, `->groupBy()`, `->window()`, and friends shape the rows. -- **Load** - `->write(...)` streams the result into a sink; `->run()` executes +- **Load** - `->write(...)` streams the result into a loader; `->run()` executes the whole thing. Data moves through the pipeline as **DataFrame → Rows** batches of columns. Every `Rows` @@ -68,7 +68,7 @@ batch carries the schema that types its columns. - [DSL Reference](/documentation/dsl/core) - every function in the DSL, with signatures and examples. -**Working with a specific source or sink** +**Working with a specific source or destination** - Browse [Adapters](/documentation/components/adapters/csv) - CSV, JSON, XML, Parquet, Avro, Excel, HTTP, PostgreSQL, Doctrine, SEAL, ChartJS. diff --git a/documentation/quick-start.md b/documentation/quick-start.md index 62bf7161d7..6289420e3f 100644 --- a/documentation/quick-start.md +++ b/documentation/quick-start.md @@ -62,7 +62,7 @@ data_frame() ->read(from_csv(__DIR__ . '/orders_flow.csv')); ``` -In this example we're using the `from_csv()` function to create a new instance of the `Flow\ETL\Adapter\CSV\CSVExtractor` class. +In this example we're using the `from_csv()` function to create a new instance of the `Flow\ETL\Adapter\CSV\IO\Extractor\CSVExtractor` class. All file-based extractors accept [glob path patterns](/documentation/components/libs/filesystem.md#glob-patterns), allowing you to read multiple files at once. @@ -101,7 +101,7 @@ You can find all available functions in the [DSL](/src/core/etl/src/Flow/ETL/DSL > [!TIP] > DSL is nothing more than a set of functions that return instances of Flow PHP objects. > You can always create objects directly, but DSL is a more convenient way to build a pipeline. -> All available ETL functions can be found in the [Function](/src/core/etl/src/Flow/ETL/Function) namespace. +> All available ETL functions can be found in the [Functions](/src/core/etl/src/Flow/ETL/Functions) namespace. [Transformations Examples](/transformations/#example) @@ -145,5 +145,5 @@ Flow ETL is using lazy execution, which means that the pipeline will not be exec ``` There are few more triggering functions, like `fetch()`. You can find which functions are `@lazy` or `@trigger` by looking at -the [DataFrame](/src/core/etl/src/Flow/ETL/DataFrame.php) source code. +the [DataFrame](/src/core/etl/src/Flow/ETL/API/DataFrame.php) source code. diff --git a/documentation/upgrading.md b/documentation/upgrading.md index c44f62c66e..868bf43bca 100644 --- a/documentation/upgrading.md +++ b/documentation/upgrading.md @@ -19,7 +19,10 @@ Upgrading from older versions: [0.35.x to 0.40.x](/documentation/upgrading-0.35- | `ListType>`, `MapType>` | `ListType`, `MapType` | | Mago `plugins = ["flow-php"]` infers `type_structure()` shapes | Built-in plugin no longer infers them - use `flow-php/mago-types-bridge` (`plugins = ["flow/types"]`) | -### 2) `flow-php/parquet` - statistics are written and read as arrow-rs does; one physical schema for one logical schema on both engines +### 2) + +`flow-php/parquet` - statistics are written and read as arrow-rs does; one physical schema for one logical schema on +both engines | Before | After | |------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -30,14 +33,16 @@ Upgrading from older versions: [0.35.x to 0.40.x](/documentation/upgrading-0.35- | the packer/unpacker encoded FLBA/BYTE_ARRAY decimals and UUIDs; `DecimalConverter` took INT-backed decimals only | `new DecimalConverter($physicalType, $precision, $scale, $byteLength)` and `UuidConverter` convert to the physical bytes; the packer/unpacker take them as-is | | `FlatColumn::int32()` / `int64()` wrote converted `INT_32` / `INT_64` | plain `INT32` / `INT64`, as arrow-rs | | a column with a logical type and no converted type written without one (DuckDB read a logical `UINT_64` as signed) | the converted type derived from the logical type, as arrow-rs; a `FlatColumn` whose converted and logical types disagree is refused, on read and write | -| the arrow writer named map entries `m.entries.key` | `m.key_value.key` (LogicalTypes.md): `COLUMNS_*` options keyed by `…key_value.key` address the same leaf on both engines | +| the arrow writer named map entries `m.entries.key` | `m.key_value.key` (LogicalTypes.md): `COLUMNS_*` options keyed by `...key_value.key` address the same leaf on both engines | | the arrow writer wrote `FlatColumn::enum()` as `STRING` | `ENUM` | | `Option::ARROW_BATCH_SIZE`, read by nothing | removed | | `Flow\Parquet\Dremel\Statistics\Comparator` | removed: `Flow\Parquet\Data\PhysicalOrder` | | `new StatisticsCounter($column)` | `new StatisticsCounter($column, new StatisticsTruncation($length))` | | `new UnsignedIntegerConverter($path, $bitWidth)` | `new UnsignedInt32Converter()`; UINT_64 is `UnsignedInt64Converter` | -### 3) `flow-php/parquet`, `flow-php/etl-adapter-parquet` - `UINT_64` reads as a decimal string on both engines (reverses rows 84 and 85 of 0.44→0.45 for `UINT_64`) +### 3) `flow-php/parquet`, `flow-php/etl-adapter-parquet` - + +`UINT_64` reads as a decimal string on both engines (reverses rows 84 and 85 of 0.44→0.45 for `UINT_64`) | Before | After | |-----------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------| @@ -46,13 +51,16 @@ Upgrading from older versions: [0.35.x to 0.40.x](/documentation/upgrading-0.35- | writes accepted an int from 0 to `PHP_INT_MAX` into `UINT_64` | an int >= 0 or a canonical digit string up to `18446744073709551615` | | `to_parquet()` refused a `numeric_string` column (`NumericStringType is not supported`) | written as a parquet `STRING` | -### 4) `flow-php/parquet` - the `arrow` extension must be rebuilt (arrow-rs 60, ABI 2) +### 4) `flow-php/parquet` - the `arrow` extension must be rebuilt (arrow-rs 60, ABI 3) | Before | After | |---------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| -| arrow-ext built against arrow-rs 59, `FLOW_ARROW_ABI` 1 | arrow-rs 60, `FLOW_ARROW_ABI` / `ArrowExtension::ABI` 2 (covering every contract change below): an older build is refused by `ArrowExtension::available()` | +| arrow-ext built against arrow-rs 59, `FLOW_ARROW_ABI` 1 | arrow-rs 60, `FLOW_ARROW_ABI` / `ArrowExtension::ABI` 3 (covering every contract change below): an older build is refused by `ArrowExtension::available()` | + +### 5) `flow-php/parquet`, `flow-php/etl-adapter-parquet` - reads take a `Query` (or, below the file, a -### 5) `flow-php/parquet`, `flow-php/etl-adapter-parquet` - reads take a `Query` (or, below the file, a `ReadPlan`) instead of limit/offset, the PHP writer writes the page index, the reader exposes its `Footer`; the `arrow` and `flow_php` extensions must be rebuilt +`ReadPlan`) instead of limit/offset, the PHP writer writes the page index, the reader exposes its `Footer`; the +`arrow` and `flow_php` extensions must be rebuilt | Before | After | |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -94,23 +102,26 @@ Upgrading from older versions: [0.35.x to 0.40.x](/documentation/upgrading-0.35- | a `numeric_string` column was comparable only with another `numeric_string` | comparable with a `string` literal too (`ref('id')->greaterThan(lit('1000'))`), and ordered numerically; still not comparable with an `integer` | | `Between\Boundary::compareValues(mixed $left, mixed $right)` | removed: `(new ValueOrder(...$operandTypes))->compare($left, $right)`; `Boundary::compare($value, $lower, $upper, ValueOrder $order = new ValueOrder())` | -### 7) `flow-php/etl` - a file extractor reads a `FileScan` and returns only rows matching its conjuncts; `Read` and `Executor\Pipeline` carry it - -| Before | After | -|--------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `FileExtractor::extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles())` | `FileExtractor::scan(FlowContext $context, FileScan $scan)`, returning only rows matching every one of `$scan->conjuncts`; `extract(FlowContext $context, ?int $limit = null)` is `Extractor`'s, `use Scanning;` makes it `scan()` with a limit | -| `$extractor->extract($context, pathFilter: $filter)` | `$extractor->scan($context, new FileScan(pathFilter: $filter))` | -| `Read::limit()`, `Read::pathFilter()`, `new Read($extractor, $limit, $pathFilter)` | `Read::scan()->limit`, `->pathFilter`, `->conjuncts`; `new Read($extractor, new FileScan($limit, $pathFilter))` | -| `Read::withPathFilter()` on any extractor | refused (`InvalidArgumentException`) on an extractor that is no `FileExtractor`, as `withFilter()` | -| `Executor\Pipeline::limit()`, `pathFilter()`, `new Pipeline(..., $limit, $pathFilter, $sources)` | `Pipeline::scan()`; `new Pipeline(..., $scan, $sources)` | -| a `filter()` over a file source stayed a `Filter` step | each deterministic part of it is pushed into the source (`FileScan::$conjuncts`) and leaves the step, an emptied step is removed; a `limit()` after it is pushed beside it, so the read stops after that many matching rows; `explain()` prints each part as `Filter: …` under `Read` | -| `new FileReadLoop($fileColumns, $schema)`, `read($sources, $batches, $batchSize, Backend $backend, $window)` | `new FileReadLoop($fileColumns, $schema, $scan->conjuncts)`, `read($sources, $batches, $batchSize, FlowContext $context, $window)` | -| `Optimizer::default()` ran `PushLimitIntoSource` before `PushFilterIntoSource` | after it | -| `Rewrite::of(Node $node)` | `of(Node $original, Node $rebuilt)`: the node as it was and rebuilt over its rewritten children | -| `new Flow\ETL\Plan\ReplaceLeaf($target, $replacement)` | `ReplaceNodes::one($target, $replacement)`, or `new ReplaceNodes($replacements, $removals)` for several in one rewrite (a removed node gives way to its child) | -| `new ScalarFunctionFilterTransformer($function, ?ScalarFunction $resolved)` | the second argument is `?Function\Evaluation\Filter`; `bind()` sets it, as before | -| `explain()`: `Select` printed `Columns: …`, other nodes `Defines columns: …` | `Keeps only: …` and `Changes columns: …` (a column defined, renamed or dropped) | -| `Redefined::$unknown` | `Redefined::isUnknown()`; `Redefined::isAllBut()` tells an `allBut()` apart | +### 7) `flow-php/etl` - a file extractor reads a `Criteria` and returns only rows matching its conjuncts; `Read` and + +`Executor\Pipeline` carry it + +| Before | After | +|--------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `FileExtractor::extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles())` | `FileExtractor::extract(Backend $backend, Criteria $criteria)`, returning only rows matching every one of `$criteria->conjuncts` | +| `$extractor->extract($context, pathFilter: $filter)` | `$extractor->extract($backend, new Criteria(pathFilter: $filter))` | +| `Read::limit()`, `Read::pathFilter()`, `new Read($extractor, $limit, $pathFilter)` | `Read::criteria()->limit`, `->pathFilter`, `->conjuncts`; `new Read($extractor, new Criteria($limit, $pathFilter))` | +| `Read::withPathFilter()` on any extractor | refused (`InvalidArgumentException`) on an extractor that is no `FileExtractor`, as `withFilter()` | +| `Executor\Pipeline::limit()`, `pathFilter()`, `new Pipeline(..., $limit, $pathFilter, $sources)` | `new Pipeline($id, Plan\Node\Read\|Pipeline\|null $input, PipelineSteps $steps)`; the read is the `Read` node, its criteria `$pipeline->input->criteria()`; the counters are an `Executor::execute()` argument (row 95) | +| a `filter()` over a file source stayed a `Filter` step | each deterministic part the source uses is pushed into it (`Criteria::$conjuncts`) and leaves the step: a part over partition columns only, on every source, and a part `FileExtractor::prunes()` accepts (Parquet: a part it translates into a Parquet filter); an emptied step is removed; a `limit()` after it is pushed beside it, so the read stops after that many matching rows; `explain()` prints each pushed part as `Filter: ...` under `Read`; a partition-only part after a kept one stays in the step and still prunes the listing (`explain()` prints it as `Files:`) | +| a custom `FileExtractor` | implements `prunes(ScalarFunction $conjunct): bool`: true only when the source skips data with the conjunct before it decodes rows; `false` keeps the conjunct in its `Filter` step | +| `new FileReadLoop($fileColumns, $schema)`, `read($sources, $batches, $batchSize, Backend $backend, $window)` | `new FileReadLoop($fileColumns, $schema, $criteria->conjuncts)`, `read($sources, $batches, $batchSize, Backend $backend, $window)` | +| `Optimizer::default()` ran `PushLimitIntoSource` before `PushFilterIntoSource` | after it | +| `Rewrite::of(Node $node)` | `of(Node $original, Node $rebuilt)`: the node as it was and rebuilt over its rewritten children | +| `new Flow\ETL\Plan\ReplaceLeaf($target, $replacement)` | `ReplaceNodes::one($target, $replacement)`, or `new ReplaceNodes($replacements, $removals)` for several in one rewrite (a removed node gives way to its child) | +| `new ScalarFunctionFilterTransformer($function, ?ScalarFunction $resolved)` | the second argument is `?Functions\Evaluation\Filter`; `bind()` sets it, as before | +| `explain()`: `Select` printed `Columns: ...`, other nodes `Defines columns: ...` | `Keeps only: ...` and `Changes columns: ...` (a column defined, renamed or dropped) | +| `Redefined::$unknown` | `Redefined::isUnknown()`; `Redefined::isAllBut()` tells an `allBut()` apart | A custom file extractor: @@ -122,15 +133,21 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi } // after -use Scanning; // extract() is scan() with only a limit +public function extract(Backend $backend, Criteria $criteria): Generator +{ + // reads with $criteria->limit and $criteria->pathFilter, returns only rows matching every one of + // $criteria->conjuncts +} -public function scan(FlowContext $context, FileScan $scan): Generator +public function prunes(ScalarFunction $conjunct): bool { - // reads with $scan->limit and $scan->pathFilter, returns only rows matching every one of $scan->conjuncts + return false; // true only when the source skips data with $conjunct } ``` -### 8) `flow-php/telemetry`, `flow-php/psr18-telemetry-bridge`, `flow-php/symfony-telemetry-bundle` - HTTP URLs are redacted, stack traces carry no argument values +### 8) `flow-php/telemetry`, `flow-php/psr18-telemetry-bridge`, + +`flow-php/symfony-telemetry-bundle` - HTTP URLs are redacted, stack traces carry no argument values | Before | After | |---------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -138,7 +155,10 @@ public function scan(FlowContext $context, FileScan $scan): Generator | Symfony HttpClient `url.full`, `url.scheme`, `server.address`, `server.port` came from the `$url` argument (`/me`, `unknown` with `base_uri`) | they come from the effective URL, `base_uri` and the `query` option included | | `exception.stacktrace` was `getTraceAsString()`, with argument values when `zend.exception_ignore_args` is off | `StackTrace::format()`: every frame renders `()`, whatever `zend.exception_ignore_args` is | -### 9) `flow-php/etl`, `flow-php/etl-adapter-*` - an empty source without a declared schema fails at the source; Parquet and Floe accept `withSchema()` +### 9) `flow-php/etl`, + +`flow-php/etl-adapter-*` - an empty source without a declared schema fails at the source; Parquet and Floe accept +`withSchema()` | Before | After | |---------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------| @@ -147,7 +167,10 @@ public function scan(FlowContext $context, FileScan $scan): Generator | `from_parquet(...)->withSchema()`, `from_floe(...)->withSchema()` threw `InvalidArgumentException` | read every file by name against the declared schema: undeclared columns are not read, a missing nullable column is null | | `from_cache($id)` without a cache instance described an empty schema | `SchemaNotDerivableException`; the plan runs unbound | -### 10) `flow-php/parquet` - the PHP engine reads and writes DELTA_LENGTH_BYTE_ARRAY / DELTA_BYTE_ARRAY; both engines write the same default encodings (dictionary on, arrow-rs fallbacks); COLUMNS_ENCODINGS is literal on both engines +### 10) + +`flow-php/parquet` - the PHP engine reads and writes DELTA_LENGTH_BYTE_ARRAY / DELTA_BYTE_ARRAY; both engines write the +same default encodings (dictionary on, arrow-rs fallbacks); COLUMNS_ENCODINGS is literal on both engines | Before | After | |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -172,31 +195,1669 @@ public function scan(FlowContext $context, FileScan $scan): Generator The `arrow` extension must be rebuilt. -### 11) `flow-php/etl`, `flow-php/etl-adapter-postgresql`, `flow-php/etl-adapter-excel`, `flow-php/etl-adapter-seal`, `flow-php/etl-adapter-json`, `flow-php/cli` - "entry" renamed to "column" - -| Before | After | -|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------| -| `DataFrame::withEntry()` / `withEntries()` | `withColumn()` / `withColumns()` (old names deprecated) | -| `with_entry()`, `Flow\ETL\WithEntry` | `with_column()`, `Flow\ETL\WithColumn` | -| `entry()` | removed, use `ref()` or `col()` | -| `Definition::entry()` | `Definition::reference()` | -| `SchemaDefinitionNotFoundException::entry()` | `column()` | -| `RenameEntryStrategy`, `Rename{Case,Map,Replace}EntryStrategy` | `RenameColumnStrategy`, `Rename{Case,Map,Replace}ColumnStrategy` | -| `RenameEntryTransformer`, `RenameEachEntryTransformer` | `RenameColumnTransformer`, `RenameEachColumnTransformer` | -| `{Select,Drop,Prune}EntriesTransformer` | `{Select,Drop,Prune}ColumnsTransformer` | -| `DuplicatedEntriesException` | `DuplicatedColumnsException` | -| `Plan\Node\{WithColumn,WindowColumn,ExpandColumn}::$entry`, `Plan\Node\{Select,Drop,Distinct,DuplicateRow}::$entries` | `$column`, `$columns` | -| `new SequenceExtractor($generator)` wrote a column named `entry` | the column name is required: `new SequenceExtractor($generator, 'name')` | -| `Flow\ETL\Adapter\PostgreSql\EntryTypesMap` | `Flow\ETL\Adapter\PostgreSql\TypesMap` | -| `ExcelLoader::withSheetNameFromEntry()` | `withSheetNameFromColumn()` | -| `SealLoader::withIdentifierEntry()` | `withIdentifierColumn()` | -| `JsonExtractor::withPointer($pointer, pointerToEntryName: true)` | `pointerToColumnName: true` | -| CLI option `--input-json-pointer-entry-name` | `--input-json-pointer-column-name` | -| named arguments `entry:` / `entries:` (`select`, `drop`, `sortBy`, `groupBy`, `duplicateRow`, `ref`, `partition_by`, …), `entry_name:` (`from_sequence_*`), `entryLeft:` / `entryRight:` (`Join\Comparison\{Equal,Identical}`) | `column:` / `columns:`, `column_name:`, `leftReference:` / `rightReference:` | -| exception messages saying `entry` / `entries` for a column (`Schema definition for entry "x" not found`, …) | say `column` / `columns` | +### 11) `flow-php/etl`, `flow-php/etl-adapter-postgresql`, `flow-php/etl-adapter-excel`, `flow-php/etl-adapter-seal`, + +`flow-php/etl-adapter-json`, `flow-php/cli` - "entry" renamed to "column" + +| Before | After | +|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------| +| `DataFrame::withEntry()` / `withEntries()` | `withColumn()` / `withColumns()` (old names deprecated) | +| `with_entry()`, `Flow\ETL\WithEntry` | `with_column()`, `Flow\ETL\Functions\WithColumn` | +| `entry()` | removed, use `ref()` or `col()` | +| `Definition::entry()` | `Definition::reference()` | +| `SchemaDefinitionNotFoundException::entry()` | `column()` | +| `RenameEntryStrategy`, `Rename{Case,Map,Replace}EntryStrategy` | `Flow\ETL\Operators\Transformer\Rename\{RenameColumnStrategy, Rename{Case,Map,Replace}ColumnStrategy}` | +| `RenameEntryTransformer`, `RenameEachEntryTransformer` | `Flow\ETL\Operators\Transformer\{RenameColumnTransformer, RenameEachColumnTransformer}` | +| `{Select,Drop,Prune}EntriesTransformer` | `Flow\ETL\Operators\Transformer\{Select,Drop,Prune}ColumnsTransformer` | +| `DuplicatedEntriesException` | `DuplicatedColumnsException` | +| `Plan\Node\{WithColumn,WindowColumn,ExpandColumn}::$entry`, `Plan\Node\{Select,Drop,Distinct,DuplicateRow}::$entries` | `$column`, `$columns` | +| `new SequenceExtractor($generator)` wrote a column named `entry` | the column name is required: `new SequenceExtractor($generator, 'name')` | +| `Flow\ETL\Adapter\PostgreSql\EntryTypesMap` | `Flow\ETL\Adapter\PostgreSql\Values\Schema\TypesMap` | +| `ExcelLoader::withSheetNameFromEntry()` | `withSheetNameFromColumn()` | +| `SealLoader::withIdentifierEntry()` | `withIdentifierColumn()` | +| `JsonExtractor::withPointer($pointer, pointerToEntryName: true)` | `pointerToColumnName: true` | +| CLI option `--input-json-pointer-entry-name` | `--input-json-pointer-column-name` | +| named arguments `entry:` / `entries:` (`select`, `drop`, `sortBy`, `groupBy`, `duplicateRow`, `ref`, `partition_by`, ...), `entry_name:` (`from_sequence_*`), `entryLeft:` / `entryRight:` (`Join\Comparison\{Equal,Identical}`) | `column:` / `columns:`, `column_name:`, `leftReference:` / `rightReference:` | +| exception messages saying `entry` / `entries` for a column (`Schema definition for entry "x" not found`, ...) | say `column` / `columns` | + +--- + +### 12) `flow-php/etl`, `flow-php/etl-adapter-csv`, `flow-php/etl-adapter-json`, `flow-php/etl-adapter-excel`, + +`flow-php/etl-adapter-google-sheet` - `SchemaSampler` + `SniffsColumnTypes` merged into `Sniffer` + +Schema inference samples a source through one contract. Each adapter builds its `Sniffer` inside `schema()`; a unit is +one +listed file (one sheet range, one in-memory dataset). A bounded `sampleSize` still drains one budget across the units in +listing order; `sampleSize(-1)` sniffs every unit whole and independently and merges the partials in unit order, so the +result does not depend on the order the units ran in. + +| Before | After | +|---------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Flow\ETL\Schema\Inference\SchemaSampler`, `SniffsColumnTypes` | `Flow\ETL\IO\Inference\Sniffer` - `names(): list`, `units(): int`, `sniff(int $unit, int $rowBudget): Sample`, `bytes(int $unit): int`, `parts(int $unit, int $targetBytes): non-empty-list`, `close(): void` | +| `new SchemaInferrer($inference, $typer)->infer($names, $samples)` | `new SchemaInferrer($inference)->infer($sniffer): Inferred` (`->schema`, `->units`) - the narrower is the sniffer's; `sampleSize(-1)` sniffs through `infer_schema()->sniffWith($launcher)`, in this process by default | +| `SchemaInferrer::sniff($names, $rows, $rowBudget)` | `ColumnTypes::sniff($names, $rows, $rowBudget, $typer)` | +| `CSVOpenSource::sniff($names, $rowBudget, $inference, $typer)` | `CSVOpenSource::sniff($names, $rowBudget, $typer)` (PHP, adaptive and `flow_php` native sources) | +| `CSVFileReader::samples()`, `CSVFileSample`, `CSVSamples` | `CSVSniffer(CSVSourceOpener $opener, $sources, InferredTypes $candidates, ?CSVHeader $header = null)` - `header()`; what a sniff read is on its `Sample` | +| `JsonFileReader::samples()`, `JsonSamples`, `new JsonFileReader(..., $sources)` | `JsonSniffer($reader, $sources)`; the reader no longer takes the listing; `JsonFileSample::bytes()` / `complete()` | +| `Flow\ETL\Adapter\CSV\{CSVSampledFiles, CSVSampledRows}` | removed: `Flow\ETL\IO\Inference\SampledUnits` (the row estimate over every sampled file) and `Flow\ETL\IO\Inference\Sample` (one file's sniff: columns, bytes, complete) | +| `Flow\ETL\Adapter\JSON\JSONMachine\{JsonSampledFiles, JsonSampledRows}` | removed: `Flow\ETL\IO\Inference\SampledUnits` and `Flow\ETL\IO\Inference\Sample`, as for CSV | +| `WorkbookSampler($workbook, $files)` | `ExcelSniffer($workbook, $files, InferredTypes $candidates, ?ExcelHeader $header = null)` | +| `GoogleSheetSampler($reader, $rowBudget)` | `GoogleSheetSniffer($reader, $rowBudget, $typer)` | +| `InMemoryRows::samples()`, `SpilledRows::samples()` | `sniff(0, $rowBudget)` - one unit, no header | +| `sampleSize(-1)`: `filesToSniff` counted files that yielded a row | `sampleSize(-1)`: every listed unit up to `filesToSniff` is sniffed whole and independently; empty files count | +| `ColumnTypes::merge()`, `ColumnTypes::schema()` | `ColumnTypes::folded(): SniffedColumns`, then `SniffedColumns::merge()` / `schema()` | + +--- + +### 13) `flow-php/etl`, `flow-php/filesystem`, `flow-php/parquet`, `flow-php/etl-adapter-csv`, + +`flow-php/etl-adapter-parquet` and every file adapter - file sources say where they may be cut + +Additive. A `SplittableExtractor` cuts its source into `FileSplit`s at plan time by arithmetic alone - one split per +file for JSON documents, text, XML, Excel and listings; byte windows for CSV, JSON lines, Parquet and Floe - and reads +one through `scan()` with the split on its `Criteria`. Splits cover every row exactly once, in source order. A CSV +window resyncs both ends to a record start, so a quoted newline never splits a record; a JSON lines window resyncs both +ends to a line start; a Parquet or Floe window owns the row groups or sections whose midpoint it holds. Through a +`Launcher` with more than one slot, set with `config_builder()->launcher()`, the splits are read +in worker processes. One launcher serves the whole run, joined frames included. + +| Before | After | +|-------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Criteria(?int $limit, Filter $pathFilter, list $conjuncts)` | `Criteria(..., ?FileSplit $split = null)`, `withSplit(FileSplit)`; `narrows()` is true with a split; a read with a split lists nothing but the split's file | +| every file extractor read whole files only | `SplittableExtractor extends Extractor { splits(Criteria, int $targetSplitBytes): non-empty-list; reader(): Reader }` on `files()`, `from_csv()`, `from_json()`, `from_json_lines()`, `from_text()`, `from_xml()`, `from_excel()`, `from_parquet()`, `from_floe()`; each split is read with `extract($backend, $criteria->withSplit($split))`, or through the config's launcher | +| `SelfDescribingFile { close(), schema(), source(), statistics() }` | `+ units(): list` - row groups (Parquet) or sections (Floe) as byte ranges; a window owns the units whose midpoint it holds | +| `ParquetFileBatches(..., array $conjuncts)`, `FloeFileBatches(..., ?DerivedSchema $derived)`, `CSVFileBatches(CSVFileReader, ?InferredColumns)` | each reads `Criteria::$split` in `batches()` (see 121) | +| `AdaptiveCSVOpenSource` opened and detected the dialect itself | `CSVSourceOpener(Filesystem, CSVReadOptions, ?FlowPhpExtension)` - `open(SourceFile)`, `openSplit(SourceFile, FileSplit, list $columns)`, `dialect()`, `over()`; `AdaptiveCSVOpenSource` delegates to it | +| `CSVDecoder(..., bool $emptyToNull)`, `RecordDecoder(..., bool $dropExtraColumns)`, `RustCSVOpenSource(..., ?int $charactersReadInLine)` | each gains `?array $headers = null` - the names of a stream that carries no header record (a window past a file's first record); `withHeader` must be false with them | + +### 14) `flow-php/filesystem-async-aws-bridge`, `-azure-bridge`, `-sftp-bridge` - filesystems take a client source + +| Before | After | +|--------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `new AsyncAWSS3Filesystem($mount, $bucket, S3Client $s3Client, $options)` | `new AsyncAWSS3Filesystem($mount, $bucket, S3ClientSource $client, $options)` - `new LiveS3Client($s3Client)` or `new ConfiguredS3Client($configuration)` | +| `new AzureBlobFilesystem($mount, BlobServiceInterface $blobService, $options)` | `new AzureBlobFilesystem($mount, BlobServiceSource $blobService, $options)` - `new LiveBlobService(...)` or `ConfiguredBlobService::fromArray([...])` | +| `new SFTPFilesystem($mount, SFTP $sftp, $options)` | `new SFTPFilesystem($mount, SFTPClientSource $client, $options)` - `new LiveSFTPClient($sftp)` or `ConfiguredSFTPClient::fromArray([...])` | +| async-aws `Options::withBlockSize()` refused 5 MiB | 5 MiB, S3's inclusive minimum part size, is accepted | + +`aws_s3_filesystem()`, `azure_filesystem()` and `sftp_filesystem()` keep their parameter names and also accept a plain +configuration array; only a filesystem built from configuration can be described for a worker process. The Symfony +filesystem bundle builds the configured form unless an HTTP client, a factory or a logger is injected. --- +### 15) `flow-php/filesystem`, `flow-php/parquet` - every filesystem and parquet engine is configurable; the + +`arrow` extension must be rebuilt (ABI 3) + +| Before | After | +|--------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `interface Filesystem` (10 methods) | `+ normalize(): ?array`, `+ static fromArray(array $configuration): static` - the plain configuration a worker rebuilds the filesystem from; `null` / throw when the instance holds live state (a client, memory). Core wraps it in `Flow\ETL\Foundation\Describe\ConfiguredFilesystem` | +| `interface ParquetEngine` (3 methods) | the same two methods (`fromArray(): self`); `PhpParquetEngine` and `AdaptiveParquetEngine` normalize `{byte_order, options}`, `RustParquetEngine` `{options}`; wrapped by `Flow\ETL\Adapter\Parquet\IO\ConfiguredParquetEngine` | +| `FLOW_ARROW_ABI` / `ArrowExtension::ABI` 2 | 3: the interface the extension registers gained the two methods; an older build is refused by `ArrowExtension::available()` | +| `Path` | `+ normalize(): PathData`, `+ static fromArray(array)`; `NativeLocalFileBlocksFactory::blockLocation()`; `OptionalType::assert()` is documented as returning `null \|T` | + +--- + +### 16) `flow-php/etl` - `BucketsStorage::close()` + +| Before | After | +|-----------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `interface BucketsStorage { append(), get(), remove(), set() }` | `+ close(string $bucketId): void` - finishes the bucket's pending writes so another process can read it; a no-op without any. `FilesystemBuckets implements Describable` | + +--- + +### 17) `flow-php/etl` - the default batch size is 2048 rows + +| Before | After | +|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `batchSize()` is 100 for every extractor using `Batches` | 2048 - the per-batch cost dominated the read at 100 rows (a parallel read handed rows back in 100-row Floe frames; 10 GiB CSV: 51 s to 39 s single-process, 28 s to 9 s at 8 workers). `withBatchSize()` is unchanged; `PostgreSql*`/`Dbal*` extractors keep 1000. `FilesystemBuckets` defaults to 2048 as well, so a handed-back batch is never re-sliced | + +### 18) `flow-php/etl` and every adapter - functions, transformers, loaders and error handlers describe themselves + +| Before | After | +|---------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `interface ScalarFunction`, `Transformer`, `Loader`, `ErrorHandler` | each `extends Describable`: implement `describe(): ?Description` and `static fromDescription(array $data, Rebuild $rebuild): static`; return `null` / throw when the instance holds live state - it then always runs in the parent process | + +### 19) `flow-php/etl`, `flow-php/etl-adapter-*` - `FileLoader` copies itself to another destination; + +`{part}` is reserved + +| Before | After | +|-------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `interface FileLoader { destination(), saveMode() }` | `interface FileLoader extends Loader`, `+ inTask(Part $task): static`, `+ commit(string $run, int $tasks): ?FileCommit` | +| `interface PartitioningLoader { partitionBy() }` | removed: `FileLoader` takes `partitionBy(Partitioning $partitioning): static` and `partitioning(): Partitioning` | +| `{part}` in a destination named a partition column `part` | `{part}` is the split part: `to_parquet('out/orders-{part}.parquet')` writes `out/orders-00000.parquet` in one process and `out/orders-0000k.parquet` per worker | +| `partitionBy('part')` with a `'.../{part}/...'` destination | throws `InvalidArgumentException` when the plan is built: rename the column | +| SaveMode on a `{part}` destination checked per file | checked over every existing part (`orders-*.parquet`): `ExceptionIfExists`/`Ignore` see any part, `Overwrite` removes every stale part, `Append` names carry a run id | + +### 20) `flow-php/filesystem` - `NativeLocalFilesystem::mv()` creates the target's parent directory + +| Before | After | +|----------------------------------------------------------|-------------------------------------------------------------| +| `mv()` into a missing directory fails (`rename()` error) | the missing parent directories are created, then `rename()` | + +### 21) `flow-php/etl` - aggregating and window functions describe themselves and expose a partial state + +| Before | After | +|--------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `interface AggregatingFunction extends FunctionTree` | `extends FunctionTree, Describable`; `+ partialType(): StructureType`, `+ partial(): array`, `+ mergePartial(array $partial, FlowContext $context): void` - every custom aggregator implements them (a group by read by workers merges partials) | +| `interface WindowFunction extends FunctionTree`, `interface WindowFrame` | each `extends Describable`; return `null` / throw when the instance holds live state - it then runs in the parent process | +| `Aggregators::cloned()` | `Aggregators::fresh()` - `withChildren(children())` per aggregator, never `clone`: a custom aggregator's `withChildren()` must return an instance without state | + +### 22) `flow-php/etl` - join expressions describe themselves; `HashJoinConfig` carries the broadcast limit + +| Before | After | +|------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| `interface Comparison` | `extends Describable`; a custom comparison returning `null` keeps the join in the parent process | +| `new HashJoinConfig($bucketing, $memoryLimit)` | `new HashJoinConfig($bucketing, $memoryLimit, $broadcastLimit)`; `hash_join()->broadcastLimit(Unit)` sets it, default 10 MB | + +### 23) `flow-php/etl` - `dropDuplicates()` runs as a processor + +| Before | After | +|-------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| `dropDuplicates()` planned as a `DropDuplicatesTransformer` step; a failure in it is offered to the error handler | planned as a `Flow\ETL\Operators\Processor\DistinctProcessor`; a failure in it propagates like any processor's; rows and streaming are unchanged | +| `explain()` prints `DropDuplicatesTransformer` among a segment's steps | `explain()` prints `DistinctProcessor` among the steps of its pipeline | + +### 24) `flow-php/etl` - the step factories take a built algorithm config + +| Before | After | +|---------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `GroupBySteps::of($groupBy, $config, ?GroupByAlgorithmBuilder $algorithm)` | `GroupBySteps::of($groupBy, $config, ?HashGroupByConfig $grouping)` - pass `$builder->build($config->cache->localFilesystemCacheDir, $config->backend())` | +| `SortSteps::of()` / `SortSteps::external($refs, $config, ?SortAlgorithmBuilder $algorithm)` | `SortSteps::of($refs, $config, null\|MemorySortConfig\|ExternalSortConfig $sort)`; `SortSteps::external($refs, ExternalSortConfig $sort, RandomValueGenerator $random): ExternalSort` | +| `RepartitionSteps::of($by, $config, ?RepartitionAlgorithmBuilder $algorithm)` | `RepartitionSteps::of($by, $config, ?HashRepartitionConfig $repartition)` | +| `JoinSteps::of($right, $on, $type, $config, ?JoinAlgorithmBuilder $algorithm)` | `JoinSteps::of($right, $on, $type, $config, ?HashJoinConfig $join)` | + +### 25) `flow-php/etl` - `count()` without a reference copies itself fresh + +| Before | After | +|------------------------------------------------------------------|--------------------------------------------------------------------------------------------------| +| `Count::withChildren([])` on `count()` returns the same instance | returns `new Count(null, $window)` without state, like every other aggregator's `withChildren()` | + +### 26) `flow-php/etl` - `Flow\Floe\Statistics` renamed to `FooterStatistics` + +| Before | After | +|------------------------|------------------------------------------| +| `Flow\Floe\Statistics` | `Flow\ETL\Storage\Floe\FooterStatistics` | + +The footer struct (`rows`, `byteSize`) is no longer a third `Statistics` beside `Flow\ETL\IO\Extractor\Statistics` and +`Flow\ETL\Runtime\Metrics\Statistics`. The Floe bytes are unchanged. + +### 27) `flow-php/filesystem`, `flow-php/etl` - `RangeSourceStream` exposes its window; the + +`flow_php` extension must be rebuilt (ABI 3) + +| Before | After | +|-------------------------------------------------------------|------------------------------------------------------------------------------------------------| +| `RangeSourceStream` reads only through `iterate()`/`read()` | `stream()`, `start()`, `end()` (exclusive) name the inner stream and the window it is cut from | +| `FlowPhpExtension::ABI` is `2` | `FlowPhpExtension::ABI` is `3`; a `flow_php` built before this change is refused | + +The native CSV reader opens a `NativeLocalSourceStream` (or a `RangeSourceStream` over one) by its path and reads the +window itself; every other stream is still read through its PHP `iterate()`. `NativeLocalFilesystem::readFrom()` no +longer creates the file's parent directory, and `NativeLocalSourceStream::size()` is the open handle's size. + +### 28) `flow-php/etl` - a Floe BATCH frame is decoded by the `Backend` + +| Before | After | +|------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| +| `(new FrameDecoder($codec))->decode($body, $schema, $backend)` | `$backend->decodeFrame($body, $schema, $codec)`; `Flow\Floe\FrameDecoder` is removed | +| `Backend` implementors: `builder`, `constant`, `decode`, `adopt`, `allocatedBytes` | also `decodeFrame(string $body, Schema $schema, Codec $codec): Rows` | + +`PhpBackend::decodeFrame()` is the former `FrameDecoder` walk. `RustBackend::decodeFrame()` reads the frame directory +natively and points the Arrow buffers into the body string instead of copying every buffer through `decode()`. + +### 29) `flow-php/etl` - a pipeline ends at a processor that needs every row; `Segment`, `Segments` and +`Materialization` removed + +| Before | After | +|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Processor` implementors: `bind`, `process` | also `breaks(): bool` - true when `process()` needs every input row before it emits one; such a processor ends its pipeline | +| a pipeline ended after a blocking node | it ends at a processor that `breaks()`: a loader after `collect()` runs in the next pipeline, a window's `WindowProcessor` opens the pipeline after its repartition | +| `Executor\Segment`, `Executor\Segments`, `Pipeline::segments()` | removed; `Pipeline::$steps` is a `PipelineSteps(list $steps, null\|Processor\|JoinStep $breaker)`, run by `StepRunner::steps()` | +| `new Pipeline(..., ?Pipeline $input)` | `new Pipeline($id, $input, $steps)`: `$input` is the pipeline read; `Pipeline::input()` stays | +| `Plan\Materialization`, `Node::materialization()` | removed | +| `explain()`: `streaming` / `blocking` on a declarations line, `Buffers all rows before passing them on` under a node | removed; `explain()->toString(Stage::physical)` shows where pipelines end | +| `Planner\PipelineSplit` | removed: `Planner::pipelines()` builds the physical plan | +| `new GroupByAggregationProcessor($groupBy, HashBucketing $bucketing, Buckets $buckets, Unit $memoryLimit, int $batchSize)` | `new GroupByAggregationProcessor($groupBy, BucketingConfig $config, RandomValueGenerator $random)` | +| `new PivotProcessor($groupBy, int $batchSize)` | `new PivotProcessor($groupBy, BucketingConfig $config)` | +| `new HashJoinProcessor($right, $executor, $on, $type, Buckets $left, Buckets $right, $random, Unit $memoryLimit, int $bucketsCount, int $batchSize)` | `new HashJoin($on, $type, HashJoinConfig $config, $random)` (row 94) | +| `new TopNProcessor($refs, $limit, ?ExternalSort $fallback)` | `new TopNProcessor($refs, $limit, MemorySortConfig\|ExternalSortConfig $config, RandomValueGenerator $random)` | +| `new MergeSortProcessor(ExternalSort $sort)` | `new MergeSortProcessor(ExternalSortConfig $config, References $refs, RandomValueGenerator $random)` | +| `InvalidLogicException::pipelineWithoutSource()` | removed - every pipeline reads an extractor or another pipeline | + +### 30) `flow-php/etl`, `flow-php/postgresql`, `flow-php/azure-sdk`, `flow-php/cli` - no traits + +| Before | After | +|--------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `use Flow\ETL\Extractor\Batches` | a `BatchSize` field set in the constructor, plus your own `withBatchSize(int): static` (`new BatchSize($size)`) and `batchSize(): int` (`->size`) | +| `use Flow\ETL\Extractor\File\FileReading` / `ListingPartitions` | a `FileColumnsSource(SourceFiles $files)` field: `columns(): FileColumns`; `new SourceFiles($filesystem, $path)` lists the files (`statuses($criteria)`) | +| `use Flow\ETL\Extractor\File\PathFiltering` | a `DerivedFromFiles` field: `fold()`, `footers()`, `from()`, `schema()`, `learn()`; `with*()` replaces it with `new DerivedFromFiles()` | +| `use Flow\ETL\Extractor\File\MetadataColumns` / `DeclaresPartitionTypes` | `FileColumnsSource::withMetadataColumns()` / `withPartitionTypes()` / `partitionTypes()`; the host replaces its field | +| `use Flow\ETL\Function\ResolvesFromChildren` | your own `resolved()` and `deterministic()`, each `true` when every child is | +| `use Flow\PostgreSql\QueryBuilder\AstToSql` | `public function toSql(): string { return (new AstDeparser())->deparse($this->toAst()); }` | +| `Statement::assert(SelectStatement::class)`, `StatementTrait` | `type_instance_of(SelectStatement::class)->assert($statement)` | +| `use Flow\Azure\SDK\Endpoints\UserAgentHeader` | removed; every endpoint options class keeps its `userAgentHeader()` / `withUserAgent()` | +| `Flow\CLI\Command\Traits\*Options`, `DBOptions::DB_CONNECTION_HELP` | `Flow\CLI\Options\*Options::configure(Command)`, `ConfigOption::configure()`, `SchemaInferenceOptions::apply()`, `DbConnectionOptions::HELP` | + +`ScalarFunctionChain` is the one trait left. `mago guard` refuses any new trait outside the test folders. + +### 31) `flow-php/parquet` - the options given to `Writer` apply on `open()` + +| Before | After | +|----------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| +| `Writer::open()` / `openForStream()` took `Option::ROW_GROUP_SIZE_CHECK_INTERVAL` from the engine's options | from the `Writer`'s options, as `write()` does | +| the arrow engine took `Option::ARROW_WRITE_BATCH_SIZE` from the options it was built with, on `open()` and `write()` alike | from the options given to `openForWrite()` (the `Writer`'s), returned by `OptionsConverter::toExtension()`, not `toEngine()` | + +### 32) `flow-php/etl` - `groupBy`, `dropDuplicates`, `repartition` and partitioned writes key rows by value identity + +| Before | After | +|----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------| +| `groupBy` / `dropDuplicates` kept `0.0` and `-0.0` apart | one key | +| `repartition()` / `partitionBy()` put the strings `"1"` and `"1.0"` in one partition | two partitions | +| a `groupBy` spilling past its memory limit returned a date key twice | once | +| `GroupBy::tag()` | removed: `(new KeyEncoder($columns))->of($rows)` keys a batch | +| `Hasher::hash(list> $values)` | `hash(list $keys)`: one key per row, from a `KeyEncoder` | +| `KeyValues::of()` returned `list>` | `list`, the join-match key | +| `new HashBucketing(list $by, ...)`, `HashBucketing::indexOf(list $values)` | `new HashBucketing(KeyEncoder $keys, ...)`; `indexOf()` removed: `index($hasher->hash([$key])[0])` | +| `new KeyGrouping(KeyValues $keys, ...)` | `new KeyGrouping(KeyEncoder $keys, Backend $backend)` | +| `RowHashes` hashed column names with the values | the values only, `KeyEncoder` over the sorted columns | + +### 33) `flow-php/etl` - `explain()` names classes through `ObjectExtractor::shortName()` + +| Before | After | +|-----------------------------------------------------------------------------------------------|------------------------------------------------------------------| +| an anonymous class showed as `Transformer@anonymous` followed by a NUL byte and its file path | `class@anonymous` | +| `Details::name(object $object)` | removed: `Flow\Telemetry\ObjectExtractor::shortName($object)` | +| `new StepDetails(Details $details, Condition $condition, StatisticsLine $statistics)` | `new StepDetails(StatisticsLine $statistics, NodeDetails $node)` | + +### 35) `flow-php/etl` - one physical plan class, planner internals simplified + +| Before | After | +|------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Executor\PhysicalPlan` interface, `Executor\Described`, `Executor\Raw` | `final readonly class PhysicalPlan(Pipeline $root, ?Schema $schema, ?SchemaNotDerivableException $refusal = null, bool $countsOnly = false)`; the config's `Launcher::read(PhysicalPlan $plan, ...): ?Launch` cuts the launch at run time | +| `$plan instanceof Described`, `Raw::$why`, `Described::$schema` | `$plan->described()`, `$plan->refusal()`, `$plan->schema()` | +| `NodeTranslator::algorithm()`, `NodeTranslator::toSteps($node, $context, $frames, $algorithm)` | removed; `steps(Node $node)` on an instance (row 103) | +| `Optimizer\TopNRewrite` | removed: `CombineSortAndLimit` is the `Rewrite` | +| `new SinkAttachment($planned, $context)`, `SinkAttachment::next()` | `new WritePlanner($planned)`, `WritePlanner::next()`; it returns `BranchPipeline` and `TransactionGroup` plan values (row 95) | +| `(new CountingProcessor())->rows($count, $backend)` / `->schema()` | `CountingProcessor::rows($count, $backend)` / `CountingProcessor::schema()` | +| logs `Error during ETL segment execution.` (and the `Skipping ...` variants) | `... ETL pipeline execution.` | + +### 36) `flow-php/etl` and the file adapters - file-read plumbing: one listing object, one read config, one header type + +| Before | After | +|--------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| `PartitionColumns` | removed: `FileColumns::declare()`; the names are listed by `FileColumnsSource::columns()` | +| `new FileColumns(PartitionColumns $columns, array $names, PartitionTypes $types, bool $metadataColumns)`, `FileColumns::fromArray($data, $filesystem)` | `new FileColumns(array $names, PartitionTypes $types, bool $metadataColumns)`, `FileColumns::fromArray($data)` | +| `new FileColumnsSource($filesystem, $path, $types, $metadataColumns)` | a fifth argument, `?array $partitionNames = null` (listed on the first `columns()` call when null) | +| `new CSVSplitReader($filesystem, $source, $fileColumns, $schema, $batchSize, ...)` (and the JSON, Excel, XML, Text, Parquet and Floe split readers) | `new FileSplitReader(new FileRead($filesystem, $source, $fileColumns, $schema, $batchSize), new CSVFileBatches(...))` (see 121) | +| `CSVFileReader` | removed: `new CSVFileBatches(CSVSourceOpener $opener, ?InferredColumns $inferred)`; `CSVSniffer::header()` | +| `CSVHeader`, `ExcelHeader` | `Flow\ETL\IO\Extractor\Grid\GridHeader` | +| `ExcelExtractor::reader(?ExcelSniffer $sampled)`, `ParquetExtractor::reader(?ParquetSourceFile $firstFile)` | `reader()` | +| a split reader rebuilt in a worker detected the CSV dialect of a file the parent had detected | `CSVSourceOpener` describes the dialects it learned | +| `DerivedFromFiles::learn(Schema $schema)` | `learn(Schema $schema, string $from = '')`; `from()` also names the learned source | +| `DerivedFromFiles::footers()` after `learn()` returned empty footer statistics | throws `InvalidLogicException` | +| `new InferredRows($source, $sample)`, `InferredRows::of(array $rows, Schema $schema, Backend $backend)` | `new InferredRows($source, $sample, Schema $schema, Backend $backend, int $batchSize)`; `add(array $record): ?Rows`, `flush(): ?Rows` | +| `JsonFileReader::batches(SourceFile $source, int $batchSize, ?FileSplit $window)` | removed: `sample($source, window: $window)` yields the records; `Flow\ETL\Values\Rows\RecordBatch` batches them | + +### 37) `flow-php/etl` - one listing extractor, Floe and row-building internals trimmed + +| Before | After | +|----------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------| +| `files()` returned `FilesExtractor` (a `SplittableExtractor`), `from_path_partitions()` returned `PathPartitionsExtractor` | both return `FileListingExtractor`; it is not a `SplittableExtractor`; `FilesSplitReader` removed | +| `from_path_partitions()` statistics: rows only | rows and the listed bytes | +| `ListedFileRow` implementors: `row()` | also `schema(): Schema`, the default body columns | +| `Flow\Floe\FrameReader` | removed: `FloeStreamReader::chunkFiller()`, `FloeStreamReader::COMPACT_THRESHOLD` | +| `new FloeSourceFile(FloeStreamReader $reader, SourceFile $source)`, `FloeSourceFile::$reader` | `new FloeSourceFile(Footer $footer, SourceFile $source)` | +| `FloeWriter::createForStream($stream, $metadata)` | removed: `(new FloeStreamWriter($schema, $backend, $options))->create($stream, $metadata)` | +| `new FloeFileSinks($filesystem, $declared, $metadata, $options)`, `FloeFileSinks::writer(): FloeWriter` | `new FloeFileWriterFactory($declared, $metadata, $options)`, `writer(): FileWriter` | +| `MidpointOwnership::rowsOf($split, $units)` | `$split->owns($units)` | +| `SchemaInference::fromArray()` clamped a size below 1 to 1 | `new SchemaInference()` and `fromArray()` refuse a size below 1 other than -1 (`InvalidArgumentException`) | +| `Rows::withColumns()` refused a wrongly sized column the schema does not name | ignores it, as it ignores the column | +| `Flow\ETL\RowsFactory` | removed | +| - | `Rows::castTo(Schema $schema, Backend $backend)`, `ReadWindow::trim()` / `reached()`, `InvalidArgumentException::unsupportedPath()` | + +### 38) `flow-php/etl` - bound steps are immutable + +| Before | After | +|-----------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `HashJoinProcessor`, `GroupByAggregationProcessor`, `PivotProcessor`, `DuplicateRowTransformer`: `final class`, `bind()` set fields on a copy | `final readonly class`; the bound state is a trailing optional constructor argument (`?Schema $left`, `?Schema $declaredRight`; `?GroupByShape $shape`; `?PivotShape $shape`; `$resolved`, `?Schema $output`) | +| `new DuplicateRowTransformer($condition, WithColumn ...$columns)` | `new DuplicateRowTransformer($condition, array $columns)` | + +### 39) `flow-php/etl` - join rules live on the join type; one join schema, one probe loop + +| Before | After | +|------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------| +| `JoinShape::of($expression, $type)->schema()->of($type, $left, $right)` | `JoinSchema::of($expression, $type)->output($left, $right)`; `JoinShape` removed | +| `JoinShape::of(...)->merger()`, `new RowMerger($prefix, $dropLeft, $dropRight)` | `JoinSchema::of(...)->merger($left, $right)`, `new RowMerger(Schema $output, list $left, list $right)` | +| `new JoinSchema($prefix, $dropLeft, $dropRight)`, `JoinSchema::of(Join $type, $left, $right)` | a fourth argument, `Join $type`; `output($left, $right)` | +| `RowMerger::merge()` threw `DuplicatedColumnsException` on a name collision | the collision is refused by `JoinSchema::output()` (`SchemaDefinitionNotUniqueException`) | +| `NullRowBuilder` | removed: `Rows::nulls(Schema $schema, Backend $backend)` | +| `JoinSide::of($rows, ?Rows $nullRow, ?Schema $schema)` | `JoinSide::of($rows, ?Schema $schema)`; `nullRow(Schema $fallback, Backend $backend)` | +| - | `Join::dropsNullKeys(Side $side)`, `Join::padsWithNullRow(Side $side)`, `enum Side` | + +### 40) `flow-php/etl` - join equality, grouping, pivot, batch size and telemetry internals + +| Before | After | +|-------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------| +| `join(..., join_on(['k' => 'k']))` matched `'1'` with `'1.0'` | no match, the same rule as `ref('k')->equals(ref('j'))` | +| `GroupBy::aggregatedValues()` | removed | +| `PivotAggregation::aggregateBound($rows, $context, $groupBy, $shape)` | `PivotAggregation::aggregate($rows, $context, $groupBy, $shape)` | +| `PivotedTable::flush(0, $context)` threw `InvalidArgumentException` | the batch size is checked by `new PivotAggregation($batchSize)` | +| `new DuplicateRowTransformer($condition, $columns, $resolved, $output)` | a fifth argument, `?array $steps` | +| - | `ReferenceResolver::bind(FunctionTree $function, Schema $schema, string $method, ?Type $returns = null)` | + +| `Flow\ETL\Extractor\BatchSize` | `Flow\ETL\BatchSize` | +| `new Transformation\BatchSize(int $size)` | `new Transformation\BatchSize(BatchSize $size)`; `batch_size(int $size)` unchanged | +| `int $batchSize` in `new Joiner()`, `new ExternalSort()`, `new KWayMerge()`, `new PivotAggregation()`, `new ExpandingProcessor()`, `new SpillingBuckets()`, `PivotedTable::flush()` | `BatchSize $batchSize` | +| `new BatchingProcessor(int $size)`, `new Plan\Node\Batch($input, int $size)`, `int $size` property | `BatchSize $size` | +| these threw `InvalidArgumentException` `Batch size must be greater than 0, given: 0` | `new BatchSize(0)` throws `Batch size must be greater than 0, got 0` | +| `TopNProcessor::followedBy($kept, $rest)`, `(new BoundedRead($limit, $backend))->followedBy($read, $rest)` | `BoundedRead::followedBy($read, $rest)` | +| `DerivedColumns::declare($schema, $definition)` | `$schema->put($definition)` | +| a transformer or loader opened its own telemetry span | the pipeline opens one span per step, user transformers and loaders included | +| `FileWriteFrame::write($rows, $context, $loader)` | `FileWrite::write(Rows $rows, Backend $backend)` | +| - | `Flow\ETL\Foundation\Telemetry\SpanAttributes::spanAttributes()`: extra attributes for a step's span | +| `OnEach::typeOver(Type $arrayType)` | `OnEach::typeOver(Type $arrayType, ScalarFunction $body)` | +| `Hashes::add()`, `Hashes::exists()` | removed: `Hashes::keep()` | +| `GroupBy\PivotSchema` | removed: `GroupBy::pivotSchema(Schema $input, array $pivotColumns, AggregatingFunction $aggregation)` | +| `new ScalarFunctionTransformer($column, $function, $resolved, $derived, $output)` | a sixth argument, `?Schema $declared` | +| `$keyGrouping->indices($keys)` | `KeyGrouping::indices($keys, Hasher $hasher)` | +| `GroupBy::references()` | `GroupBy::refs()->all()` | + +### 42) `flow-php/etl`, adapters - file extractor and reader trims + +| Before | After | +|-----------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| `declaredPartitionTypes()` on the CSV, JSON, Text, XML, Excel, Parquet and Floe extractors, `FileColumnsSource::partitionTypes()` | removed | +| `new AdaptiveCSVOpenSource($filesystem, $source, $options, $extension)` | `(new CSVSourceOpener($filesystem, $options, extension: $extension))->open($source)` | +| `ParquetSchemaPassFile` | removed: `DerivedFromFiles::fold()` leaves the first file open for its owner | +| - | `ParquetSourceFile::forSource(SourceFile $source)`, `FileSplit::window(SourceStream $stream, ?int $start, ?int $end)` | +| `JsonLineResync::window($stream, $split)` | `$split->window($stream, $resync->after(...), $resync->after(...))` | +| `new JsonFileBatches($filesystem, $reader, $format, $pointer, $split)` | `new JsonFileBatches($filesystem, $reader)`; the split arrives through `batches()` (see 121) | +| - | `JsonFileReader::pointsInto()`, `JsonFileReader::window(?FileSplit $split)` | +| `ParquetStoredType::of($normalized)` | `ParquetSchemaConformance::storedType($normalized)` | +| `WorkbookSheetSample::rows()` | removed | + +### 43) `flow-php/parquet`, `flow-php/arrow-ext`, `flow-php/flow-php-ext` - reader and encoder trims + +| Before | After | +|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| +| `ResolvedLeaf` (`?PhysicalIntervals $intervals`), `LeafResolver::resolve()` | `ExactLeaf` (`PhysicalIntervals $intervals`), `InexactLeaf`, `BoundFilter::resolve()` | +| `StatisticsReader`; `statistics(): ?StatisticsReader` on `ColumnChunk`, `DataPageHeader`, `DataPageHeaderV2` | `statistics(): ?Statistics`; `Statistics::min()`, `max()`, `minValue()`, `maxValue()`; counts and exactness as properties (`$statistics->nullCount`) | +| engine configuration `byte_order` as a case name (`'LITTLE_ENDIAN'`) | the value (`'V'`, `'N'`) | +| `ReadFlatColumnValues::rawValues(): Generator` | `rawValues(): array` | +| `new RustCSVEncoder($separator, $enclosure, $escape, $newLineSeparator, $dateTimeFormat, $dateFormat, $php)` | `new RustCSVEncoder(CSVWriteOptions $options, PhpCSVEncoder $php)` | +| `new RustJsonEncoder($flags, $dateTimeFormat, $dateFormat, $php)` | `new RustJsonEncoder(PhpJsonEncoder $php)`; `PhpJsonEncoder::$flags`, `$dateTimeFormat`, `$dateFormat` are public | +| `ParquetFile\Page\ColumnData`, `RowGroup\FlowColumnChunk`, `RowGroup::addColumnChunk()`, `RowGroup::setRowsCount()`, `ColumnChunkBuilders::builders()`, `PageContainer::dataSize()`, `WriteFlatColumnValues::definitionLevelsCount()`, `addRaw()`, `skipRows()`, `splitByRows()` | removed | +| `ColumnRef` threw `\InvalidArgumentException` | `Flow\Parquet\Exception\InvalidArgumentException` | +| - | `Reader\ChunkBuffer`, `RustRowGroupChunks::decoded()` | + +### 44) `flow-php/etl` - `NativePHPHash::xxh128()` removed + +| Before | After | +|----------------------------------|----------------------------------------------------------------------| +| `NativePHPHash::xxh128($string)` | `hash('xxh128', $string)`, or `(new NativePHPHash())->hash($string)` | + +### 45) `flow-php/etl` - foundation and memory accounting moved to their layer namespaces + +`Flow\Calculator`, `Flow\Clock` and `Flow\ArrayComparison` move under `Flow\ETL\Foundation`, memory accounting +under `Flow\ETL\Values\Memory`. Class names do not change. + +| Before | After | +|--------------------------------------------------------|-----------------------------------------------------------------------| +| `Flow\ArrayComparison\ArrayComparison` | `Flow\ETL\Foundation\ArrayComparison\ArrayComparison` | +| `Flow\ArrayComparison\ArraySortByKey` | `Flow\ETL\Foundation\ArrayComparison\ArraySortByKey` | +| `Flow\Calculator\Calculator` | `Flow\ETL\Foundation\Calculator\Calculator` | +| `Flow\Calculator\Exception\Exception` | `Flow\ETL\Foundation\Calculator\Exception\Exception` | +| `Flow\Calculator\Exception\InvalidExponentException` | `Flow\ETL\Foundation\Calculator\Exception\InvalidExponentException` | +| `Flow\Calculator\Exception\InvalidScaleException` | `Flow\ETL\Foundation\Calculator\Exception\InvalidScaleException` | +| `Flow\Calculator\Exception\NonNumericValueException` | `Flow\ETL\Foundation\Calculator\Exception\NonNumericValueException` | +| `Flow\Calculator\Exception\RoundingNecessaryException` | `Flow\ETL\Foundation\Calculator\Exception\RoundingNecessaryException` | +| `Flow\Calculator\NumberNormalizer` | `Flow\ETL\Foundation\Calculator\NumberNormalizer` | +| `Flow\Calculator\Rounding` | `Flow\ETL\Foundation\Calculator\Rounding` | +| `Flow\Calculator\RunningSum` | `Flow\ETL\Foundation\Calculator\RunningSum` | +| `Flow\Clock\FakeClock` | `Flow\ETL\Foundation\Clock\FakeClock` | +| `Flow\Clock\SystemClock` | `Flow\ETL\Foundation\Clock\SystemClock` | +| `Flow\ETL\Dataset\Memory\BoundedRead` | `Flow\ETL\Values\Memory\BoundedRead` | +| `Flow\ETL\Dataset\Memory\Configuration` | `Flow\ETL\Values\Memory\Configuration` | +| `Flow\ETL\Dataset\Memory\Consumption` | `Flow\ETL\Values\Memory\Consumption` | +| `Flow\ETL\Dataset\Memory\MemoryBudget` | `Flow\ETL\Values\Memory\MemoryBudget` | +| `Flow\ETL\Dataset\Memory\Unit` | `Flow\ETL\Values\Memory\Unit` | +| `Flow\ETL\FlowPhpExtension` | `Flow\ETL\Foundation\FlowPhpExtension` | +| `Flow\ETL\NativePHPRandomValueGenerator` | `Flow\ETL\Foundation\NativePHPRandomValueGenerator` | +| `Flow\ETL\RandomValueGenerator` | `Flow\ETL\Foundation\RandomValueGenerator` | + +### 46) `flow-php/etl` - value types moved to the `Flow\ETL\Values` namespace + +Rows, schemas, definitions, columns and their physical layouts move under `Flow\ETL\Values`; the Floe frame codec +contract moves to `Flow\ETL\Values\Column\Frame`. Class names do not change. + +| Before | After | +|---------------------------------------------------------------|----------------------------------------------------------------------| +| `Flow\ETL\BatchSize` | `Flow\ETL\Values\BatchSize` | +| `Flow\ETL\Cardinality` | `Flow\ETL\Values\Cardinality` | +| `Flow\ETL\Column\AdaptiveBackend` | `Flow\ETL\Values\Column\AdaptiveBackend` | +| `Flow\ETL\Column\Backend` | `Flow\ETL\Values\Column\Backend` | +| `Flow\ETL\Column\Column` | `Flow\ETL\Values\Column\Column` | +| `Flow\ETL\Column\ColumnBuilder` | `Flow\ETL\Values\Column\ColumnBuilder` | +| `Flow\ETL\Column\ComparableValues` | `Flow\ETL\Values\Column\ComparableValues` | +| `Flow\ETL\Column\DateFormatPattern` | `Flow\ETL\Values\Column\DateFormatPattern` | +| `Flow\ETL\Column\Elements` | `Flow\ETL\Values\Column\Elements` | +| `Flow\ETL\Column\Layout\BooleanLayout` | `Flow\ETL\Values\Column\Layout\BooleanLayout` | +| `Flow\ETL\Column\Layout\BufferLayout` | `Flow\ETL\Values\Column\Layout\BufferLayout` | +| `Flow\ETL\Column\Layout\Buffers` | `Flow\ETL\Values\Column\Layout\Buffers` | +| `Flow\ETL\Column\Layout\FixedBinary16Layout` | `Flow\ETL\Values\Column\Layout\FixedBinary16Layout` | +| `Flow\ETL\Column\Layout\Float64Layout` | `Flow\ETL\Values\Column\Layout\Float64Layout` | +| `Flow\ETL\Column\Layout\Int32Layout` | `Flow\ETL\Values\Column\Layout\Int32Layout` | +| `Flow\ETL\Column\Layout\Int64Layout` | `Flow\ETL\Values\Column\Layout\Int64Layout` | +| `Flow\ETL\Column\Layout\Layout` | `Flow\ETL\Values\Column\Layout\Layout` | +| `Flow\ETL\Column\Layout\NullCount` | `Flow\ETL\Values\Column\Layout\NullCount` | +| `Flow\ETL\Column\Layout\Offsets` | `Flow\ETL\Values\Column\Layout\Offsets` | +| `Flow\ETL\Column\Layout\Utf8Layout` | `Flow\ETL\Values\Column\Layout\Utf8Layout` | +| `Flow\ETL\Column\Layout\Validity` | `Flow\ETL\Values\Column\Layout\Validity` | +| `Flow\ETL\Column\PhpBackend` | `Flow\ETL\Values\Column\PhpBackend` | +| `Flow\ETL\Column\Php\CastingColumnBuilder` | `Flow\ETL\Values\Column\Php\CastingColumnBuilder` | +| `Flow\ETL\Column\Php\ColumnDecoder` | `Flow\ETL\Values\Column\Php\ColumnDecoder` | +| `Flow\ETL\Column\Php\ConstantColumn` | `Flow\ETL\Values\Column\Php\ConstantColumn` | +| `Flow\ETL\Column\Php\ListColumn` | `Flow\ETL\Values\Column\Php\ListColumn` | +| `Flow\ETL\Column\Php\ListColumnBuilder` | `Flow\ETL\Values\Column\Php\ListColumnBuilder` | +| `Flow\ETL\Column\Php\MapColumn` | `Flow\ETL\Values\Column\Php\MapColumn` | +| `Flow\ETL\Column\Php\MapColumnBuilder` | `Flow\ETL\Values\Column\Php\MapColumnBuilder` | +| `Flow\ETL\Column\Php\NullColumnBuilder` | `Flow\ETL\Values\Column\Php\NullColumnBuilder` | +| `Flow\ETL\Column\Php\PhpColumnBuilder` | `Flow\ETL\Values\Column\Php\PhpColumnBuilder` | +| `Flow\ETL\Column\Php\ScalarColumn` | `Flow\ETL\Values\Column\Php\ScalarColumn` | +| `Flow\ETL\Column\Php\ScalarColumnBuilder` | `Flow\ETL\Values\Column\Php\ScalarColumnBuilder` | +| `Flow\ETL\Column\Php\StructColumn` | `Flow\ETL\Values\Column\Php\StructColumn` | +| `Flow\ETL\Column\Php\StructColumnBuilder` | `Flow\ETL\Values\Column\Php\StructColumnBuilder` | +| `Flow\ETL\Column\Physical\DatePhysical` | `Flow\ETL\Values\Column\Physical\DatePhysical` | +| `Flow\ETL\Column\Physical\DateTimePhysical` | `Flow\ETL\Values\Column\Physical\DateTimePhysical` | +| `Flow\ETL\Column\Physical\DaysFromCivil` | `Flow\ETL\Values\Column\Physical\DaysFromCivil` | +| `Flow\ETL\Column\Physical\ElementPosition` | `Flow\ETL\Values\Column\Physical\ElementPosition` | +| `Flow\ETL\Column\Physical\EnumPhysical` | `Flow\ETL\Values\Column\Physical\EnumPhysical` | +| `Flow\ETL\Column\Physical\HtmlDocumentPhysical` | `Flow\ETL\Values\Column\Physical\HtmlDocumentPhysical` | +| `Flow\ETL\Column\Physical\HtmlElementPhysical` | `Flow\ETL\Values\Column\Physical\HtmlElementPhysical` | +| `Flow\ETL\Column\Physical\IdentityPhysical` | `Flow\ETL\Values\Column\Physical\IdentityPhysical` | +| `Flow\ETL\Column\Physical\JsonFactory` | `Flow\ETL\Values\Column\Physical\JsonFactory` | +| `Flow\ETL\Column\Physical\JsonPhysical` | `Flow\ETL\Values\Column\Physical\JsonPhysical` | +| `Flow\ETL\Column\Physical\ListPhysical` | `Flow\ETL\Values\Column\Physical\ListPhysical` | +| `Flow\ETL\Column\Physical\MapPhysical` | `Flow\ETL\Values\Column\Physical\MapPhysical` | +| `Flow\ETL\Column\Physical\NullPhysical` | `Flow\ETL\Values\Column\Physical\NullPhysical` | +| `Flow\ETL\Column\Physical\Physical` | `Flow\ETL\Values\Column\Physical\Physical` | +| `Flow\ETL\Column\Physical\PhysicalFor` | `Flow\ETL\Values\Column\Physical\PhysicalFor` | +| `Flow\ETL\Column\Physical\StructPhysical` | `Flow\ETL\Values\Column\Physical\StructPhysical` | +| `Flow\ETL\Column\Physical\TimePhysical` | `Flow\ETL\Values\Column\Physical\TimePhysical` | +| `Flow\ETL\Column\Physical\TimeZonePhysical` | `Flow\ETL\Values\Column\Physical\TimeZonePhysical` | +| `Flow\ETL\Column\Physical\TimeZones` | `Flow\ETL\Values\Column\Physical\TimeZones` | +| `Flow\ETL\Column\Physical\UuidPhysical` | `Flow\ETL\Values\Column\Physical\UuidPhysical` | +| `Flow\ETL\Column\Physical\XmlCharacterReferences` | `Flow\ETL\Values\Column\Physical\XmlCharacterReferences` | +| `Flow\ETL\Column\Physical\XmlDocumentPhysical` | `Flow\ETL\Values\Column\Physical\XmlDocumentPhysical` | +| `Flow\ETL\Column\Physical\XmlElementPhysical` | `Flow\ETL\Values\Column\Physical\XmlElementPhysical` | +| `Flow\ETL\Column\Retype` | `Flow\ETL\Values\Column\Retype` | +| `Flow\ETL\Column\RustBackend` | `Flow\ETL\Values\Column\RustBackend` | +| `Flow\ETL\Column\RustColumn` | `Flow\ETL\Values\Column\RustColumn` | +| `Flow\ETL\Column\RustColumnBuilder` | `Flow\ETL\Values\Column\RustColumnBuilder` | +| `Flow\ETL\Column\RustColumnsBatch` | `Flow\ETL\Values\Column\RustColumnsBatch` | +| `Flow\ETL\Column\TextValues` | `Flow\ETL\Values\Column\TextValues` | +| `Flow\ETL\Column\ValueColumn` | `Flow\ETL\Values\Column\ValueColumn` | +| `Flow\ETL\Row\ColumnName` | `Flow\ETL\Values\Column\ColumnName` | +| `Flow\ETL\Row\NullsOrder` | `Flow\ETL\Values\NullsOrder` | +| `Flow\ETL\Row\SortOrder` | `Flow\ETL\Values\SortOrder` | +| `Flow\ETL\Rows` | `Flow\ETL\Values\Rows` | +| `Flow\ETL\Rows\ColumnRetyping` | `Flow\ETL\Values\Rows\ColumnRetyping` | +| `Flow\ETL\Rows\RecordBatch` | `Flow\ETL\Values\Rows\RecordBatch` | +| `Flow\ETL\Rows\RowsBuilder` | `Flow\ETL\Values\Rows\RowsBuilder` | +| `Flow\ETL\Schema` | `Flow\ETL\Values\Schema` | +| `Flow\ETL\SchemaValidator` | `Flow\ETL\Values\SchemaValidator` | +| `Flow\ETL\Schema\Definition` | `Flow\ETL\Values\Schema\Definition` | +| `Flow\ETL\Schema\Definition\BooleanDefinition` | `Flow\ETL\Values\Schema\Definition\BooleanDefinition` | +| `Flow\ETL\Schema\Definition\DateDefinition` | `Flow\ETL\Values\Schema\Definition\DateDefinition` | +| `Flow\ETL\Schema\Definition\DateTimeDefinition` | `Flow\ETL\Values\Schema\Definition\DateTimeDefinition` | +| `Flow\ETL\Schema\Definition\ElementCompatibility` | `Flow\ETL\Values\Schema\Definition\ElementCompatibility` | +| `Flow\ETL\Schema\Definition\EnumDefinition` | `Flow\ETL\Values\Schema\Definition\EnumDefinition` | +| `Flow\ETL\Schema\Definition\FloatDefinition` | `Flow\ETL\Values\Schema\Definition\FloatDefinition` | +| `Flow\ETL\Schema\Definition\HTMLDefinition` | `Flow\ETL\Values\Schema\Definition\HTMLDefinition` | +| `Flow\ETL\Schema\Definition\HTMLElementDefinition` | `Flow\ETL\Values\Schema\Definition\HTMLElementDefinition` | +| `Flow\ETL\Schema\Definition\IntegerDefinition` | `Flow\ETL\Values\Schema\Definition\IntegerDefinition` | +| `Flow\ETL\Schema\Definition\JsonDefinition` | `Flow\ETL\Values\Schema\Definition\JsonDefinition` | +| `Flow\ETL\Schema\Definition\ListDefinition` | `Flow\ETL\Values\Schema\Definition\ListDefinition` | +| `Flow\ETL\Schema\Definition\MapDefinition` | `Flow\ETL\Values\Schema\Definition\MapDefinition` | +| `Flow\ETL\Schema\Definition\NullDefinition` | `Flow\ETL\Values\Schema\Definition\NullDefinition` | +| `Flow\ETL\Schema\Definition\StringDefinition` | `Flow\ETL\Values\Schema\Definition\StringDefinition` | +| `Flow\ETL\Schema\Definition\StructureDefinition` | `Flow\ETL\Values\Schema\Definition\StructureDefinition` | +| `Flow\ETL\Schema\Definition\TimeDefinition` | `Flow\ETL\Values\Schema\Definition\TimeDefinition` | +| `Flow\ETL\Schema\Definition\TimeZoneDefinition` | `Flow\ETL\Values\Schema\Definition\TimeZoneDefinition` | +| `Flow\ETL\Schema\Definition\TypeProjection` | `Flow\ETL\Values\Schema\Definition\TypeProjection` | +| `Flow\ETL\Schema\Definition\UuidDefinition` | `Flow\ETL\Values\Schema\Definition\UuidDefinition` | +| `Flow\ETL\Schema\Definition\XMLDefinition` | `Flow\ETL\Values\Schema\Definition\XMLDefinition` | +| `Flow\ETL\Schema\Definition\XMLElementDefinition` | `Flow\ETL\Values\Schema\Definition\XMLElementDefinition` | +| `Flow\ETL\Schema\Metadata` | `Flow\ETL\Values\Schema\Metadata` | +| `Flow\ETL\Schema\SameSchemas` | `Flow\ETL\Values\Schema\SameSchemas` | +| `Flow\ETL\Schema\SimilarNames` | `Flow\ETL\Foundation\SimilarNames` | +| `Flow\ETL\Schema\SortingStrategy` | `Flow\ETL\Values\Schema\SortingStrategy` | +| `Flow\ETL\Schema\SortingStrategy\AlphabeticalStrategy` | `Flow\ETL\Values\Schema\SortingStrategy\AlphabeticalStrategy` | +| `Flow\ETL\Schema\SortingStrategy\CombinedStrategy` | `Flow\ETL\Values\Schema\SortingStrategy\CombinedStrategy` | +| `Flow\ETL\Schema\SortingStrategy\MetadataStrategy` | `Flow\ETL\Values\Schema\SortingStrategy\MetadataStrategy` | +| `Flow\ETL\Schema\SortingStrategy\TypeStrategy` | `Flow\ETL\Values\Schema\SortingStrategy\TypeStrategy` | +| `Flow\ETL\Schema\SortingStrategy\TypeStrategy\TypePriorities` | `Flow\ETL\Values\Schema\SortingStrategy\TypeStrategy\TypePriorities` | +| `Flow\ETL\Schema\Validator\EvolvingValidator` | `Flow\ETL\Values\Schema\Validator\EvolvingValidator` | +| `Flow\ETL\Schema\Validator\MismatchedDefinition` | `Flow\ETL\Values\Schema\Validator\MismatchedDefinition` | +| `Flow\ETL\Schema\Validator\SelectiveValidator` | `Flow\ETL\Values\Schema\Validator\SelectiveValidator` | +| `Flow\ETL\Schema\Validator\StrictValidator` | `Flow\ETL\Values\Schema\Validator\StrictValidator` | +| `Flow\ETL\Schema\Validator\ValidationContext` | `Flow\ETL\Values\Schema\Validator\ValidationContext` | +| `Flow\Floe\Codec` | `Flow\ETL\Values\Column\Frame\Codec` | +| `Flow\Floe\Codec\NoopCodec` | `Flow\ETL\Values\Column\Frame\NoopCodec` | +| `Flow\Floe\Exception\FloeException` | `Flow\ETL\Values\Column\Frame\FloeException` | +| `Flow\Floe\Exception\FrameTooLarge` | `Flow\ETL\Values\Column\Frame\FrameTooLarge` | + +### 47) `flow-php/etl` - columns are addressed by name in `Schema`, `Definition` and `Rows::reduceToArray()` + +`Schema` methods take column names (`string`), no longer `Reference`. `Definition::reference()` becomes +`Definition::name()`, `Schema::references()` becomes `Schema::names()`. + +| Before | After | +|-------------------------------------------------------------------------------|-------------------------------------| +| `$definition->reference()->name()` | `$definition->name()` | +| `$schema->references()` | `$schema->names()` - `list` | +| `$schema->get(ref('id'))`, also `findDefinition`, `keep`, `remove`, `reorder` | `$schema->get('id')` | +| `$schema->moveAfter(ref('a'), ref('b'))`, also `addAfter`, `moveTo`, `rename` | `$schema->moveAfter('a', 'b')` | +| `$rows->reduceToArray(ref('id'))` | `$rows->reduceToArray('id')` | + +### 48) `flow-php/etl` - `Rows::concat()` is static, `Rows::sortBy()` is removed + +| Before | After | +|------------------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| `$rows->concat($backend, $other)` | `Rows::concat($schema, $backend, $rows, $other)` | +| `$rows->sortBy(ref('id')->desc())` | `$rows->gather($order->permutation($order->keys($rows), $rows->count()))` with `$order = new RowOrder([ref('id')->desc()])` | + +`Rows::concat()` takes the schema every part must carry, so an empty list of parts still has one. + +### 49) `flow-php/etl` - `RowsBuilder::append()` and `RowsBuilder::appendProjected()` are removed + +| Before | After | +|------------------------------------|--------------------------------------------------------------------------------------------| +| `$builder->append($row)` | `$builder->appendRows([$row])` | +| `$builder->appendProjected($rows)` | `$builder->appendRows($rows)` - records keyed by position fill the column at that position | + +### 50) `flow-php/etl` - one type table in `PhysicalFor` + +`LayoutFor`, `PhysicalKind` and `PhysicalBuilderFor` merge into `PhysicalFor`. + +| Before | After | +|---------------------------------------------|---------------------------------------------------| +| `(new PhysicalFor())->type($type)` | `(new PhysicalFor())->physical($type)` | +| `(new LayoutFor())->type($type)` | `(new PhysicalFor())->layout($type)` | +| `(new PhysicalKind())->accepts($type, $v)` | `(new PhysicalFor())->accepts($type, $v)` | +| `(new PhysicalKind())->all($type, $values)` | `(new PhysicalFor())->acceptsAll($type, $values)` | +| `(new PhysicalKind())->elements($type, $v)` | `(new PhysicalFor())->acceptsElements($type, $v)` | +| `(new PhysicalBuilderFor())->type($type)` | `(new PhysicalFor())->builder($type)` | + +### 51) `flow-php/etl` - definition merging goes through `Definitions` + +`Definitions::of()` maps a type to its definition (what `definition_from_type()` does, which now delegates to it), +`Definitions::merged()` widens two definitions of one column. `CommonType` and `ValueMatch` are removed. + +| Before | After | +|--------------|------------------------------------------------| +| `CommonType` | removed - `Definitions::merged($left, $right)` | +| `ValueMatch` | removed - `$definition->matches($value)` | + +### 52) `flow-php/etl` - schema errors past an inference sample report the row in the source + +`InferredRows` is removed: memory extractors and the native CSV / JSON readers report a refused row at its 0-based +position in the whole source, not in the batch. + +| Before | After | +|--------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| `InferredRows` | removed - the extractors' `RecordBatch` keeps the source offset | +| `InferredSchemaException::pastTheSample($source, $row, $inference, $mismatch)` | `InferredSchemaException::whenPastTheSample($source, $sampleSize, $mismatch)` - returns `$mismatch` within the sample | +| `SchemaMismatchException::$rowIndex` - position in the batch | position in the whole source | + +### 53) `flow-php/etl` - a refused value in a typed column throws `ColumnMismatchException` + +| Before | After | +|-----------------------------------------------------------------|----------------------------------| +| `CastingColumnBuilder::append()` threw `Flow\Types\Exception\*` | throws `ColumnMismatchException` | + +--- + +### 54) `flow-php/etl` - storage types moved to the `Flow\ETL\Storage` namespace + +Batch storage, the Floe file encoding, the serializers and `Memory` move under `Flow\ETL\Storage`; +`TelemetryAttributes` moves to `Flow\ETL\Foundation\Telemetry`. The bucket storages are renamed after the one +contract they implement (row 55). The Floe source and sink (`FloeExtractor`, `FloeLoader`) move to +`Flow\ETL\IO\Floe` (row 112); the `Flow\Floe\DSL` and `Flow\Serializer\DSL` functions keep their namespace. + +| Before | After | +|----------------------------------------------------|----------------------------------------------------------------| +| `Flow\ETL\Bucketing\BucketsStorage` | `Flow\ETL\Storage\RowsStorage` | +| `Flow\ETL\Bucketing\ResidentBucketsStorage` | `Flow\ETL\Storage\ResidentRowsStorage` | +| `Flow\ETL\Bucketing\Storage\FilesystemBuckets` | `Flow\ETL\Storage\FilesystemRowsStorage` | +| `Flow\ETL\Bucketing\Storage\MemoryBuckets` | `Flow\ETL\Storage\MemoryRowsStorage` | +| `Flow\ETL\Bucketing\Storage\PSRCacheBuckets` | `Flow\ETL\Storage\PsrRowsStorage` | +| `Flow\ETL\Bucketing\Storage\SpillingBuckets` | `Flow\ETL\Storage\SpillingRowsStorage` | +| `Flow\ETL\Cache\Implementation\ApcuCache` | `Flow\ETL\Storage\ApcuRowsStorage` | +| `Flow\ETL\Cache\Implementation\TraceableCache` | `Flow\ETL\Runtime\Telemetry\TraceableRowsStorage` | +| `Flow\ETL\Config\Telemetry\TelemetryAttributes` | `Flow\ETL\Foundation\Telemetry\TelemetryAttributes` | +| `Flow\ETL\Memory\ArrayMemory` | `Flow\ETL\Storage\Memory\ArrayMemory` | +| `Flow\ETL\Memory\Memory` | `Flow\ETL\Storage\Memory\Memory` | +| `Flow\Floe\Exception\IncompatibleSchemaException` | `Flow\ETL\Storage\Floe\Exception\IncompatibleSchemaException` | +| `Flow\Floe\FloeMerger` | `Flow\ETL\Storage\Floe\FloeMerger` | +| `Flow\Floe\FloeSerializer` | `Flow\ETL\Storage\Floe\FloeSerializer` | +| `Flow\Floe\FloeStreamReader` | `Flow\ETL\Storage\Floe\FloeStreamReader` | +| `Flow\Floe\FloeStreamWriter` | `Flow\ETL\Storage\Floe\FloeStreamWriter` | +| `Flow\Floe\Footer` | `Flow\ETL\Storage\Floe\Footer` | +| `Flow\Floe\FooterLocation` | `Flow\ETL\Storage\Floe\FooterLocation` | +| `Flow\Floe\FooterReader` | `Flow\ETL\Storage\Floe\FooterReader` | +| `Flow\Floe\Format` | `Flow\ETL\Storage\Floe\Format` | +| `Flow\Floe\FrameWriter` | `Flow\ETL\Storage\Floe\FrameWriter` | +| `Flow\Floe\Options` | `Flow\ETL\Storage\Floe\Options` | +| `Flow\Floe\Section` | `Flow\ETL\Storage\Floe\Section` | +| `Flow\Floe\SplittingFrameEncoder` | `Flow\ETL\Storage\Floe\SplittingFrameEncoder` | +| `Flow\Serializer\CompressingSerializer` | `Flow\ETL\Storage\Serializer\CompressingSerializer` | +| `Flow\Serializer\Exception\SerializationException` | `Flow\ETL\Storage\Serializer\Exception\SerializationException` | +| `Flow\Serializer\Serializer` | `Flow\ETL\Storage\Serializer\Serializer` | + +### 55) `flow-php/etl` - one `RowsStorage` contract for the cache, spilling and shuffles + +`Cache` and `BucketsStorage` merge into `Flow\ETL\Storage\RowsStorage`: a key holds batches appended in order. +`cache()`, `from_cache()`, `config_builder()->cache()` and `filesystem_cache()` keep their names and take or build a +`RowsStorage`. The default cache writes one Floe file per key with the configured backend, so the configured +`Serializer` no longer reaches it. Entries cached by 0.46 are not read: they are written again on the next run. +The filesystem cache moves from `/////` to `/flow-php-rows//.floe`. +Remove the old entries under `` by hand. + +| Before | After | +|-----------------------------------------------------------------------|-------------------------------------------------------------------------------------------| +| `Flow\ETL\Cache` | `Flow\ETL\Storage\RowsStorage` | +| `$cache->set($key, $rows)`, `$cache->get($key)` | `$storage->append($key, $rows)`, `$storage->close($key)`, `$storage->get($key)` - batches | +| `$cache->schema($key)` - throws for a missing key | `$storage->schema($key)` - `null` for a missing key | +| `$cache->delete($key)`, `$cache->clear()` | `$storage->remove($key)` - no `clear()` | +| `$buckets->set($key, $rows)` | `$storage->append($key, $rows)` then `$storage->close($key)` on a fresh key | +| `KeyNotInCacheException` | removed - `has()` answers, `get()` of a missing key yields nothing | +| `InMemoryCache` | `MemoryRowsStorage` | +| `FilesystemCache`, `filesystem_cache($serializer, $dir, $filesystem)` | `FilesystemRowsStorage`, `filesystem_cache($dir, $filesystem, $backend)` | +| `PSRSimpleCache($cache, $serializer, $ttl)` | `PsrRowsStorage($cache, $serializer, ttl: $ttl)` - keys under the `flow.rows` prefix | +| `PSRCacheBuckets($cache, $serializer, 'flow.buckets', $ttl)` | `PsrRowsStorage($cache, $serializer, 'flow.rows', $ttl)` | +| `TraceableCache` | `TraceableRowsStorage` | +| `CacheIndex` | removed - a key holds its batches | +| `FilesystemBuckets::rows($key, $offset, $limit)` | `FilesystemRowsStorage::slice($key, $offset, $limit)` | +| `SpillingBuckets::around($bucketing, $memoryLimit, $backend)` | `$bucketing->spilling($memoryLimit, $backend)` | + +A filesystem key whose file has no footer, because the process writing it was killed, counts as missing, so the next +`cache()` run writes it again. + +### 56) `flow-php/etl` - `NativePHPSerializer`, `Base64Serializer` and PHP serialization of `Rows` are removed + +`NativePHPSerializer` decoded every batch into the default backend, whatever the configuration said. + +| Before | After | +|---------------------------------------------|---------------------------------------------------------------------------| +| `NativePHPSerializer` | `FloeSerializer` | +| `Base64Serializer($serializer)` | `base64_encode(serialize_to_string($serializer, $rows))` | +| `serialize($rows)`, `unserialize($payload)` | `serialize_to_string($serializer, $rows)`, `unserialize_from_string(...)` | + +### 57) `flow-php/etl` - `FloeFiles` opens Floe files; `FloeStreamWriter` is created open + +| Before | After | +|------------------------------------------------------------------------------------|---------------------------------------------------------------------------------| +| `new FloeReader($filesystem, $backend, $codec)->read($path)` | `new FloeFiles($filesystem, $backend, new Options(codec: $codec))->read($path)` | +| `$writer = new FloeWriter($filesystem, $schema, $backend); $writer->create($path)` | `$writer = new FloeFiles($filesystem, $backend)->create($path, $schema)` | +| `$writer->append($path, $metadata)` on a `FloeWriter` | `new FloeFiles($filesystem, $backend)->append($path, $schema, $metadata)` | +| `$writer = new FloeStreamWriter($schema, $backend); $writer->create($stream)` | `$writer = FloeStreamWriter::create($stream, $schema, $backend)` | +| `$writer->resume($stream, $footer, $length)` | `FloeStreamWriter::resume($stream, $footer, $length, $schema, $backend)` | +| `FloeStreamReader::head($count)` | `$reader->rows($batchSize, 0, $count)` | +| `FloeStreamReader::tail($count)` | `$reader->rows($batchSize, max(0, $reader->totalRows() - $count))` | +| `FloeStreamReader::byteSize()` | `$reader->footer()->statistics->byteSize` | + +A closed writer refuses `write()` and `close()` with `Floe writer session is closed`. + +### 58) `flow-php/etl` - `Flow\ETL\Values\Memory\Configuration` is removed + +`MemoryLimit::default()` reads PHP's `memory_limit` itself. `Configuration::isConsumptionBelow()`, `isInfinite()` and +`isLessThan()` had no callers. + +### 59) `flow-php/etl` - `ApcuCache` renamed to `ApcuRowsStorage` + +`ApcuCache` implements `RowsStorage` (row 55) and is named after it, like the other storages. There is no DSL function +for it. + +| Before | After | +|------------------------------------------|------------------------------------------------| +| `new ApcuCache($serializer, $namespace)` | `new ApcuRowsStorage($serializer, $namespace)` | +| `$cache->clear()` | removed - `$storage->remove($key)` per key | + +### 60) `flow-php/etl` - kernel types moved to the `Flow\ETL\Kernels` namespace, `Join` renamed to `JoinType` + +The join type enum is a value used by the API, the functions and the kernels. It moves to `Flow\ETL\Values\Join` and is +renamed `JoinType`. + +| Before | After | +|-------------------------------------------------------|---------------------------------------------------------------| +| `Flow\ETL\Join\Join` | `Flow\ETL\Values\Join\JoinType` | +| `Flow\ETL\Join\JoinSchema` | `Flow\ETL\Kernels\Join\JoinSchema` | +| `Flow\ETL\Join\HashJoin\*` | `Flow\ETL\Kernels\HashJoin\*` | +| `Flow\ETL\Bucketing\{Bucket, BucketRun, Buckets}` | `Flow\ETL\Kernels\Bucketing\{Bucket, BucketRun, Buckets}` | +| `Flow\ETL\Bucketing\{HashBucketing, KeyGrouping}` | `Flow\ETL\Kernels\Bucketing\{HashBucketing, KeyGrouping}` | +| `Flow\ETL\Hash\{Algorithm, NativePHPHash, PlainText}` | `Flow\ETL\Kernels\Hash\{Algorithm, NativePHPHash, PlainText}` | +| `Flow\ETL\Sort\{ExternalSort, RowOrder, SortKey}` | `Flow\ETL\Kernels\Sort\{ExternalSort, RowOrder, SortKey}` | +| `Flow\ETL\Sort\Merge\{BucketCursor, KWayMerge}` | `Flow\ETL\Kernels\Sort\Merge\{BucketCursor, KWayMerge}` | +| `$df->join($right, $on, Join::left)` | `$df->join($right, $on, JoinType::left)` | + +### 61) `flow-php/etl` - `=` and join keys of two types compare in one type + +When the plan is built, comparisons and join keys of different types are cast to one type, as DuckDB does. Numbers +widen to float, a date widens to a datetime, and a string casts to the other side's type. An integral string casts +exactly, other numeric strings round half away from zero for an integer side. A value that cannot be cast now throws +instead of not matching. `===` and `!==` still compare without a cast. + +| Before | After | +|-------------------------------------------------------|---------------------------------------------------------------------------------| +| `ref('id')->equals(lit('x'))` on an int column: false | throws `EvaluationException`: `Column "id" value 'x' cannot be cast to integer` | +| join of an int key and a string key `'x'`: no match | throws the same exception | +| join keys keyed loosely (`KeyValues`) | join keys cast to one type, then keyed exactly (`KeyEncoder`) | + +### 62) `flow-php/etl` - NaN is the largest value, uuid, json, xml, enum and html sort by their stored form + +NaN equals NaN and orders after every number, as in DuckDB. This holds in `=`, `>`, `<` and `between`. It also holds +in `sortBy()`, `groupBy()`, `batchBy()` and window peers. + +| Before | After | +|-------------------------------------------------------|-------------------------------------------------------------------------------------| +| `f > 2.0` drops a NaN row | keeps it | +| `f = NAN` matches nothing | matches NaN | +| `batchBy('f')` puts two NaN rows in two batches | one batch | +| `sortBy()` on uuid, json, xml, enum keeps input order | sorts by bytes: uuid by its 16 bytes, json and xml by their text, enum by case name | +| `rank()` over a uuid order: input order | the uuid's byte order | + +### 63) `flow-php/etl` - one set of seen keys for distinct rows, unique constraints and statistics + +`SeenKeys` holds every key a stream brought. It counts the distinct values in statistics and in the CLI +`file:analyze` output ("Distinct Values"). The counts now tell types apart and keep datetime microseconds. They count +-0.0 as 0.0, and they count every column type, json included. + +| Before | After | +|-------------------------------------------------------------------|----------------------------------------| +| `Flow\ETL\Transformer\DropDuplicates\Hashes` | `Flow\ETL\Kernels\Bucketing\SeenKeys` | +| `Flow\ETL\Rows\RowHashes` | removed - `KeyEncoder` keys the rows | +| `Flow\ETL\Constraint\UniqueConstraint\{Storage, InMemoryStorage}` | `SeenKeys`, inside the constraint | +| `UniqueConstraint::withStorage(Storage $storage)` | removed | +| `Flow\ETL\Dataset\Statistics\DistinctCounter` | `SeenKeys`, inside `Statistics\Column` | +| two datetimes 1 µs apart: 1 distinct value | 2 | +| a json column: 0 distinct values | its distinct values | + +### 64) `flow-php/etl` - one fold for aggregate partials + +A custom `AggregatingFunction` drops `merge(AggregatingFunction $other, FlowContext $context)`, and `mergePartial()` +is renamed `merge()`. Group by merges a spilled aggregator through its `partial()`. + +```php +// before +public function mergePartial(array $partial, FlowContext $context): void { /* fold $partial */ } + +public function merge(AggregatingFunction $other, FlowContext $context): void +{ + $this->mergePartial($other->partial(), $context); +} + +// after +public function merge(array $partial, FlowContext $context): void { /* fold $partial */ } +``` + +### 65) `flow-php/etl` - kernels take column names and values + +Kernels take column names, `SortColumn` and `Backend`, never references or the flow context. Hashing runs once per +distinct key. + +| Before | After | +|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------| +| `new KeyValues(list)` | `new KeyEncoder(list $columns, array $types = [], ?Schema $declared = null)` | +| `Bucketing\{Hasher, NativeHasher, SingleBucketHasher, KeyEncoder}` (interface) | removed | +| `new HashBucketing($keys, $count, $hasher, $random, ...)` | `new HashBucketing(KeyEncoder $keys, $count, $random, ...)` | +| `HashBucketing::index(string $hash)` | `HashBucketing::indexOf(string $key)`, `HashBucketing::id($namespace, $runId, $index)` | +| `KeyGrouping::indices(array $keys, Hasher $hasher)` | `KeyGrouping::indices(array $keys)` | +| `new RowOrder(list)` | `new RowOrder(list)`, `$references->sortColumns()` | +| `Window\PeerComparator`, `Row\TypedValueComparator` | `RowOrder::ties(Rows $partition)` | +| `ExternalSort::sort($rows, FlowContext $context, $limit)` | `ExternalSort::sort($rows, Backend $backend, $limit)`, with `list` | +| `KWayMerge::merge(list)`, `KWayMerge::mergeStreams($streams)` | `KWayMerge::merge(list> $streams)` | +| `new Joiner(Expression $expression, Join $type, Backend $backend, ...)` | `new Joiner(JoinType $type, JoinSchema $schema, ?JoinKeys $keys, PairPredicate $condition, Backend $backend, ...)` | +| `JoinKeys::fromComparison($comparison)`, `JoinSchema::of($expression, $type)` | `$expression->keys()`, `$expression->schema($type)` | +| `NullKeyFilter(list)::rows($generator)` | `NullKeyFilter(list)::of($batch)` | +| `RepartitionProcessor::$hasher`, `new PartitionRouter($partitioning, $hasher)` | removed | +| explain: `Hasher: NativeHasher` under a repartition | removed | +| `ComparableValues::orderedByPhysical($type)` | `ComparableValues::equalByPhysical($type)` | + +### 66) `flow-php/etl` - `isIn()` casts each candidate as `=` does (row 61) + +| Before | After | +|------------------------------------------------------|---------------------------------------------------------------------------------------| +| `ref('n')->isIn(lit(['x']))` on an int column: false | throws `EvaluationException`: `Column "haystack" value 'x' cannot be cast to integer` | +| `ref('n')->isIn(lit(['1', 'x']))` with `n` = 1: true | throws the same exception | +| `lit('a')->isIn(lit([1, 'a']))`: true | throws: `Column "needle" value 'a' cannot be cast to integer` | + +A haystack of mixed types throws when one candidate cannot be cast. Cast the haystack, or make it one type. + +### 67) `flow-php/etl` - one `Comparison` function replaces the 8 comparison classes + +| Before | After | +|----------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------| +| `new Equals($a, $b)`, `NotEquals`, `Same`, `NotSame` | `new Comparison($a, Operator::EQUAL, $b)`, `NOT_EQUAL`, `IDENTICAL`, `NOT_IDENTICAL` | +| `new GreaterThan($a, $b)`, `GreaterThanEqual`, `LessThan`, `LessThanEqual` | `new Comparison($a, Operator::GREATER_THAN, $b)`, `GREATER_THAN_EQUAL`, `LESS_THAN`, `LESS_THAN_EQUAL` | +| `$function instanceof Equals` | `$function instanceof Comparison && $function->operator === Operator::EQUAL` | + +`Operator` is `Flow\Types\Type\Comparison\Operator`. `equals()`, `greaterThan()` and the other DSL methods keep their +names and now return `Comparison`. + +### 68) `flow-php/etl` - window functions return one value per partition row + +| Before | After | +|--------------------------------------------------------|-----------------------------------------------------------------| +| `WindowFunction::apply(WindowContext $window): mixed` | `WindowFunction::values(Rows $partition): list` | +| `PartitionRanking::rankPartition(Rows $partition)` | `WindowFunction::values(Rows $partition)` | +| `FrameAccumulating::accumulator(FlowContext $context)` | `FrameAccumulating::accumulator()` | +| `new SumAccumulator($ref, $exact, $context)` | `new SumAccumulator($ref, $exact)`, same for `Count`, `Average` | +| `Window\WindowContext`, `Function\PartitionRanking` | removed | + +A frame-accumulating function returns `(new FrameAccumulation($this->window()->frame()))->values($this, $partition)`. + +### 69) `flow-php/etl` - functions take the `Backend`, not the `FlowContext` + +| Before | After | +|---------------------------------------------------------------------------|---------------------------------------------------------------------| +| `ScalarFunction::eval(Rows $rows, FlowContext $context)` | `ScalarFunction::eval(Rows $rows, Backend $backend)` | +| `AggregatingFunction::aggregate($rows, $indices, FlowContext $context)` | `AggregatingFunction::aggregate($rows, $indices, Backend $backend)` | +| `AggregatingFunction::merge(array $partial, FlowContext $context)` | `AggregatingFunction::merge(array $partial, Backend $backend)` | +| `Parameter::as*($rows, FlowContext $context, ...)`, `Parameter::values()` | the same with `Backend $backend` | +| `Parameter::column($rows, $context)` | `$function->eval($rows, $backend)` | +| `Selection::evaluate($function, $rows, FlowContext $context)` | `Selection::evaluate($function, $rows, Backend $backend)` | +| `Filter::filter(Rows $rows, FlowContext $context)` | `Filter::filter(Rows $rows, Backend $backend)` | +| `$context->calculator()` inside a function | `new Calculator()` | + +In a custom function, use `$backend` where you called `$context->backend()`. Callers pass `$context->backend()`. + +### 70) `flow-php/etl` - an alias never picks the column that is read + +| Before | After | +|-------------------------------------------------------------|-------------------------------------------------| +| `select(ref('a')->as('b'))`: column b's values, named b | column a's values, named b | +| `select(ref('a')->as('b'), ref('b'))`: one column b | throws `select() would output column "b" twice` | +| `dropDuplicates(ref('a')->as('b'))`: dedupes by b | dedupes by a | +| `drop(ref('a')->as('b'))`: drops b | drops a | +| `repartition(ref('a')->as('b'))`, `partitionBy()`: by b | by a | +| `groupBy(ref('a')->as('k'))`: throws `column "k" not found` | groups by a, the key column is named a | +| `References::names()`: each alias | each column read (`Reference::to()`) | + +Use `ref('b')` to read column b. + +### 71) `flow-php/etl` - sort order lives on `SortColumn`, not on `Reference` + +| Before | After | +|---------------------------------------------------------------------------------|--------------------------------------------------------------------| +| `ref('a')->desc()`, `->asc()`: a `Reference` | a `SortColumn` (`name`, `order`, `nulls`) | +| `Reference::sort()`, `Reference::nulls()` | removed: read `$sortColumn->order`, `$sortColumn->nulls` | +| `References::sortColumns()` | removed | +| `sortBy()`, `Window::orderBy()`, `constraint_sorted_by()`: `Reference` | `Reference`, `SortColumn` or `string`; a reference sorts ascending | +| `new Sort($input, References $refs)`, `new TopN(...)`, sort processors: `$refs` | `list $columns` | +| `Window::order()`: `list` | `list` | + +### 72) `flow-php/etl` - function types moved to the `Flow\ETL\Functions` namespace, `Join\Comparison` renamed to `Join\Condition` + +| Before | After | +|--------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------| +| `Flow\ETL\Function\*` | `Flow\ETL\Functions\*` | +| `Flow\ETL\Window`, `Flow\ETL\Window\*` | `Flow\ETL\Functions\Window`, `Flow\ETL\Functions\Window\*` | +| `Flow\ETL\Row\{Reference, References, ResolvedReference, UnresolvedReference}` | `Flow\ETL\Functions\Reference\{Reference, References, ResolvedReference, UnresolvedReference}` | +| `Flow\ETL\Join\Comparison` | `Flow\ETL\Functions\Join\Condition` | +| `Flow\ETL\Join\Comparison\{All, Any, Equal, Identical}` | `Flow\ETL\Functions\Join\Condition\{All, Any, Equal, Identical}` | +| `Flow\ETL\Join\Expression` | `Flow\ETL\Functions\Join\Expression` | +| `Flow\ETL\String\StringStyles` | `Flow\ETL\Functions\String\StringStyles` | + +DSL function namespaces (`Flow\ETL\DSL`) do not change. + +### 73) `flow-php/etl-adapter-json`, `flow-php/etl-adapter-parquet` - a 0-row `load()` writes no file + +| Before | After | +|---------------------------------------------------------|---------| +| `to_json()`: `load()` of 0 rows, then `closure()` | no file | +| `to_json_lines()`: `load()` of 0 rows, then `closure()` | no file | +| `to_parquet()`: `load()` of 0 rows, then `closure()` | no file | + +Before: `[]`, an empty `.jsonl` and a 0-row parquet file. CSV, Text, XML and Excel already wrote no file. + +### 74) `flow-php/etl` - a file source lists its files once per run + +| Before | After | +|------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------| +| a frame run again: the inferred schema, partitions and statistics of the first run | listed and inferred again on every run; an `explain()` or `schema()` before the run shares its listing | +| `new FileColumnsSource($filesystem, $path, $types, $metadata, $names)` | `new FileColumnsSource(new SourceFiles($filesystem, $path), $types, $metadata)` | +| `new FileRead($filesystem, $path, $columns, $schema, $batchSize)` | `new FileRead(new SourceFiles($filesystem, $path), $columns, $schema, $batchSize)` | +| `FileExtractor` | extends `Flow\ETL\IO\Extractor\Refreshable` (`refresh(): void`) | + +An extractor that wraps a file source implements `Flow\ETL\IO\Extractor\Refreshable` and calls `refresh()` on the +wrapped source. + +### 75) `flow-php/etl` - a partition value that cannot be cast is reported one way + +| Before | After | +|---------------------------------------------------------------------------|-------------------------------------------------------------------------| +| read of a null partition value: `path value "__HIVE_DEFAULT_PARTITION__"` | `path value NULL` | +| read of a file without the partition: `path value ""` | `path value NULL` | +| `new ScalarFunctionFilter($function, $partitions, $context)` | `new ScalarFunctionFilter($function, $partitions, $context->backend())` | + +### 76) `flow-php/etl` - `batches()`, `batched_by()` and the collecting extractor are removed + +| Before | After | +|------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------| +| `read(batches($extractor, $n))` | `read($extractor)->batchSize($n)` | +| `read(batched_by($extractor, $column, $min))` | `read($extractor)->batchBy($column, $min)` | +| `batches($child, $n)` inside `from_all(...)` | `$child->withBatchSize($n)` when the child is a `BatchableExtractor`, otherwise `->batchSize($n)` on the frame | +| `read(new CollectingExtractor($extractor))` | `read($extractor)->collect()` | +| `Flow\ETL\Extractor\{BatchExtractor, BatchByExtractor, CollectingExtractor}` | removed | + +### 77) `flow-php/etl` and every adapter - sources and sinks take the `Backend`, not the `FlowContext` + +| Before | After | +|----------------------------------------------------------------|----------------------------------------------------------------------------------| +| `Extractor::extract(FlowContext $context, ?int $limit = null)` | `extract(Backend $backend, Criteria $criteria)`, the limit is `$criteria->limit` | +| `Loader::load(Rows $rows, FlowContext $context)` | `load(Rows $rows, Backend $backend)` | +| `Loader\Closure::closure(FlowContext $context)` | `closure()` | +| `Loader\Discardable::discard(FlowContext $context)` | `discard()` | +| `$extractor->extract(flow_context())` | `$extractor->extract(flow_context()->backend(), new Criteria())` | + +A custom extractor and loader: + +```php +// before +public function extract(FlowContext $context, ?int $limit = null): Generator +{ + yield rows(...); // stops after $limit rows +} + +public function load(Rows $rows, FlowContext $context): void +{ + $this->write($rows->toArray()); +} + +// after +public function extract(Backend $backend, Criteria $criteria): Generator +{ + yield rows(...); // stops after $criteria->limit rows +} + +public function load(Rows $rows, Backend $backend): void +{ + $this->write($rows->toArray()); +} +``` + +### 78) `flow-php/etl`, `flow-php/etl-adapter-parquet`, `flow-php/etl-adapter-excel`, `flow-php/etl-adapter-doctrine`, + +`flow-php/cli` - `withOffset()` is gone; `offset()` is pushed into file sources + +| Before | After | +|-----------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `from_parquet($path)->withOffset($n)`, `from_parquet($path, offset: $n)` | `df()->read(from_parquet($path))->offset($n)` | +| `from_floe($path)->withOffset($n)` | `df()->read(from_floe($path))->offset($n)` | +| `from_excel($path)->withOffset($n)` (a start row, the header counted) | `df()->read(from_excel($path))->offset($n - 2)` with a header, `->offset($n - 1)` without one | +| `ExcelReadOptions(bool $withHeader, bool $convertEmptyToNull, int $offset, ...)`, `withOffset()` | `ExcelReadOptions(bool $withHeader, bool $convertEmptyToNull, ?string $sheetName, ?ExcelReader $format)`; `SheetCells(SheetInterface $sheet, bool $withHeader)` | +| `from_dbal_limit_offset_qb($connection, $queryBuilder, offset: $n)`, `DbalLimitOffsetExtractor::withOffset($n)` | `$queryBuilder->setFirstResult($n)`; without a maximum it read no rows, it now reads from the offset | +| `ExplainedRows::of(..., ?int $maximum, int $offset)` | `of(..., ?int $maximum)` | +| `FooterStatistics::of(int $listedFiles, int $offset)` | `of(int $listedFiles)` | +| `flow` CLI `--input-parquet-offset`, `--input-excel-offset` | `--input-file-offset` | +| a custom `FileExtractor` | skips `$criteria->offset` rows matching every one of `$criteria->conjuncts` before it yields | + +### 79) `flow-php/etl` and every adapter - `isRepeatable()` is on `Extractor`; `RewindableExtractor`, + +`OverridingExtractor` and `Repeatability` are removed + +| Before | After | +|--------------------------------------------------------------|-------------------------------------------------------------------------------------------| +| `implements RewindableExtractor` + `isRepeatable(): bool` | `Extractor::isRepeatable(): bool` on every extractor; a one-shot stream returns `false` | +| an extractor that did not implement `RewindableExtractor` | implements `isRepeatable()` returning `false` | +| `OverridingExtractor::extractors()` | removed; a wrapper answers `isRepeatable()` for what it reads (`from_all()`: every child) | +| `(new Repeatability())->of($extractor)` | `$extractor->isRepeatable()` | +| `(new Repeatability())->ofPlan($plan)` | `$plan->isRepeatable()` | +| `from_cache($id)->withFallbackExtractor($e)->isRepeatable()` | `false` also when `$e` is not repeatable | + +### 80) `flow-php/etl` - `MemoryExtractor` merges into `ArrayExtractor` + +| Before | After | +|-------------------------------------------------------------------------------|------------------------------------------------------------------------| +| `Flow\ETL\Extractor\MemoryExtractor`, `from_memory($memory): MemoryExtractor` | `from_memory($memory): ArrayExtractor` (`new ArrayExtractor($memory)`) | +| `from_memory($memory)->statistics()`: unknown rows | the exact `Memory::count()`, read again on every plan | + +### 81) `flow-php/etl`, `flow-php/etl-adapter-doctrine` - `DataFrameFactory` becomes `ExtractorFactory` + +| Before | After | +|---------------------------------------------------------------|----------------------------------------------------------------------------------------------------------| +| `Flow\ETL\DataFrameFactory { from(Rows): DataFrame }` | `Flow\ETL\IO\Extractor\ExtractorFactory { from(Rows): Extractor }` | +| `Flow\ETL\Adapter\Doctrine\DbalDataFrameFactory` | `Flow\ETL\Adapter\Doctrine\IO\Extractor\DbalExtractorFactory`, `from()` returns the `DbalQueryExtractor` | +| `joinEach(DataFrameFactory $factory, ...)` | `joinEach(ExtractorFactory $factory, ...)`: the right rows of a left batch are the extractor's rows | +| `dbal_dataframe_factory($connection, $query, ...$parameters)` | `dbal_extractor_factory($connection, $query, ...$parameters)` | + +### 82) `flow-php/etl-adapter-json`, `flow-php/etl-adapter-parquet`, `flow-php/etl-adapter-xml`, `flow-php/etl-adapter-csv` - + +one framing enum, one converter vocabulary, a required XML writer, one copy of the CSV dialect checks + +| Before | After | +|-------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------| +| `Flow\ETL\Adapter\JSON\JSONMachine\JsonFormat::Document` | `Flow\ETL\Adapter\JSON\IO\JsonFraming::Array` (a read of `Array` or `ArrayLines` is a document) | +| `JsonFormat::Lines` | `JsonFraming::Lines` | +| `Flow\ETL\Adapter\Parquet\SchemaConverter::toFlow()`, `toParquet()` | `toFlowSchema()`, `toParquetSchema()` | +| `Flow\ETL\Adapter\JSON\SchemaConverter::toFlow()` | `toFlowSchema()` | +| `new XMLEncoder()` without a writer, then `encode()` threw `RuntimeException` | `XMLEncoder(XMLWriter $xmlWriter, ...)`: the writer is required | +| `new Detector\Option($separator, $enclosure, 'ab')` (any escape) | refused: an escape is empty or one character, as `CSVDialect::assertParts()` checks everywhere | + +### 83) `flow-php/etl` - the ASCII table is one formatter + +| Before | After | +|----------------------------------------------------------------------------------------|------------------------------------------------------------------| +| `Flow\ETL\Formatter\ASCII\{ASCIITable, Headers, Body, ASCIIHeaders, ASCIIBody}` | removed; `(new AsciiTableFormatter())->format($rows, $truncate)` | +| `AsciiTableFormatter::format()`, `to_output()` of a batch: time grew with rows squared | one pass over the cells | + +### 84) `flow-php/etl-adapter-csv`, `flow-php/etl-adapter-json`, `flow-php/etl-adapter-parquet` - the `Adaptive*` lane + +wrappers are gone, the class that builds the sink or source picks the lane + +| Before | After | +|-------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------| +| `Flow\ETL\Adapter\CSV\AdaptiveCSVEncoder` | removed; `CSVLoader::open()` picks `RustCSVEncoder` or `PhpCSVEncoder` | +| `Flow\ETL\Adapter\JSON\AdaptiveJsonEncoder` | removed; `JsonLoader::open()` picks `RustJsonEncoder` or `PhpJsonEncoder` | +| `Flow\ETL\Adapter\JSON\AdaptiveJsonOpenSource` | removed; `JsonFileBatches::open(SourceFile $source, ?FileSplit $split = null): JsonOpenSource` picks the lane | +| `Flow\ETL\Adapter\Parquet\AdaptiveParquetOpenSource` | removed; `ParquetSourceFile::open()` picks the lane | +| `Flow\ETL\Adapter\Parquet\AdaptiveParquetOpenSink` | removed; `ParquetFileWriterFactory::writer(): FileWriter` picks the lane | +| `new CSVLoader($path, $filesystem)`, `new JsonLoader($path, $filesystem, $framing)` | a last `?FlowPhpExtension $extension = null` | + +### 85) `flow-php/etl` and every adapter - IO types moved to `Flow\ETL\IO`, adapter types to their layer namespaces (`IO`, `Values`, `Functions`, `Foundation`) + +| Before | After | +|--------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Flow\ETL\{Extractor, Formatter, Loader, Transaction}` | `Flow\ETL\IO\{Extractor, Formatter, Loader, Transaction}` | +| `Flow\ETL\RustIterator` | `Flow\ETL\Values\RustIterator` | +| `Flow\ETL\Extractor\{ArrayExtractor, BatchableExtractor, CacheExtractor, ChainExtractor}` | `Flow\ETL\IO\Extractor\{ArrayExtractor, BatchableExtractor, CacheExtractor, ChainExtractor}` | +| `Flow\ETL\Extractor\{FeedExtractor, InfersSchema, RowsExtractor, SequenceExtractor, Signal}` | `Flow\ETL\IO\Extractor\{FeedExtractor, InfersSchema, RowsExtractor, SequenceExtractor, Signal}` | +| `Flow\ETL\Extractor\Statistics` | `Flow\ETL\IO\Extractor\Statistics` | +| `Flow\ETL\Extractor\File\{DerivedSchema, FileBatches, FileColumns, FileConstants, FileExtractor}` | `Flow\ETL\IO\Extractor\File\{DerivedSchema, FileBatches, FileColumns, FileConstants, FileExtractor}` | +| `Flow\ETL\Extractor\File\{FileReadLoop, FooterStatistics, InferredColumns, MetadataColumnsExtractor, OffsetSkippingFileBatches}` | `Flow\ETL\IO\Extractor\File\{FileReadLoop, FooterStatistics, InferredColumns, MetadataColumnsExtractor, OffsetSkippingFileBatches}` | +| `Flow\ETL\Extractor\File\{PartitionTypes, ReadWindow, SelfDescribingFile, SourceFile}` | `Flow\ETL\IO\Extractor\File\{PartitionTypes, ReadWindow, SelfDescribingFile, SourceFile}` | +| `Flow\ETL\Extractor\Grid\{HeaderNames, RecordDecoder}` | `Flow\ETL\IO\Extractor\Grid\{HeaderNames, RecordDecoder}` | +| `Flow\ETL\Extractor\Memory\{InMemoryRows, SpillState, SpilledRows}` | `Flow\ETL\IO\Extractor\Memory\{InMemoryRows, SpillState, SpilledRows}` | +| `Flow\ETL\Extractor\SequenceGenerator\{DatePeriodSequenceGenerator, NumberSequenceGenerator, SequenceGenerator}` | `Flow\ETL\IO\Extractor\SequenceGenerator\{DatePeriodSequenceGenerator, NumberSequenceGenerator, SequenceGenerator}` | +| `Flow\ETL\Filesystem\{SaveMode, ScalarFunctionFilter}` | `Flow\ETL\IO\Filesystem\{SaveMode, ScalarFunctionFilter}` | +| `Flow\ETL\Formatter\AsciiTableFormatter` | `Flow\ETL\IO\Formatter\AsciiTableFormatter` | +| `Flow\ETL\Formatter\ASCII\ASCIIValue` | `Flow\ETL\IO\Formatter\ASCII\ASCIIValue` | +| `Flow\ETL\Loader\{ArrayLoader, Closure, Discardable, MemoryLoader, Partitioning}` | `Flow\ETL\IO\Loader\{ArrayLoader, Closure, Discardable, MemoryLoader, Partitioning}` | +| `Flow\ETL\Loader\{PartitioningLoader, RowPartitions, SchemaValidationLoader, StreamLoader}` | `Flow\ETL\IO\Loader\{RowPartitions, SchemaValidationLoader, StreamLoader}` (`PartitioningLoader` removed, see 19) | +| `Flow\ETL\Loader\File\{FileLoader, FileSink, FileSinks, FileWriteFrame, FilesSink}` | `Flow\ETL\IO\Loader\File\{FileLoader, FileWriter, FileWriterFactory, FileWrite, FileCommit}` | +| `Flow\ETL\Loader\File\PartitionRouter` | `Flow\ETL\IO\Loader\File\PartitionRouter` | +| `Flow\ETL\Loader\StreamLoader\{Output, Type}` | `Flow\ETL\IO\Loader\StreamLoader\{Output, Type}` | +| `Flow\ETL\Row\Formatter\ASCIISchemaFormatter` | `Flow\ETL\IO\Formatter\Row\ASCIISchemaFormatter` | +| `Flow\ETL\Schema\SchemaFormatter` | `Flow\ETL\IO\Formatter\Schema\SchemaFormatter` | +| `Flow\ETL\Schema\Formatter\{InlineSchemaFormatter, JsonSchemaFormatter, PHPSchemaFormatter}` | `Flow\ETL\IO\Formatter\Schema\{InlineSchemaFormatter, JsonSchemaFormatter, PHPSchemaFormatter}` | +| `Flow\ETL\Schema\Formatter\PHPFormatter\{TypeFormatter, ValueFormatter}` | `Flow\ETL\IO\Formatter\Schema\PHPFormatter\{TypeFormatter, ValueFormatter}` | +| `Flow\ETL\Schema\Inference\{ColumnTypes, InferredTypes, SchemaInference, SchemaInferenceBuilder, SchemaInferrer}` | `Flow\ETL\IO\Inference\{ColumnTypes, InferredTypes, SchemaInference, SchemaInferenceBuilder, SchemaInferrer}` | +| `Flow\ETL\Schema\Inference\TypeFloor` | `Flow\ETL\IO\Inference\TypeFloor` | +| `Flow\ETL\Adapter\Avro\FlixTech\AvroLoader` | `Flow\ETL\Adapter\Avro\IO\Loader\AvroLoader` | +| `Flow\ETL\Adapter\Avro\FlixTech\AvroExtractor` | `Flow\ETL\Adapter\Avro\IO\Extractor\AvroExtractor` | +| `Flow\ETL\Adapter\CSV\CSVDialect` | `Flow\ETL\Adapter\CSV\IO\CSVDialect` | +| `Flow\ETL\Adapter\CSV\{CSVEncoder, CSVLoader, CSVOpenSink, CSVWriteOptions, PhpCSVEncoder}` | `Flow\ETL\Adapter\CSV\IO\Loader\{CSVEncoder, CSVLoader, CSVFileWriter, CSVWriteOptions, PhpCSVEncoder}` | +| `Flow\ETL\Adapter\CSV\RustCSVEncoder` | `Flow\ETL\Adapter\CSV\IO\Loader\RustCSVEncoder` | +| `Flow\ETL\Adapter\CSV\{CSVDecoder, CSVDetector, CSVEnclosureScan, CSVExtractor, CSVFileBatches}` | `Flow\ETL\Adapter\CSV\IO\Extractor\{CSVDecoder, CSVDetector, CSVEnclosureScan, CSVExtractor, CSVFileBatches}` | +| `Flow\ETL\Adapter\CSV\{CSVLineReader, CSVOpenSource, CSVReadOptions, CSVRecordBoundary, PhpCSVOpenSource}` | `Flow\ETL\Adapter\CSV\IO\Extractor\{CSVLineReader, CSVOpenSource, CSVReadOptions, CSVRecordBoundary, PhpCSVOpenSource}` | +| `Flow\ETL\Adapter\CSV\RustCSVOpenSource` | `Flow\ETL\Adapter\CSV\IO\Extractor\RustCSVOpenSource` | +| `Flow\ETL\Adapter\CSV\Detector\{Option, Options}` | `Flow\ETL\Adapter\CSV\IO\Extractor\Detector\{Option, Options}` | +| `Flow\ETL\Adapter\CSV\Exception\CantDetectCSVOptions` | `Flow\ETL\Adapter\CSV\Foundation\Exception\CantDetectCSVOptions` | +| `Flow\ETL\Adapter\ChartJS\{Chart, ChartJSLoader}` | `Flow\ETL\Adapter\ChartJS\IO\Loader\{Chart, ChartJSLoader}` | +| `Flow\ETL\Adapter\ChartJS\Chart\{BarChart, LineChart, PieChart}` | `Flow\ETL\Adapter\ChartJS\IO\Loader\Chart\{BarChart, LineChart, PieChart}` | +| `Flow\ETL\Adapter\Doctrine\{DbalMetadata, SchemaConverter, TypesMap}` | `Flow\ETL\Adapter\Doctrine\Values\Schema\{DbalMetadata, SchemaConverter, TypesMap}` | +| `Flow\ETL\Adapter\Doctrine\{DbalEncoder, DbalLoader, DbalTransaction, IsolationLevelRestore}` | `Flow\ETL\Adapter\Doctrine\IO\Loader\{DbalEncoder, DbalLoader, DbalTransaction, IsolationLevelRestore}` | +| `Flow\ETL\Adapter\Doctrine\{DbalKeySetExtractor, DbalLimitOffsetExtractor, DbalQueryExtractor, DbalResultSchema, DescribeQuery}` | `Flow\ETL\Adapter\Doctrine\IO\Extractor\{DbalKeySetExtractor, DbalLimitOffsetExtractor, DbalQueryExtractor, DbalResultSchema, DescribeQuery}` | +| `Flow\ETL\Adapter\Doctrine\{LiteralParameter, MysqliPlaceholders, MysqliResultColumns, MysqliTypesMap, NativePlaceholders}` | `Flow\ETL\Adapter\Doctrine\IO\Extractor\{LiteralParameter, MysqliPlaceholders, MysqliResultColumns, MysqliTypesMap, NativePlaceholders}` | +| `Flow\ETL\Adapter\Doctrine\{NativeTypes, Order, OrderBy, Pages, Parameter}` | `Flow\ETL\Adapter\Doctrine\IO\Extractor\{NativeTypes, Order, OrderBy, Pages, Parameter}` | +| `Flow\ETL\Adapter\Doctrine\{ParametersSet, PdoSqliteColumnNames, PgSqlResultColumns, PgSqlTypesMap, PlaceholderDialect}` | `Flow\ETL\Adapter\Doctrine\IO\Extractor\{ParametersSet, PdoSqliteColumnNames, PgSqlResultColumns, PgSqlTypesMap, PlaceholderDialect}` | +| `Flow\ETL\Adapter\Doctrine\{PlaceholderRewriter, PostgreSqlPlaceholders, QueryParameter, ResultColumn}` | `Flow\ETL\Adapter\Doctrine\IO\Extractor\{PlaceholderRewriter, PostgreSqlPlaceholders, QueryParameter, ResultColumn}` | +| `Flow\ETL\Adapter\Doctrine\ProbeRefusal` | `Flow\ETL\Adapter\Doctrine\Foundation\Exception\ProbeRefusal` | +| `Flow\ETL\Adapter\Doctrine\{RewrittenSql, Sqlite3ColumnNames, SqliteResultSchema, SqliteTypes, Table}` | `Flow\ETL\Adapter\Doctrine\IO\Extractor\{RewrittenSql, Sqlite3ColumnNames, SqliteResultSchema, SqliteTypes, Table}` | +| `Flow\ETL\Adapter\Doctrine\TypedColumns` | `Flow\ETL\Adapter\Doctrine\IO\Extractor\TypedColumns` | +| `Flow\ETL\Adapter\Doctrine\Explain\{ExplainedRows, MySqlExplainedRows, PostgreSqlExplainedRows}` | `Flow\ETL\Adapter\Doctrine\IO\Extractor\Explain\{ExplainedRows, MySqlExplainedRows, PostgreSqlExplainedRows}` | +| `Flow\ETL\Adapter\Doctrine\Pagination\{Key, KeySet, KeySetPage, KeyValues, Order}` | `Flow\ETL\Adapter\Doctrine\IO\Extractor\Pagination\{Key, KeySet, KeySetPage, KeyValues, Order}` | +| `Flow\ETL\Adapter\Excel\{CellStyler, ExcelEncoder, ExcelLoader, ExcelWriter, WorkbookManager}` | `Flow\ETL\Adapter\Excel\IO\Loader\{CellStyler, ExcelEncoder, ExcelLoader, ExcelWriter, WorkbookManager}` | +| `Flow\ETL\Adapter\Excel\CellTypeNarrower` | `Flow\ETL\Adapter\Excel\IO\Inference\CellTypeNarrower` | +| `Flow\ETL\Adapter\Excel\{ExcelExtractor, ExcelFileBatches, ExcelFormatDetector, ExcelReadOptions}` | `Flow\ETL\Adapter\Excel\IO\Extractor\{ExcelExtractor, ExcelFileBatches, ExcelFormatDetector, ExcelReadOptions}` | +| `Flow\ETL\Adapter\Excel\{ExcelReader, WorkbookReader, WorkbookSheet}` | `Flow\ETL\Adapter\Excel\IO\Extractor\{ExcelReader, WorkbookReader, WorkbookSheet}` | +| `Flow\ETL\Adapter\Excel\WorkbookSheetSample` | `Flow\ETL\Adapter\Excel\IO\Inference\WorkbookSheetSample` | +| `Flow\ETL\Adapter\Excel\Sheet\SheetNameAssertion` | `Flow\ETL\Adapter\Excel\Functions\SheetNameAssertion` | +| `Flow\ETL\Adapter\Excel\Sheet\{OpenSheet, SheetCells, SheetsManager, XlsxSheetRows}` | `Flow\ETL\Adapter\Excel\IO\Extractor\Sheet\{OpenSheet, SheetCells, SheetsManager, XlsxSheetRows}` | +| `Flow\ETL\Adapter\Excel\Function\IsValidExcelSheetName` | `Flow\ETL\Adapter\Excel\Functions\IsValidExcelSheetName` | +| `Flow\ETL\Adapter\GoogleSheet\{Columns, GoogleSheetExtractor, GoogleSheetReadOptions, GoogleSheetReader}` | `Flow\ETL\Adapter\GoogleSheet\IO\Extractor\{Columns, GoogleSheetExtractor, GoogleSheetReadOptions, GoogleSheetReader}` | +| `Flow\ETL\Adapter\GoogleSheet\GoogleSheetSample` | `Flow\ETL\Adapter\GoogleSheet\IO\Inference\GoogleSheetSample` | +| `Flow\ETL\Adapter\GoogleSheet\SheetRange` | `Flow\ETL\Adapter\GoogleSheet\IO\Extractor\SheetRange` | +| `Flow\ETL\Adapter\Http\{ContentTypeDetector, HttpDecoder, HttpExchange, PsrHttpClientDynamicExtractor, PsrHttpClientPaginatedExtractor}` | `Flow\ETL\Adapter\Http\IO\Extractor\{ContentTypeDetector, HttpDecoder, HttpExchange, PsrHttpClientDynamicExtractor, PsrHttpClientPaginatedExtractor}` | +| `Flow\ETL\Adapter\Http\{PsrHttpClientStaticExtractor, ResponseType}` | `Flow\ETL\Adapter\Http\IO\Extractor\{PsrHttpClientStaticExtractor, ResponseType}` | +| `Flow\ETL\Adapter\Http\DynamicExtractor\NextRequestFactory` | `Flow\ETL\Adapter\Http\IO\Extractor\NextRequestFactory` | +| `Flow\ETL\Adapter\Http\Pagination\{DecodedResponse, InjectInto, LinkHeaderParser, PageState, PaginationStrategy}` | `Flow\ETL\Adapter\Http\IO\Extractor\Pagination\{DecodedResponse, InjectInto, LinkHeaderParser, PageState, PaginationStrategy}` | +| `Flow\ETL\Adapter\Http\Pagination\{Paginator, RelativeUriResolver, RequestComparator, RequestOption, ResultCounter}` | `Flow\ETL\Adapter\Http\IO\Extractor\Pagination\{Paginator, RelativeUriResolver, RequestComparator, RequestOption, ResultCounter}` | +| `Flow\ETL\Adapter\Http\Pagination\StopWhen` | `Flow\ETL\Adapter\Http\IO\Extractor\Pagination\StopWhen` | +| `Flow\ETL\Adapter\Http\Pagination\StopWhen\{AllOf, AnyOf, EmptyPath, EmptyResponseBody, FlagFalse}` | `Flow\ETL\Adapter\Http\IO\Extractor\Pagination\StopWhen\{AllOf, AnyOf, EmptyPath, EmptyResponseBody, FlagFalse}` | +| `Flow\ETL\Adapter\Http\Pagination\StopWhen\{FlagTrue, MaxPages, MaxResults, NeverStop, PathMissing}` | `Flow\ETL\Adapter\Http\IO\Extractor\Pagination\StopWhen\{FlagTrue, MaxPages, MaxResults, NeverStop, PathMissing}` | +| `Flow\ETL\Adapter\Http\Pagination\StopWhen\TotalReached` | `Flow\ETL\Adapter\Http\IO\Extractor\Pagination\StopWhen\TotalReached` | +| `Flow\ETL\Adapter\Http\Pagination\Strategy\{CursorFromBody, LastRecordCursor, LinkHeader, NextUrlFromBody, OffsetLimit}` | `Flow\ETL\Adapter\Http\IO\Extractor\Pagination\Strategy\{CursorFromBody, LastRecordCursor, LinkHeader, NextUrlFromBody, OffsetLimit}` | +| `Flow\ETL\Adapter\Http\Pagination\Strategy\PageNumber` | `Flow\ETL\Adapter\Http\IO\Extractor\Pagination\Strategy\PageNumber` | +| `Flow\ETL\Adapter\JSON\JsonFraming` | `Flow\ETL\Adapter\JSON\IO\JsonFraming` | +| `Flow\ETL\Adapter\JSON\SchemaConverter` | `Flow\ETL\Adapter\JSON\IO\Schema\SchemaConverter` | +| `Flow\ETL\Adapter\JSON\{JsonEncoder, JsonLoader, JsonOpenSink, PhpJsonEncoder, RustJsonEncoder}` | `Flow\ETL\Adapter\JSON\IO\Loader\{JsonEncoder, JsonLoader, JsonFileWriter, PhpJsonEncoder, RustJsonEncoder}` | +| `Flow\ETL\Adapter\JSON\{JsonOpenSource, PhpJsonOpenSource, RustJsonOpenSource}` | `Flow\ETL\Adapter\JSON\IO\Extractor\{JsonOpenSource, PhpJsonOpenSource, RustJsonOpenSource}` | +| `Flow\ETL\Adapter\JSON\JSONMachine\{JsonArrayElements, JsonExtractor, JsonFileBatches, JsonFileReader}` | `Flow\ETL\Adapter\JSON\IO\Extractor\{JsonArrayElements, JsonExtractor, JsonFileBatches, JsonFileReader}` | +| `Flow\ETL\Adapter\JSON\JSONMachine\JsonFileSample` | `Flow\ETL\Adapter\JSON\IO\Inference\JsonFileSample` | +| `Flow\ETL\Adapter\JSON\JSONMachine\JsonReadBytes` | `Flow\ETL\Adapter\JSON\IO\Extractor\JsonReadBytes` | +| `Flow\ETL\Adapter\JSON\JsonSchema\{JsonSchemaMetadata, ResolvedReference}` | `Flow\ETL\Adapter\JSON\Values\Schema\JsonSchema\{JsonSchemaMetadata, ResolvedReference}` | +| `Flow\ETL\Adapter\JSON\JsonSchema\ReferenceResolver` | `Flow\ETL\Adapter\JSON\IO\Schema\JsonSchema\ReferenceResolver` | +| `Flow\ETL\Adapter\JSON\JsonSchema\Exception\{CircularReferenceException, UnresolvableReferenceException, UnsupportedKeywordException}` | `Flow\ETL\Adapter\JSON\Foundation\Exception\{CircularReferenceException, UnresolvableReferenceException, UnsupportedKeywordException}` | +| `Flow\ETL\Adapter\Logger\PsrLoggerLoader` | `Flow\ETL\Adapter\Logger\IO\Loader\PsrLoggerLoader` | +| `Flow\ETL\Adapter\Logger\Logger\DumpLogger` | `Flow\ETL\Adapter\Logger\IO\Loader\DumpLogger` | +| `Flow\ETL\Adapter\Parquet\SchemaConverter` | `Flow\ETL\Adapter\Parquet\Values\Schema\SchemaConverter` | +| `Flow\ETL\Adapter\Parquet\{ParquetEncoder, ParquetFileSink, ParquetFileSinks, ParquetLoader, ParquetOpenSink}` | `Flow\ETL\Adapter\Parquet\IO\Loader\{ParquetEncoder, ParquetFileWriter, ParquetFileWriterFactory, ParquetLoader}` | +| `Flow\ETL\Adapter\Parquet\{ParquetSchemaConformance, PhpParquetOpenSink, RustParquetOpenSink}` | `Flow\ETL\Adapter\Parquet\IO\Loader\{ParquetSchemaConformance, RustParquetFileWriter}` | +| `Flow\ETL\Adapter\Parquet\{ParquetExtractor, ParquetFileBatches, ParquetOpenSource, ParquetSourceFile, ParquetSourceFileOpener}` | `Flow\ETL\Adapter\Parquet\IO\Extractor\{ParquetExtractor, ParquetFileBatches, ParquetOpenSource, ParquetSourceFile, ParquetSourceFileOpener}` | +| `Flow\ETL\Adapter\Parquet\{PhpParquetOpenSource, RustParquetOpenSource}` | `Flow\ETL\Adapter\Parquet\IO\Extractor\{PhpParquetOpenSource, RustParquetOpenSource}` | +| `Flow\ETL\Adapter\Parquet\ValueConverter\{ElementsValueConverter, JsonValueConverter, StructValueConverter, UuidValueConverter, ValueConverter}` | `Flow\ETL\Adapter\Parquet\IO\Loader\{ElementsValueConverter, JsonValueConverter, StructValueConverter, UuidValueConverter, ValueConverter}` | +| `Flow\ETL\Adapter\Parquet\ValueConverter\ValueConverters` | `Flow\ETL\Adapter\Parquet\IO\Loader\ValueConverters` | +| `Flow\ETL\Adapter\PostgreSql\{PostgreSqlMetadata, SchemaConverter}` | `Flow\ETL\Adapter\PostgreSql\Values\Schema\{PostgreSqlMetadata, SchemaConverter}` | +| `Flow\ETL\Adapter\PostgreSql\{Operation, PostgreSqlEncoder, PostgreSqlLoader, PostgreSqlTransaction}` | `Flow\ETL\Adapter\PostgreSql\IO\Loader\{Operation, PostgreSqlEncoder, PostgreSqlLoader, PostgreSqlTransaction}` | +| `Flow\ETL\Adapter\PostgreSql\{ExplainedRows, PostgreSqlCursorExtractor, PostgreSqlKeySetExtractor, PostgreSqlLimitOffsetExtractor, ReadQuery}` | `Flow\ETL\Adapter\PostgreSql\IO\Extractor\{ExplainedRows, PostgreSqlCursorExtractor, PostgreSqlKeySetExtractor, PostgreSqlLimitOffsetExtractor, ReadQuery}` | +| `Flow\ETL\Adapter\PostgreSql\ResultSchema` | `Flow\ETL\Adapter\PostgreSql\IO\Extractor\ResultSchema` | +| `Flow\ETL\Adapter\PostgreSql\LoaderOptions\{DeleteOptions, InsertOptions, UpdateOptions}` | `Flow\ETL\Adapter\PostgreSql\IO\Loader\LoaderOptions\{DeleteOptions, InsertOptions, UpdateOptions}` | +| `Flow\ETL\Adapter\PostgreSql\Pagination\{Key, KeySet, KeySetPage, KeyValues, Order}` | `Flow\ETL\Adapter\PostgreSql\IO\Extractor\Pagination\{Key, KeySet, KeySetPage, KeyValues, Order}` | +| `Flow\ETL\Adapter\PostgreSql\QueryBuilder\{DeleteQueryBuilder, InsertQueryBuilder, UpdateQueryBuilder}` | `Flow\ETL\Adapter\PostgreSql\IO\Loader\QueryBuilder\{DeleteQueryBuilder, InsertQueryBuilder, UpdateQueryBuilder}` | +| `Flow\ETL\Adapter\PostgreSql\Schema\SortingStrategy\TypeStrategy` | `Flow\ETL\Adapter\PostgreSql\Values\Schema\SortingStrategy\TypeStrategy` | +| `Flow\ETL\Adapter\PostgreSql\ValueConverter\{EnumConverter, HTMLConverter, XMLConverter}` | `Flow\ETL\Adapter\PostgreSql\IO\Loader\{EnumConverter, HTMLConverter, XMLConverter}` | +| `Flow\ETL\Adapter\PostgreSql\Exception\RuntimeException` | removed: the adapter throws `Flow\ETL\Foundation\Exception\RuntimeException`, its parent | +| `Flow\ETL\Adapter\PostgreSql\Exception\TypeMappingException` | `Flow\ETL\Adapter\PostgreSql\Foundation\Exception\TypeMappingException` | +| `Flow\ETL\Adapter\Seal\{SchemaConverter, SealMetadata}` | `Flow\ETL\Adapter\Seal\Values\Schema\{SchemaConverter, SealMetadata}` | +| `Flow\ETL\Adapter\Seal\{Operation, SealEncoder, SealLoader}` | `Flow\ETL\Adapter\Seal\IO\Loader\{Operation, SealEncoder, SealLoader}` | +| `Flow\ETL\Adapter\Text\{TextEncoder, TextLoader, TextOpenSink}` | `Flow\ETL\Adapter\Text\IO\Loader\{TextEncoder, TextLoader, TextFileWriter}` | +| `Flow\ETL\Adapter\Text\{TextExtractor, TextFileBatches}` | `Flow\ETL\Adapter\Text\IO\Extractor\{TextExtractor, TextFileBatches}` | +| `Flow\ETL\Adapter\XML\{XMLEncoder, XMLOpenSink, XMLValueNodes, XMLWriter}` | `Flow\ETL\Adapter\XML\IO\Loader\{XMLEncoder, XMLFileWriter, XMLValueNodes, XMLWriter}` | +| `Flow\ETL\Adapter\XML\{RootlessDocument, XMLFileBatches, XMLNodeBatches, XMLNodeCursor, XMLNodes}` | `Flow\ETL\Adapter\XML\IO\Extractor\{RootlessDocument, XMLFileBatches, XMLNodeBatches, XMLNodeCursor, XMLNodes}` | +| `Flow\ETL\Adapter\XML\XMLParserExtractor` | `Flow\ETL\Adapter\XML\IO\Extractor\XMLParserExtractor` | +| `Flow\ETL\Adapter\XML\Abstraction\{XMLAttribute, XMLNode, XMLNodeType}` | `Flow\ETL\Adapter\XML\IO\Loader\{XMLAttribute, XMLNode, XMLNodeType}` | +| `Flow\ETL\Adapter\XML\Loader\XMLLoader` | `Flow\ETL\Adapter\XML\IO\Loader\XMLLoader` | +| `Flow\ETL\Adapter\XML\XMLWriter\{DOMDocumentWriter, StringXMLWriter}` | `Flow\ETL\Adapter\XML\IO\Loader\XMLWriter\{DOMDocumentWriter, StringXMLWriter}` | + +DSL function namespaces do not change. + +### 86) `flow-php/etl` - constraints start every run fresh, `Stateful` no longer extends `Transformer` + +| Before | After | +|-----------------------------------------------------------------------------------------|--------------------------------------------------------------| +| a frame with `constraint_unique()` or `constraint_sorted_by()` failed on its second run | passes | +| `interface Stateful extends Transformer { public function fresh(): self; }` | `interface Stateful { public function fresh(): static; }` | +| `final class MyTransformer implements Stateful` | `final class MyTransformer implements Stateful, Transformer` | +| a `Constraint` with state between batches kept it across runs | `final class MyConstraint implements Constraint, Stateful` | + +### 87) `flow-php/etl` - `DropDuplicatesTransformer` is removed + +| Before | After | +|---------------------------------------------------------------|-------------------------------------| +| `$df->transform(new DropDuplicatesTransformer('id', 'name'))` | `$df->dropDuplicates('id', 'name')` | +| `Flow\ETL\Transformer\DropDuplicatesTransformer` | removed | + +### 88) `flow-php/etl` - the memory limit is part of `BucketingConfig`, one builder for group by and repartition + +| Before | After | +|-------------------------------------------------------------------------------------|---------------------------------------------------------------------------------| +| `new BucketingConfig($storage, $bucketsCount, $batchSize)` | `new BucketingConfig($storage, $bucketsCount, $batchSize, $memoryLimit)` | +| `BucketingConfig::spilling(Unit $memoryLimit, Backend $backend)` | `BucketingConfig::spilling(Backend $backend)` | +| `Flow\ETL\Config\Grouping\HashGroupByConfig` | `Flow\ETL\Storage\BucketingConfig` | +| `Flow\ETL\Config\Repartition\HashRepartitionConfig` | `Flow\ETL\Storage\BucketingConfig` | +| `new HashJoinConfig($bucketing, $memoryLimit, $broadcastLimit)` | `new HashJoinConfig($bucketing, $broadcastLimit)` | +| `new ExternalSortConfig($bucketing, $memoryLimit, $merge)` | `new ExternalSortConfig($bucketing, $merge)` | +| `$config->memoryLimit` of a join, sort, group by or repartition config | `$config->bucketing->memoryLimit` (join, sort), `$config->memoryLimit` (others) | +| `hash_group_by(): HashGroupByBuilder`, `hash_repartition(): HashRepartitionBuilder` | both return `Flow\ETL\API\Config\Bucketing\BucketingConfigBuilder` | +| `Flow\ETL\Config\Grouping\HashGroupByBuilder` | removed: `BucketingConfigBuilder::groupBy()` | +| `Flow\ETL\Config\Repartition\HashRepartitionBuilder` | removed: `BucketingConfigBuilder::repartition()` | +| `Join memory limit must be greater than 0 bytes` (and `Sort`, `Group by`, ...) | `Bucketing memory limit must be greater than 0 bytes` | + +### 89) `flow-php/etl` - `GroupBy` is immutable, a group by is placed by `aggregate()` + +| Before | After | +|--------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------| +| `$g = new GroupBy('a'); $g->aggregate(sum(ref('v'))); $g->pivot($ref, $values)` | `new GroupBy(refs('a'), new Aggregators(sum(ref('v'))), new Pivot($ref, $values))` | +| `GroupBy::aggregate()`, `GroupBy::pivot()` | removed | +| `PivotValues::resolve(DataFrame $source, Reference $pivot)` | `PivotValues::resolve(iterable $batches, Reference $pivot)` (`iterable`) | +| - | `DataFrame::aggregateBy(GroupBy $groupBy, ?BucketingAlgorithmBuilder $algorithm)` | +| `$df->groupBy('a');` without `->aggregate()` grouped `$df` to its distinct keys | `$df` is unchanged: call `->aggregate(...)` | +| `$grouped->aggregate(sum(...), count(...))` then `$grouped->pivot(...)` pivoted the aggregated frame on `sum` only | the aggregated frame is not pivoted: call `->pivot(...)->aggregate(...)` | + +### 90) `flow-php/etl` - transformers and processors take a `Backend`, not a `FlowContext` + +| Before | After | +|------------------------------------------------------------------------|-----------------------------------------------------------------------------------| +| `Transformer::transform(Rows $rows, FlowContext $context): Rows` | `Transformer::transform(Rows $rows, Backend $backend): Rows` | +| `Processor::process(Generator $rows, FlowContext $context): Generator` | `Processor::process(Generator $rows, Backend $backend): Generator` | +| `new SerializeTransformer($target, $standalone)` | `new SerializeTransformer($target, $serializer, $standalone)` | +| `new UnserializeTransformer($source, $target, $merge, $mergePrefix)` | `new UnserializeTransformer($source, $target, $serializer, $merge, $mergePrefix)` | +| `new CachingProcessor(?string $id = null, ?RowsStorage $cache = null)` | `new CachingProcessor(string $id, RowsStorage $cache)` | + +`explain()` of a cache step always prints the storage in use as `Cache: `. + +A transformer that read the config at run time takes what it needs in its constructor. + +Before: + +```php +final class MyTransformer implements Transformer +{ + public function transform(Rows $rows, FlowContext $context): Rows + { + $serializer = $context->config->serializer(); + // ... + } +} +``` + +After: + +```php +final class MyTransformer implements Transformer +{ + public function __construct(private Serializer $serializer) + { + } + + public function transform(Rows $rows, Backend $backend): Rows + { + // ... + } +} +``` + +### 91) `flow-php/etl` - operator types moved to `Flow\ETL\Operators`, their configs to `Storage`, `Kernels` and + +`Operators\Join` + +| Before | After | +|--------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Flow\ETL\{BoundStep, Constraint, GroupBy, Processor, Transformer}` | `Flow\ETL\Operators\{BoundStep, Constraint, GroupBy, Processor, Transformer}` | +| `Flow\ETL\Config\Bucketing\BucketingConfig` | `Flow\ETL\Storage\BucketingConfig` | +| `Flow\ETL\Config\Join\HashJoinConfig` | `Flow\ETL\Operators\Join\HashJoinConfig` | +| `Flow\ETL\Config\Sort\{ExternalSortConfig, MemorySortConfig}` | `Flow\ETL\Kernels\Sort\{ExternalSortConfig, MemorySortConfig}` | +| `Flow\ETL\Constraint\{SortedByConstraint, UniqueConstraint}` | `Flow\ETL\Operators\Constraint\{SortedByConstraint, UniqueConstraint}` | +| `Flow\ETL\GroupBy\{AggregatedGroups, Aggregators, DeclaredPivotValues, DiscoveredPivotValues, Group}` | `Flow\ETL\Operators\GroupBy\{AggregatedGroups, Aggregators, DeclaredPivotValues, DiscoveredPivotValues, Group}` | +| `Flow\ETL\GroupBy\{GroupByShape, GroupKey, Pivot, PivotShape, PivotValues}` | `Flow\ETL\Operators\GroupBy\{GroupByShape, GroupKey, Pivot, PivotShape, PivotValues}` | +| `Flow\ETL\GroupBy\PivotedTable` | `Flow\ETL\Operators\GroupBy\PivotedTable` | +| `Flow\ETL\Plan\RequiredColumns` | `Flow\ETL\Operators\Processor\RequiredColumns` | +| `Flow\ETL\Processor\{BatchingByProcessor, BatchingProcessor, BoundExpansion, CachingProcessor, CollectingProcessor}` | `Flow\ETL\Operators\Processor\{BatchingByProcessor, BatchingProcessor, BoundExpansion, CachingProcessor, CollectingProcessor}` | +| `Flow\ETL\Processor\{ConstrainedProcessor, CountingProcessor, ExpandingProcessor, GroupByAggregationProcessor, HashJoinProcessor}` | `Flow\ETL\Operators\Processor\{ConstrainedProcessor, CountingProcessor, ExpandingProcessor, GroupByAggregationProcessor}`, `HashJoinProcessor` -> `Flow\ETL\Operators\Join\HashJoin` (row 94) | +| `Flow\ETL\Processor\{MemorySortProcessor, MergeSortProcessor, OffsetProcessor, PivotProcessor, RepartitionProcessor}` | `Flow\ETL\Operators\Processor\{MemorySortProcessor, MergeSortProcessor, OffsetProcessor, PivotProcessor, RepartitionProcessor}` | +| `Flow\ETL\Processor\{TopNProcessor, VoidProcessor, WindowProcessor}` | `Flow\ETL\Operators\Processor\{TopNProcessor, VoidProcessor, WindowProcessor}` | +| `Flow\ETL\Transformation\AddRowIndex\StartFrom` | `Flow\ETL\Operators\Transformer\AddRowIndex\StartFrom` | +| `Flow\ETL\Transformer\{AddRowIndexTransformer, CollectReferencesTransformer, CrossJoinRowsTransformer, DerivedColumns, DuplicateRowTransformer}` | `Flow\ETL\Operators\Transformer\{AddRowIndexTransformer, CollectReferencesTransformer, DerivedColumns, DuplicateRowTransformer}` | +| `Flow\ETL\Transformer\{Expansion, JoinEachRowsTransformer, LimitTransformer, ScalarFunctionFilterTransformer, ScalarFunctionTransformer}` | `Flow\ETL\Operators\Transformer\{Expansion, JoinEachRowsTransformer, LimitTransformer, ScalarFunctionFilterTransformer, ScalarFunctionTransformer}` | +| `Flow\ETL\Transformer\{SerializeTransformer, SerializedPayloadDecoder, Stateful, UnpackedColumns, UnserializeTransformer}` | `Flow\ETL\Operators\Transformer\{SerializeTransformer, SerializedPayloadDecoder, Stateful, UnpackedColumns, UnserializeTransformer}` | +| `Flow\ETL\Transformer\UntilTransformer` | `Flow\ETL\Operators\Transformer\UntilTransformer` | + +`ConstrainedProcessor` refuses a non-constraint with `Pipeline constraints must be of type Flow\ETL\Operators\Constraint`. + +### 92) `flow-php/etl` - the `*Steps` factories are removed + +| Before | After | +|----------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Flow\ETL\GroupBy\GroupBySteps::of()` | removed: the planner builds `[PruneColumnsTransformer, GroupByAggregationProcessor]` (the prune only when `GroupBy::readColumns(): ?list` is not null) or `[PivotProcessor]` | +| `Flow\ETL\Join\JoinSteps::of()` | removed: the planner builds a `JoinStep` over a `HashJoin` (row 94) | +| `Flow\ETL\Repartition\RepartitionSteps::of()` | removed: `new RepartitionProcessor(References $by, BucketingConfig $config, RandomValueGenerator $random)` | +| `new RepartitionProcessor($by, HashBucketing $bucketing, Buckets $buckets, Unit $memoryLimit, Hasher $hasher)` | `new RepartitionProcessor($by, BucketingConfig $config, RandomValueGenerator $random)`: it builds its bucketing and its spilling buckets from `$config`; `$memoryLimit` is `$config->memoryLimit` | +| `Flow\ETL\Sort\SortSteps::of()` | removed: `new MemorySortProcessor($columns)` or `new MergeSortProcessor($config, $columns, $random)` | +| `SortSteps::external()` | `Flow\ETL\Kernels\Sort\ExternalSort::of($columns, ExternalSortConfig $config, RandomValueGenerator $random)` | + +### 93) `flow-php/etl` - the optimizer walks are methods of their rule + +| Before | After | +|----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------| +| `Flow\ETL\Optimizer\LimitWalk::of($from, $leaf)` | `Flow\ETL\Planning\Optimizer\Rule\PushLimitIntoSource::limitAt($from, $leaf)` | +| `Flow\ETL\Optimizer\FilterWalk::reachedByEveryConsumer()`, `::predicateAtLeaf()` | `Flow\ETL\Planning\Optimizer\Rule\PushFilterIntoSource::reachedByEveryConsumer()`, `::predicateAtLeaf()` | +| `Flow\ETL\Optimizer\ProjectionWalk::below($node, $above)` | `Flow\ETL\Planning\Optimizer\Rule\PushProjectionIntoExpand::below($node, $above)` | + +### 94) `flow-php/etl` - the join operators are fed by the runtime + +| Before | After | +|----------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Flow\ETL\Processor\HashJoinProcessor` (a `Processor`) | `Flow\ETL\Operators\Join\HashJoin`, not a `Processor`: `join(Generator $left, Generator $right, Backend $backend): Generator`, `bind(Schema $left, Schema $right): BoundStep` | +| `new HashJoinProcessor(PhysicalPlan $right, Executor $executor, $on, $type, $config, $random, $backend)` | `new HashJoin($on, $type, $config, $random)`; the right plan is not the operator's | +| `Flow\ETL\Transformer\CrossJoinRowsTransformer` (a `Transformer`) | `Flow\ETL\Operators\Join\CrossJoin`, not a `Transformer`: `join(Generator $left, Generator $right, Backend $backend): Generator`, `bind(Schema $left, Schema $right): BoundStep` | +| `new CrossJoinRowsTransformer(PhysicalPlan $right, Executor $executor, string $prefix)` | `new CrossJoin(string $prefix)` | +| a join step holds its right plan | the plan holds `Flow\ETL\Planning\Physical\JoinStep(HashJoin\|CrossJoin $join, PhysicalPlan $right)`; the runtime runs the right plan once and hands its rows to `join()` | +| `crossJoin()`: a failure in it is offered to the error handler (`onTransformation()`) | a failure in it propagates like any processor's, a failure of the right side included; the error handler is not asked | +| `crossJoin()`: one transformation span per batch | no span, as for every processor and for `join()` | +| `BoundStep::$step` is `Processor\|Transformer` | `Processor\|Transformer\|HashJoin\|CrossJoin`, typed by `@template`: `Processor::bind()` returns `BoundStep`, `Transformer::bind()` `BoundStep` | +| `explain()` prints `Processor: HashJoinProcessor`, `Transformer: CrossJoinRowsTransformer` | `Join: HashJoin`, `Join: CrossJoin` | + +### 95) `flow-php/etl` - the physical plan holds plan data, the runtime holds the run + +| Before | After | +|---------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Planner::plan(LogicalPlan $logical, FlowContext $context, ?SourceRows $sources)` | `plan(LogicalPlan $logical)` (row 103); the counters go to `Executor::execute()` | +| `Executor::execute(PhysicalPlan $plan)`, `fetch(PhysicalPlan $plan)` | `execute(LogicalPlan $logical, FlowContext $context, ?SourceCounts $sources = null)`, `fetch(LogicalPlan $logical, FlowContext $context)`: the executor plans, and `execute()` returns the `PhysicalPlan` (`getReturn()`) | +| `Executor::executePlan(PhysicalPlan $plan)`, `executePipeline(Pipeline $pipeline)` | `executePlan(PhysicalPlan $plan, FlowContext $context, ?SourceCounts $sources = null)`, `executePipeline(Pipeline $pipeline, FlowContext $context, ?Extractor $source = null)` | +| `Executor::merge(Generator $batches, PhysicalPlan $plan)` | `merge(Generator $batches, Backend $backend)`: an empty result takes its schema from the `PhysicalPlan` `$batches` returns | +| `Flow\ETL\Planner\SinkFeedFactory` (built the sink pipelines at plan time) | `Flow\ETL\Runtime\Executor\WriteLoaderFactory`: `feed(BranchPipeline, ErrorHandler, FlowContext): BranchLoader`, `transactional(TransactionGroup, FlowContext): TransactionLoader`; the plan holds `BranchPipeline`/`TransactionGroup` and the runtime builds the loaders each run | +| `new SinkRun(Pipeline $side, Executor $executor)` | `new BranchFiber(Pipeline $side, Executor $executor, FlowContext $context, FeedExtractor $feed)` | +| `explain()` prints `Loader: SinkFeed`, `Loader: TransactionalSinks` for a sink with steps of its own, a transaction | `Branch: BranchPipeline`, `Transaction: TransactionGroup` | + +### 96) `flow-php/etl` - `explain()` formats a join and a sort once for both plans + +| Before | After | +|-------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| `Flow\ETL\Plan\Explain\Details` | `Flow\ETL\Planning\Explain\NodeDetails` | +| `Flow\ETL\Plan\Explain\Outline` | `Flow\ETL\Planning\Explain\LogicalOutline` | +| `Flow\ETL\Plan\Explain\Branches`, `Branches::of($entry, $indent)` | removed: `TreeLayout::branches($entry, $indent)` | +| `Details::join(Node\Join $join)`, `Details::crossJoin(Node\CrossJoin $join)`, `Details::references()` | `NodeDetails::join(JoinType $type, Expression $on)`, `NodeDetails::crossJoin(string $prefix)`; `references()` removed | +| `StepDetails::sort()`, `StepDetails::columns(References $refs)` | `NodeDetails::sort(list $columns)`, `NodeDetails::columns($refs->all())` | +| `explain()` logical join: `Left on: id, tag` / `Right on: user_id, tag` | `On: id = user_id` / `On: tag = tag`, one line per condition, as the physical plan prints it | +| `explain()` physical join: `Join: left`, `Join: cross` | `Type: left`, `Type: cross`, as the logical plan prints it | +| `explain()` logical sort: `Sort by: id, name` (names only) | `Sort by: id desc, name asc nulls last` (direction, and a nulls order that is not the default) | +| `explain()` physical sort and top N: `Sort: id asc` | `Sort by: id asc` | + +### 97) `flow-php/etl` - planning types moved to `Flow\ETL\Planning` + +| Before | After | +|--------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------| +| `Flow\ETL\Plan` | `Flow\ETL\Planning\Logical\Plan` | +| `Flow\ETL\Optimizer` | `Flow\ETL\Planning\Optimizer\Optimizer` | +| `Flow\ETL\Planner` | `Flow\ETL\Planning\Physical\Planner` | +| `Flow\ETL\Config\Grouping\GroupByAlgorithmBuilder` | `Flow\ETL\Planning\Logical\Node\BucketingAlgorithmBuilder` (see 121) | +| `Flow\ETL\Config\Join\JoinAlgorithmBuilder` | `Flow\ETL\Planning\Logical\Node\JoinAlgorithmBuilder` | +| `Flow\ETL\Config\Repartition\RepartitionAlgorithmBuilder` | `Flow\ETL\Planning\Logical\Node\BucketingAlgorithmBuilder` (see 121) | +| `Flow\ETL\Config\Sort\SortAlgorithmBuilder` | `Flow\ETL\Planning\Logical\Node\SortAlgorithmBuilder` | +| `Flow\ETL\Executor\{PhysicalPlan, Pipeline}` | `Flow\ETL\Planning\Physical\{PhysicalPlan, Pipeline}` | +| `Flow\ETL\Optimizer\{JoinSides, Rule}` | `Flow\ETL\Planning\Optimizer\{JoinSides, Rule}` | +| `Flow\ETL\Optimizer\Rule\{CombineLimits, CombineSortAndLimit, CountFromStatistics, PushFilterIntoSource, PushLimitIntoSource}` | `Flow\ETL\Planning\Optimizer\Rule\{CombineLimits, CombineSortAndLimit, CountFromStatistics, PushFilterIntoSource, PushLimitIntoSource}` | +| `Flow\ETL\Optimizer\Rule\PushProjectionIntoExpand` | `Flow\ETL\Planning\Optimizer\Rule\PushProjectionIntoExpand` | +| `Flow\ETL\Plan\{Explain, Format, LogicalPlan, Node, Redefined}` | `Flow\ETL\Planning\Logical\{Explain, Format, LogicalPlan, Node, Redefined}` | +| `Flow\ETL\Plan\{Rewrite, RowCount, Sinks, Stage, TransformUp}` | `Flow\ETL\Planning\Logical\{Rewrite, RowCount, Stage, TransformUp, Writes}` | +| `Flow\ETL\Plan\{Transparency, Trigger}` | `Flow\ETL\Planning\Logical\{Transparency, Trigger}` | +| `Flow\ETL\Plan\Explain\{BoxLayout, Condition, Entry, FlowLayout, Layout}` | `Flow\ETL\Planning\Explain\{BoxLayout, Condition, Entry, FlowLayout, Layout}` | +| `Flow\ETL\Plan\Explain\{PhysicalOutline, StatisticsLine, StepDetails, TreeLayout}` | `Flow\ETL\Planning\Explain\{PhysicalOutline, StatisticsLine, StepDetails, TreeLayout}` | +| `Flow\ETL\Plan\Node\{Aggregate, Batch, BatchBy, Cache, Collect}` | `Flow\ETL\Planning\Logical\Node\{Aggregate, Batch, BatchBy, Cache, Collect}` | +| `Flow\ETL\Plan\Node\{CollectRefs, Constrain, Count, CrossJoin, Discard}` | `Flow\ETL\Planning\Logical\Node\{CollectRefs, Constrain, Count, CrossJoin, Discard}` | +| `Flow\ETL\Plan\Node\{Distinct, Drop, DuplicateRow, ExpandColumn, Filter}` | `Flow\ETL\Planning\Logical\Node\{Distinct, Drop, DuplicateRow, ExpandColumn, Filter}` | +| `Flow\ETL\Plan\Node\{Join, JoinEach, JoinsFrame, Limit, Offset}` | `Flow\ETL\Planning\Logical\Node\{Join, JoinEach, JoinsFrame, Limit, Offset}` | +| `Flow\ETL\Plan\Node\{Outputs, Read, Rename, RenameEach, Repartition}` | `Flow\ETL\Planning\Logical\Node\{Outputs, Read, Rename, RenameEach, Repartition}` | +| `Flow\ETL\Plan\Node\{Result, Select, Sort, TopN, Transaction}` | `Flow\ETL\Planning\Logical\Node\{Result, Select, Sort, TopN, Transaction}` | +| `Flow\ETL\Plan\Node\{Transform, Until, Validate, WindowColumn, WithColumn}` | `Flow\ETL\Planning\Logical\Node\{Transform, Until, Validate, WindowColumn, WithColumn}` | +| `Flow\ETL\Plan\Node\Write` | `Flow\ETL\Planning\Logical\Node\Write` | +| `Flow\ETL\Planner\{NodeTranslator, PlannedNode, PlannedNodes, SinkAttachment}` | `Flow\ETL\Planning\Physical\{NodeTranslator, PlannedNode, PlannedNodes, WritePlanner}` | + +### 98) `flow-php/etl` - `FlowContext` and `Config` hold no calculator + +| Before | After | +|--------------------------------------------------------|-------------------------------------------| +| `$context->calculator()`, `$config->calculator()` | `new Calculator()` | +| `new Config(..., $backend, $calculator, $random, ...)` | `new Config(..., $backend, $random, ...)` | + +### 99) `flow-php/etl` - runtime types renamed + +| Before | After | +|--------------------------------|--------------------------------------------| +| `Flow\ETL\Executor\SinkRun` | `Flow\ETL\Runtime\Executor\BranchFiber` | +| `Flow\ETL\Executor\SinkOffers` | `Flow\ETL\Runtime\Executor\OfferedFailure` | + +### 100) `flow-php/etl` - `FlowContext::setErrorHandler()` removed, `onError()` changes only its own frame + +| Before | After | +|----------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------| +| `$context->setErrorHandler($handler)` | `$context = $context->withErrorHandler($handler)` | +| `onError()` on a frame built with `new DataFrame(..., $context)` changed `$context` and every frame sharing it | `onError()` changes only that frame, `$context` keeps its handler | +| `$df->groupBy(...)`, then `$df->onError($h)`, then `->pivot(...)`: discovery ran under `$h` | discovery runs under the handler the frame had at `groupBy()` | + +### 101) `flow-php/etl` - `InferredSchemaException::columnsDiverge()` takes the sample bounds + +| Before | After | +|---------------------------------------------------------------------------|-------------------------------------------------------------------------------------------| +| `columnsDiverge($source, $inferredFrom, $inferred, $columns, $inference)` | `columnsDiverge($source, $inferredFrom, $inferred, $columns, $sampleSize, $filesToSniff)` | + +### 102) `flow-php/etl` - sink failures follow the main pipeline's error-handler rule + +The error handler gets a loader or transformer failure once, against that step. Any other failure propagates. This +holds in a `Transactional` child too. + +| Before | After | +|-----------------------------------------------------------------------------------------------------|---------------------------------------------------| +| a sink's consumer `closure()` failure was offered to `onLoading()` against an internal `SinkFeed` | not offered: it propagates, as in a main pipeline | +| a processor failure in a sink was offered the same way | not offered: it propagates | +| a failing `Transaction::begin()` or `commit()` was offered against an internal `TransactionalSinks` | not offered: it propagates | +| a loader or transformer failure in a `Transactional` child named an internal `SinkFeed` | offered once, naming that loader or transformer | + +### 103) `flow-php/etl` - planning takes neither `FlowContext` nor `Config`: rules get the `Backend`, the planner holds its settings + +| Before | After | +|--------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------| +| `Rule::apply(LogicalPlan $plan, FlowContext $context)` | `apply(LogicalPlan $plan, Backend $backend)` | +| `Optimizer::optimize(LogicalPlan $plan, FlowContext $context)` | `optimize(LogicalPlan $plan, Backend $backend)` | +| `new JoinSides(Optimizer $optimizer, FlowContext $context)` | `new JoinSides(Optimizer $optimizer, Backend $backend)` | +| `new Planner(Optimizer $optimizer = new Optimizer())` | `new Planner(Optimizer $optimizer, NodeTranslator $translator, Backend $backend)`; `Config::planner()` returns the one the config builds | +| `Planner::plan()` (row 95) | `plan(LogicalPlan $logical)`; `Planner::optimize(LogicalPlan $logical)` runs the planner's optimizer alone | +| `NodeTranslator::steps()` (row 35), static | `new NodeTranslator($backend, $randomValueGenerator, $id, $cache, $cacheDir, $sort, $grouping, $join, $repartition)`, `steps(Node $node)` | +| `Plan::of(LogicalPlan $logical, FlowContext $context)`, `Plan::$context` | `Plan::of(LogicalPlan $logical, Planner $planner)`, `Plan::$planner` | + +A custom optimizer rule: + +```php +// before +public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan + +// after +public function apply(LogicalPlan $plan, Backend $backend): LogicalPlan +``` + +### 104) `flow-php/etl` - `TelemetryContext` takes the data frame id and name, one step trio for every step kind + +| Before | After | +|------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| +| `new TelemetryContext($backend, $logger, $tracer, $meter, $options)` | `new TelemetryContext(string $dataFrameId, string $dataFrameName, $backend, $logger, $tracer, $meter, $options)` | +| `dataFrameStarted(FlowContext $context)` | `dataFrameStarted(array $configuration)`, the map `Config::telemetryAttributes()` returns | +| `dataFrameCompleted(FlowContext $context, array $attributes = [])` | `dataFrameCompleted(array $attributes = [])` | +| `dataFrameFailed(FlowContext $context, Throwable $exception, array $attributes = [])` | `dataFrameFailed(Throwable $exception, array $attributes = [])` | +| `dataFrameBatchProcessed(Rows $rows, FlowContext $context)` | `dataFrameBatchProcessed(Rows $rows)` | +| `loadingStarted(Loader $loader, array $attributes = [])` | with `$options->traceLoading`: `$step = stepStarted(StepKind::loading, $loader::class, $attributes)` | +| `transformationStarted(Transformer $transformer, array $attributes = [])` | with `$options->traceTransformations`: `$step = stepStarted(StepKind::transformation, $transformer::class, $attributes)` | +| `loadingCompleted($loader, $attributes)`, `transformationCompleted($transformer, $attributes)` | `stepCompleted($step, $attributes)` | +| `loadingFailed($loader, $exception)`, `transformationFailed($transformer, $exception)` | `stepFailed($step, StepKind::loading, $loader::class, $exception)`, the same with `StepKind::transformation` | +| `SpanStack::push(Span $span, Scope $scope)`, `SpanStack::pop()` | `push(SpanScope $spanScope)`, `remove(SpanScope $spanScope): bool` | +| `new SpanScope(Span $span, Scope $scope)` | `new SpanScope(Span $span, ?Scope $scope)` | +| `trace_transformations`: one span per transformer | also one span per processor and join (`flow.etl.operator.class`). A failing one logs `Operator failed` | + +### 105) `flow-php/etl` - runtime types moved to `Flow\ETL\Runtime`, telemetry types to `Flow\ETL\Runtime\Telemetry` + +| Before | After | +|--------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------| +| `Flow\ETL\{Analyze, Config, ErrorHandler, FlowContext}` | `Flow\ETL\Runtime\{Analyze, Config, ErrorHandler, FlowContext}` | +| `Flow\ETL\Config\Cache\CacheConfig` | `Flow\ETL\Runtime\CacheConfig` | +| `Flow\ETL\Config\MemoryLimit` | `Flow\ETL\Runtime\MemoryLimit` | +| `Flow\ETL\Config\Telemetry\{SpanScope, SpanStack, TelemetryConfig, TelemetryContext, TelemetryOptions}` | `Flow\ETL\Runtime\Telemetry\{SpanScope, SpanStack, TelemetryConfig, TelemetryContext, TelemetryOptions}` | +| `Flow\ETL\Dataset\{Report, SourceStatistics, Statistics}` | `Flow\ETL\Runtime\Metrics\{Report, SourceStatistics, Statistics}` | +| `Flow\ETL\Dataset\Statistics\{Column, Columns, ExecutionTime}` | `Flow\ETL\Runtime\Metrics\Statistics\{Column, Columns, ExecutionTime}` | +| `Flow\ETL\Dataset\Statistics\HighResolutionTime` | `Flow\ETL\Foundation\Clock\HighResolutionTime` | +| `Flow\ETL\ErrorHandler\{ExtractionAction, ExtractionError, IgnoreError, LoadingAction, LoadingError}` | `Flow\ETL\Runtime\ErrorHandler\{ExtractionAction, ExtractionError, IgnoreError, LoadingAction, LoadingError}` | +| `Flow\ETL\ErrorHandler\{SkipRows, ThrowError, TransformationAction, TransformationError}` | `Flow\ETL\Runtime\ErrorHandler\{SkipRows, ThrowError, TransformationAction, TransformationError}` | +| `Flow\ETL\Exception\SinkFailure` | removed: a failure out of a branch or a transaction surfaces as the user's own exception | +| `Flow\ETL\Executor` | `Flow\ETL\Runtime\Executor\Executor` | +| `Flow\ETL\Executor\{AdoptedBatches, SinkFeed, StatisticsCollector, TransactionalSinks}` | `Flow\ETL\Runtime\Executor\{AdoptedBatches, BranchLoader, StatisticsCollector, TransactionLoader}` | +| `Flow\ETL\Executor\TransactionRollback` | removed: `TransactionLoader` rolls back itself and logs a failing rollback | +| `Flow\ETL\Executor\SourceRows` | `Flow\ETL\Runtime\Executor\SourceCounts` | +| `new Flow\ETL\Exception\TransactionRolledBack(Loader $loader, Throwable $cause)` | removed: the rolled-back error is offered once to the plan's error handler | + +### 106) `flow-php/etl` - the serialize transformers lose their mode flags + +| Before | After | +|----------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------| +| `new SerializeTransformer($target, $serializer, standalone: true)` | `->with(new SerializeTransformer($target, $serializer))->select($target)` | +| `new UnserializeTransformer($source, $target, $serializer, merge: false)` | `->with(new UnserializeTransformer($source, $target, $serializer))->select(...$target->names())` | +| `new UnserializeTransformer($source, $target, $serializer, true, $prefix)` | `new UnserializeTransformer($source, $target, $serializer, $prefix)` | + +### 107) `flow-php/etl` - `from_data_frame()` is removed + +| Before | After | +|---------------------------------------------------------------|------------------------------------------------------------------------------------------------------------| +| `df()->read(from_data_frame($orders))->limit(5)` | `$orders->limit(5)` | +| `from_data_frame($orders)` to read one frame in two pipelines | build the frame in a function: `$orders = fn (): DataFrame => df()->read(...)`, then `$orders()->limit(5)` | +| `Flow\ETL\Extractor\DataFrameExtractor` | removed | + +### 108) `flow-php/symfony-http-foundation-bridge` - `http_stream_open()` takes a DataFrame + +| Before | After | +|--------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------| +| `http_stream_open(from_data_frame($df))` | `http_stream_open($df)`: the frame runs under its own config, and the response adds its steps to `$df` | +| `http_stream_open(from_data_frame($df))->config($config)` | throws `InvalidArgumentException`: set the config on the frame, `df($config)->read(...)` | +| `new DataStream(extractor: ...)`, `new FlowBufferedResponse(extractor: ...)`, `new FlowStreamedResponse(extractor: ...)` | `source: ...` | + +### 109) `flow-php/etl` - one name per `DataFrame` and `Flow` verb + +| Before | After | +|-----------------------------------------------------------|-------------------------------------| +| `$df->withEntry($name, $function)` | `$df->withColumn($name, $function)` | +| `$df->withEntries([...])` | `$df->withColumns([...])` | +| `$df->rows($transformer)`, `$df->transform($transformer)` | `$df->with($transformer)` | +| `$df->load($sink)` | `$df->write($sink)` | +| `df()->from($extractor)`, `df()->extract($extractor)` | `df()->read($extractor)` | + +### 110) `flow-php/etl` - `Config::builder()` and `Config::default()` are removed + +| Before | After | +|---------------------|----------------------------------------------| +| `Config::builder()` | `config_builder()`, or `new ConfigBuilder()` | +| `Config::default()` | `config()`, or `config_builder()->build()` | + +### 111) `flow-php/etl` - `Processor` extends `Describable` + +| Before | After | +|--------------------------|-----------------------------------------------------------------------------------------------------| +| `Processor` implementors | also `describe(): ?Description` and `static fromDescription(array $data, Rebuild $rebuild): static` | + +If the processor holds live state, return `null` from `describe()`. + +### 112) `flow-php/etl` - every namespace is a layer root + +`Flow\ETL\Flow` and the DSL function namespaces keep their names. + +| Before | After | +|------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------| +| `Flow\ETL\{DataFrame, Sink, Transformation, Transformations}` | `Flow\ETL\API\{Branch, DataFrame, Transformation, Transformations}` | +| `Flow\ETL\DataFrame\GroupedDataFrame` | `Flow\ETL\API\GroupedDataFrame` | +| `Flow\ETL\Config\ConfigBuilder` | `Flow\ETL\API\Config\ConfigBuilder` | +| `Flow\ETL\Config\{Bucketing\BucketingConfigBuilder, Cache\CacheConfigBuilder, Join\HashJoinBuilder}` | `Flow\ETL\API\Config\{Bucketing\BucketingConfigBuilder, Cache\CacheConfigBuilder, Join\HashJoinBuilder}` | +| `Flow\ETL\Config\Sort\{ExternalSortBuilder, MemorySortBuilder}` | `Flow\ETL\API\Config\Sort\{ExternalSortBuilder, MemorySortBuilder}` | +| `Flow\ETL\Sink\{Branched, Transactional, Transformed}` | `Flow\ETL\API\{FilteredBranch, Transactional, TransformedBranch}` | +| `Flow\ETL\Transformation\{AddRowIndex, BatchBy, BatchSize, Drop, Limit, MaskColumns, Select}` | `Flow\ETL\API\Transformation\{AddRowIndex, BatchBy, BatchSize, Drop, Limit, MaskColumns, Select}` | +| `Flow\ETL\Exception\*` | `Flow\ETL\Foundation\Exception\*` | +| `Flow\ETL\Exception\LimitReachedException` | `Flow\ETL\Operators\LimitReachedException` | +| `Flow\Floe\{FloeExtractor, FloeSourceFile, FloeFileBatches}` | `Flow\ETL\IO\Floe\{FloeExtractor, FloeSourceFile, FloeFileBatches}` | +| `Flow\Floe\{FloeLoader, FloeFileSink, FloeFileSinks}` | `Flow\ETL\IO\Floe\{FloeLoader, FloeFileWriter, FloeFileWriterFactory}` | +| `Flow\Calculator\Exception\Exception extends \Exception` | extends `Flow\ETL\Foundation\Exception\Exception`: a `catch (Flow\ETL\Foundation\Exception\Exception)` now also catches calculator errors | + +### 113) `flow-php/etl` - exception factories take names and type strings + +| Before | After | +|---------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| `ColumnMismatchException::missingColumn($definition)` | `ColumnMismatchException::missingColumn($definition->name())` | +| `ColumnMismatchException::retype($definition, $reason)` | `ColumnMismatchException::retype($definition->name(), $definition->type()->toString(), $reason)` | +| `ColumnMismatchException::unsupportedType($definition, $reason)` | `ColumnMismatchException::unsupportedType($definition->name(), $definition->type()->toString(), $reason)` | +| `ColumnMismatchException::untypedColumn($definition)` | `ColumnMismatchException::untypedColumn($definition->name())` | +| `ColumnMismatchException::valueDoesNotMatch($definition, $value, $reason)` | `ColumnMismatchException::valueDoesNotMatch($definition->name(), $definition->type()->toString(), $value, $reason)` | +| `InferredSchemaException::columnsDiverge($source, $inferredFrom, $inferred, $columns, ...)` | `InferredSchemaException::columnsDiverge($source, $inferredFrom, $inferred->names(), $columns, ...)` | +| `InferredSchemaException::filesDiverge($source, $readFrom, $validation)` | `InferredSchemaException::filesDiverge($source, $readFrom, $validation->toString())` | +| `new SchemaValidationException($expected, $given, $validation)` | `new SchemaValidationException($validation->toString())` | +| `SchemaValidationException::context()`, `given()`, `schema()` | removed: the message carries the mismatches | + +### 114) `flow-php/etl` - the user's side path is a `Branch`, a transaction group is not a branch + +"Sink" is gone: `Sink` is `Branch`, the plan's write roots are `Writes`. `Transactional` no longer implements `Branch` +and has no `write()`: pass it to `DataFrame::write()`, `to_branch()` or `to_transformation()`. A `Transactional` +directly inside a `Transactional` is a `TypeError`. The named argument of `DataFrame::write()`, `to_branch()`, +`to_transformation()` is `$write`, of `to_dbal_transaction()` and `to_pgsql_transaction()` it is `$writes`. + +| Before | After | +|----------------------------------------------------------------------------------------------|---------------------------------------------------------------------------| +| `DataFrame::write(Loader\|Sink $sink)` | `DataFrame::write(Loader\|Branch\|Transactional $write)` | +| `Transactional::sinks()` | `Transactional::writes()` | +| `Transactional::write(DataFrame $prefix)` | removed: `$frame->write($transactional)` | +| `LogicalPlan::sinks()`, `LogicalPlan::sinksOnSpine()`, `Node\Outputs::sinks()` | `writes()`, `writesOnSpine()`, `writes()` | +| `SinkBranch::consumers()` | `BranchPipeline::writes()` | +| `InvalidLogicException::sinkNotOnSpine()`, `sinkRootRewritten()`, `errorHandlerInsideSink()` | `writeNotOnSpine()`, `writeRootRewritten()`, `errorHandlerInsideBranch()` | + +### 115) `flow-php/etl` - a file loader writes through a `FileWriter` + +One format written to one open stream is a `FileWriter`, made by a `FileWriterFactory`. The factory takes the schema of +the stream's first batch, so a writer opens its file at once. `ParquetFileWriter` conforms and writes on the PHP lane and +`RustParquetFileWriter` does the same on the native lane. Both implement `FileWriter`, so `ParquetOpenSink` and +`PhpParquetOpenSink` are gone. A batch without rows opens nothing, for every file loader (`to_floe()` wrote an empty file +before). + +| Before | After | +|------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| +| `FileSink`, `FileSinks::open(DestinationStream $stream, Backend $backend)` | `FileWriter`, `FileWriterFactory::writer(DestinationStream $stream, Schema $schema, Backend $backend)` | +| `CSVOpenSink`, `JsonOpenSink`, `TextOpenSink`, `XMLOpenSink` | `CSVFileWriter`, `JsonFileWriter`, `TextFileWriter`, `XMLFileWriter` | +| `FloeFileSink`, `FloeFileSinks` | `FloeFileWriter`, `FloeFileWriterFactory` | +| `ParquetFileSink`, `PhpParquetOpenSink`, `ParquetOpenSink` | `ParquetFileWriter`; `ParquetOpenSink` removed: both lanes implement `FileWriter` | +| `ParquetFileSinks::conform()`, `ParquetFileSinks::writer(DestinationStream)` | removed: `ParquetFileWriterFactory::writer(DestinationStream, Schema, Backend)` returns the lane | +| `new RustParquetOpenSink(RustParquetFileWriter $writer)` | `new RustParquetFileWriter(ArrowRustParquetFileWriter $writer, ParquetSchemaConformance $conformance, Backend $backend)` | + +### 116) `flow-php/filesystem` - `MemoryFilesystem::mv()` moves files and directories + +| Before | After | +|-------------------------------------------------------------------------|--------------------------------------------------------------------------------------------| +| `mv()` throws `RuntimeException` `Cannot move files around in memory` | a file: its content moves to the target, which it replaces; the source is removed | +| - | a directory: every file under it moves to the same relative path under the target | +| - | a missing source returns `false` | +| `to_csv('memory://...')->saveMode(overwrite())` throws at the end | the write replaces the file | + +### 117) `flow-php/etl`, `flow-php/etl-adapter-*` - one file commit protocol; readers skip hidden paths + +Every file write runs through one `FileCommit`. Files that replace others (Overwrite) and every parallel task's files +are written under a temporary output inside the destination, then moved into place; the other modes write in place. +The save mode decides once for the whole destination. Readers skip the temporary output of a running or crashed write +by Spark's hidden-path rule. + +| Before | After | +|-------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `FilesSink` | `FileCommit`: a task stages its files, the job commits them; a serial write is one task | +| Overwrite wrote `._flow_php_tmp.` beside each file | every Overwrite and every parallel task writes under `/.flow-tmp--/` first; an aborted parallel write leaves the destination directory behind, empty | +| Append beside an existing plain file: `_.` | `-.` | +| Ignore / ExceptionIfExists on a partitioned plain destination: decided per partition file | decided once over `/**/.`: an existing file in any partition skips (Ignore) or refuses (ExceptionIfExists) the whole write, as Spark and DuckDB do | +| Overwrite removed `*.` in each written partition (`data_backup.csv` too) | removes only `.` and `-.`, partitioned or not; files an older Append named `_.` stay | +| Overwrite removed the stale files, then moved the new ones | moves first, then removes: a failed commit leaves duplicates, never loses the old files | +| a write without rows touched nothing | Overwrite empties an unpartitioned destination (a partitioned one keeps every partition); ExceptionIfExists over an existing destination throws; Ignore and Append write nothing | +| `to_excel()->saveMode(ignore())` over an existing workbook overwrote it | keeps it | +| file extractors and `files()` list every file the path matches | skip a file under a `.` or `_` path segment below the path's static part, except segments with `=`, `_metadata` and `_common_metadata` (Spark's rule); a path you name yourself is read | + +### 118) `flow-php/etl` - a limit ends the stream without an exception + +| Before | After | +|---------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| +| `Flow\ETL\Operators\LimitReachedException` thrown by `transform()` to end the stream | removed: a `Limiting` transformer returns the trimmed batch and `reached()` returns `true` | +| - | `Flow\ETL\Operators\Transformer\Limiting extends Transformer { reached(): bool }`, implemented by `LimitTransformer` and `UntilTransformer` | +| `Limit reached` log attribute `limit` (`0` for `until()`) | attribute `step`: the class of the transformer that reached it | + +### 119) `flow-php/etl` - `count(ref)` skips nulls everywhere, window frames reuse the aggregate functions + +| Before | After | +|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------| +| group-by and pivot `count(ref('x'))` over `1, null, 3`: `3` | `2`, the same as the window `count(ref('x'))` and SQL. `count()` without a reference still counts rows | +| `FrameAccumulating::accumulator(): FrameAccumulator` | `FrameAccumulating::accumulator(): AggregatingFunction`, a fresh instance with no window | +| `WindowFunction::values(Rows $partition)` | `WindowFunction::values(Rows $partition, Backend $backend)` | +| `FrameAccumulation::values($function, $partition)` | `FrameAccumulation::values($function, $partition, $backend)` | +| `Window\FrameAccumulator`, `Window\Accumulator\{Count,Sum,Average}Accumulator` | removed: a frame feeds the function's own `aggregate()` | + +### 121) `flow-php/etl`, `flow-php/etl-adapter-*` - one mechanism per operation + +| Before | After | +|--------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `FileBatches { batches(SourceFile $source, Schema $body, int $batchSize, Backend $backend, ReadWindow $window) }` | `FileBatches extends Describable { batches(FileRead $read, Criteria $criteria, SourceFile $source, Schema $body, Backend $backend, ReadWindow $window); close() }` - each format's batches holds its settings and describes itself; the batch size is `$read->batchSize`, the split `$criteria->split` | +| `ConfigBuilder::groupBy(GroupByAlgorithmBuilder $algorithm)`, `ConfigBuilder::repartition(RepartitionAlgorithmBuilder $algorithm)` | `ConfigBuilder::groupBy(BucketingAlgorithmBuilder $algorithm)`, `ConfigBuilder::repartition(BucketingAlgorithmBuilder $algorithm)` | +| `DataFrame::aggregate(array $aggregations, ?GroupByAlgorithmBuilder $algorithm)`, `DataFrame::groupBy(..., ?GroupByAlgorithmBuilder $algorithm)` | `?BucketingAlgorithmBuilder $algorithm`; `BucketingConfigBuilder` implements it | +| `to_branch($condition, $write)->withTransformation($transformation)` | `to_branch($condition, to_transformation($transformation, $write))` | +| `abstract class StopWhen { shouldStop(); and(); or() }` (`flow-php/etl-adapter-http`) | `interface StopWhen { shouldStop(DecodedResponse $response, PageState $state): bool }` - a custom condition `implements StopWhen` | +| `http_stop_when_flag_false('has_more')->or(http_stop_when_max_pages(1000))` | `http_stop_when_any_of(http_stop_when_flag_false('has_more'), http_stop_when_max_pages(1000))` | +| `$a->and($b)` on a stop condition | `http_stop_when_all_of($a, $b)` | + ## Upgrading from 0.45.x to 0.46.x ### 1) `flow-php/etl-adapter-postgresql`, `flow-php/etl-adapter-doctrine` - keyset reads reject NULL keys up front @@ -206,7 +1867,8 @@ The `arrow` extension must be rebuilt. | a NULL key: rows lost or `NULL value found in column ...`, depending on the batch size | one check query before the first page, then `RuntimeException` `Keyset pagination requires non-null keys ...` | | no query before the first page | `SELECT 1 FROM () ... WHERE k IS NULL LIMIT 1` (DBAL: the builder plus `IS NULL` conditions) | -Filter NULL keys out with `IS NOT NULL`, or page by non-null columns. Index the keys - on an unindexed key the check is one full scan. +Filter NULL keys out with `IS NOT NULL`, or page by non-null columns. Index the keys - on an unindexed key the check is +one full scan. ### 2) `flow-php/etl-adapter-postgresql`, `flow-php/etl-adapter-doctrine` - keyset reads reject duplicate keys @@ -268,7 +1930,8 @@ Page by non-null key columns. | `created_at > $1 OR (created_at = $1 AND id > $2)` | `(created_at, id) > ($1, $2)` | | all DESC: `created_at < $1 OR (created_at = $1 AND id < $2)` | `(created_at, id) < ($1, $2)` | -Nothing to change; the row comparison is an index range bound. Mixed ASC/DESC keys keep the `OR` form. Update assertions on the generated SQL. +Nothing to change; the row comparison is an index range bound. Mixed ASC/DESC keys keep the `OR` form. Update assertions +on the generated SQL. ### 9) `flow-php/etl-adapter-doctrine` - a non-scalar keyset value is rejected @@ -303,7 +1966,9 @@ day, and `overwrite()` replaces only the new one - rewrite the dataset, or parti $df->withEntry('date', ref('date')->cast(type_date())) ``` -### 12) `flow-php/etl-adapter-parquet`, `flow-php/etl` - a partition column the file holds is typed from the file in a pushed filter +### 12) `flow-php/etl-adapter-parquet`, + +`flow-php/etl` - a partition column the file holds is typed from the file in a pushed filter | Before | After | |------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------| @@ -348,7 +2013,8 @@ $df->withEntry('item', array_expand(ref('items'))); ### 17) `flow-php/etl` - a serialized `Rows` holds a list of frames -A `Rows` serialized by 0.45 (e.g. a `FilesystemCache` entry) is not read by 0.46: clear such caches after upgrading. A batch +A `Rows` serialized by 0.45 (e.g. a `FilesystemCache` entry) is not read by 0.46: clear such caches after upgrading. A +batch over 2 GiB of strings in one column is now split across frames instead of throwing `OffsetOverflow`. --- @@ -382,13 +2048,17 @@ over 2 GiB of strings in one column is now split across frames instead of throwi | `declare_cursor('c', 'INSERT INTO t VALUES (1) RETURNING id')->toSql()` - segfault | `InvalidArgumentException` | | `declare_cursor('c', 'SELECT 1; SELECT 2')` - the second statement silently dropped | `InvalidArgumentException` | -### 4) `flow-php/postgresql` - `SelectStatement::hasIntoClause()` sees `SELECT ... INTO` in a `UNION` / `INTERSECT` / `EXCEPT` +### 4) `flow-php/postgresql` - `SelectStatement::hasIntoClause()` sees `SELECT ... INTO` in a `UNION` / `INTERSECT` / + +`EXCEPT` | `sql_parse($sql)->statements()->first()->hasIntoClause()`, `$sql` | Before | After | |-------------------------------------------------------------------|---------|--------| | `SELECT id INTO t FROM x UNION SELECT 1` | `false` | `true` | -### 5) `flow-php/postgresql` - `sql_to_*_query()` and the pagination modifiers take exactly one read-only `SELECT` or `VALUES` +### 5) `flow-php/postgresql` - `sql_to_*_query()` and the pagination modifiers take exactly one read-only `SELECT` or + +`VALUES` | Before | After | |--------------------------------------------------------------------------------------|-----------------------------| @@ -421,7 +2091,9 @@ Applies to `PaginationModifier`, `CountModifier` and `KeysetPaginationModifier` | `InvalidLogicException::cyclicPlanOnDescribe()` | removed | | telemetry debug-log field `optimizers` | `optimizer_rules` | -### 8) `flow-php/etl` - `extract()` receives the pushed limit and path filter, `LimitPushDown` and `withPathFilter()` removed +### 8) `flow-php/etl` - `extract()` receives the pushed limit and path filter, `LimitPushDown` and + +`withPathFilter()` removed | Before | After | |----------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------| @@ -480,7 +2152,9 @@ Applies to `PaginationModifier`, `CountModifier` and `KeysetPaginationModifier` | `SkipRows`: a drain failure of `to_transformation()` / `to_branch()` at `closure()` was rethrown | offered to `onTransformation()` once - `SkipRows` drops the buffered batch | | `$df->load(loader: $l)`, `$df->write(loader: $l)`, `to_transformation($t, loader: $l)`, `to_branch($c, loader: $l)` | the named argument is `sink:` | -### 13) `flow-php/etl-adapter-doctrine`, `-postgresql` - `to_dbal_transaction()` / `to_pgsql_transaction()` are transaction roots +### 13) `flow-php/etl-adapter-doctrine`, `-postgresql` - `to_dbal_transaction()` / + +`to_pgsql_transaction()` are transaction roots | Before | After | |-----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -574,7 +2248,9 @@ after it shift by one. `Stage::physical` is new - see the core documentation. | `a,b` / `"x\"y",1` / `"p",2` | 1 row: `{"a":"x\\\"y","b":"1\n\"p\""}` | 2 rows: `{"a":"x\\\"y","b":1}`, `{"a":"p","b":2}` | | `a,b` / `x"y,1` / `"p",2` | 1 row: `{"a":"x\"y","b":"1\n\"p\""}` | 2 rows: `{"a":"x\"y","b":1}`, `{"a":"p","b":2}` | -### 23) `flow-php/etl-adapter-csv` - `CSVOpenSource` is an interface, `CSVSourceOpener` removed, `CSVLineReader` takes the separator and escape +### 23) `flow-php/etl-adapter-csv` - `CSVOpenSource` is an interface, `CSVSourceOpener` removed, + +`CSVLineReader` takes the separator and escape | Before | After | |--------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -737,7 +2413,9 @@ The same applies to map values and structure elements. |-----------------------------------------------------------------------|------------------| | `int4[]` → `list`, Floe and Parquet refuse to write it | `list` | -### 39) `flow-php/etl-adapter-json` - `anyOf` / `oneOf` / multi-type `type` inside `items`, `properties` or `additionalProperties` +### 39) `flow-php/etl-adapter-json` - `anyOf` / `oneOf` / multi-type `type` inside `items`, `properties` or + +`additionalProperties` | Before | After | |------------------------------------------------------------------------------------------|---------------------------------------------------| @@ -745,29 +2423,31 @@ The same applies to map values and structure elements. ### 40) `flow-php/types`, `flow-php/etl` - a `datetime` column has one time zone -| Before | After | -|-------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `type_datetime()->cast('2026-01-02 03:04')` in `date.timezone` | in the column zone: UTC, or `type_datetime('Europe/Warsaw')` | -| `cast('2026-01-02T03:04:05+02:00')` keeps `+02:00`, `cast('…Z')` keeps `Z` | the same instant in the column zone (`2026-01-02 01:04:05 UTC`) | -| a `DateTimeInterface` value in `rows()`, `from_rows()`, a cast or a Floe read keeps its own zone | converted to the column zone, instant unchanged | -| `to_csv()` / `to_json()` / `to_xml()` write each value's own offset (`+02:00` in, `+02:00` out) | the column zone's offset (`2026-01-02T01:04:05+00:00`); declare `datetime_schema('at', zone: 'Europe/Warsaw')` on the extractor to write a local one | -| `from_dbal_*()` naive `timestamp` / MySQL `datetime` text, and naive CSV/JSON text, read in `date.timezone` | read in the column zone (UTC); `->withSchema(schema(datetime_schema('at', zone: 'Europe/Warsaw')))` keeps a local reading | -| `type_datetime('CEST')`, `to_timezone($ref, 'PST')`, `'EST'`, `'CET'` | `InvalidArgumentException` - an IANA region name, `UTC` or `+HH:MM` | -| `to_timezone($v, lit('UTC'))` | `TypeError` - pass `'UTC'` or a `DateTimeZone` | -| `to_timezone($ref, ref('tz'))` - a per-row zone | `TypeError` - the zone of `to_timezone()` and `new ToTimeZone()` is `DateTimeZone\|string` | -| `now($tz)`, `to_date_time(…, $tz)`, `to_timezone(…, $tz)` return a `datetime` (UTC) column | a `datetime` column in `$tz`; a `ScalarFunction` zone in `now()` / `to_date_time()` yields UTC; `now(new DateTimeZone('EST'))`, `to_date_time(…, new DateTimeZone('PST'))` throw `InvalidArgumentException` (abbreviation zones) | -| `to_date_time($dateTimeObject)` truncates to midnight | keeps the time, converted to the zone | -| merging `datetime` columns ignores zones | equal zones kept, different zones become `UTC` | -| merging `time` with `date` / `datetime` gives `datetime` | `string` | -| `?T` inside a list, map or structure passes a valid value through `cast()` unchanged | cast like `T`: a `?datetime` moves to its zone, a `?structure` keeps only its declared keys, in declared order | -| `type_date()->cast('2026-01-02T23:00:00-05:00')` midnight `-05:00` | `2026-01-02 00:00:00 UTC`; every non-object date input lands at midnight UTC | -| `type_date()->cast($dateTime)` sets the caller's `DateTime` to midnight | returns a new `DateTimeImmutable` | -| `type_time()->cast(new DateTimeImmutable('10:00+05:00'))` = `05:00:00` | `10:00:00` - the value's wall clock | -| Excel date cells in the host zone | UTC wall clock | -| `new DateTimeDefinition('at', true)` | `new DateTimeDefinition('at', type_datetime(), true)` or `datetime_schema('at', nullable: true, zone: 'Europe/Warsaw')` | -| a `flow_php` extension older than this release | Composer refuses it (`conflict`) - see 67) | - -### 41) `flow-php/parquet` - `Converter::isFor()` replaced by `static Converter::forColumn()`, `Int32DateTimeConverter` removed +| Before | After | +|-------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `type_datetime()->cast('2026-01-02 03:04')` in `date.timezone` | in the column zone: UTC, or `type_datetime('Europe/Warsaw')` | +| `cast('2026-01-02T03:04:05+02:00')` keeps `+02:00`, `cast('...Z')` keeps `Z` | the same instant in the column zone (`2026-01-02 01:04:05 UTC`) | +| a `DateTimeInterface` value in `rows()`, `from_rows()`, a cast or a Floe read keeps its own zone | converted to the column zone, instant unchanged | +| `to_csv()` / `to_json()` / `to_xml()` write each value's own offset (`+02:00` in, `+02:00` out) | the column zone's offset (`2026-01-02T01:04:05+00:00`); declare `datetime_schema('at', zone: 'Europe/Warsaw')` on the extractor to write a local one | +| `from_dbal_*()` naive `timestamp` / MySQL `datetime` text, and naive CSV/JSON text, read in `date.timezone` | read in the column zone (UTC); `->withSchema(schema(datetime_schema('at', zone: 'Europe/Warsaw')))` keeps a local reading | +| `type_datetime('CEST')`, `to_timezone($ref, 'PST')`, `'EST'`, `'CET'` | `InvalidArgumentException` - an IANA region name, `UTC` or `+HH:MM` | +| `to_timezone($v, lit('UTC'))` | `TypeError` - pass `'UTC'` or a `DateTimeZone` | +| `to_timezone($ref, ref('tz'))` - a per-row zone | `TypeError` - the zone of `to_timezone()` and `new ToTimeZone()` is `DateTimeZone\|string` | +| `now($tz)`, `to_date_time(..., $tz)`, `to_timezone(..., $tz)` return a `datetime` (UTC) column | a `datetime` column in `$tz`; a `ScalarFunction` zone in `now()` / `to_date_time()` yields UTC; `now(new DateTimeZone('EST'))`, `to_date_time(..., new DateTimeZone('PST'))` throw `InvalidArgumentException` (abbreviation zones) | +| `to_date_time($dateTimeObject)` truncates to midnight | keeps the time, converted to the zone | +| merging `datetime` columns ignores zones | equal zones kept, different zones become `UTC` | +| merging `time` with `date` / `datetime` gives `datetime` | `string` | +| `?T` inside a list, map or structure passes a valid value through `cast()` unchanged | cast like `T`: a `?datetime` moves to its zone, a `?structure` keeps only its declared keys, in declared order | +| `type_date()->cast('2026-01-02T23:00:00-05:00')` midnight `-05:00` | `2026-01-02 00:00:00 UTC`; every non-object date input lands at midnight UTC | +| `type_date()->cast($dateTime)` sets the caller's `DateTime` to midnight | returns a new `DateTimeImmutable` | +| `type_time()->cast(new DateTimeImmutable('10:00+05:00'))` = `05:00:00` | `10:00:00` - the value's wall clock | +| Excel date cells in the host zone | UTC wall clock | +| `new DateTimeDefinition('at', true)` | `new DateTimeDefinition('at', type_datetime(), true)` or `datetime_schema('at', nullable: true, zone: 'Europe/Warsaw')` | +| a `flow_php` extension older than this release | Composer refuses it (`conflict`) - see 67) | + +### 41) `flow-php/parquet` - `Converter::isFor()` replaced by `static Converter::forColumn()`, + +`Int32DateTimeConverter` removed | Before | After | |--------------------------------------------------------------------|------------------------------------------------------------------------------------| @@ -944,7 +2624,6 @@ different instant; it now stays a `datetime` unless it is cast to a date explici |---------------------------------------------------|--------------------| | `new Comparator(StructureComparison $structures)` | `new Comparator()` | - ### 60) `flow-php/etl` - the row-shaped reader/writer API is removed | Before | After | @@ -1027,6 +2706,7 @@ Project with `from_parquet(..., columns: [...])`. Change a type after reading, e | a codec wraps every frame body | a codec wraps each buffer of a BATCH frame behind an i64 uncompressed length (`-1` = stored raw); the directory stays plaintext | | a null in a NOT NULL column refused by the Floe writer (`SchemaMismatchException`) | refused by `Rows::fromColumns()` / `withColumns()` with the same exception and message, whatever the sink | | an XML/HTML value that cannot be converted to or from its string while building a column throws `FloeException` | throws `Flow\ETL\Exception\InvalidArgumentException`, same message | + ### 67) `flow-php/flow-php-ext`, `flow-php/etl`, `flow-php/etl-adapter-csv` - the extension is the column backend | Before | After | @@ -1034,7 +2714,7 @@ Project with `from_parquet(..., columns: [...])`. Change a type after reading, e | `Flow\Floe\RustFloeEncoderNative`, `Flow\ETL\Row\RustRowHydratorNative` | removed | | - | `flow_php` registers `Flow\ETL\Column\{Backend, Column, ColumnBuilder, RustBackend, RustColumn, RustColumnBuilder}` | | any `flow_php` installs beside `flow-php/etl` | Composer refuses `ext-flow_php` < 0.45 (`conflict`); forced past it: `Class "Flow\ETL\Column\RustBackend" not found` | -| untagged builds report `0.44.1+65.g…` | `0.45.0-dev+65.g…` (next minor, pre-release) | +| untagged builds report `0.44.1+65.g...` | `0.45.0-dev+65.g...` (next minor, pre-release) | | - | `Backend`: `builder()`, `constant()`, `decode()`, `adopt(Definition, Column): Column`, `allocatedBytes(): int` | | `Consumption` / `analyze()` / telemetry memory = `memory_get_usage()` | plus the configured backend's `allocatedBytes()` (native columns are not bound by `memory_limit`) | | `new Consumption(bool $realMemory = true)` | `new Consumption(Backend, bool $realMemory = true)` | @@ -1076,14 +2756,14 @@ Project with `from_parquet(..., columns: [...])`. Change a type after reading, e ### 70) `flow-php/etl` - `ScalarFunction` evaluates a batch -| Before | After | -|-------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------| -| `ScalarFunction::eval(Row, FlowContext): mixed` | `eval(Rows, FlowContext): Column` - one value per row, built by `ResultColumn` | -| `ExpandResults::eval(Row): array`, `UnpackResults::eval(Row): array` | marker interfaces - `eval(Rows): Column` returns a list (expand) or structure (unpack) column | -| `Parameter::{as, asArray, asBoolean, asEnum, asFloat, asInstanceOf, asInt}(Row, …)` | `{asTypes, asArrays, asBooleans, asEnums, asFloats, asInstancesOf, asInts}(Rows, …)` - one list per batch | -| `Parameter::{asListOfObjects, asNumber, asObject, asString}(Row, …)` | `{asListsOfObjects, asNumbers, asObjects, asStrings}(Rows, …)` | -| `Parameter::eval(Row)`, `asValue(Row)` | `values(Rows, FlowContext)`, `column(Rows, FlowContext)` | -| `$function->eval($rows, $context)` on an unresolved tree | `InvalidLogicException` - `(new ReferenceResolver())->resolve($function, $rows->schema())->eval($rows, $context)` | +| Before | After | +|---------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------| +| `ScalarFunction::eval(Row, FlowContext): mixed` | `eval(Rows, FlowContext): Column` - one value per row, built by `ResultColumn` | +| `ExpandResults::eval(Row): array`, `UnpackResults::eval(Row): array` | marker interfaces - `eval(Rows): Column` returns a list (expand) or structure (unpack) column | +| `Parameter::{as, asArray, asBoolean, asEnum, asFloat, asInstanceOf, asInt}(Row, ...)` | `{asTypes, asArrays, asBooleans, asEnums, asFloats, asInstancesOf, asInts}(Rows, ...)` - one list per batch | +| `Parameter::{asListOfObjects, asNumber, asObject, asString}(Row, ...)` | `{asListsOfObjects, asNumbers, asObjects, asStrings}(Rows, ...)` | +| `Parameter::eval(Row)`, `asValue(Row)` | `values(Rows, FlowContext)`, `column(Rows, FlowContext)` | +| `$function->eval($rows, $context)` on an unresolved tree | `InvalidLogicException` - `(new ReferenceResolver())->resolve($function, $rows->schema())->eval($rows, $context)` | Before: @@ -1125,10 +2805,10 @@ public function eval(Rows $rows, FlowContext $context): Column ### 72) `flow-php/etl` - function errors carry the batch row -| Before | After | -|----------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------| -| an exception thrown by a function reaches the caller as-is | `Flow\ETL\Exception\EvaluationException` (extends `InvalidArgumentException`), message `… (row N)`, the original as `getPrevious()` | -| `catch (SomeSpecificException $e)` around `withEntry()` / `filter()` | `catch (EvaluationException $e)` and read `$e->getPrevious()` | +| Before | After | +|----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| an exception thrown by a function reaches the caller as-is | `Flow\ETL\Exception\EvaluationException` (extends `InvalidArgumentException`), message `... (row N)`, the original as `getPrevious()` | +| `catch (SomeSpecificException $e)` around `withEntry()` / `filter()` | `catch (EvaluationException $e)` and read `$e->getPrevious()` | ### 73) `flow-php/etl` - uuid, json, enum, xml, html and date/time values compare by value @@ -1181,7 +2861,7 @@ public function eval(Rows $rows, FlowContext $context): Column | group-by / repartition / join wrote one bucket frame per bucket per input batch | buckets stay in memory under the limit; past it every frame holds `batchSize` rows | | `GroupBySteps::of()` / `RepartitionSteps::of()` - `[BucketingProcessor, GroupByAggregationProcessor]` / `[BucketingProcessor, RepartitionProcessor]` | `[GroupByAggregationProcessor]` / `[RepartitionProcessor]`; both take `HashBucketing` and `Unit $memoryLimit`; `GroupBy\BucketAggregation` removed | | `new ExternalSortConfig($bucketing, ?BucketsStorage $merge = null, int $runSize)` | `new ExternalSortConfig($bucketing, Unit $memoryLimit, ?BucketsStorage $merge = null)` | -| `new HashJoinConfig($bucketing)`, `new HashGroupByConfig($bucketing)`, `new HashRepartitionConfig($bucketing)` | `new …Config($bucketing, Unit $memoryLimit)` | +| `new HashJoinConfig($bucketing)`, `new HashGroupByConfig($bucketing)`, `new HashRepartitionConfig($bucketing)` | `new ...Config($bucketing, Unit $memoryLimit)` | | `new HashJoinProcessor($right, $executor, $on, $type, $left, $right, $random, $bucketsCount, $batchSize)` | `new HashJoinProcessor($right, $executor, $on, $type, $left, $right, $random, Unit $memoryLimit, $bucketsCount, $batchSize)` | | `new MergeSortProcessor(References, Buckets, Buckets, RandomValueGenerator, int, int)` | `new MergeSortProcessor(ExternalSort)` | | `SortSteps::of()` - `[BucketingProcessor, MergeSortProcessor]` for the external sort | `[MergeSortProcessor]`; `SortSteps::external(): ?ExternalSort` | @@ -1198,13 +2878,15 @@ public function eval(Rows $rows, FlowContext $context): Column | Before | After | |---------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| | `sum(ref('a'), exact: ref('is_exact'))` - exactness chosen per row | `sum(ref('a'), exact: true)` - one `bool` for the aggregate, as `average()` and `plus/minus/multiply/power` | -| `sum(…, exact: ScalarFunction)` disabled spill column pruning (`references()` was `null`) | `sum()` always enumerates its reference; a custom `AggregatingFunction` may still return `null` | -| `batched_by(from_…, 'at')` - equal datetimes / uuids split into one batch per row, a `null` value joined the next group | value equality, as `batchBy()`; nulls are a group of their own | +| `sum(..., exact: ScalarFunction)` disabled spill column pruning (`references()` was `null`) | `sum()` always enumerates its reference; a custom `AggregatingFunction` may still return `null` | +| `batched_by(from_..., 'at')` - equal datetimes / uuids split into one batch per row, a `null` value joined the next group | value equality, as `batchBy()`; nulls are a group of their own | | `string_agg(ref('code'), ',', SortOrder::ASC)` over `'9', '10', '100'` - `9,10,100` (numeric-looking strings numerically) | `10,100,9` - strings by bytes, as `sortBy()` | | `discover_pivot_values()` over `'9', '10', '100'` - columns `9, 10, 100` | `10, 100, 9`; integer pivot values stay numeric | | `->stringMatch()` / `->stringMatchAll()` turned a PHP `\Error` into an exception `optional()` swallowed | an `\Error` propagates; `->jsonDecode()`, `cast()`, `->arrayPathExists()`, `->stringMatch*()` errors keep their cause as `getPrevious()` | -### 79) `flow-php/etl-adapter-parquet` - with `flow_php` and arrow-ext loaded and no engine given, Parquet is read and written as native columns +### 79) `flow-php/etl-adapter-parquet` - with + +`flow_php` and arrow-ext loaded and no engine given, Parquet is read and written as native columns | Before | After | |--------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -1219,14 +2901,18 @@ public function eval(Rows $rows, FlowContext $context): Column |---------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------| | `float_schema('amount')` written as Parquet `FLOAT` (32-bit): `0.14` read back as `0.14000000059604645` | written as `DOUBLE` (64-bit), read back exactly; readers of the file see a `DOUBLE` column | -### 81) `flow-php/etl-adapter-parquet` - with `flow_php` and arrow-ext, a string that is not valid UTF-8 is refused on write +### 81) `flow-php/etl-adapter-parquet` - with + +`flow_php` and arrow-ext, a string that is not valid UTF-8 is refused on write | Before | After | |------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | without arrow-ext, `PhpParquetEngine` wrote `"\xff\xfe"` into a `STRING` column other readers reject (DuckDB: `is not valid UTF8`) | `Flow\Parquet\Exception\RuntimeException` `Parquet column "name" row 0 holds a string that is not valid UTF-8; Parquet STRING columns require UTF-8` - the row is 0-based within the batch written | | with arrow-ext: `Column 'name': expected string, got string` | the same refusal; invalid UTF-8 into `STRING` / `JSON` / `ENUM` is refused by both engines; `Option::VALIDATE_DATA = false` on `PhpParquetEngine` writes it unvalidated (a file other readers reject); arrow-ext always refuses | -### 82) `flow-php/parquet` - `DELTA_BINARY_PACKED` follows the spec; affected files written by `PhpParquetEngine` must be rewritten +### 82) `flow-php/parquet` - `DELTA_BINARY_PACKED` follows the spec; affected files written by + +`PhpParquetEngine` must be rewritten | Before | After | |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| @@ -1243,7 +2929,9 @@ public function eval(Rows $rows, FlowContext $context): Column | `new ParquetFile($stream, $byteOrder, $options, $engine)` | `new ParquetFile($stream, $options, $engine->openForRead($stream))` | | a `ParquetFile` closed its stream only when destroyed | `ParquetFile::close()` closes its reader (once); `ParquetFile::reader()` returns it | -### 84) `flow-php/parquet` - the arrow engine (`RustParquetEngine`, was `ArrowParquetEngine`) reads and writes through arrow-ext's new Parquet core +### 84) `flow-php/parquet` - the arrow engine (`RustParquetEngine`, was + +`ArrowParquetEngine`) reads and writes through arrow-ext's new Parquet core | Before | After | |----------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -1255,11 +2943,13 @@ public function eval(Rows $rows, FlowContext $context): Column | `TIME` writes dropped the interval's `d` | kept | | an offset read decoded every row group before the offset; page headers were read to the end of the file | only the row groups a read needs, each column chunk read once | | an `INTERVAL`, a path into a `LIST` / `MAP`, `LZO` or `INT96` under `Option::INT_96_AS_DATETIME = false` failed inside arrow-ext | refused before the first batch naming the column and `\Flow\Parquet\Reader::php()` | -| `Reader::read()` returned without touching the file; an invalid file failed at `metadata()` | the footer is read when the file is opened: `Reader::read()` / `readStream()` throw `InvalidArgumentException` `Given file is not valid Parquet file: …` | +| `Reader::read()` returned without touching the file; an invalid file failed at `metadata()` | the footer is read when the file is opened: `Reader::read()` / `readStream()` throw `InvalidArgumentException` `Given file is not valid Parquet file: ...` | | `Flow\Arrow\Parquet\Reader` / `Writer` / `Exception` | removed: `Flow\Parquet\Engine\{RustParquetFileReader, RustParquetFileWriter}`, `Flow\Arrow\Parquet\{RustColumnsReader, RustBatchReader}`; refusals are `Flow\Parquet\Exception\*` | | `flow-php/parquet` installable with any arrow-ext version | `conflict: ext-arrow <0.45` | -### 85) `flow-php/parquet` - `PhpParquetEngine` reads unsigned integers and INT96 exactly; `FlatColumn::enum()` writes the ENUM logical type; values it wrote wrongly are refused +### 85) `flow-php/parquet` - `PhpParquetEngine` reads unsigned integers and INT96 exactly; + +`FlatColumn::enum()` writes the ENUM logical type; values it wrote wrongly are refused | Before | After | |-----------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -1267,7 +2957,7 @@ public function eval(Rows $rows, FlowContext $context): Column | `INT96` 1960-01-01 23:59:59.999999999 read as 1960-01-02 00:00:00 (rounded) | 1960-01-01 23:59:59.999999 (floored, integer arithmetic) | | `FlatColumn::enum()` wrote logical `STRING` + converted `ENUM`, a file arrow-rs refuses | logical `ENUM` + converted `ENUM` | | `from_parquet()` refused a logical `ENUM` column (`ENUM is not supported`) | read as a `string` column, as `STRING` | -| an int outside an `INT_8/16/32` or `UINT_8/16/32/64` range written wrapped (4000000000 into `INT32` → `-294967296`), negatives into `UINT` accepted | `ValidationException` `Column "" row : … is out of the … range` | +| an int outside an `INT_8/16/32` or `UINT_8/16/32/64` range written wrapped (4000000000 into `INT32` → `-294967296`), negatives into `UINT` accepted | `ValidationException` `Column "" row : ... is out of the ... range` | | a `UUID` / `FIXED_LEN_BYTE_ARRAY(n)` value of another byte length written cut or short | refused naming column and row | | invalid UTF-8 into `STRING` / `JSON` / `ENUM` written (other readers reject the file) | refused naming column and row | | these checks | run under `Option::VALIDATE_DATA` (default `true`), as every other validator check | @@ -1287,7 +2977,9 @@ public function eval(Rows $rows, FlowContext $context): Column | `true` → `1`, `false` → an empty field, the same bytes as `null` | `true` / `false`; `null` stays an empty field | | a NOT NULL boolean column could not be read back by `from_csv()` | reads back equal; a nullable column keeps `false` and `null` apart | -### 88) `flow-php/etl-adapter-csv`, `-xml`, `-text`, `-json` - a float is written as the shortest text that reads back as the same float +### 88) `flow-php/etl-adapter-csv`, `-xml`, `-text`, + +`-json` - a float is written as the shortest text that reads back as the same float | Before | After | |-----------------------------------------------------------------------------------------------|---------------------------------------------------------------| @@ -1301,20 +2993,24 @@ Writing a float column forces `serialize_precision = -1` for the render and rest holds another value, `ini_set()` must be allowed; otherwise the writers throw `RuntimeException('Writing floats requires serialize_precision = -1 and ini_set() cannot change it')`. -### 89) `flow-php/etl-adapter-csv`, `-json`, `-xml`, `-text` - a nested value is written by its type, as a column of that type is +### 89) `flow-php/etl-adapter-csv`, `-json`, `-xml`, + +`-text` - a nested value is written by its type, as a column of that type is -| Before | After | -|---------------------------------------------------------------------------------------------------------|----------------------------------------------------------------| -| CSV `list`: `[{"date":"2026-01-02 03:04:05.000000","timezone_type":3,"timezone":"UTC"}]` | `["2026-01-02T03:04:05+00:00"]` (the loader's datetime format) | -| CSV `list`: `[{}]`; `list