sql-dialect-fixes

dialect/api/src/main/java/org/eclipse/daanse/sql/dialect/api/generator/AggregationGenerator.java

@@ -27,6 +27,17 @@ public interface AggregationGenerator extends IdentifierQuoter {
     String IGNORE_NULLS = " IGNORE NULLS ";
     String RESPECT_NULLS = " RESPECT NULLS ";
 
+    // -------------------- AVG --------------------
+
+    /**
+     * The operand of an {@code avg} aggregate as this dialect needs it to compute the mean: the operand itself, unless
+     * the engine averages whole numbers to a whole number. SQL Server returns 31 for the mean of 31 and 32 over an
+     * INTEGER column, where every other engine, MDX and SSAS give 31.5 (database run 19./20.10., F-10.6 2).
+     */
+    default CharSequence averageOperand(CharSequence operand) {
+        return operand;
+    }
+
     // -------------------- list / string aggregation --------------------
 
     default java.util.Optional<String> generateListAgg(CharSequence operand, boolean distinct, String separator,

dialect/api/src/main/java/org/eclipse/daanse/sql/dialect/api/generator/FunctionGenerator.java

@@ -11,6 +11,15 @@
 
 public interface FunctionGenerator {
 
+    /**
+     * The dividend of a division as this dialect needs it to divide without losing the fraction: the dividend itself,
+     * unless the engine cuts a quotient's scale. Derby divides DECIMAL(31,4) by DECIMAL(31,4) at scale 0, so a native
+     * ratio of 1.9 came out as 1 (a native Filter over a calculated measure read no row, database run 20.10.).
+     */
+    default CharSequence quotientDividend(CharSequence dividend) {
+        return dividend;
+    }
+
     default StringBuilder wrapIntoSqlUpperCaseFunction(CharSequence sqlExpression) {
         return new StringBuilder("UPPER(").append(sqlExpression).append(")");
     }

dialect/db/derby/src/main/java/org/eclipse/daanse/sql/dialect/db/derby/DerbyDialect.java

@@ -45,6 +45,23 @@ public String booleanTypeName() {
         return "SMALLINT";
     }
 
+    /**
+     * {@code 1} and {@code 0}: this dialect stores a truth value as SMALLINT ({@link #booleanTypeName()}), and Derby does
+     * not compare SMALLINT with BOOLEAN ("ERROR 42818: Comparisons between 'SMALLINT' and 'BOOLEAN' are not supported",
+     * a boolean level's key, database run 20.10.).
+     */
+    @Override
+    public void quoteBooleanLiteral(StringBuilder buf, String value) {
+        String literal = value.trim();
+        if (literal.equalsIgnoreCase("TRUE") || literal.equals("1")) {
+            buf.append('1');
+        } else if (literal.equalsIgnoreCase("FALSE") || literal.equals("0")) {
+            buf.append('0');
+        } else {
+            throw new NumberFormatException("Illegal BOOLEAN literal:  " + value);
+        }
+    }
+
     /** JDBC-free constructor for SQL generation. */
     public DerbyDialect() {
         super(org.eclipse.daanse.sql.dialect.api.DialectInitData.ansiDefaults());
@@ -141,6 +158,34 @@ public boolean requiresAliasForFromQuery() {
         return true;
     }
 
+    /**
+     * Derby averages an integer column to an integer (31 for 31 and 32, the Measure Aggregator Base tutorial, database
+     * run 20.10.); the operand is read as DOUBLE, as on SQL Server.
+     */
+    @Override
+    public CharSequence averageOperand(CharSequence operand) {
+        return "CAST(" + operand + " AS DOUBLE)";
+    }
+
+    /**
+     * Derby divides decimals at a scale its precision leaves over: DECIMAL(31,4) by DECIMAL(31,4) at scale 0, so a
+     * ratio of 1.9 is 1; integers divide to an integer. The dividend is read as DOUBLE (a native Filter over a calculated
+     * measure, FilterTest#testCmNativeFilter, database run 20.10.).
+     */
+    @Override
+    public CharSequence quotientDividend(CharSequence dividend) {
+        return "CAST(" + dividend + " AS DOUBLE)";
+    }
+
+    /**
+     * One writer: Derby locks its system tables for DDL, and tables and indexes created at once deadlock ("A lock could
+     * not be obtained due to a deadlock", CREATE INDEX beside CREATE TABLE while FoodMart is loaded, database run 20.10.).
+     */
+    @Override
+    public boolean supportsParallelLoading() {
+        return false;
+    }
+
     @Override
     public boolean allowsMultipleCountDistinct() {
         // Derby allows at most one distinct-count per query.

dialect/db/derby/src/test/java/org/eclipse/daanse/sql/dialect/db/derby/DerbyDialectTest.java

@@ -39,6 +39,36 @@ protected void setUp() throws Exception {
         dialect = new DerbyDialect(org.eclipse.daanse.sql.dialect.api.DialectInitData.fromConnection(connection));
     }
 
+    /** A truth value is stored as SMALLINT, so its literal is 1 or 0: Derby does not compare SMALLINT with BOOLEAN. */
+    @Test
+    void aTruthValueIsWrittenAsTheNumberItIsStoredAs() {
+        for (String[] pair : new String[][] { { "true", "1" }, { "FALSE", "0" }, { "1", "1" }, { "0", "0" } }) {
+            StringBuilder buf = new StringBuilder();
+            dialect.quoteBooleanLiteral(buf, pair[0]);
+            org.junit.jupiter.api.Assertions.assertEquals(pair[1], buf.toString(), pair[0]);
+        }
+    }
+
+    /** Derby cuts a quotient's scale; the dividend is read as DOUBLE. */
+    @Test
+    void aDivisionReadsItsDividendAsDouble() {
+        org.junit.jupiter.api.Assertions.assertEquals("CAST(x AS DOUBLE)",
+                dialect.functionGenerator().quotientDividend("x").toString());
+    }
+
+    /** AVG over an INTEGER column is an integer in Derby; the operand is read as DOUBLE. */
+    @Test
+    void anAverageReadsItsOperandAsDouble() {
+        org.junit.jupiter.api.Assertions.assertEquals("CAST(x AS DOUBLE)",
+                dialect.aggregationGenerator().averageOperand("x").toString());
+    }
+
+    /** Derby deadlocks on DDL run at once; a loader writes one statement at a time. */
+    @Test
+    void loadingIsOneWriterAtATime() {
+        org.junit.jupiter.api.Assertions.assertFalse(dialect.supportsParallelLoading());
+    }
+
     @Test
     void testGetDialectName() {
         assertEquals("derby", dialect.name());

dialect/db/h2/src/main/java/org/eclipse/daanse/sql/dialect/db/h2/H2Dialect.java

@@ -109,6 +109,16 @@ public H2Dialect(org.eclipse.daanse.sql.dialect.api.DialectInitData init) {
         super(init);
     }
 
+    /**
+     * H2 divides integers to an integer. MDX divides with a fraction, and a native condition over a quotient has to as
+     * well: the dividend is read as DOUBLE (F-10.9, a native Filter over Units Shipped / Units Ordered, database run
+     * 20.10.).
+     */
+    @Override
+    public CharSequence quotientDividend(CharSequence dividend) {
+        return "CAST(" + dividend + " AS DOUBLE)";
+    }
+
     @Override
     public String name() {
         return SUPPORTED_PRODUCT_NAME.toLowerCase();

dialect/db/h2/src/test/java/org/eclipse/daanse/sql/dialect/db/h2/H2DialectTest.java

@@ -48,6 +48,13 @@ protected void setUp() throws Exception {
         dialect = new H2Dialect(org.eclipse.daanse.sql.dialect.api.DialectInitData.fromConnection(connection));
     }
 
+    /** H2 divides integers to an integer; the dividend is read as DOUBLE. */
+    @Test
+    void aQuotientReadsItsDividendAsDouble() {
+        org.junit.jupiter.api.Assertions.assertEquals("CAST(x AS DOUBLE)",
+                dialect.functionGenerator().quotientDividend("x").toString());
+    }
+
     @Test
     void testGetDialectName() {
         assertEquals("h2", dialect.name());

dialect/db/mssqlserver/src/main/java/org/eclipse/daanse/sql/dialect/db/mssqlserver/MicrosoftSqlServerDialect.java

@@ -441,6 +441,24 @@ public boolean supportsPercentileCont() {
         return true;
     }
 
+    /**
+     * SQL Server divides integers to an integer. MDX divides with a fraction, and a native condition over a quotient has to as well: the dividend is read as
+     * float (F-10.9, a native Filter over Units Shipped / Units Ordered, database run 20.10.).
+     */
+    @Override
+    public CharSequence quotientDividend(CharSequence dividend) {
+        return "CAST(" + dividend + " AS float)";
+    }
+
+    /**
+     * SQL Server averages an integer column to an integer (the result type of AVG is the operand's); the operand is
+     * read as float, the double way every measure value takes.
+     */
+    @Override
+    public CharSequence averageOperand(CharSequence operand) {
+        return "CAST(" + operand + " AS float)";
+    }
+
     @Override
     public java.util.Optional<String> generateListAgg(CharSequence operand, boolean distinct, String separator,
             String coalesce, String onOverflowTruncate, List<OrderedColumn> columns) {

dialect/db/mssqlserver/src/test/java/org/eclipse/daanse/sql/dialect/db/mssqlserver/MicrosoftSqlServerGeneratorsTest.java

@@ -27,6 +27,19 @@ class MicrosoftSqlServerGeneratorsTest {
 
     private final MicrosoftSqlServerDialect d = new MicrosoftSqlServerDialect();
 
+    /** SQL Server divides integers to an integer; the dividend is read as float. */
+    @Test
+    void a_quotient_reads_its_dividend_as_float() {
+        assertThat(d.functionGenerator().quotientDividend("x")).hasToString("CAST(x AS float)");
+    }
+
+    /** AVG over an INTEGER column is an integer on SQL Server; the operand is read as float. */
+    @Test
+    void avg_reads_its_operand_as_float() {
+        assertThat(d.aggregationGenerator().averageOperand("\"t\".\"x\""))
+                .hasToString("CAST(\"t\".\"x\" AS float)");
+    }
+
     @Test
     void output_clause() {
         assertThat(d.returningGenerator().supportsReturning()).isTrue();

dialect/db/mysql/src/main/java/org/eclipse/daanse/sql/dialect/db/mysql/MySqlDialect.java

@@ -424,9 +424,12 @@ public Optional<String> generateRegularExpression(String source, String javaRege
         }
         sb.append(" REGEXP ");
         // MDX MATCHES is anchored (Pattern.matches); REGEXP is contains —
-        // anchor AFTER the case fold, or the fold would turn \z into \Z
+        // anchor AFTER the case fold, or the fold would turn \z into \Z.
+        // REGEXP follows the collation of its operand, case-insensitive by default (utf8mb4_0900_ai_ci): a
+        // case-sensitive match says so in the pattern. (?-i) is read by ICU (MySQL 8.0.4+) and PCRE (MariaDB), the
+        // engines \A and \z already need; without it "a.*" matched "A..." natively, not when interpreted.
         if (caseSensitive) {
-            quoteStringLiteral(sb, "\\A(?:" + javaRegex + ")\\z");
+            quoteStringLiteral(sb, "(?-i)\\A(?:" + javaRegex + ")\\z");
         } else {
             quoteStringLiteral(sb, "\\A(?:" + javaRegex.toUpperCase() + ")\\z");
         }
@@ -492,9 +495,12 @@ public java.util.Optional<String> generateBitAggregation(BitOperation operation,
         case AND -> buf.append("BIT_AND(").append(operand).append(")");
         case OR -> buf.append("BIT_OR(").append(operand).append(")");
         case XOR -> buf.append("BIT_XOR(").append(operand).append(")");
-        case NAND -> buf.append("NOT(BIT_AND(").append(operand).append("))");
-        case NOR -> buf.append("NOT(BIT_OR(").append(operand).append("))");
-        case NXOR -> buf.append("NOT(BIT_XOR(").append(operand).append("))");
+        // NOT is logical in MySQL and MariaDB: NOT(BIT_AND(1)) is 0, which the tutorial oracle read silently as the
+        // NAND (database run 20.10.). ~ is bitwise, but on the unsigned 64-bit result of BIT_AND (~1 is
+        // 18446744073709551614); read as SIGNED it is -2, as DuckDB writes it and the provider computes it
+        case NAND -> buf.append("CAST(~BIT_AND(").append(operand).append(") AS SIGNED)");
+        case NOR -> buf.append("CAST(~BIT_OR(").append(operand).append(") AS SIGNED)");
+        case NXOR -> buf.append("CAST(~BIT_XOR(").append(operand).append(") AS SIGNED)");
         };
         return java.util.Optional.of(result.toString());
     }

dialect/db/mysql/src/test/java/org/eclipse/daanse/sql/dialect/db/mysql/MySqlDialect3Test.java

@@ -76,7 +76,8 @@ void testGenerateRegularExpression_CaseInsensitive() throws Exception {
     @Test
     void testGenerateRegularExpression_CaseSensitive() throws Exception {
         String sql = dialect.regexGenerator().generateRegularExpression("table.column", ".*a.*").get().toString();
-        assertEquals("table.column IS NOT NULL AND table.column REGEXP '\\\\A(?:.*a.*)\\\\z'", sql);
+        // REGEXP follows the collation, case-insensitive by default: the pattern asks for case
+        assertEquals("table.column IS NOT NULL AND table.column REGEXP '(?-i)\\\\A(?:.*a.*)\\\\z'", sql);
     }
 
     @Test

dialect/db/mysql/src/test/java/org/eclipse/daanse/sql/dialect/db/mysql/MySqlGeneratorsTest.java

@@ -32,6 +32,21 @@ class MySqlGeneratorsTest {
 
     private final MySqlDialect d = new MySqlDialect();
 
+    /**
+     * A negated bit aggregate is the signed bitwise complement: MySQL's NOT is logical (NOT(BIT_AND(1)) is 0), and ~ on
+     * the unsigned 64-bit BIT_AND result is 18446744073709551614 where the other arts answer -2.
+     */
+    @Test
+    void a_negated_bit_aggregate_is_the_signed_bitwise_complement() {
+        var g = d.aggregationGenerator();
+        assertThat(g.generateBitAggregation(org.eclipse.daanse.sql.model.sql.BitOperation.NAND, "x").orElseThrow())
+                .isEqualTo("CAST(~BIT_AND(x) AS SIGNED)");
+        assertThat(g.generateBitAggregation(org.eclipse.daanse.sql.model.sql.BitOperation.NOR, "x").orElseThrow())
+                .isEqualTo("CAST(~BIT_OR(x) AS SIGNED)");
+        assertThat(g.generateBitAggregation(org.eclipse.daanse.sql.model.sql.BitOperation.NXOR, "x").orElseThrow())
+                .isEqualTo("CAST(~BIT_XOR(x) AS SIGNED)");
+    }
+
     @Test
     void pagination_offset_limit() {
         assertThat(d.paginationGenerator().paginate(OptionalLong.of(20), OptionalLong.of(40)))

dialect/db/postgresql/src/main/java/org/eclipse/daanse/sql/dialect/db/postgresql/PostgreSqlDialect.java

@@ -394,6 +394,43 @@ protected boolean supportsNullsOrdering() {
         return productVersion != null && productVersion.compareTo("8.3") >= 0;
     }
 
+    /**
+     * PostgreSQL divides integers to an integer. MDX divides with a fraction, and a native condition over a quotient has to as well: the dividend is read as
+     * double precision (F-10.9, a native Filter over Units Shipped / Units Ordered, database run 20.10.).
+     */
+    @Override
+    public CharSequence quotientDividend(CharSequence dividend) {
+        return "CAST(" + dividend + " AS double precision)";
+    }
+
+    /**
+     * {@code STRING_AGG(CAST(operand AS TEXT), 'separator' ORDER BY ...)}. PostgreSQL had no list aggregation here, and
+     * every text aggregation failed in the SQL builder and in the recorder alike (the writeback tutorials on
+     * PostgreSQL, database run 20.10.). The operand is cast, because STRING_AGG takes text only. DISTINCT with an ORDER BY
+     * over other columns is an error in PostgreSQL (the ORDER BY must name the arguments); DISTINCT is then left out, as
+     * DuckDB does, so that both write the same list.
+     */
+    @Override
+    public java.util.Optional<String> generateListAgg(CharSequence operand, boolean distinct, String separator,
+            String coalesce, String onOverflowTruncate, List<OrderedColumn> columns) {
+        boolean ordered = columns != null && !columns.isEmpty();
+        StringBuilder buf = new StringBuilder(64).append("STRING_AGG(");
+        if (distinct && !ordered) {
+            buf.append("DISTINCT ");
+        }
+        buf.append("CAST(").append(operand).append(" AS TEXT), '")
+                .append((separator != null ? separator : ",").replace("'", "''")).append('\'');
+        if (ordered) {
+            buf.append(" ORDER BY ").append(buildOrderedColumnsClause(columns));
+        }
+        return java.util.Optional.of(buf.append(')').toString());
+    }
+
+    @Override
+    public boolean supportsListAgg() {
+        return true;
+    }
+
     // Unified BitOperation methods
 
     @Override
@@ -403,9 +440,12 @@ public java.util.Optional<String> generateBitAggregation(BitOperation operation,
         case AND -> buf.append("bit_and(").append(operand).append(")");
         case OR -> buf.append("bit_or(").append(operand).append(")");
         case XOR -> buf.append("bit_xor(").append(operand).append(")");
-        case NAND -> buf.append("NOT(bit_and(").append(operand).append("))");
-        case NOR -> buf.append("NOT(bit_or(").append(operand).append("))");
-        case NXOR -> buf.append("NOT(bit_xor(").append(operand).append("))");
+        // NOT is logical in PostgreSQL and takes a boolean, so NOT(bit_and(x)) failed on every integer column
+        // (tutorial oracle, database run 20.10.); the bitwise complement is ~, as DuckDB writes it and the provider
+        // computes it (the NAND of 1 is -2)
+        case NAND -> buf.append("~(bit_and(").append(operand).append("))");
+        case NOR -> buf.append("~(bit_or(").append(operand).append("))");
+        case NXOR -> buf.append("~(bit_xor(").append(operand).append("))");
         };
         return java.util.Optional.of(result.toString());
     }

dialect/db/postgresql/src/test/java/org/eclipse/daanse/sql/dialect/db/postgresql/PostgreSqlGeneratorsTest.java

@@ -35,6 +35,43 @@ class PostgreSqlGeneratorsTest {
 
     private final PostgreSqlDialect d = new PostgreSqlDialect();
 
+    /** PostgreSQL divides integers to an integer; the dividend is read as double precision. */
+    @Test
+    void a_quotient_reads_its_dividend_as_double_precision() {
+        assertThat(d.functionGenerator().quotientDividend("x")).hasToString("CAST(x AS double precision)");
+    }
+
+    /** A negated bit aggregate is the bitwise complement: PostgreSQL's NOT is logical and takes no integer. */
+    @Test
+    void a_negated_bit_aggregate_is_the_bitwise_complement() {
+        org.eclipse.daanse.sql.dialect.api.generator.AggregationGenerator g = d.aggregationGenerator();
+        assertThat(g.generateBitAggregation(org.eclipse.daanse.sql.model.sql.BitOperation.NAND, "x").orElseThrow())
+                .isEqualTo("~(bit_and(x))");
+        assertThat(g.generateBitAggregation(org.eclipse.daanse.sql.model.sql.BitOperation.NOR, "x").orElseThrow())
+                .isEqualTo("~(bit_or(x))");
+        assertThat(g.generateBitAggregation(org.eclipse.daanse.sql.model.sql.BitOperation.NXOR, "x").orElseThrow())
+                .isEqualTo("~(bit_xor(x))");
+    }
+
+    /** PostgreSQL lists text with STRING_AGG; the operand is cast, the separator quoted. */
+    @Test
+    void list_aggregation_is_string_agg() {
+        assertThat(d.aggregationGenerator().supportsListAgg()).isTrue();
+        assertThat(d.aggregationGenerator().generateListAgg("\"t\".\"c\"", true, " | ", null, null, List.of())
+                .orElseThrow()).isEqualTo("STRING_AGG(DISTINCT CAST(\"t\".\"c\" AS TEXT), ' | ')");
+        assertThat(d.aggregationGenerator().generateListAgg("x", false, "it's", null, null,
+                List.of(new org.eclipse.daanse.sql.model.sql.OrderedColumn("y", null))).orElseThrow())
+                .isEqualTo("STRING_AGG(CAST(x AS TEXT), 'it''s' ORDER BY \"y\")");
+    }
+
+    /** DISTINCT beside an ORDER BY over other columns is an error in PostgreSQL; it is left out, as in DuckDB. */
+    @Test
+    void a_distinct_ordered_list_is_written_without_distinct() {
+        assertThat(d.aggregationGenerator().generateListAgg("x", true, null, null, null,
+                List.of(new org.eclipse.daanse.sql.model.sql.OrderedColumn("y", null))).orElseThrow())
+                .isEqualTo("STRING_AGG(CAST(x AS TEXT), ',' ORDER BY \"y\")");
+    }
+
     @Test
     void pagination_limit_offset() {
         assertThat(d.paginationGenerator().paginate(OptionalLong.of(20), OptionalLong.of(40)))