DuckDB: support QUALIFY after WINDOW, OVER (w ...) and FILTER without WHERE (#2803)

* DuckDB: support QUALIFY after WINDOW, OVER (w ...) and FILTER without WHERE

- QUALIFY may follow the WINDOW clause and prints back there
  (`PlainSelect.isQualifyAfterWindow()`).
- A window specification may start with the name of the window it
  extends, `OVER (w ORDER BY b)`, also in WINDOW definitions
  (`WindowDefinition.getBaseWindowName()`).
- `FILTER (b > 1)` without WHERE is accepted except under the
  PostgreSQL dialect (`AnalyticExpression.isFilterWhereKeyword()`).

* Keep window specifications parseable inside lookahead

- The base window name is a choice rather than an optional, so that
  lookahead scans through a window specification keep its check:
  `JSON_VALUE(... DEFAULT sum(a) OVER (PARTITION BY b) ON EMPTY)` parses
  again.
- Only an OVER window may extend a named window, not WITHIN GROUP.
- QUALIFY after WINDOW requires a WINDOW clause.

Author: guizmaii

src/main/java/net/sf/jsqlparser/expression/AnalyticExpression.java

@@ -41,6 +41,7 @@ public class AnalyticExpression extends ASTNodeAccessImpl implements Expression
     private boolean unique = false;
     private boolean ignoreNullsOutside = false; // IGNORE NULLS outside function parameters
     private Expression filterExpression = null;
+    private boolean filterWhereKeyword = true;
     private List<OrderByElement> funcOrderBy = null;
     private String onOverflowTruncate = null;
 
@@ -391,12 +392,22 @@ public String toString() {
     /** Renders the filter using the caller's expression writer. */
     public void appendFilterTo(StringBuilder builder, Consumer<Expression> expressionPrinter) {
         if (filterExpression != null) {
-            builder.append("FILTER (WHERE ");
+            builder.append(filterWhereKeyword ? "FILTER (WHERE " : "FILTER (");
             expressionPrinter.accept(filterExpression);
             builder.append(')');
         }
     }
 
+    /** Whether the FILTER clause has WHERE, which DuckDB allows to leave out. */
+    public boolean isFilterWhereKeyword() {
+        return filterWhereKeyword;
+    }
+
+    public AnalyticExpression setFilterWhereKeyword(boolean filterWhereKeyword) {
+        this.filterWhereKeyword = filterWhereKeyword;
+        return this;
+    }
+
     public boolean isAllColumns() {
         return allColumns;
     }

src/main/java/net/sf/jsqlparser/expression/WindowDefinition.java

@@ -23,6 +23,7 @@ public class WindowDefinition implements Serializable {
     final PartitionByClause partitionBy = new PartitionByClause();
     WindowElement windowElement = null;
     private String windowName;
+    private String baseWindowName;
 
     public OrderByClause getOrderBy() {
         return orderBy;
@@ -61,6 +62,31 @@ public void setPartitionExpressionList(ExpressionList<Expression> partitionExpre
         partitionBy.setExpressions(partitionExpressionList, brackets);
     }
 
+    /** DuckDB: the window this one extends, as w in OVER (w ORDER BY b), or {@code null}. */
+    public String getBaseWindowName() {
+        return baseWindowName;
+    }
+
+    public WindowDefinition setBaseWindowName(String baseWindowName) {
+        this.baseWindowName = baseWindowName;
+        return this;
+    }
+
+    /** Appends the base window name, separated from any following part. */
+    public void appendBaseWindowNameTo(StringBuilder builder) {
+        if (baseWindowName != null) {
+            builder.append(baseWindowName);
+            boolean more = partitionBy.getPartitionExpressionList() != null
+                    && !partitionBy.getPartitionExpressionList().isEmpty()
+                    || orderBy.getOrderByElements() != null
+                            && !orderBy.getOrderByElements().isEmpty()
+                    || windowElement != null;
+            if (more) {
+                builder.append(' ');
+            }
+        }
+    }
+
     public String getWindowName() {
         return windowName;
     }
@@ -105,6 +131,7 @@ public String toString() {
             b.append(windowName).append(" AS ");
         }
         b.append("(");
+        appendBaseWindowNameTo(b);
         partitionBy.toStringPartitionBy(b);
         orderBy.toStringOrderByElements(b);
 

src/main/java/net/sf/jsqlparser/statement/select/PlainSelect.java

@@ -45,6 +45,7 @@ public class PlainSelect extends Select {
     private GroupByElement groupBy;
     private Expression having;
     private Expression qualify;
+    private boolean qualifyAfterWindow;
     private OptimizeFor optimizeFor;
     private Skip skip;
     private final EnumSet<MySqlSelectModifier> mySqlSelectModifiers =
@@ -442,6 +443,16 @@ public Expression getQualify() {
         return qualify;
     }
 
+    /** Whether QUALIFY follows the WINDOW clause, as in DuckDB, rather than preceding it. */
+    public boolean isQualifyAfterWindow() {
+        return qualifyAfterWindow;
+    }
+
+    public PlainSelect setQualifyAfterWindow(boolean qualifyAfterWindow) {
+        this.qualifyAfterWindow = qualifyAfterWindow;
+        return this;
+    }
+
     public PlainSelect setQualify(Expression qualify) {
         this.qualify = qualify;
         return this;
@@ -595,14 +606,17 @@ public StringBuilder appendSelectBodyTo(StringBuilder builder) {
         if (having != null) {
             builder.append(" HAVING ").append(having);
         }
-        if (qualify != null) {
+        if (qualify != null && !qualifyAfterWindow) {
             builder.append(" QUALIFY ").append(qualify);
         }
         if (windowDefinitions != null) {
             builder.append(" WINDOW ");
             builder.append(windowDefinitions.stream().map(WindowDefinition::toString)
                     .collect(joining(", ")));
         }
+        if (qualify != null && qualifyAfterWindow) {
+            builder.append(" QUALIFY ").append(qualify);
+        }
         appendEmitClauseTo(builder);
         if (intoTempTable != null) {
             builder.append(" INTO TEMP ").append(intoTempTable);

src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java

@@ -1346,6 +1346,9 @@ public <S> StringBuilder visit(AnalyticExpression analyticExpression, S context)
             builder.append(" ").append(analyticExpression.getWindowName());
         } else if (analyticExpression.getType() != AnalyticType.WITHIN_GROUP_OVER) {
             builder.append(" (");
+            if (analyticExpression.getWindowDefinition() != null) {
+                analyticExpression.getWindowDefinition().appendBaseWindowNameTo(builder);
+            }
 
             if (partitionExpressionList != null
                     && !partitionExpressionList.isEmpty()) {

src/main/java/net/sf/jsqlparser/util/deparser/SelectDeParser.java

@@ -233,7 +233,7 @@ public <S> StringBuilder visit(PlainSelect plainSelect, S context) {
             builder.append(" HAVING ");
             plainSelect.getHaving().accept(expressionVisitor, context);
         }
-        if (plainSelect.getQualify() != null) {
+        if (plainSelect.getQualify() != null && !plainSelect.isQualifyAfterWindow()) {
             builder.append(" QUALIFY ");
             plainSelect.getQualify().accept(expressionVisitor, context);
         }
@@ -242,6 +242,10 @@ public <S> StringBuilder visit(PlainSelect plainSelect, S context) {
             builder.append(plainSelect.getWindowDefinitions().stream()
                     .map(WindowDefinition::toString).collect(joining(", ")));
         }
+        if (plainSelect.getQualify() != null && plainSelect.isQualifyAfterWindow()) {
+            builder.append(" QUALIFY ");
+            plainSelect.getQualify().accept(expressionVisitor, context);
+        }
         Alias alias = plainSelect.getAlias();
         if (alias != null) {
             builder.append(alias);

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -1833,6 +1833,17 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
         }
     }
 
+    /** Whether a window specification starts with the name of the window it extends. */
+    private boolean isBaseWindowNameAhead() {
+        Token token = getToken(1);
+        switch (token.kind) {
+            case K_PARTITION: case K_ORDER: case K_ROWS: case K_RANGE: case K_GROUPS:
+                return false;
+            default:
+                return isIdentifierishNameToken(token);
+        }
+    }
+
     private boolean isAliasAhead() {
         if (isAsOfJoinAhead() || isSemiAntiJoinAhead() || isApproxNearestAhead()) {
             return false;
@@ -8408,6 +8419,9 @@ PlainSelect PlainSelect() #PlainSelect:
         windowName = RelObjectName() <K_AS> winDef = windowDefinition() { List<WindowDefinition> winDefs = new ArrayList<WindowDefinition>(); winDefs.add(winDef.withWindowName(windowName)); }
         ( LOOKAHEAD(2) "," windowName = RelObjectName() <K_AS> winDef = windowDefinition() { winDefs.add(winDef.withWindowName(windowName)); } )*
         { plainSelect.setWindowDefinitions(winDefs); }
+        // DuckDB: QUALIFY after WINDOW
+        [ LOOKAHEAD(2, { plainSelect.getQualify() == null }) qualify=Qualify()
+            { plainSelect.setQualify(qualify); plainSelect.setQualifyAfterWindow(true); } ]
     ]
     [ LOOKAHEAD(<K_ORDER> <K_BY>) orderByElements = OrderByElements() { plainSelect.setOrderByElements(orderByElements); } ]
     [ LOOKAHEAD(2) interpolateElements = InterpolateClause() { plainSelect.setInterpolate(interpolateElements); } ]
@@ -13479,7 +13493,13 @@ void windowFun(AnalyticExpression retval):{
     (
         windowName = RelObjectName() { retval.setWindowName(windowName); }
         |
-        winDef = windowDefinition() { retval.setWindowDefinition(winDef); }
+        winDef = windowDefinition()
+        {
+            if (winDef.getBaseWindowName() != null && retval.getType() != AnalyticType.OVER) {
+                throw new ParseException("Only an OVER window may extend a named window");
+            }
+            retval.setWindowDefinition(winDef);
+        }
 
         [
             LOOKAHEAD(2) <K_OVER> "("
@@ -13498,8 +13518,17 @@ WindowDefinition windowDefinition() : {
     WindowElement windowElement = null;
     boolean partitionByBrackets = false;
     WindowDefinition winDef = new WindowDefinition();
+    String baseWindowName;
 } {
     "("
+    // DuckDB: (w ORDER BY b) extends the named window w. A choice, not an optional, so that
+    // lookahead scans through a window specification keep the check
+    (
+        LOOKAHEAD({ isBaseWindowNameAhead() }) baseWindowName=RelObjectName()
+        { winDef.setBaseWindowName(baseWindowName); }
+        |
+        {}
+    )
     [<K_PARTITION> <K_BY>
         (LOOKAHEAD(3) expressionList=ComplexExpressionList()
         | "(" {partitionByBrackets = true;} expressionList=ComplexExpressionList() ")" )
@@ -13523,7 +13552,7 @@ AnalyticExpression AnalyticExpression(Function function) :
 {
     (
         (
-            filter=AggregateFilter() { retval.setType(AnalyticType.FILTER_ONLY); }
+            filter=AggregateFilter(retval) { retval.setType(AnalyticType.FILTER_ONLY); }
             [ LOOKAHEAD(2) windowFun(retval) ]
             {
                 if (Dialect.POSTGRESQL.name().equals(getAsString(Feature.dialect))
@@ -13535,19 +13564,29 @@ AnalyticExpression AnalyticExpression(Function function) :
         )
         | windowFun(retval)
           [ LOOKAHEAD({ retval.getType() == AnalyticType.WITHIN_GROUP
-                  && getToken(1).kind == K_FILTER }) filter=AggregateFilter() ]
+                  && getToken(1).kind == K_FILTER }) filter=AggregateFilter(retval) ]
     )
     {
         retval.setFilterExpression(filter);
         if (true) { return retval; }
     }
 }
 
-/** Shared FILTER predicate for ordinary, window, and ordered-set aggregates. */
-Expression AggregateFilter():
+/**
+ * Shared FILTER predicate for ordinary, window, and ordered-set aggregates. DuckDB also accepts
+ * it without WHERE, FILTER (x > 1), but PostgreSQL does not.
+ */
+Expression AggregateFilter(AnalyticExpression analyticExpression):
 { Expression filter; }
 {
-    <K_FILTER> "(" <K_WHERE> filter=Expression() ")"
+    <K_FILTER> "("
+    (
+        <K_WHERE>
+        |
+        LOOKAHEAD({ !Dialect.POSTGRESQL.name().equals(getAsString(Feature.dialect)) })
+        { analyticExpression.setFilterWhereKeyword(false); }
+    )
+    filter=Expression() ")"
     { return filter; }
 }
 

src/test/java/net/sf/jsqlparser/statement/select/DuckDBTest.java

@@ -1072,4 +1072,38 @@ void testTrailingCommaInSelectList() throws JSQLParserException {
             TestUtils.assertSqlCanBeParsedAndDeparsed("SELECT a, " + item + " FROM t", true);
         }
     }
+
+    @Test
+    void testWindowAndFilterForms() throws JSQLParserException {
+        for (String sqlStr : new String[] {
+                "SELECT a FROM t WINDOW w AS (ORDER BY a) QUALIFY row_number() OVER w = 1",
+                "SELECT a FROM t QUALIFY row_number() OVER w = 1 WINDOW w AS (ORDER BY a)",
+                "SELECT sum(a) OVER (w ROWS 2 PRECEDING) FROM t WINDOW w AS (ORDER BY b)",
+                "SELECT sum(a) OVER (w) FROM t WINDOW w AS (ORDER BY b)",
+                "SELECT sum(a) OVER w2 FROM t WINDOW w AS (PARTITION BY c), w2 AS (w ORDER BY b)",
+                "SELECT sum(a) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM t",
+                "SELECT sum(a) FILTER (b > 1) FROM t",
+                "SELECT count(*) FILTER (b > 1) OVER () FROM t",
+                // window specifications inside syntactic lookahead
+                "SELECT JSON_VALUE(j, '$.a' DEFAULT sum(a) OVER (PARTITION BY b) ON EMPTY) FROM t",
+                "SELECT JSON_VALUE(j, '$.a' DEFAULT sum(a) OVER (w ORDER BY b) ON EMPTY) FROM t"}) {
+            TestUtils.assertSqlCanBeParsedAndDeparsed(sqlStr, true);
+        }
+        PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse(
+                "SELECT sum(a) FILTER (b > 1) OVER (w ORDER BY c) FROM t WINDOW w AS (PARTITION BY d)");
+        net.sf.jsqlparser.expression.AnalyticExpression sum =
+                (net.sf.jsqlparser.expression.AnalyticExpression) select.getSelectItem(0)
+                        .getExpression();
+        Assertions.assertFalse(sum.isFilterWhereKeyword());
+        Assertions.assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil.parse(
+                "SELECT percentile_cont(0.5) WITHIN GROUP (w ORDER BY a) FROM t"));
+        Assertions.assertThrows(JSQLParserException.class,
+                () -> CCJSqlParserUtil.parse("SELECT a FROM t ORDER SIBLINGS BY a QUALIFY x = 1"));
+        // PostgreSQL requires WHERE
+        Assertions.assertThrows(JSQLParserException.class,
+                () -> CCJSqlParserUtil.parse("SELECT sum(a) FILTER (b > 1) FROM t",
+                        parser -> parser.withDialect(
+                                net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect.POSTGRESQL)));
+        Assertions.assertEquals("w", sum.getWindowDefinition().getBaseWindowName());
+    }
 }