Feature #2058 - Window Function: frame exclude clause

doc/sql.extensions/README.window_functions.md

@@ -4,7 +4,7 @@ By the SQL specification, window functions (also know as analytical functions) a
 
 That sort of functions are used with the `OVER` clause. Window functions may appear only in the select list or the `ORDER BY` clause of a query.
 
-Additional to the `OVER` clause, Firebird window functions may use partitions, order and frames (FB 4.0).
+Additional to the `OVER` clause, Firebird window functions may use partitions, order, frames (FB 4.0) and frame exclusions (FB 6.0).
 
 Syntax:
 
@@ -23,7 +23,7 @@ Syntax:
   ORDER BY <expr> [<direction>] [<nulls placement>] [, <expr> [<direction>] [<nulls placement>]] ...
 
 <window frame> ::=
-  {RANGE | ROWS} <window frame extent>
+  {RANGE | ROWS} <window frame extent> [<window frame exclusion>]
 
 <window frame extent> ::=
   {<window frame start> | <window frame between>}
@@ -35,6 +35,9 @@ Syntax:
   BETWEEN {UNBOUNDED PRECEDING | <expr> PRECEDING | <expr> FOLLOWING | CURRENT ROW} AND
           {UNBOUNDED FOLLOWING | <expr> PRECEDING | <expr> FOLLOWING | CURRENT ROW}
 
+<window frame exclusion> ::=
+  EXCLUDE {NO OTHERS | CURRENT ROW | GROUP | TIES}
+
 <direction> ::=
   {ASC | DESC}
 
@@ -279,6 +282,28 @@ With `ROWS`, order expressions is not limited by number or types. In this case,
 
 The frame syntax with `<window frame start>` specifies the start frame, with the end frame being `CURRENT ROW`.
 
+The optional frame exclusion clause (FB 6.0) is part of the frame clause and removes rows from the frame after its bounds have been evaluated. It can only be specified together with an explicit `ROWS` or `RANGE` frame. `EXCLUDE NO OTHERS` is the default and keeps the frame unchanged.
+
+`EXCLUDE CURRENT ROW` removes only the current row from the frame.
+
+`EXCLUDE GROUP` removes the current row and all its peers. Peers are rows with the same `ORDER BY` values as the current row. If there is no `ORDER BY`, all rows in the partition are peers.
+
+`EXCLUDE TIES` removes the peers of the current row, but keeps the current row itself. If there is no `ORDER BY`, all other rows in the partition are removed from the frame.
+
+For example, with duplicate salaries, the following query sums the full ordered partition except the current salary peer group:
+
+```sql
+select
+    id,
+    salary,
+    sum(salary) over (
+      order by salary
+      rows between unbounded preceding and unbounded following
+      exclude group
+    ) sum_other_salaries
+  from employee
+  order by salary;
+```
 
 
 When `ORDER BY` window clause is used but frame clause is omitted, it defaults to `RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`. This fact makes the query below to produce "weird" behaviour for the "sum_salary" column. It sums from the partition start to the current key, instead of sum the whole partition.
@@ -346,9 +371,9 @@ select
 |  5 |  10.00 |           3 |
 |  2 |  12.00 |           1 |
 
-Some window functions discard frames. `ROW_NUMBER`, `LAG` and `LEAD` always work as `ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`. And `DENSE_RANK`, `RANK`, `PERCENT_RANK` and `CUME_DIST` always work as `RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`.
+Some window functions discard frames and frame exclusions. `ROW_NUMBER`, `LAG` and `LEAD` always work as `ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`. And `DENSE_RANK`, `RANK`, `PERCENT_RANK` and `CUME_DIST` always work as `RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`.
 
-`FIRST_VALUE`, `LAST_VALUE` and `NTH_VALUE` respect frames, but the `RANGE` unit works identically as `ROWS`.
+`FIRST_VALUE`, `LAST_VALUE` and `NTH_VALUE` respect frames and frame exclusions, but the `RANGE` unit works identically as `ROWS`.
 
 ## 6. Named windows (FB 4.0)
 

src/dsql/ExprNodes.cpp

@@ -9644,12 +9644,14 @@ ValueExprNode* OverNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 	const AggNode* aggNode = nodeAs<AggNode>(node->aggExpr);
 
 	if (node->window &&
-		node->window->extent &&
 		aggNode &&
+		(node->window->extent || node->window->exclusion != WindowClause::Exclusion::NO_OTHERS) &&
 		(aggNode->getCapabilities() & AggNode::CAP_RESPECTS_WINDOW_FRAME) !=
 			AggNode::CAP_RESPECTS_WINDOW_FRAME)
 	{
-		node->window->extent = WindowClause::FrameExtent::createDefault(dsqlScratch->getPool());
+		if (node->window->extent)
+			node->window->extent = WindowClause::FrameExtent::createDefault(dsqlScratch->getPool());
+
 		node->window->exclusion = WindowClause::Exclusion::NO_OTHERS;
 	}
 

src/dsql/WinNodes.cpp

@@ -436,7 +436,7 @@ void FirstValueWinNode::aggInit(thread_db* tdbb, Request* request) const
 
 dsc* FirstValueWinNode::winPass(thread_db* tdbb, Request* request, SlidingWindow* window) const
 {
-	if (!window->moveWithinFrame(-window->getInFrameOffset()))
+	if (!window->moveToFrameStart())
 		return nullptr;
 
 	dsc* desc = EVL_expr(tdbb, request, arg);
@@ -497,7 +497,7 @@ void LastValueWinNode::aggInit(thread_db* tdbb, Request* request) const
 
 dsc* LastValueWinNode::winPass(thread_db* tdbb, Request* request, SlidingWindow* window) const
 {
-	if (!window->moveWithinFrame(window->getFrameEnd() - window->getRecordPosition()))
+	if (!window->moveToFrameEnd())
 		return nullptr;
 
 	dsc* desc = EVL_expr(tdbb, request, arg);
@@ -587,11 +587,11 @@ dsc* NthValueWinNode::winPass(thread_db* tdbb, Request* request, SlidingWindow*
 	const SLONG fromPos = desc ? MOV_get_long(tdbb, desc, 0) : FROM_FIRST;
 
 	if (fromPos == FROM_FIRST)
-		records -= window->getInFrameOffset() + 1;
+		--records;
 	else
-		records = window->getFrameEnd() - window->getRecordPosition() - records + 1;
+		records = window->getEffectiveFrameSize() - records;
 
-	if (!window->moveWithinFrame(records))
+	if (!window->moveToFrameOffset(records))
 		return nullptr;
 
 	desc = EVL_expr(tdbb, request, arg);

src/dsql/parse-conflicts.txt

@@ -1 +1 @@
-140 shift/reduce conflicts, 10 reduce/reduce conflicts.
+138 shift/reduce conflicts, 10 reduce/reduce conflicts.

src/dsql/parse.y

@@ -8977,6 +8977,13 @@ window_clause
 	: symbol_window_name_opt
 			window_partition_opt
 			order_clause_opt
+		{
+			$$ = newNode<WindowClause>($1, $2, $3,
+				static_cast<WindowClause::FrameExtent*>(NULL), WindowClause::Exclusion::NO_OTHERS);
+		}
+	| symbol_window_name_opt
+			window_partition_opt
+			order_clause_opt
 			window_frame_extent
 			window_frame_exclusion_opt
 		{
@@ -8992,9 +8999,7 @@ window_partition_opt
 
 %type <windowClauseFrameExtent> window_frame_extent
 window_frame_extent
-	: /* nothing */
-		{ $$ = NULL; }
-	| RANGE
+	: RANGE
 		{ $$ = newNode<WindowClause::FrameExtent>(WindowClause::FrameExtent::Unit::RANGE); }
 		window_frame($2)
 		{ $$ = $2; }

src/jrd/RecordSourceNodes.cpp

@@ -2656,11 +2656,6 @@ void WindowSourceNode::parseWindow(thread_db* tdbb, CompilerScratch* csb)
 				break;
 
 			case blr_window_win_exclusion:
-				//// TODO: CORE-5338 - write code for execution.
-				PAR_error(csb,
-					Arg::Gds(isc_wish_list) <<
-					Arg::Gds(isc_random) << "window EXCLUDE clause");
-
 				window.exclusion = (WindowClause::Exclusion) csb->csb_blr_reader.getByte();
 
 				switch (window.exclusion)

src/jrd/recsrc/RecordSource.h

@@ -770,41 +770,43 @@ namespace Jrd
 	{
 	public:
 		SlidingWindow(thread_db* aTdbb, const BaseBufferedStream* aStream, Request* request,
-			FB_UINT64 aPartitionStart, FB_UINT64 aPartitionEnd,
-			FB_UINT64 aFrameStart, FB_UINT64 aFrameEnd);
+			SINT64 aPartitionStart, SINT64 aPartitionEnd,
+			SINT64 aFrameStart, SINT64 aFrameEnd,
+			SINT64 aExclusionStart1, SINT64 aExclusionEnd1,
+			SINT64 aExclusionStart2, SINT64 aExclusionEnd2);
 		~SlidingWindow();
 
-		FB_UINT64 getPartitionStart() const
+		SINT64 getPartitionStart() const
 		{
 			return partitionStart;
 		}
 
-		FB_UINT64 getPartitionEnd() const
+		SINT64 getPartitionEnd() const
 		{
 			return partitionEnd;
 		}
 
-		FB_UINT64 getPartitionSize() const
+		SINT64 getPartitionSize() const
 		{
 			return partitionEnd - partitionStart + 1;
 		}
 
-		FB_UINT64 getFrameStart() const
+		SINT64 getFrameStart() const
 		{
 			return frameStart;
 		}
 
-		FB_UINT64 getFrameEnd() const
+		SINT64 getFrameEnd() const
 		{
 			return frameEnd;
 		}
 
-		FB_UINT64 getFrameSize() const
+		SINT64 getFrameSize() const
 		{
-			return frameEnd - frameStart + 1;
+			return getEffectiveFrameSize();
 		}
 
-		FB_UINT64 getRecordPosition() const
+		SINT64 getRecordPosition() const
 		{
 			return savedPosition;
 		}
@@ -825,15 +827,37 @@ namespace Jrd
 
 		bool moveWithinPartition(SINT64 delta);
 		bool moveWithinFrame(SINT64 delta);
+		bool moveToFrameStart();
+		bool moveToFrameEnd();
+		bool moveToFrameOffset(SINT64 offset);
+		SINT64 getEffectiveFrameSize() const;
+
+	private:
+		bool hasFrame() const
+		{
+			return frameStart <= frameEnd && frameStart >= partitionStart && frameEnd <= partitionEnd;
+		}
+
+		bool isExcluded(SINT64 position) const
+		{
+			return (position >= exclusionStart1 && position <= exclusionEnd1) ||
+				(position >= exclusionStart2 && position <= exclusionEnd2);
+		}
+
+		bool moveToFramePosition(SINT64 position);
 
 	private:
 		thread_db* tdbb;
 		const BaseBufferedStream* const stream;
-		FB_UINT64 partitionStart;
-		FB_UINT64 partitionEnd;
-		FB_UINT64 frameStart;
-		FB_UINT64 frameEnd;
-		FB_UINT64 savedPosition;
+		SINT64 partitionStart;
+		SINT64 partitionEnd;
+		SINT64 frameStart;
+		SINT64 frameEnd;
+		SINT64 exclusionStart1;
+		SINT64 exclusionEnd1;
+		SINT64 exclusionStart2;
+		SINT64 exclusionEnd2;
+		SINT64 savedPosition;
 		bool moved = false;
 	};
 
@@ -1032,6 +1056,12 @@ namespace Jrd
 			void getFrameValue(thread_db* tdbb, Request* request,
 				const Frame* frame, impure_value_ex* impureValue) const;
 
+			Block getPeerBlock(thread_db* tdbb, Request* request, Impure* impure,
+				SINT64 position) const;
+			void getExclusionBlocks(thread_db* tdbb, Request* request, Impure* impure,
+				SINT64 position, Block* exclusion1, Block* exclusion2) const;
+			bool isExcluded(SINT64 position, const Block& exclusion1, const Block& exclusion2) const;
+
 			SINT64 locateFrameRange(thread_db* tdbb, Request* request, Impure* impure,
 				const Frame* frame, const dsc* offsetDesc, SINT64 position) const;