Fix SETTINGS placement in EXPLAIN AST for FORMAT...SETTINGS syntax

CLAUDE.md

@@ -22,13 +22,18 @@ The tests are very fast. If a test is timing out, it indicates a bug (likely an
 
 ## Checking for Newly Passing Explain Tests
 
-After implementing parser/explain changes, run:
+**IMPORTANT:** After implementing parser/explain changes, ALWAYS run check-explain to update metadata files:
 
 ```bash
 go test ./parser/... -check-explain -v 2>&1 | grep "EXPLAIN PASSES NOW"
 ```
 
-Tests that output `EXPLAIN PASSES NOW` can have their statement removed from `explain_todo` in `metadata.json`.
+This command:
+1. Runs all explain tests including those in `explain_todo`
+2. Automatically updates `metadata.json` files to remove passing statements from `explain_todo`
+3. Reports which tests now pass
+
+**You must run this after every change to parser or explain code**, then commit the updated metadata.json files along with your code changes.
 
 ## Test Structure
 

ast/ast.go

@@ -75,7 +75,8 @@ type SelectQuery struct {
 	LimitBy          []Expression          `json:"limit_by,omitempty"`
 	LimitByHasLimit  bool                  `json:"limit_by_has_limit,omitempty"` // true if LIMIT BY was followed by another LIMIT
 	Offset           Expression            `json:"offset,omitempty"`
-	Settings    []*SettingExpr        `json:"settings,omitempty"`
+	Settings           []*SettingExpr        `json:"settings,omitempty"`
+	SettingsAfterFormat bool                 `json:"settings_after_format,omitempty"` // true if SETTINGS came after FORMAT
 	IntoOutfile *IntoOutfileClause    `json:"into_outfile,omitempty"`
 	Format      *Identifier           `json:"format,omitempty"`
 }
@@ -425,6 +426,26 @@ type AlterCommand struct {
 	Settings       []*SettingExpr       `json:"settings,omitempty"`
 	Where          Expression           `json:"where,omitempty"`       // For DELETE WHERE
 	Assignments    []*Assignment        `json:"assignments,omitempty"` // For UPDATE
+	Projection     *Projection          `json:"projection,omitempty"`  // For ADD PROJECTION
+	ProjectionName string               `json:"projection_name,omitempty"` // For DROP/MATERIALIZE/CLEAR PROJECTION
+}
+
+// Projection represents a projection definition.
+type Projection struct {
+	Position token.Position            `json:"-"`
+	Name     string                    `json:"name"`
+	Select   *ProjectionSelectQuery    `json:"select"`
+}
+
+func (p *Projection) Pos() token.Position { return p.Position }
+func (p *Projection) End() token.Position { return p.Position }
+
+// ProjectionSelectQuery represents the SELECT part of a projection.
+type ProjectionSelectQuery struct {
+	Position token.Position   `json:"-"`
+	Columns  []Expression     `json:"columns"`
+	GroupBy  []Expression     `json:"group_by,omitempty"`
+	OrderBy  *Identifier      `json:"order_by,omitempty"` // Single column for ORDER BY
 }
 
 // Assignment represents a column assignment in UPDATE.
@@ -456,16 +477,20 @@ const (
 	AlterMaterializeIndex  AlterCommandType = "MATERIALIZE_INDEX"
 	AlterAddConstraint     AlterCommandType = "ADD_CONSTRAINT"
 	AlterDropConstraint    AlterCommandType = "DROP_CONSTRAINT"
-	AlterModifyTTL         AlterCommandType = "MODIFY_TTL"
-	AlterModifySetting     AlterCommandType = "MODIFY_SETTING"
-	AlterDropPartition     AlterCommandType = "DROP_PARTITION"
-	AlterDetachPartition   AlterCommandType = "DETACH_PARTITION"
-	AlterAttachPartition   AlterCommandType = "ATTACH_PARTITION"
-	AlterReplacePartition  AlterCommandType = "REPLACE_PARTITION"
-	AlterFreezePartition   AlterCommandType = "FREEZE_PARTITION"
-	AlterFreeze            AlterCommandType = "FREEZE"
-	AlterDeleteWhere       AlterCommandType = "DELETE_WHERE"
-	AlterUpdate            AlterCommandType = "UPDATE"
+	AlterModifyTTL          AlterCommandType = "MODIFY_TTL"
+	AlterModifySetting      AlterCommandType = "MODIFY_SETTING"
+	AlterDropPartition      AlterCommandType = "DROP_PARTITION"
+	AlterDetachPartition    AlterCommandType = "DETACH_PARTITION"
+	AlterAttachPartition    AlterCommandType = "ATTACH_PARTITION"
+	AlterReplacePartition   AlterCommandType = "REPLACE_PARTITION"
+	AlterFreezePartition    AlterCommandType = "FREEZE_PARTITION"
+	AlterFreeze             AlterCommandType = "FREEZE"
+	AlterDeleteWhere        AlterCommandType = "DELETE_WHERE"
+	AlterUpdate             AlterCommandType = "UPDATE"
+	AlterAddProjection      AlterCommandType = "ADD_PROJECTION"
+	AlterDropProjection     AlterCommandType = "DROP_PROJECTION"
+	AlterMaterializeProjection AlterCommandType = "MATERIALIZE_PROJECTION"
+	AlterClearProjection    AlterCommandType = "CLEAR_PROJECTION"
 )
 
 // TruncateQuery represents a TRUNCATE statement.

internal/explain/select.go

@@ -36,21 +36,17 @@ func explainSelectWithUnionQuery(sb *strings.Builder, n *ast.SelectWithUnionQuer
 		}
 	}
 	// FORMAT clause - check if any SelectQuery has Format set
-	var hasFormat bool
 	for _, sel := range n.Selects {
 		if sq, ok := sel.(*ast.SelectQuery); ok && sq.Format != nil {
 			Node(sb, sq.Format, depth+1)
-			hasFormat = true
 			break
 		}
 	}
 	// When FORMAT is present, SETTINGS is output at SelectWithUnionQuery level
-	if hasFormat {
-		for _, sel := range n.Selects {
-			if sq, ok := sel.(*ast.SelectQuery); ok && len(sq.Settings) > 0 {
-				fmt.Fprintf(sb, "%s Set\n", indent)
-				break
-			}
+	for _, sel := range n.Selects {
+		if sq, ok := sel.(*ast.SelectQuery); ok && sq.Format != nil && len(sq.Settings) > 0 {
+			fmt.Fprintf(sb, "%s Set\n", indent)
+			break
 		}
 	}
 }
@@ -126,7 +122,8 @@ func explainSelectQuery(sb *strings.Builder, n *ast.SelectQuery, indent string,
 			Node(sb, expr, depth+2)
 		}
 	}
-	// SETTINGS - output here if there's no FORMAT, otherwise it's at SelectWithUnionQuery level
+	// SETTINGS - output at SelectQuery level only if there's no FORMAT
+	// When FORMAT is present, SETTINGS is at SelectWithUnionQuery level instead
 	if len(n.Settings) > 0 && n.Format == nil {
 		fmt.Fprintf(sb, "%s Set\n", indent)
 	}
@@ -235,21 +232,17 @@ func countSelectUnionChildren(n *ast.SelectWithUnionQuery) int {
 		}
 	}
 	// Check if any SelectQuery has Format set
-	var hasFormat bool
 	for _, sel := range n.Selects {
 		if sq, ok := sel.(*ast.SelectQuery); ok && sq.Format != nil {
 			count++
-			hasFormat = true
 			break
 		}
 	}
-	// When FORMAT is present, SETTINGS is counted at this level
-	if hasFormat {
-		for _, sel := range n.Selects {
-			if sq, ok := sel.(*ast.SelectQuery); ok && len(sq.Settings) > 0 {
-				count++
-				break
-			}
+	// When FORMAT is present, SETTINGS is counted at SelectWithUnionQuery level
+	for _, sel := range n.Selects {
+		if sq, ok := sel.(*ast.SelectQuery); ok && sq.Format != nil && len(sq.Settings) > 0 {
+			count++
+			break
 		}
 	}
 	return count
@@ -390,8 +383,8 @@ func countSelectQueryChildren(n *ast.SelectQuery) int {
 	if n.Offset != nil {
 		count++
 	}
-	// SETTINGS is counted here only if there's no FORMAT
-	// If FORMAT is present, SETTINGS is at SelectWithUnionQuery level
+	// SETTINGS is counted at SelectQuery level only if there's no FORMAT
+	// When FORMAT is present, SETTINGS is at SelectWithUnionQuery level instead
 	if len(n.Settings) > 0 && n.Format == nil {
 		count++
 	}

internal/explain/statements.go

@@ -21,6 +21,9 @@ func explainInsertQuery(sb *strings.Builder, n *ast.InsertQuery, indent string,
 	} else if n.Table != "" {
 		children++ // Table identifier
 	}
+	if len(n.Columns) > 0 {
+		children++ // Column list
+	}
 	if n.Select != nil {
 		children++
 	}
@@ -49,6 +52,14 @@ func explainInsertQuery(sb *strings.Builder, n *ast.InsertQuery, indent string,
 		fmt.Fprintf(sb, "%s Identifier %s\n", indent, name)
 	}
 
+	// Column list
+	if len(n.Columns) > 0 {
+		fmt.Fprintf(sb, "%s ExpressionList (children %d)\n", indent, len(n.Columns))
+		for _, col := range n.Columns {
+			fmt.Fprintf(sb, "%s  Identifier %s\n", indent, col.Parts[len(col.Parts)-1])
+		}
+	}
+
 	if n.Select != nil {
 		Node(sb, n.Select, depth+1)
 	}
@@ -672,13 +683,61 @@ func explainAlterCommand(sb *strings.Builder, cmd *ast.AlterCommand, indent stri
 		if cmd.Where != nil {
 			Node(sb, cmd.Where, depth+1)
 		}
+	case ast.AlterAddProjection:
+		if cmd.Projection != nil {
+			explainProjection(sb, cmd.Projection, indent, depth+1)
+		}
+	case ast.AlterDropProjection, ast.AlterMaterializeProjection, ast.AlterClearProjection:
+		if cmd.ProjectionName != "" {
+			fmt.Fprintf(sb, "%s Identifier %s\n", indent, cmd.ProjectionName)
+		}
 	default:
 		if cmd.Partition != nil {
 			Node(sb, cmd.Partition, depth+1)
 		}
 	}
 }
 
+func explainProjection(sb *strings.Builder, p *ast.Projection, indent string, depth int) {
+	children := 0
+	if p.Select != nil {
+		children++
+	}
+	fmt.Fprintf(sb, "%s Projection (children %d)\n", indent, children)
+	if p.Select != nil {
+		explainProjectionSelectQuery(sb, p.Select, indent+"  ", depth+1)
+	}
+}
+
+func explainProjectionSelectQuery(sb *strings.Builder, q *ast.ProjectionSelectQuery, indent string, depth int) {
+	children := 0
+	if len(q.Columns) > 0 {
+		children++
+	}
+	if q.OrderBy != nil {
+		children++
+	}
+	if len(q.GroupBy) > 0 {
+		children++
+	}
+	fmt.Fprintf(sb, "%sProjectionSelectQuery (children %d)\n", indent, children)
+	if len(q.Columns) > 0 {
+		fmt.Fprintf(sb, "%s ExpressionList (children %d)\n", indent, len(q.Columns))
+		for _, col := range q.Columns {
+			Node(sb, col, depth+2)
+		}
+	}
+	if q.OrderBy != nil {
+		fmt.Fprintf(sb, "%s Identifier %s\n", indent, q.OrderBy.Parts[len(q.OrderBy.Parts)-1])
+	}
+	if len(q.GroupBy) > 0 {
+		fmt.Fprintf(sb, "%s ExpressionList (children %d)\n", indent, len(q.GroupBy))
+		for _, expr := range q.GroupBy {
+			Node(sb, expr, depth+2)
+		}
+	}
+}
+
 func countAlterCommandChildren(cmd *ast.AlterCommand) int {
 	children := 0
 	switch cmd.Type {
@@ -743,6 +802,14 @@ func countAlterCommandChildren(cmd *ast.AlterCommand) int {
 		if cmd.Where != nil {
 			children++
 		}
+	case ast.AlterAddProjection:
+		if cmd.Projection != nil {
+			children++
+		}
+	case ast.AlterDropProjection, ast.AlterMaterializeProjection, ast.AlterClearProjection:
+		if cmd.ProjectionName != "" {
+			children++
+		}
 	default:
 		if cmd.Partition != nil {
 			children++