Add deduplicate suffix to OPTIMIZE query explain output (#114)

Author: kyleconroy

ast/ast.go

@@ -80,16 +80,16 @@ func explainDictionaryDefinition(sb *strings.Builder, n *ast.DictionaryDefinitio
 		explainDictionaryLifetime(sb, n.Lifetime, indent+" ", depth+1)
 	}
 
-	// RANGE (if present, comes before LAYOUT)
-	if n.Range != nil {
-		explainDictionaryRange(sb, n.Range, indent+" ", depth+1)
-	}
-
-	// LAYOUT
+	// LAYOUT (comes before RANGE in EXPLAIN output)
 	if n.Layout != nil {
 		explainDictionaryLayout(sb, n.Layout, indent+" ", depth+1)
 	}
 
+	// RANGE
+	if n.Range != nil {
+		explainDictionaryRange(sb, n.Range, indent+" ", depth+1)
+	}
+
 	// SETTINGS
 	if len(n.Settings) > 0 {
 		fmt.Fprintf(sb, "%s Set\n", indent)

internal/explain/dictionary.go

@@ -155,6 +155,12 @@ func Node(sb *strings.Builder, node interface{}, depth int) {
 		fmt.Fprintf(sb, "%sCreateSettingsProfileQuery\n", indent)
 	case *ast.DropSettingsProfileQuery:
 		fmt.Fprintf(sb, "%sDROP SETTINGS PROFILE query\n", indent)
+	case *ast.CreateNamedCollectionQuery:
+		fmt.Fprintf(sb, "%sCreateNamedCollectionQuery\n", indent)
+	case *ast.AlterNamedCollectionQuery:
+		fmt.Fprintf(sb, "%sAlterNamedCollectionQuery\n", indent)
+	case *ast.DropNamedCollectionQuery:
+		fmt.Fprintf(sb, "%sDropNamedCollectionQuery\n", indent)
 	case *ast.ShowCreateSettingsProfileQuery:
 		// Use PROFILES (plural) when multiple profiles are specified
 		queryName := "SHOW CREATE SETTINGS PROFILE query"
@@ -334,6 +340,9 @@ func Column(sb *strings.Builder, col *ast.ColumnDeclaration, depth int) {
 	if len(col.Settings) > 0 {
 		children++
 	}
+	if col.Comment != "" {
+		children++
+	}
 	if children > 0 {
 		fmt.Fprintf(sb, "%sColumnDeclaration %s (children %d)\n", indent, col.Name, children)
 	} else {
@@ -360,6 +369,9 @@ func Column(sb *strings.Builder, col *ast.ColumnDeclaration, depth int) {
 	if len(col.Settings) > 0 {
 		fmt.Fprintf(sb, "%s Set\n", indent)
 	}
+	if col.Comment != "" {
+		fmt.Fprintf(sb, "%s Literal \\'%s\\'\n", indent, col.Comment)
+	}
 }
 
 // explainCodecExpr handles CODEC expressions in column declarations

internal/explain/explain.go

@@ -2,6 +2,7 @@ package explain
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/sqlc-dev/doubleclick/ast"
@@ -402,8 +403,9 @@ func collectLogicalOperands(n *ast.BinaryExpr) []ast.Expression {
 
 func explainUnaryExpr(sb *strings.Builder, n *ast.UnaryExpr, indent string, depth int) {
 	// Handle negate of literal numbers - output as negative literal instead of function
+	// BUT only if the literal is NOT parenthesized (e.g., -1 folds, but -(1) stays as negate function)
 	if n.Op == "-" {
-		if lit, ok := n.Operand.(*ast.Literal); ok {
+		if lit, ok := n.Operand.(*ast.Literal); ok && !lit.Parenthesized {
 			switch lit.Type {
 			case ast.LiteralInteger:
 				// Convert positive integer to negative
@@ -433,6 +435,19 @@ func explainUnaryExpr(sb *strings.Builder, n *ast.UnaryExpr, indent string, dept
 				s := FormatFloat(-val)
 				fmt.Fprintf(sb, "%sLiteral Float64_%s\n", indent, s)
 				return
+			case ast.LiteralString:
+				// Handle BigInt - very large numbers stored as strings
+				// ClickHouse converts these to Float64 in scientific notation
+				if lit.IsBigInt {
+					if strVal, ok := lit.Value.(string); ok {
+						// Parse the string as float64 and negate it
+						if f, err := strconv.ParseFloat(strVal, 64); err == nil {
+							s := FormatFloat(-f)
+							fmt.Fprintf(sb, "%sLiteral Float64_%s\n", indent, s)
+							return
+						}
+					}
+				}
 			}
 		}
 	}
@@ -477,8 +492,13 @@ func explainAliasedExpr(sb *strings.Builder, n *ast.AliasedExpr, depth int) {
 						needsFunctionFormat = true
 						break
 					}
-					// Also check if nested arrays/tuples contain non-literal elements
+					// Check if tuple contains array literals - these need Function tuple format
 					if lit, ok := expr.(*ast.Literal); ok {
+						if lit.Type == ast.LiteralArray {
+							needsFunctionFormat = true
+							break
+						}
+						// Also check if nested arrays/tuples contain non-literal elements
 						if containsNonLiteralInNested(lit) {
 							needsFunctionFormat = true
 							break

internal/explain/expressions.go

@@ -160,9 +160,22 @@ func formatArrayLiteral(val interface{}) string {
 				if lit.Type == ast.LiteralInteger {
 					switch val := lit.Value.(type) {
 					case int64:
-						parts = append(parts, fmt.Sprintf("Int64_%d", -val))
+						negVal := -val
+						// ClickHouse normalizes -0 to UInt64_0
+						if negVal == 0 {
+							parts = append(parts, "UInt64_0")
+						} else if negVal > 0 {
+							parts = append(parts, fmt.Sprintf("UInt64_%d", negVal))
+						} else {
+							parts = append(parts, fmt.Sprintf("Int64_%d", negVal))
+						}
 					case uint64:
-						parts = append(parts, fmt.Sprintf("Int64_-%d", val))
+						// ClickHouse normalizes -0 to UInt64_0
+						if val == 0 {
+							parts = append(parts, "UInt64_0")
+						} else {
+							parts = append(parts, fmt.Sprintf("Int64_-%d", val))
+						}
 					default:
 						parts = append(parts, fmt.Sprintf("Int64_-%v", lit.Value))
 					}
@@ -195,8 +208,19 @@ func formatNumericExpr(e ast.Expression) (string, bool) {
 		if lit, ok := unary.Operand.(*ast.Literal); ok {
 			switch val := lit.Value.(type) {
 			case int64:
-				return fmt.Sprintf("Int64_%d", -val), true
+				negVal := -val
+				// ClickHouse normalizes -0 to UInt64_0
+				if negVal == 0 {
+					return "UInt64_0", true
+				} else if negVal > 0 {
+					return fmt.Sprintf("UInt64_%d", negVal), true
+				}
+				return fmt.Sprintf("Int64_%d", negVal), true
 			case uint64:
+				// ClickHouse normalizes -0 to UInt64_0
+				if val == 0 {
+					return "UInt64_0", true
+				}
 				return fmt.Sprintf("Int64_%d", -int64(val)), true
 			case float64:
 				return fmt.Sprintf("Float64_%s", FormatFloat(-val)), true
@@ -289,6 +313,13 @@ func FormatDataType(dt *ast.DataType) string {
 		} else if ident, ok := p.(*ast.Identifier); ok {
 			// Identifier (e.g., function name in AggregateFunction types)
 			params = append(params, ident.Name())
+		} else if unary, ok := p.(*ast.UnaryExpr); ok {
+			// Unary expression (e.g., -1 for negative numbers)
+			if lit, ok := unary.Operand.(*ast.Literal); ok {
+				params = append(params, fmt.Sprintf("%s%v", unary.Op, lit.Value))
+			} else {
+				params = append(params, fmt.Sprintf("%v", p))
+			}
 		} else {
 			params = append(params, fmt.Sprintf("%v", p))
 		}
@@ -469,7 +500,7 @@ func formatExprAsString(expr ast.Expression) string {
 		case ast.LiteralNull:
 			return "NULL"
 		case ast.LiteralArray:
-			return formatArrayAsString(e.Value)
+			return formatArrayAsStringFromLiteral(e)
 		case ast.LiteralTuple:
 			return formatTupleAsString(e.Value)
 		default:
@@ -519,6 +550,28 @@ func formatExprAsString(expr ast.Expression) string {
 	}
 }
 
+// formatArrayAsStringFromLiteral formats an array literal as a string for :: cast syntax
+// It preserves original spacing from the source
+func formatArrayAsStringFromLiteral(lit *ast.Literal) string {
+	exprs, ok := lit.Value.([]ast.Expression)
+	if !ok {
+		return "[]"
+	}
+	var parts []string
+	for _, e := range exprs {
+		parts = append(parts, formatElementAsString(e))
+	}
+	separator := ","
+	if lit.SpacedCommas {
+		separator = ", "
+	}
+	// Use outer spaces when source had whitespace after [ (e.g., for multi-line arrays)
+	if lit.SpacedBrackets {
+		return "[ " + strings.Join(parts, separator) + " ]"
+	}
+	return "[" + strings.Join(parts, separator) + "]"
+}
+
 // formatArrayAsString formats an array literal as a string for :: cast syntax
 func formatArrayAsString(val interface{}) string {
 	exprs, ok := val.([]ast.Expression)
@@ -555,9 +608,14 @@ func formatElementAsString(expr ast.Expression) string {
 		case ast.LiteralFloat:
 			return fmt.Sprintf("%v", e.Value)
 		case ast.LiteralString:
+			s := e.Value.(string)
+			// Check if this is a big integer stored as string (too large for int64/uint64)
+			// These should NOT be quoted when formatted in arrays
+			if e.IsBigInt {
+				return s
+			}
 			// Quote strings with single quotes, triple-escape for nested context
 			// Expected output format is \\\' (three backslashes + quote)
-			s := e.Value.(string)
 			// Triple-escape single quotes for nested string literal context
 			s = strings.ReplaceAll(s, "'", "\\\\\\'")
 			return "\\\\\\'" + s + "\\\\\\'"
@@ -569,7 +627,7 @@ func formatElementAsString(expr ast.Expression) string {
 		case ast.LiteralNull:
 			return "NULL"
 		case ast.LiteralArray:
-			return formatArrayAsString(e.Value)
+			return formatArrayAsStringFromLiteral(e)
 		case ast.LiteralTuple:
 			return formatTupleAsString(e.Value)
 		default:

internal/explain/format.go

@@ -115,13 +115,35 @@ func explainFunctionCallWithAlias(sb *strings.Builder, n *ast.FunctionCall, alia
 	if n.Distinct {
 		fnName = fnName + "Distinct"
 	}
+	// Append "If" if the function has a FILTER clause
+	if n.Filter != nil {
+		fnName = fnName + "If"
+	}
 	if alias != "" {
 		fmt.Fprintf(sb, "%sFunction %s (alias %s) (children %d)\n", indent, fnName, alias, children)
 	} else {
 		fmt.Fprintf(sb, "%sFunction %s (children %d)\n", indent, fnName, children)
 	}
 	// Arguments (Settings are included as part of argument count)
-	argCount := len(n.Arguments)
+	// FILTER condition is appended to arguments for -If suffix functions
+	// count(name) FILTER (WHERE cond) -> countIf(name, cond) - 2 args
+	// count(*) FILTER (WHERE cond) -> countIf(cond) - 1 arg (asterisk dropped)
+	var argCount int
+	filterArgs := n.Arguments
+	if n.Filter != nil {
+		// Filter condition is appended as an extra argument
+		// But first, remove any Asterisk arguments (count(*) case)
+		var nonAsteriskArgs []ast.Expression
+		for _, arg := range n.Arguments {
+			if _, isAsterisk := arg.(*ast.Asterisk); !isAsterisk {
+				nonAsteriskArgs = append(nonAsteriskArgs, arg)
+			}
+		}
+		filterArgs = nonAsteriskArgs
+		argCount = len(filterArgs) + 1 // +1 for filter condition
+	} else {
+		argCount = len(n.Arguments)
+	}
 	if len(n.Settings) > 0 {
 		argCount++ // Set is counted as one argument
 	}
@@ -130,7 +152,12 @@ func explainFunctionCallWithAlias(sb *strings.Builder, n *ast.FunctionCall, alia
 		fmt.Fprintf(sb, " (children %d)", argCount)
 	}
 	fmt.Fprintln(sb)
-	for _, arg := range n.Arguments {
+	// Output arguments (filterArgs excludes Asterisk when FILTER is present)
+	argsToOutput := filterArgs
+	if n.Filter == nil {
+		argsToOutput = n.Arguments
+	}
+	for _, arg := range argsToOutput {
 		// For view() table function, unwrap Subquery wrapper
 		// Also reset the subquery context since view() SELECT is not in a Subquery node
 		if strings.ToLower(n.Name) == "view" {
@@ -144,6 +171,10 @@ func explainFunctionCallWithAlias(sb *strings.Builder, n *ast.FunctionCall, alia
 		}
 		Node(sb, arg, depth+2)
 	}
+	// Append filter condition at the end
+	if n.Filter != nil {
+		Node(sb, n.Filter, depth+2)
+	}
 	// Settings appear as Set node inside ExpressionList
 	if len(n.Settings) > 0 {
 		fmt.Fprintf(sb, "%s  Set\n", indent)
@@ -567,8 +598,8 @@ func explainCastExprWithAlias(sb *strings.Builder, n *ast.CastExpr, alias string
 			if lit.Type == ast.LiteralArray || lit.Type == ast.LiteralTuple {
 				if useArrayFormat {
 					fmt.Fprintf(sb, "%s  Literal %s\n", indent, FormatLiteral(lit))
-				} else if containsCastExpressions(lit) {
-					// Array contains CastExpr elements - output as Function array with children
+				} else if containsCastExpressions(lit) || !containsOnlyLiterals(lit) {
+					// Array contains CastExpr or non-literal elements - output as Function array with children
 					Node(sb, n.Expr, depth+2)
 				} else {
 					// Simple literals (including negative numbers) - format as string
@@ -738,6 +769,7 @@ func containsCastExpressions(lit *ast.Literal) bool {
 }
 
 // containsOnlyLiterals checks if a literal array/tuple contains only literal values (no expressions)
+// This includes negated literals (UnaryExpr with Op="-" and Literal operand)
 func containsOnlyLiterals(lit *ast.Literal) bool {
 	var exprs []ast.Expression
 	switch lit.Type {
@@ -752,16 +784,24 @@ func containsOnlyLiterals(lit *ast.Literal) bool {
 	}
 
 	for _, e := range exprs {
-		innerLit, ok := e.(*ast.Literal)
-		if !ok {
-			return false
+		// Check if it's a direct literal
+		if innerLit, ok := e.(*ast.Literal); ok {
+			// Nested arrays/tuples need recursive check
+			if innerLit.Type == ast.LiteralArray || innerLit.Type == ast.LiteralTuple {
+				if !containsOnlyLiterals(innerLit) {
+					return false
+				}
+			}
+			continue
 		}
-		// Nested arrays/tuples need recursive check
-		if innerLit.Type == ast.LiteralArray || innerLit.Type == ast.LiteralTuple {
-			if !containsOnlyLiterals(innerLit) {
-				return false
+		// Check if it's a negated literal (e.g., -1)
+		if unary, ok := e.(*ast.UnaryExpr); ok && unary.Op == "-" {
+			if _, isLit := unary.Operand.(*ast.Literal); isLit {
+				continue
 			}
 		}
+		// Not a literal or negated literal
+		return false
 	}
 	return true
 }
@@ -986,10 +1026,11 @@ func explainInExpr(sb *strings.Builder, n *ast.InExpr, indent string, depth int)
 					// Check if this tuple contains only primitive literals (including unary negation)
 					if !containsOnlyPrimitiveLiteralsWithUnary(lit) {
 						allTuplesArePrimitive = false
+						allPrimitiveLiterals = false // Non-primitive tuple breaks the mixed literal check too
 					}
 				}
-				// Check if it's a primitive literal type (not a tuple or complex type)
-				if lit.Type == ast.LiteralTuple || lit.Type == ast.LiteralArray {
+				// Arrays break the primitive literals check
+				if lit.Type == ast.LiteralArray {
 					allPrimitiveLiterals = false
 				}
 			} else if isNumericExpr(item) {
@@ -1133,7 +1174,8 @@ func explainInExprWithAlias(sb *strings.Builder, n *ast.InExpr, alias string, in
 		allBooleansOrNull := true
 		allTuples := true
 		allTuplesArePrimitive := true
-		hasNonNull := false // Need at least one non-null value
+		allPrimitiveLiterals := true // Any mix of primitive literals (numbers, strings, booleans, null, primitive tuples)
+		hasNonNull := false          // Need at least one non-null value
 		for _, item := range n.List {
 			if lit, ok := item.(*ast.Literal); ok {
 				if lit.Type == ast.LiteralNull {
@@ -1155,6 +1197,7 @@ func explainInExprWithAlias(sb *strings.Builder, n *ast.InExpr, alias string, in
 				} else {
 					if !containsOnlyPrimitiveLiterals(lit) {
 						allTuplesArePrimitive = false
+						allPrimitiveLiterals = false
 					}
 				}
 			} else if isNumericExpr(item) {
@@ -1167,10 +1210,11 @@ func explainInExprWithAlias(sb *strings.Builder, n *ast.InExpr, alias string, in
 				allStringsOrNull = false
 				allBooleansOrNull = false
 				allTuples = false
+				allPrimitiveLiterals = false
 				break
 			}
 		}
-		canBeTupleLiteral = hasNonNull && (allNumericOrNull || (allStringsOrNull && len(n.List) <= maxStringTupleSizeWithAlias) || allBooleansOrNull || (allTuples && allTuplesArePrimitive))
+		canBeTupleLiteral = hasNonNull && (allNumericOrNull || (allStringsOrNull && len(n.List) <= maxStringTupleSizeWithAlias) || allBooleansOrNull || (allTuples && allTuplesArePrimitive) || allPrimitiveLiterals)
 	}
 
 	// Count arguments