From be74b991c9739662cce7ab8968f3922892e52364 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Tue, 25 Aug 2026 18:28:15 +0300 Subject: [PATCH] Keep no-match rows when pulling up a correlated aggregate subquery With the Postgres planner (optimizer=off or an ORCA fallback), a correlated scalar subquery with an aggregate is pulled up into an INNER join with a grouped subquery (convert_EXPR_to_join), which drops outer rows that have no match. The original subquery keeps them: it computes the aggregate over empty input, so e.g. COUNT yields 0 there: select ... from t1 where t1.a > (select count(*) from t2 where t2.a = t1.d); A row with no match in t2 must be compared as "t1.a > 0" and can pass, but the INNER join dropped it. To fix this, pull the subquery up into a LEFT join, so no-match rows survive as null-extended rows, and rewrite the comparison to return the same value the subquery would: outer OP CASE WHEN match_flag THEN expr ELSE empty_input_default END match_flag is a constant TRUE column added to the subquery. For a matched row the CASE returns the real expression; for a null-extended row the flag is NULL and the CASE returns the empty-input default (0 for COUNT, NULL for other aggregates). The comparison runs above the LEFT join as a filter, not as the join condition: as a join qual it would null-extend matched rows that fail it, and the default would let them back in. The LEFT join is not always needed. If a no-match row cannot pass the comparison anyway -- e.g. "1 = (select count(*) ...)" turns into "1 = 0" for it -- dropping it is fine and the INNER join is kept as before. This is detected by substituting the empty-input default into the comparison and constant-folding it. Ordinary sum/avg/min/max comparisons fall into this group: their empty-input value is NULL, and a comparison with NULL does not pass, so those plans do not change. If the comparison cannot be placed above the join (the sublink is in an outer join's ON clause) or the subquery's targetlist is correlated, the pull-up bails out and the sublink runs as a SubPlan, as before. Ported from open-gpdb/gpdb#397 with two PostgreSQL 16 adaptations: - count(*) / count(any) are recognized via the F_COUNT_ / F_COUNT_ANY macros from fmgroids.h: genbki no longer allows custom OID symbols for pg_proc entries. - Since the removal of outerjoin_delayed, a clause is kept above an outer join only when its Vars carry the join's relid in varnullingrels; otherwise the comparison would be pushed down to the subquery rel and the LEFT join removed as useless, so the no-match default would never apply. Give the planner-built LEFT join a real RTE_JOIN range-table entry and mark the Vars used by the comparison as nulled by it. Co-Authored-By: excaliiibur --- src/backend/cdb/cdbsubselect.c | 223 +++++++++++++++++- src/backend/optimizer/prep/prepjointree.c | 28 +++ src/test/regress/expected/subselect_gp.out | 19 ++ .../expected/subselect_gp_optimizer.out | 19 ++ src/test/regress/sql/subselect_gp.sql | 15 ++ 5 files changed, 302 insertions(+), 2 deletions(-) diff --git a/src/backend/cdb/cdbsubselect.c b/src/backend/cdb/cdbsubselect.c index 4813439de0f..31d1cf13050 100644 --- a/src/backend/cdb/cdbsubselect.c +++ b/src/backend/cdb/cdbsubselect.c @@ -29,6 +29,7 @@ #include "parser/parse_relation.h" /* addRangeTableEntryForSubquery() */ #include "parser/parsetree.h" /* rt_fetch() */ #include "rewrite/rewriteManip.h" +#include "utils/fmgroids.h" /* F_COUNT_ANY, F_COUNT_ */ #include "utils/lsyscache.h" /* get_op_btree_interpretation() */ #include "utils/syscache.h" #include "cdb/cdbsubselect.h" /* me */ @@ -42,6 +43,11 @@ static JoinExpr *make_join_expr(Node *larg, int r_rtindex, int join_type); static Node *make_lasj_quals(PlannerInfo *root, SubLink *sublink, int subquery_indx); static Node *add_null_match_clause(Node *clause); +static Expr *build_match_flag_case_expr(Var *flagVar, Var *aggVar, Expr *defaultExpr); +static Expr *build_empty_input_default_expr(Node *expr); +static Node *replace_agg_with_empty_default_mutator(Node *node, void *context); +static bool no_match_row_survives(PlannerInfo *root, OpExpr *opexp, + Expr *defaultExpr); typedef struct NonNullableVarsContext { @@ -538,6 +544,16 @@ safe_to_convert_EXPR(SubLink *sublink, ConvertSubqueryToJoinContext *ctx1) if (!subselect->hasAggs) return false; + /** + * A window function is evaluated after aggregation, over the aggregate's + * single-row result, so no join row can carry the value a no-match row + * would see, and the empty-input default of the expression cannot be + * computed outside the subquery either (a WindowFunc is only valid + * inside its own WindowAgg node). Let it run as a SubPlan. + */ + if (subselect->hasWindowFuncs) + return false; + /** * A LIMIT or OFFSET could interfere with the transformation of the * correlated qual to GROUP BY. (LIMIT >0 in a subquery that contains a @@ -560,6 +576,14 @@ safe_to_convert_EXPR(SubLink *sublink, ConvertSubqueryToJoinContext *ctx1) if (list_length(subselect->targetList) != 1) return false; + /** + * Correlation in the targetlist cannot be handled: the pulled-up + * expression (and the empty-input default derived from it) would carry + * upper-level Vars out of the subquery. + */ + if (contain_vars_of_level_or_above((Node *) subselect->targetList, 1)) + return false; + /** * Walk the quals of the subquery to do a more fine grained check as to whether this subquery @@ -623,6 +647,51 @@ convert_EXPR_to_join(PlannerInfo *root, OpExpr *opexp) subselect->jointree->quals = ctx1.innerQual; + /* + * An INNER join drops outer rows that have no matching inner + * rows. Without the pull-up they are kept: the subquery + * computes its expression over empty input (COUNT = 0, other + * aggregates NULL) and the comparison may still pass. + * + * So plug the empty-input value into the comparison and run + * eval_const_expressions() on it. FALSE or NULL means no-match + * rows cannot pass and the INNER join is correct; otherwise use + * a LEFT join to keep them. + */ + Expr *defaultExpr; + TargetEntry *flagTLE = NULL; + bool use_left_join; + + defaultExpr = build_empty_input_default_expr((Node *) origSubqueryTLE->expr); + use_left_join = no_match_row_survives(root, opexp, defaultExpr); + + if (use_left_join) + { + /* + * After the LEFT join the expression column is NULL both for a + * no-match row and for a matched group whose expression is + * genuinely NULL. To tell them apart, add a constant-TRUE + * match-flag column to the subquery: it can be NULL only when + * the LEFT join found no match and filled the subquery's + * columns with NULLs. + * + * The flag goes BEFORE the expression column: with this + * order the planner can drop the SubqueryScan node from the + * plan. + */ + TargetEntry *aggTLE = (TargetEntry *) llast(subselect->targetList); + + flagTLE = makeTargetEntry((Expr *) makeBoolConst(true, false), + aggTLE->resno, + pstrdup("csq_count_flag"), + false); + aggTLE->resno++; + subselect->targetList = list_truncate(subselect->targetList, + list_length(subselect->targetList) - 1); + subselect->targetList = lappend(subselect->targetList, flagTLE); + subselect->targetList = lappend(subselect->targetList, aggTLE); + } + /** * Construct a new range table entry for the new pulled up subquery. */ @@ -644,7 +713,8 @@ convert_EXPR_to_join(PlannerInfo *root, OpExpr *opexp) join_expr->quals = joinQual; - TargetEntry *subselectAggTLE = (TargetEntry *) list_nth(subselect->targetList, list_length(subselect->targetList) - 1); + /* The pulled-up expression column is last in either layout. */ + TargetEntry *subselectAggTLE = (TargetEntry *) llast(subselect->targetList); /** * modify the op expr to involve the column that has the computed aggregate that needs to compared. @@ -656,7 +726,44 @@ convert_EXPR_to_join(PlannerInfo *root, OpExpr *opexp) exprCollation((Node *) subselectAggTLE->expr), 0); - list_nth_replace(opexp->args, 1, aggVar); + if (use_left_join) + { + Var *flagVar; + RangeTblEntry *joinRTE; + int joinRTIndex; + + join_expr->jointype = JOIN_LEFT; + + /* + * Give the outer join a range table entry and mark the Vars the + * comparison uses as nulled by it. Since the removal of + * outerjoin_delayed the planner keeps a clause above an outer + * join only when the clause's Vars carry the join's relid in + * varnullingrels; without this the comparison would be pushed + * down to the subquery rel (or the join removed as useless) and + * the no-match default would never apply. + */ + joinRTE = makeNode(RangeTblEntry); + joinRTE->rtekind = RTE_JOIN; + joinRTE->jointype = JOIN_LEFT; + joinRTE->joinmergedcols = 0; + joinRTE->eref = makeAlias("unnamed_join", NIL); + joinRTE->inFromCl = false; + root->parse->rtable = lappend(root->parse->rtable, joinRTE); + joinRTIndex = list_length(root->parse->rtable); + join_expr->rtindex = joinRTIndex; + + flagVar = (Var *) makeVar(rteIndex, flagTLE->resno, BOOLOID, -1, + InvalidOid, 0); + flagVar->varnullingrels = bms_make_singleton(joinRTIndex); + aggVar->varnullingrels = bms_make_singleton(joinRTIndex); + list_nth_replace(opexp->args, 1, + build_match_flag_case_expr(flagVar, aggVar, defaultExpr)); + } + else + { + list_nth_replace(opexp->args, 1, aggVar); + } return join_expr; } @@ -664,6 +771,118 @@ convert_EXPR_to_join(PlannerInfo *root, OpExpr *opexp) return NULL; } +/* + * Build "CASE WHEN flagVar THEN aggVar ELSE defaultExpr END". + * + * flagVar is the subquery's match-flag column: TRUE for a matched group, + * NULL for a null-extended no-match row. + */ +static Expr * +build_match_flag_case_expr(Var *flagVar, Var *aggVar, Expr *defaultExpr) +{ + CaseWhen *casewhen; + CaseExpr *caseexpr; + + Assert(flagVar != NULL); + Assert(aggVar != NULL); + Assert(defaultExpr != NULL); + + casewhen = makeNode(CaseWhen); + casewhen->expr = (Expr *) flagVar; + casewhen->result = (Expr *) aggVar; + casewhen->location = -1; + + caseexpr = makeNode(CaseExpr); + caseexpr->casetype = exprType((Node *) aggVar); + caseexpr->casecollid = exprCollation((Node *) aggVar); + caseexpr->arg = NULL; + caseexpr->args = list_make1(casewhen); + caseexpr->defresult = defaultExpr; + caseexpr->location = -1; + + return (Expr *) caseexpr; +} + +static Expr * +build_empty_input_default_expr(Node *expr) +{ + Node *rewritten; + + rewritten = replace_agg_with_empty_default_mutator(copyObject(expr), NULL); + return (Expr *) rewritten; +} + +static Node * +replace_agg_with_empty_default_mutator(Node *node, void *context) +{ + Aggref *aggref; + Oid default_type; + Oid default_collation; + int16 typlen; + bool typbyval; + + if (node == NULL) + return NULL; + + if (IsA(node, Aggref)) + { + bool is_count; + + aggref = (Aggref *) node; + is_count = (aggref->aggfnoid == F_COUNT_ANY || + aggref->aggfnoid == F_COUNT_); + if (is_count) + { + default_type = INT8OID; + default_collation = InvalidOid; + } + else + { + default_type = aggref->aggtype; + default_collation = exprCollation((Node *) aggref); + } + + /* + * COUNT is 0 over empty input; every other aggregate is NULL. The + * choice must follow the aggregate, not its result type: sum(int4) + * also returns int8 but its empty-input value is NULL. + */ + get_typlenbyval(default_type, &typlen, &typbyval); + return (Node *) makeConst(default_type, -1, default_collation, typlen, + is_count ? Int64GetDatum(0) : (Datum) 0, + !is_count, typbyval); + } + + return expression_tree_mutator(node, replace_agg_with_empty_default_mutator, + context); +} + +/* + * no_match_row_survives + * + * Could a no-match row satisfy "outerExpr OP (subquery)"? Plug defaultExpr in + * for the subquery and constant-fold: false if it folds to FALSE/NULL, else true. + */ +static bool +no_match_row_survives(PlannerInfo *root, OpExpr *opexp, Expr *defaultExpr) +{ + OpExpr *testexpr = (OpExpr *) copyObject(opexp); + Node *folded; + + list_nth_replace(testexpr->args, 1, copyObject(defaultExpr)); + folded = eval_const_expressions(root, (Node *) testexpr); + + if (IsA(folded, Const)) + { + Const *c = (Const *) folded; + + if (c->constisnull || !DatumGetBool(c->constvalue)) + return false; + } + + return true; +} + /* NOTIN subquery transformation -start */ /* check if NOT IN conversion to antijoin is possible */ diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index dde7402c572..139349c40f8 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -858,11 +858,39 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, if (IsA(rarg, SubLink)) { + /* + * The pulled-up join is spliced in at *jtlink1, and in the + * LEFT-join case the comparison itself moves there too, so + * every Var of this query level used by the clause must be + * available at that attach point. Otherwise (e.g. an outer + * join's ON clause referencing the non-nullable side) leave + * the sublink to be planned as a SubPlan. + */ + if (!bms_is_subset(pull_varnos(root, node), available_rels1)) + return node; + j = convert_EXPR_to_join(root, opexp); if (j) { /* Yes, insert the new join node into the join tree */ j->larg = *jtlink1; + + if (j->jointype == JOIN_LEFT) + { + /* + * COUNT-preserving pull-up (see convert_EXPR_to_join). + * opexp must run ABOVE the LEFT JOIN, not as its join + * condition: as a join qual a matched row that fails it + * would be treated as unmatched, null-extended, and let + * back in by the no-match default of the CASE built by + * convert_EXPR_to_join. Wrap the join in a FromExpr so + * opexp stays a post-join filter. + */ + *jtlink1 = (Node *) makeFromExpr(list_make1(j), node); + return NULL; + } + + /* Inner-join case: opexp stays as an ordinary qual. */ *jtlink1 = (Node *) j; } return node; diff --git a/src/test/regress/expected/subselect_gp.out b/src/test/regress/expected/subselect_gp.out index 3e4ea0d16c9..36b86b0d93d 100644 --- a/src/test/regress/expected/subselect_gp.out +++ b/src/test/regress/expected/subselect_gp.out @@ -4015,3 +4015,22 @@ reset optimizer; drop table outer_foo; drop table inner_bar; drop table t; + +-- A window function in a correlated aggregate subquery: the pull-up must +-- bail out, because no join row carries the value a no-match row would see +-- and a WindowFunc cannot be evaluated outside the subquery. The sublink +-- runs as a SubPlan. +create table t_csq_win_out(a int, b int) distributed by (a); +create table t_csq_win_in(a int) distributed by (a); +insert into t_csq_win_out values (1, 5); +set optimizer=off; +-- count()=0 over empty input, then count() over () = 1 over the single +-- aggregate row: "1 > 0 + 1" is false and no row may be returned +select * from t_csq_win_out +where a > (select count(*) + count(*) over () from t_csq_win_in where t_csq_win_in.a = t_csq_win_out.b); + a | b +---+--- +(0 rows) + +reset optimizer; +drop table t_csq_win_out, t_csq_win_in; diff --git a/src/test/regress/expected/subselect_gp_optimizer.out b/src/test/regress/expected/subselect_gp_optimizer.out index c847505320f..f76a04cdfb0 100644 --- a/src/test/regress/expected/subselect_gp_optimizer.out +++ b/src/test/regress/expected/subselect_gp_optimizer.out @@ -4153,3 +4153,22 @@ reset optimizer; drop table outer_foo; drop table inner_bar; drop table t; + +-- A window function in a correlated aggregate subquery: the pull-up must +-- bail out, because no join row carries the value a no-match row would see +-- and a WindowFunc cannot be evaluated outside the subquery. The sublink +-- runs as a SubPlan. +create table t_csq_win_out(a int, b int) distributed by (a); +create table t_csq_win_in(a int) distributed by (a); +insert into t_csq_win_out values (1, 5); +set optimizer=off; +-- count()=0 over empty input, then count() over () = 1 over the single +-- aggregate row: "1 > 0 + 1" is false and no row may be returned +select * from t_csq_win_out +where a > (select count(*) + count(*) over () from t_csq_win_in where t_csq_win_in.a = t_csq_win_out.b); + a | b +---+--- +(0 rows) + +reset optimizer; +drop table t_csq_win_out, t_csq_win_in; diff --git a/src/test/regress/sql/subselect_gp.sql b/src/test/regress/sql/subselect_gp.sql index d8edea7d4e0..1a49184e392 100644 --- a/src/test/regress/sql/subselect_gp.sql +++ b/src/test/regress/sql/subselect_gp.sql @@ -1515,3 +1515,18 @@ reset optimizer; drop table outer_foo; drop table inner_bar; drop table t; + +-- A window function in a correlated aggregate subquery: the pull-up must +-- bail out, because no join row carries the value a no-match row would see +-- and a WindowFunc cannot be evaluated outside the subquery. The sublink +-- runs as a SubPlan. +create table t_csq_win_out(a int, b int) distributed by (a); +create table t_csq_win_in(a int) distributed by (a); +insert into t_csq_win_out values (1, 5); +set optimizer=off; +-- count()=0 over empty input, then count() over () = 1 over the single +-- aggregate row: "1 > 0 + 1" is false and no row may be returned +select * from t_csq_win_out +where a > (select count(*) + count(*) over () from t_csq_win_in where t_csq_win_in.a = t_csq_win_out.b); +reset optimizer; +drop table t_csq_win_out, t_csq_win_in;