From ab50793fe034a394d3ee5d8128a61cb3ceaca57a Mon Sep 17 00:00:00 2001 From: Qingqing Zhou Date: Tue, 14 Apr 2026 15:35:30 -0700 Subject: [PATCH] Realize MarkJoin as HashJoin for IN/NOT IN subqueries (issue #272) Replace nested-loop execution with hash-based execution for IN/NOT IN derived MarkJoin. Hash on correlation (residual) keys, then evaluate the marker predicate per-row for three-valued NULL logic. Falls back to nested loop when no hashable residual keys exist. EXISTS/NOT EXISTS MarkJoin continues to use nested loop. Co-Authored-By: Claude Opus 4.6 --- qpmodel/subquery.cs | 262 +++++++++++++++++------ test/regress/expect/subqueryd_hashmj.txt | 68 ++++++ test/regress/expect/subqueryd_or.txt | 4 +- test/regress/sql/subqueryd_hashmj.sql | 22 ++ 4 files changed, 292 insertions(+), 64 deletions(-) create mode 100644 test/regress/expect/subqueryd_hashmj.txt create mode 100644 test/regress/sql/subqueryd_hashmj.sql diff --git a/qpmodel/subquery.cs b/qpmodel/subquery.cs index be74220..61a4b0b 100644 --- a/qpmodel/subquery.cs +++ b/qpmodel/subquery.cs @@ -1712,95 +1712,233 @@ int findMarkerOrdinal() public override void Exec(Action callback) { var isDerivedFromInClause = filterHasMarkerBinExpr(logic_.filter_); - Exec(callback, isDerivedFromInClause); + if (isDerivedFromInClause) + ExecHash(callback); + else + ExecNested(callback); } - public void Exec(Action callback, bool isDerivedFromInClause) + + // Hash-based execution for IN/NOT IN derived MarkJoin (issue #272). + // + // The filter has two parts: + // - Marker BinExpr: the IN-list equality (e.g., a2=b2) + // - Residual: the correlation predicate (e.g., b1=a1) + // + // We hash the RIGHT side on the RESIDUAL equi-join keys (the correlation + // condition), then for each left row probe the hash table and evaluate + // the marker predicate for three-valued IN/NOT IN logic. + void ExecHash(Action callback) { ExecContext context = context_; var logic = logic_ as LogicMarkJoin; var filter = logic.filter_; bool semi = (logic_ is LogicMarkSemiJoin); - bool antisemi = (logic_ is LogicMarkAntiSemiJoin); - bool lIsNull = false; // l represent one row - bool RHasNull = false; // R represent a set of Row - bool RisEmpty = true; - Value marker = false; //false true null - int markerOrdinal = 0; - Debug.Assert(filter != null); + // Decompose the filter into marker predicate and residual. + var andList = filter.FilterToAndList(); + var markerExpr = andList.Find(x => x is BinExpr xB && xB.IsMarkerBinExpr()) as BinExpr; + Debug.Assert(markerExpr != null); + andList.Remove(markerExpr); + + // Extract equi-join keys from residual predicates for hashing. + var ltabrefs = lchild_().logic_.InclusiveTableRefs(); + var leftKeys = new List(); + var rightKeys = new List(); + var nonEquiResidual = new List(); + + foreach (var pred in andList) + { + if (pred is BinExpr be && be.op_ == "=") + { + var lrefs = be.lchild_().tableRefs_; + var rrefs = be.rchild_().tableRefs_; + bool lOnLeft = lrefs != null && !lrefs.Except(ltabrefs).Any(); + bool rOnLeft = rrefs != null && !rrefs.Except(ltabrefs).Any(); + + if (lOnLeft && !rOnLeft) + { + leftKeys.Add(be.lchild_()); + rightKeys.Add(be.rchild_()); + continue; + } + else if (rOnLeft && !lOnLeft) + { + leftKeys.Add(be.rchild_()); + rightKeys.Add(be.lchild_()); + continue; + } + } + nonEquiResidual.Add(pred); + } + + Expr nonEquiFilter = nonEquiResidual.Count > 0 ? FilterHelper.AndListToExpr(nonEquiResidual) : null; + int lColCount = lchild_().logic_.output_.Count; + int rColCount = rchild_().logic_.output_.Count; + + // If no hashable residual keys, fall back to nested loop. + if (leftKeys.Count == 0) + { + ExecNestedInClause(callback, markerExpr, andList.Count > 0 ? FilterHelper.AndListToExpr(andList) : null); + return; + } + + // Build hash table from right (subquery) side on residual keys. + var hm = new Dictionary>(); + + rchild_().Exec(r => + { + Row fakeLeft = new Row(lColCount); + Row combined = new Row(fakeLeft, r); + var keys = KeyList.ComputeKeys(context, rightKeys, combined); + if (keys.ColsHasNull()) + return; // NULL correlation keys never match + if (hm.TryGetValue(keys, out List existing)) + existing.Add(r); + else + hm.Add(keys, new List { r }); + }); + + // Probe with left (outer) side. lchild_().Exec(l => { - lIsNull = l.ColsHasNull(); - marker = false; - rchild_().Exec(r => + bool lIsNull = l.ColsHasNull(); + Value marker = false; + bool RHasNull = false; + bool RisEmpty = true; + + var keys = KeyList.ComputeKeys(context, leftKeys, l); + if (!keys.ColsHasNull() && hm.TryGetValue(keys, out List matches)) { - Row n = new Row(l, r); - if (isDerivedFromInClause) + // Hash hit on correlation keys — now evaluate marker and non-equi residual + foreach (var r in matches) { - if (!(r is null)) - RHasNull = r.ColsHasNull(); - var andList = filter.FilterToAndList(); - - // SELECT a1 FROM a WHERE a1 = 3 and a2 NOT IN (SELECT b2 FROM b WHERE a1 < b1); - // a1 < b1 will not produce marker - // a2 = b2 will produce marker - // if the markjoin is derived from IN clause, we need to judge if it is a empty - // - if (andList.Count >= 2) + Row combined = new Row(l, r); + + // Check non-equi residual first + if (nonEquiFilter != null) { - var markerExpr = andList.Find(x => x is BinExpr xB && xB.IsMarkerBinExpr()); - andList.Remove(markerExpr); - var excludeMarkerExpr = FilterHelper.AndListToExpr(andList); - var flagE = excludeMarkerExpr.Exec(context, n); + var flag = nonEquiFilter.Exec(context, combined); + if (!(flag is true)) + continue; + } - if (flagE is true) - RisEmpty = false; - else - return; + RisEmpty = false; + if (r.ColsHasNull()) + RHasNull = true; - // there is at least one match, mark true - if (markerExpr.Exec(context, n) is true) - marker = true; - } - else if (filter.Exec(context, n) is true) - marker = true; - } - else if (!(marker is true) && !isDerivedFromInClause) - { - if (filter.Exec(context, n) is true) + // Evaluate the marker (IN-list equality) predicate + if (markerExpr.Exec(context, combined) is true) { marker = true; - n = ExecProject(n); - fixMarkerValue(n, semi); - callback(n); + break; // One match is enough for IN } } - }); + } + + // Three-valued NULL logic (same as original nested loop): + if (marker is false && RHasNull) + marker = null; + if (lIsNull && RisEmpty) + marker = false; + + Row rr = new Row(rColCount); + Row n = new Row(l, rr); + n = ExecProject(n); - if (isDerivedFromInClause) + if (marker is null) + fixMarkerValue(n, false); + else { - if (marker is false && RHasNull) - marker = null; + bool boolMarker = marker is true; + fixMarkerValue(n, semi ? boolMarker : !boolMarker); + } + + callback(n); + }); + } - if (lIsNull && RisEmpty) - marker = false; + // Nested-loop fallback for IN-clause MarkJoin when no hashable keys found. + void ExecNestedInClause(Action callback, BinExpr markerExpr, Expr residualFilter) + { + ExecContext context = context_; + bool semi = (logic_ is LogicMarkSemiJoin); + int rColCount = rchild_().logic_.output_.Count; - Row r = new Row(rchild_().logic_.output_.Count); - Row n = new Row(l, r); - n = ExecProject(n); + lchild_().Exec(l => + { + bool lIsNull = l.ColsHasNull(); + Value marker = false; + bool RHasNull = false; + bool RisEmpty = true; - markerOrdinal = findMarkerOrdinal(); - if (marker is null) - fixMarkerValue(n, false); - else + rchild_().Exec(r => + { + Row combined = new Row(l, r); + + if (residualFilter != null) { - bool boolMarker = marker is true; - fixMarkerValue(n, semi ? boolMarker : !boolMarker); + var flag = residualFilter.Exec(context, combined); + if (!(flag is true)) + return; } - callback(n); + RisEmpty = false; + if (r.ColsHasNull()) + RHasNull = true; + + if (markerExpr.Exec(context, combined) is true) + marker = true; + }); + + if (marker is false && RHasNull) + marker = null; + if (lIsNull && RisEmpty) + marker = false; + + Row rr = new Row(rColCount); + Row n = new Row(l, rr); + n = ExecProject(n); + + if (marker is null) + fixMarkerValue(n, false); + else + { + bool boolMarker = marker is true; + fixMarkerValue(n, semi ? boolMarker : !boolMarker); } - else if (!(marker is true) && !isDerivedFromInClause) + + callback(n); + }); + } + + // Nested-loop execution for EXISTS/NOT EXISTS derived MarkJoin. + void ExecNested(Action callback) + { + ExecContext context = context_; + var logic = logic_ as LogicMarkJoin; + var filter = logic.filter_; + bool semi = (logic_ is LogicMarkSemiJoin); + Value marker = false; + + lchild_().Exec(l => + { + marker = false; + rchild_().Exec(r => + { + if (!(marker is true)) + { + Row n = new Row(l, r); + if (filter.Exec(context, n) is true) + { + marker = true; + n = ExecProject(n); + fixMarkerValue(n, semi); + callback(n); + } + } + }); + + if (!(marker is true)) { Row r = new Row(rchild_().logic_.output_.Count); Row n = new Row(l, r); diff --git a/test/regress/expect/subqueryd_hashmj.txt b/test/regress/expect/subqueryd_hashmj.txt new file mode 100644 index 0000000..499fcac --- /dev/null +++ b/test/regress/expect/subqueryd_hashmj.txt @@ -0,0 +1,68 @@ +select a1 from a where a2 in (select b2 from b where b2 = a1) +PhysicFilter (actual rows=0) + Output: a.a1[0] + Filter: {#marker@1}[1] + -> PhysicMarkJoin Left (actual rows=3) + Output: a.a1[0],{#marker@1}[1] + Filter: (a.a2[1]=b.b2[2] and b.b2[2]=a.a1[0]) + -> PhysicScanTable a (actual rows=3) + Output: a.a1[0],a.a2[1] + -> PhysicScanTable b (actual rows=3) + Output: b.b2[1] + + +select a1 from a where a2 not in (select b2 from b where b2 = a1) +PhysicFilter (actual rows=3) + Output: a.a1[0] + Filter: {#marker@1}[1] + -> PhysicMarkJoin Left (actual rows=3) + Output: a.a1[0],{#marker@1}[1] + Filter: (a.a2[1]=b.b2[2] and b.b2[2]=a.a1[0]) + -> PhysicScanTable a (actual rows=3) + Output: a.a1[0],a.a2[1] + -> PhysicScanTable b (actual rows=3) + Output: b.b2[1] +0 +1 +2 + +select a1 from a where a1 in (select b1 from b where b2 = a2 and b3 > 2) +PhysicFilter (actual rows=2) + Output: a.a1[0] + Filter: {#marker@1}[1] + -> PhysicMarkJoin Left (actual rows=3) + Output: a.a1[0],{#marker@1}[1] + Filter: (a.a1[0]=b.b1[2] and b.b2[3]=a.a2[1]) + -> PhysicScanTable a (actual rows=3) + Output: a.a1[0],a.a2[1] + -> PhysicScanTable b (actual rows=2) + Output: b.b1[0],b.b2[1] + Filter: b.b3[2]>2 +1 +2 + +select a1 from a where a1 not in (select b1 from b where b2 = a2 and b3 > 2) +PhysicFilter (actual rows=1) + Output: a.a1[0] + Filter: {#marker@1}[1] + -> PhysicMarkJoin Left (actual rows=3) + Output: a.a1[0],{#marker@1}[1] + Filter: (a.a1[0]=b.b1[2] and b.b2[3]=a.a2[1]) + -> PhysicScanTable a (actual rows=3) + Output: a.a1[0],a.a2[1] + -> PhysicScanTable b (actual rows=2) + Output: b.b1[0],b.b2[1] + Filter: b.b3[2]>2 +0 + +select a1 from a where a2 in (select b2 from b where b3 > 2) +PhysicScanTable a (actual rows=2) + Output: a.a1[0] + Filter: a.a2[1] in @1 + cached 1 + -> PhysicScanTable b (actual rows=2) + Output: b.b2[1] + Filter: b.b3[2]>2 +1 +2 + diff --git a/test/regress/expect/subqueryd_or.txt b/test/regress/expect/subqueryd_or.txt index d38b1da..c8e1fc9 100644 --- a/test/regress/expect/subqueryd_or.txt +++ b/test/regress/expect/subqueryd_or.txt @@ -7,7 +7,7 @@ PhysicFilter (actual rows=1) Filter: (a.a2[1]=b.b2[2] and b.b2[2]=a.a1[0]) -> PhysicScanTable a (actual rows=3) Output: a.a1[0],a.a2[1] - -> PhysicScanTable b (actual rows=3, loops=3) + -> PhysicScanTable b (actual rows=3) Output: b.b2[1] 2 @@ -20,7 +20,7 @@ PhysicFilter (actual rows=3) Filter: (a.a2[1]=b.b2[2] and b.b2[2]=a.a1[0]) -> PhysicScanTable a (actual rows=3) Output: a.a1[0],a.a2[1] - -> PhysicScanTable b (actual rows=3, loops=3) + -> PhysicScanTable b (actual rows=3) Output: b.b2[1] 0 1 diff --git a/test/regress/sql/subqueryd_hashmj.sql b/test/regress/sql/subqueryd_hashmj.sql new file mode 100644 index 0000000..3cd0ece --- /dev/null +++ b/test/regress/sql/subqueryd_hashmj.sql @@ -0,0 +1,22 @@ +-- Issue #272: realize MarkJoin as HashJoin for IN/NOT IN +-- These queries exercise the hash-based execution path in PhysicMarkJoin. +-- The hash is built on correlation (residual) keys, not the IN-list marker. + +set enable_subquery_unnest = true; +set enable_dependent_join_pushdown = false; +set enable_neumann_full_decorrelation = false; + +-- basic correlated IN: hash on b2=a1 (correlation), marker is a2=b2 +select a1 from a where a2 in (select b2 from b where b2 = a1); + +-- basic correlated NOT IN: same keys, anti-semi semantics +select a1 from a where a2 not in (select b2 from b where b2 = a1); + +-- correlated IN with additional filter on subquery side +select a1 from a where a1 in (select b1 from b where b2 = a2 and b3 > 2); + +-- NOT IN with additional filter +select a1 from a where a1 not in (select b1 from b where b2 = a2 and b3 > 2); + +-- IN with no correlation predicate: falls back to nested loop +select a1 from a where a2 in (select b2 from b where b3 > 2);