Repository navigation
Realize MarkJoin as HashJoin for IN/NOT IN (issue #272) #323
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1503,7 +1503,7 @@ | |
|
|
||
| public class LogicCteProducer : LogicNode | ||
| { | ||
| internal CteExpr cte_; | ||
|
Check warning on line 1506 in qpmodel/subquery.cs
|
||
|
|
||
| public CteInfoEntry cteInfoEntry_; | ||
| // represent the id of CTE, it should match the related CteProducer | ||
|
|
@@ -1712,95 +1712,233 @@ | |
| public override void Exec(Action<Row> callback) | ||
| { | ||
| var isDerivedFromInClause = filterHasMarkerBinExpr(logic_.filter_); | ||
| if (isDerivedFromInClause) | ||
| ExecHash(callback); | ||
| else | ||
| ExecNested(callback); | ||
| } | ||
|
|
||
| // Hash-based execution for IN/NOT IN derived MarkJoin (issue #272). | ||
| // | ||
| // The filter has two parts: | ||
| // - Marker BinExpr: the IN-list equality (e.g., a2=b2) | ||
| // - Residual: the correlation predicate (e.g., b1=a1) | ||
| // | ||
| // We hash the RIGHT side on the RESIDUAL equi-join keys (the correlation | ||
| // condition), then for each left row probe the hash table and evaluate | ||
| // the marker predicate for three-valued IN/NOT IN logic. | ||
| void ExecHash(Action<Row> callback) | ||
| { | ||
| ExecContext context = context_; | ||
| var logic = logic_ as LogicMarkJoin; | ||
| var filter = logic.filter_; | ||
| bool semi = (logic_ is LogicMarkSemiJoin); | ||
|
|
||
| // Decompose the filter into marker predicate and residual. | ||
| var andList = filter.FilterToAndList(); | ||
| var markerExpr = andList.Find(x => x is BinExpr xB && xB.IsMarkerBinExpr()) as BinExpr; | ||
| Debug.Assert(markerExpr != null); | ||
| andList.Remove(markerExpr); | ||
|
|
||
| // Extract equi-join keys from residual predicates for hashing. | ||
| var ltabrefs = lchild_().logic_.InclusiveTableRefs(); | ||
| var leftKeys = new List<Expr>(); | ||
| var rightKeys = new List<Expr>(); | ||
| var nonEquiResidual = new List<Expr>(); | ||
|
|
||
| foreach (var pred in andList) | ||
| { | ||
| if (pred is BinExpr be && be.op_ == "=") | ||
| { | ||
| var lrefs = be.lchild_().tableRefs_; | ||
| var rrefs = be.rchild_().tableRefs_; | ||
| bool lOnLeft = lrefs != null && !lrefs.Except(ltabrefs).Any(); | ||
| bool rOnLeft = rrefs != null && !rrefs.Except(ltabrefs).Any(); | ||
|
|
||
| if (lOnLeft && !rOnLeft) | ||
| { | ||
| leftKeys.Add(be.lchild_()); | ||
| rightKeys.Add(be.rchild_()); | ||
| continue; | ||
| } | ||
| else if (rOnLeft && !lOnLeft) | ||
| { | ||
| leftKeys.Add(be.rchild_()); | ||
| rightKeys.Add(be.lchild_()); | ||
| continue; | ||
| } | ||
| } | ||
| nonEquiResidual.Add(pred); | ||
| } | ||
|
|
||
| Expr nonEquiFilter = nonEquiResidual.Count > 0 ? FilterHelper.AndListToExpr(nonEquiResidual) : null; | ||
| int lColCount = lchild_().logic_.output_.Count; | ||
| int rColCount = rchild_().logic_.output_.Count; | ||
|
|
||
| // If no hashable residual keys, fall back to nested loop. | ||
| if (leftKeys.Count == 0) | ||
| { | ||
| ExecNestedInClause(callback, markerExpr, andList.Count > 0 ? FilterHelper.AndListToExpr(andList) : null); | ||
| return; | ||
| } | ||
|
|
||
| // Build hash table from right (subquery) side on residual keys. | ||
| var hm = new Dictionary<KeyList, List<Row>>(); | ||
|
|
||
| rchild_().Exec(r => | ||
| { | ||
| Row fakeLeft = new Row(lColCount); | ||
| Row combined = new Row(fakeLeft, r); | ||
| var keys = KeyList.ComputeKeys(context, rightKeys, combined); | ||
| if (keys.ColsHasNull()) | ||
| return; // NULL correlation keys never match | ||
| if (hm.TryGetValue(keys, out List<Row> existing)) | ||
| existing.Add(r); | ||
| else | ||
| hm.Add(keys, new List<Row> { r }); | ||
| }); | ||
|
|
||
| // Probe with left (outer) side. | ||
| lchild_().Exec(l => | ||
| { | ||
| bool lIsNull = l.ColsHasNull(); | ||
| Value marker = false; | ||
| bool RHasNull = false; | ||
| bool RisEmpty = true; | ||
|
|
||
| var keys = KeyList.ComputeKeys(context, leftKeys, l); | ||
| if (!keys.ColsHasNull() && hm.TryGetValue(keys, out List<Row> matches)) | ||
| { | ||
| // Hash hit on correlation keys — now evaluate marker and non-equi residual | ||
| foreach (var r in matches) | ||
| { | ||
| Row combined = new Row(l, r); | ||
|
|
||
| // Check non-equi residual first | ||
| if (nonEquiFilter != null) | ||
| { | ||
| var flag = nonEquiFilter.Exec(context, combined); | ||
| if (!(flag is true)) | ||
| continue; | ||
| } | ||
|
|
||
| RisEmpty = false; | ||
| if (r.ColsHasNull()) | ||
| RHasNull = true; | ||
|
|
||
| // Evaluate the marker (IN-list equality) predicate | ||
| if (markerExpr.Exec(context, combined) is true) | ||
| { | ||
| marker = true; | ||
| break; // One match is enough for IN | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Three-valued NULL logic (same as original nested loop): | ||
| if (marker is false && RHasNull) | ||
| marker = null; | ||
| if (lIsNull && RisEmpty) | ||
| marker = false; | ||
|
|
||
| Row rr = new Row(rColCount); | ||
| Row n = new Row(l, rr); | ||
| n = ExecProject(n); | ||
|
|
||
| if (marker is null) | ||
| fixMarkerValue(n, false); | ||
| else | ||
| { | ||
| bool boolMarker = marker is true; | ||
| fixMarkerValue(n, semi ? boolMarker : !boolMarker); | ||
| } | ||
|
|
||
| callback(n); | ||
| }); | ||
| } | ||
|
|
||
| // Nested-loop fallback for IN-clause MarkJoin when no hashable keys found. | ||
| void ExecNestedInClause(Action<Row> callback, BinExpr markerExpr, Expr residualFilter) | ||
| { | ||
| ExecContext context = context_; | ||
| bool semi = (logic_ is LogicMarkSemiJoin); | ||
| int rColCount = rchild_().logic_.output_.Count; | ||
|
|
||
| lchild_().Exec(l => | ||
| { | ||
| bool lIsNull = l.ColsHasNull(); | ||
| Value marker = false; | ||
| bool RHasNull = false; | ||
| bool RisEmpty = true; | ||
|
|
||
| rchild_().Exec(r => | ||
| { | ||
| Row combined = new Row(l, r); | ||
|
|
||
| if (residualFilter != null) | ||
| { | ||
| var flag = residualFilter.Exec(context, combined); | ||
| if (!(flag is true)) | ||
| return; | ||
| } | ||
|
|
||
| RisEmpty = false; | ||
| if (r.ColsHasNull()) | ||
| RHasNull = true; | ||
|
|
||
| if (markerExpr.Exec(context, combined) is true) | ||
| marker = true; | ||
| }); | ||
|
Comment on lines
+1874
to
+1891
There was a problem hiding this comment. ExecNestedInClause scans the entire right side even after |
||
|
|
||
| if (marker is false && RHasNull) | ||
| marker = null; | ||
| if (lIsNull && RisEmpty) | ||
| marker = false; | ||
|
|
||
| Row rr = new Row(rColCount); | ||
| Row n = new Row(l, rr); | ||
| n = ExecProject(n); | ||
|
|
||
| if (marker is null) | ||
| fixMarkerValue(n, false); | ||
| else | ||
| { | ||
| bool boolMarker = marker is true; | ||
| fixMarkerValue(n, semi ? boolMarker : !boolMarker); | ||
| } | ||
|
|
||
| callback(n); | ||
| }); | ||
| } | ||
|
|
||
| // Nested-loop execution for EXISTS/NOT EXISTS derived MarkJoin. | ||
| void ExecNested(Action<Row> callback) | ||
| { | ||
| ExecContext context = context_; | ||
| var logic = logic_ as LogicMarkJoin; | ||
| var filter = logic.filter_; | ||
| bool semi = (logic_ is LogicMarkSemiJoin); | ||
| Value marker = false; | ||
|
|
||
| lchild_().Exec(l => | ||
| { | ||
| marker = false; | ||
| rchild_().Exec(r => | ||
| { | ||
| if (!(marker is true)) | ||
| { | ||
| Row n = new Row(l, r); | ||
| if (filter.Exec(context, n) is true) | ||
| { | ||
| marker = true; | ||
| n = ExecProject(n); | ||
| fixMarkerValue(n, semi); | ||
| callback(n); | ||
| } | ||
| } | ||
| }); | ||
|
|
||
| if (!(marker is true)) | ||
| { | ||
| Row r = new Row(rchild_().logic_.output_.Count); | ||
| Row n = new Row(l, r); | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| select a1 from a where a2 in (select b2 from b where b2 = a1) | ||
| PhysicFilter (actual rows=0) | ||
| Output: a.a1[0] | ||
| Filter: {#marker@1}[1] | ||
| -> PhysicMarkJoin Left (actual rows=3) | ||
| Output: a.a1[0],{#marker@1}[1] | ||
| Filter: (a.a2[1]=b.b2[2] and b.b2[2]=a.a1[0]) | ||
| -> PhysicScanTable a (actual rows=3) | ||
| Output: a.a1[0],a.a2[1] | ||
| -> PhysicScanTable b (actual rows=3) | ||
| Output: b.b2[1] | ||
|
|
||
|
|
||
| select a1 from a where a2 not in (select b2 from b where b2 = a1) | ||
| PhysicFilter (actual rows=3) | ||
| Output: a.a1[0] | ||
| Filter: {#marker@1}[1] | ||
| -> PhysicMarkJoin Left (actual rows=3) | ||
| Output: a.a1[0],{#marker@1}[1] | ||
| Filter: (a.a2[1]=b.b2[2] and b.b2[2]=a.a1[0]) | ||
| -> PhysicScanTable a (actual rows=3) | ||
| Output: a.a1[0],a.a2[1] | ||
| -> PhysicScanTable b (actual rows=3) | ||
| Output: b.b2[1] | ||
| 0 | ||
| 1 | ||
| 2 | ||
|
|
||
| select a1 from a where a1 in (select b1 from b where b2 = a2 and b3 > 2) | ||
| PhysicFilter (actual rows=2) | ||
| Output: a.a1[0] | ||
| Filter: {#marker@1}[1] | ||
| -> PhysicMarkJoin Left (actual rows=3) | ||
| Output: a.a1[0],{#marker@1}[1] | ||
| Filter: (a.a1[0]=b.b1[2] and b.b2[3]=a.a2[1]) | ||
| -> PhysicScanTable a (actual rows=3) | ||
| Output: a.a1[0],a.a2[1] | ||
| -> PhysicScanTable b (actual rows=2) | ||
| Output: b.b1[0],b.b2[1] | ||
| Filter: b.b3[2]>2 | ||
| 1 | ||
| 2 | ||
|
|
||
| select a1 from a where a1 not in (select b1 from b where b2 = a2 and b3 > 2) | ||
| PhysicFilter (actual rows=1) | ||
| Output: a.a1[0] | ||
| Filter: {#marker@1}[1] | ||
| -> PhysicMarkJoin Left (actual rows=3) | ||
| Output: a.a1[0],{#marker@1}[1] | ||
| Filter: (a.a1[0]=b.b1[2] and b.b2[3]=a.a2[1]) | ||
| -> PhysicScanTable a (actual rows=3) | ||
| Output: a.a1[0],a.a2[1] | ||
| -> PhysicScanTable b (actual rows=2) | ||
| Output: b.b1[0],b.b2[1] | ||
| Filter: b.b3[2]>2 | ||
| 0 | ||
|
|
||
| select a1 from a where a2 in (select b2 from b where b3 > 2) | ||
| PhysicScanTable a (actual rows=2) | ||
| Output: a.a1[0] | ||
| Filter: a.a2[1] in @1 | ||
| <InSubqueryExpr> cached 1 | ||
| -> PhysicScanTable b (actual rows=2) | ||
| Output: b.b2[1] | ||
| Filter: b.b3[2]>2 | ||
| 1 | ||
| 2 | ||
|
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In ExecHash,
fakeLeftis allocated inside therchild_().Execcallback, so a new all-null Row is created for every right-side row. This is unnecessary allocation/GC pressure since the left placeholder is constant; consider allocatingfakeLeftonce outside the loop (or avoid the combined row entirely ifrightKeysare guaranteed right-only).