From 9f060e12a03623cf6b28306015a677060ca7a390 Mon Sep 17 00:00:00 2001
From: Joe Wicentowski <joewiz@gmail.com>
Date: Sun, 10 May 2026 11:46:10 -0400
Subject: [PATCH] [ci] compare-results.xslt: surface runner-JAR drift warning
 when build SHAs differ

Adds a `cr:warnings` block to the comparison report that fires when the
previous and current XQTS runs were produced by different runner JAR
builds. Without this, develop-branch deltas, runner-source bugfixes,
and shaded-`exist-core` differences all silently appear as if they
were caused by the PR's source change.

Why: see https://github.com/eXist-db/exist/issues/6326. A 2026-05-09
investigation found the same eXist SHA producing +831 XQTS passes and
a 25x wallclock improvement when re-run against a different runner
JAR build (the runner-side `applyVersionHint` cap-at-3.1 patch). The
existing comparison report attributes those deltas to the PR's source
change, which is wrong. This makes runner-JAR drift directly
attributable rather than masquerading as engine improvements.

What changed:
- Reads `runner-info.xml` from each run's output dir root (two levels
  up from the junit-data path). The metadata is emitted by the
  companion `exist-xqts-runner` change; older runs predating that
  change will have the file absent.
- Emits a `cr:warning kind="runner-drift"` element when the runner
  JAR's git-sha or sha256 differs between the runs, including the
  full `runner-jar` payloads from each side for transparency.
- Emits a `cr:warning kind="embedded-exist-core-drift"` when the
  runner JAR appears identical but the shaded `exist-core` version
  differs (the second drift mode surfaced during PR #6331's
  perf-6322 investigation).
- Graceful degradation: when either side's `runner-info.xml` is
  missing or malformed (`doc-available` returns false), no warning
  is emitted and the rest of the comparison runs unchanged. Older
  baselines and contributor environments without the runner-side
  patch keep working with no behaviour change.

Hand-tested with synthetic inputs against Saxon-HE 9.9 (the same
version CI uses), covering: SHAs differ, SHAs match, missing on one
side, malformed XML, and embedded-exist-core-only drift.

`ci-xqts.yml` needs no edit -- the artifact upload uploads the entire
`/tmp/xqts-output` dir and `runner-info.xml` round-trips automatically.

Partially addresses #6326 (mitigation A only). Mitigations B (runner
version pin) and C (N-run flake filtering) remain open.

Companion runner PR: https://github.com/eXist-db/exist-xqts-runner/pull/<TBD>

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 exist-xqts/src/main/xslt/compare-results.xslt | 84 ++++++++++++++++++-
 1 file changed, 80 insertions(+), 4 deletions(-)

diff --git a/exist-xqts/src/main/xslt/compare-results.xslt b/exist-xqts/src/main/xslt/compare-results.xslt
index 5872dc35c28..28203f1421d 100644
--- a/exist-xqts/src/main/xslt/compare-results.xslt
+++ b/exist-xqts/src/main/xslt/compare-results.xslt
@@ -25,7 +25,8 @@
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
     xmlns:xs="http://www.w3.org/2001/XMLSchema"
     xmlns:cr="http://exist-db.org/exist-xqts/compare-results"
-    exclude-result-prefixes="xs"
+    xmlns:ri="http://exist-db.org/exist-xqts-runner/runner-info"
+    exclude-result-prefixes="xs ri"
     version="2.0">
 
     <xsl:param name="xqts.previous.junit-data-path" as="xs:string" required="yes"/>
@@ -38,8 +39,16 @@
     <xsl:template name="compare-results" as="document-node(element(cr:comparison))">
         <xsl:variable name="previous-summary" select="cr:summarise-results($xqts.previous.junit-data-path)" as="document-node(element(cr:results))"/>
         <xsl:variable name="current-summary" select="cr:summarise-results($xqts.current.junit-data-path)" as="document-node(element(cr:results))"/>
+        <xsl:variable name="previous-runner-info" as="document-node()?" select="cr:load-runner-info($xqts.previous.junit-data-path)"/>
+        <xsl:variable name="current-runner-info" as="document-node()?" select="cr:load-runner-info($xqts.current.junit-data-path)"/>
         <xsl:document>
             <cr:comparison>
+                <xsl:variable name="warnings" as="element(cr:warning)*" select="cr:drift-warnings($previous-runner-info, $current-runner-info)"/>
+                <xsl:if test="exists($warnings)">
+                    <cr:warnings>
+                        <xsl:sequence select="$warnings"/>
+                    </cr:warnings>
+                </xsl:if>
                 <cr:previous>
                     <xsl:copy select="$previous-summary/cr:results">
                         <xsl:copy-of select="@*"/>
@@ -92,10 +101,10 @@
         <xsl:param name="previous-results" as="element(cr:results)" required="yes"/>
         <xsl:param name="current-results" as="element(cr:results)" required="yes"/>
         <xsl:param name="attr-name" as="xs:string" required="yes"/>
-        
+
         <xsl:variable name="previous-attr" select="$previous-results/@*[local-name(.) eq $attr-name]"/>
         <xsl:variable name="current-attr" select="$current-results/@*[local-name(.) eq $attr-name]"/>
-        
+
         <xsl:attribute name="{$attr-name}" select="$current-attr - $previous-attr"/>
         <xsl:attribute name="{$attr-name}-pct" select="(($current-attr - $previous-attr) div $previous-attr) * 100"/>
     </xsl:function>
@@ -118,4 +127,71 @@
         </xsl:copy>
     </xsl:template>
 
-</xsl:stylesheet>
\ No newline at end of file
+    <!--
+        Locate `runner-info.xml` next to the run's output dir. The junit data
+        path is `<output>/junit/data`, so the metadata file sits two levels up.
+        Returns the empty sequence if the file is missing or unparseable, so
+        the warning step degrades to a no-op rather than failing the comparison.
+    -->
+    <xsl:function name="cr:load-runner-info" as="document-node()?">
+        <xsl:param name="junit-data-path" as="xs:string"/>
+        <xsl:variable name="uri" select="concat($junit-data-path, '/../../runner-info.xml')"/>
+        <xsl:choose>
+            <xsl:when test="doc-available($uri)">
+                <xsl:sequence select="doc($uri)"/>
+            </xsl:when>
+            <xsl:otherwise>
+                <xsl:sequence select="()"/>
+            </xsl:otherwise>
+        </xsl:choose>
+    </xsl:function>
+
+    <!--
+        Emit a `cr:warning` per drift kind detected. The two kinds:
+          - runner-drift: runner-JAR git-sha or sha256 differs between the runs
+          - embedded-exist-core-drift: embedded `exist-core` version differs
+            even when the runner JAR git-sha matches
+        See https://github.com/eXist-db/exist/issues/6326.
+    -->
+    <xsl:function name="cr:drift-warnings" as="element(cr:warning)*">
+        <xsl:param name="previous" as="document-node()?"/>
+        <xsl:param name="current" as="document-node()?"/>
+        <xsl:if test="exists($previous) and exists($current)">
+            <xsl:variable name="prev-jar-sha" select="string($previous//ri:runner-jar/ri:git-sha[not(@unknown='true')])"/>
+            <xsl:variable name="curr-jar-sha" select="string($current//ri:runner-jar/ri:git-sha[not(@unknown='true')])"/>
+            <xsl:variable name="prev-jar-hash" select="string($previous//ri:runner-jar/ri:sha256[not(@unknown='true')])"/>
+            <xsl:variable name="curr-jar-hash" select="string($current//ri:runner-jar/ri:sha256[not(@unknown='true')])"/>
+            <xsl:variable name="prev-core-ver" select="string($previous//ri:embedded-exist-core/ri:version[not(@unknown='true')])"/>
+            <xsl:variable name="curr-core-ver" select="string($current//ri:embedded-exist-core/ri:version[not(@unknown='true')])"/>
+
+            <xsl:variable name="runner-drift" as="xs:boolean" select="
+                ($prev-jar-sha ne '' and $curr-jar-sha ne '' and $prev-jar-sha ne $curr-jar-sha)
+                or ($prev-jar-hash ne '' and $curr-jar-hash ne '' and $prev-jar-hash ne $curr-jar-hash)"/>
+
+            <xsl:if test="$runner-drift">
+                <cr:warning kind="runner-drift">
+                    <cr:summary>Runner JAR build SHA or sha256 differs between the previous and current XQTS runs. Test deltas may include runner-side effects unrelated to this PR. See https://github.com/eXist-db/exist/issues/6326.</cr:summary>
+                    <cr:previous>
+                        <xsl:copy-of select="$previous//ri:runner-jar"/>
+                    </cr:previous>
+                    <cr:current>
+                        <xsl:copy-of select="$current//ri:runner-jar"/>
+                    </cr:current>
+                </cr:warning>
+            </xsl:if>
+
+            <xsl:if test="not($runner-drift) and $prev-core-ver ne '' and $curr-core-ver ne '' and $prev-core-ver ne $curr-core-ver">
+                <cr:warning kind="embedded-exist-core-drift">
+                    <cr:summary>The runner JAR appears identical, but the embedded `exist-core` version differs between the runs. Test deltas may include `exist-core` shading effects rather than the eXist source under test.</cr:summary>
+                    <cr:previous>
+                        <xsl:copy-of select="$previous//ri:embedded-exist-core"/>
+                    </cr:previous>
+                    <cr:current>
+                        <xsl:copy-of select="$current//ri:embedded-exist-core"/>
+                    </cr:current>
+                </cr:warning>
+            </xsl:if>
+        </xsl:if>
+    </xsl:function>
+
+</xsl:stylesheet>