fix: update the Leios setup to the latest prototype

.github/workflows/regression-leios.yaml

@@ -14,10 +14,6 @@ on:
         type: boolean
         default: false
         description: "Allow unstable error messages"
-      skip_deselect:
-        type: boolean
-        default: false
-        description: "Run also the tests that are known to fail on Leios"
 
 permissions:
   contents: read
@@ -35,7 +31,6 @@ jobs:
       cluster_era: dijkstra 12
       markexpr: testnets or leios
       allow_unstable_error_msgs: ${{ inputs.allow_unstable_error_msgs }}
-      skip_deselect: ${{ inputs.skip_deselect }}
       env-path: runner/env_leios
     secrets:
       CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}

.github/workflows/regression_reusable.yaml

@@ -43,10 +43,6 @@ on:
         required: false
         type: boolean
         default: false
-      skip_deselect:
-        required: false
-        type: boolean
-        default: false
       testrun_name:
         required: false
         type: string
@@ -113,11 +109,6 @@ jobs:
           ENV_PATH: ${{ inputs.env-path }}
         run: |
           if [ -e "$ENV_PATH" ]; then cat "$ENV_PATH" >> "$GITHUB_ENV"; fi
-      - name: Clear deselect file
-        # Undo a `DESELECT_FROM_FILE` that came from the env file. Runs before
-        # the tcache step below, so `skip_passed` still wins if both are used.
-        if: inputs.skip_deselect
-        run: echo "DESELECT_FROM_FILE=" >> "$GITHUB_ENV"
       - name: Get previous test results
         if: inputs.testrun_name && inputs.skip_passed
         env:
@@ -131,8 +122,6 @@ jobs:
             # handed to pytest-select as a list of test names.
             if ! curl -sf -u "$TCACHE_BASIC_AUTH" "$TCACHE_URL/${testrun_name_strip}/pypassed" \
                    > deselected_tests.txt; then
-              # `DESELECT_FROM_FILE` is left alone here, so a value coming from
-              # the env file still applies.
               echo "::warning::Could not fetch previous test results from the" \
                    "tcache, not skipping previously passed tests."
               rm -f deselected_tests.txt

cardano_node_tests/tests/test_leios_blocks.py

@@ -116,11 +116,12 @@
 # rate of the `local_fast` variant (and its 4x shorter epoch) drains the mempool.
 MIN_BLOCK_INTERVAL_SEC = 10
 
-# The first epoch in which a pool can be a member of the Leios voting committee. The
-# committee is drawn from a stake distribution snapshot that is empty for the whole
-# lifetime of a freshly started cluster instance until this epoch, so up to then every
-# pool answers every EB announcement with `NotOnCommittee` and no EB can be voted on or
-# certified.
+# The first epoch in which a pool registered by a transaction can be a member of the
+# Leios voting committee. The committee is drawn from a stake distribution snapshot
+# that is empty for the whole lifetime of a freshly started cluster instance until this
+# epoch, so up to then every pool answers every EB announcement with `NotOnCommittee`
+# and no EB can be voted on or certified. It doesn't apply to a testnet whose pools
+# come with their BLS key straight from the genesis, see `_is_committee_seated_in_genesis`.
 VOTING_START_EPOCH = 3
 
 # Number of seconds between two searches of the logs while waiting for the pools to
@@ -395,6 +396,34 @@ def _collect_pre_voting_msgs(
     return {p: s.found for p, s in searches.items()}, list(log_errors.values())
 
 
+def _is_committee_seated_in_genesis(*, genesis: dict) -> bool:
+    """Check whether the Leios voting committee is seated from epoch 0.
+
+    A pool can vote once its BLS key is in a stake distribution snapshot the committee
+    is drawn from. A pool registered by a transaction on a freshly started cluster
+    instance gets there only in `VOTING_START_EPOCH`, while a pool whose parameters,
+    BLS key included, are already in the genesis is in the initial ledger state and so
+    on the committee from the first slot.
+
+    The genesis always lists the pools created by `genesis create-staked --gen-pools`,
+    but their entries carry a BLS key only when the cluster start script wrote the real
+    pool parameters into them instead of submitting a registration transaction. The key
+    is therefore what tells the two setups apart, not the presence of the pools. The
+    entries live under `extraConfig` since cardano-cli 11.2 and under `staking` before
+    that.
+
+    Args:
+        genesis: The Shelley genesis of the cluster instance.
+
+    Returns:
+        `True` when a pool is on the voting committee from epoch 0.
+    """
+    pools: dict = genesis.get("extraConfig", {}).get("stakePools", {}).get("data") or genesis.get(
+        "staking", {}
+    ).get("pools", {})
+    return any(p.get("blsKey") for p in pools.values())
+
+
 class TestLeios:
     """Tests for Leios endorser blocks."""
 
@@ -446,6 +475,8 @@ def test_no_voting_before_committee_epoch(
     ):
         """Check that no EB is voted on before the voting committee becomes active.
 
+        * Skip when the genesis seats the voting committee from epoch 0, as there is
+          then no epoch in which a vote would be premature
         * Skip when no epoch before `VOTING_START_EPOCH` has room left for the whole
           search window
         * Wait for a point in an epoch where the window fits before the next epoch
@@ -458,6 +489,12 @@ def test_no_voting_before_committee_epoch(
         cluster = cluster_leios
         common.get_test_id(cluster)
 
+        if _is_committee_seated_in_genesis(genesis=cluster.genesis):
+            pytest.skip(
+                "The pools are on the Leios voting committee from epoch 0, as their BLS keys "
+                "come from the genesis, so there is no epoch in which a vote is premature"
+            )
+
         state_dir = cluster_nodes.get_cluster_env().state_dir
         pool_logs = sorted(state_dir.glob("pool*.stdout"))
         assert pool_logs, f"No pool log files found in '{state_dir}'"

cardano_node_tests/utils/clusterlib_utils.py

@@ -1698,13 +1698,33 @@ def get_snapshot_rec(*, ledger_snapshot: dict) -> dict[str, int | list]:
     return hashes
 
 
+def unwrap_snapshot(*, ledger_snapshot: dict) -> dict:
+    """Get the stake distribution out of an `esSnapshots` entry of the ledger state.
+
+    On a Leios enabled node the entry is a record that holds the stake distribution
+    under `snapShot`, next to `epochNo` and `leiosCommitteeSize`. On older nodes the
+    entry is the stake distribution itself.
+
+    Args:
+        ledger_snapshot: A `pstakeMark`, `pstakeSet` or `pstakeGo` entry of the
+            `esSnapshots` record of the ledger state.
+
+    Returns:
+        The stake distribution, with the `snapShot` wrapper removed when there is one.
+    """
+    snapshot = ledger_snapshot.get("snapShot")
+    return snapshot if isinstance(snapshot, dict) else ledger_snapshot
+
+
 def get_stake_rec(*, stake_snapshot: dict) -> dict:
     """Get uniform record for stake key snapshot."""
+    unwrapped = unwrap_snapshot(ledger_snapshot=stake_snapshot)
+
     stake_rec: dict
-    if "activeStake" in stake_snapshot:
-        stake_rec = stake_snapshot["activeStake"]  # In cardano-node 10.7.0+
-    elif "stake" in stake_snapshot:
-        stake_rec = stake_snapshot["stake"]
+    if "activeStake" in unwrapped:
+        stake_rec = unwrapped["activeStake"]  # In cardano-node 10.7.0+
+    elif "stake" in unwrapped:
+        stake_rec = unwrapped["stake"]
     else:
         err = "Neither 'activeStake' nor 'stake' found in stake snapshot"
         raise KeyError(err)
@@ -1714,9 +1734,10 @@ def get_stake_rec(*, stake_snapshot: dict) -> dict:
 def get_snapshot_delegations(*, ledger_snapshot: dict) -> dict[str, list[str]]:
     """Get delegations data from ledger state snapshot."""
     delegations: dict[str, list[str]] = {}
+    unwrapped = unwrap_snapshot(ledger_snapshot=ledger_snapshot)
 
-    if "activeStake" in ledger_snapshot:
-        deleg_rec = ledger_snapshot["activeStake"]
+    if "activeStake" in unwrapped:
+        deleg_rec = unwrapped["activeStake"]
 
         for rk, rv in deleg_rec.items():
             # In node 10.7+ the format is a dict like
@@ -1732,8 +1753,8 @@ def get_snapshot_delegations(*, ledger_snapshot: dict) -> dict[str, list[str]]:
             else:
                 delegations[r_pool_id] = [r_hash]
 
-    elif "delegations" in ledger_snapshot:
-        deleg_rec = ledger_snapshot["delegations"]
+    elif "delegations" in unwrapped:
+        deleg_rec = unwrapped["delegations"]
 
         for rk, rv in deleg_rec.items():
             # In node 8.4+ the format is a dict like

doc/leios_tests.md

@@ -15,7 +15,6 @@ The setup lives in [`runner/env_leios`](https://github.com/IntersectMBO/cardano-
 | `PROTOCOL_VERSION`    | `12`                                   | Dijkstra protocol version.                              |
 | `ENABLE_TX_FIREHOSE`  | `true`                                 | Tx load generator, needed to fill the mempool.          |
 | `TX_TPS`              | `15`                                   | Tx rate ceiling of the load generator.                  |
-| `DESELECT_FROM_FILE`  | `scripts/deselected_leios_tests.txt`   | Skips the tests that are known to fail here.            |
 
 ## Running in CI
 
@@ -68,13 +67,6 @@ already exported value wins, so the node branch can be switched without touching
 NODE_REV=my-leios-branch ./scripts/test_leios.sh
 ```
 
-To run also the tests that are known to fail on Leios, clear the deselect file (or point it
-to your own list):
-
-```sh
-./scripts/test_leios.sh DESELECT_FROM_FILE=
-```
-
 Only `VAR=VALUE` arguments are accepted; anything else is rejected with a usage message.
 Results land in `run_workdir`.
 
@@ -95,11 +87,3 @@ On a failed testrun the workflow runs the analysis automatically and surfaces it
 * the **run summary** of the workflow run, and a foldable `Preliminary failure analysis`
   group in the log of the `Read failure analysis into env` step,
 * `failure_analysis.md` in the `testrun-files` artifact.
-
-## Known Failures
-
-`scripts/deselected_leios_tests.txt` lists the tests that are known to fail in this setup,
-grouped by cause, each group with the upstream issue link (missing PlutusV2 cost model in
-Dijkstra genesis, `transaction assemble` rejecting a Dijkstra `TxWitness`, missing required
-signers in the Plutus script context, new Dijkstra protocol params, mandatory Leios BLS
-signing keys, ...).

framework_tests/test_clusterlib_utils.py

@@ -14,6 +14,7 @@
 
 KEY_HASH1 = "9e1156acae8bd72bc1815d0be9fcb64e2d50e61f4204c45b901dad6b"
 KEY_HASH2 = "7c2086ea4ebaa880c6e6c70604c0deb37ffbaa0567aec0bea8564055"
+POOL_ID1 = "2e35bc3cae0fa3b642932a8e45602318027772192a2e215a537a6a8a"
 
 
 def write_script(*, script: dict, dest_dir: pl.Path) -> pl.Path:
@@ -166,3 +167,86 @@ def test_unsupported_type(self, tmp_path: pl.Path):
         script_file = write_script(script={"type": "unknown"}, dest_dir=tmp_path)
         with pytest.raises(ValueError, match="Unsupported simple script type: unknown"):
             clusterlib_utils.get_reference_script_size(script_file=script_file)
+
+
+class TestLedgerStateSnapshot:
+    """Tests for reading the stake distribution snapshots of the ledger state.
+
+    The `esSnapshots` entries have three formats to support: the `swd*` records of
+    cardano-node 10.7+, the flat records before that, and the `snapShot` wrapper that
+    a Leios enabled node puts around either of them.
+    """
+
+    ACTIVE_STAKE: tp.ClassVar[dict] = {
+        f"keyHash-{KEY_HASH1}": {"swdDelegation": POOL_ID1, "swdStake": 10},
+        f"keyHash-{KEY_HASH2}": {"swdDelegation": POOL_ID1, "swdStake": 20},
+    }
+    LEIOS_SNAPSHOT: tp.ClassVar[dict] = {
+        "epochNo": 6,
+        "leiosCommitteeSize": 900,
+        "snapShot": {"activeStake": ACTIVE_STAKE, "stakePoolsSnapShot": {}},
+    }
+
+    def test_unwrap_leios(self):
+        """Strip the `snapShot` wrapper of a Leios enabled node."""
+        assert clusterlib_utils.unwrap_snapshot(ledger_snapshot=self.LEIOS_SNAPSHOT) == {
+            "activeStake": self.ACTIVE_STAKE,
+            "stakePoolsSnapShot": {},
+        }
+
+    def test_unwrap_unwrapped(self):
+        """Keep a snapshot that has no `snapShot` wrapper as it is."""
+        snapshot = {"activeStake": self.ACTIVE_STAKE}
+        assert clusterlib_utils.unwrap_snapshot(ledger_snapshot=snapshot) == snapshot
+
+    def test_stake_rec_leios(self):
+        """Get the stake record from a wrapped snapshot."""
+        stake_rec = clusterlib_utils.get_stake_rec(stake_snapshot=self.LEIOS_SNAPSHOT)
+        assert stake_rec == self.ACTIVE_STAKE
+
+    def test_stake_rec_active_stake(self):
+        """Get the stake record from an unwrapped cardano-node 10.7+ snapshot."""
+        stake_rec = clusterlib_utils.get_stake_rec(
+            stake_snapshot={"activeStake": self.ACTIVE_STAKE}
+        )
+        assert stake_rec == self.ACTIVE_STAKE
+
+    def test_stake_rec_legacy(self):
+        """Get the stake record from a snapshot that predates `activeStake`."""
+        stake = {f"keyHash-{KEY_HASH1}": 10}
+        assert clusterlib_utils.get_stake_rec(stake_snapshot={"stake": stake}) == stake
+
+    def test_stake_rec_unknown(self):
+        """Fail on a snapshot that holds no stake record."""
+        with pytest.raises(KeyError, match="Neither 'activeStake' nor 'stake' found"):
+            clusterlib_utils.get_stake_rec(stake_snapshot={"epochNo": 6})
+
+    def test_delegations_leios(self):
+        """Get the delegations from a wrapped snapshot."""
+        delegations = clusterlib_utils.get_snapshot_delegations(ledger_snapshot=self.LEIOS_SNAPSHOT)
+        assert delegations == {POOL_ID1: [KEY_HASH1, KEY_HASH2]}
+
+    def test_delegations_active_stake(self):
+        """Get the delegations from an unwrapped cardano-node 10.7+ snapshot."""
+        delegations = clusterlib_utils.get_snapshot_delegations(
+            ledger_snapshot={"activeStake": self.ACTIVE_STAKE}
+        )
+        assert delegations == {POOL_ID1: [KEY_HASH1, KEY_HASH2]}
+
+    def test_delegations_legacy(self):
+        """Get the delegations from a snapshot that predates `activeStake`."""
+        delegations = clusterlib_utils.get_snapshot_delegations(
+            ledger_snapshot={"delegations": {f"keyHash-{KEY_HASH1}": POOL_ID1}}
+        )
+        assert delegations == {POOL_ID1: [KEY_HASH1]}
+
+    def test_delegations_unknown(self):
+        """Fail on a snapshot that holds no delegations."""
+        with pytest.raises(KeyError, match="Neither 'stakePoolsSnapShot' nor 'delegations' found"):
+            clusterlib_utils.get_snapshot_delegations(ledger_snapshot={"epochNo": 6})
+
+    def test_snapshot_rec_leios(self):
+        """Sum the stake amounts of a wrapped snapshot."""
+        stake_rec = clusterlib_utils.get_stake_rec(stake_snapshot=self.LEIOS_SNAPSHOT)
+        hashes = clusterlib_utils.get_snapshot_rec(ledger_snapshot=stake_rec)
+        assert hashes == {KEY_HASH1: 10, KEY_HASH2: 20}

pyproject.toml

@@ -26,7 +26,7 @@ classifiers = [
 dependencies = [
     "allure-pytest (>=2.16.0,<3.0.0)",
     "cardano-clusterlib (>=0.10.6,<0.11.0)",
-    "cardonnay (>=0.4.6,<0.5.0)",
+    "cardonnay (>=0.4.7,<0.5.0)",
     "cbor2 (>=6.1.4,<7.0.0)",
     "filelock (>=3.32.5,<4.0.0)",
     "hypothesis (>=6.167.1,<7.0.0)",

runner/env_leios

@@ -1,8 +1,8 @@
 TESTNET_VARIANT=leios_fast
 ENABLE_TX_FIREHOSE=true
+POOLS_IN_GENESIS=true
 TX_TPS=15
 COMMAND_ERA=dijkstra
 PROTOCOL_VERSION=12
 CI_ARGS=-k 'not submit_api'
 SESSION_TIMEOUT=4h
-DESELECT_FROM_FILE=scripts/deselected_leios_tests.txt

runner/run_tests.sh

@@ -120,9 +120,9 @@ compute_common_args() {
     assert_deselect_file || exit 3
 
     # `pytest-select` has no comment syntax - it reads every line of the file
-    # as a test name. Hand it a stripped copy so that annotated lists (e.g.
-    # `scripts/deselected_leios_tests.txt`, which groups the tests by cause)
-    # don't end up in its "Not all deselected tests exist" warning.
+    # as a test name. Hand it a stripped copy, so that a list annotated with
+    # comments and blank lines doesn't end up in its "Not all deselected tests
+    # exist" warning.
     stripped_deselect="$(mktemp -t deselected_tests.XXXXXX.txt)"
     trap 'rm -f "$stripped_deselect"' EXIT
     sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e '/^#/d' -e '/^$/d' \