From 076b13fda9c2b7fb3f0dc3987bf2d23699706ad0 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Tue, 27 May 2025 18:25:27 +0200 Subject: [PATCH 01/25] ENH: Use shlex.join for pretty printing of shell commands --- contrib/ci_engine.py | 3 ++- pelita/game.py | 3 ++- pelita/scripts/pelita_server.py | 4 ++-- pelita/scripts/pelita_tournament.py | 27 ++------------------------- pelita/team.py | 3 ++- 5 files changed, 10 insertions(+), 30 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index bbbac8e4c..62906bb91 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -46,6 +46,7 @@ import itertools import json import logging +import shlex import sqlite3 import subprocess import sys @@ -70,7 +71,7 @@ def hash_team(team_spec): 'pelita.scripts.pelita_player', 'hash-team', team_spec] - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) res = subprocess.run(external_call, capture_output=True, text=True) return res.stdout.strip().split("\n")[-1].strip() diff --git a/pelita/game.py b/pelita/game.py index 5651ae884..71c596545 100644 --- a/pelita/game.py +++ b/pelita/game.py @@ -3,6 +3,7 @@ import logging import math import os +import shlex import subprocess import sys import time @@ -96,7 +97,7 @@ def _run_external_viewer(self, subscribe_sock, controller, geometry, delay, stop external_call = [sys.executable, '-m', tkviewer] + viewer_args - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) # os.setsid will keep the viewer from closing when the main process exits # a better solution might be to decouple the viewer from the main process if _mswindows: diff --git a/pelita/scripts/pelita_server.py b/pelita/scripts/pelita_server.py index 3db613a1a..2591bc97d 100755 --- a/pelita/scripts/pelita_server.py +++ b/pelita/scripts/pelita_server.py @@ -506,7 +506,7 @@ def play_remote(team_spec, pair_addr, silent_bots=False): pair_addr, *(['--silent-bots'] if silent_bots else []), ] - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) sub = subprocess.Popen(external_call) return sub @@ -517,7 +517,7 @@ def _check_team(team_spec): 'pelita.scripts.pelita_player', 'check-team', team_spec] - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) res = subprocess.run(external_call, capture_output=True, text=True) return res.stdout.strip() diff --git a/pelita/scripts/pelita_tournament.py b/pelita/scripts/pelita_tournament.py index 7515561c1..e53635da5 100755 --- a/pelita/scripts/pelita_tournament.py +++ b/pelita/scripts/pelita_tournament.py @@ -37,29 +37,6 @@ def firstNN(*args): """ return next(filter(lambda x: x is not None, args), None) - -def shlex_unsplit(cmd): - """ - Translates a list of command arguments into bash-like ‘human’ readable form. - Pseudo-reverses shlex.split() - - Example - ------- - >>> shlex_unsplit(["command", "-f", "Hello World"]) - "command -f 'Hello World'" - - Parameters - ---------- - cmd : list of string - command + parameter list - - Returns - ------- - string - """ - return " ".join(shlex.quote(arg) for arg in cmd) - - def create_directory(prefix): for suffix in itertools.count(0): path = Path('{}-{:02d}'.format(prefix, suffix)) @@ -139,9 +116,9 @@ def setup(): print("Please enter the location of the sound-giving binary:") sound_path = input() elif res == "s": - sound_path = shlex_unsplit(sound["say"]) + sound_path = shlex.join(sound["say"]) elif res == "f": - sound_path = shlex_unsplit(sound["flite"]) + sound_path = shlex.join(sound["flite"]) else: continue diff --git a/pelita/team.py b/pelita/team.py index a8dee66f1..83e3d9a43 100644 --- a/pelita/team.py +++ b/pelita/team.py @@ -1,6 +1,7 @@ import logging import os +import shlex import subprocess import sys import time @@ -495,7 +496,7 @@ def _call_pelita_player(self, team_spec, address, color='', store_output=False): team_spec, address] - _logger.debug("Executing: %r", external_call) + _logger.debug("Executing: %r", shlex.join(external_call)) if store_output == subprocess.DEVNULL: return (subprocess.Popen(external_call, stdout=store_output), None, None) elif store_output: From 3c769f10ebe3838539130520fa47f9c6fb0d0bc0 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Mon, 26 May 2025 17:11:56 +0200 Subject: [PATCH 02/25] NF: Add thread count to cli --- contrib/ci_engine.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 62906bb91..16c63f15c 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -169,7 +169,7 @@ def run_game(self, p1, p2): self.dbwrapper.add_gameresult(p1_name, p2_name, result, final_state, stdout, stderr) - def start(self, n): + def start(self, n, thread_count): """Start the Engine. This method will start and infinite loop, testing each agent @@ -810,11 +810,12 @@ def get_wins_losses(self, team=None): type=click.File('r'), help='Configuration file') @click.option('-n', help='run N times', type=int, default=0) +@click.option('--thread-count', '-t', help='run in parallel', type=int, default=0) @click.option('--print', is_flag=True, default=False, help='Print scores and exit.') @click.option('--nohash', is_flag=True, default=False, help='Do not hash the players') -def main(log, config, n, print, nohash): +def main(log, config, n, thread_count, print, nohash): if log is not None: start_logging(log, __name__) start_logging(log, 'pelita') @@ -825,7 +826,7 @@ def main(log, config, n, print, nohash): else: if not nohash: ci_engine.load_players() - ci_engine.start(n) + ci_engine.start(n, thread_count) if __name__ == '__main__': main() From bd12a787eed81d42d3aad19e12ac0bae90cf4f2e Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Fri, 6 Jun 2025 13:13:58 +0200 Subject: [PATCH 03/25] NF: Restructure ci_engine to employ concurrent worker threads Adds an optional event flag to call_pelita to break the loop --- contrib/ci_engine.py | 81 ++++++++++++++++++++++++++++++++--- pelita/tournament/__init__.py | 7 ++- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 16c63f15c..47afd060c 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -46,10 +46,13 @@ import itertools import json import logging +import queue import shlex +import signal import sqlite3 import subprocess import sys +import threading from random import Random import click @@ -65,6 +68,15 @@ # the path of the configuration file CFG_FILE = './ci.cfg' +EXIT = threading.Event() + +def signal_handler(signal, frame): + _logger.warning('Program terminated by kill or ctrl-c') + EXIT.set() + sys.exit(0) + +signal.signal(signal.SIGINT, signal_handler) + def hash_team(team_spec): external_call = [sys.executable, '-m', @@ -145,13 +157,20 @@ def run_game(self, p1, p2): """ team_specs = [self.players[i]['path'] for i in (p1, p2)] - print(f"Playing {self.players[p1]['name']} against {self.players[p2]['name']}.") final_state, stdout, stderr = call_pelita(team_specs, rounds=self.rounds, size=self.size, viewer=self.viewer, - seed=self.seed) + seed=self.seed, + exit_flag=EXIT + ) + + if not final_state: + print(stdout, stderr) + p1_name, p2_name = self.players[p1]['name'], self.players[p2]['name'] + res = (p1_name, p2_name, None, final_state, stdout, stderr) + return res if final_state['whowins'] == 2: result = -1 @@ -166,7 +185,8 @@ def run_game(self, p1, p2): if stderr: _logger.warning('Stderr: %r', stderr) p1_name, p2_name = self.players[p1]['name'], self.players[p2]['name'] - self.dbwrapper.add_gameresult(p1_name, p2_name, result, final_state, stdout, stderr) + res = (p1_name, p2_name, result, final_state, stdout, stderr) + return res def start(self, n, thread_count): @@ -187,7 +207,29 @@ def start(self, n, thread_count): loop = itertools.repeat(None) if n == 0 else itertools.repeat(None, n) rng = Random() - for _ in loop: + def worker(q, r, lock=threading.Lock()): + for task in iter(q.get, None): # blocking get until None is received + try: + # print(task) + count, slf, p1, p2 = task + + print(f"Playing #{count}: {self.players[p1]['name']} against {self.players[p2]['name']}.") + + res = slf.run_game(p1, p2) + r.put((count, (p1, p2), res)) + #with lock: + finally: + q.task_done() + + worker_count = thread_count + q = queue.Queue(maxsize=thread_count) + r = queue.Queue() + threads = [threading.Thread(target=worker, args=[q, r], daemon=False) + for _ in range(worker_count)] + for t in threads: + t.start() + + for count, _ in enumerate(loop): # choose the player with the least number of played game, # match with another random player # mix the sides and let them play @@ -199,9 +241,30 @@ def start(self, n, thread_count): players = [a, b] rng.shuffle(players) - self.run_game(players[0], players[1]) - self.pretty_print_results(highlight=[self.players[players[0]]['name'], self.players[players[1]]['name']]) - print('------------------------------') + q.put((count, self, players[0], players[1])) + + try: + count, players, res = r.get_nowait() + print(f"Storing #{count}: {self.players[players[0]]['name']} against {self.players[players[1]]['name']}.") + self.dbwrapper.add_gameresult(*res) + except queue.Empty: + pass + + q.join() # block until all spawned tasks are done + + while True: + try: + count, players, res = r.get_nowait() + print(f"Storing #{count}: {self.players[players[0]]['name']} against {self.players[players[1]]['name']}.") + self.dbwrapper.add_gameresult(*res) + except queue.Empty: + break + + for _ in threads: # signal workers to quit + q.put(None) + + for t in threads: # wait until workers exit + t.join() def get_results(self, idx, idx2=None): @@ -308,12 +371,14 @@ def elo_change(a, b, outcome): FROM games """).fetchall() for p1, p2, result in g: + change = 0 if result == 0: change = elo_change(elo[p1], elo[p2], 1) if result == 1: change = elo_change(elo[p1], elo[p2], 0) if result == -1: change = elo_change(elo[p1], elo[p2], 0.5) + elo[p1] += change elo[p2] -= change @@ -583,6 +648,8 @@ def add_gameresult(self, p1_name, p2_name, result, final_state, std_out, std_err STDOUT and STDERR of the game """ + if not final_state: + return self.cursor.execute(""" INSERT INTO games VALUES (?, ?, ?, ?, ?, ?) diff --git a/pelita/tournament/__init__.py b/pelita/tournament/__init__.py index a2193edac..4f90c962e 100644 --- a/pelita/tournament/__init__.py +++ b/pelita/tournament/__init__.py @@ -98,7 +98,7 @@ def run_and_terminate_process(args, **kwargs): p.kill() -def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, write_replay=False, store_output=False): +def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, write_replay=False, store_output=False, exit_flag=None): """ Starts a new process with the given command line arguments and waits until finished. Returns @@ -167,6 +167,11 @@ def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, writ while True: evts = dict(poll.poll(1000)) + if exit_flag and exit_flag.is_set(): + # An external process tells us to quit + _logger.info("Received exit signal") + break + if not evts and proc.poll() is not None: # no more events and proc has finished. # we break the loop From 6a3b1ce6777fbc888e83b7face20d466301c15f8 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Fri, 6 Jun 2025 13:19:02 +0200 Subject: [PATCH 04/25] RF: Remove click --- contrib/ci_engine.py | 64 +++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 47afd060c..98d73373c 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -41,7 +41,7 @@ """ - +import argparse import configparser import itertools import json @@ -55,7 +55,6 @@ import threading from random import Random -import click from rich.console import Console from rich.table import Table @@ -868,32 +867,41 @@ def get_wins_losses(self, team=None): return self.cursor.execute(query).fetchall() -@click.command() -@click.option('--log', - is_flag=False, flag_value="-", default=None, metavar='LOGFILE', - help="print debugging log information to LOGFILE (default 'stderr')") -@click.option('--config', - default=CFG_FILE, - type=click.File('r'), - help='Configuration file') -@click.option('-n', help='run N times', type=int, default=0) -@click.option('--thread-count', '-t', help='run in parallel', type=int, default=0) -@click.option('--print', is_flag=True, default=False, - help='Print scores and exit.') -@click.option('--nohash', is_flag=True, default=False, - help='Do not hash the players') -def main(log, config, n, thread_count, print, nohash): - if log is not None: - start_logging(log, __name__) - start_logging(log, 'pelita') - - ci_engine = CI_Engine(config) - if print: - ci_engine.pretty_print_results() - else: - if not nohash: +def run(args): + with open(args.config) as f: + ci_engine = CI_Engine(f) + if not args.no_hash: ci_engine.load_players() - ci_engine.start(n, thread_count) + ci_engine.start(args.n, args.thread_count) + +def print_scores(args): + with open(args.config) as f: + ci_engine = CI_Engine(f) + ci_engine.pretty_print_results() + if __name__ == '__main__': - main() + parser = argparse.ArgumentParser() + parser.add_argument('--log', help="Print debugging log information to LOGFILE (default 'stderr').", + metavar='LOGFILE', const='-', nargs='?') + parser.add_argument('--config', help="Print debugging log information to LOGFILE (default 'stderr').", + metavar='FILE', default=CFG_FILE) + + subparsers = parser.add_subparsers(required=True) + + parser_run = subparsers.add_parser('run') + parser_run.add_argument('-n', help='run N times', type=int, default=0) + parser_run.add_argument('--thread-count', '-t', help='run in parallel', type=int, default=1) + parser_run.add_argument('--no-hash', help='Do not hash the players prior to running', type=bool, default=False) + parser_run.set_defaults(func=run) + + parser_print_scores = subparsers.add_parser('print-scores') + parser_print_scores.set_defaults(func=print_scores) + + args = parser.parse_args() + + if args.log is not None: + start_logging(args.log, __name__) + start_logging(args.log, 'pelita') + + args.func(args) From 366c8a1d60b5fd4d91e04d37c4a9d41dc8ce1972 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Fri, 6 Jun 2025 13:21:51 +0200 Subject: [PATCH 05/25] ENH: Improve counting logic --- contrib/ci_engine.py | 72 +++++++++++++++++++++++++++++++++------ contrib/test_ci_engine.py | 2 ++ 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 98d73373c..0dbd608cc 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -42,7 +42,9 @@ """ import argparse +import collections import configparser +import heapq import itertools import json import logging @@ -166,7 +168,6 @@ def run_game(self, p1, p2): ) if not final_state: - print(stdout, stderr) p1_name, p2_name = self.players[p1]['name'], self.players[p2]['name'] res = (p1_name, p2_name, None, final_state, stdout, stderr) return res @@ -206,10 +207,22 @@ def start(self, n, thread_count): loop = itertools.repeat(None) if n == 0 else itertools.repeat(None, n) rng = Random() + game_counts = self.dbwrapper.get_game_counts() + game_count_heap = [] + for player_id, player in enumerate(self.players): + if player.get('error'): + continue + + count = game_counts[player['name']] + + tie_breaker = rng.random() + val = [count, tie_breaker, player_id] + heapq.heappush(game_count_heap, val) + + def worker(q, r, lock=threading.Lock()): for task in iter(q.get, None): # blocking get until None is received try: - # print(task) count, slf, p1, p2 = task print(f"Playing #{count}: {self.players[p1]['name']} against {self.players[p2]['name']}.") @@ -232,16 +245,20 @@ def worker(q, r, lock=threading.Lock()): # choose the player with the least number of played game, # match with another random player # mix the sides and let them play - broken_players = {idx for idx, player in enumerate(self.players) if player.get('error')} - game_count = [(self.dbwrapper.get_game_count(p['name']), idx) for idx, p in enumerate(self.players)] - players_sorted = [idx for count, idx in sorted(game_count) if idx not in broken_players] - a, rest = players_sorted[0], players_sorted[1:] - b = rng.choice(rest) - players = [a, b] + + a = heapq.heappop(game_count_heap) + b_i = rng.randrange(len(game_count_heap)) + b = game_count_heap[b_i] + players = [a[2], b[2]] rng.shuffle(players) q.put((count, self, players[0], players[1])) + del game_count_heap[b_i] + game_count_heap.append([b[0] + 1, rng.random(), b[2]]) + heapq.heapify(game_count_heap) + heapq.heappush(game_count_heap, [a[0] + 1, rng.random(), a[2]]) + try: count, players, res = r.get_nowait() print(f"Storing #{count}: {self.players[players[0]]['name']} against {self.players[players[1]]['name']}.") @@ -363,7 +380,7 @@ def elo_change(a, b, outcome): return k * (outcome - expected) from collections import defaultdict - elo = defaultdict(lambda: 1500) + elo = defaultdict(lambda: 1500.) g = self.dbwrapper.cursor.execute(""" SELECT player1, player2, result @@ -707,6 +724,32 @@ def get_team_name(self, p_name): raise ValueError('Player %s does not exist in database.' % p_name) return res[0] + def get_game_counts(self): + """Get number of games per player. + + Returns + ------- + relevant_results : dict[name, int] + + """ + self.cursor.execute(""" + SELECT p.name, COUNT(g.player) AS num_games + FROM + players p + LEFT JOIN + ( + SELECT player1 AS player FROM games + UNION ALL + SELECT player2 AS player FROM games + ) g + ON p.name = g.player + GROUP BY p.name + """) + counts = collections.Counter() + for name, val in self.cursor.fetchall(): + counts[name] += val + return counts + def get_game_count(self, p1_name, p2_name=None): """Get number of games involving player1 (AND player2 if specified). @@ -879,6 +922,11 @@ def print_scores(args): ci_engine = CI_Engine(f) ci_engine.pretty_print_results() +def hash_teams(args): + with open(args.config) as f: + ci_engine = CI_Engine(f) + ci_engine.load_players() + if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -892,12 +940,16 @@ def print_scores(args): parser_run = subparsers.add_parser('run') parser_run.add_argument('-n', help='run N times', type=int, default=0) parser_run.add_argument('--thread-count', '-t', help='run in parallel', type=int, default=1) - parser_run.add_argument('--no-hash', help='Do not hash the players prior to running', type=bool, default=False) + parser_run.add_argument('--no-hash', help='Do not hash the players prior to running', action='store_true', default=False) parser_run.set_defaults(func=run) parser_print_scores = subparsers.add_parser('print-scores') parser_print_scores.set_defaults(func=print_scores) + parser_hash = subparsers.add_parser('hash-teams') + parser_hash.set_defaults(func=hash_teams) + parser_hash.add_argument('--thread-count', '-t', help='run in parallel', type=int, default=1) + args = parser.parse_args() if args.log is not None: diff --git a/contrib/test_ci_engine.py b/contrib/test_ci_engine.py index 2e59e5218..22f5a24ef 100644 --- a/contrib/test_ci_engine.py +++ b/contrib/test_ci_engine.py @@ -137,3 +137,5 @@ def test_wins_losses(db_wrapper): assert db_wrapper.get_game_count('p1', 'p2') == 3 assert db_wrapper.get_game_count('p2', 'p1') == 3 assert db_wrapper.get_game_count('p3', 'p1') == 1 + + assert db_wrapper.get_game_counts() == dict(p1=4, p2=3, p3=1) From 246807da320097f0a30d96f92596f93b5da621a2 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Wed, 4 Jun 2025 22:13:22 +0200 Subject: [PATCH 06/25] ENH: Use exit flag in main loop --- contrib/ci_engine.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 0dbd608cc..0baeff35a 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -71,10 +71,10 @@ EXIT = threading.Event() -def signal_handler(signal, frame): +def signal_handler(_signal, _frame): _logger.warning('Program terminated by kill or ctrl-c') EXIT.set() - sys.exit(0) + sys.exit() signal.signal(signal.SIGINT, signal_handler) @@ -265,6 +265,11 @@ def worker(q, r, lock=threading.Lock()): self.dbwrapper.add_gameresult(*res) except queue.Empty: pass + except Exception: + pass + + if EXIT.is_set(): + break q.join() # block until all spawned tasks are done From 1eca2a13d2124a9f160034e1f6649151978a6e95 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Fri, 6 Jun 2025 15:56:42 +0200 Subject: [PATCH 07/25] BLD: Fix test --- .github/workflows/test_pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_pytest.yml b/.github/workflows/test_pytest.yml index 6ef157b3f..8e7af391e 100644 --- a/.github/workflows/test_pytest.yml +++ b/.github/workflows/test_pytest.yml @@ -129,5 +129,5 @@ jobs: - name: Run ci_engine session run: | cd contrib - python ci_engine.py -n 5 + python ci_engine.py run -n 5 timeout-minutes: 5 From 854ea9c6b192be5b0b81a3db4b1cdda4633e9495 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Sat, 7 Jun 2025 00:32:01 +0200 Subject: [PATCH 08/25] RF: Elo in sqlite --- contrib/ci_engine.py | 83 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 0baeff35a..820e9df2b 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -498,7 +498,8 @@ def batched(iterable, n): table.add_column("Error count") table.add_column("# Fatal Errors") - elo = self.gen_elo() + elo = dict(self.dbwrapper.get_elo()) + # elo = self.gen_elo() result.sort(reverse=True) for [score, win, draw, loss, name, team_name, error_count, fatalerror_count] in result: @@ -511,7 +512,7 @@ def batched(iterable, n): f"{draw}", f"{loss}", f"{score:6.3f}", - f"{elo[name]: >4.0f}", + f"{elo.get(name, 0): >4.0f}", f"{error_count}", f"{fatalerror_count}", style=style, @@ -915,6 +916,84 @@ def get_wins_losses(self, team=None): return self.cursor.execute(query).fetchall() + def get_elo(self): + query = """ + WITH RECURSIVE + ordered_matches AS ( + SELECT + ROW_NUMBER() OVER (ORDER BY rowid) AS match_num, + player1, + player2, + result + FROM games + ), + + -- Initialize with first match + elo_recursive(match_num, player1, player2, result, + rating1, rating2, + rating_json) AS ( + SELECT + match_num, + player1, + player2, + result, + 1500.0, + 1500.0, + json_object(player1, 1500.0, player2, 1500.0) + FROM ordered_matches + WHERE match_num = 1 + + UNION ALL + + SELECT + om.match_num, + om.player1, + om.player2, + om.result, + + -- Get ratings from JSON state + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0), + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0), + + -- Update JSON state with new ratings + json_set( + er.rating_json, + '$.' || om.player1, + ROUND( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0) + + 32 * ((CASE om.result WHEN 0 THEN 1.0 WHEN -1 THEN 0.5 ELSE 0.0 END) - + 1.0 / (1 + pow(10, ( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0) - + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0) + ) / 400.0))), 2), + '$.' || om.player2, + ROUND( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0) + + 32 * ((CASE om.result WHEN 0 THEN 0.0 WHEN -1 THEN 0.5 ELSE 1.0 END) - + 1.0 / (1 + pow(10, ( + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player1) AS REAL), 1500.0) - + IFNULL(CAST(json_extract(er.rating_json, '$.' || om.player2) AS REAL), 1500.0) + ) / 400.0))), 2) + ) + FROM ordered_matches om + JOIN elo_recursive er ON om.match_num = er.match_num + 1 + ), + + final AS ( + SELECT rating_json + FROM elo_recursive + ORDER BY match_num DESC + LIMIT 1 + ) + SELECT + key AS player, + ROUND(value, 2) AS rating + FROM final, json_each(rating_json) + ORDER BY rating DESC; + + """ + return self.cursor.execute(query).fetchall() + def run(args): with open(args.config) as f: ci_engine = CI_Engine(f) From 6a01ff5120e35cca66a97656485b058bd3380f64 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Sat, 7 Jun 2025 12:49:15 +0200 Subject: [PATCH 09/25] NF: Use asyncio for hashing --- contrib/ci_engine.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 820e9df2b..a9457a197 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -42,6 +42,7 @@ """ import argparse +import asyncio import collections import configparser import heapq @@ -52,7 +53,6 @@ import shlex import signal import sqlite3 -import subprocess import sys import threading from random import Random @@ -78,15 +78,21 @@ def signal_handler(_signal, _frame): signal.signal(signal.SIGINT, signal_handler) -def hash_team(team_spec): +async def hash_team(team_spec, semaphore): external_call = [sys.executable, '-m', 'pelita.scripts.pelita_player', 'hash-team', team_spec] - _logger.debug("Executing: %r", shlex.join(external_call)) - res = subprocess.run(external_call, capture_output=True, text=True) - return res.stdout.strip().split("\n")[-1].strip() + async with semaphore: + _logger.debug("Executing: %r", shlex.join(external_call)) + proc = await asyncio.create_subprocess_exec(*external_call, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await proc.communicate() + + return stdout.decode().strip().split("\n")[-1].strip() class CI_Engine: """Continuous Integration Engine.""" @@ -106,7 +112,7 @@ def __init__(self, cfgfile): self.db_file = config.get('general', 'db_file') self.dbwrapper = DB_Wrapper(self.db_file) - def load_players(self): + def load_players(self, concurrency=1): hash_cache = {} # remove players from db which are not in the config anymore @@ -115,19 +121,27 @@ def load_players(self): _logger.debug('Removing %s from database, because it is not among the current players.' % (pname)) self.dbwrapper.remove_player(pname) + semaphore = asyncio.Semaphore(concurrency) + + async def do_hash(): + tasks = [asyncio.create_task(hash_team(player['path'], semaphore)) for player in self.players] + hashes = await asyncio.gather(*tasks) + return {player['path']: hash for (player, hash) in zip(self.players, hashes)} + + hash_cache = asyncio.run(do_hash()) + # add new players into db for player in self.players: pname, path = player['name'], player['path'] if pname not in self.dbwrapper.get_players(): _logger.debug('Adding %s to database.' % pname) - hash_cache[path] = hash_team(path) self.dbwrapper.add_player(pname, hash_cache[path]) # reset players where the directory hash changed for player in self.players: path = player['path'] pname = player['name'] - new_hash = hash_cache.get(path, hash_team(path)) + new_hash = hash_cache.get(path) if new_hash != self.dbwrapper.get_player_hash(pname): _logger.debug('Resetting %s because its module hash changed.' % pname) self.dbwrapper.remove_player(pname) @@ -998,7 +1012,7 @@ def run(args): with open(args.config) as f: ci_engine = CI_Engine(f) if not args.no_hash: - ci_engine.load_players() + ci_engine.load_players(concurrency=args.thread_count) ci_engine.start(args.n, args.thread_count) def print_scores(args): @@ -1009,7 +1023,7 @@ def print_scores(args): def hash_teams(args): with open(args.config) as f: ci_engine = CI_Engine(f) - ci_engine.load_players() + ci_engine.load_players(concurrency=args.thread_count) if __name__ == '__main__': From c722a09f83894e88f1e19fb17f73d4fef6e1b04d Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Sat, 7 Jun 2025 12:57:10 +0200 Subject: [PATCH 10/25] ENH: Simpler sorting logic --- contrib/ci_engine.py | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index a9457a197..bb4f353a2 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -45,10 +45,10 @@ import asyncio import collections import configparser -import heapq import itertools import json import logging +import operator import queue import shlex import signal @@ -222,17 +222,6 @@ def start(self, n, thread_count): rng = Random() game_counts = self.dbwrapper.get_game_counts() - game_count_heap = [] - for player_id, player in enumerate(self.players): - if player.get('error'): - continue - - count = game_counts[player['name']] - - tie_breaker = rng.random() - val = [count, tie_breaker, player_id] - heapq.heappush(game_count_heap, val) - def worker(q, r, lock=threading.Lock()): for task in iter(q.get, None): # blocking get until None is received @@ -260,18 +249,17 @@ def worker(q, r, lock=threading.Lock()): # match with another random player # mix the sides and let them play - a = heapq.heappop(game_count_heap) - b_i = rng.randrange(len(game_count_heap)) - b = game_count_heap[b_i] - players = [a[2], b[2]] + players_sorted = sorted(list(game_counts.items()), key=operator.itemgetter(1)) + a = players_sorted[0][0] + b = rng.choice(players_sorted[1:])[0] + + players = [a, b] rng.shuffle(players) q.put((count, self, players[0], players[1])) - del game_count_heap[b_i] - game_count_heap.append([b[0] + 1, rng.random(), b[2]]) - heapq.heapify(game_count_heap) - heapq.heappush(game_count_heap, [a[0] + 1, rng.random(), a[2]]) + game_counts[a] += 1 + game_counts[b] += 1 try: count, players, res = r.get_nowait() From acafc9abe1c761cc892adfc5070caf7a5076f986 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Sat, 7 Jun 2025 13:28:58 +0200 Subject: [PATCH 11/25] RF: Change internal data structure --- contrib/ci_engine.py | 85 ++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index bb4f353a2..c407d1615 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -98,11 +98,11 @@ class CI_Engine: """Continuous Integration Engine.""" def __init__(self, cfgfile): - self.players = [] + self.players = {} config = configparser.ConfigParser() config.read_file(cfgfile) for name, path in config.items('agents'): - self.players.append({'name': name, 'path': path}) + self.players[name]= {'path': path} self.rounds = config['general'].getint('rounds', None) self.size = config['general'].get('size', None) @@ -117,39 +117,38 @@ def load_players(self, concurrency=1): # remove players from db which are not in the config anymore for pname in self.dbwrapper.get_players(): - if pname not in [p['name'] for p in self.players]: + if pname not in self.players: _logger.debug('Removing %s from database, because it is not among the current players.' % (pname)) self.dbwrapper.remove_player(pname) semaphore = asyncio.Semaphore(concurrency) async def do_hash(): - tasks = [asyncio.create_task(hash_team(player['path'], semaphore)) for player in self.players] + players = [(pname, player['path']) for pname, player in self.players.items()] + tasks = [asyncio.create_task(hash_team(player[1], semaphore)) for player in players] hashes = await asyncio.gather(*tasks) - return {player['path']: hash for (player, hash) in zip(self.players, hashes)} + return {player[0]: hash for (player, hash) in zip(players, hashes)} hash_cache = asyncio.run(do_hash()) # add new players into db - for player in self.players: - pname, path = player['name'], player['path'] + for pname, player in self.players.items(): + path = player['path'] if pname not in self.dbwrapper.get_players(): _logger.debug('Adding %s to database.' % pname) - self.dbwrapper.add_player(pname, hash_cache[path]) + self.dbwrapper.add_player(pname, hash_cache[pname]) # reset players where the directory hash changed - for player in self.players: + for pname, player in self.players.items(): path = player['path'] - pname = player['name'] - new_hash = hash_cache.get(path) + new_hash = hash_cache[pname] if new_hash != self.dbwrapper.get_player_hash(pname): _logger.debug('Resetting %s because its module hash changed.' % pname) self.dbwrapper.remove_player(pname) self.dbwrapper.add_player(pname, new_hash) - for player in self.players: + for pname, player in self.players.items(): path = player['path'] - pname = player['name'] try: _logger.debug('Querying team name for %s.' % pname) team_name = check_team(player['path']) @@ -171,7 +170,7 @@ def run_game(self, p1, p2): the indices of the players """ - team_specs = [self.players[i]['path'] for i in (p1, p2)] + team_specs = [self.players[p1]['path'], self.players[p2]['path']] final_state, stdout, stderr = call_pelita(team_specs, rounds=self.rounds, @@ -182,8 +181,7 @@ def run_game(self, p1, p2): ) if not final_state: - p1_name, p2_name = self.players[p1]['name'], self.players[p2]['name'] - res = (p1_name, p2_name, None, final_state, stdout, stderr) + res = (p1, p2, None, final_state, stdout, stderr) return res if final_state['whowins'] == 2: @@ -198,8 +196,7 @@ def run_game(self, p1, p2): _logger.debug('Stdout: %r', stdout) if stderr: _logger.warning('Stderr: %r', stderr) - p1_name, p2_name = self.players[p1]['name'], self.players[p2]['name'] - res = (p1_name, p2_name, result, final_state, stdout, stderr) + res = (p1, p2, result, final_state, stdout, stderr) return res @@ -228,7 +225,7 @@ def worker(q, r, lock=threading.Lock()): try: count, slf, p1, p2 = task - print(f"Playing #{count}: {self.players[p1]['name']} against {self.players[p2]['name']}.") + print(f"Playing #{count}: {p1} against {p2}.") res = slf.run_game(p1, p2) r.put((count, (p1, p2), res)) @@ -263,7 +260,11 @@ def worker(q, r, lock=threading.Lock()): try: count, players, res = r.get_nowait() - print(f"Storing #{count}: {self.players[players[0]]['name']} against {self.players[players[1]]['name']}.") + final_state = res[3] + if final_state: + print(f"Storing #{count}: {players[0]} against {players[1]}.") + else: + print(f"Not storing #{count}: {players[0]} against {players[1]}.") self.dbwrapper.add_gameresult(*res) except queue.Empty: pass @@ -278,7 +279,11 @@ def worker(q, r, lock=threading.Lock()): while True: try: count, players, res = r.get_nowait() - print(f"Storing #{count}: {self.players[players[0]]['name']} against {self.players[players[1]]['name']}.") + final_state = res[3] + if final_state: + print(f"Storing #{count}: {players[0]} against {players[1]}.") + else: + print(f"Not storing #{count}: {players[0]} against {players[1]}.") self.dbwrapper.add_gameresult(*res) except queue.Empty: break @@ -290,7 +295,7 @@ def worker(q, r, lock=threading.Lock()): t.join() - def get_results(self, idx, idx2=None): + def get_results(self, p1_name, p2_name=None): """Get the results so far. This method goes through the internal list of of all game @@ -332,18 +337,16 @@ def get_results(self, idx, idx2=None): """ win, loss, draw = 0, 0, 0 - p1_name = self.players[idx]['name'] - p2_name = None if idx2 is None else self.players[idx2]['name'] relevant_results = self.dbwrapper.get_results(p1_name, p2_name) for p1, p2, r in relevant_results: - if (idx2 is None and p1_name == p1) or (idx2 is not None and p1_name == p1 and p2_name == p2): + if (p2_name is None and p1_name == p1) or (p2_name is not None and p1_name == p1 and p2_name == p2): if r == 0: win += 1 elif r == 1: loss += 1 elif r == -1: draw += 1 - if (idx2 is None and p1_name == p2) or (idx2 is not None and p1_name == p2 and p2_name == p1): + if (p2_name is None and p1_name == p2) or (p2_name is not None and p1_name == p2 and p2_name == p1): if r == 1: win += 1 elif r == 0: @@ -352,7 +355,7 @@ def get_results(self, idx, idx2=None): draw += 1 return win, loss, draw - def get_errorcount(self, idx): + def get_errorcount(self, p_name): """Gets the error count for team idx Parameters @@ -366,17 +369,15 @@ def get_errorcount(self, idx): the number of errors for this player """ - p_name = self.players[idx]['name'] error_count, fatalerror_count = self.dbwrapper.get_errorcount(p_name) return error_count, fatalerror_count - def get_team_name(self, idx): + def get_team_name(self, p_name): """Get last registered team name. team_name : string """ - p_name = self.players[idx]['name'] return self.dbwrapper.get_team_name(p_name) def gen_elo(self): @@ -424,8 +425,8 @@ def pretty_print_results(self, highlight=None): res = self.dbwrapper.get_wins_losses() rows = { k: list(v) for k, v in itertools.groupby(res, key=lambda x:x[0]) } - good_players = [p for p in self.players if not p.get('error')] - bad_players = [p for p in self.players if p.get('error')] + good_players = [p for p, player in self.players.items() if not player.get('error')] + bad_players = [p for p, player in self.players.items() if player.get('error')] num_rows_per_player = (len(good_players) // MAX_COLUMNS) + 1 row_style = [*([""] * num_rows_per_player), *(["dim"] * num_rows_per_player)] @@ -455,26 +456,26 @@ def batched(iterable, n): yield batch result = [] - for idx, p in enumerate(good_players): - win, loss, draw = self.get_results(idx) - error_count, fatalerror_count = self.get_errorcount(idx) + for idx, pname in enumerate(good_players): + win, loss, draw = self.get_results(pname) + error_count, fatalerror_count = self.get_errorcount(pname) try: - team_name = self.get_team_name(idx) + team_name = self.get_team_name(pname) except ValueError: team_name = None score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) - result.append([score, win, draw, loss, p['name'], team_name, error_count, fatalerror_count]) + result.append([score, win, draw, loss, pname, team_name, error_count, fatalerror_count]) wdl = f"{win:3d},{draw:3d},{loss:3d}" try: - row = rows[p['name']] + row = rows[pname] except KeyError: continue vals = { k: (w,l,d) for _p1, k, w, l, d in row } cross_results = [] - for idx2, p2 in enumerate(good_players): - win, loss, draw = vals.get(p2['name'], (0, 0, 0)) + for idx2, p2name in enumerate(good_players): + win, loss, draw = vals.get(p2name, (0, 0, 0)) if idx == idx2: cross_results.append(" - - - ") else: @@ -482,7 +483,7 @@ def batched(iterable, n): for c, r in enumerate(batched(cross_results, MAX_COLUMNS)): if c == 0: - table.add_row(f"{idx}", p['name'], f"{score:.2f}", wdl, *r) + table.add_row(f"{idx}", pname, f"{score:.2f}", wdl, *r) else: table.add_row("", "", "", "", *r) @@ -523,7 +524,7 @@ def batched(iterable, n): console.print(table) for p in bad_players: - print("% 30s ***%30s***" % (p['name'], p['error'])) + print("% 30s ***%30s***" % (p, self.players[p]['error'])) class DB_Wrapper: From 0d1260c7352bd335bcf1754b4ee6f4727fbc6075 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Sat, 7 Jun 2025 14:45:53 +0200 Subject: [PATCH 12/25] ENH: Use ThreadPoolExecutor to query for team names --- contrib/ci_engine.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index c407d1615..14a17c22c 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -44,6 +44,7 @@ import argparse import asyncio import collections +from concurrent.futures import ThreadPoolExecutor import configparser import itertools import json @@ -147,16 +148,27 @@ async def do_hash(): self.dbwrapper.remove_player(pname) self.dbwrapper.add_player(pname, new_hash) - for pname, player in self.players.items(): - path = player['path'] + def check_team_name(args): + pname, path = args try: _logger.debug('Querying team name for %s.' % pname) - team_name = check_team(player['path']) - self.dbwrapper.add_team_name(pname, team_name) + team_name = check_team(path) + return { 'team_name': team_name } except RemotePlayerFailure as e: e_type, e_msg = e.args - _logger.debug(f'Could not import {player} at path {path} ({e_type}): {e_msg}') - player['error'] = e.args + _logger.debug(f'Could not import {pname} at path {path} ({e_type}): {e_msg}') + return { 'error': e.args } + + with ThreadPoolExecutor(max_workers=concurrency) as executor: + players = [(pname, player['path']) for pname, player in self.players.items()] + team_names = executor.map(check_team_name, players) + + for (pname, path), team_name in zip(players, team_names): + if 'error' in team_name: + self.players[pname]['error'] = team_name['error'] + else: + self.dbwrapper.add_team_name(pname, team_name['team_name']) + def run_game(self, p1, p2): """Run a single game. From c90819f081585871b9bf7b031d2e0289bf9675e5 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Sat, 7 Jun 2025 22:21:58 +0200 Subject: [PATCH 13/25] ENH: Delete players that have an error --- contrib/ci_engine.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 14a17c22c..6edb84271 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -169,6 +169,12 @@ def check_team_name(args): else: self.dbwrapper.add_team_name(pname, team_name['team_name']) + for pname in self.players: + if 'error' in self.players[pname]: + print(pname, self.players[pname]) + else: + print(pname, self.players[pname], self.dbwrapper.get_team_name(pname)) + def run_game(self, p1, p2): """Run a single game. @@ -232,6 +238,10 @@ def start(self, n, thread_count): game_counts = self.dbwrapper.get_game_counts() + for pname, player in self.players.items(): + if "error" in player and pname in game_counts: + del game_counts[pname] + def worker(q, r, lock=threading.Lock()): for task in iter(q.get, None): # blocking get until None is received try: @@ -259,6 +269,7 @@ def worker(q, r, lock=threading.Lock()): # mix the sides and let them play players_sorted = sorted(list(game_counts.items()), key=operator.itemgetter(1)) + a = players_sorted[0][0] b = rng.choice(players_sorted[1:])[0] From 9bf8c7313688ce81438bfa0bf2984ab0dc42227c Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Wed, 10 Sep 2025 01:14:33 +0200 Subject: [PATCH 14/25] ENH: Use longer initial timeout --- contrib/ci_engine.py | 4 +++- pelita/tournament/__init__.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 6edb84271..94ad8570c 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -152,7 +152,7 @@ def check_team_name(args): pname, path = args try: _logger.debug('Querying team name for %s.' % pname) - team_name = check_team(path) + team_name = check_team(path, timeout=6*concurrency) return { 'team_name': team_name } except RemotePlayerFailure as e: e_type, e_msg = e.args @@ -195,6 +195,8 @@ def run_game(self, p1, p2): size=self.size, viewer=self.viewer, seed=self.seed, + timeout=10, + initial_timeout=120, exit_flag=EXIT ) diff --git a/pelita/tournament/__init__.py b/pelita/tournament/__init__.py index 4f90c962e..decd9a27c 100644 --- a/pelita/tournament/__init__.py +++ b/pelita/tournament/__init__.py @@ -98,7 +98,8 @@ def run_and_terminate_process(args, **kwargs): p.kill() -def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, write_replay=False, store_output=False, exit_flag=None): +def call_pelita(team_specs, *, rounds, size, viewer, seed, timeout=3, initial_timeout=6, + team_infos=None, write_replay=False, store_output=False, exit_flag=None): """ Starts a new process with the given command line arguments and waits until finished. Returns @@ -134,6 +135,8 @@ def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, writ size = ['--size', size] if size else [] viewer = ['--' + viewer] if viewer else [] seed = ['--seed', seed] if seed else [] + timeout = ['--timeout', str(timeout)] + initial_timeout = ['--initial-timeout', str(initial_timeout)] write_replay = ['--write-replay', write_replay] if write_replay else [] store_output = ['--store-output', store_output] if store_output else [] append_blue = ['--append-blue', team_infos[0]] if team_infos[0] else [] @@ -148,6 +151,8 @@ def call_pelita(team_specs, *, rounds, size, viewer, seed, team_infos=None, writ *size, *viewer, *seed, + *timeout, + *initial_timeout, *write_replay, *store_output] From b1a812b983c4621909808cde229548f56d6010dd Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Wed, 10 Sep 2025 17:55:53 +0200 Subject: [PATCH 15/25] RF: Make --full optional --- contrib/ci_engine.py | 152 +++++++++++++++++++++++-------------------- 1 file changed, 82 insertions(+), 70 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 94ad8570c..e622d088b 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -433,86 +433,86 @@ def elo_change(a, b, outcome): return elo - def pretty_print_results(self, highlight=None): + def pretty_print_results(self, full=False, highlight=None): """Pretty print the current results. """ if highlight is None: highlight = [] - console = Console() - # Some guesswork in here - MAX_COLUMNS = (console.width - 40) // 12 - if MAX_COLUMNS < 4: - # Let’s be honest: You should enlarge your terminal window even before that - MAX_COLUMNS = 4 - - res = self.dbwrapper.get_wins_losses() - rows = { k: list(v) for k, v in itertools.groupby(res, key=lambda x:x[0]) } - good_players = [p for p, player in self.players.items() if not player.get('error')] bad_players = [p for p, player in self.players.items() if player.get('error')] - num_rows_per_player = (len(good_players) // MAX_COLUMNS) + 1 - row_style = [*([""] * num_rows_per_player), *(["dim"] * num_rows_per_player)] - - table = Table(row_styles=row_style, title="Cross results") - table.add_column("") - table.add_column("Name") - table.add_column("Score", justify="right") - table.add_column("W/D/L") - - column_players = [[] for _idx in range(min(MAX_COLUMNS, len(good_players)))] - # if we have more good_players than allowed columns, we must wrap around - for idx, _p in enumerate(good_players): - column_players[idx % MAX_COLUMNS].append(idx) - - for midx in column_players: - table.add_column('\n'.join(map(str, midx))) - - - def batched(iterable, n): - # Backport from Python 3.12 - # batched('ABCDEFG', 3) → ABC DEF G - if n < 1: - raise ValueError('n must be at least one') - iterator = iter(iterable) - while batch := tuple(itertools.islice(iterator, n)): - yield batch - - result = [] - for idx, pname in enumerate(good_players): - win, loss, draw = self.get_results(pname) - error_count, fatalerror_count = self.get_errorcount(pname) - try: - team_name = self.get_team_name(pname) - except ValueError: - team_name = None - score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) - result.append([score, win, draw, loss, pname, team_name, error_count, fatalerror_count]) - wdl = f"{win:3d},{draw:3d},{loss:3d}" - - try: - row = rows[pname] - except KeyError: - continue - vals = { k: (w,l,d) for _p1, k, w, l, d in row } - - cross_results = [] - for idx2, p2name in enumerate(good_players): - win, loss, draw = vals.get(p2name, (0, 0, 0)) - if idx == idx2: - cross_results.append(" - - - ") - else: - cross_results.append(f"{win:2d},{draw:2d},{loss:2d}") + console = Console() - for c, r in enumerate(batched(cross_results, MAX_COLUMNS)): - if c == 0: - table.add_row(f"{idx}", pname, f"{score:.2f}", wdl, *r) - else: - table.add_row("", "", "", "", *r) + if full: + # Some guesswork in here + MAX_COLUMNS = (console.width - 40) // 12 + if MAX_COLUMNS < 4: + # Let’s be honest: You should enlarge your terminal window even before that + MAX_COLUMNS = 4 + + res = self.dbwrapper.get_wins_losses() + rows = { k: list(v) for k, v in itertools.groupby(res, key=lambda x:x[0]) } + + num_rows_per_player = (len(good_players) // MAX_COLUMNS) + 1 + row_style = [*([""] * num_rows_per_player), *(["dim"] * num_rows_per_player)] + + table = Table(row_styles=row_style, title="Cross results") + table.add_column("") + table.add_column("Name") + table.add_column("Score", justify="right") + table.add_column("W/D/L") + + column_players = [[] for _idx in range(min(MAX_COLUMNS, len(good_players)))] + # if we have more good_players than allowed columns, we must wrap around + for idx, _p in enumerate(good_players): + column_players[idx % MAX_COLUMNS].append(idx) + + for midx in column_players: + table.add_column('\n'.join(map(str, midx))) + + + def batched(iterable, n): + # Backport from Python 3.12 + # batched('ABCDEFG', 3) → ABC DEF G + if n < 1: + raise ValueError('n must be at least one') + iterator = iter(iterable) + while batch := tuple(itertools.islice(iterator, n)): + yield batch + + for idx, pname in enumerate(good_players): + win, loss, draw = self.get_results(pname) + error_count, fatalerror_count = self.get_errorcount(pname) + try: + team_name = self.get_team_name(pname) + except ValueError: + team_name = None + score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) + wdl = f"{win:3d},{draw:3d},{loss:3d}" - console.print(table) + try: + row = rows[pname] + except KeyError: + continue + vals = { k: (w,l,d) for _p1, k, w, l, d in row } + + cross_results = [] + for idx2, p2name in enumerate(good_players): + win, loss, draw = vals.get(p2name, (0, 0, 0)) + if idx == idx2: + cross_results.append(" - - - ") + else: + cross_results.append(f"{win:2d},{draw:2d},{loss:2d}") + + for c, r in enumerate(batched(cross_results, MAX_COLUMNS)): + if c == 0: + table.add_row(f"{idx}", pname, f"{score:.2f}", wdl, *r) + else: + table.add_row("", "", "", "", *r) + + console.print(table) table = Table(title="Bot ranking") @@ -529,6 +529,17 @@ def batched(iterable, n): elo = dict(self.dbwrapper.get_elo()) # elo = self.gen_elo() + result = [] + for idx, pname in enumerate(good_players): + win, loss, draw = self.get_results(pname) + error_count, fatalerror_count = self.get_errorcount(pname) + try: + team_name = self.get_team_name(pname) + except ValueError: + team_name = None + score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) + result.append([score, win, draw, loss, pname, team_name, error_count, fatalerror_count]) + result.sort(reverse=True) for [score, win, draw, loss, name, team_name, error_count, fatalerror_count] in result: style = "bold" if name in highlight else None @@ -1032,7 +1043,7 @@ def run(args): def print_scores(args): with open(args.config) as f: ci_engine = CI_Engine(f) - ci_engine.pretty_print_results() + ci_engine.pretty_print_results(full=args.full) def hash_teams(args): with open(args.config) as f: @@ -1056,6 +1067,7 @@ def hash_teams(args): parser_run.set_defaults(func=run) parser_print_scores = subparsers.add_parser('print-scores') + parser_print_scores.add_argument('--full', help='show full pair statistics', action='store_true', default=False) parser_print_scores.set_defaults(func=print_scores) parser_hash = subparsers.add_parser('hash-teams') From 64b4ad413d17598cf6ce04b64c83443dde786bd2 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Wed, 10 Sep 2025 22:18:02 +0200 Subject: [PATCH 16/25] RF: Show stats for a single team --- contrib/ci_engine.py | 136 +++++++++++++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 44 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index e622d088b..b8e3c1974 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -433,7 +433,7 @@ def elo_change(a, b, outcome): return elo - def pretty_print_results(self, full=False, highlight=None): + def pretty_print_results(self, full=False, team=None, highlight=None): """Pretty print the current results. """ @@ -445,6 +445,56 @@ def pretty_print_results(self, full=False, highlight=None): console = Console() + + table = Table(title="Bot ranking") + + table.add_column("Name") + table.add_column("# Matches") + table.add_column("# Wins") + table.add_column("# Draws") + table.add_column("# Losses") + table.add_column("Score") + table.add_column("ELO") + table.add_column("Error count") + table.add_column("# Fatal Errors") + + elo = dict(self.dbwrapper.get_elo()) + # elo = self.gen_elo() + + result = [] + for idx, pname in enumerate(good_players): + win, loss, draw = self.get_results(pname) + error_count, fatalerror_count = self.get_errorcount(pname) + try: + team_name = self.get_team_name(pname) + except ValueError: + team_name = None + score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) + result.append([score, win, draw, loss, pname, team_name, error_count, fatalerror_count]) + + result.sort(reverse=True) + for [score, win, draw, loss, name, team_name, error_count, fatalerror_count] in result: + style = "bold" if name in highlight else None + display_name = f"{name} ({team_name})" if team_name else f"{name}" + table.add_row( + display_name, + f"{win+draw+loss}", + f"{win}", + f"{draw}", + f"{loss}", + f"{score:6.3f}", + f"{elo.get(name, 0): >4.0f}", + f"{error_count}", + f"{fatalerror_count}", + style=style, + ) + + console.print(table) + + for p in bad_players: + print("% 30s ***%30s***" % (p, self.players[p]['error'])) + + if full: # Some guesswork in here MAX_COLUMNS = (console.width - 40) // 12 @@ -514,53 +564,49 @@ def batched(iterable, n): console.print(table) - table = Table(title="Bot ranking") + elif team: + MAX_COLUMNS = (console.width - 40) // 12 + if MAX_COLUMNS < 4: + # Let’s be honest: You should enlarge your terminal window even before that + MAX_COLUMNS = 4 - table.add_column("Name") - table.add_column("# Matches") - table.add_column("# Wins") - table.add_column("# Draws") - table.add_column("# Losses") - table.add_column("Score") - table.add_column("ELO") - table.add_column("Error count") - table.add_column("# Fatal Errors") + res = self.dbwrapper.get_wins_losses(team=team) + rows = {k: list(v) for k, v in itertools.groupby(res, key=lambda x:x[1])} - elo = dict(self.dbwrapper.get_elo()) - # elo = self.gen_elo() + row_style = ["", "dim"] - result = [] - for idx, pname in enumerate(good_players): - win, loss, draw = self.get_results(pname) - error_count, fatalerror_count = self.get_errorcount(pname) - try: - team_name = self.get_team_name(pname) - except ValueError: - team_name = None - score = 0 if (win+loss+draw) == 0 else (win-loss) / (win+loss+draw) - result.append([score, win, draw, loss, pname, team_name, error_count, fatalerror_count]) + table = Table(row_styles=row_style, title=f"Match results for team {team}") + table.add_column("Name") + table.add_column("# Matches") + table.add_column("# Wins") + table.add_column("# Draws") + table.add_column("# Losses") - result.sort(reverse=True) - for [score, win, draw, loss, name, team_name, error_count, fatalerror_count] in result: - style = "bold" if name in highlight else None - display_name = f"{name} ({team_name})" if team_name else f"{name}" - table.add_row( - display_name, - f"{win+draw+loss}", - f"{win}", - f"{draw}", - f"{loss}", - f"{score:6.3f}", - f"{elo.get(name, 0): >4.0f}", - f"{error_count}", - f"{fatalerror_count}", - style=style, - ) + for idx, pname in enumerate(good_players): + try: + team_name = self.get_team_name(pname) + except ValueError: + team_name = None - console.print(table) + try: + row = rows[pname] + except KeyError: + continue - for p in bad_players: - print("% 30s ***%30s***" % (p, self.players[p]['error'])) + for r in row: # there should only be one row + p1, p2, win, loss, draw = r + + display_name = f"{pname} ({team_name})" if team_name else f"{pname}" + + table.add_row( + display_name, + f"{win+draw+loss}", + f"{win}", + f"{draw}", + f"{loss}", + ) + + console.print(table) class DB_Wrapper: @@ -1043,7 +1089,7 @@ def run(args): def print_scores(args): with open(args.config) as f: ci_engine = CI_Engine(f) - ci_engine.pretty_print_results(full=args.full) + ci_engine.pretty_print_results(full=args.full, team=args.team) def hash_teams(args): with open(args.config) as f: @@ -1067,7 +1113,9 @@ def hash_teams(args): parser_run.set_defaults(func=run) parser_print_scores = subparsers.add_parser('print-scores') - parser_print_scores.add_argument('--full', help='show full pair statistics', action='store_true', default=False) + full_or_team = parser_print_scores.add_mutually_exclusive_group() + full_or_team.add_argument('--full', help='show full pair statistics', action='store_true', default=False) + full_or_team.add_argument('--team', help='show statistics for team', type=str, default=None) parser_print_scores.set_defaults(func=print_scores) parser_hash = subparsers.add_parser('hash-teams') From cfcd92704f9f8c93153f2070ba36eaee0532a89e Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Wed, 10 Sep 2025 23:03:58 +0200 Subject: [PATCH 17/25] ENH: Save number of errors directly --- contrib/ci_engine.py | 51 +++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index b8e3c1974..5d433fc63 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -455,7 +455,7 @@ def pretty_print_results(self, full=False, team=None, highlight=None): table.add_column("# Losses") table.add_column("Score") table.add_column("ELO") - table.add_column("Error count") + table.add_column("# Timeouts") table.add_column("# Fatal Errors") elo = dict(self.dbwrapper.get_elo()) @@ -646,7 +646,12 @@ def create_tables(self): """) self.cursor.execute(""" CREATE TABLE IF NOT EXISTS games - (player1 text, player2 text, result int, final_state text, stdout text, stderr text, + (player1 text, player2 text, result int, final_state text, + player1_timeouts int, player2_timeouts int, + player1_fatal_errors int, player2_fatal_errors int, + stdout text, stderr text, + player1_stdout text, player1_stderr text, + player2_stdout text, player2_stderr text, FOREIGN KEY(player1) REFERENCES players(name) ON DELETE CASCADE, FOREIGN KEY(player2) REFERENCES players(name) ON DELETE CASCADE) """) @@ -759,8 +764,12 @@ def add_gameresult(self, p1_name, p2_name, result, final_state, std_out, std_err return self.cursor.execute(""" INSERT INTO games - VALUES (?, ?, ?, ?, ?, ?) - """, [p1_name, p2_name, result, json.dumps(final_state), std_out, std_err]) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, [p1_name, p2_name, result, json.dumps(final_state), + final_state['num_errors'][0], final_state['num_errors'][1], + len(final_state['fatal_errors'][0]), len(final_state['fatal_errors'][1]), + std_out, std_err, + "", "", "", ""]) self.connection.commit() def get_results(self, p1_name, p2_name=None): @@ -882,45 +891,29 @@ def get_errorcount(self, p1_name): Returns ------- error_count, fatalerror_count : errorcount - """ self.cursor.execute(""" - SELECT sum(c) FROM + SELECT sum(timeouts), sum(fatal_errors) FROM ( - SELECT sum(json_extract(final_state, '$.num_errors[0]')) AS c + SELECT + sum(player1_timeouts) AS timeouts, + sum(player1_fatal_errors) AS fatal_errors FROM games WHERE player1 = :p1 UNION ALL - SELECT sum(json_extract(final_state, '$.num_errors[1]')) AS c + SELECT + sum(player2_timeouts) AS timeouts, + sum(player2_fatal_errors) AS fatal_errors FROM games WHERE player2 = :p1 ) """, dict(p1=p1_name)) - error_count, = self.cursor.fetchone() - - self.cursor.execute(""" - SELECT sum(c) FROM - ( - SELECT count(*) AS c - FROM games - WHERE player1 = :p1 AND - json_extract(final_state, '$.fatal_errors[0]') != '[]' - - UNION ALL - - SELECT count(*) AS c - FROM games - WHERE player2 = :p1 AND - json_extract(final_state, '$.fatal_errors[1]') != '[]' - ) - """, - dict(p1=p1_name)) - fatal_errorcount, = self.cursor.fetchone() + timeouts, fatal_errorcount = self.cursor.fetchone() - return error_count, fatal_errorcount + return timeouts, fatal_errorcount def get_wins_losses(self, team=None): """ Get all wins and losses combined in a table of From 0dc42947bad614b5ff7cfafedcbdb6276179a748 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Wed, 10 Sep 2025 23:37:01 +0200 Subject: [PATCH 18/25] ENH: Collect individual player output --- contrib/ci_engine.py | 71 +++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 5d433fc63..386d299ad 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -50,12 +50,14 @@ import json import logging import operator +from pathlib import Path import queue import shlex import signal import sqlite3 import sys import threading +from tempfile import TemporaryDirectory from random import Random from rich.console import Console @@ -190,34 +192,44 @@ def run_game(self, p1, p2): """ team_specs = [self.players[p1]['path'], self.players[p2]['path']] - final_state, stdout, stderr = call_pelita(team_specs, - rounds=self.rounds, - size=self.size, - viewer=self.viewer, - seed=self.seed, - timeout=10, - initial_timeout=120, - exit_flag=EXIT - ) + with TemporaryDirectory() as tmpdir: - if not final_state: - res = (p1, p2, None, final_state, stdout, stderr) - return res + final_state, stdout, stderr = call_pelita(team_specs, + rounds=self.rounds, + size=self.size, + viewer=self.viewer, + seed=self.seed, + store_output=tmpdir, + timeout=10, + initial_timeout=120, + exit_flag=EXIT + ) - if final_state['whowins'] == 2: - result = -1 - else: - result = final_state['whowins'] + if not final_state: + res = (p1, p2, None, final_state, stdout, stderr) + return res + + if final_state['whowins'] == 2: + result = -1 + else: + result = final_state['whowins'] + + del final_state['walls'] + del final_state['food'] - del final_state['walls'] - del final_state['food'] + _logger.info('Final state: %r', final_state) + _logger.debug('Stdout: %r', stdout) + if stderr: + _logger.warning('Stderr: %r', stderr) - _logger.info('Final state: %r', final_state) - _logger.debug('Stdout: %r', stdout) - if stderr: - _logger.warning('Stderr: %r', stderr) - res = (p1, p2, result, final_state, stdout, stderr) - return res + p1_stdout = (Path(tmpdir) / 'blue.out').read_text() + p1_stderr = (Path(tmpdir) / 'blue.err').read_text() + + p2_stdout = (Path(tmpdir) / 'red.out').read_text() + p2_stderr = (Path(tmpdir) / 'red.err').read_text() + + res = (p1, p2, result, final_state, [stdout, stderr], [p1_stdout, p1_stderr], [p2_stdout, p2_stderr]) + return res def start(self, n, thread_count): @@ -745,7 +757,7 @@ def remove_player(self, pname): WHERE name = ?""", (pname,)) self.connection.commit() - def add_gameresult(self, p1_name, p2_name, result, final_state, std_out, std_err): + def add_gameresult(self, p1_name, p2_name, result, final_state, std, p1_out, p2_out): """Add a new game result to the database. Parameters @@ -760,6 +772,11 @@ def add_gameresult(self, p1_name, p2_name, result, final_state, std_out, std_err STDOUT and STDERR of the game """ + + stdout, stderr = std + p1_stdout, p1_stderr = p1_out + p2_stdout, p2_stderr = p2_out + if not final_state: return self.cursor.execute(""" @@ -768,8 +785,8 @@ def add_gameresult(self, p1_name, p2_name, result, final_state, std_out, std_err """, [p1_name, p2_name, result, json.dumps(final_state), final_state['num_errors'][0], final_state['num_errors'][1], len(final_state['fatal_errors'][0]), len(final_state['fatal_errors'][1]), - std_out, std_err, - "", "", "", ""]) + stdout, stderr, + p1_stdout, p1_stderr, p2_stdout, p2_stderr]) self.connection.commit() def get_results(self, p1_name, p2_name=None): From 63902bd19ba3ab35ccfdb2f21a0fbd927e1e1959 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Sun, 14 Sep 2025 20:18:15 +0200 Subject: [PATCH 19/25] ENH: Add id field to db --- contrib/ci_engine.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 386d299ad..7e86d6d7a 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -658,14 +658,21 @@ def create_tables(self): """) self.cursor.execute(""" CREATE TABLE IF NOT EXISTS games - (player1 text, player2 text, result int, final_state text, + ( + id INTEGER PRIMARY KEY, + player1 text, player2 text, result int, final_state text, player1_timeouts int, player2_timeouts int, player1_fatal_errors int, player2_fatal_errors int, + FOREIGN KEY(player1) REFERENCES players(name) ON DELETE CASCADE, + FOREIGN KEY(player2) REFERENCES players(name) ON DELETE CASCADE) + """) + self.cursor.execute(""" + CREATE TABLE IF NOT EXISTS game_output + (game_id int, stdout text, stderr text, player1_stdout text, player1_stderr text, player2_stdout text, player2_stderr text, - FOREIGN KEY(player1) REFERENCES players(name) ON DELETE CASCADE, - FOREIGN KEY(player2) REFERENCES players(name) ON DELETE CASCADE) + FOREIGN KEY(game_id) REFERENCES games(id) ON DELETE CASCADE) """) self.connection.commit() @@ -781,10 +788,19 @@ def add_gameresult(self, p1_name, p2_name, result, final_state, std, p1_out, p2_ return self.cursor.execute(""" INSERT INTO games - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (player1, player2, result, final_state, + player1_timeouts, player2_timeouts, + player1_fatal_errors, player2_fatal_errors) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + RETURNING id """, [p1_name, p2_name, result, json.dumps(final_state), final_state['num_errors'][0], final_state['num_errors'][1], - len(final_state['fatal_errors'][0]), len(final_state['fatal_errors'][1]), + len(final_state['fatal_errors'][0]), len(final_state['fatal_errors'][1])]) + game_id, = self.cursor.fetchone() + self.cursor.execute(""" + INSERT INTO game_output + VALUES (?, ?, ?, ?, ?, ?, ?) + """, [game_id, stdout, stderr, p1_stdout, p1_stderr, p2_stdout, p2_stderr]) self.connection.commit() From bd2413307ca403adb4fb5a3ba557194694e29a48 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Mon, 2 Feb 2026 23:31:27 +0100 Subject: [PATCH 20/25] RF: Clearer naming for timeouts --- contrib/ci_engine.py | 41 ++++++++++++++++++++++++++------------- contrib/test_ci_engine.py | 21 +++++++++++--------- pelita/game.py | 4 ++-- pelita/ui/tk_canvas.py | 2 +- 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 7e86d6d7a..488de253b 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -209,10 +209,14 @@ def run_game(self, p1, p2): res = (p1, p2, None, final_state, stdout, stderr) return res - if final_state['whowins'] == 2: - result = -1 + if final_state['game_phase'] != 'FINISHED': + _logger.info("Game finished in phase %s", final_state['game_phase']) + result = -2 else: - result = final_state['whowins'] + if final_state['whowins'] == 2: + result = -1 + else: + result = final_state['whowins'] del final_state['walls'] del final_state['food'] @@ -661,8 +665,8 @@ def create_tables(self): ( id INTEGER PRIMARY KEY, player1 text, player2 text, result int, final_state text, - player1_timeouts int, player2_timeouts int, - player1_fatal_errors int, player2_fatal_errors int, + player1_num_timeouts int, player2_num_timeouts int, + player1_had_fatal_error bool, player2_had_fatal_error bool, FOREIGN KEY(player1) REFERENCES players(name) ON DELETE CASCADE, FOREIGN KEY(player2) REFERENCES players(name) ON DELETE CASCADE) """) @@ -775,6 +779,7 @@ def add_gameresult(self, p1_name, p2_name, result, final_state, std, p1_out, p2_ 0 if player 1 won 1 of player 2 won -1 if draw + -2 if anything other than game_phase FINISHED std_out, std_err : str STDOUT and STDERR of the game @@ -786,16 +791,24 @@ def add_gameresult(self, p1_name, p2_name, result, final_state, std, p1_out, p2_ if not final_state: return + + final_state_str = json.dumps(final_state) + player1_num_timeouts, player2_num_timeouts = final_state['num_timeouts'] + + player1_had_fatal_error = len(final_state['fatal_errors'][0]) != 0 + player2_had_fatal_error = len(final_state['fatal_errors'][1]) != 0 + self.cursor.execute(""" INSERT INTO games (player1, player2, result, final_state, - player1_timeouts, player2_timeouts, - player1_fatal_errors, player2_fatal_errors) + player1_num_timeouts, player2_num_timeouts, + player1_had_fatal_error, player2_had_fatal_error) VALUES (?, ?, ?, ?, ?, ?, ?, ?) RETURNING id - """, [p1_name, p2_name, result, json.dumps(final_state), - final_state['num_errors'][0], final_state['num_errors'][1], - len(final_state['fatal_errors'][0]), len(final_state['fatal_errors'][1])]) + """, [p1_name, p2_name, result, final_state_str, + player1_num_timeouts, player2_num_timeouts, + player1_had_fatal_error, player2_had_fatal_error]) + game_id, = self.cursor.fetchone() self.cursor.execute(""" INSERT INTO game_output @@ -929,16 +942,16 @@ def get_errorcount(self, p1_name): SELECT sum(timeouts), sum(fatal_errors) FROM ( SELECT - sum(player1_timeouts) AS timeouts, - sum(player1_fatal_errors) AS fatal_errors + sum(player1_num_timeouts) AS timeouts, + sum(player1_had_fatal_error) AS fatal_errors FROM games WHERE player1 = :p1 UNION ALL SELECT - sum(player2_timeouts) AS timeouts, - sum(player2_fatal_errors) AS fatal_errors + sum(player2_num_timeouts) AS timeouts, + sum(player2_had_fatal_error) AS fatal_errors FROM games WHERE player2 = :p1 ) diff --git a/contrib/test_ci_engine.py b/contrib/test_ci_engine.py index 22f5a24ef..cc3939d0f 100644 --- a/contrib/test_ci_engine.py +++ b/contrib/test_ci_engine.py @@ -8,6 +8,9 @@ def db_wrapper(): wrapper.create_tables() return wrapper +def make_simple_gameresult(p1, p2, result): + return [p1, p2, result, {'num_timeouts': [0, 0], 'fatal_errors': [[], []]}, ['', ''], ['', ''], ['', '']] + """Tests for the DB_Wrapper class.""" def test_foreign_keys_enabled(db_wrapper): @@ -26,9 +29,9 @@ def test_remove_player(db_wrapper): db_wrapper.add_player('p1', 'h1') db_wrapper.add_player('p2', 'h2') db_wrapper.add_player('p3', 'h3') - db_wrapper.add_gameresult('p1', 'p2', 0, '{}', '', '') - db_wrapper.add_gameresult('p2', 'p1', 0, '{}', '', '') - db_wrapper.add_gameresult('p2', 'p3', 0, '{}', '', '') + db_wrapper.add_gameresult(*make_simple_gameresult('p1', 'p2', 0)) + db_wrapper.add_gameresult(*make_simple_gameresult('p2', 'p1', 0)) + db_wrapper.add_gameresult(*make_simple_gameresult('p2', 'p3', 0)) # player2 has three games assert len(db_wrapper.get_results('p2')) == 3 db_wrapper.remove_player('p1') @@ -63,13 +66,13 @@ def test_get_results(db_wrapper): db_wrapper.add_player('p2', 'h2') # empty list if no results are available assert db_wrapper.get_results('p1') == [] - db_wrapper.add_gameresult('p1', 'p2', 0, '{}', '', '') + db_wrapper.add_gameresult(*make_simple_gameresult('p1', 'p2', 0)) result = db_wrapper.get_results('p1')[0] # check for correct values assert result[0] == 'p1' assert result[1] == 'p2' assert result[2] == 0 - db_wrapper.add_gameresult('p2', 'p1', 0, '{}', '', '') + db_wrapper.add_gameresult(*make_simple_gameresult('p2', 'p1', 0)) # check for correct number of results results = db_wrapper.get_results('p1') assert len(results) == 2 @@ -97,22 +100,22 @@ def test_wins_losses(db_wrapper): db_wrapper.add_player('p1', 'h1') db_wrapper.add_player('p2', 'h2') db_wrapper.add_player('p3', 'h3') - db_wrapper.add_gameresult('p1', 'p2', 0, "{}", "", "") + db_wrapper.add_gameresult(*make_simple_gameresult('p1', 'p2', 0)) assert db_wrapper.get_wins_losses() == [ ('p1', 'p2', 1, 0, 0), ('p2', 'p1', 0, 1, 0) ] - db_wrapper.add_gameresult('p1', 'p2', -1, "{}", "", "") + db_wrapper.add_gameresult(*make_simple_gameresult('p1', 'p2', -1)) assert db_wrapper.get_wins_losses() == [ ('p1', 'p2', 1, 0, 1), ('p2', 'p1', 0, 1, 1) ] - db_wrapper.add_gameresult('p2', 'p1', 1, "{}", "", "") + db_wrapper.add_gameresult(*make_simple_gameresult('p2', 'p1', 1)) assert db_wrapper.get_wins_losses() == [ ('p1', 'p2', 2, 0, 1), ('p2', 'p1', 0, 2, 1) ] - db_wrapper.add_gameresult('p3', 'p1', 1, "{}", "", "") + db_wrapper.add_gameresult(*make_simple_gameresult('p3', 'p1', 1)) assert db_wrapper.get_wins_losses() == [ ('p1', 'p2', 2, 0, 1), ('p1', 'p3', 1, 0, 0), diff --git a/pelita/game.py b/pelita/game.py index 71c596545..29d0aac1a 100644 --- a/pelita/game.py +++ b/pelita/game.py @@ -805,7 +805,7 @@ def prepare_viewer_state(game_state): # game_state["timeouts"] has a tuple as a dict key # that cannot be serialized in json. # To fix this problem, we only send the current error - # and add another attribute "num_errors" + # and add another attribute "num_timeouts" # to the final dict. # the key for the current round, turn @@ -817,7 +817,7 @@ def prepare_viewer_state(game_state): ] # add the number of errors - viewer_state["num_errors"] = [ + viewer_state["num_timeouts"] = [ len(team_errors) for team_errors in game_state["timeouts"] ] diff --git a/pelita/ui/tk_canvas.py b/pelita/ui/tk_canvas.py index 547e57ac8..a3993dc4b 100644 --- a/pelita/ui/tk_canvas.py +++ b/pelita/ui/tk_canvas.py @@ -870,7 +870,7 @@ def status(team_idx): # sum the deaths of both bots in this team deaths = game_state['deaths'][team_idx] + game_state['deaths'][team_idx+2] kills = game_state['kills'][team_idx] + game_state['kills'][team_idx+2] - ret = "Timeouts: %d, Kills: %d, Deaths: %d, Time: %.2f" % (game_state["num_errors"][team_idx], kills, deaths, game_state["team_time"][team_idx]) + ret = "Timeouts: %d, Kills: %d, Deaths: %d, Time: %.2f" % (game_state["num_timeouts"][team_idx], kills, deaths, game_state["team_time"][team_idx]) return ret except TypeError: return "" From 218997ce63414de98fce3dafb764e70c4d3c8007 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Sun, 14 Sep 2025 22:26:48 +0200 Subject: [PATCH 21/25] RF: Use ThreadPoolExecutor for the matches --- contrib/ci_engine.py | 257 +++++++++++++++++++++---------------------- 1 file changed, 127 insertions(+), 130 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 488de253b..1257a22c1 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -51,7 +51,6 @@ import logging import operator from pathlib import Path -import queue import shlex import signal import sqlite3 @@ -61,6 +60,7 @@ from random import Random from rich.console import Console +from rich.progress import (Progress, SpinnerColumn, TextColumn, TimeElapsedColumn) from rich.table import Table from pelita.network import RemotePlayerFailure @@ -97,6 +97,65 @@ async def hash_team(team_spec, semaphore): return stdout.decode().strip().split("\n")[-1].strip() + +def run_game(team_specs, config): + """Run a single game. + + This method runs a single game and returns the result. + + Parameters + ---------- + p1, p2 : int + the indices of the players + + """ + + with TemporaryDirectory() as tmpdir: + final_state, stdout, stderr = call_pelita(team_specs, + rounds=config['rounds'], + size=config['size'], + viewer=config['viewer'], + seed=config['seed'], + store_output=tmpdir, + timeout=10, + initial_timeout=120, + exit_flag=EXIT + ) + + if not final_state: + result = None + + if final_state['game_phase'] != 'FINISHED': + _logger.info("Game finished in phase %s", final_state['game_phase']) + result = -2 + else: + if final_state['whowins'] == 2: + result = -1 + else: + result = final_state['whowins'] + + try: + del final_state['walls'] + del final_state['food'] + except IndexError: + pass + + _logger.info('Final state: %r', final_state) + _logger.debug('Stdout: %r', stdout) + if stderr: + _logger.warning('Stderr: %r', stderr) + + p1_stdout = (Path(tmpdir) / 'blue.out').read_text() + p1_stderr = (Path(tmpdir) / 'blue.err').read_text() + + p2_stdout = (Path(tmpdir) / 'red.out').read_text() + p2_stderr = (Path(tmpdir) / 'red.err').read_text() + + res = (result, final_state, [stdout, stderr], [p1_stdout, p1_stderr], [p2_stdout, p2_stderr]) + return res + + + class CI_Engine: """Continuous Integration Engine.""" @@ -177,69 +236,10 @@ def check_team_name(args): else: print(pname, self.players[pname], self.dbwrapper.get_team_name(pname)) - - def run_game(self, p1, p2): - """Run a single game. - - This method runs a single game ``p1`` vs ``p2`` and internally - stores the result. - - Parameters - ---------- - p1, p2 : int - the indices of the players - - """ - team_specs = [self.players[p1]['path'], self.players[p2]['path']] - - with TemporaryDirectory() as tmpdir: - - final_state, stdout, stderr = call_pelita(team_specs, - rounds=self.rounds, - size=self.size, - viewer=self.viewer, - seed=self.seed, - store_output=tmpdir, - timeout=10, - initial_timeout=120, - exit_flag=EXIT - ) - - if not final_state: - res = (p1, p2, None, final_state, stdout, stderr) - return res - - if final_state['game_phase'] != 'FINISHED': - _logger.info("Game finished in phase %s", final_state['game_phase']) - result = -2 - else: - if final_state['whowins'] == 2: - result = -1 - else: - result = final_state['whowins'] - - del final_state['walls'] - del final_state['food'] - - _logger.info('Final state: %r', final_state) - _logger.debug('Stdout: %r', stdout) - if stderr: - _logger.warning('Stderr: %r', stderr) - - p1_stdout = (Path(tmpdir) / 'blue.out').read_text() - p1_stderr = (Path(tmpdir) / 'blue.err').read_text() - - p2_stdout = (Path(tmpdir) / 'red.out').read_text() - p2_stderr = (Path(tmpdir) / 'red.err').read_text() - - res = (p1, p2, result, final_state, [stdout, stderr], [p1_stdout, p1_stderr], [p2_stdout, p2_stderr]) - return res - - - def start(self, n, thread_count): + def start(self, n, concurrency): """Start the Engine. - This method will start and infinite loop, testing each agent + This method will start and run n matches, testing each agent randomly against another one. The result is printed after each game. @@ -251,89 +251,88 @@ def start(self, n, thread_count): >>> ci.start() """ - loop = itertools.repeat(None) if n == 0 else itertools.repeat(None, n) - rng = Random() + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + TimeElapsedColumn() + ) as progress: - game_counts = self.dbwrapper.get_game_counts() + lock = threading.Lock() - for pname, player in self.players.items(): - if "error" in player and pname in game_counts: - del game_counts[pname] + def worker(count, p1, p2): + with lock: + progress_task = progress.add_task(f"Playing #{count}: {p1} against {p2}.") - def worker(q, r, lock=threading.Lock()): - for task in iter(q.get, None): # blocking get until None is received - try: - count, slf, p1, p2 = task + config = { + 'rounds': self.rounds, + 'size': self.size, + 'viewer': self.viewer, + 'seed': None, # TODO + } - print(f"Playing #{count}: {p1} against {p2}.") + team_specs = [self.players[p1]['path'], self.players[p2]['path']] + res = run_game(team_specs, config) - res = slf.run_game(p1, p2) - r.put((count, (p1, p2), res)) - #with lock: - finally: - q.task_done() + with lock: + progress.update(progress_task, completed=True, visible=False) - worker_count = thread_count - q = queue.Queue(maxsize=thread_count) - r = queue.Queue() - threads = [threading.Thread(target=worker, args=[q, r], daemon=False) - for _ in range(worker_count)] - for t in threads: - t.start() + return count, (p1, p2), res - for count, _ in enumerate(loop): - # choose the player with the least number of played game, - # match with another random player - # mix the sides and let them play + def producer(): + rng = Random() - players_sorted = sorted(list(game_counts.items()), key=operator.itemgetter(1)) + game_counts = self.dbwrapper.get_game_counts() - a = players_sorted[0][0] - b = rng.choice(players_sorted[1:])[0] + for count in range(n): + for pname, player in self.players.items(): + if "error" in player and pname in game_counts: + del game_counts[pname] - players = [a, b] - rng.shuffle(players) + # choose the player with the least number of played games, + # match with another random player + # shuffle the sides and let them play - q.put((count, self, players[0], players[1])) + players_sorted = sorted(list(game_counts.items()), key=operator.itemgetter(1)) - game_counts[a] += 1 - game_counts[b] += 1 + a = players_sorted[0][0] + b = rng.choice(players_sorted[1:])[0] - try: - count, players, res = r.get_nowait() - final_state = res[3] - if final_state: - print(f"Storing #{count}: {players[0]} against {players[1]}.") - else: - print(f"Not storing #{count}: {players[0]} against {players[1]}.") - self.dbwrapper.add_gameresult(*res) - except queue.Empty: - pass - except Exception: - pass + players = [a, b] + rng.shuffle(players) - if EXIT.is_set(): - break + _logger.debug(f"Adding match {count} ({players[0]} vs {players[1]}) to worker queue") + task = (count, players[0], players[1]) - q.join() # block until all spawned tasks are done + yield task - while True: - try: - count, players, res = r.get_nowait() - final_state = res[3] - if final_state: - print(f"Storing #{count}: {players[0]} against {players[1]}.") - else: - print(f"Not storing #{count}: {players[0]} against {players[1]}.") - self.dbwrapper.add_gameresult(*res) - except queue.Empty: - break + game_counts[a] += 1 + game_counts[b] += 1 - for _ in threads: # signal workers to quit - q.put(None) - for t in threads: # wait until workers exit - t.join() + with ThreadPoolExecutor(max_workers=concurrency) as executor: + if sys.version_info < (3, 14): + _logger.warning(f"Generating all {n} match partners. Use Python 3.14+ to do this lazily.") + buffersize = {} + else: + buffersize = {'buffersize': concurrency} + + for result in executor.map(lambda args: worker(*args), producer(), **buffersize): + count, players, res = result + + p1_name, p2_name = players + winner, final_state, out, p1_out, p2_out = res + + if final_state: + match final_state["whowins"]: + case 0: + progress.console.print(f"Storing #{count}: [u]{players[0]}[/u] against {players[1]}.") + case 1: + progress.console.print(f"Storing #{count}: {players[0]} against [u]{players[1]}[/u].") + case _: + progress.console.print(f"Storing #{count}: {players[0]} against {players[1]}.") + else: + progress.console.print(f"Not storing #{count}: {players[0]} against {players[1]}.") + self.dbwrapper.add_gameresult(p1_name, p2_name, winner, final_state, out, p1_out, p2_out) def get_results(self, p1_name, p2_name=None): @@ -363,7 +362,6 @@ def get_results(self, p1_name, p2_name=None): the number of wins, losses and draws for this player or combination of players - Examples -------- @@ -1135,7 +1133,6 @@ def hash_teams(args): ci_engine = CI_Engine(f) ci_engine.load_players(concurrency=args.thread_count) - if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--log', help="Print debugging log information to LOGFILE (default 'stderr').", @@ -1146,7 +1143,7 @@ def hash_teams(args): subparsers = parser.add_subparsers(required=True) parser_run = subparsers.add_parser('run') - parser_run.add_argument('-n', help='run N times', type=int, default=0) + parser_run.add_argument('-n', help='run N times', type=int, default=1000) parser_run.add_argument('--thread-count', '-t', help='run in parallel', type=int, default=1) parser_run.add_argument('--no-hash', help='Do not hash the players prior to running', action='store_true', default=False) parser_run.set_defaults(func=run) From f4a23174ae318f85f6e87140641d9a25ac28e028 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Thu, 28 May 2026 14:24:49 +0200 Subject: [PATCH 22/25] ENH: Allow CLI override of database location. --- contrib/ci_engine.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 1257a22c1..01499ea7f 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -159,7 +159,7 @@ def run_game(team_specs, config): class CI_Engine: """Continuous Integration Engine.""" - def __init__(self, cfgfile): + def __init__(self, cfgfile, database=None): self.players = {} config = configparser.ConfigParser() config.read_file(cfgfile) @@ -171,7 +171,7 @@ def __init__(self, cfgfile): self.viewer = config['general'].get('viewer', 'null') self.seed = config['general'].get('seed', None) - self.db_file = config.get('general', 'db_file') + self.db_file = database or config.get('general', 'db_file') self.dbwrapper = DB_Wrapper(self.db_file) def load_players(self, concurrency=1): @@ -638,6 +638,7 @@ def __init__(self, dbfile): """ self.db_file = dbfile + _logger.info("Using sqlite database file ‘%s’.", self.db_file) self.connection = sqlite3.connect(self.db_file) self.cursor = self.connection.cursor() self.cursor.execute("PRAGMA foreign_keys = ON;") @@ -1118,19 +1119,19 @@ def get_elo(self): def run(args): with open(args.config) as f: - ci_engine = CI_Engine(f) + ci_engine = CI_Engine(f, args.database) if not args.no_hash: ci_engine.load_players(concurrency=args.thread_count) ci_engine.start(args.n, args.thread_count) def print_scores(args): with open(args.config) as f: - ci_engine = CI_Engine(f) + ci_engine = CI_Engine(f, args.database) ci_engine.pretty_print_results(full=args.full, team=args.team) def hash_teams(args): with open(args.config) as f: - ci_engine = CI_Engine(f) + ci_engine = CI_Engine(f, args.database) ci_engine.load_players(concurrency=args.thread_count) if __name__ == '__main__': @@ -1139,6 +1140,8 @@ def hash_teams(args): metavar='LOGFILE', const='-', nargs='?') parser.add_argument('--config', help="Print debugging log information to LOGFILE (default 'stderr').", metavar='FILE', default=CFG_FILE) + parser.add_argument('--database', help="Database location", + metavar='FILE', default=None) subparsers = parser.add_subparsers(required=True) From e52a3edfbe294e81dce4a1058128693a692d3d77 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Thu, 28 May 2026 22:19:18 +0200 Subject: [PATCH 23/25] ENH: Make player paths relative to cfg file --- contrib/ci_engine.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index 01499ea7f..b2c046d82 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -160,11 +160,14 @@ class CI_Engine: """Continuous Integration Engine.""" def __init__(self, cfgfile, database=None): + self.cfg_path = Path(cfgfile) + self.players = {} config = configparser.ConfigParser() config.read_file(cfgfile) for name, path in config.items('agents'): - self.players[name]= {'path': path} + self.players[name]= {'path': self.cfg_path.parent / path} + self.rounds = config['general'].getint('rounds', None) self.size = config['general'].get('size', None) From 210fb74b3caa519fb3b172d496ff0bb47e1adfe5 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Thu, 28 May 2026 22:25:48 +0200 Subject: [PATCH 24/25] RF: Read cfg with read_file --- contrib/ci_engine.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index b2c046d82..bf9500c67 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -164,9 +164,9 @@ def __init__(self, cfgfile, database=None): self.players = {} config = configparser.ConfigParser() - config.read_file(cfgfile) + config.read(cfgfile) for name, path in config.items('agents'): - self.players[name]= {'path': self.cfg_path.parent / path} + self.players[name]= {'path': str(self.cfg_path.parent / path)} self.rounds = config['general'].getint('rounds', None) @@ -1121,21 +1121,18 @@ def get_elo(self): return self.cursor.execute(query).fetchall() def run(args): - with open(args.config) as f: - ci_engine = CI_Engine(f, args.database) - if not args.no_hash: - ci_engine.load_players(concurrency=args.thread_count) - ci_engine.start(args.n, args.thread_count) + ci_engine = CI_Engine(args.config, args.database) + if not args.no_hash: + ci_engine.load_players(concurrency=args.thread_count) + ci_engine.start(args.n, args.thread_count) def print_scores(args): - with open(args.config) as f: - ci_engine = CI_Engine(f, args.database) - ci_engine.pretty_print_results(full=args.full, team=args.team) + ci_engine = CI_Engine(args.config, args.database) + ci_engine.pretty_print_results(full=args.full, team=args.team) def hash_teams(args): - with open(args.config) as f: - ci_engine = CI_Engine(f, args.database) - ci_engine.load_players(concurrency=args.thread_count) + ci_engine = CI_Engine(args.config, args.database) + ci_engine.load_players(concurrency=args.thread_count) if __name__ == '__main__': parser = argparse.ArgumentParser() From 5ceb4aa7ba5e6fa1f28591a4a0992cf13868d147 Mon Sep 17 00:00:00 2001 From: Rike-Benjamin Schuppner Date: Fri, 29 May 2026 21:04:28 +0200 Subject: [PATCH 25/25] NF: Simple HTML export --- contrib/ci_engine.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/contrib/ci_engine.py b/contrib/ci_engine.py index bf9500c67..7fc2d19a0 100755 --- a/contrib/ci_engine.py +++ b/contrib/ci_engine.py @@ -450,7 +450,7 @@ def elo_change(a, b, outcome): return elo - def pretty_print_results(self, full=False, team=None, highlight=None): + def pretty_print_results(self, full=False, team=None, highlight=None, html_export=None): """Pretty print the current results. """ @@ -460,8 +460,7 @@ def pretty_print_results(self, full=False, team=None, highlight=None): good_players = [p for p, player in self.players.items() if not player.get('error')] bad_players = [p for p, player in self.players.items() if player.get('error')] - console = Console() - + console = Console(record=True) table = Table(title="Bot ranking") @@ -625,6 +624,9 @@ def batched(iterable, n): console.print(table) + if html_export: + console.save_html(html_export) + class DB_Wrapper: """Wrapper around the games database.""" @@ -1128,7 +1130,7 @@ def run(args): def print_scores(args): ci_engine = CI_Engine(args.config, args.database) - ci_engine.pretty_print_results(full=args.full, team=args.team) + ci_engine.pretty_print_results(full=args.full, team=args.team, html_export=args.html_export) def hash_teams(args): ci_engine = CI_Engine(args.config, args.database) @@ -1152,6 +1154,7 @@ def hash_teams(args): parser_run.set_defaults(func=run) parser_print_scores = subparsers.add_parser('print-scores') + parser_print_scores.add_argument('--html-export', help='output as HTML', default=False) full_or_team = parser_print_scores.add_mutually_exclusive_group() full_or_team.add_argument('--full', help='show full pair statistics', action='store_true', default=False) full_or_team.add_argument('--team', help='show statistics for team', type=str, default=None)